Skip to content

Commit c954ebf

Browse files
p-x9p-x9
authored
Merge pull request #306 from p-x9/feature/benchmark-workflow
Add manual benchmark workflow
2 parents 2934815 + 42abdca commit c954ebf

2 files changed

Lines changed: 359 additions & 0 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: Benchmark
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
mode:
7+
description: Run one ref or compare it with another ref
8+
type: choice
9+
options:
10+
- compare
11+
- run
12+
default: compare
13+
baseline_ref:
14+
description: Branch, tag, or SHA used as the comparison baseline
15+
type: string
16+
default: main
17+
filter:
18+
description: Optional swift-benchmark regular expression
19+
type: string
20+
default: ""
21+
pairs:
22+
description: Number of AB/BA pairs in compare mode
23+
type: choice
24+
options:
25+
- "1"
26+
- "3"
27+
default: "1"
28+
29+
permissions:
30+
contents: read
31+
32+
concurrency:
33+
group: benchmark-${{ github.repository }}
34+
cancel-in-progress: false
35+
36+
env:
37+
DEVELOPER_DIR: /Applications/Xcode_16.2.app/Contents/Developer
38+
BENCHMARK_DISABLE_JEMALLOC: "1"
39+
40+
jobs:
41+
benchmark:
42+
name: ${{ inputs.mode == 'compare' && 'Compare refs' || 'Run benchmarks' }}
43+
runs-on: macos-15
44+
timeout-minutes: 120
45+
env:
46+
BENCHMARK_MODE: ${{ inputs.mode }}
47+
BENCHMARK_FILTER: ${{ inputs.filter }}
48+
BENCHMARK_PAIRS: ${{ inputs.pairs }}
49+
50+
steps:
51+
- name: Checkout candidate
52+
uses: actions/checkout@v4
53+
with:
54+
path: candidate
55+
56+
- name: Checkout baseline
57+
if: inputs.mode == 'compare'
58+
uses: actions/checkout@v4
59+
with:
60+
ref: ${{ inputs.baseline_ref }}
61+
path: baseline
62+
63+
- name: Select Xcode 16.2
64+
run: sudo xcode-select -s /Applications/Xcode_16.2.app
65+
66+
- name: Run benchmark workflow
67+
shell: bash
68+
env:
69+
CANDIDATE_ROOT: ${{ github.workspace }}/candidate
70+
BASELINE_ROOT: ${{ github.workspace }}/baseline
71+
BENCHMARK_RESULTS_DIR: ${{ runner.temp }}/benchmark-results
72+
MACHOKIT_BENCH_MACHO: ${{ runner.temp }}/MachOKitBenchmarks-fixture
73+
run: bash candidate/scripts/benchmark.sh
74+
75+
- name: Upload benchmark results
76+
if: always()
77+
uses: actions/upload-artifact@v4
78+
with:
79+
name: benchmark-${{ inputs.mode }}-${{ github.run_id }}
80+
path: ${{ runner.temp }}/benchmark-results
81+
if-no-files-found: warn
82+
retention-days: 30

‎scripts/benchmark.sh‎

Lines changed: 277 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,277 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
# This script can run directly from a checkout or from GitHub Actions.
6+
#
7+
# Run the current checkout:
8+
# scripts/benchmark.sh
9+
#
10+
# Compare the current checkout with another checkout:
11+
# BENCHMARK_MODE=compare BASELINE_ROOT=/path/to/baseline scripts/benchmark.sh
12+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
13+
REPOSITORY_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
14+
15+
BENCHMARK_MODE="${BENCHMARK_MODE:-run}"
16+
BENCHMARK_FILTER="${BENCHMARK_FILTER:-}"
17+
BENCHMARK_PAIRS="${BENCHMARK_PAIRS:-1}"
18+
CANDIDATE_ROOT="${CANDIDATE_ROOT:-$REPOSITORY_ROOT}"
19+
BASELINE_ROOT="${BASELINE_ROOT:-}"
20+
BENCHMARK_RESULTS_DIR="${BENCHMARK_RESULTS_DIR:-/tmp/machokit-benchmark-results}"
21+
MACHOKIT_BENCH_MACHO="${MACHOKIT_BENCH_MACHO:-$BENCHMARK_RESULTS_DIR/MachOKitBenchmarks-fixture}"
22+
BENCHMARK_DISABLE_JEMALLOC="${BENCHMARK_DISABLE_JEMALLOC:-1}"
23+
24+
export BENCHMARK_DISABLE_JEMALLOC
25+
export MACHOKIT_BENCH_MACHO
26+
27+
fail() {
28+
echo "benchmark: $*" >&2
29+
exit 1
30+
}
31+
32+
canonical_directory() {
33+
local path="$1"
34+
[[ -d "$path" ]] || fail "directory does not exist: $path"
35+
(cd "$path" && pwd)
36+
}
37+
38+
case "$BENCHMARK_MODE" in
39+
run | compare)
40+
;;
41+
*)
42+
fail "BENCHMARK_MODE must be 'run' or 'compare'"
43+
;;
44+
esac
45+
46+
case "$BENCHMARK_PAIRS" in
47+
1 | 3)
48+
;;
49+
*)
50+
fail "BENCHMARK_PAIRS must be '1' or '3'"
51+
;;
52+
esac
53+
54+
CANDIDATE_ROOT="$(canonical_directory "$CANDIDATE_ROOT")"
55+
[[ -d "$CANDIDATE_ROOT/Benchmarks" ]] || fail "candidate has no Benchmarks directory"
56+
57+
if [[ "$BENCHMARK_MODE" == "compare" ]]; then
58+
[[ -n "$BASELINE_ROOT" ]] || fail "BASELINE_ROOT is required in compare mode"
59+
BASELINE_ROOT="$(canonical_directory "$BASELINE_ROOT")"
60+
[[ -e "$BASELINE_ROOT/.git" ]] || fail "baseline must be a Git checkout"
61+
[[ -d "$BASELINE_ROOT/Benchmarks" ]] || fail "baseline has no Benchmarks directory"
62+
[[ "$BASELINE_ROOT" != "$CANDIDATE_ROOT" ]] || fail "baseline and candidate must be different directories"
63+
fi
64+
65+
mkdir -p "$BENCHMARK_RESULTS_DIR"
66+
SUMMARY_FILE="$BENCHMARK_RESULTS_DIR/summary.md"
67+
: > "$SUMMARY_FILE"
68+
69+
append_summary() {
70+
local section_file="$1"
71+
cat "$section_file" >> "$SUMMARY_FILE"
72+
if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
73+
cat "$section_file" >> "$GITHUB_STEP_SUMMARY"
74+
fi
75+
}
76+
77+
prepare_benchmark_sources() {
78+
# The baseline checkout is disposable. Mirror the candidate's benchmark
79+
# definitions and dependency lock so only the MachOKit source ref differs.
80+
rsync -a --delete \
81+
--exclude .benchmarkBaselines/ \
82+
--exclude .build/ \
83+
--exclude .swiftpm/ \
84+
--exclude result.txt \
85+
"$CANDIDATE_ROOT/Benchmarks/" \
86+
"$BASELINE_ROOT/Benchmarks/"
87+
}
88+
89+
build_benchmarks() {
90+
local checkout="$1"
91+
local label="$2"
92+
(
93+
cd "$checkout/Benchmarks"
94+
swift build -c release --product MachOKitBenchmarks
95+
) 2>&1 | tee "$BENCHMARK_RESULTS_DIR/build-$label.log"
96+
}
97+
98+
create_fixed_fixture() {
99+
local candidate_bin_path
100+
candidate_bin_path="$(
101+
cd "$CANDIDATE_ROOT/Benchmarks"
102+
swift build -c release --show-bin-path
103+
)"
104+
cp "$candidate_bin_path/MachOKitBenchmarks" "$MACHOKIT_BENCH_MACHO"
105+
}
106+
107+
record_environment() {
108+
local section_file="$BENCHMARK_RESULTS_DIR/environment.md"
109+
local cpu
110+
local candidate_sha
111+
local baseline_sha="not applicable"
112+
local filter_display
113+
114+
cpu="$(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m)"
115+
candidate_sha="$(git -C "$CANDIDATE_ROOT" rev-parse HEAD)"
116+
if [[ "$BENCHMARK_MODE" == "compare" ]]; then
117+
baseline_sha="$(git -C "$BASELINE_ROOT" rev-parse HEAD)"
118+
fi
119+
120+
filter_display="${BENCHMARK_FILTER//$'\n'/ }"
121+
filter_display="${filter_display//$'\r'/ }"
122+
filter_display="${filter_display//\`/\\\`}"
123+
filter_display="${filter_display//|/\\|}"
124+
125+
{
126+
echo "## Benchmark environment"
127+
echo
128+
echo "| Item | Value |"
129+
echo "| --- | --- |"
130+
echo "| Mode | \`$BENCHMARK_MODE\` |"
131+
echo "| Candidate | \`$candidate_sha\` |"
132+
echo "| Baseline | \`$baseline_sha\` |"
133+
echo "| Runner | \`${RUNNER_NAME:-local} (${RUNNER_OS:-$(uname -s)})\` |"
134+
echo "| Runner image | \`${ImageOS:-unknown} ${ImageVersion:-unknown}\` |"
135+
echo "| CPU | \`$cpu\` |"
136+
echo "| Memory | \`$(sysctl -n hw.memsize) bytes\` |"
137+
echo "| Xcode | \`$(xcodebuild -version | tr '\n' ' ')\` |"
138+
echo "| Fixture SHA-256 | \`$(shasum -a 256 "$MACHOKIT_BENCH_MACHO" | awk '{print $1}')\` |"
139+
echo "| Filter | \`${filter_display:-all benchmarks}\` |"
140+
if [[ "$BENCHMARK_MODE" == "compare" ]]; then
141+
echo "| Pairs | \`$BENCHMARK_PAIRS\` |"
142+
else
143+
echo "| Pairs | \`not applicable\` |"
144+
fi
145+
echo
146+
} > "$section_file"
147+
148+
append_summary "$section_file"
149+
}
150+
151+
benchmark_args=(--no-progress --quiet)
152+
if [[ -n "$BENCHMARK_FILTER" ]]; then
153+
benchmark_args+=(--filter "$BENCHMARK_FILTER")
154+
fi
155+
156+
run_baseline() {
157+
local checkout="$1"
158+
local baseline_name="$2"
159+
local log_file="$3"
160+
(
161+
cd "$checkout/Benchmarks"
162+
swift package --allow-writing-to-package-directory benchmark baseline update \
163+
"$baseline_name" \
164+
"${benchmark_args[@]}"
165+
) 2>&1 | tee "$log_file"
166+
}
167+
168+
copy_baseline_to_candidate() {
169+
local baseline_name="$1"
170+
local source_dir="$BASELINE_ROOT/Benchmarks/.benchmarkBaselines/MachOKitBenchmarks/$baseline_name"
171+
local destination_dir="$CANDIDATE_ROOT/Benchmarks/.benchmarkBaselines/MachOKitBenchmarks/$baseline_name"
172+
mkdir -p "$destination_dir"
173+
cp "$source_dir/results.json" "$destination_dir/results.json"
174+
}
175+
176+
export_baseline() {
177+
local baseline_name="$1"
178+
local output_dir="$2"
179+
mkdir -p "$output_dir/samples"
180+
(
181+
cd "$CANDIDATE_ROOT/Benchmarks"
182+
swift package --allow-writing-to-directory "$output_dir/samples" \
183+
benchmark baseline read "$baseline_name" \
184+
--format histogramSamples \
185+
--path "$output_dir/samples" \
186+
--no-progress \
187+
--quiet
188+
)
189+
cp "$CANDIDATE_ROOT/Benchmarks/.benchmarkBaselines/MachOKitBenchmarks/$baseline_name/results.json" \
190+
"$output_dir/$baseline_name.results.json"
191+
}
192+
193+
run_single() {
194+
local section_file="$BENCHMARK_RESULTS_DIR/run.md"
195+
196+
run_baseline "$CANDIDATE_ROOT" candidate "$BENCHMARK_RESULTS_DIR/run.log"
197+
198+
{
199+
echo "## Benchmark results"
200+
echo
201+
(
202+
cd "$CANDIDATE_ROOT/Benchmarks"
203+
swift package benchmark baseline read candidate \
204+
--format markdown \
205+
--no-progress
206+
)
207+
} > "$section_file"
208+
append_summary "$section_file"
209+
210+
export_baseline candidate "$BENCHMARK_RESULTS_DIR"
211+
}
212+
213+
run_comparison() {
214+
local pair
215+
local pair_dir
216+
local baseline_name
217+
local candidate_name
218+
local measurement_order
219+
220+
for ((pair = 1; pair <= BENCHMARK_PAIRS; pair++)); do
221+
pair_dir="$BENCHMARK_RESULTS_DIR/pair-$pair"
222+
baseline_name="baseline-pair-$pair"
223+
candidate_name="candidate-pair-$pair"
224+
mkdir -p "$pair_dir"
225+
226+
if ((pair % 2 == 1)); then
227+
measurement_order="baseline → candidate"
228+
run_baseline "$BASELINE_ROOT" "$baseline_name" "$pair_dir/baseline.log"
229+
run_baseline "$CANDIDATE_ROOT" "$candidate_name" "$pair_dir/candidate.log"
230+
else
231+
measurement_order="candidate → baseline"
232+
run_baseline "$CANDIDATE_ROOT" "$candidate_name" "$pair_dir/candidate.log"
233+
run_baseline "$BASELINE_ROOT" "$baseline_name" "$pair_dir/baseline.log"
234+
fi
235+
236+
copy_baseline_to_candidate "$baseline_name"
237+
238+
{
239+
echo "## Comparison pair $pair"
240+
echo
241+
echo "Measurement order: $measurement_order"
242+
echo
243+
(
244+
cd "$CANDIDATE_ROOT/Benchmarks"
245+
swift package benchmark baseline compare \
246+
"$baseline_name" \
247+
"$candidate_name" \
248+
--format markdown \
249+
--no-progress
250+
)
251+
} > "$pair_dir/comparison.md"
252+
append_summary "$pair_dir/comparison.md"
253+
254+
export_baseline "$baseline_name" "$pair_dir"
255+
export_baseline "$candidate_name" "$pair_dir"
256+
done
257+
}
258+
259+
if [[ "$BENCHMARK_MODE" == "compare" ]]; then
260+
prepare_benchmark_sources
261+
fi
262+
263+
build_benchmarks "$CANDIDATE_ROOT" candidate
264+
if [[ "$BENCHMARK_MODE" == "compare" ]]; then
265+
build_benchmarks "$BASELINE_ROOT" baseline
266+
fi
267+
268+
create_fixed_fixture
269+
record_environment
270+
271+
if [[ "$BENCHMARK_MODE" == "compare" ]]; then
272+
run_comparison
273+
else
274+
run_single
275+
fi
276+
277+
echo "Benchmark results: $BENCHMARK_RESULTS_DIR"

0 commit comments

Comments
 (0)