ci: benchmarking pipeline #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| # A/B benchmark of two fuzzamoto-libafl versions: run each version N times for a fixed | |
| # duration in parallel, then aggregate and statistically compare the campaigns. | |
| # | |
| # Bitcoin Core (the target) is built once into a shared image and held constant; only the | |
| # bind-mounted fuzzamoto source differs between the two arms, so the comparison isolates | |
| # the fuzzer change. See ci/benchmark-evaluation.py for the metrics. | |
| # | |
| # Runner specs are inlined in each job's `runs-on:` label (runs-on.com syntax); there is | |
| # no .github/runs-on.yml. Campaign runners are pinned to a single exact instance type so | |
| # every campaign runs on identical hardware (see the campaign job). | |
| # | |
| # Triggers: | |
| # - workflow_dispatch: compare two arbitrary refs (full control over N / duration). | |
| # - PR label 'needs benchmark': full run, N=10, 3600s/campaign (PR base vs head). | |
| # - PR label 'needs benchmark smoke': cheap wiring test, N=2, 300s/campaign. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| baseline_ref: | |
| description: "Baseline fuzzamoto git ref" | |
| default: "master" | |
| treatment_ref: | |
| description: "Treatment fuzzamoto git ref" | |
| required: true | |
| runs: | |
| description: "Campaigns per version (N)" | |
| default: "10" | |
| duration_secs: | |
| description: "Seconds per campaign" | |
| default: "3600" | |
| pull_request: | |
| # Gated on the 'needs benchmark' / 'needs benchmark smoke' labels (checked in setup's `if:`). | |
| types: [labeled] | |
| permissions: | |
| contents: read | |
| pull-requests: write # for the PR comment | |
| jobs: | |
| setup: | |
| if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'needs benchmark' || github.event.label.name == 'needs benchmark smoke' | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - cpu=2+4 | |
| - family=m7i+c7i | |
| - image=ubuntu22-full-x64 | |
| - extras=s3-cache | |
| outputs: | |
| baseline: ${{ steps.refs.outputs.baseline }} | |
| treatment: ${{ steps.refs.outputs.treatment }} | |
| duration: ${{ steps.refs.outputs.duration }} | |
| runs_json: ${{ steps.refs.outputs.runs_json }} # e.g. "[1,2,...,10]" | |
| steps: | |
| - id: refs | |
| run: | | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| echo "baseline=${{ github.event.pull_request.base.sha }}" >> "$GITHUB_OUTPUT" | |
| echo "treatment=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" | |
| if [ "${{ github.event.label.name }}" = "needs benchmark smoke" ]; then | |
| N=2 | |
| echo "duration=300" >> "$GITHUB_OUTPUT" | |
| else | |
| N=10 | |
| echo "duration=3600" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| echo "baseline=${{ inputs.baseline_ref }}" >> "$GITHUB_OUTPUT" | |
| echo "treatment=${{ inputs.treatment_ref }}" >> "$GITHUB_OUTPUT" | |
| N=${{ inputs.runs }} | |
| echo "duration=${{ inputs.duration_secs }}" >> "$GITHUB_OUTPUT" | |
| fi | |
| echo "runs_json=$(seq -s, 1 "$N" | sed 's/^/[/;s/$/]/')" >> "$GITHUB_OUTPUT" | |
| build: | |
| needs: setup | |
| # Fat instance to build the shared image (Bitcoin Core + AFL++ + depends). | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - cpu=16+32 | |
| - family=c7i+m7i+c7a | |
| - image=ubuntu22-full-x64 | |
| - volume=120gb:gp3 | |
| - extras=s3-cache # S3-backed docker layer cache (no 10GB cap) | |
| - spot=false | |
| steps: | |
| - uses: actions/checkout@v4 # Bitcoin Core build is version-independent | |
| - uses: docker/setup-buildx-action@v3 | |
| - name: Build & cache the shared image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./Dockerfile.libafl | |
| push: false | |
| tags: fuzzamoto-libafl:bench | |
| platforms: linux/amd64 | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max # backed by runs-on s3-cache (no 10GB cap) | |
| campaign: | |
| needs: [setup, build] | |
| # nested-virt exposes /dev/kvm (required by Nyx). IMPORTANT: pinned to a single, | |
| # exact instance type so every campaign (both arms, all N runs) runs on identical | |
| # hardware - otherwise a coverage/exec-rate delta could reflect instance variance | |
| # rather than the fuzzer change. spot=false so an interruption can't void a 1h run. | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=c7i.2xlarge # exact type; adjust to one nested-virt type in your account | |
| - image=ubuntu22-full-x64 | |
| - nested-virt | |
| - volume=80gb:gp3 | |
| - extras=s3-cache | |
| - spot=false | |
| timeout-minutes: 90 # 60m campaign + build/compile headroom | |
| strategy: | |
| fail-fast: false # a dead campaign must not cancel the others | |
| matrix: | |
| version: [baseline, treatment] | |
| run: ${{ fromJSON(needs.setup.outputs.runs_json) }} | |
| steps: | |
| - name: Checkout fuzzer version under test | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ matrix.version == 'baseline' && needs.setup.outputs.baseline || needs.setup.outputs.treatment }} | |
| - uses: docker/setup-buildx-action@v3 | |
| - name: Restore image from cache | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./Dockerfile.libafl | |
| load: true | |
| tags: fuzzamoto-libafl:bench | |
| cache-from: type=gha # near-instant; layers built by `build` | |
| - name: Run campaign | |
| run: | | |
| rm -rf /tmp/out && mkdir -p /tmp/out | |
| docker run --privileged --device=/dev/kvm \ | |
| -e BENCH_DURATION=${{ needs.setup.outputs.duration }} \ | |
| -v ./:/fuzzamoto -v /tmp/out:/tmp/out \ | |
| fuzzamoto-libafl:bench just -f /ci/libafl.justfile bench | |
| - name: Collect results | |
| run: | | |
| mkdir -p artifact | |
| cp /tmp/out/bench/bench-cpu_000.csv artifact/ \ | |
| || echo "no bench csv (campaign may have died early)" | |
| cp -r /tmp/out/cpu_000/crashes artifact/crashes 2>/dev/null || true | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-${{ matrix.version }}-${{ matrix.run }} | |
| path: artifact | |
| retention-days: 14 | |
| compare: | |
| needs: [setup, campaign] | |
| if: always() | |
| # Lightweight runner for report aggregation; does not affect measurements. | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - cpu=2+4 | |
| - family=m7i+c7i | |
| - image=ubuntu22-full-x64 | |
| - extras=s3-cache | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: bench-* | |
| path: dl | |
| - name: Arrange into baseline/treatment layout | |
| run: | | |
| shopt -s nullglob | |
| for d in dl/bench-baseline-*; do | |
| n=${d##*-}; mkdir -p results/baseline/ir/$n | |
| cp "$d/bench-cpu_000.csv" results/baseline/ir/$n/ 2>/dev/null || true | |
| done | |
| for d in dl/bench-treatment-*; do | |
| n=${d##*-}; mkdir -p results/treatment/ir/$n | |
| cp "$d/bench-cpu_000.csv" results/treatment/ir/$n/ 2>/dev/null || true | |
| done | |
| - name: Install evaluation dependencies | |
| run: pip install numpy pandas matplotlib seaborn scipy statsmodels tabulate | |
| - name: Evaluate | |
| run: | | |
| python3 ci/benchmark-evaluation.py results --out report --hours min | |
| { | |
| echo "## fuzzamoto-libafl benchmark" | |
| echo | |
| echo "Baseline: \`${{ needs.setup.outputs.baseline }}\` • Treatment: \`${{ needs.setup.outputs.treatment }}\`" | |
| echo | |
| cat report/evaluation_report.md | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-report | |
| path: report | |
| retention-days: 90 | |
| - name: Comment on PR | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const body = fs.readFileSync('report/evaluation_report.md', 'utf8'); | |
| await github.rest.issues.createComment({ | |
| ...context.repo, | |
| issue_number: context.issue.number, | |
| body, | |
| }); |