benchmark.yml

name: Benchmark

# A/B benchmark of two fuzzamoto-libafl versions: run each version N times for a fixed
# duration in parallel, then aggregate and statistically compare the campaigns.
#
# Bitcoin Core (the target) is built once into a shared image and held constant; only the
# bind-mounted fuzzamoto source differs between the two arms, so the comparison isolates
# the fuzzer change. See ci/benchmark-evaluation.py for the metrics.
#
# Runner specs are inlined in each job's `runs-on:` label (runs-on.com syntax); there is
# no .github/runs-on.yml. Campaign runners are pinned to a single exact instance type so
# every campaign runs on identical hardware (see the campaign job).
#
# Triggers:
#   - workflow_dispatch: compare two arbitrary refs (full control over N / duration).
#   - PR label 'needs benchmark':        full run, N=10, 3600s/campaign (PR base vs head).
#   - PR label 'needs benchmark smoke':  cheap wiring test, N=2, 300s/campaign.

on:
  workflow_dispatch:
    inputs:
      baseline_ref:
        description: "Baseline fuzzamoto git ref"
        default: "master"
      treatment_ref:
        description: "Treatment fuzzamoto git ref"
        required: true
      runs:
        description: "Campaigns per version (N)"
        default: "10"
      duration_secs:
        description: "Seconds per campaign"
        default: "3600"
  pull_request:
    # Gated on the 'needs benchmark' / 'needs benchmark smoke' labels (checked in setup's `if:`).
    types: [labeled]

permissions:
  contents: read
  pull-requests: write # for the PR comment

jobs:
  setup:
    if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'needs benchmark' || github.event.label.name == 'needs benchmark smoke'
    runs-on:
      - runs-on=${{ github.run_id }}
      - cpu=2+4
      - family=m7i+c7i
      - image=ubuntu22-full-x64
      - extras=s3-cache
    outputs:
      baseline: ${{ steps.refs.outputs.baseline }}
      treatment: ${{ steps.refs.outputs.treatment }}
      duration: ${{ steps.refs.outputs.duration }}
      runs_json: ${{ steps.refs.outputs.runs_json }} # e.g. "[1,2,...,10]"
    steps:
      - id: refs
        run: |
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            echo "baseline=${{ github.event.pull_request.base.sha }}"  >> "$GITHUB_OUTPUT"
            echo "treatment=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"
            if [ "${{ github.event.label.name }}" = "needs benchmark smoke" ]; then
              N=2
              echo "duration=300" >> "$GITHUB_OUTPUT"
            else
              N=10
              echo "duration=3600" >> "$GITHUB_OUTPUT"
            fi
          else
            echo "baseline=${{ inputs.baseline_ref }}"   >> "$GITHUB_OUTPUT"
            echo "treatment=${{ inputs.treatment_ref }}" >> "$GITHUB_OUTPUT"
            N=${{ inputs.runs }}
            echo "duration=${{ inputs.duration_secs }}" >> "$GITHUB_OUTPUT"
          fi
          echo "runs_json=$(seq -s, 1 "$N" | sed 's/^/[/;s/$/]/')" >> "$GITHUB_OUTPUT"

  build:
    needs: setup
    # Fat instance to build the shared image (Bitcoin Core + AFL++ + depends).
    runs-on:
      - runs-on=${{ github.run_id }}
      - cpu=16+32
      - family=c7i+m7i+c7a
      - image=ubuntu22-full-x64
      - volume=120gb:gp3
      - extras=s3-cache # S3-backed docker layer cache (no 10GB cap)
      - spot=false
    steps:
      - uses: actions/checkout@v4 # Bitcoin Core build is version-independent
      - uses: docker/setup-buildx-action@v3
      - name: Build & cache the shared image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./Dockerfile.libafl
          push: false
          tags: fuzzamoto-libafl:bench
          platforms: linux/amd64
          cache-from: type=gha
          cache-to: type=gha,mode=max # backed by runs-on s3-cache (no 10GB cap)

  campaign:
    needs: [setup, build]
    # nested-virt exposes /dev/kvm (required by Nyx). IMPORTANT: pinned to a single,
    # exact instance type so every campaign (both arms, all N runs) runs on identical
    # hardware - otherwise a coverage/exec-rate delta could reflect instance variance
    # rather than the fuzzer change. spot=false so an interruption can't void a 1h run.
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=c7i.2xlarge # exact type; adjust to one nested-virt type in your account
      - image=ubuntu22-full-x64
      - nested-virt
      - volume=80gb:gp3
      - extras=s3-cache
      - spot=false
    timeout-minutes: 90 # 60m campaign + build/compile headroom
    strategy:
      fail-fast: false # a dead campaign must not cancel the others
      matrix:
        version: [baseline, treatment]
        run: ${{ fromJSON(needs.setup.outputs.runs_json) }}
    steps:
      - name: Checkout fuzzer version under test
        uses: actions/checkout@v4
        with:
          ref: ${{ matrix.version == 'baseline' && needs.setup.outputs.baseline || needs.setup.outputs.treatment }}
      - uses: docker/setup-buildx-action@v3
      - name: Restore image from cache
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./Dockerfile.libafl
          load: true
          tags: fuzzamoto-libafl:bench
          cache-from: type=gha # near-instant; layers built by `build`
      - name: Run campaign
        run: |
          rm -rf /tmp/out && mkdir -p /tmp/out
          docker run --privileged --device=/dev/kvm \
            -e BENCH_DURATION=${{ needs.setup.outputs.duration }} \
            -v ./:/fuzzamoto -v /tmp/out:/tmp/out \
            fuzzamoto-libafl:bench just -f /ci/libafl.justfile bench
      - name: Collect results
        run: |
          mkdir -p artifact
          cp /tmp/out/bench/bench-cpu_000.csv artifact/ \
            || echo "no bench csv (campaign may have died early)"
          cp -r /tmp/out/cpu_000/crashes artifact/crashes 2>/dev/null || true
      - uses: actions/upload-artifact@v4
        with:
          name: bench-${{ matrix.version }}-${{ matrix.run }}
          path: artifact
          retention-days: 14

  compare:
    needs: [setup, campaign]
    if: always()
    # Lightweight runner for report aggregation; does not affect measurements.
    runs-on:
      - runs-on=${{ github.run_id }}
      - cpu=2+4
      - family=m7i+c7i
      - image=ubuntu22-full-x64
      - extras=s3-cache
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          pattern: bench-*
          path: dl
      - name: Arrange into baseline/treatment layout
        run: |
          shopt -s nullglob
          for d in dl/bench-baseline-*; do
            n=${d##*-}; mkdir -p results/baseline/ir/$n
            cp "$d/bench-cpu_000.csv" results/baseline/ir/$n/ 2>/dev/null || true
          done
          for d in dl/bench-treatment-*; do
            n=${d##*-}; mkdir -p results/treatment/ir/$n
            cp "$d/bench-cpu_000.csv" results/treatment/ir/$n/ 2>/dev/null || true
          done
      - name: Install evaluation dependencies
        run: pip install numpy pandas matplotlib seaborn scipy statsmodels tabulate
      - name: Evaluate
        run: |
          python3 ci/benchmark-evaluation.py results --out report --hours min
          {
            echo "## fuzzamoto-libafl benchmark"
            echo
            echo "Baseline: \`${{ needs.setup.outputs.baseline }}\`  •  Treatment: \`${{ needs.setup.outputs.treatment }}\`"
            echo
            cat report/evaluation_report.md
          } >> "$GITHUB_STEP_SUMMARY"
      - uses: actions/upload-artifact@v4
        with:
          name: benchmark-report
          path: report
          retention-days: 90
      - name: Comment on PR
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const body = fs.readFileSync('report/evaluation_report.md', 'utf8');
            await github.rest.issues.createComment({
              ...context.repo,
              issue_number: context.issue.number,
              body,
            });