Benchmark report #371
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark report | |
| # Trusted-side half of the benchmark PR feedback (see benchmark.yml). A | |
| # `pull_request` run from a fork gets a read-only GITHUB_TOKEN, so it can't | |
| # label or comment — it renders the report and uploads it as an artifact | |
| # instead. This workflow fires on that run's completion via `workflow_run`, | |
| # which always executes from the default branch with the base repo's | |
| # read/write token, and does the writes. | |
| # | |
| # Security: `workflow_run` handlers must never check out or execute the head | |
| # fork's code. This job only downloads the report artifact (plain data: | |
| # rendered markdown, a verdict JSON, the PR number) and calls the GitHub API. | |
| # The PR number is validated as an integer and the comment body is passed via | |
| # a file, so nothing from the fork reaches a shell or an eval. | |
| on: | |
| workflow_run: | |
| workflows: [Benchmark] | |
| types: [completed] | |
| # Collapse re-runs for the same PR source, mirroring benchmark.yml's own | |
| # cancel-in-progress. Key on head repo + branch: head_branch alone would | |
| # collapse unrelated PRs from two forks that happen to share a branch name | |
| # (e.g. both `patch-1`). | |
| concurrency: | |
| group: >- | |
| benchmark-report-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} | |
| cancel-in-progress: true | |
| jobs: | |
| report: | |
| name: Label and comment | |
| runs-on: ubuntu-latest | |
| # Only PR runs upload a report bundle, and only a successful benchmark | |
| # produced one — skip pushes to main and failed/cancelled runs. | |
| if: | | |
| github.event.workflow_run.event == 'pull_request' | |
| && github.event.workflow_run.conclusion == 'success' | |
| permissions: | |
| actions: read # download the artifact from the triggering run | |
| pull-requests: write # upsert the comparison comment | |
| issues: write # create/apply the benchmark:regression / benchmark:improvement labels | |
| steps: | |
| # Land outside the workspace: the docs warn against extracting a | |
| # cross-workflow artifact where a checkout's scripts could sit. There's | |
| # no checkout here, but keeping it in runner.temp keeps that guarantee. | |
| - name: Download report bundle | |
| id: download | |
| continue-on-error: true | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: benchmark-report | |
| path: ${{ runner.temp }}/report | |
| run-id: ${{ github.event.workflow_run.id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # A run whose paths filter skipped the benchmark job uploads no bundle; | |
| # download fails and there's simply nothing to report. Stop cleanly. | |
| - name: Check bundle present | |
| id: bundle | |
| run: | | |
| if [ -f "$RUNNER_TEMP/report/pr-number" ] && [ -f "$RUNNER_TEMP/report/report.md" ]; then | |
| echo "present=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "no report bundle (benchmark job likely skipped by paths filter); nothing to do" | |
| echo "present=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Validate the PR number is a bare integer before it touches any API call. | |
| # A malformed number can't be trusted as an issue target, but this is a | |
| # best-effort reporter — flip `valid` off and let the writes skip rather | |
| # than failing the whole run red (matching the continue-on-error stance | |
| # of every other real-work step here). | |
| - name: Read PR number | |
| id: pr | |
| if: steps.bundle.outputs.present == 'true' | |
| run: | | |
| pr=$(tr -d '[:space:]' < "$RUNNER_TEMP/report/pr-number") | |
| case "$pr" in | |
| ''|*[!0-9]*) | |
| echo "::warning::skipping report: invalid PR number in artifact: '$pr'" | |
| echo "valid=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| ;; | |
| esac | |
| echo "number=$pr" >> "$GITHUB_OUTPUT" | |
| echo "valid=true" >> "$GITHUB_OUTPUT" | |
| - name: Label PR on significant deltas | |
| if: steps.pr.outputs.valid == 'true' | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| verdict="$RUNNER_TEMP/report/verdict.json" | |
| regression=$(jq -r '.regression' "$verdict" 2>/dev/null || echo false) | |
| improvement=$(jq -r '.improvement' "$verdict" 2>/dev/null || echo false) | |
| repo='${{ github.repository }}' | |
| pr='${{ steps.pr.outputs.number }}' | |
| gh label create benchmark:regression --repo "$repo" --force \ | |
| --color D93F0B --description "CI benchmark: significant resource regression vs main" | |
| gh label create benchmark:improvement --repo "$repo" --force \ | |
| --color 0E8A16 --description "CI benchmark: significant resource improvement vs main" | |
| # Keep labels in sync with the latest run: apply when the verdict | |
| # says so, remove a stale one when a newer push clears it. | |
| if [ "$regression" = "true" ]; then | |
| gh pr edit "$pr" --repo "$repo" --add-label benchmark:regression | |
| else | |
| gh pr edit "$pr" --repo "$repo" --remove-label benchmark:regression || true | |
| fi | |
| if [ "$improvement" = "true" ]; then | |
| gh pr edit "$pr" --repo "$repo" --add-label benchmark:improvement | |
| else | |
| gh pr edit "$pr" --repo "$repo" --remove-label benchmark:improvement || true | |
| fi | |
| - name: Comment on PR | |
| if: steps.pr.outputs.valid == 'true' | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| marker='<!-- macterm-benchmark -->' | |
| printf '%s\n\n' "$marker" > comment.md | |
| cat "$RUNNER_TEMP/report/report.md" >> comment.md | |
| repo='${{ github.repository }}' | |
| pr='${{ steps.pr.outputs.number }}' | |
| comment_id=$(gh api "repos/$repo/issues/$pr/comments" --paginate \ | |
| --jq "[.[] | select(.body | startswith(\"$marker\"))][0].id // empty") | |
| if [ -n "$comment_id" ]; then | |
| gh api -X PATCH "repos/$repo/issues/comments/$comment_id" -F body=@comment.md --silent | |
| else | |
| gh api "repos/$repo/issues/$pr/comments" -F body=@comment.md --silent | |
| fi |