Avoid interpolating config variable values into error messages #1165

Workflow file for this run

.github/workflows/regression-benchmark.yml at 47e7596

	name: Regression Benchmark

	# Benchmarks Hardhat's E2E regression scenarios and compares the results
	# against the baselines recorded from main.

	on:
	# Records baselines for main. Deliberately do not run on merge_group.
	push:
	branches:
	- "main"
	# Manual runs of the dispatched ref's HEAD.
	workflow_dispatch:
	inputs:
	scenario-filter:
	description: "Glob(s) selecting which projects/scenarios to run, by directory name (e.g. `1inch`). Comma-separated. Defaults to `` (all projects). Forwarded to `bench:regression --scenarios`."
	required: false
	type: string
	default: "*"
	benchmark-filter:
	description: "Glob(s) selecting which benchmarks to run within each project, by name — a command or step name, i.e. the part after `<project> /` in a report (e.g. `test solidity`, `cold compile`, `compile`). Comma-separated. Defaults to `*` (the full suite). Forwarded to `bench:regression --benchmarks`."
	required: false
	type: string
	default: "*"
	# ChatOps: comment `/bench` on a same-repo PR, optionally with
	# `scenarios="<glob>[,<glob>...]"` and/or `benchmarks="<glob>[,<glob>...]"`
	# to select which projects/benchmarks run (quote values containing spaces).
	# Authorization + gating happens in `setup`.
	issue_comment:
	types: [created]

	concurrency:
	# Group by PR (comment events) or ref (dispatch) so re-triggering `/bench`
	# on a PR supersedes the previous run.
	#
	# Pushes to main are the exception: every commit must record its own baseline
	# entry, so each push gets a group keyed by commit SHA. `cancel-in-progress`
	# only protects the in-progress run, not runs waiting in the queue. A
	# unique-per-commit group collides with nothing, so every commit's run is
	# preserved and serialized by the self-hosted runner.
	#
	# Every comment triggers a workflow run. The job-level `if` skips non-`/bench`
	# ones, but that runs after concurrency is evaluated, so an unrelated comment
	# can cancel an in-progress benchmark. Give those skipped runs a unique group
	# so they collide with nothing.
	group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha \|\| github.event.issue.number \|\| github.ref }}${{ (github.event_name == 'issue_comment' && !startsWith(github.event.comment.body, '/bench')) && format('-skip-{0}', github.run_id) \|\| '' }}
	# Don't cancel in-progress baseline runs on main so baselines aren't lost.
	cancel-in-progress: ${{ github.event_name != 'push' }}

	jobs:
	setup:
	name: Resolve ref and authorize
	runs-on: ubuntu-latest
	timeout-minutes: 40
	permissions:
	contents: read # read PR head / checkout metadata
	pull-requests: write # pulls.get + post status comments/reactions on the PR
	issues: write # post status comments + reactions on the PR
	actions: read # list CI workflow runs for the CI-green gate
	# For comment events, only proceed for `/bench` comments on a PR. Other
	# events (push/workflow_dispatch) always evaluate inside the script.
	if: >-
	github.event_name != 'issue_comment' \|\| (github.event.issue.pull_request != null &&
	startsWith(github.event.comment.body, '/bench'))

	outputs:
	should_run: ${{ steps.resolve.outputs.should_run }}
	bench_ref: ${{ steps.resolve.outputs.bench_ref }}
	is_baseline: ${{ steps.resolve.outputs.is_baseline }}
	scenario_filter: ${{ steps.resolve.outputs.scenario_filter }}
	benchmark_filter: ${{ steps.resolve.outputs.benchmark_filter }}
	steps:
	# Sparse-checkout only the workflow scripts so the github-script step can
	# require the resolver module below.
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	sparse-checkout: .github/scripts
	persist-credentials: false

	- id: resolve
	uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
	with:
	script: \|
	const { resolveRegressionTrigger } = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/resolve-regression-trigger.ts`);
	await resolveRegressionTrigger({ github, context, core });

	regression-benchmark:
	name: Regression benchmark
	needs: setup
	if: needs.setup.outputs.should_run == 'true'
	environment: github-action-benchmark
	# Use a self-hosted runner for stable benchmark measurements; the shared
	# GitHub runners produce noisy results that will generate false positives
	# against the configured alert-threshold.
	runs-on: hardhat-linux-amd64-self-hosted
	timeout-minutes: 180
	permissions:
	# Lets the GITHUB_TOKEN used by "Compare benchmark result against
	# baseline" read commit metadata via the REST API. Only needed on
	# dispatch and `/bench` runs; push events carry the commit in the
	# event payload.
	contents: read
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	ref: ${{ needs.setup.outputs.bench_ref }}
	persist-credentials: false
	fetch-depth: 0
	fetch-tags: true

	# cspell:disable-next-line
	- uses: socketdev/action@ba6de6cc0565af1f42295590380973573297e31f # v1.3.2
	with:
	mode: firewall

	- uses: ./.github/actions/setup-env

	- name: Install system dependencies
	# `time` is GNU /usr/bin/time — bench:regression wraps each benchmark in
	# it to capture peak RSS (memory) alongside timing.
	run: \|
	sudo apt-get update
	sudo apt-get install -y hyperfine time

	- name: Configure E2E clone directory
	# Clone scenarios into the runner's temp dir instead of the default
	# /tmp/end-to-end, which persists across runs on the self-hosted
	# runner. The "Clean up temp dir" step below removes it after the run.
	run: echo "E2E_CLONE_DIR=${RUNNER_TEMP:?}/hardhat-e2e-clones" >> "$GITHUB_ENV"

	- name: Configure per-run package-manager caches
	# The benchmark republishes deterministic versions (e.g. hardhat@3.9.1)
	# whose content changes each run (new @nomicfoundation/edr pin), so any
	# package-manager cache that persists on the self-hosted runner can
	# resolve a prior run's stale republish. Point these caches at temp dirs
	# that are cleaned up at the end of the job (see "Clean up temp dir").
	#
	# Yarn berry: the global folder holds both the zip cache and the npm
	# packument metadata cache. YARN_* env vars override any .yarnrc.yml.
	#
	# Yarn classic: no override is needed. It doesn't have a persistent
	# metadata cache, and its tarball cache slug includes the tarball's
	# sha1.
	#
	# All caches live under PACKAGE_MANAGER_CACHES so the cleanup step can
	# remove that single directory without being updated for every cache
	# added here.
	run: \|
	set -euo pipefail
	PACKAGE_MANAGER_CACHES="${RUNNER_TEMP:?}/package-manager-caches"
	{
	echo "PACKAGE_MANAGER_CACHES=$PACKAGE_MANAGER_CACHES"
	echo "BUN_INSTALL_CACHE_DIR=$PACKAGE_MANAGER_CACHES/bun-install"
	echo "YARN_GLOBAL_FOLDER=$PACKAGE_MANAGER_CACHES/yarn-global"
	} >> "$GITHUB_ENV"

	- name: Install packages
	run: sfw pnpm install --frozen-lockfile --prefer-offline

	- name: Build
	run: pnpm build

	- name: Stamp the hardhat package for validation
	# Embed a unique per-run marker in the hardhat that gets published to
	# Verdaccio. Also record the pre-run version so the validation step
	# can tell whether hardhat was republished at all.
	run: \|
	echo "BENCH_RUN_STAMP=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_ENV"
	echo "HH_PRE_VER=$(node -p "require('./packages/hardhat/package.json').version")" >> "$GITHUB_ENV"
	npm pkg set benchRunStamp="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" --prefix packages/hardhat

	# pnpm 11 skips creating bin symlinks whose target isn't built at install
	# time; relink workspace bins (e.g. hardhat) now that the build output
	# exists. See pnpm/pnpm#10524.
	- name: Recreate workspace bin links
	run: pnpm rebuild -r

	- name: Clear stale pnpm metadata cache
	# The self-hosted runner reuses ~/.cache/pnpm across runs. The benchmark
	# republishes deterministic versions (e.g. hardhat@3.9.1) whose content
	# can change between runs (new @nomicfoundation/edr pin), so a prior
	# run's cached registry metadata makes pnpm-based scenario installs
	# resolve a stale hardhat with outdated dependencies.
	#
	# pnpm 11 ignores `npm_config_cache_dir`, so the metadata cache can't be
	# relocated.
	#
	# The content-addressable store is separate and kept on purpose: it's
	# keyed by integrity hash, so with fresh metadata pnpm derives the correct
	# hash and any stale entry is simply never matched. Keeping it is safe and
	# preserves install speed.
	run: rm -rf "${HOME:?}/.cache/pnpm"

	- name: Run regression benchmark
	env:
	ALCHEMY_URL: ${{ secrets.ALCHEMY_URL }}
	# Scenario/benchmark glob filters, always set by the setup job (`*`
	# matches all). Forwarded verbatim to bench:regression, which matches
	# them against scenario directory / benchmark (command or step) names.
	SCENARIO_FILTER: ${{ needs.setup.outputs.scenario_filter }}
	BENCHMARK_FILTER: ${{ needs.setup.outputs.benchmark_filter }}
	run: \|
	set -euo pipefail
	pnpm bench:regression \
	--use-local \
	--force-checkout \
	--output regression-report.json \
	--scenarios "$SCENARIO_FILTER" \
	--benchmarks "$BENCHMARK_FILTER"

	- name: Validate scenarios used the local Hardhat build
	# Diagnostic only: verifies each E2E scenario resolved the hardhat
	# published to Verdaccio by THIS run — identified by the stamp embedded
	# above — rather than a stale copy from a package-manager cache on the
	# self-hosted runner or the public npm registry. Emits warnings instead
	# of errors so it can never regress the benchmark runner.
	run: node .github/scripts/validate-local-hardhat-install.ts

	# Persist the (possibly partial) benchmark output in the job log so it
	# survives cleanup and is available for debugging.
	- name: Dump regression report on failure
	if: failure()
	run: \|
	echo "== regression-report.json =="
	cat regression-report.json 2>/dev/null \
	\|\| echo "(no regression-report.json produced)"

	# Comparing and recording are split so that a regression can be told
	# apart from a failed write. This invocation only reads, so it cannot
	# fail the way "Record baseline in the results repo" can.
	#
	# It can still fail on a fetch or clone error instead of on the alert.
	# Nothing here distinguishes the two, so the Slack message calls both a
	# regression. The step's error annotation on the run says which it was.
	#
	# The comparison is against the last recorded data point, i.e. the
	# previous push to main. So the baseline moves: a regression alerts
	# once, then becomes the new normal.
	- name: Compare benchmark result against baseline
	id: compare
	uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
	# A regression must not break main, so absorb the alert on baseline
	# runs and let the Slack step report it instead. Comparison runs
	# (`/bench`, dispatch) still fail.
	continue-on-error: ${{ needs.setup.outputs.is_baseline == 'true' }}
	with:
	tool: customSmallerIsBetter
	output-file-path: regression-report.json
	gh-repository: github.com/nomic-foundation-automation/hardhat-benchmark-results
	gh-pages-branch: main
	benchmark-data-dir-path: hardhat3
	# This invocation only reads the (public) results repo, so the
	# short-lived GITHUB_TOKEN suffices on every trigger. A token is
	# still required: with `gh-repository` set, the action rejects an
	# empty `github-token` during config validation.
	github-token: ${{ secrets.GITHUB_TOKEN }}
	auto-push: false
	alert-threshold: "110%"
	fail-on-alert: true
	summary-always: true

	- name: Clear the results-repo clone
	# "Compare benchmark result against baseline" leaves its clone of the
	# results repo in the workspace, and `git clone` refuses a non-empty
	# destination, so remove it before "Record baseline in the results
	# repo" clones again. `always()` because a comparison run that
	# fails on a regression has to clean up too.
	if: always()
	run: rm -rf "${GITHUB_WORKSPACE:?}/benchmark-data-repository"

	# Record the data point whether or not it regressed, so main's baseline
	# always tracks the newest commit. `fail-on-alert` is off here, so the
	# only way this fails is a genuine store failure (clone, auth or push
	# error) — which fails the run and gets the generic Slack notification
	# rather than the regression one.
	- name: Record baseline in the results repo
	id: record
	if: ${{ needs.setup.outputs.is_baseline == 'true' }}
	uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
	# `tool` through `benchmark-data-dir-path` must stay identical to
	# "Compare benchmark result against baseline" — the two invocations
	# have to address the same dataset.
	with:
	tool: customSmallerIsBetter
	output-file-path: regression-report.json
	gh-repository: github.com/nomic-foundation-automation/hardhat-benchmark-results
	gh-pages-branch: main
	benchmark-data-dir-path: hardhat3
	# Pushing to another repository needs the cross-repo PAT.
	github-token: ${{ secrets.BENCHMARK_GITHUB_TOKEN }}
	auto-push: true
	fail-on-alert: false
	# The comparison above already wrote the summary.
	summary-always: false

	- name: Notify failures and regressions
	# Fires on baseline runs for a failed run, or for a regression that
	# `continue-on-error` deliberately left green.
	#
	# Two details in the condition that look removable but are not:
	# 1. `failure()` suppresses the implicit `success()` that GitHub wraps
	# every step `if:` in. Without it, this step is skipped on the
	# failed runs it exists to report.
	# 2. It reads `outcome`, not `conclusion`, because `continue-on-error`
	# rewrites `conclusion` to `success`.
	#
	# The regression wording needs the record step to have succeeded too.
	# An infra error that trips both steps therefore reads as a plain
	# failure.
	if: ${{ needs.setup.outputs.is_baseline == 'true' && (failure() \|\| steps.compare.outcome == 'failure') }}
	uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
	with:
	webhook: ${{ secrets.GH_ACTION_NOTIFICATIONS_SLACK_WEBHOOK_URL }}
	webhook-type: webhook-trigger
	# Slack is the only channel a regression is reported on, so a
	# rejected delivery must fail the step rather than just warn — even
	# though on the green regression path that reddens the run.
	errors: true
	payload: \|
	{
	"workflow_name": "${{ github.workflow }}${{ (steps.compare.outcome == 'failure' && steps.record.outcome == 'success') && ' — perf regression on main (baseline updated)' \|\| '' }}",
	"run_url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
	}

	- name: Clean up temp dir
	# Self-hosted runners don't automatically wipe the E2E_CLONE_DIR and
	# PACKAGE_MANAGER_CACHES written to RUNNER_TEMP between jobs, so we do
	# this manually to avoid polluting future runs.
	if: always()
	run: rm -rf "$E2E_CLONE_DIR" "$PACKAGE_MANAGER_CACHES"

	report:
	name: Report result to PR
	needs: [setup, regression-benchmark]
	# 1. run on benchmark failure but skip when the run is superseded (concurrency)
	# 2. Only report back to the PR for `/bench` comment triggers
	# 3. Only report back if the benchmark actually started
	# prettier-ignore
	if: >-
	!cancelled()
	&& github.event_name == 'issue_comment'
	&& needs.regression-benchmark.result != 'skipped'
	runs-on: ubuntu-latest
	permissions:
	issues: write # post the result comment on the PR
	pull-requests: write # post status comments/reactions on the PR
	steps:
	- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
	env:
	RESULT: ${{ needs.regression-benchmark.result }}
	BENCH_REF: ${{ needs.setup.outputs.bench_ref }}
	with:
	script: \|
	const { RESULT, BENCH_REF } = process.env;
	const runUrl =
	`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` +
	`/actions/runs/${context.runId}`;
	const target = `\`${BENCH_REF.slice(0, 12)}\``;
	const body =
	RESULT === "success"
	? `✅ Regression benchmark passed for ${target}.\n\n` +
	`[View workflow run](${runUrl})`
	: `❌ Regression benchmark failed for ${target}. This is either ` +
	`a detected performance regression or an infrastructure ` +
	`failure — see the run for details.\n\n` +
	`[View workflow run](${runUrl})`;
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.payload.issue.number,
	body,
	});

Navigation Menu

Uh oh!

Avoid interpolating config variable values into error messages #1165

Workflow file

Avoid interpolating config variable values into error messages #1165

Uh oh!

Workflow file for this run