publish: release 0.15.0 #995

Workflow file for this run

.github/workflows/test-e2e.yml at ec3c924

	# End-to-end transparency test suite — resource-intensive, runs for active non-draft PRs.
	#
	# Trigger strategy
	# ----------------
	# • Automatic: fires when a PR is opened, re-opened, marked ready for review,
	# or receives new commits.
	# • Draft PRs: the workflow may still be triggered by the PR event, but the job
	# itself is skipped until the PR is no longer a draft.
	# • Manual: use the "Run workflow" button in the Actions UI (workflow_dispatch)
	# whenever you need to re-run on demand.
	name: Transparency E2E

	on:
	pull_request:
	types: [opened, synchronize, reopened, ready_for_review]
	workflow_dispatch:
	inputs:
	reason:
	description: "Why are you running the E2E suite manually?"
	required: false
	default: "manual re-run after new commits"

	concurrency:
	# One run per PR at a time; do NOT cancel in-progress runs so a partial
	# result is never silently discarded.
	group: "e2e-${{ github.head_ref \|\| github.ref_name }}"
	cancel-in-progress: false

	jobs:
	transparency-e2e-linux:
	name: "E2E / Transparency"
	if: github.event_name != 'pull_request' \|\| !github.event.pull_request.draft
	# pinned: pytest-mpl baselines are generated on this exact image (FreeType/Agg rendering is version-sensitive)
	runs-on: ubuntu-24.04
	timeout-minutes: 30
	env:
	MPLBACKEND: Agg

	steps:
	- name: Checkout code
	uses: actions/checkout@v7

	- name: Set up environment
	uses: ./.github/actions/setup-raitap-env
	with:
	python-version: "3.13"
	# torchattacks + foolbox added so the Parity step can exercise the
	# robustness adversarial-attack parity tests in this 3.13 lane.
	sync-command: uv sync --group dev --extra torch-cpu --extra transparency --extra mlflow --extra metrics --extra onnx-cpu --extra torchattacks --extra foolbox

	# Split by marker so each category renders as its own collapsible section
	# in the job log. ``if: always()`` keeps later categories running after an
	# earlier failure; any failing step still fails the job.
	- name: Visual regression
	run: uv run pytest -m "e2e and visual" -v --tb=long --mpl

	- name: Parity
	if: always()
	# Captum/torchattacks/foolbox parity run here; marabou parity collects
	# but skips (maraboupy is absent in this 3.13 lane) — it runs in the
	# E2E / Robustness lane below.
	run: uv run pytest -m "e2e and parity" -v --tb=long

	- name: Transparency E2E (remaining)
	if: always()
	run: uv run pytest -m "e2e and not visual and not parity and not robustness" -v --tb=long

	detection-e2e-linux:
	# Drives the bundled ``contributor-configs/fasterrcnn-udacity/`` config
	# end-to-end: torchvision ``fasterrcnn_resnet50_fpn_v2`` on four Udacity
	# dashcam frames, per-box Integrated Gradients with the ``detection_image``
	# visualiser, HTMLReporter output. Catches regressions in the detection
	# task family (TorchBackend autodetect, per-box explain phase, detection
	# visualiser, DetectionMetrics) at the integrated pipeline level.
	#
	# CPU runs of fasterrcnn + IG OOM the GitHub-hosted ubuntu-latest runner
	# (~7 GB RAM). Gated behind ``ENABLE_HOSTED_GPU_CI`` so the job lights up
	# only when an org-level GPU lane exists. Until then, contributors run
	# the config locally (``contributor-configs/fasterrcnn-udacity/run.sh``).
	name: "Detection E2E (Linux)"
	if: ${{ vars.ENABLE_HOSTED_GPU_CI == 'true' && (github.event_name != 'pull_request' \|\| !github.event.pull_request.draft) }}
	runs-on: ubuntu-latest
	timeout-minutes: 30
	env:
	MPLBACKEND: Agg

	steps:
	- name: Checkout code
	uses: actions/checkout@v7

	- name: Set up environment
	uses: ./.github/actions/setup-raitap-env
	with:
	python-version: "3.13"
	# Mirror ``contributor-configs/fasterrcnn-udacity/run.sh`` so this
	# job exercises the same extras set a contributor gets locally.
	sync-command: uv sync --group dev --extra torch-cpu --extra captum --extra metrics --extra reporting

	- name: Cache torch hub weights + raitap sample data
	# fasterrcnn_resnet50_fpn_v2 weights (~170MB) + UdacitySelfDriving
	# samples (small) — both downloaded on first pipeline run,
	# deterministic per cfg. Key on pyproject.toml + the assessment.yaml
	# so a torchvision bump or cfg change cleanly invalidates.
	uses: actions/cache@v6
	with:
	path: \|
	~/.cache/torch
	~/.cache/raitap
	key: detection-e2e-assets-v1-${{ hashFiles('pyproject.toml', 'contributor-configs/fasterrcnn-udacity/assessment.yaml') }}
	restore-keys: \|
	detection-e2e-assets-v1-

	- name: Pre-fetch Udacity sample images
	# Warm ~/.cache/raitap before the pipeline runs so a cache miss on the
	# samples download surfaces as its own failing step rather than
	# masquerading as a pipeline error.
	run: uv run python -c "from raitap.data.samples import _load_sample; _load_sample('UdacitySelfDriving')"

	- name: Run detection E2E pipeline
	# --acknowledge-preprocessing-off: torchvision detection models do
	# their own internal resize/normalise; a RAITAP-level preprocessing
	# step would break box-coord alignment. Mirrors run.sh.
	run: \|
	set -o pipefail
	uv run raitap \
	--config-dir contributor-configs/fasterrcnn-udacity \
	--config-name assessment \
	--acknowledge-preprocessing-off 2>&1 \| tee detection-run.log

	- name: Assert DetectionMetrics resolved (in metrics metadata.json)
	# Pipeline persists ``resolved_target`` to ``metrics/metadata.json``;
	# checking the file is more robust than scraping log output.
	run: \|
	find outputs -path "*/metrics/metadata.json" -print \| head -1 \| xargs grep -F "raitap.metrics.DetectionMetrics"

	- name: Assert outputs directory exists
	run: test -d outputs

	- name: Assert detection-image PNG produced
	run: find outputs -type f -name "*.png" \| head \| grep -q .

	- name: Assert HTMLReporter output exists
	# ``reporting.filename`` in the contributor config is
	# ``fasterrcnn_udacity_report``; HTMLReporter appends ``.html``.
	run: find outputs -type f -name "fasterrcnn_udacity_report.html" \| grep -q .

	- name: Upload outputs on failure
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: detection-e2e-outputs
	path: outputs
	retention-days: 7

	marabou-e2e-linux:
	name: "E2E / Robustness"
	if: github.event_name != 'pull_request' \|\| !github.event.pull_request.draft
	runs-on: ubuntu-latest
	timeout-minutes: 10
	env:
	MPLBACKEND: Agg

	steps:
	- name: Checkout code
	uses: actions/checkout@v7

	- name: Set up environment
	uses: ./.github/actions/setup-raitap-env
	with:
	# maraboupy 2.0.0 publishes cp311 wheels only — pin Python to 3.11.
	python-version: "3.11"
	sync-command: uv sync --group dev --extra torch-cpu --extra marabou

	- name: Install onnxruntime for ONNX inference in tests
	# The marabou extra pulls onnx (graph format) but not onnxruntime
	# (inference engine). The e2e tests run the original ACAS Xu
	# forward pass via onnxruntime to pick the target class, so install
	# it explicitly in this lane.
	run: uv pip install "onnxruntime>=1.17"

	- name: Run Marabou E2E tests
	# Scope collection to the marabou file so we don't import unrelated
	# test modules whose deps (torchmetrics, etc.) aren't installed in
	# this minimal lane.
	run: uv run pytest src/raitap/robustness/tests/test_e2e_marabou_acas_xu.py -m e2e -v --tb=long

	- name: Marabou parity
	if: always()
	# Verdict parity vs a direct maraboupy solve. Scoped to the parity file
	# for the same minimal-deps reason as the e2e step above.
	run: uv run pytest src/raitap/robustness/tests/test_marabou_parity.py -m parity -v --tb=long

	example-consumer-e2e-linux:
	# Drives `example/` as a standalone consumer end-to-end: `uv sync` against
	# the consumer pyproject, then full pipeline via the `raitap`
	# console-script. Covers the bits that only show up at the
	# consumer-project boundary: console-script entry, `RaitapSearchPathPlugin`
	# discovering bundled groups from outside the raitap source tree,
	# `tool.uv.package = false` + `tool.uv.sources` pattern, lazy_import
	# contract under a real install, and the live auto-deps bootstrap
	# (`uv add` + `os.execv` re-exec) with extras already pinned.
	#
	# The bootstrap-from-zero path (bare ``pip install raitap`` then
	# ``raitap --demo``) is covered by ``bare-bootstrap-from-zero-linux``
	# below.
	name: "example/ consumer E2E (Linux)"
	if: github.event_name != 'pull_request' \|\| !github.event.pull_request.draft
	runs-on: ubuntu-latest
	timeout-minutes: 15
	env:
	MPLBACKEND: Agg

	steps:
	- name: Checkout code
	uses: actions/checkout@v7

	- name: Set up Python
	uses: actions/setup-python@v6
	with:
	python-version: "3.13"

	- name: Install uv (with wheel cache)
	uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: "example/uv.lock"

	- name: Cache torch hub weights + raitap sample data
	# vit_b_32 weights (~350MB) + imagenet_samples (small) — both
	# downloaded on first pipeline run, deterministic per cfg.
	uses: actions/cache@v6
	with:
	path: \|
	~/.cache/torch
	~/.cache/raitap
	key: example-e2e-assets-v1-${{ hashFiles('example/assessment.yaml') }}
	restore-keys: \|
	example-e2e-assets-v1-

	- name: Swap torch backend for CI host (runner-only)
	# Committed `example/pyproject.toml` pins `torch-intel` (the Intel-laptop
	# default for this project). GH ubuntu runners have no XPU; pulling
	# IPEX would waste ~200MB and never be used under `hardware=cpu`.
	# Swap to `torch-cpu` in the workspace copy only — never committed.
	working-directory: example
	run: \|
	sed -i 's/torch-intel/torch-cpu/g' pyproject.toml
	rm -f uv.lock
	cat pyproject.toml

	- name: Sync consumer venv
	working-directory: example
	run: uv sync

	- name: Run example via console-script (bootstrap + re-exec live)
	# `--allow-project-edit` consents to the inferred `uv add` call.
	# Bootstrap always re-asserts the full inferred extras set (it
	# doesn't diff against the currently-pinned set), so even with
	# everything already pinned the call goes through — `uv add` is
	# idempotent at the resolver level, but `os.execv` + the sentinel
	# short-circuit + the post-relaunch pipeline import all execute.
	# `hardware=cpu` Hydra override pins the CPU backend regardless of
	# host probe.
	working-directory: example
	run: \|
	set -o pipefail
	uv run raitap --config-name assessment hardware=cpu --allow-project-edit 2>&1 \| tee run.log

	- name: Assert bootstrap fired and pipeline produced outputs
	working-directory: example
	run: \|
	grep -F "uv add raitap[" run.log
	test -d outputs
	find outputs -type f -name 'report*' \| grep -q .

	bare-bootstrap-from-zero-linux:
	# Proves the headline UX promise from ``docs/using-raitap/get-it-running.md``:
	# ``pip install raitap`` (no extras), ``raitap --demo``, RAITAP figures out
	# the rest. This exercises the bootstrap-from-zero path end-to-end:
	# - compose Hydra cfg on a venv with NO torch / torchvision / onnxruntime
	# - infer extras (host probe picks ``torch-cpu`` here)
	# - install via ``uv add raitap[<extras>]``
	# - ``os.execv`` re-exec under the sentinel
	# - second-pass compose with the freshly installed backend
	# - run the demo pipeline + reports
	#
	# Bootstrap composes the cfg by importing every adapter family
	# ``__init__.py`` — the ``lazy_import`` contract in ``raitap.utils.lazy``
	# is what lets that happen torch-free. A regression here means a
	# contributor added a top-level ``import torch`` / ``import torchvision``
	# somewhere in the registration chain. The per-family
	# ``test_partial_extras_safe.py`` tests pinpoint the offender;
	# ``deps/tests/test_bootstrap_from_zero.py`` covers the bootstrap call
	# path itself. This job is the integrated proof.
	name: "Bare bootstrap-from-zero (Linux)"
	if: github.event_name != 'pull_request' \|\| !github.event.pull_request.draft
	runs-on: ubuntu-latest
	timeout-minutes: 15
	env:
	MPLBACKEND: Agg

	steps:
	- name: Checkout code
	uses: actions/checkout@v7

	- name: Set up Python
	uses: actions/setup-python@v6
	with:
	python-version: "3.13"

	- name: Install uv (with wheel cache)
	uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: "uv.lock"

	- name: Cache torch hub weights + raitap sample data
	uses: actions/cache@v6
	with:
	path: \|
	~/.cache/torch
	~/.cache/raitap
	key: bare-bootstrap-assets-v1-${{ hashFiles('src/raitap/configs/demo.yaml') }}
	restore-keys: \|
	bare-bootstrap-assets-v1-

	- name: Create bare consumer project (outside the raitap repo)
	# ``uv add`` needs a project pyproject.toml in cwd to operate. Build a
	# throwaway one in /tmp that depends on bare ``raitap`` from the
	# checkout — no extras pinned. This is the exact "first-time user"
	# state the get-it-running docs imply.
	run: \|
	mkdir -p /tmp/bare-consumer
	cd /tmp/bare-consumer
	cat > pyproject.toml <<EOF
	[project]
	name = "bare-consumer"
	version = "0.0.0"
	requires-python = ">=3.11,<3.14"
	dependencies = ["raitap"]

	[tool.uv]
	package = false

	[tool.uv.sources]
	raitap = { path = "${GITHUB_WORKSPACE}", editable = false }
	EOF
	uv sync

	- name: Run demo end-to-end via bootstrap-from-zero
	# ``-y`` consents to ``uv add`` mutating the throwaway pyproject. The
	# bootstrap probes the GH ubuntu host (no GPU) → infers ``torch-cpu``
	# + adapter extras from the demo cfg → installs → re-execs → runs.
	working-directory: /tmp/bare-consumer
	run: \|
	set -o pipefail
	uv run raitap --demo -y 2>&1 \| tee run.log

	- name: Assert bootstrap fired with torch-cpu + pipeline produced outputs
	working-directory: /tmp/bare-consumer
	run: \|
	grep -F "torch-cpu" run.log
	grep -F "uv add raitap[" run.log
	test -d outputs
	find outputs -type f \| head -20

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

publish: release 0.15.0 #995

Workflow file

publish: release 0.15.0 #995

Uh oh!

Workflow file for this run