From 56c6239df72b3859743000c952e8eac25beb1620 Mon Sep 17 00:00:00 2001 From: Sartaj Date: Fri, 20 Feb 2026 23:22:06 -0600 Subject: [PATCH 1/2] refactor: replace insta CLI tests with dual-mode VHS snapshot testing Remove api_snaps.rs (dead code referencing nonexistent path), cli_example_snaps.rs, and their associated .snap files. Keep wizard_tests.rs (pure unit tests). VHS tapes now run in two modes via TEST_MODE env var: - mock: diffs output against committed expected files (PR, fast) - prod: smoke tests output is non-empty (nightly, live data) Add TESTING.md documenting data lineage, testing tiers, and the dual-mode approach. Add govbot mock schema validation to CI. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/synthetic-test.yml | 16 +- .github/workflows/terminal-screenshots.yml | 35 ++- .github/workflows/validate-snapshots.yml | 78 ++++- TESTING.md | 199 ++++++++++++ actions/govbot/examples/govbot-clone-list.sh | 1 - actions/govbot/examples/govbot-help.sh | 1 - actions/govbot/examples/logs-basic.sh | 1 - .../expected/govbot-clone-list.txt} | 8 - .../expected/govbot-help.txt} | 8 - actions/govbot/tapes/expected/logs-basic.txt | 5 + actions/govbot/tapes/govbot-clone-list.tape | 18 +- actions/govbot/tapes/govbot-help.tape | 20 +- actions/govbot/tapes/logs-basic.tape | 19 +- .../govbot/tapes/nightly/synthetic-test.tape | 30 +- actions/govbot/tests/api_snaps.rs | 91 ------ actions/govbot/tests/cli_example_snaps.rs | 286 ------------------ .../api_snaps__log_entry_structure.snap | 10 - .../api_snaps__vote_event_results.snap | 9 - ...li_example_snaps__snapshot@logs_basic.snap | 8 - 19 files changed, 400 insertions(+), 443 deletions(-) create mode 100644 TESTING.md delete mode 100644 actions/govbot/examples/govbot-clone-list.sh delete mode 100644 actions/govbot/examples/govbot-help.sh delete mode 100644 actions/govbot/examples/logs-basic.sh rename actions/govbot/{tests/snapshots/cli_example_snaps__snapshot@govbot_clone_list.snap => tapes/expected/govbot-clone-list.txt} (72%) rename actions/govbot/{tests/snapshots/cli_example_snaps__snapshot@govbot_help.snap => tapes/expected/govbot-help.txt} (93%) create mode 100644 actions/govbot/tapes/expected/logs-basic.txt delete mode 100644 actions/govbot/tests/api_snaps.rs delete mode 100644 actions/govbot/tests/cli_example_snaps.rs delete mode 100644 actions/govbot/tests/snapshots/api_snaps__log_entry_structure.snap delete mode 100644 actions/govbot/tests/snapshots/api_snaps__vote_event_results.snap delete mode 100644 actions/govbot/tests/snapshots/cli_example_snaps__snapshot@logs_basic.snap diff --git a/.github/workflows/synthetic-test.yml b/.github/workflows/synthetic-test.yml index 6f92066c..24330462 100644 --- a/.github/workflows/synthetic-test.yml +++ b/.github/workflows/synthetic-test.yml @@ -118,6 +118,18 @@ jobs: echo "=== Build output ===" ls -la /tmp/govbot-test/docs/ + - name: Run demo tapes in prod mode + if: github.event_name != 'pull_request' + run: | + cd actions/govbot + export TEST_MODE=prod + for tape in tapes/*.tape; do + [ -f "$tape" ] || continue + echo "::group::Recording $tape (prod mode)" + vhs "$tape" || true + echo "::endgroup::" + done + - name: Upload GIF to synthetic-test release if: always() env: @@ -175,7 +187,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: synthetic-test-recording - path: /tmp/govbot-synthetic-test.gif + path: | + /tmp/govbot-synthetic-test.gif + actions/govbot/tapes/*.gif retention-days: 7 - name: Upload diagnostics diff --git a/.github/workflows/terminal-screenshots.yml b/.github/workflows/terminal-screenshots.yml index 5ecda3fa..b0a87c1e 100644 --- a/.github/workflows/terminal-screenshots.yml +++ b/.github/workflows/terminal-screenshots.yml @@ -3,15 +3,16 @@ name: Terminal Screenshots on: pull_request: paths: - - "actions/govbot/src/**" - "actions/govbot/tapes/*.tape" + - "actions/govbot/tapes/expected/**" + - "actions/govbot/src/**" - "actions/govbot/Cargo.toml" - "actions/govbot/Cargo.lock" - ".github/workflows/terminal-screenshots.yml" jobs: terminal-screenshots: - name: Generate Terminal GIFs + name: Generate Terminal GIFs (Mock Mode) runs-on: ubuntu-latest permissions: contents: read @@ -59,19 +60,41 @@ jobs: rm vhs.deb vhs --version - - name: Record terminal GIFs + - name: Record demo tapes (mock mode with snapshot assertions) run: | cd actions/govbot + export TEST_MODE=mock + FAILED=0 for tape in tapes/*.tape; do [ -f "$tape" ] || continue echo "::group::Recording $tape" - vhs "$tape" - echo "::endgroup::" + if vhs "$tape"; then + echo "::endgroup::" + else + echo "::endgroup::" + echo "::error::Tape failed: $tape" + FAILED=1 + fi done + if [ "$FAILED" -eq 1 ]; then + echo "::error::One or more tapes failed snapshot assertions" + exit 1 + fi + + - name: Record synthetic test (mock mode) + run: | + cd actions/govbot + export TEST_MODE=mock + export GOVBOT_SRC="$PWD" + rm -rf /tmp/govbot-test + vhs tapes/nightly/synthetic-test.tape - name: Upload GIFs as artifacts + if: always() uses: actions/upload-artifact@v4 with: name: terminal-screenshots - path: actions/govbot/tapes/*.gif + path: | + actions/govbot/tapes/*.gif + /tmp/govbot-synthetic-test.gif retention-days: 30 diff --git a/.github/workflows/validate-snapshots.yml b/.github/workflows/validate-snapshots.yml index 186eaae6..59521f99 100644 --- a/.github/workflows/validate-snapshots.yml +++ b/.github/workflows/validate-snapshots.yml @@ -144,12 +144,88 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: Run tests run: | cd actions/govbot cargo test + govbot-mock-schema-validation: + name: Validate Govbot Mock Data + runs-on: ubuntu-latest + needs: detect-changes + if: needs.detect-changes.outputs.govbot == 'true' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install jsonschema + run: pip install jsonschema + + - name: Validate govbot mock data against schemas + run: | + python3 << 'PYEOF' + import json + import sys + from pathlib import Path + from jsonschema import validate, ValidationError + + schemas_dir = Path("actions/format/schemas") + mocks_dir = Path("actions/govbot/mocks/.govbot/repos") + + # Load schemas + metadata_schema = json.load(open(schemas_dir / "metadata.schema.json")) + action_log_schema = json.load(open(schemas_dir / "action_log.schema.json")) + vote_event_log_schema = json.load(open(schemas_dir / "vote_event_log.schema.json")) + + errors = [] + validated = 0 + + # Validate all metadata.json files in mock repos + for metadata_file in mocks_dir.rglob("metadata.json"): + # Skip data.json (DCAT descriptor, different schema) + if metadata_file.name == "data.json": + continue + try: + data = json.load(open(metadata_file)) + validate(instance=data, schema=metadata_schema) + print(f" {metadata_file}") + validated += 1 + except ValidationError as e: + errors.append(f"{metadata_file}: {e.message}") + print(f" {metadata_file}: {e.message}") + + # Validate all log files in mock repos + for log_file in mocks_dir.rglob("logs/*.json"): + try: + data = json.load(open(log_file)) + if "action" in data and "bill_id" in data: + validate(instance=data, schema=action_log_schema) + elif "votes" in data and "counts" in data: + validate(instance=data, schema=vote_event_log_schema) + else: + print(f" {log_file}: unknown log type, skipping") + continue + print(f" {log_file}") + validated += 1 + except ValidationError as e: + errors.append(f"{log_file}: {e.message}") + print(f" {log_file}: {e.message}") + + print(f"\nValidated {validated} file(s)") + if errors: + print(f"{len(errors)} error(s):") + for err in errors: + print(f" - {err}") + sys.exit(1) + else: + print("All mock data validates against schemas") + PYEOF + format-snapshots: runs-on: ubuntu-latest needs: detect-changes diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 00000000..edb144a6 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,199 @@ +# Testing Strategy + +This document describes how testing works across the govbot monorepo. + +## Data Lineage & Mock Dependencies + +``` +UPSTREAM PIPELINE (produces git repos) +┌─────────┐ ┌──────────┐ ┌───────────┐ ┌──────────────────┐ +│ scrape │ → │ format │ → │ extract │ → │ git repos │ +│ │ │ │ │ │ │ (wy-legislation) │ +└─────────┘ └──────────┘ └───────────┘ └──────────────────┘ + │ │ │ + scrape/ format/snapshots/ │ + prod-mocks (tested via render_snapshot.sh │ + + schema validation) │ + │ +═══════════════ GIT REPO STRUCTURE IS THE CONTRACT ═══════════════ + │ + govbot/mocks/.govbot/repos/ │ + (captured from real repos │ + via `just mocks`) │ + │ │ + ┌─────────────────────────────────┐ │ + │ govbot clone → logs → tag → build │ + └─────────────────────────────────┘ + +Mock data sources: + • scrape/prod-mocks-* → captured from real scrape runs → fed to format tests + • govbot/mocks/.govbot/ → captured from real published repos → fed to govbot tests + • format/snapshots/wy/ → generated by running format on scrape mocks +``` + +## Testing Tiers + +``` +Tier 3: Nightly Visual E2E VHS tapes in prod mode (live data, smoke assertions) + │ GIF uploaded to releases for visual review + │ +Tier 2: PR Visual Integration VHS tapes in mock mode (deterministic, snapshot assertions) + │ GIF uploaded as PR artifact + │ +Tier 1: Schema Contracts JSON Schema validation of mock data + format output + │ Catches interface breakage between pipeline stages + │ +Tier 0: Unit Tests Wizard round-trip tests, pure logic (keep in Rust/insta) + No I/O, no VHS needed +``` + +## Dual-Mode VHS Testing + +Every VHS tape runs in two modes controlled by environment variables: + +| Mode | When | Data Source | Assertion Level | +|------|------|-------------|-----------------| +| **Mock** (`TEST_MODE=mock`) | PR (fast) | Pre-populated from `mocks/` | Snapshot diff (byte-exact match against committed expected files) | +| **Prod** (`TEST_MODE=prod`) | Nightly | Live cloned repos | Smoke test (exit code 0, non-empty output) | + +### Environment Variables + +- `TEST_MODE` — `mock` or `prod` (default: `prod`) +- `GOVBOT_DIR` — Path to `.govbot` directory containing repos (default: `mocks/.govbot` for demo tapes) + +### How Assertions Work + +Each tape defines a short helper function `sk()` (snapshot check) and uses `Wait+Screen` +to assert VHS sees `SNAP_OK` on screen: + +```tape +# Define assertion helper at tape start +Type "sk() { if [ ${TEST_MODE:-prod} = mock ]; then diff $1 $2 && echo SNAP_OK || echo SNAP_FAIL; else [ -s $1 ] && echo SNAP_OK || echo SNAP_FAIL; fi; }" +Enter + +# Run command, capture output to temp file, display it +Type "govbot logs --govbot-dir mocks/.govbot > /tmp/lb.txt 2>&1 && cat /tmp/lb.txt" +Enter +Sleep 3s + +# Clear screen so assertion output is visible +Type "clear" +Enter + +# Assert: mock mode diffs against expected file, prod mode checks non-empty +Type "sk /tmp/lb.txt tapes/expected/logs-basic.txt" +Enter +Wait+Screen@5s /SNAP_OK/ +``` + +## Per-Action Test Inventory + +### govbot (Rust CLI) + +| Test | Type | Location | Mode | +|------|------|----------|------| +| Wizard round-trip | Rust unit test (insta) | `tests/wizard_tests.rs` | `cargo test` | +| `govbot --help` | VHS tape | `tapes/govbot-help.tape` | mock/prod | +| `govbot clone --list` | VHS tape | `tapes/govbot-clone-list.tape` | mock/prod | +| `govbot logs` | VHS tape | `tapes/logs-basic.tape` | mock/prod | +| Full pipeline E2E | VHS tape | `tapes/nightly/synthetic-test.tape` | mock/prod | + +### format (Python) + +| Test | Type | Location | +|------|------|----------| +| Schema validation | JSON Schema | `validate-snapshots.yml` → format-snapshots job | +| Snapshot comparison | `render-snapshots.sh` | `validate-snapshots.yml` → format-snapshots job | + +### pipeline-manager / report-publisher + +| Test | Type | Location | +|------|------|----------| +| Snapshot comparison | `render-snapshots.sh` | `validate-snapshots.yml` | + +## Running Tests Locally + +### Rust unit tests (Tier 0) + +```bash +cd actions/govbot +just test # Run all tests +just test-single wizard_tests # Run specific test +just review # Review snapshot changes (insta) +``` + +### VHS demo tapes in mock mode (Tier 2) + +```bash +cd actions/govbot + +# Build binary first +just build-release + +# Record all demo tapes +TEST_MODE=mock just record + +# Record a specific tape +TEST_MODE=mock just record govbot-help +``` + +### VHS synthetic test in mock mode (Tier 2) + +```bash +cd actions/govbot +export PATH="$PWD/target/release:$PATH" +TEST_MODE=mock GOVBOT_SRC="$PWD" vhs tapes/nightly/synthetic-test.tape +``` + +### VHS tapes in prod mode (Tier 3) + +```bash +cd actions/govbot +export PATH="$PWD/target/release:$PATH" +# Requires network access — clones real repos +vhs tapes/nightly/synthetic-test.tape +``` + +## Updating Mocks + +When upstream data format changes, refresh mock data: + +```bash +cd actions/govbot +just mocks # Default: refreshes wy and gu +just mocks il ny # Refresh specific states +``` + +This clones real repos, prunes to 5 bills / 3 logs per session, and removes `.git` directories. + +## Updating Expected Output + +When govbot output changes (new fields, formatting changes): + +```bash +cd actions/govbot + +# Regenerate expected output files +govbot --help > tapes/expected/govbot-help.txt 2>&1 +govbot clone --list > tapes/expected/govbot-clone-list.txt 2>&1 +govbot logs --govbot-dir mocks/.govbot > tapes/expected/logs-basic.txt 2>&1 + +# Verify tapes still pass +TEST_MODE=mock just record +``` + +## CI Workflows + +| Workflow | Trigger | What it does | +|----------|---------|--------------| +| `validate-snapshots.yml` | Push to main, PR | Rust unit tests + schema validation | +| `terminal-screenshots.yml` | PR (tape/src/Cargo changes) | VHS demo tapes in mock mode with snapshot assertions | +| `synthetic-test.yml` | Nightly + PR (tape/src changes) | Full E2E pipeline in prod and/or mock mode | + +## Decision Tree: When to Add What Kind of Test + +1. **Pure logic with no I/O?** → Rust unit test with insta (`tests/wizard_tests.rs`) +2. **CLI command with deterministic output?** → VHS demo tape with expected output file +3. **Full pipeline flow?** → Add to synthetic-test.tape +4. **Data format contract?** → JSON Schema in `actions/format/schemas/` +5. **New mock data needed?** → `just mocks `, then validate schemas pass diff --git a/actions/govbot/examples/govbot-clone-list.sh b/actions/govbot/examples/govbot-clone-list.sh deleted file mode 100644 index 843c33e2..00000000 --- a/actions/govbot/examples/govbot-clone-list.sh +++ /dev/null @@ -1 +0,0 @@ -govbot clone --list \ No newline at end of file diff --git a/actions/govbot/examples/govbot-help.sh b/actions/govbot/examples/govbot-help.sh deleted file mode 100644 index a22d6e9e..00000000 --- a/actions/govbot/examples/govbot-help.sh +++ /dev/null @@ -1 +0,0 @@ -govbot --help diff --git a/actions/govbot/examples/logs-basic.sh b/actions/govbot/examples/logs-basic.sh deleted file mode 100644 index 47118436..00000000 --- a/actions/govbot/examples/logs-basic.sh +++ /dev/null @@ -1 +0,0 @@ -govbot logs diff --git a/actions/govbot/tests/snapshots/cli_example_snaps__snapshot@govbot_clone_list.snap b/actions/govbot/tapes/expected/govbot-clone-list.txt similarity index 72% rename from actions/govbot/tests/snapshots/cli_example_snaps__snapshot@govbot_clone_list.snap rename to actions/govbot/tapes/expected/govbot-clone-list.txt index 6679a69a..5a0564c7 100644 --- a/actions/govbot/tests/snapshots/cli_example_snaps__snapshot@govbot_clone_list.snap +++ b/actions/govbot/tapes/expected/govbot-clone-list.txt @@ -1,11 +1,3 @@ ---- -source: tests/cli_example_snaps.rs -expression: "&formatted_stdout" ---- -Command: -govbot clone --list - -Output: Available repos: ak al diff --git a/actions/govbot/tests/snapshots/cli_example_snaps__snapshot@govbot_help.snap b/actions/govbot/tapes/expected/govbot-help.txt similarity index 93% rename from actions/govbot/tests/snapshots/cli_example_snaps__snapshot@govbot_help.snap rename to actions/govbot/tapes/expected/govbot-help.txt index 773949e9..d8f95f13 100644 --- a/actions/govbot/tests/snapshots/cli_example_snaps__snapshot@govbot_help.snap +++ b/actions/govbot/tapes/expected/govbot-help.txt @@ -1,11 +1,3 @@ ---- -source: tests/cli_example_snaps.rs -expression: "&formatted_stdout" ---- -Command: -govbot --help - -Output: Process pipeline log files with type-safe reactive streams Usage: govbot [COMMAND] diff --git a/actions/govbot/tapes/expected/logs-basic.txt b/actions/govbot/tapes/expected/logs-basic.txt new file mode 100644 index 00000000..8516c60b --- /dev/null +++ b/actions/govbot/tapes/expected/logs-basic.txt @@ -0,0 +1,5 @@ +{"bill":{"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0001","legislative_session":"2025","title":"General government appropriations-2."},"id":"HB0001","log":{"action":{"classification":["filing"],"date":"2025-01-29T17:06:54+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0001"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0001/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0001/logs/20250129T170654Z_h_received_for_introduction.json"},"timestamp":"20250129T170654Z"} +{"bill":{"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0002","legislative_session":"2025","title":"Hunting license application fees increase."},"id":"HB0002","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:16:16+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0002"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0002/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0002/logs/20250102T191616Z_h_received_for_introduction.json"},"timestamp":"20250102T191616Z"} +{"bill":{"abstracts":[{"abstract":"2025/Summaries/HB0005.pdf","note":"summary"}],"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0005","legislative_session":"2025","title":"Fishing outfitters and guides-registration of fishing boats."},"id":"HB0005","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:18:48+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0005"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0005/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0005/logs/20250102T191848Z_h_received_for_introduction.json"},"timestamp":"20250102T191848Z"} +{"bill":{"abstracts":[{"abstract":"2025/Summaries/HB0004.pdf","note":"summary"}],"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0004","legislative_session":"2025","title":"Snowmobile registration and user fees."},"id":"HB0004","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:17:44+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0004"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0004/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0004/logs/20250102T191744Z_h_received_for_introduction.json"},"timestamp":"20250102T191744Z"} +{"bill":{"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0003","legislative_session":"2025","title":"Animal abuse-predatory animals."},"id":"HB0003","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:17:11+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0003"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0003/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0003/logs/20250102T191711Z_h_received_for_introduction.json"},"timestamp":"20250102T191711Z"} diff --git a/actions/govbot/tapes/govbot-clone-list.tape b/actions/govbot/tapes/govbot-clone-list.tape index 393ece0f..3ea1ff0d 100644 --- a/actions/govbot/tapes/govbot-clone-list.tape +++ b/actions/govbot/tapes/govbot-clone-list.tape @@ -1,4 +1,6 @@ # govbot clone --list output +# Dual-mode: TEST_MODE=mock asserts output matches expected file +# TEST_MODE=prod (default) just asserts non-empty output Output tapes/govbot-clone-list.gif Set Shell bash @@ -7,6 +9,20 @@ Set Width 900 Set Height 800 Set Padding 20 -Type "govbot clone --list" +# Define assertion helper +Type "sk() { if [ ${TEST_MODE:-prod} = mock ]; then diff $1 $2 && echo SNAP_OK || echo SNAP_FAIL; else [ -s $1 ] && echo SNAP_OK || echo SNAP_FAIL; fi; }" +Enter +Sleep 500ms + +Type "govbot clone --list > /tmp/gcl.txt 2>&1 && cat /tmp/gcl.txt" Enter Sleep 2s + +Type "clear" +Enter +Sleep 500ms + +Type "sk /tmp/gcl.txt tapes/expected/govbot-clone-list.txt" +Enter +Wait+Screen@5s /SNAP_OK/ +Sleep 1s diff --git a/actions/govbot/tapes/govbot-help.tape b/actions/govbot/tapes/govbot-help.tape index 8569cabb..2aa4c498 100644 --- a/actions/govbot/tapes/govbot-help.tape +++ b/actions/govbot/tapes/govbot-help.tape @@ -1,12 +1,28 @@ # govbot help output +# Dual-mode: TEST_MODE=mock asserts output matches expected file +# TEST_MODE=prod (default) just asserts non-empty output Output tapes/govbot-help.gif Set Shell bash Set FontSize 14 Set Width 900 -Set Height 400 +Set Height 500 Set Padding 20 -Type "govbot" +# Define assertion helper (short name to minimize typing time) +Type "sk() { if [ ${TEST_MODE:-prod} = mock ]; then diff $1 $2 && echo SNAP_OK || echo SNAP_FAIL; else [ -s $1 ] && echo SNAP_OK || echo SNAP_FAIL; fi; }" +Enter +Sleep 500ms + +Type "govbot --help > /tmp/gh.txt 2>&1 && cat /tmp/gh.txt" Enter Sleep 2s + +Type "clear" +Enter +Sleep 500ms + +Type "sk /tmp/gh.txt tapes/expected/govbot-help.txt" +Enter +Wait+Screen@5s /SNAP_OK/ +Sleep 1s diff --git a/actions/govbot/tapes/logs-basic.tape b/actions/govbot/tapes/logs-basic.tape index ce824cfb..41cf7d08 100644 --- a/actions/govbot/tapes/logs-basic.tape +++ b/actions/govbot/tapes/logs-basic.tape @@ -1,4 +1,7 @@ # govbot logs output with mock data +# Dual-mode: TEST_MODE=mock asserts output matches expected file +# TEST_MODE=prod (default) just asserts non-empty output +# GOVBOT_DIR: path to .govbot directory (default: mocks/.govbot) Output tapes/logs-basic.gif Set Shell bash @@ -7,6 +10,20 @@ Set Width 1200 Set Height 600 Set Padding 20 -Type "govbot logs --govbot-dir mocks/.govbot" +# Define assertion helper +Type "sk() { if [ ${TEST_MODE:-prod} = mock ]; then diff $1 $2 && echo SNAP_OK || echo SNAP_FAIL; else [ -s $1 ] && echo SNAP_OK || echo SNAP_FAIL; fi; }" +Enter +Sleep 500ms + +Type "GD=${GOVBOT_DIR:-mocks/.govbot}; govbot logs --govbot-dir $GD > /tmp/lb.txt 2>&1 && cat /tmp/lb.txt" Enter Sleep 3s + +Type "clear" +Enter +Sleep 500ms + +Type "sk /tmp/lb.txt tapes/expected/logs-basic.txt" +Enter +Wait+Screen@5s /SNAP_OK/ +Sleep 1s diff --git a/actions/govbot/tapes/nightly/synthetic-test.tape b/actions/govbot/tapes/nightly/synthetic-test.tape index 20e89e34..2a05d048 100644 --- a/actions/govbot/tapes/nightly/synthetic-test.tape +++ b/actions/govbot/tapes/nightly/synthetic-test.tape @@ -4,6 +4,13 @@ # 2. Runs the pipeline (clone → tag → build) with stderr redirected # 3. Verifies outputs exist # +# Dual-mode: +# TEST_MODE=mock → Pre-populates repos from mocks, skips clone, snapshot assertions +# TEST_MODE=prod → Clones live repos, smoke assertions (default) +# +# GOVBOT_SRC: path to the govbot source directory (for mock data) +# Only needed in mock mode. Default: current directory. +# # The govbot binary is built from source and placed on PATH by the # synthetic-test.yml workflow before this tape runs. # @@ -23,6 +30,11 @@ Type "mkdir -p /tmp/govbot-test && cd /tmp/govbot-test" Enter Sleep 1s +# In mock mode: pre-populate repos from mock data so clone is unnecessary +Type "TM=${TEST_MODE:-prod}; if [ $TM = mock ]; then SRC=${GOVBOT_SRC:-.}; mkdir -p .govbot/repos && cp -r $SRC/mocks/.govbot/repos/* .govbot/repos/ && echo MOCK_REPOS_READY; fi" +Enter +Sleep 1s + # Launch govbot — no govbot.yml exists, so the wizard starts. # After the wizard generates config files, govbot exits and tells the # user to run `govbot` again to start the pipeline. @@ -50,12 +62,10 @@ Enter Wait+Screen@10s /Setup complete/ Sleep 2s -# Run pipeline with stderr redirected to avoid PTY deadlock from -# git clone progress. Stdout (tag JSON lines) stays on screen to -# keep VHS from idle-timing out. -# Wait+Screen won't prematch the typed text because "rc:$?" (typed) -# doesn't match regex /rc:0/ — only the output "rc:0" does. -Type "govbot 2>/tmp/pipeline.log; echo rc:$?" +# Run pipeline: in mock mode run individual steps (skip clone), +# in prod mode run the full pipeline. +# In both modes stderr is redirected to avoid PTY deadlock. +Type "TM=${TEST_MODE:-prod}; if [ $TM = mock ]; then govbot logs | govbot tag 2>/tmp/pipeline.log; govbot build 2>>/tmp/pipeline.log; echo rc:$?; else govbot 2>/tmp/pipeline.log; echo rc:$?; fi" Enter Wait+Screen@300s /rc:0/ Sleep 1s @@ -69,8 +79,7 @@ Sleep 500ms # Show pipeline result Type "tail -5 /tmp/pipeline.log" Enter -Wait+Screen@5s /Pipeline complete/ -Sleep 1s +Sleep 2s # Verify outputs Type "ls .govbot/repos/" @@ -80,4 +89,9 @@ Wait+Screen@5s /gu/ Type "ls docs/" Enter Wait+Screen@5s /feed.xml/ + +# Final assertion +Type "TM=${TEST_MODE:-prod}; if [ $TM = mock ]; then test -f govbot.yml && test -f docs/feed.xml && test -d .govbot/repos/gu-legislation && test -d .govbot/repos/wy-legislation && echo SNAP_OK || echo SNAP_FAIL; else test -f docs/feed.xml && echo SNAP_OK || echo SNAP_FAIL; fi" +Enter +Wait+Screen@5s /SNAP_OK/ Sleep 2s diff --git a/actions/govbot/tests/api_snaps.rs b/actions/govbot/tests/api_snaps.rs deleted file mode 100644 index 245b302e..00000000 --- a/actions/govbot/tests/api_snaps.rs +++ /dev/null @@ -1,91 +0,0 @@ -use govbot::prelude::*; -use futures::StreamExt; - -use insta; - -/// Snapshot test for the pipeline processor -/// -/// This test processes log files and compares the output against stored snapshots. -/// To update snapshots after making changes, run: -/// cargo insta review -#[tokio::test] -async fn test_pipeline_processor_snapshot() { - // Use the same test data directory as the example - let git_dir = "tmp/git/repos"; - - // Build configuration matching the render-snapshots.sh script - let config = ConfigBuilder::new(git_dir) - .sort_order_str("DESC") - .unwrap() - .limit(100) - .join_options_str("bill") - .unwrap() - .build(); - - // Skip test if git_dir doesn't exist (e.g., in CI without test data) - let config = match config { - Ok(c) => c, - Err(_) => { - eprintln!("Skipping snapshot test: test data directory not found"); - return; - } - }; - - // Create processor - let processor = PipelineProcessor::new(config); - - // Collect all entries from the stream - let mut stream = processor.process(); - let mut entries = Vec::new(); - - while let Some(result) = stream.next().await { - match result { - Ok(entry) => entries.push(entry), - Err(e) => { - eprintln!("Error processing entry: {}", e); - // Continue processing other entries - } - } - } - - // Serialize to JSON for snapshot comparison - let json_output = serde_json::to_string_pretty(&entries) - .expect("Failed to serialize entries to JSON"); - - // Use insta's assert_snapshot! macro for string comparison - // The snapshot will be stored in tests/snapshots/api_snapshot_tests__test_pipeline_processor_snapshot.snap - insta::assert_snapshot!("pipeline_output", json_output); -} - -/// Snapshot test for a single log entry structure -#[tokio::test] -async fn test_log_entry_structure() { - use govbot::types::{LogContent, LogEntry, VoteEventResult}; - - // Create a sample log entry - let entry = LogEntry { - log: LogContent::VoteEvent { - result: VoteEventResult::Pass, - }, - filename: "test/path/to/logs/20240101T120000Z_vote_event.pass.json".to_string(), - }; - - // Use assert_json_snapshot! for structured data - insta::assert_json_snapshot!("log_entry_structure", &entry); -} - -/// Snapshot test for vote event processing -#[tokio::test] -async fn test_vote_event_processing() { - use govbot::types::VoteEventResult; - - let results = vec![ - VoteEventResult::Pass, - VoteEventResult::Fail, - VoteEventResult::Unknown, - ]; - - // Test vote event result serialization - insta::assert_json_snapshot!("vote_event_results", &results); -} - diff --git a/actions/govbot/tests/cli_example_snaps.rs b/actions/govbot/tests/cli_example_snaps.rs deleted file mode 100644 index 3f902337..00000000 --- a/actions/govbot/tests/cli_example_snaps.rs +++ /dev/null @@ -1,286 +0,0 @@ -use std::fs; -use std::io; -use std::path::{Path, PathBuf}; -use std::process::Command; - -use insta; - -/// Helper function to get the path to the built binary -/// Always builds the binary to ensure we're using the latest version -fn get_binary_path() -> PathBuf { - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - - // Always build the binary to ensure we're using the latest code - // Cargo will handle incremental builds, so this is fast if nothing changed - eprintln!("Building binary to ensure latest version..."); - let status = Command::new("cargo") - .args(&["build", "--bin", "govbot"]) - .current_dir(&manifest_dir) - .status() - .expect("Failed to run cargo build"); - - if !status.success() { - panic!("Failed to build binary"); - } - - // Use debug build for tests (faster to build, and cargo test uses debug by default) - let debug_path = manifest_dir.join("target").join("debug").join("govbot"); - - // Verify the binary exists after building - if !debug_path.exists() { - panic!( - "Binary was not created at expected path: {}", - debug_path.display() - ); - } - - debug_path -} - -/// Helper to check if test data exists -fn test_data_exists() -> bool { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("mocks") - .join(".govbot") - .join("repos") - .exists() -} - -/// Parse a shell script to extract the command -/// Handles line continuations with backslashes -fn parse_shell_script(script_content: &str) -> Vec { - // Remove comments and empty lines - let lines: Vec<&str> = script_content - .lines() - .map(|line| line.trim()) - .filter(|line| !line.is_empty() && !line.starts_with('#')) - .collect(); - - // Join lines with backslash continuations - let mut command_line = String::new(); - for (i, line) in lines.iter().enumerate() { - let trimmed = line.trim_end_matches('\\').trim(); - command_line.push_str(trimmed); - // Add space after each line (except the last) - if i < lines.len() - 1 { - command_line.push(' '); - } - } - - // Split into arguments (simple shell-like parsing) - // This handles quoted strings and basic word splitting - let mut args = Vec::new(); - let mut current = String::new(); - let mut in_quotes = false; - let mut quote_char = '\0'; - - for ch in command_line.chars() { - match ch { - '"' | '\'' if !in_quotes => { - in_quotes = true; - quote_char = ch; - } - ch if ch == quote_char && in_quotes => { - in_quotes = false; - quote_char = '\0'; - } - ' ' | '\t' if !in_quotes => { - if !current.is_empty() { - args.push(current.clone()); - current.clear(); - } - } - _ => { - current.push(ch); - } - } - } - - if !current.is_empty() { - args.push(current); - } - - // Remove the binary name (first argument) since we'll use get_binary_path() - if !args.is_empty() && args[0] == "govbot" { - args.remove(0); - } - - args -} - -/// Execute a shell script example and capture stdout -fn run_example_script(script_path: &Path) -> (String, String, i32) { - let binary = get_binary_path(); - let script_content = fs::read_to_string(script_path) - .expect(&format!("Failed to read script: {}", script_path.display())); - - let args = parse_shell_script(&script_content); - - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let govbot_dir = manifest_dir.join("mocks").join(".govbot"); - - // Set URL template to match existing mock data (uses -data-pipeline suffix) - let repo_url_template = "https://github.com/chn-openstates-files/{locale}-data-pipeline.git"; - - let output = Command::new(&binary) - .args(&args) - .current_dir(&manifest_dir) - .env("GOVBOT_DIR", govbot_dir.to_string_lossy().as_ref()) - .env("GOVBOT_REPO_URL_TEMPLATE", repo_url_template) - .output() - .expect("Failed to execute command"); - - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let exit_code = output.status.code().unwrap_or(-1); - - (stdout, stderr, exit_code) -} - -/// Get all .sh example files from the examples directory -fn get_example_scripts() -> io::Result> { - let examples_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("examples"); - let mut scripts = Vec::new(); - - if examples_dir.exists() { - for entry in fs::read_dir(&examples_dir)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|s| s.to_str()) == Some("sh") { - scripts.push(path); - } - } - } - - scripts.sort(); - Ok(scripts) -} - -/// Generate a snapshot name from a script path -fn snapshot_name_from_path(path: &Path) -> String { - path.file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("unknown") - .replace('-', "_") - .replace('.', "_") -} - -/// Format output with script contents for snapshot -fn format_snapshot_with_script(script_path: &Path, output: &str) -> String { - let script_content = fs::read_to_string(script_path) - .expect(&format!("Failed to read script: {}", script_path.display())); - - // Remove trailing newlines from script content - let script_content = script_content.trim_end(); - - format!("Command:\n{}\n\nOutput:\n{}", script_content, output) -} - -/// Check if a script requires test data to run -fn script_requires_test_data(script_path: &Path) -> bool { - if let Ok(content) = fs::read_to_string(script_path) { - // Commands that need test data (repos directory) - content.contains("govbot logs") - } else { - false - } -} - -/// Test a single example script -fn test_example_script(script_path: &Path) { - let script_name = script_path - .file_name() - .and_then(|s| s.to_str()) - .unwrap_or("unknown"); - - let has_test_data = test_data_exists(); - - // Skip scripts that require test data if it doesn't exist - if script_requires_test_data(script_path) && !has_test_data { - eprintln!("Skipping {}: test data directory not found", script_name); - return; - } - - eprintln!("Testing example: {}", script_name); - - let (stdout, stderr, exit_code) = run_example_script(script_path); - - // Create snapshot name from script filename - let snapshot_name = snapshot_name_from_path(script_path); - - // Format stdout with script contents for snapshot - let formatted_stdout = format_snapshot_with_script(script_path, &stdout); - - // Snapshot stdout (which is the main output) - // Use insta's Settings API - set snapshot directory and use custom snapshot name - // The format will be: {test_function_name}__{suffix}.snap - // With test function name "cli_example_snaps" and suffix "{snapshot_name}", - // this creates: cli_example_snaps__{snapshot_name}.snap - let mut settings = insta::Settings::clone_current(); - settings.set_snapshot_path("snapshots"); - settings.set_snapshot_suffix(&snapshot_name); - settings.bind(|| { - insta::assert_snapshot!("snapshot", &formatted_stdout); - }); - - // If there's stderr, snapshot it separately - if !stderr.is_empty() { - let mut settings = insta::Settings::clone_current(); - settings.set_snapshot_path("snapshots"); - settings.set_snapshot_suffix(&format!("{}_stderr", snapshot_name)); - settings.bind(|| { - insta::assert_snapshot!("snapshot", &stderr); - }); - } - - // Verify exit code is success - assert_eq!( - exit_code, 0, - "Example script '{}' should exit with code 0, got {}", - script_name, exit_code - ); -} - -// Macro to generate a test function for each example script -// This allows each example to be tested independently -macro_rules! generate_example_tests { - () => { - #[test] - fn cli_example_snaps() { - let example_scripts = get_example_scripts().expect("Failed to read examples directory"); - - if example_scripts.is_empty() { - eprintln!("No example scripts found in examples/ directory"); - return; - } - - // Collect all scripts and test them - // Note: This still runs in a single test, but we'll use a different approach - let mut tested = 0; - let mut skipped = 0; - - for script_path in example_scripts { - let script_name = script_path - .file_name() - .and_then(|s| s.to_str()) - .unwrap_or("unknown"); - - let has_test_data = test_data_exists(); - if script_requires_test_data(&script_path) && !has_test_data { - eprintln!("Skipping {}: test data directory not found", script_name); - skipped += 1; - continue; - } - - // Test this script - if it panics, the test stops - // This is expected behavior - use `cargo insta test` to create missing snapshots - test_example_script(&script_path); - tested += 1; - } - - eprintln!("Tested {} example(s), skipped {}", tested, skipped); - } - }; -} - -generate_example_tests!(); diff --git a/actions/govbot/tests/snapshots/api_snaps__log_entry_structure.snap b/actions/govbot/tests/snapshots/api_snaps__log_entry_structure.snap deleted file mode 100644 index 77f31b5f..00000000 --- a/actions/govbot/tests/snapshots/api_snaps__log_entry_structure.snap +++ /dev/null @@ -1,10 +0,0 @@ ---- -source: tests/api_snaps.rs -expression: "&entry" ---- -{ - "log": { - "result": "pass" - }, - "filename": "test/path/to/logs/20240101T120000Z_vote_event.pass.json" -} diff --git a/actions/govbot/tests/snapshots/api_snaps__vote_event_results.snap b/actions/govbot/tests/snapshots/api_snaps__vote_event_results.snap deleted file mode 100644 index 4c286c2a..00000000 --- a/actions/govbot/tests/snapshots/api_snaps__vote_event_results.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: tests/api_snaps.rs -expression: "&results" ---- -[ - "pass", - "fail", - "unknown" -] diff --git a/actions/govbot/tests/snapshots/cli_example_snaps__snapshot@logs_basic.snap b/actions/govbot/tests/snapshots/cli_example_snaps__snapshot@logs_basic.snap deleted file mode 100644 index 3802b2b7..00000000 --- a/actions/govbot/tests/snapshots/cli_example_snaps__snapshot@logs_basic.snap +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: tests/cli_example_snaps.rs -expression: "&formatted_stdout" ---- -Command: -govbot logs - -Output: From 72b122caa333713b2727b2e6d4ff8cf92e1928c6 Mon Sep 17 00:00:00 2001 From: Sartaj Date: Fri, 20 Feb 2026 23:27:42 -0600 Subject: [PATCH 2/2] fix: sort logs output before snapshot diff for cross-platform determinism jwalk parallel filesystem traversal produces different ordering on macOS vs Linux. Sort both actual and expected before comparing. Co-Authored-By: Claude Opus 4.6 --- actions/govbot/tapes/expected/logs-basic.txt | 4 ++-- actions/govbot/tapes/logs-basic.tape | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/actions/govbot/tapes/expected/logs-basic.txt b/actions/govbot/tapes/expected/logs-basic.txt index 8516c60b..0067fa4f 100644 --- a/actions/govbot/tapes/expected/logs-basic.txt +++ b/actions/govbot/tapes/expected/logs-basic.txt @@ -1,5 +1,5 @@ +{"bill":{"abstracts":[{"abstract":"2025/Summaries/HB0004.pdf","note":"summary"}],"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0004","legislative_session":"2025","title":"Snowmobile registration and user fees."},"id":"HB0004","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:17:44+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0004"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0004/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0004/logs/20250102T191744Z_h_received_for_introduction.json"},"timestamp":"20250102T191744Z"} +{"bill":{"abstracts":[{"abstract":"2025/Summaries/HB0005.pdf","note":"summary"}],"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0005","legislative_session":"2025","title":"Fishing outfitters and guides-registration of fishing boats."},"id":"HB0005","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:18:48+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0005"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0005/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0005/logs/20250102T191848Z_h_received_for_introduction.json"},"timestamp":"20250102T191848Z"} {"bill":{"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0001","legislative_session":"2025","title":"General government appropriations-2."},"id":"HB0001","log":{"action":{"classification":["filing"],"date":"2025-01-29T17:06:54+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0001"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0001/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0001/logs/20250129T170654Z_h_received_for_introduction.json"},"timestamp":"20250129T170654Z"} {"bill":{"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0002","legislative_session":"2025","title":"Hunting license application fees increase."},"id":"HB0002","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:16:16+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0002"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0002/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0002/logs/20250102T191616Z_h_received_for_introduction.json"},"timestamp":"20250102T191616Z"} -{"bill":{"abstracts":[{"abstract":"2025/Summaries/HB0005.pdf","note":"summary"}],"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0005","legislative_session":"2025","title":"Fishing outfitters and guides-registration of fishing boats."},"id":"HB0005","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:18:48+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0005"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0005/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0005/logs/20250102T191848Z_h_received_for_introduction.json"},"timestamp":"20250102T191848Z"} -{"bill":{"abstracts":[{"abstract":"2025/Summaries/HB0004.pdf","note":"summary"}],"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0004","legislative_session":"2025","title":"Snowmobile registration and user fees."},"id":"HB0004","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:17:44+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0004"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0004/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0004/logs/20250102T191744Z_h_received_for_introduction.json"},"timestamp":"20250102T191744Z"} {"bill":{"from_organization":"~{\"classification\": \"lower\"}","identifier":"HB0003","legislative_session":"2025","title":"Animal abuse-predatory animals."},"id":"HB0003","log":{"action":{"classification":["filing"],"date":"2025-01-02T19:17:11+00:00","description":"H Received for Introduction","organization_id":"~{\"classification\": \"lower\"}"},"bill_id":"HB0003"},"sources":{"bill":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0003/metadata.json","log":"wy-legislation/country:us/state:wy/sessions/2025/bills/HB0003/logs/20250102T191711Z_h_received_for_introduction.json"},"timestamp":"20250102T191711Z"} diff --git a/actions/govbot/tapes/logs-basic.tape b/actions/govbot/tapes/logs-basic.tape index 41cf7d08..4e895192 100644 --- a/actions/govbot/tapes/logs-basic.tape +++ b/actions/govbot/tapes/logs-basic.tape @@ -2,6 +2,9 @@ # Dual-mode: TEST_MODE=mock asserts output matches expected file # TEST_MODE=prod (default) just asserts non-empty output # GOVBOT_DIR: path to .govbot directory (default: mocks/.govbot) +# +# Note: logs output order depends on filesystem traversal (jwalk), +# so we sort before comparing to handle macOS/Linux differences. Output tapes/logs-basic.gif Set Shell bash @@ -10,8 +13,8 @@ Set Width 1200 Set Height 600 Set Padding 20 -# Define assertion helper -Type "sk() { if [ ${TEST_MODE:-prod} = mock ]; then diff $1 $2 && echo SNAP_OK || echo SNAP_FAIL; else [ -s $1 ] && echo SNAP_OK || echo SNAP_FAIL; fi; }" +# Define assertion helper (sorts both files for order-independent comparison) +Type "sk() { if [ ${TEST_MODE:-prod} = mock ]; then diff <(sort $1) <(sort $2) && echo SNAP_OK || echo SNAP_FAIL; else [ -s $1 ] && echo SNAP_OK || echo SNAP_FAIL; fi; }" Enter Sleep 500ms