fix(eval): judge final deliverable state; guard against contradictory… #196
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Runs scenarios 01+02 (curl + TypeScript SDK) with heuristic + LLM judge. | |
| # Sets EVAL_LOCAL_DOCS=1 so the agent reads repo docs under docs/ (not production WebFetch). | |
| # Injects OUTPOST_API_KEY (and related env) for the agent run so smoke tests hit live Outpost; | |
| # step 2 re-runs the saved artifacts deterministically. | |
| # Triggers: workflow_dispatch, or push (main) / pull_request when docs / OpenAPI / agent-eval / TS SDK paths change. | |
| # Each run bills Anthropic (agent + judge). | |
| # Requires repo secrets: ANTHROPIC_API_KEY, EVAL_TEST_DESTINATION_URL, OUTPOST_API_KEY | |
| # (OUTPOST_TEST_WEBHOOK_URL uses the same URL as EVAL_TEST_DESTINATION_URL in CI.) | |
| # Env is scoped per step — see each run step's env block (no job-wide secret export). | |
| # See docs/agent-evaluation/README.md § CI (recommended slice). | |
| name: Docs agent eval (CI slice) | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "docs/content/**" | |
| - "docs/apis/**" | |
| - "docs/agent-evaluation/**" | |
| - "docs/README.md" | |
| - "docs/AGENTS.md" | |
| - "sdks/outpost-typescript/**" | |
| - ".github/workflows/docs-agent-eval-ci.yml" | |
| pull_request: | |
| paths: | |
| - "docs/content/**" | |
| - "docs/apis/**" | |
| - "docs/agent-evaluation/**" | |
| - "docs/README.md" | |
| - "docs/AGENTS.md" | |
| - "sdks/outpost-typescript/**" | |
| - ".github/workflows/docs-agent-eval-ci.yml" | |
| jobs: | |
| eval-ci: | |
| # One live Outpost mutation at a time: execute-ci-artifacts deletes a shared CI tenant on EXIT. | |
| # Without this, parallel runs could DELETE the same tenant while another job is mid-test. | |
| concurrency: | |
| group: outpost-docs-agent-eval-live-outpost | |
| cancel-in-progress: false | |
| # Fork PRs cannot use repository secrets; skip instead of failing a required-looking job. | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| defaults: | |
| run: | |
| working-directory: docs/agent-evaluation | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: "24" | |
| cache: npm | |
| cache-dependency-path: docs/agent-evaluation/package-lock.json | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Run eval CI slice (scenarios 01, 02) | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| EVAL_TEST_DESTINATION_URL: ${{ secrets.EVAL_TEST_DESTINATION_URL }} | |
| EVAL_LOCAL_DOCS: "1" | |
| OUTPOST_API_KEY: ${{ secrets.OUTPOST_API_KEY }} | |
| OUTPOST_TEST_WEBHOOK_URL: ${{ secrets.EVAL_TEST_DESTINATION_URL }} | |
| OUTPOST_API_BASE_URL: https://api.outpost.hookdeck.com/2025-07-01 | |
| run: ./scripts/ci-eval.sh | |
| - name: Execute generated curl + TypeScript artifacts (live Outpost) | |
| env: | |
| OUTPOST_API_KEY: ${{ secrets.OUTPOST_API_KEY }} | |
| OUTPOST_TEST_WEBHOOK_URL: ${{ secrets.EVAL_TEST_DESTINATION_URL }} | |
| OUTPOST_API_BASE_URL: https://api.outpost.hookdeck.com/2025-07-01 | |
| OUTPOST_CI_PUBLISH_TOPIC: user.created | |
| # Agent eval 01/02 fixtures use this tenant; delete before+after so destination caps do not break CI. | |
| OUTPOST_CI_CLEANUP_TENANT: customer_acme_001 | |
| run: ./scripts/execute-ci-artifacts.sh | |
| # Transcripts, heuristic + LLM scores, generated scripts — present after eval; execute step may add logs in-place. | |
| - name: Redact secrets in eval artifacts (best effort) | |
| if: always() | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| EVAL_TEST_DESTINATION_URL: ${{ secrets.EVAL_TEST_DESTINATION_URL }} | |
| OUTPOST_API_KEY: ${{ secrets.OUTPOST_API_KEY }} | |
| OUTPOST_TEST_WEBHOOK_URL: ${{ secrets.EVAL_TEST_DESTINATION_URL }} | |
| run: node --import tsx scripts/redact-eval-artifacts.ts | |
| - name: Upload agent eval outputs (debug) | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: docs-agent-eval-runs-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: docs/agent-evaluation/results/runs | |
| if-no-files-found: warn | |
| retention-days: 14 |