bench: add standalone search abstention evaluation (#151) #180
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # CI for Palinode — hosted runners only. | |
| # | |
| # This replaces the previous ci.yml / main-ci.yml / test.yml trio, whose | |
| # private-runner jobs could never run here and left every run queued until it | |
| # expired. All jobs below run on GitHub-hosted runners; unit tests cover Linux | |
| # plus one macOS lane for filesystem and platform compatibility. | |
| # | |
| # All embeddings / LLM calls are mocked in the unit suite — no external | |
| # services are needed. | |
| name: CI | |
| on: | |
| push: | |
| branches: [ main ] | |
| tags: [ "v*" ] | |
| pull_request: | |
| branches: [ main ] | |
| jobs: | |
| unit-tests: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| python-version: "3.11" | |
| - os: ubuntu-latest | |
| python-version: "3.12" | |
| - os: macos-latest | |
| python-version: "3.12" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} (Linux) | |
| if: runner.os == 'Linux' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Set up Python 3.12 (macOS — extension-loading build) | |
| if: runner.os == 'macOS' | |
| run: | | |
| brew install python@3.12 | |
| "$(brew --prefix python@3.12)/bin/python3.12" -m venv "$RUNNER_TEMP/palinode-venv" | |
| echo "$RUNNER_TEMP/palinode-venv/bin" >> "$GITHUB_PATH" | |
| - name: Verify loadable SQLite extensions (macOS) | |
| if: runner.os == 'macOS' | |
| run: | | |
| python -c 'import sqlite3; assert hasattr(sqlite3.connect(":memory:"), "enable_load_extension")' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Assert palinode resolves to the checked-out tree | |
| # Regression guard for editable installs: palinode.__file__ must | |
| # resolve under GITHUB_WORKSPACE, not some other site-packages path. | |
| run: | | |
| RESOLVED=$(python -c "import palinode; print(palinode.__file__)") | |
| echo "palinode.__file__ = $RESOLVED" | |
| if [[ "$RESOLVED" != "$GITHUB_WORKSPACE"/* ]]; then | |
| echo "ERROR: palinode resolves outside the workspace ($GITHUB_WORKSPACE)" | |
| exit 1 | |
| fi | |
| - name: Run unit tests | |
| run: pytest tests/ -v --tb=short --ignore=tests/integration | |
| security-scan: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: "pip" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| pip install bandit pip-audit | |
| - name: Run bandit (static security analysis) | |
| run: | | |
| status=0 | |
| bandit -r palinode/ -ll || status=$? | |
| bandit -r palinode/ -ll -f json -o bandit-report.json || status=$? | |
| exit "$status" | |
| continue-on-error: true | |
| - name: Run pip-audit (dependency vulnerability check) | |
| run: | | |
| status=0 | |
| pip-audit || status=$? | |
| pip-audit -f json -o pip-audit-report.json || status=$? | |
| exit "$status" | |
| continue-on-error: true | |
| - name: Upload security scan reports | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: security-scan-reports | |
| path: | | |
| bandit-report.json | |
| pip-audit-report.json | |
| if-no-files-found: warn | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: "pip" | |
| - name: Install ruff | |
| run: python -m pip install "ruff~=0.16.0" | |
| - name: Run ruff | |
| run: ruff check palinode/ tests/ scripts/ | |
| httpx-monopoly: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Enforce the CLI HTTP-client boundary | |
| run: bash scripts/check-httpx-monopoly.sh | |
| - name: Run monopoly-linter test suite | |
| run: bash tests/test_check_httpx_monopoly.sh | |
| write-choke-point: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Enforce the memory mutation choke point | |
| run: bash scripts/check-write-choke-point.sh | |
| - name: Run choke-point-linter test suite | |
| run: bash tests/test_check_write_choke_point.sh |