Skip to content
Test

docs: add claim extraction example for fact-checking pipelines #6142

Sign in to view logs

docs: add claim extraction example for fact-checking pipelines

docs: add claim extraction example for fact-checking pipelines #6142

Workflow file for this run

.github/workflows/test.yml at 4f0a664
name: Test
on:
pull_request:
push:
branches:
- main
permissions:
contents: read
jobs:
# Core tests without LLM providers
core-tests:
name: Core Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
python-version: "3.11"
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --frozen --all-extras
- name: Run core tests
run: >-
uv run --frozen pytest tests/ --asyncio-mode=auto -n auto
--ignore=tests/coverage
--ignore=tests/test_batch_processor_coverage.py
-k 'not test_core_providers and not test_openai and not test_anthropic
and not test_gemini and not test_genai and not test_writer and not
test_vertexai and not docs'
env:
INSTRUCTOR_ENV: CI
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
# Offline coverage tests must not be filtered by provider names.
offline-coverage-tests:
name: Offline Coverage Tests (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
# xai-sdk requires Python 3.10+, so Python 3.9 is a smoke job.
# Only SDK-dependent xAI tests and source are omitted there.
- python-version: "3.9"
pytest-args: >-
--ignore=tests/coverage/test_xai_client_coverage.py
--ignore=tests/coverage/test_xai_handlers_coverage.py
-k 'not test_xai_builder_forwards_real_client_and_mode and
not test_xai_builder_reports_unavailable_factory and
not (test_tail_builder_factory_failure_is_logged_and_propagated
and _build_xai)'
coverage-omit: instructor/v2/providers/xai/*
- python-version: "3.10"
pytest-args: ""
coverage-omit: ""
- python-version: "3.11"
pytest-args: ""
coverage-omit: ""
- python-version: "3.12"
pytest-args: ""
coverage-omit: ""
- python-version: "3.13"
pytest-args: ""
coverage-omit: ""
env:
INSTRUCTOR_ENV: CI
PYTHONWARNINGS: error::ResourceWarning
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
python-version: ${{ matrix.python-version }}
cache-suffix: offline-py${{ matrix.python-version }}
- name: Set up Python
run: uv python install ${{ matrix.python-version }}
- name: Install the project
run: uv sync --frozen --all-extras
- name: Run offline coverage tests
run: |
uv run --frozen coverage erase
uv run --frozen coverage run -m pytest \
tests/coverage tests/test_batch_processor_coverage.py \
--asyncio-mode=auto --strict-config --strict-markers \
-W error::pytest.PytestUnraisableExceptionWarning \
-W error::pytest.PytestUnhandledThreadExceptionWarning \
${{ matrix.pytest-args }}
if [ "${{ matrix.python-version }}" = "3.9" ]; then
uv run --frozen coverage report \
--omit="${{ matrix.coverage-omit }}" --show-missing
else
uv run --frozen coverage report \
--omit="${{ matrix.coverage-omit }}" --show-missing --fail-under=90
fi
# Enforce repository-wide statement and branch coverage without provider secrets.
full-coverage:
name: Full Coverage (Python 3.11)
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
python-version: "3.11"
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --frozen --all-extras
- name: Run full offline coverage
run: |
uv run --frozen coverage erase
uv run --frozen coverage run --branch -m pytest tests/ \
--asyncio-mode=auto --strict-config --strict-markers \
-k 'not docs'
uv run --frozen coverage report --show-missing --fail-under=99
uv run --frozen coverage json -o "${RUNNER_TEMP}/coverage.json"
uv run --frozen python -c 'import json, os; totals = json.load(open(os.path.join(os.environ["RUNNER_TEMP"], "coverage.json")))["totals"]; missing = totals["missing_lines"]; print(f"Missing statements: {missing}"); raise SystemExit(1 if missing else 0)'
env:
INSTRUCTOR_ENV: CI
# Core provider tests for OpenAI
core-openai:
name: Core Provider Tests (OpenAI)
runs-on: ubuntu-latest
needs: core-tests
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
python-version: "3.11"
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --frozen --all-extras
- name: Skip core provider tests (OpenAI)
if: ${{ env.OPENAI_API_KEY == '' }}
run: echo "Skipping OpenAI core provider tests (missing OPENAI_API_KEY)."
- name: Run core provider tests (OpenAI)
if: ${{ env.OPENAI_API_KEY != '' }}
run: |
set +e
uv run --frozen pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "openai"
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
# Core provider tests for Anthropic
core-anthropic:
name: Core Provider Tests (Anthropic)
runs-on: ubuntu-latest
needs: core-tests
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
python-version: "3.11"
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --frozen --all-extras
- name: Skip core provider tests (Anthropic)
if: ${{ env.ANTHROPIC_API_KEY == '' }}
run: echo "Skipping Anthropic core provider tests (missing ANTHROPIC_API_KEY)."
- name: Run core provider tests (Anthropic)
if: ${{ env.ANTHROPIC_API_KEY != '' }}
run: |
set +e
uv run --frozen pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "anthropic"
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
# Core provider tests for Google
core-google:
name: Core Provider Tests (Google)
runs-on: ubuntu-latest
needs: core-tests
env:
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
GOOGLE_GENAI_MODEL: ${{ secrets.GOOGLE_GENAI_MODEL }}
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
python-version: "3.11"
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --frozen --all-extras
- name: Skip core provider tests (Google)
if: ${{ env.GOOGLE_API_KEY == '' || env.GOOGLE_GENAI_MODEL == '' }}
run: echo "Skipping Google core provider tests (missing GOOGLE_API_KEY or GOOGLE_GENAI_MODEL)."
- name: Run core provider tests (Google)
if: ${{ env.GOOGLE_API_KEY != '' && env.GOOGLE_GENAI_MODEL != '' }}
run: |
set +e
uv run --frozen pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "google"
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
# Core provider tests for other providers
core-other:
name: Core Provider Tests (Other)
runs-on: ubuntu-latest
needs: core-tests
env:
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
WRITER_API_KEY: ${{ secrets.WRITER_API_KEY }}
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
python-version: "3.11"
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --frozen --all-extras
- name: Skip core provider tests (Other)
if: >-
${{ env.COHERE_API_KEY == '' && env.XAI_API_KEY == ''
&& env.MISTRAL_API_KEY == '' && env.CEREBRAS_API_KEY == ''
&& env.FIREWORKS_API_KEY == '' && env.WRITER_API_KEY == ''
&& env.PERPLEXITY_API_KEY == '' }}
run: echo "Skipping core provider tests (Other) (missing provider secrets)."
- name: Run core provider tests (Cohere, xAI, Mistral, etc)
if: >-
${{ env.COHERE_API_KEY != '' || env.XAI_API_KEY != ''
|| env.MISTRAL_API_KEY != '' || env.CEREBRAS_API_KEY != ''
|| env.FIREWORKS_API_KEY != '' || env.WRITER_API_KEY != ''
|| env.PERPLEXITY_API_KEY != '' }}
run: |
set +e
uv run --frozen pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "cohere or xai or mistral or cerebras or fireworks or writer or perplexity"
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
WRITER_API_KEY: ${{ secrets.WRITER_API_KEY }}
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
# Provider tests run in parallel
provider-tests:
name: ${{ matrix.provider.name }} Tests
runs-on: ubuntu-latest
needs: [core-openai, core-anthropic, core-google, core-other]
env:
PROVIDER_API_KEY: ${{ secrets[matrix.provider.env_key] }}
GOOGLE_GENAI_MODEL: ${{ secrets.GOOGLE_GENAI_MODEL }}
strategy:
fail-fast: false
matrix:
provider:
- name: OpenAI
env_key: OPENAI_API_KEY
test_path: tests/llm/test_openai
- name: Anthropic
env_key: ANTHROPIC_API_KEY
test_path: tests/llm/test_anthropic
- name: Gemini
env_key: GOOGLE_API_KEY
test_path: tests/llm/test_gemini
- name: Google GenAI
env_key: GOOGLE_API_KEY
test_path: tests/llm/test_genai
- name: Vertex AI
env_key: GOOGLE_API_KEY
test_path: tests/llm/test_vertexai
- name: Writer
env_key: WRITER_API_KEY
test_path: tests/llm/test_writer
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
python-version: "3.11"
cache-suffix: provider-${{ matrix.provider.name }}-py311
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --frozen --all-extras
- name: Skip ${{ matrix.provider.name }} tests
if: >-
${{ env.PROVIDER_API_KEY == '' ||
((matrix.provider.name == 'Gemini' || matrix.provider.name == 'Google GenAI'
|| matrix.provider.name == 'Vertex AI') && env.GOOGLE_GENAI_MODEL == '') }}
run: >-
echo "Skipping ${{ matrix.provider.name }} tests
(missing ${{ matrix.provider.env_key }} or GOOGLE_GENAI_MODEL)."
- name: Run ${{ matrix.provider.name }} tests
if: >-
${{ env.PROVIDER_API_KEY != '' &&
((matrix.provider.name != 'Gemini' && matrix.provider.name != 'Google GenAI'
&& matrix.provider.name != 'Vertex AI') || env.GOOGLE_GENAI_MODEL != '') }}
run: |
set +e
uv run --frozen pytest ${{ matrix.provider.test_path }} --asyncio-mode=auto -n auto
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
${{ matrix.provider.env_key }}: ${{ secrets[matrix.provider.env_key] }}
# Auto client needs multiple providers
auto-client-test:
name: Auto Client Tests
runs-on: ubuntu-latest
needs: [core-openai, core-anthropic, core-google, core-other]
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
python-version: "3.11"
- name: Set up Python
run: uv python install 3.11
- name: Install the project
run: uv sync --frozen --all-extras
- name: Skip Auto Client tests
if: >-
${{ env.OPENAI_API_KEY == '' || env.GOOGLE_API_KEY == ''
|| env.COHERE_API_KEY == '' || env.ANTHROPIC_API_KEY == ''
|| env.XAI_API_KEY == '' }}
run: echo "Skipping Auto Client tests (missing one or more provider secrets)."
- name: Run Auto Client tests
if: >-
${{ env.OPENAI_API_KEY != '' && env.GOOGLE_API_KEY != ''
&& env.COHERE_API_KEY != '' && env.ANTHROPIC_API_KEY != ''
&& env.XAI_API_KEY != '' }}
run: |
set +e
uv run --frozen pytest tests/providers/test_auto_client.py --asyncio-mode=auto -n auto
status=$?
set -e
if [ $status -eq 5 ]; then
echo "No tests collected; treating as success."
exit 0
fi
exit $status
env:
INSTRUCTOR_ENV: CI
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}