evalwire/.github/workflows/ci.yml at 2c8b0c7459c9696a5c8b3f965aaec651fa483008 · zurfjereluhmie/evalwire · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

jobs:
  lint:
    name: Lint, format & typecheck
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v6.0.3

      - name: Install uv
        uses: astral-sh/setup-uv@v8.2.0
        with:
          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version-file: .python-version

      - name: Install dependencies
        run: uv sync --group dev

      - name: Lint (ruff check)
        run: uv run ruff check src tests demo

      - name: Format (ruff format)
        run: uv run ruff format --check src tests demo

      - name: Typecheck (ty)
        run: uv run ty check

  test:
    name: Test (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13"]

    steps:
      - uses: actions/checkout@v6.0.3

      - name: Install uv
        uses: astral-sh/setup-uv@v8.2.0
        with:
          enable-cache: true

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: uv sync --group dev

      - name: Test with coverage
        run: uv run pytest tests/ -q --cov=evalwire --cov-report=term-missing --cov-fail-under=85

  test-integration:
    name: Integration tests (phoenix v${{ matrix.phoenix-major }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        phoenix-major: [13, 14, 15]

    steps:
      - uses: actions/checkout@v6.0.3

      - name: Install uv
        uses: astral-sh/setup-uv@v8.2.0
        with:
          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version-file: .python-version

      - name: Install dependencies
        run: uv sync --group dev

      - name: Pin arize-phoenix to major ${{ matrix.phoenix-major }}
        run: |
          MAJOR=${{ matrix.phoenix-major }}
          NEXT=$((MAJOR + 1))
          if [ "$MAJOR" -lt 14 ]; then
            uv pip install "arize-phoenix>=${MAJOR},<${NEXT}" "arize-phoenix-evals<3.0" --prerelease=allow
          else
            uv pip install "arize-phoenix>=${MAJOR},<${NEXT}"
          fi

      - name: Run integration tests
        run: uv run pytest tests/ -q -m integration

  # https://github.com/marketplace/actions/alls-green#why
  ci-success: # This job does nothing and is only used for the branch protection
    name: CI Success
    if: always()
    needs:
      - lint
      - test
      - test-integration
      - docs-check
    runs-on: ubuntu-latest
    steps:
      - name: Decide whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@release/v1
        with:
          allowed-skips: docs-check
          jobs: ${{ toJSON(needs) }}

  docs-check:
    name: Docs build check
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v6.0.3

      - uses: dorny/paths-filter@v4
        id: filter
        with:
          filters: |
            docs:
              - docs/**
              - mkdocs.yml
              - src/**
              - .github/workflows/docs.yml

      - name: Install uv
        if: steps.filter.outputs.docs == 'true'
        uses: astral-sh/setup-uv@v8.2.0
        with:
          enable-cache: true

      - name: Set up Python
        if: steps.filter.outputs.docs == 'true'
        uses: actions/setup-python@v6
        with:
          python-version-file: .python-version

      - name: Install dependencies
        if: steps.filter.outputs.docs == 'true'
        run: uv sync --group dev

      - name: Build docs (dry run)
        if: steps.filter.outputs.docs == 'true'
        run: uv run mkdocs build --clean --strict