diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 921745a..2cf4900 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,13 +76,11 @@ jobs: enable-cache: true - name: Test ${{ matrix.service }} - # Unit (coverage-gated) + integration (ungated). e2e is marked but not in - # the per-build gate — it's orchestration-heavy; run it via `ci test --tier e2e`. - # Keyed on the build context so multi-image stacks test the right subtree; - # no-ops for units with no pyproject (e.g. devbox). - run: | - uv run ci test "${{ matrix.context }}" --tier unit - uv run ci test "${{ matrix.context }}" --tier integration + # Runs unit + integration together; the project's --cov-fail-under applies + # to their COMBINED coverage. e2e is orchestration-heavy and runs separately + # (e2e.yml / `ci test --tier e2e`). Keyed on the build context so multi-image + # stacks test the right subtree; no-ops for units with no pyproject (devbox). + run: uv run ci test "${{ matrix.context }}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/stacks/apps/takeout-manager/manager/pyproject.toml b/stacks/apps/takeout-manager/manager/pyproject.toml index 699f16f..f5a22cd 100644 --- a/stacks/apps/takeout-manager/manager/pyproject.toml +++ b/stacks/apps/takeout-manager/manager/pyproject.toml @@ -32,4 +32,4 @@ markers = [ "integration: needs external services (real process / fakes / docker)", "e2e: full black-box end-to-end (docker compose)", ] -addopts = "--strict-markers --cov=backend --cov-report=term-missing --cov-fail-under=35" +addopts = "--strict-markers --cov=backend --cov-report=term-missing --cov-fail-under=88" diff --git a/stacks/monitoring/custom-exporter/pyproject.toml b/stacks/monitoring/custom-exporter/pyproject.toml index 999ff51..ebfa203 100644 --- a/stacks/monitoring/custom-exporter/pyproject.toml +++ b/stacks/monitoring/custom-exporter/pyproject.toml @@ -35,5 +35,5 @@ addopts = [ "--cov=iperf3_exporter", "--cov-report=term-missing", "--cov-report=html", - "--cov-fail-under=30", + "--cov-fail-under=75", ] diff --git a/tools/ci/ci/apptests.py b/tools/ci/ci/apptests.py index 6830e5d..0cd5364 100644 --- a/tools/ci/ci/apptests.py +++ b/tools/ci/ci/apptests.py @@ -3,8 +3,13 @@ Discovery is structural: any dir under ``stacks/`` with a ``pyproject.toml`` is a testable project (each declares its own pytest dev-group, so ``uv run pytest`` self-bootstraps). Tiers are ``tests/{unit,integration,e2e}`` subdirs, run if they -exist. Coverage gating is a unit-tier concern, so non-unit tiers clear the -project's ``addopts`` (otherwise ``--cov-fail-under`` trips on a partial run). +exist. + +The default (gated) run executes unit + integration **together in one pytest**, so +the project's ``--cov-fail-under`` applies to the *combined* coverage of both tiers. +A single explicit ``--tier`` runs that tier alone and clears ``addopts`` (a partial +run shouldn't trip the coverage gate). e2e is not in the default suite — it's run +explicitly via ``--tier e2e`` (the e2e workflow). The selection logic is pure and unit-tested; only :func:`run_tests` shells out. """ @@ -15,6 +20,8 @@ from pathlib import Path TIERS = ("unit", "integration", "e2e") +# The gated default suite: unit + integration, coverage measured across both. +DEFAULT_TIERS = ("unit", "integration") def discover_test_projects(repo_root: str | Path) -> list[str]: @@ -42,32 +49,39 @@ def select_projects(projects: list[str], selector: str | None) -> list[str]: return selected -def tiers_to_run(tier: str) -> list[str]: - if tier == "all": - return list(TIERS) +def tiers_to_run(tier: str | None) -> list[str]: + """No tier → the default gated suite (unit+integration); else that single tier.""" + if tier is None: + return list(DEFAULT_TIERS) if tier in TIERS: return [tier] - raise ValueError(f"unknown tier {tier!r} (want unit|integration|e2e|all)") + raise ValueError(f"unknown tier {tier!r} (want unit|integration|e2e)") + +def run_tests( + repo_root: str | Path, projects: list[str], tiers: list[str], gated: bool = True, runner=subprocess.run +) -> int: + """Run each project's existing tiers in one pytest invocation. Returns an exit code. -def run_tests(repo_root: str | Path, projects: list[str], tiers: list[str], runner=subprocess.run) -> int: - """Run ``uv run pytest tests/`` for each existing (project, tier). Returns an exit code.""" + ``gated`` keeps the project's ``addopts`` so ``--cov-fail-under`` applies to the + combined coverage of all tiers run together; otherwise ``addopts`` is cleared + (a single explicit tier is a partial run and shouldn't be coverage-gated). + """ root = Path(repo_root) rc = 0 ran_any = False for rel in projects: proj = root / rel - for tier in tiers: - if not (proj / "tests" / tier).is_dir(): - continue - ran_any = True - print(f"==> {rel} : {tier}") - # Coverage gate (e.g. --cov-fail-under) is a unit-tier concern; a partial - # run of integration/e2e would otherwise fail it on ~0% coverage. - extra = [] if tier == "unit" else ["-o", "addopts="] - result = runner(["uv", "run", "pytest", f"tests/{tier}", *extra], cwd=proj) - if result.returncode != 0: - rc = 1 + present = [t for t in tiers if (proj / "tests" / t).is_dir()] + if not present: + continue + ran_any = True + print(f"==> {rel} : {' '.join(present)}") + paths = [f"tests/{t}" for t in present] + extra = [] if gated else ["-o", "addopts="] + result = runner(["uv", "run", "pytest", *paths, *extra], cwd=proj) + if result.returncode != 0: + rc = 1 if not ran_any: print("No matching test tiers.") return rc diff --git a/tools/ci/ci/cli.py b/tools/ci/ci/cli.py index ea5303f..6ea5cf3 100644 --- a/tools/ci/ci/cli.py +++ b/tools/ci/ci/cli.py @@ -36,7 +36,10 @@ def _cmd_test(args: argparse.Namespace) -> int: selected = sorted({p for c in contexts for p in apptests.select_projects(projects, c)}) else: selected = apptests.select_projects(projects, args.selector) - return apptests.run_tests(args.repo_root, selected, apptests.tiers_to_run(args.tier)) + # No --tier → the gated default suite (unit+integration, combined coverage). + return apptests.run_tests( + args.repo_root, selected, apptests.tiers_to_run(args.tier), gated=args.tier is None + ) def _cmd_images(args: argparse.Namespace) -> int: @@ -61,7 +64,7 @@ def build_parser() -> argparse.ArgumentParser: test = sub.add_parser("test", help="run app pytest suites by tier") test.add_argument("selector", nargs="?", default=None, help="app name or repo-relative path") - test.add_argument("--tier", default="all", choices=["unit", "integration", "e2e", "all"]) + test.add_argument("--tier", default=None, choices=["unit", "integration", "e2e"]) test.add_argument("--affected", action="store_true", help="only projects changed vs --base") test.add_argument("--base", default="origin/main") test.add_argument("--repo-root", default=".") diff --git a/tools/ci/tests/test_apptests.py b/tools/ci/tests/test_apptests.py index b9b4337..c911af8 100644 --- a/tools/ci/tests/test_apptests.py +++ b/tools/ci/tests/test_apptests.py @@ -46,8 +46,9 @@ def test_select_no_match_returns_empty(): def test_tiers_to_run(): - assert tiers_to_run("all") == ["unit", "integration", "e2e"] + assert tiers_to_run(None) == ["unit", "integration"] # default gated suite assert tiers_to_run("unit") == ["unit"] + assert tiers_to_run("e2e") == ["e2e"] import pytest with pytest.raises(ValueError): @@ -63,38 +64,48 @@ def test_discover_finds_pyproject_dirs_and_skips_venv(tmp_path): assert discover_test_projects(tmp_path) == ["stacks/apps/warden/app"] -def _fake_runner(calls, rc_for=None): +def _fake_runner(calls, returncode=0): def runner(cmd, cwd=None): calls.append((cmd, str(cwd))) class R: - returncode = (rc_for or {}).get(cmd[-1], 0) + pass + R.returncode = returncode return R() return runner -def test_run_tests_skips_missing_tiers_and_clears_addopts_for_nonunit(tmp_path): +def test_gated_run_combines_existing_tiers_in_one_call_keeping_addopts(tmp_path): proj = tmp_path / "stacks/apps/warden/app" (proj / "tests" / "unit").mkdir(parents=True) (proj / "tests" / "integration").mkdir(parents=True) - # no tests/e2e → that tier is skipped + # no tests/e2e → excluded calls: list = [] rc = run_tests(tmp_path, ["stacks/apps/warden/app"], ["unit", "integration", "e2e"], - runner=_fake_runner(calls)) - invoked_tiers = [cmd[3] for cmd, _ in calls] # "tests/" - assert invoked_tiers == ["tests/unit", "tests/integration"] - # unit keeps the project's addopts; integration clears them - assert "-o" not in calls[0][0] - assert "-o" in calls[1][0] and "addopts=" in calls[1][0] + gated=True, runner=_fake_runner(calls)) + assert len(calls) == 1 # one combined invocation + cmd = calls[0][0] + assert cmd[:3] == ["uv", "run", "pytest"] + assert "tests/unit" in cmd and "tests/integration" in cmd and "tests/e2e" not in cmd + assert "-o" not in cmd # gated → keep the project's addopts (coverage on the combined run) assert rc == 0 +def test_ungated_run_clears_addopts(tmp_path): + proj = tmp_path / "stacks/apps/warden/app" + (proj / "tests" / "integration").mkdir(parents=True) + calls: list = [] + run_tests(tmp_path, ["stacks/apps/warden/app"], ["integration"], gated=False, + runner=_fake_runner(calls)) + cmd = calls[0][0] + assert "-o" in cmd and "addopts=" in cmd # partial run → no coverage gate + + def test_run_tests_propagates_failure(tmp_path): proj = tmp_path / "stacks/apps/warden/app" (proj / "tests" / "unit").mkdir(parents=True) - calls: list = [] rc = run_tests(tmp_path, ["stacks/apps/warden/app"], ["unit"], - runner=_fake_runner(calls, rc_for={"tests/unit": 1})) + runner=_fake_runner([], returncode=1)) assert rc == 1