diff --git a/.github/workflows/periodic_tests.yml b/.github/workflows/periodic_tests.yml index 62bffca07c..34dd74f186 100644 --- a/.github/workflows/periodic_tests.yml +++ b/.github/workflows/periodic_tests.yml @@ -156,6 +156,29 @@ jobs: # Save cache with the current date (ENV set in numba_cache action) key: numba-test-no-soft-deps-${{ runner.os }}-3.12-${{ env.CURRENT_DATE }} + test-soft-dep-skips: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install aeon and dependencies + uses: ./.github/actions/cpu_all_extras + with: + additional_extras: "dev,unstable_extras" + + - name: Show dependencies + run: python -m pip list + + - name: Run tests + run: python -m pytest --check-soft-dependency-skips -q + pytest: runs-on: ${{ matrix.os }} diff --git a/.github/workflows/pr_pytest.yml b/.github/workflows/pr_pytest.yml index 0cc08e3a23..9b0b1b00cf 100644 --- a/.github/workflows/pr_pytest.yml +++ b/.github/workflows/pr_pytest.yml @@ -48,6 +48,29 @@ jobs: - name: Run tests run: python -m pytest -n logical --prtesting ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'full pytest actions') }} + test-soft-dep-skips: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install aeon and dependencies + uses: ./.github/actions/cpu_all_extras + with: + additional_extras: "dev,unstable_extras" + + - name: Show dependencies + run: python -m pip list + + - name: Run tests + run: python -m pytest --prtesting ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'full pytest actions') }} --check-soft-dependency-skips -q + pytest: runs-on: ${{ matrix.os }} diff --git a/aeon/datasets/tests/test_monster_loader.py b/aeon/datasets/tests/test_monster_loader.py index 56a1b2cde1..f020122a70 100644 --- a/aeon/datasets/tests/test_monster_loader.py +++ b/aeon/datasets/tests/test_monster_loader.py @@ -12,7 +12,7 @@ @pytest.mark.skipif( - not _check_soft_dependencies("huggingface-hub", severity="none"), + not _check_soft_dependencies("huggingface_hub", severity="none"), reason="required soft dependency huggingface-hub not available", ) @pytest.mark.xfail(raises=CONNECTION_ERRORS) @@ -25,7 +25,7 @@ def test_monster_dataset_names(): @pytest.mark.skipif( - not _check_soft_dependencies("huggingface-hub", severity="none"), + not _check_soft_dependencies("huggingface_hub", severity="none"), reason="required soft dependency huggingface-hub not available", ) @pytest.mark.xfail(raises=CONNECTION_ERRORS) diff --git a/aeon/testing/estimator_checking/_estimator_checking.py b/aeon/testing/estimator_checking/_estimator_checking.py index e529ef2d27..1783afebfa 100644 --- a/aeon/testing/estimator_checking/_estimator_checking.py +++ b/aeon/testing/estimator_checking/_estimator_checking.py @@ -88,7 +88,7 @@ class is passed. ): # wrap check to skip if necessary (missing dependencies, on an exclude list # etc.) - checks.append(_check_if_xfail(est, check, has_dependencies)) + checks.append(_check_if_skip_pytest(est, check, has_dependencies)) # return a pytest parametrize decorator with custom ids return pytest.mark.parametrize( @@ -204,7 +204,7 @@ class is passed. has_dependencies=True, ): # wrap check to skip if necessary (on an exclude list etc.) - checks.append(_check_if_skip(estimator, check, True)) + checks.append(_check_if_skip_wrapper(estimator, check, True)) # process run/exclude lists to filter checks if not isinstance(checks_to_run, (list, tuple)) and checks_to_run is not None: @@ -275,18 +275,26 @@ class is passed. return results -def _check_if_xfail(estimator, check, has_dependencies): - """Check if a check should be xfailed.""" +def _check_if_skip_pytest(estimator, check, has_dependencies): + """Check if a check should be skipped in a pytest setting.""" import pytest - skip, reason, _ = _should_be_skipped(estimator, check, has_dependencies) + skip, reason, check_name = _should_be_skipped(estimator, check, has_dependencies) if skip: - return pytest.param(check, marks=pytest.mark.xfail(reason=reason)) + est_name = ( + estimator.__name__ if isclass(estimator) else estimator.__class__.__name__ + ) + return pytest.param( + check, + marks=pytest.mark.skip( + reason=f"Skipping test {check_name} for {est_name}: {reason}" + ), + ) return check -def _check_if_skip(estimator, check, has_dependencies): +def _check_if_skip_wrapper(estimator, check, has_dependencies): """Check if a check should be skipped by raising a SkipTest exception.""" skip, reason, check_name = _should_be_skipped(estimator, check, has_dependencies) if skip: @@ -298,7 +306,7 @@ def wrapped(*args, **kwargs): if isclass(estimator) else estimator.__class__.__name__ ) - raise SkipTest(f"Skipping {check_name} for {est_name}: {reason}") + raise SkipTest(f"Skipping test {check_name} for {est_name}: {reason}") return wrapped return check @@ -312,7 +320,12 @@ def _should_be_skipped(estimator, check, has_dependencies): # check estimator dependencies if not has_dependencies and "softdep" not in check_name: - return True, "Incompatible dependencies or Python version", check_name + return ( + True, + "Incompatible dependencies or Python version. Check may require " + "a soft dependency.", + check_name, + ) # check aeon exclude lists if est_name in EXCLUDE_ESTIMATORS: diff --git a/conftest.py b/conftest.py index 015e145ae5..39cc2e0a6e 100644 --- a/conftest.py +++ b/conftest.py @@ -10,6 +10,8 @@ __maintainer__ = ["MatthewMiddlehurst"] +import pytest + def pytest_addoption(parser): """Pytest command line parser options adder.""" @@ -35,6 +37,15 @@ def pytest_addoption(parser): "version." ), ) + parser.addoption( + "--check-soft-dependency-skips", + action="store_true", + default=False, + help=( + "Fail tests skipped by soft dependency checks. Skips all other tests. " + "Use only in environments where soft dependencies are installed." + ), + ) def pytest_configure(config): @@ -80,3 +91,53 @@ def pytest_configure(config): from aeon.testing import testing_config testing_config.PR_TESTING = True + + if config.getoption("--check-soft-dependency-skips"): + config.pluginmanager.register( + _CheckSoftDependencySkips(), + name="check-soft-dependency-skips", + ) + + +class _CheckSoftDependencySkips: + _SOFT_DEPENDENCY_TERMS = ( + "soft dependency", + "soft dependencies", + "soft-dependency", + "soft-dependencies", + ) + + @pytest.hookimpl(trylast=True) + def pytest_runtest_setup(self, item): + pytest.skip("Tests are not run in current testing setup.") + + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + outcome = yield + report = outcome.get_result() + + if self._is_soft_dependency_skip(report): + skip_reason = self._get_skip_reason(report) + report.outcome = "failed" + report.longrepr = ( + "Test skipped because a soft dependency check failed while " + "--check-soft-dependency-skips is enabled. " + f"Original skip reason: {skip_reason}" + ) + + @staticmethod + def _get_skip_reason(report): + """Extract the skip reason across pytest longrepr formats.""" + if isinstance(report.longrepr, tuple) and len(report.longrepr) >= 3: + return str(report.longrepr[2]) + + return str(report.longrepr) + + @classmethod + def _is_soft_dependency_skip(cls, report): + """Check if a soft dependency related test is skipped.""" + if not report.skipped: + return False + + reason = cls._get_skip_reason(report).lower() + return any(term in reason for term in cls._SOFT_DEPENDENCY_TERMS) diff --git a/pyproject.toml b/pyproject.toml index b45c090e92..9df314d4d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -187,8 +187,9 @@ addopts = [ "--dist=worksteal", "--reruns=3", "--reruns-delay=3", - "--rerun-except=(?