[MNT] Add test to check that soft dependency skipping is working as intended

.github/workflows/periodic_tests.yml

@@ -156,6 +156,29 @@ jobs:
           # Save cache with the current date (ENV set in numba_cache action)
           key: numba-test-no-soft-deps-${{ runner.os }}-3.12-${{ env.CURRENT_DATE }}
 
+  test-soft-dep-skips:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install aeon and dependencies
+        uses: ./.github/actions/cpu_all_extras
+        with:
+          additional_extras: "dev,unstable_extras"
+
+      - name: Show dependencies
+        run: python -m pip list
+
+      - name: Run tests
+        run: python -m pytest --check-soft-dependency-skips -q
+
   pytest:
     runs-on: ${{ matrix.os }}
 

.github/workflows/pr_pytest.yml

@@ -48,6 +48,29 @@ jobs:
       - name: Run tests
         run: python -m pytest -n logical --prtesting ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'full pytest actions') }}
 
+  test-soft-dep-skips:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install aeon and dependencies
+        uses: ./.github/actions/cpu_all_extras
+        with:
+          additional_extras: "dev,unstable_extras"
+
+      - name: Show dependencies
+        run: python -m pip list
+
+      - name: Run tests
+        run: python -m pytest --prtesting ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'full pytest actions') }} --check-soft-dependency-skips -q
+
   pytest:
     runs-on: ${{ matrix.os }}
 

aeon/datasets/tests/test_monster_loader.py

@@ -12,7 +12,7 @@
 
 
 @pytest.mark.skipif(
-    not _check_soft_dependencies("huggingface-hub", severity="none"),
+    not _check_soft_dependencies("huggingface_hub", severity="none"),
     reason="required soft dependency huggingface-hub not available",
 )
 @pytest.mark.xfail(raises=CONNECTION_ERRORS)
@@ -25,7 +25,7 @@ def test_monster_dataset_names():
 
 
 @pytest.mark.skipif(
-    not _check_soft_dependencies("huggingface-hub", severity="none"),
+    not _check_soft_dependencies("huggingface_hub", severity="none"),
     reason="required soft dependency huggingface-hub not available",
 )
 @pytest.mark.xfail(raises=CONNECTION_ERRORS)

aeon/testing/estimator_checking/_estimator_checking.py

@@ -88,7 +88,7 @@ class is passed.
         ):
             # wrap check to skip if necessary (missing dependencies, on an exclude list
             # etc.)
-            checks.append(_check_if_xfail(est, check, has_dependencies))
+            checks.append(_check_if_skip_pytest(est, check, has_dependencies))
 
     # return a pytest parametrize decorator with custom ids
     return pytest.mark.parametrize(
@@ -204,7 +204,7 @@ class is passed.
         has_dependencies=True,
     ):
         # wrap check to skip if necessary (on an exclude list etc.)
-        checks.append(_check_if_skip(estimator, check, True))
+        checks.append(_check_if_skip_wrapper(estimator, check, True))
 
     # process run/exclude lists to filter checks
     if not isinstance(checks_to_run, (list, tuple)) and checks_to_run is not None:
@@ -275,18 +275,26 @@ class is passed.
     return results
 
 
-def _check_if_xfail(estimator, check, has_dependencies):
-    """Check if a check should be xfailed."""
+def _check_if_skip_pytest(estimator, check, has_dependencies):
+    """Check if a check should be skipped in a pytest setting."""
     import pytest
 
-    skip, reason, _ = _should_be_skipped(estimator, check, has_dependencies)
+    skip, reason, check_name = _should_be_skipped(estimator, check, has_dependencies)
     if skip:
-        return pytest.param(check, marks=pytest.mark.xfail(reason=reason))
+        est_name = (
+            estimator.__name__ if isclass(estimator) else estimator.__class__.__name__
+        )
+        return pytest.param(
+            check,
+            marks=pytest.mark.skip(
+                reason=f"Skipping test {check_name} for {est_name}: {reason}"
+            ),
+        )
 
     return check
 
 
-def _check_if_skip(estimator, check, has_dependencies):
+def _check_if_skip_wrapper(estimator, check, has_dependencies):
     """Check if a check should be skipped by raising a SkipTest exception."""
     skip, reason, check_name = _should_be_skipped(estimator, check, has_dependencies)
     if skip:
@@ -298,7 +306,7 @@ def wrapped(*args, **kwargs):
                 if isclass(estimator)
                 else estimator.__class__.__name__
             )
-            raise SkipTest(f"Skipping {check_name} for {est_name}: {reason}")
+            raise SkipTest(f"Skipping test {check_name} for {est_name}: {reason}")
 
         return wrapped
     return check
@@ -312,7 +320,12 @@ def _should_be_skipped(estimator, check, has_dependencies):
 
     # check estimator dependencies
     if not has_dependencies and "softdep" not in check_name:
-        return True, "Incompatible dependencies or Python version", check_name
+        return (
+            True,
+            "Incompatible dependencies or Python version. Check may require "
+            "a soft dependency.",
+            check_name,
+        )
 
     # check aeon exclude lists
     if est_name in EXCLUDE_ESTIMATORS:

conftest.py

@@ -10,6 +10,8 @@
 
 __maintainer__ = ["MatthewMiddlehurst"]
 
+import pytest
+
 
 def pytest_addoption(parser):
     """Pytest command line parser options adder."""
@@ -35,6 +37,15 @@ def pytest_addoption(parser):
             "version."
         ),
     )
+    parser.addoption(
+        "--check-soft-dependency-skips",
+        action="store_true",
+        default=False,
+        help=(
+            "Fail tests skipped by soft dependency checks. Skips all other tests. "
+            "Use only in environments where soft dependencies are installed."
+        ),
+    )
 
 
 def pytest_configure(config):
@@ -80,3 +91,53 @@ def pytest_configure(config):
         from aeon.testing import testing_config
 
         testing_config.PR_TESTING = True
+
+    if config.getoption("--check-soft-dependency-skips"):
+        config.pluginmanager.register(
+            _CheckSoftDependencySkips(),
+            name="check-soft-dependency-skips",
+        )
+
+
+class _CheckSoftDependencySkips:
+    _SOFT_DEPENDENCY_TERMS = (
+        "soft dependency",
+        "soft dependencies",
+        "soft-dependency",
+        "soft-dependencies",
+    )
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_runtest_setup(self, item):
+        pytest.skip("Tests are not run in current testing setup.")
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtest_makereport(self, item, call):
+        outcome = yield
+        report = outcome.get_result()
+
+        if self._is_soft_dependency_skip(report):
+            skip_reason = self._get_skip_reason(report)
+            report.outcome = "failed"
+            report.longrepr = (
+                "Test skipped because a soft dependency check failed while "
+                "--check-soft-dependency-skips is enabled. "
+                f"Original skip reason: {skip_reason}"
+            )
+
+    @staticmethod
+    def _get_skip_reason(report):
+        """Extract the skip reason across pytest longrepr formats."""
+        if isinstance(report.longrepr, tuple) and len(report.longrepr) >= 3:
+            return str(report.longrepr[2])
+
+        return str(report.longrepr)
+
+    @classmethod
+    def _is_soft_dependency_skip(cls, report):
+        """Check if a soft dependency related test is skipped."""
+        if not report.skipped:
+            return False
+
+        reason = cls._get_skip_reason(report).lower()
+        return any(term in reason for term in cls._SOFT_DEPENDENCY_TERMS)

pyproject.toml

@@ -187,8 +187,9 @@ addopts = [
     "--dist=worksteal",
     "--reruns=3",
     "--reruns-delay=3",
-    "--rerun-except=(?<!Permission)Error",
-    "--rerun-except=Exception",
+    "--rerun-except=^(?!PermissionError:).*Error:",
+    "--rerun-except=Exception:",
+    "--rerun-except=Skipping test",
 ]
 filterwarnings = [
     "ignore::UserWarning",