Deprecate 'FCAS Providers' static table — AEMO migrated the data (#92)

README.md

@@ -230,9 +230,17 @@ You can view the static tables available by printing the tables in NEMOSIS' defa
 from nemosis import defaults
 
 print(defaults.static_tables)
-# ['ELEMENTS_FCAS_4_SECOND', 'VARIABLES_FCAS_4_SECOND', 'Generators and Scheduled Loads', 'FCAS Providers']
+# ['ELEMENTS_FCAS_4_SECOND', 'VARIABLES_FCAS_4_SECOND', 'Generators and Scheduled Loads']
 ```
 
+Note: `'FCAS Providers'` was previously listed here but has been
+deprecated — AEMO migrated the underlying data out of the
+`NEM Registration and Exemption List.xlsx` workbook to a weekly archive
+at <https://www.nemweb.com.au/REPORTS/CURRENT/ANCILLARY_SERVICES_REPORTS/>.
+See issue [#92](https://github.com/UNSW-CEEM/NEMOSIS/issues/92) for
+status. Calls to `static_table("FCAS Providers", ...)` now raise a
+`UserInputError` pointing at the new endpoint.
+
 #### static_table
 
 The `static_table` function can be used to access these tables

src/nemosis/data_fetch_methods.py

@@ -299,6 +299,23 @@ def static_table(
     Returns:
         data (pd.Dataframe)
     """
+    if table_name == "FCAS Providers":
+        # Deprecated: AEMO emptied the `Ancillary Services` sheet of
+        # `NEM Registration and Exemption List.xlsx` and moved the data
+        # to a weekly archive on nemweb. The existing NEMOSIS handler
+        # reads the (now-empty) sheet and chokes on the dedup step.
+        # See issue #92.
+        raise UserInputError(
+            "The 'FCAS Providers' static table is no longer available "
+            "via NEMOSIS. AEMO migrated this data out of "
+            "'NEM Registration and Exemption List.xlsx' to a weekly "
+            "archive at "
+            "https://www.nemweb.com.au/REPORTS/CURRENT/ANCILLARY_SERVICES_REPORTS/ "
+            "(file pattern: PUBLIC_ANCILLARY_SERVICES_YYYYMMDD.zip). "
+            "See issue #92 for tracking. "
+            "The 'Generators and Scheduled Loads' table still works."
+        )
+
     if raw_data_location is None:
         raise UserInputError("The raw_data_location provided is None.")
 

src/nemosis/defaults.py

@@ -125,7 +125,10 @@
     "ELEMENTS_FCAS_4_SECOND",
     "VARIABLES_FCAS_4_SECOND",
     "Generators and Scheduled Loads",
-    "FCAS Providers",
+    # "FCAS Providers" was deprecated when AEMO migrated the underlying
+    # data out of the registration XLS workbook. See issue #92.
+    # static_table() raises UserInputError early with a pointer to the
+    # new endpoint, so calls to it never reach this list anyway.
 ]
 
 nem_web_domain_url = "https://nemweb.com.au/"

tests/end_to_end_table_tests/test_static_tables.py

@@ -1,15 +1,23 @@
 """Tests for AEMO's static (time-independent) tables.
 
-These four tables share one test file because they have no date dimension —
-each is downloaded once as a snapshot and cached verbatim. The fixtures are
-frozen copies of what AEMO was serving when build.py last ran.
+These tables have no date dimension — each is downloaded once as a
+snapshot and cached verbatim. The fixtures are frozen copies of what
+AEMO was serving when build.py last ran.
 
   - ELEMENTS_FCAS_4_SECOND      — CSV, scraped from a directory index
   - VARIABLES_FCAS_4_SECOND     — CSV, direct download
   - Generators and Scheduled Loads — XLS workbook, direct download
-  - FCAS Providers              — different tab of the same XLS workbook
+
+"FCAS Providers" was a fourth static table sharing the registration XLS
+workbook with "Generators and Scheduled Loads". AEMO has since emptied
+the underlying sheet and migrated the data to a weekly archive on
+nemweb — see issue #92. The handler now raises early; coverage is the
+deprecation-error tests below.
 """
+import pytest
+
 from nemosis import static_table
+from nemosis.custom_errors import UserInputError
 
 
 def test_elements_fcas_returns_non_empty_frame(nemosis_fixture):
@@ -27,14 +35,24 @@ def test_generators_and_scheduled_loads_returns_non_empty_frame(nemosis_fixture)
     assert not data.empty
 
 
-def test_fcas_providers_returns_non_empty_frame(nemosis_fixture):
-    # "FCAS Providers" reads a different sheet of the same XLS workbook as
-    # "Generators and Scheduled Loads". NEMOSIS's static_downloader_map has
-    # no entry for "FCAS Providers", so the first fetch errors on an empty
-    # cache — prime it by fetching the sibling table.
-    static_table("Generators and Scheduled Loads", str(nemosis_fixture))
-    data = static_table("FCAS Providers", str(nemosis_fixture))
-    assert not data.empty
+def test_fcas_providers_raises_deprecation_error(nemosis_fixture):
+    """FCAS Providers was deprecated when AEMO migrated the data out of
+    the registration XLS workbook (issue #92). The handler raises
+    immediately, regardless of cache state, with a pointer to the new
+    nemweb endpoint."""
+    with pytest.raises(UserInputError, match="no longer available"):
+        static_table("FCAS Providers", str(nemosis_fixture))
+
+
+def test_fcas_providers_deprecation_message_points_to_new_endpoint(nemosis_fixture):
+    """The deprecation error must include the new AEMO endpoint URL so
+    users have somewhere to go. This test locks the URL into the
+    message; updating where the data lives requires updating this test."""
+    with pytest.raises(UserInputError) as excinfo:
+        static_table("FCAS Providers", str(nemosis_fixture))
+    msg = str(excinfo.value)
+    assert "ANCILLARY_SERVICES_REPORTS" in msg
+    assert "issue #92" in msg
 
 
 # ---------------------------------------------------------------------------
@@ -76,13 +94,6 @@ def test_generators_filter_narrows(nemosis_fixture):
     assert set(filtered["Region"]) == {"SA1"}
 
 
-def test_fcas_providers_filter_narrows(nemosis_fixture):
-    static_table("Generators and Scheduled Loads", str(nemosis_fixture))
-    cols = ["Participant", "Region", "DUID", "Bid Type"]
-    full = static_table("FCAS Providers", str(nemosis_fixture), select_columns=cols)
-    filtered = static_table(
-        "FCAS Providers", str(nemosis_fixture),
-        select_columns=cols, filter_cols=["Region"], filter_values=[["SA1"]],
-    )
-    assert 0 < len(filtered) < len(full)
-    assert set(filtered["Region"]) == {"SA1"}
+# FCAS Providers filter test removed — the table is deprecated (see
+# issue #92), so there's no longer a happy path to filter against.
+# The deprecation-error tests above cover the new behaviour.