Enhance API client to support JSON strings for ARC submissions and improve error handling for invalid JSON inputs

Co-authored-by: Copilot <copilot@github.com>

Author: Zalfsten

middleware/api_client/README.md

@@ -51,13 +51,13 @@ async def main():
         # Send a single ARC
         response = await client.create_or_update_arc(
             rdi="my-rdi",
-            arc=arc,
+            arc=arc,  # Can be ARC object, dict, or JSON string
         )
         print(f"ARC status: {response.status}")
 
         # Or run a harvest workflow
         async def arc_stream():
-            yield arc
+            yield arc  # Can yield ARC objects, dicts, or JSON strings
 
         harvest = await client.harvest_arcs(
             rdi="my-rdi",
@@ -85,22 +85,22 @@ asyncio.run(main())
 
 ## API Methods
 
-### `create_or_update_arc(rdi: str, arc: ARC | dict) -> ArcResult`
+### `create_or_update_arc(rdi: str, arc: ARC | dict | str) -> ArcResult`
 
 Create or update one ARC in the Middleware API.
 
 **Parameters:**
 
 - `rdi` (str): The RDI identifier (e.g., "edaphobase").
-- `arc` (ARC | dict): ARC object from arctrl or pre-serialised RO-Crate dict.
+- `arc` (ARC | dict | str): ARC object from arctrl, pre-serialised RO-Crate dict, or JSON string.
 
 **Returns:**
 
 - `ArcResult`: Contains the result of the operation.
 
 **Raises:**
 
-- `ApiClientError`: If the request fails due to HTTP errors or network issues.
+- `ApiClientError`: If the request fails due to HTTP errors, network issues, or invalid JSON.
 
 **Example:**
 
@@ -112,17 +112,18 @@ arc = ARC.from_arc_investigation(inv)
 
 response = await client.create_or_update_arc(
     rdi="edaphobase",
-    arc=arc,
+    arc=arc,  # Can also be dict or JSON string
 )
 ```
 
-### `harvest_arcs(rdi: str, arcs: AsyncIterator[ARC | dict], expected_datasets: int | None = None) -> HarvestResult`
+### `harvest_arcs(rdi: str, arcs: AsyncIterator[ARC | dict | str], expected_datasets: int | None = None) -> HarvestResult`
 
 Convenience workflow to create a harvest, upload all ARCs from an async iterator, and complete the harvest.
 
 - Uses `config.max_concurrency` by default.
 - Continues on item-level submission errors and skips failed items.
 - Cancels the harvest only for catastrophic errors.
+- Supports ARC objects, pre-serialised RO-Crate dicts, and JSON strings.
 
 All errors are raised as `ApiClientError` exceptions:
 
@@ -132,24 +133,7 @@ from middleware.api_client import ApiClientError
 try:
     response = await client.create_or_update_arc(
         rdi="my-rdi",
-        arc=arc,
+        arc=arc,  # Can be ARC object, dict, or JSON string
     )
 except ApiClientError as e:
     print(f"API Error: {e}")
-```
-
-## Configuration via Environment Variables
-
-You can override configuration values using environment variables:
-
-```bash
-export API_URL="https://production-api:8000"
-export CLIENT_CERT_PATH="/secure/certs/prod-cert.pem"
-export CLIENT_KEY_PATH="/secure/certs/prod-key.pem"
-```
-
-Or use Docker secrets in `/run/secrets/`.
-
-## License
-
-This is part of the FAIRagro Advanced Middleware project.

middleware/api_client/src/middleware/api_client/api_client.py

@@ -262,13 +262,13 @@ def _process_completed_arc_tasks(
     async def _submit_arcs_parallel(
         self,
         harvest_id: str,
-        arcs: "AsyncGenerator[ARC | dict[str, Any], None] | AsyncIterator[ARC | dict[str, Any]]",
+        arcs: "AsyncGenerator[ARC | dict[str, Any] | str, None] | AsyncIterator[ARC | dict[str, Any] | str]",
     ) -> int:
         """Submit all ARCs in bounded parallelism and return number of skipped ARC submissions."""
         pending_tasks: set[asyncio.Task[None]] = set()
         failed_submissions = 0
 
-        async def submit_one(arc_item: "ARC | dict[str, Any]") -> None:
+        async def submit_one(arc_item: "ARC | dict[str, Any] | str") -> None:
             await self.submit_arc_in_harvest(harvest_id, arc_item)
 
         async for arc in arcs:
@@ -446,10 +446,15 @@ async def _delete(self, path: str) -> None:
     # ------------------------------------------------------------------
 
     @classmethod
-    def _serialize_arc(cls, arc: "ARC | dict[str, Any]") -> dict[str, Any]:
-        """Serialize an ARC object to a plain RO-Crate JSON dict."""
+    def _serialize_arc(cls, arc: "ARC | dict[str, Any] | str") -> dict[str, Any]:
+        """Serialize an ARC object, dict, or JSON string to a plain RO-Crate JSON dict."""
         if isinstance(arc, dict):
             return arc
+        if isinstance(arc, str):
+            try:
+                return cast(dict[str, Any], json.loads(arc))
+            except json.JSONDecodeError as e:
+                raise ApiClientError(f"Invalid JSON string provided for ARC: {e}") from e
         return cast(dict[str, Any], json.loads(arc.ToROCrateJsonString()))
 
     @classmethod
@@ -473,7 +478,7 @@ def _parse_harvest_response(cls, data: Any) -> HarvestResult:
     async def create_or_update_arc(
         self,
         rdi: str,
-        arc: "ARC | dict[str, Any]",
+        arc: "ARC | dict[str, Any] | str",
     ) -> ArcResult:
         """Create or update an ARC.
 
@@ -483,7 +488,7 @@ async def create_or_update_arc(
 
         Args:
             rdi: RDI identifier.
-            arc: ARC object or a pre-serialised RO-Crate JSON dict.
+            arc: ARC object, a pre-serialised RO-Crate JSON dict, or a JSON string.
 
         Returns:
             :class:`ArcResult` with the result of the operation.
@@ -579,7 +584,7 @@ async def cancel_harvest(self, harvest_id: str) -> None:
     async def submit_arc_in_harvest(
         self,
         harvest_id: str,
-        arc: "ARC | dict[str, Any]",
+        arc: "ARC | dict[str, Any] | str",
     ) -> ArcResult:
         """Submit an ARC within an active harvest run.
 
@@ -588,7 +593,7 @@ async def submit_arc_in_harvest(
 
         Args:
             harvest_id: Harvest identifier.
-            arc: ARC object or a pre-serialised RO-Crate JSON dict.
+            arc: ARC object, a pre-serialised RO-Crate JSON dict, or a JSON string.
 
         Returns:
             :class:`ArcResult` with the result of the operation.
@@ -601,7 +606,7 @@ async def submit_arc_in_harvest(
     async def harvest_arcs(
         self,
         rdi: str,
-        arcs: "AsyncGenerator[ARC | dict[str, Any], None] | AsyncIterator[ARC | dict[str, Any]]",
+        arcs: "AsyncGenerator[ARC | dict[str, Any] | str, None] | AsyncIterator[ARC | dict[str, Any] | str]",
         expected_datasets: int | None = None,
     ) -> HarvestResult:
         """Create a harvest, upload all ARCs from an async generator, then complete it.
@@ -618,8 +623,8 @@ async def harvest_arcs(
 
         Args:
             rdi: RDI identifier for the harvest.
-            arcs: Async generator or async iterator yielding ARC objects or
-                pre-serialised RO-Crate dicts.
+            arcs: Async generator or async iterator yielding ARC objects,
+                pre-serialised RO-Crate dicts, or JSON strings.
             expected_datasets: Optional hint about the total number of ARCs.
 
         Returns:
@@ -631,7 +636,7 @@ async def harvest_arcs(
 
         Example::
 
-            async def my_arcs() -> AsyncGenerator[dict, None]:
+            async def my_arcs() -> AsyncGenerator[dict | str, None]:
                 for arc in source:
                     yield arc
 

middleware/api_client/tests/unit/test_client.py

@@ -226,6 +226,28 @@ async def test_create_or_update_arc_with_dict(client_config: Config) -> None:
     assert isinstance(response, ArcResult)
 
 
+@pytest.mark.asyncio
+@respx.mock
+async def test_create_or_update_arc_with_json_string(client_config: Config) -> None:
+    """Test create_or_update_arc with a JSON string."""
+    route = respx.post(f"{client_config.api_url}v3/arcs").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_ARC_RESPONSE)
+    )
+    async with ApiClient(client_config) as client:
+        response = await client.create_or_update_arc(rdi="test-rdi", arc='{"id": "mock-arc"}')
+    assert route.called
+    assert isinstance(response, ArcResult)
+    assert response.arc_id == "arc-123"
+
+
+@pytest.mark.asyncio
+async def test_create_or_update_arc_with_invalid_json_string(client_config: Config) -> None:
+    """Test create_or_update_arc with an invalid JSON string."""
+    async with ApiClient(client_config) as client:
+        with pytest.raises(ApiClientError, match="Invalid JSON string provided for ARC"):
+            await client.create_or_update_arc(rdi="test-rdi", arc='{"id": "mock-arc"')
+
+
 @pytest.mark.asyncio
 @respx.mock
 async def test_create_or_update_arc_http_error(client_config: Config) -> None:
@@ -495,13 +517,35 @@ async def test_submit_arc_in_harvest_invalid_response(client_config: Config) ->
             await client.submit_arc_in_harvest("harvest-456", arc={"id": "mock"})
 
 
+@pytest.mark.asyncio
+@respx.mock
+async def test_submit_arc_in_harvest_with_json_string(client_config: Config) -> None:
+    """Test submit_arc_in_harvest with a JSON string."""
+    route = respx.post(f"{client_config.api_url}v3/harvests/harvest-456/arcs").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_ARC_RESPONSE)
+    )
+    async with ApiClient(client_config) as client:
+        response = await client.submit_arc_in_harvest("harvest-456", arc='{"id": "mock-arc"}')
+    assert route.called
+    assert isinstance(response, ArcResult)
+    assert response.arc_id == "arc-123"
+
+
+@pytest.mark.asyncio
+async def test_submit_arc_in_harvest_with_invalid_json_string(client_config: Config) -> None:
+    """Test submit_arc_in_harvest with an invalid JSON string."""
+    async with ApiClient(client_config) as client:
+        with pytest.raises(ApiClientError, match="Invalid JSON string provided for ARC"):
+            await client.submit_arc_in_harvest("harvest-456", arc='{"id": "mock-arc"')
+
+
 # ---------------------------------------------------------------------------
 # harvest_arcs
 # ---------------------------------------------------------------------------
 
 
-async def _arc_gen(*arcs: "dict[str, Any]") -> AsyncGenerator["dict[str, Any]", None]:
-    """Yield the provided arc dicts as an async generator."""
+async def _arc_gen(*arcs: "dict[str, Any] | str | ARC") -> AsyncGenerator["dict[str, Any] | str | ARC", None]:
+    """Yield the provided arc dicts, JSON strings, or ARC objects as an async generator."""
     for arc in arcs:
         yield arc
 
@@ -650,6 +694,52 @@ async def test_harvest_arcs_cancels_on_catastrophic_error(client_config: Config)
     assert cancel_route.called
 
 
+@pytest.mark.asyncio
+@respx.mock
+async def test_harvest_arcs_with_json_string(client_config: Config) -> None:
+    """harvest_arcs supports JSON strings in async generator."""
+    completed_response = {**_HARVEST_RESPONSE, "status": "COMPLETED", "completed_at": "2024-01-01T01:00:00Z"}
+    respx.post(f"{client_config.api_url}v3/harvests").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_HARVEST_RESPONSE)
+    )
+    respx.post(f"{client_config.api_url}v3/harvests/harvest-456/arcs").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_ARC_RESPONSE)
+    )
+    respx.post(f"{client_config.api_url}v3/harvests/harvest-456/complete").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=completed_response)
+    )
+
+    arcs = _arc_gen(
+        '{"id": "arc-1-string"}',
+        {"id": "arc-2-dict"},
+        ARC.from_arc_investigation(ArcInvestigation.create(identifier="test", title="Test")),
+    )
+    async with ApiClient(client_config) as client:
+        result = await client.harvest_arcs("test-rdi", arcs, expected_datasets=3)
+
+    assert isinstance(result, HarvestResult)
+    assert result.status == "COMPLETED"
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_harvest_arcs_with_invalid_json_string(client_config: Config) -> None:
+    """harvest_arcs raises ApiClientError when JSON string is invalid."""
+    # Mock the harvest creation endpoint to prevent actual HTTP requests
+    respx.post(f"{client_config.api_url}v3/harvests").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_HARVEST_RESPONSE)
+    )
+    # Mock the harvest cancellation endpoint
+    respx.delete(f"{client_config.api_url}v3/harvests/harvest-456").mock(
+        return_value=httpx.Response(http.HTTPStatus.NO_CONTENT)
+    )
+
+    async with ApiClient(client_config) as client:
+        arcs = _arc_gen('{"id": "arc-1"')  # Single invalid JSON string
+        with pytest.raises(ApiClientError, match="Invalid JSON string provided for ARC"):
+            await client.harvest_arcs("test-rdi", arcs)
+
+
 @pytest.mark.asyncio
 @respx.mock
 async def test_harvest_arcs_cancel_failure_does_not_mask_original_error(client_config: Config) -> None: