From cb740cd0919678f9af010b610bf5b766bf3398bc Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 11:12:13 +0000 Subject: [PATCH 01/29] Add mock mode for ConfigClient --- src/daq_config_server/app/client.py | 149 +++++++++++++++++++++++----- 1 file changed, 124 insertions(+), 25 deletions(-) diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index 955523da..d3e1b642 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -1,10 +1,11 @@ +import json import logging import operator from collections.abc import Callable from logging import Logger, getLogger from pathlib import Path from threading import RLock -from typing import Any, TypeVar, get_origin, overload +from typing import Any, Protocol, TypeVar, get_origin, overload import requests from cachetools import TTLCache, cachedmethod @@ -21,10 +22,122 @@ TModel = TypeVar("TModel", bound=ConfigModel) TNonModel = TypeVar("TNonModel", str, bytes, dict[str, Any]) +T = TypeVar("T", str, dict[str, Any], ConfigModel) + +ConverterDict = dict[str, Callable[[str], ConfigModel]] + class TypeConversionError(Exception): ... +class MockResponse: + def __init__( + self, + data: Any, + content_type: ValidAcceptHeaders, + status_code: int = 200, + ): + self._data = data + self.headers = {"content-type": content_type} + self.status_code = status_code + + def raise_for_status(self): + if self.status_code >= 400: + raise requests.exceptions.HTTPError() + + def json(self) -> Any: + return self._data + + @property + def text(self) -> str: + if isinstance(self._data, str): + return self._data + return json.dumps(self._data) + + @property + def content(self) -> bytes: + if isinstance(self._data, bytes): + return self._data + if isinstance(self._data, str): + return self._data.encode() + return json.dumps(self._data).encode() + + +ResponseType = Response | MockResponse + + +class ServerResponse(Protocol): + def get_response( + self, + endpoint: str, + accept_header: ValidAcceptHeaders, + file_path: Path, + ) -> ResponseType: ... + + +class MockServerResponse(ServerResponse): + def __init__(self, mock_data_converters: ConverterDict | None = None): + self.mock_data_converters = mock_data_converters or {} + + def get_response( + self, + endpoint: str, + accept_header: ValidAcceptHeaders, + file_path: Path, + ) -> MockResponse: + raw = file_path.read_text() + + if accept_header == ValidAcceptHeaders.JSON: + return MockResponse(json.loads(raw), accept_header) + + if accept_header == ValidAcceptHeaders.PLAIN_TEXT: + return MockResponse(raw, accept_header) + + return MockResponse(raw.encode(), accept_header) + + +class RealServerResponse(ServerResponse): + def __init__(self, url: str, log: Logger): + self._url = url + self._log = log + + def get_response( + self, + endpoint: str, + accept_header: ValidAcceptHeaders, + file_path: Path, + ) -> ResponseType: + """ + Get data from the config server and cache it. + + Args: + endpoint: API endpoint. + accept_header: Accept header MIME type + file_path: absolute path to the file which will be read + + Returns: + The response data. + """ + + request_url = self._url + endpoint + (f"/{file_path}") + r = requests.get(request_url, headers={"Accept": accept_header}) + # Intercept http exceptions from server so that the client + # can include the response `detail` sent by the server + try: + r.raise_for_status() + except requests.exceptions.HTTPError as err: + try: + error_detail = r.json().get("detail") + self._log.error(error_detail) + raise HTTPError(error_detail) from err + except ValueError: + self._log.error("Response raised HTTP error but no details provided") + raise HTTPError from err + + self._log.debug(f"Cache set for {request_url}.") + return r + + def _get_mime_type( requested_return_type: type[TModel | TNonModel], ) -> ValidAcceptHeaders: @@ -64,6 +177,10 @@ def __init__( maxsize=cache_size, ttl=cache_lifetime_s ) self._lock = RLock() + self._server: ServerResponse = RealServerResponse(url, self._log) + + def setup_mock(self, converters: ConverterDict) -> None: + self._server = MockServerResponse(converters) @cachedmethod( cache=operator.attrgetter("_cache"), lock=operator.attrgetter("_lock") @@ -73,7 +190,7 @@ def _cached_get( endpoint: str, accept_header: ValidAcceptHeaders, file_path: Path, - ) -> Response: + ) -> ResponseType: """ Get data from the config server and cache it. @@ -87,28 +204,15 @@ def _cached_get( """ request_url = self._url + endpoint + (f"/{file_path}") - r = requests.get(request_url, headers={"Accept": accept_header}) - # Intercept http exceptions from server so that the client - # can include the response `detail` sent by the server - try: - r.raise_for_status() - except requests.exceptions.HTTPError as err: - try: - error_detail = r.json().get("detail") - self._log.error(error_detail) - raise HTTPError(error_detail) from err - except ValueError: - self._log.error("Response raised HTTP error but no details provided") - raise HTTPError from err - + r = self._server.get_response(endpoint, accept_header, file_path) self._log.debug(f"Cache set for {request_url}.") return r def _get( self, endpoint: str, - accept_header: ValidAcceptHeaders, file_path: Path, + accept_header: ValidAcceptHeaders, reset_cached_result: bool = False, ): """ @@ -116,6 +220,7 @@ def _get( the content-type response header to format the return value. If data parsing fails, return the response contents in bytes """ + cache_key = (endpoint, accept_header, file_path) if reset_cached_result: with self._lock: @@ -201,12 +306,6 @@ def get_file_contents( file_path = Path(file_path) if force_parser: - LOGGER.warning( - "The force_parser argument should only be used for testing or " - "as a temporary measure. Add your file and parser to the " - "FILE_TO_CONVERTER_MAP. See " - "https://github.com/DiamondLightSource/daq-config-server/blob/main/docs/how-to/config-server-guide.md#file-converters" - ) # force accept header to string so conversion is done client side accept_header = _get_mime_type(str) else: @@ -214,8 +313,8 @@ def get_file_contents( result = self._get( ENDPOINTS.CONFIG, - accept_header, - file_path, + accept_header=accept_header, + file_path=file_path, reset_cached_result=reset_cached_result, ) if force_parser: From 0bae8172114e87e8efb31cff191ada5bcff8b09a Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 12:31:44 +0000 Subject: [PATCH 02/29] Update test to use mock config_client --- src/daq_config_server/app/client.py | 57 +++++++++++-------- .../models/test_beamline_parameters.py | 18 ++++-- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index d3e1b642..a6e2881d 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -24,7 +24,7 @@ T = TypeVar("T", str, dict[str, Any], ConfigModel) -ConverterDict = dict[str, Callable[[str], ConfigModel]] +ConverterDict = dict[Path | str, Callable[[str], Any]] class TypeConversionError(Exception): ... @@ -33,11 +33,11 @@ class TypeConversionError(Exception): ... class MockResponse: def __init__( self, - data: Any, + body: str | bytes, content_type: ValidAcceptHeaders, status_code: int = 200, ): - self._data = data + self._body = body self.headers = {"content-type": content_type} self.status_code = status_code @@ -46,21 +46,22 @@ def raise_for_status(self): raise requests.exceptions.HTTPError() def json(self) -> Any: - return self._data + """Match requests.Response: JSON is parsed from text/bytes.""" + if isinstance(self._body, bytes): + return json.loads(self._body.decode()) + return json.loads(self._body) @property def text(self) -> str: - if isinstance(self._data, str): - return self._data - return json.dumps(self._data) + if isinstance(self._body, bytes): + return self._body.decode() + return self._body @property def content(self) -> bytes: - if isinstance(self._data, bytes): - return self._data - if isinstance(self._data, str): - return self._data.encode() - return json.dumps(self._data).encode() + if isinstance(self._body, bytes): + return self._body + return self._body.encode() ResponseType = Response | MockResponse @@ -75,9 +76,12 @@ def get_response( ) -> ResponseType: ... -class MockServerResponse(ServerResponse): +class MockServerResponse: def __init__(self, mock_data_converters: ConverterDict | None = None): - self.mock_data_converters = mock_data_converters or {} + self._mock_data_converters = mock_data_converters or {} + + def _load_file(self, file_path: Path) -> str: + return file_path.read_text() def get_response( self, @@ -85,15 +89,20 @@ def get_response( accept_header: ValidAcceptHeaders, file_path: Path, ) -> MockResponse: - raw = file_path.read_text() - - if accept_header == ValidAcceptHeaders.JSON: - return MockResponse(json.loads(raw), accept_header) - - if accept_header == ValidAcceptHeaders.PLAIN_TEXT: - return MockResponse(raw, accept_header) - - return MockResponse(raw.encode(), accept_header) + raw = self._load_file(file_path) + # Apply optional converter hook + if file_path in self._mock_data_converters: + converted = self._mock_data_converters[file_path](raw) + # If it's a Pydantic model, serialize properly + if isinstance(converted, ConfigModel): + raw = converted.model_dump_json() + + elif isinstance(converted, dict): + raw = json.dumps(converted) + # otherwise assume already string-like + else: + raw = str(converted) + return MockResponse(raw, accept_header) class RealServerResponse(ServerResponse): @@ -179,7 +188,7 @@ def __init__( self._lock = RLock() self._server: ServerResponse = RealServerResponse(url, self._log) - def setup_mock(self, converters: ConverterDict) -> None: + def setup_mock(self, converters: ConverterDict | None = None) -> None: self._server = MockServerResponse(converters) @cachedmethod( diff --git a/tests/unit_tests/models/test_beamline_parameters.py b/tests/unit_tests/models/test_beamline_parameters.py index b433e027..57d94308 100644 --- a/tests/unit_tests/models/test_beamline_parameters.py +++ b/tests/unit_tests/models/test_beamline_parameters.py @@ -3,6 +3,7 @@ import pytest +from daq_config_server import ConfigClient from daq_config_server.models.beamline_parameters import ( _parse_value, beamline_parameters_to_dict, @@ -10,12 +11,21 @@ from tests.constants import TestDataPaths -def test_beamline_parameters_to_dict_gives_expected_result(): - with open(TestDataPaths.TEST_BEAMLINE_PARAMETERS_PATH) as f: - contents = f.read() +@pytest.fixture +def config_client() -> ConfigClient: + client = ConfigClient() + client.setup_mock( + {TestDataPaths.TEST_BEAMLINE_PARAMETERS_PATH: beamline_parameters_to_dict} + ) + return client + + +def test_beamline_parameters_to_dict_gives_expected_result(config_client: ConfigClient): with open(TestDataPaths.EXPECTED_BEAMLINE_PARAMETERS_JSON_PATH) as f: expected = json.load(f) - result = beamline_parameters_to_dict(contents) + result = config_client.get_file_contents( + TestDataPaths.TEST_BEAMLINE_PARAMETERS_PATH, desired_return_type=dict + ) assert result == expected From fe142eab513424ce1c3f4ef6679dfe9425cd63b9 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 12:40:59 +0000 Subject: [PATCH 03/29] Restructure and move response classes to own file --- src/daq_config_server/app/_server_response.py | 129 +++++++++++++++++ src/daq_config_server/app/client.py | 131 ++---------------- 2 files changed, 137 insertions(+), 123 deletions(-) create mode 100644 src/daq_config_server/app/_server_response.py diff --git a/src/daq_config_server/app/_server_response.py b/src/daq_config_server/app/_server_response.py new file mode 100644 index 00000000..76070941 --- /dev/null +++ b/src/daq_config_server/app/_server_response.py @@ -0,0 +1,129 @@ +import json +from collections.abc import Callable +from logging import Logger +from pathlib import Path +from typing import Any, Protocol + +import requests +from requests import Response as RealResponse +from requests.exceptions import HTTPError + +from daq_config_server.models.base_model import ConfigModel + +from ._routes import ValidAcceptHeaders + +ConverterDict = dict[Path, Callable[[str], Any]] + + +class MockResponse: + def __init__( + self, + body: str | bytes, + content_type: ValidAcceptHeaders, + status_code: int = 200, + ): + self._body = body + self.headers = {"content-type": content_type} + self.status_code = status_code + + def raise_for_status(self): + if self.status_code >= 400: + raise requests.exceptions.HTTPError() + + def json(self) -> Any: + """Match requests.Response: JSON is parsed from text/bytes.""" + if isinstance(self._body, bytes): + return json.loads(self._body.decode()) + return json.loads(self._body) + + @property + def text(self) -> str: + if isinstance(self._body, bytes): + return self._body.decode() + return self._body + + @property + def content(self) -> bytes: + if isinstance(self._body, bytes): + return self._body + return self._body.encode() + + +ResponseType = RealResponse | MockResponse + + +class ServerResponse(Protocol): + def get_response( + self, + endpoint: str, + accept_header: ValidAcceptHeaders, + file_path: Path, + ) -> ResponseType: ... + + +class MockServerResponse(ServerResponse): + def __init__(self, mock_data_converters: ConverterDict | None = None): + self._mock_data_converters = mock_data_converters or {} + + def get_response( + self, + endpoint: str, + accept_header: ValidAcceptHeaders, + file_path: Path, + ) -> MockResponse: + raw = file_path.read_text() + # Apply optional converter hook + if file_path in self._mock_data_converters: + converted = self._mock_data_converters[file_path](raw) + # If it's a Pydantic model, serialize properly + if isinstance(converted, ConfigModel): + raw = converted.model_dump_json() + + elif isinstance(converted, dict): + raw = json.dumps(converted) + # otherwise assume already string-like + else: + raw = str(converted) + return MockResponse(raw, accept_header) + + +class RealServerResponse(ServerResponse): + def __init__(self, url: str, log: Logger): + self._url = url + self._log = log + + def get_response( + self, + endpoint: str, + accept_header: ValidAcceptHeaders, + file_path: Path, + ) -> ResponseType: + """ + Get data from the config server and cache it. + + Args: + endpoint: API endpoint. + accept_header: Accept header MIME type + file_path: absolute path to the file which will be read + + Returns: + The response data. + """ + + request_url = self._url + endpoint + (f"/{file_path}") + r = requests.get(request_url, headers={"Accept": accept_header}) + # Intercept http exceptions from server so that the client + # can include the response `detail` sent by the server + try: + r.raise_for_status() + except requests.exceptions.HTTPError as err: + try: + error_detail = r.json().get("detail") + self._log.error(error_detail) + raise HTTPError(error_detail) from err + except ValueError: + self._log.error("Response raised HTTP error but no details provided") + raise HTTPError from err + + self._log.debug(f"Cache set for {request_url}.") + return r diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index a6e2881d..7d87f565 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -1,21 +1,25 @@ -import json import logging import operator from collections.abc import Callable from logging import Logger, getLogger from pathlib import Path from threading import RLock -from typing import Any, Protocol, TypeVar, get_origin, overload +from typing import Any, TypeVar, get_origin, overload -import requests from cachetools import TTLCache, cachedmethod from pydantic import TypeAdapter from requests import Response -from requests.exceptions import HTTPError from daq_config_server.models.base_model import ConfigModel from ._routes import ENDPOINTS, ValidAcceptHeaders +from ._server_response import ( + ConverterDict, + MockServerResponse, + RealServerResponse, + ResponseType, + ServerResponse, +) LOGGER = logging.getLogger(__name__) @@ -24,129 +28,10 @@ T = TypeVar("T", str, dict[str, Any], ConfigModel) -ConverterDict = dict[Path | str, Callable[[str], Any]] - class TypeConversionError(Exception): ... -class MockResponse: - def __init__( - self, - body: str | bytes, - content_type: ValidAcceptHeaders, - status_code: int = 200, - ): - self._body = body - self.headers = {"content-type": content_type} - self.status_code = status_code - - def raise_for_status(self): - if self.status_code >= 400: - raise requests.exceptions.HTTPError() - - def json(self) -> Any: - """Match requests.Response: JSON is parsed from text/bytes.""" - if isinstance(self._body, bytes): - return json.loads(self._body.decode()) - return json.loads(self._body) - - @property - def text(self) -> str: - if isinstance(self._body, bytes): - return self._body.decode() - return self._body - - @property - def content(self) -> bytes: - if isinstance(self._body, bytes): - return self._body - return self._body.encode() - - -ResponseType = Response | MockResponse - - -class ServerResponse(Protocol): - def get_response( - self, - endpoint: str, - accept_header: ValidAcceptHeaders, - file_path: Path, - ) -> ResponseType: ... - - -class MockServerResponse: - def __init__(self, mock_data_converters: ConverterDict | None = None): - self._mock_data_converters = mock_data_converters or {} - - def _load_file(self, file_path: Path) -> str: - return file_path.read_text() - - def get_response( - self, - endpoint: str, - accept_header: ValidAcceptHeaders, - file_path: Path, - ) -> MockResponse: - raw = self._load_file(file_path) - # Apply optional converter hook - if file_path in self._mock_data_converters: - converted = self._mock_data_converters[file_path](raw) - # If it's a Pydantic model, serialize properly - if isinstance(converted, ConfigModel): - raw = converted.model_dump_json() - - elif isinstance(converted, dict): - raw = json.dumps(converted) - # otherwise assume already string-like - else: - raw = str(converted) - return MockResponse(raw, accept_header) - - -class RealServerResponse(ServerResponse): - def __init__(self, url: str, log: Logger): - self._url = url - self._log = log - - def get_response( - self, - endpoint: str, - accept_header: ValidAcceptHeaders, - file_path: Path, - ) -> ResponseType: - """ - Get data from the config server and cache it. - - Args: - endpoint: API endpoint. - accept_header: Accept header MIME type - file_path: absolute path to the file which will be read - - Returns: - The response data. - """ - - request_url = self._url + endpoint + (f"/{file_path}") - r = requests.get(request_url, headers={"Accept": accept_header}) - # Intercept http exceptions from server so that the client - # can include the response `detail` sent by the server - try: - r.raise_for_status() - except requests.exceptions.HTTPError as err: - try: - error_detail = r.json().get("detail") - self._log.error(error_detail) - raise HTTPError(error_detail) from err - except ValueError: - self._log.error("Response raised HTTP error but no details provided") - raise HTTPError from err - - self._log.debug(f"Cache set for {request_url}.") - return r - - def _get_mime_type( requested_return_type: type[TModel | TNonModel], ) -> ValidAcceptHeaders: From 8c7465f69218f917d9e1d70013d9253294ff69f4 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 12:42:46 +0000 Subject: [PATCH 04/29] Remove duplicate log message --- src/daq_config_server/app/_server_response.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/daq_config_server/app/_server_response.py b/src/daq_config_server/app/_server_response.py index 76070941..b6656c2b 100644 --- a/src/daq_config_server/app/_server_response.py +++ b/src/daq_config_server/app/_server_response.py @@ -124,6 +124,4 @@ def get_response( except ValueError: self._log.error("Response raised HTTP error but no details provided") raise HTTPError from err - - self._log.debug(f"Cache set for {request_url}.") return r From fdc6347939c0f468700d0946727be6a2acd6cfa4 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 12:49:12 +0000 Subject: [PATCH 05/29] Update type signature --- src/daq_config_server/app/_server_response.py | 21 ++++++------------- src/daq_config_server/app/client.py | 5 +---- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/daq_config_server/app/_server_response.py b/src/daq_config_server/app/_server_response.py index b6656c2b..9c2ebaa1 100644 --- a/src/daq_config_server/app/_server_response.py +++ b/src/daq_config_server/app/_server_response.py @@ -22,12 +22,12 @@ def __init__( content_type: ValidAcceptHeaders, status_code: int = 200, ): - self._body = body self.headers = {"content-type": content_type} - self.status_code = status_code + self._body = body + self._status_code = status_code def raise_for_status(self): - if self.status_code >= 400: + if self._status_code >= 400: raise requests.exceptions.HTTPError() def json(self) -> Any: @@ -54,10 +54,7 @@ def content(self) -> bytes: class ServerResponse(Protocol): def get_response( - self, - endpoint: str, - accept_header: ValidAcceptHeaders, - file_path: Path, + self, endpoint: str, accept_header: ValidAcceptHeaders, file_path: Path ) -> ResponseType: ... @@ -66,10 +63,7 @@ def __init__(self, mock_data_converters: ConverterDict | None = None): self._mock_data_converters = mock_data_converters or {} def get_response( - self, - endpoint: str, - accept_header: ValidAcceptHeaders, - file_path: Path, + self, endpoint: str, accept_header: ValidAcceptHeaders, file_path: Path ) -> MockResponse: raw = file_path.read_text() # Apply optional converter hook @@ -93,10 +87,7 @@ def __init__(self, url: str, log: Logger): self._log = log def get_response( - self, - endpoint: str, - accept_header: ValidAcceptHeaders, - file_path: Path, + self, endpoint: str, accept_header: ValidAcceptHeaders, file_path: Path ) -> ResponseType: """ Get data from the config server and cache it. diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index 7d87f565..b8495cba 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -80,10 +80,7 @@ def setup_mock(self, converters: ConverterDict | None = None) -> None: cache=operator.attrgetter("_cache"), lock=operator.attrgetter("_lock") ) def _cached_get( - self, - endpoint: str, - accept_header: ValidAcceptHeaders, - file_path: Path, + self, endpoint: str, accept_header: ValidAcceptHeaders, file_path: Path ) -> ResponseType: """ Get data from the config server and cache it. From c0aca4cbf49db6f8c59006b5910b557eb39f2dd9 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 13:01:49 +0000 Subject: [PATCH 06/29] Add additional tests --- src/daq_config_server/app/client.py | 2 -- tests/unit_tests/app/test_client.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index b8495cba..2e35c6c7 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -119,7 +119,6 @@ def _get( del self._cache[cache_key] r = self._cached_get(*cache_key) - content_type = r.headers["content-type"].split(";")[0].strip() if content_type != accept_header: @@ -127,7 +126,6 @@ def _get( f"Server failed to parse the file as requested. Requested " f"{accept_header} but response came as content-type {content_type}" ) - try: match content_type: case ValidAcceptHeaders.JSON: diff --git a/tests/unit_tests/app/test_client.py b/tests/unit_tests/app/test_client.py index 7bf9dd3e..9e265c34 100644 --- a/tests/unit_tests/app/test_client.py +++ b/tests/unit_tests/app/test_client.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from typing import Any from unittest.mock import MagicMock, patch @@ -252,3 +253,53 @@ def test_reset_cache( str, ) assert result != new_result + + +def test_mock_config_client_get_file_contents_as_dict_gives_expected_result( + tmp_path: Path, +): + file = tmp_path / "beamline.json" + + expected_data = {"x": 1, "y": "test"} + file.write_text(json.dumps(expected_data)) + + client = ConfigClient() + client.setup_mock() + + result = client.get_file_contents(file, desired_return_type=dict) + assert result == expected_data + + +def test_mock_config_client_get_file_contents_as_str_gives_expected_result( + tmp_path: Path, +): + file = tmp_path / "beamline.json" + + expected_data = '{"x": 1, "y": "test"}' + file.write_text(expected_data) + + client = ConfigClient() + client.setup_mock() + + result = client.get_file_contents(file) + assert result == expected_data + + +class MyModel(ConfigModel): + x: float = 1.5 + y: str = "test" + z: list[int] = [1, 4, 5] + + +def test_mock_config_client_get_file_contents_as_config_model_gives_expected_result( + tmp_path: Path, +): + file = tmp_path / "beamline.json" + expected_data = MyModel().model_dump_json() + file.write_text(expected_data) + + client = ConfigClient() + client.setup_mock() + + result = client.get_file_contents(file) + assert result == expected_data From e553b8586df0b24295c9213f63c0cbfac9f0b93b Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 13:06:26 +0000 Subject: [PATCH 07/29] Fix tests from import change --- tests/unit_tests/app/test_client.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/unit_tests/app/test_client.py b/tests/unit_tests/app/test_client.py index 9e265c34..917a05c0 100644 --- a/tests/unit_tests/app/test_client.py +++ b/tests/unit_tests/app/test_client.py @@ -27,6 +27,8 @@ ) from daq_config_server.testing import make_test_response +REQUEST_PATCH = "daq_config_server.app._server_response.requests.get" + test_path = Path("test") @@ -35,7 +37,7 @@ def client() -> ConfigClient: return ConfigClient("url") -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_get_file_contents_default_header( mock_request: MagicMock, client: ConfigClient ): @@ -51,7 +53,7 @@ def test_config_client_get_file_contents_default_header( ) -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_get_file_contents_with_bytes( mock_request: MagicMock, client: ConfigClient ): @@ -65,7 +67,7 @@ def test_config_client_get_file_contents_with_bytes( ) -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_get_file_contents_gives_exception_on_invalid_json( mock_request: MagicMock, client: ConfigClient, @@ -79,7 +81,7 @@ def test_config_client_get_file_contents_gives_exception_on_invalid_json( client.get_file_contents(test_path, desired_return_type=dict[Any, Any]) -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_get_file_contents_caching( mock_request: MagicMock, client: ConfigClient, @@ -95,7 +97,7 @@ def test_config_client_get_file_contents_caching( assert client.get_file_contents(test_path, reset_cached_result=False) == "2nd_read" -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_bad_responses_no_details_raises_error( mock_request: MagicMock, client: ConfigClient ): @@ -111,7 +113,7 @@ def test_config_client_bad_responses_no_details_raises_error( ) -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_bad_responses_with_details_raises_error( mock_request: MagicMock, client: ConfigClient ): @@ -131,7 +133,7 @@ def test_config_client_bad_responses_with_details_raises_error( client._log.error.assert_called_once_with(detail) -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_get_file_contents_with_untyped_dict( mock_request: MagicMock, client: ConfigClient ): @@ -161,7 +163,7 @@ def test_get_mime_type(input: type[TModel | TNonModel], expected: ValidAcceptHea assert _get_mime_type(input) == expected -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_get_file_contents_with_force_parser_requests_str_from_server_and_converts( # noqa: E501 mock_request: MagicMock, client: ConfigClient, @@ -190,7 +192,7 @@ def test_config_client_get_file_contents_with_force_parser_requests_str_from_ser (BeamlinePitchLookupTable, pydantic.ValidationError), ], ) -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_get_file_contents_with_force_parser_still_validates_desired_return_type( # noqa: E501 mock_request: MagicMock, client: ConfigClient, @@ -217,7 +219,7 @@ def test_config_client_get_file_contents_with_force_parser_still_validates_desir assert result == expected_result -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_config_client_get_file_contents_with_bad_force_parser_errors( mock_request: MagicMock, client: ConfigClient ): @@ -232,7 +234,7 @@ def test_config_client_get_file_contents_with_bad_force_parser_errors( ) -@patch("daq_config_server.app.client.requests.get") +@patch(REQUEST_PATCH) def test_reset_cache( mock_request: MagicMock, ): From d8f8546383321917a33d4fe0ee9e08b005c97920 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 13:23:03 +0000 Subject: [PATCH 08/29] Add additional test, rename mock func to configure_mock --- src/daq_config_server/app/__init__.py | 3 +- src/daq_config_server/app/_server_response.py | 5 ++- src/daq_config_server/app/client.py | 8 ++-- tests/unit_tests/app/test_client.py | 45 ++++++++++++------- .../models/test_beamline_parameters.py | 2 +- 5 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/daq_config_server/app/__init__.py b/src/daq_config_server/app/__init__.py index d6cd5f69..8d823765 100644 --- a/src/daq_config_server/app/__init__.py +++ b/src/daq_config_server/app/__init__.py @@ -1,3 +1,4 @@ +from ._server_response import MockPathToConverterDict from .api import main -__all__ = ["main"] +__all__ = ["MockPathToConverterDict", "main"] diff --git a/src/daq_config_server/app/_server_response.py b/src/daq_config_server/app/_server_response.py index 9c2ebaa1..0b1859be 100644 --- a/src/daq_config_server/app/_server_response.py +++ b/src/daq_config_server/app/_server_response.py @@ -12,7 +12,8 @@ from ._routes import ValidAcceptHeaders -ConverterDict = dict[Path, Callable[[str], Any]] +NonModel = str | bytes | dict[str, Any] +MockPathToConverterDict = dict[Path, Callable[[str], ConfigModel | NonModel]] class MockResponse: @@ -59,7 +60,7 @@ def get_response( class MockServerResponse(ServerResponse): - def __init__(self, mock_data_converters: ConverterDict | None = None): + def __init__(self, mock_data_converters: MockPathToConverterDict | None = None): self._mock_data_converters = mock_data_converters or {} def get_response( diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index 2e35c6c7..be107076 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -14,7 +14,7 @@ from ._routes import ENDPOINTS, ValidAcceptHeaders from ._server_response import ( - ConverterDict, + MockPathToConverterDict, MockServerResponse, RealServerResponse, ResponseType, @@ -73,7 +73,7 @@ def __init__( self._lock = RLock() self._server: ServerResponse = RealServerResponse(url, self._log) - def setup_mock(self, converters: ConverterDict | None = None) -> None: + def configure_mock(self, converters: MockPathToConverterDict | None = None) -> None: self._server = MockServerResponse(converters) @cachedmethod( @@ -186,9 +186,7 @@ def get_file_contents( force_parser: Optionally provide a function to convert the contents of a config file to the desired return type. This overides whatever converter is specified for that file in the FILE_TO_CONVERTER_MAP, and can be used - if the config file isn't in the FILE_TO_CONVERTER_MAP at all. This - should only be used for testing or when waiting on a release that will - add the file to the FILE_TO_CONVERTER_MAP. + if the config file isn't in the FILE_TO_CONVERTER_MAP at all. Returns: The file contents, in the format specified. """ diff --git a/tests/unit_tests/app/test_client.py b/tests/unit_tests/app/test_client.py index 917a05c0..1e144cc9 100644 --- a/tests/unit_tests/app/test_client.py +++ b/tests/unit_tests/app/test_client.py @@ -69,8 +69,7 @@ def test_config_client_get_file_contents_with_bytes( @patch(REQUEST_PATCH) def test_config_client_get_file_contents_gives_exception_on_invalid_json( - mock_request: MagicMock, - client: ConfigClient, + mock_request: MagicMock, client: ConfigClient ): content_type = ValidAcceptHeaders.JSON bad_json = "bad_dict}" @@ -83,8 +82,7 @@ def test_config_client_get_file_contents_gives_exception_on_invalid_json( @patch(REQUEST_PATCH) def test_config_client_get_file_contents_caching( - mock_request: MagicMock, - client: ConfigClient, + mock_request: MagicMock, client: ConfigClient ): """Test reset_cached_result=False and reset_cached_result=True.""" mock_request.side_effect = [ @@ -165,8 +163,7 @@ def test_get_mime_type(input: type[TModel | TNonModel], expected: ValidAcceptHea @patch(REQUEST_PATCH) def test_config_client_get_file_contents_with_force_parser_requests_str_from_server_and_converts( # noqa: E501 - mock_request: MagicMock, - client: ConfigClient, + mock_request: MagicMock, client: ConfigClient ): mock_config = "mock_config" mock_request.return_value = make_test_response(mock_config) @@ -235,19 +232,16 @@ def test_config_client_get_file_contents_with_bad_force_parser_errors( @patch(REQUEST_PATCH) -def test_reset_cache( - mock_request: MagicMock, -): +def test_reset_cache(mock_request: MagicMock): mock_config = "Units eV mm\n5700 5.4606\n#24500 7.2\n" mock_request.return_value = make_test_response(mock_config) server = ConfigClient("url") - result = server.get_file_contents( - test_path, - str, - ) + result = server.get_file_contents(test_path, str) + assert server._cache.currsize == 1 server.reset_cache() assert server._cache.currsize == 0 + new_mock_config = "Units eV mm\n6800 5.4606\n#24500 7.2\n" mock_request.return_value = make_test_response(new_mock_config) new_result = server.get_file_contents( @@ -266,7 +260,7 @@ def test_mock_config_client_get_file_contents_as_dict_gives_expected_result( file.write_text(json.dumps(expected_data)) client = ConfigClient() - client.setup_mock() + client.configure_mock() result = client.get_file_contents(file, desired_return_type=dict) assert result == expected_data @@ -281,7 +275,7 @@ def test_mock_config_client_get_file_contents_as_str_gives_expected_result( file.write_text(expected_data) client = ConfigClient() - client.setup_mock() + client.configure_mock() result = client.get_file_contents(file) assert result == expected_data @@ -301,7 +295,26 @@ def test_mock_config_client_get_file_contents_as_config_model_gives_expected_res file.write_text(expected_data) client = ConfigClient() - client.setup_mock() + client.configure_mock() result = client.get_file_contents(file) assert result == expected_data + + +def test_mock_config_client_converter_table_to_json(tmp_path: Path): + file = tmp_path / "beamline.txt" + + # "table" format (header + row) + file.write_text("x|y\n1|test") + + def table_to_dict(contents: str) -> dict[str, Any]: + lines = contents.strip().splitlines() + headers = lines[0].split("|") + values = lines[1].split("|") + return dict(zip(headers, values, strict=True)) + + client = ConfigClient() + client.configure_mock({file: table_to_dict}) + + result = client.get_file_contents(file, desired_return_type=dict) + assert result == {"x": "1", "y": "test"} diff --git a/tests/unit_tests/models/test_beamline_parameters.py b/tests/unit_tests/models/test_beamline_parameters.py index 57d94308..d7cd029b 100644 --- a/tests/unit_tests/models/test_beamline_parameters.py +++ b/tests/unit_tests/models/test_beamline_parameters.py @@ -14,7 +14,7 @@ @pytest.fixture def config_client() -> ConfigClient: client = ConfigClient() - client.setup_mock( + client.configure_mock( {TestDataPaths.TEST_BEAMLINE_PARAMETERS_PATH: beamline_parameters_to_dict} ) return client From d5fdf309d77be0fc3ba0f6f26e1ae50161c8b494 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 13:29:45 +0000 Subject: [PATCH 09/29] Add doc strings --- src/daq_config_server/app/_server_response.py | 27 +++++++++++++++++++ src/daq_config_server/app/client.py | 26 +++++++++++++----- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/daq_config_server/app/_server_response.py b/src/daq_config_server/app/_server_response.py index 0b1859be..e665c5b5 100644 --- a/src/daq_config_server/app/_server_response.py +++ b/src/daq_config_server/app/_server_response.py @@ -17,6 +17,16 @@ class MockResponse: + """Lightweight stand-in for requests.Response used in unit tests. + + This class emulates the minimal interface of a real HTTP response + required by ConfigClient, without performing any network operations. + + This allows tests to simulate server responses at different encoding + layers (JSON, plain text, or raw bytes) while keeping behaviour + consistent with real requests.Response objects. + """ + def __init__( self, body: str | bytes, @@ -54,12 +64,23 @@ def content(self) -> bytes: class ServerResponse(Protocol): + """Interface for retrieving configuration data from either a real server or a local + mock implementation. + """ + def get_response( self, endpoint: str, accept_header: ValidAcceptHeaders, file_path: Path ) -> ResponseType: ... class MockServerResponse(ServerResponse): + """Mock implementation of ServerResponse used for unit testing. + + This class simulates a config server by reading local files instead of + performing HTTP requests. It supports optional per-file converter functions + that can transform raw file contents before they are returned. + """ + def __init__(self, mock_data_converters: MockPathToConverterDict | None = None): self._mock_data_converters = mock_data_converters or {} @@ -83,6 +104,12 @@ def get_response( class RealServerResponse(ServerResponse): + """Real HTTP implementation of ServerResponse used in production. + + This class communicates with a remote configuration server via HTTP + requests and retrieves file contents from a deployed service. + """ + def __init__(self, url: str, log: Logger): self._url = url self._log = log diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index be107076..c8abc147 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -48,8 +48,13 @@ def _get_mime_type( class ConfigClient: - """Client to communicate with a deployed config service with a configurable cache - and logger""" + """Client for retrieving configuration data from a config service with + support for caching, flexible return types, and pluggable backends. + + This client abstracts access to configuration files stored either on: + - a remote configuration server (production mode), or + - a local mock filesystem (test mode) + """ def __init__( self, @@ -58,12 +63,19 @@ def __init__( cache_size: int = 10, cache_lifetime_s: int = 3600, ) -> None: - """ + """Switch the client into mock mode using a local filesystem backend. + + This replaces the real HTTP server implementation with a mock + server that reads configuration data directly from local files. + + Optional converters can be provided to simulate server-side parsing + or transformation logic on a per-file basis. + Args: - url: Base URL of the config server. Defaults to central service. - log: Optional logger instance. - cache_size: Size of the cache (maximum number of items can be stored). - cache_lifetime_s: Lifetime of the cache (in seconds). + converters: + Optional mapping of file paths to converter functions. + Each function receives raw file contents as a string and + returns a transformed object (e.g. dict, ConfigModel, etc.). """ self._url = url.rstrip("/") self._log = log if log else getLogger("daq_config_server.client") From 0331de583f6146553c76f446ae9f71115d9d1692 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 13:44:35 +0000 Subject: [PATCH 10/29] Update docs --- docs/how-to/config-server-guide.md | 68 +++++++++++++++++++ .../reference/current_and_planned_features.md | 1 + 2 files changed, 69 insertions(+) diff --git a/docs/how-to/config-server-guide.md b/docs/how-to/config-server-guide.md index d6985f8a..4e620018 100644 --- a/docs/how-to/config-server-guide.md +++ b/docs/how-to/config-server-guide.md @@ -64,3 +64,71 @@ whitelist: # Reading sensitive information If you need to read a file which contains sensitive information, or `dls-dasc` doesn't have the permissions to read your file, you should encrypt this file as a [sealed secret](https://github.com/bitnami-labs/sealed-secrets) on your beamline cluster, and mount this in your BlueAPI service. + +# Mocking the Config Client (for tests and offline development) + +The ConfigClient can be configured to run in a fully offline mode for unit tests and local development. In this mode, no HTTP requests are made. Instead, file reads are intercepted and optionally transformed using mock converters. + +This allows you to simulate server-side conversion behaviour (e.g. JSON → dict, table → JSON, or custom Pydantic models) without requiring a running config service. + +## Enabling mock mode +```python +config_client = ConfigClient() +config_client.configure_mock() +``` +Once enabled, all file access goes through the mock layer instead of the real server. + +### Mock converters + +Mock converters allow you to simulate the server-side converter_map behaviour locally. + +A converter is a function with the signature: + +```python +Callable[[str], ConfigModel | str | bytes | dict[str, Any]] +``` + +You register converters keyed by Path: + +```python +from pathlib import Path + +config_client.setup_mock( + { + Path("/tmp/my_file.txt"): my_converter_function + } +) +``` +Example: converting a table to JSON + +You can simulate a file containing a table-like format (e.g. whitespace-separated columns) and convert it into JSON. + +### Example file content +``` +key value +x 1 +y test +``` +### Converter function +```python +import json + +def table_to_json(contents: str) -> str: + lines = contents.strip().splitlines() + headers = lines[0].split() + + result = {} + for line in lines[1:]: + key, value = line.split() + # attempt numeric conversion + if value.isdigit(): + value = int(value) + result[key] = value + + return json.dumps(result) +``` +### Test setup +```python +file_path = Path("/path/to/data.txt") +client.setup_mock({file_path: table_to_json}) +``` diff --git a/docs/reference/current_and_planned_features.md b/docs/reference/current_and_planned_features.md index b2aab4a5..2d4364ce 100644 --- a/docs/reference/current_and_planned_features.md +++ b/docs/reference/current_and_planned_features.md @@ -7,6 +7,7 @@ - Provide a client module for users to easily communicate with the server, with caching. - Have this service hosted on diamond's central argus cluster - with url `https://daq-config-server.diamond.ac.uk` - Provide server-side json and pydantic model formatting for commonly used configuration files - eg `beamline_parameters.txt` should be returned as a dictionary or pydantic model. +- Supports mock behaviour so in test environments that rely on ConfigClient as a dependency can still use a local file system for tests. ## Future features From f5278e72fb16d763ceea31652704dc555f672a7f Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 13:59:46 +0000 Subject: [PATCH 11/29] Up code coverage --- src/daq_config_server/app/_server_response.py | 6 ------ tests/unit_tests/app/test_server_response.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 tests/unit_tests/app/test_server_response.py diff --git a/src/daq_config_server/app/_server_response.py b/src/daq_config_server/app/_server_response.py index e665c5b5..01ff6165 100644 --- a/src/daq_config_server/app/_server_response.py +++ b/src/daq_config_server/app/_server_response.py @@ -31,15 +31,9 @@ def __init__( self, body: str | bytes, content_type: ValidAcceptHeaders, - status_code: int = 200, ): self.headers = {"content-type": content_type} self._body = body - self._status_code = status_code - - def raise_for_status(self): - if self._status_code >= 400: - raise requests.exceptions.HTTPError() def json(self) -> Any: """Match requests.Response: JSON is parsed from text/bytes.""" diff --git a/tests/unit_tests/app/test_server_response.py b/tests/unit_tests/app/test_server_response.py new file mode 100644 index 00000000..2e0c4324 --- /dev/null +++ b/tests/unit_tests/app/test_server_response.py @@ -0,0 +1,13 @@ +import json + +from daq_config_server.app._routes import ValidAcceptHeaders +from daq_config_server.app._server_response import MockResponse + + +def test_mock_response_using_bytes(): + data = {"x": 1, "y": "test"} + body = json.dumps(data).encode() + resp = MockResponse(body=body, content_type=ValidAcceptHeaders.JSON) + assert resp.json() == data + assert resp.text == json.dumps(data) + assert resp.content == body From eff67be83aa42be2197d81d32606dd47dce10733 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 14:14:38 +0000 Subject: [PATCH 12/29] Correct RealServerResponse url --- src/daq_config_server/app/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index c8abc147..ba87d780 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -83,7 +83,7 @@ def __init__( maxsize=cache_size, ttl=cache_lifetime_s ) self._lock = RLock() - self._server: ServerResponse = RealServerResponse(url, self._log) + self._server: ServerResponse = RealServerResponse(self._url, self._log) def configure_mock(self, converters: MockPathToConverterDict | None = None) -> None: self._server = MockServerResponse(converters) From f6e1b4391a80de5c14c23e5b0d869cfccdeb4450 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 14:18:00 +0000 Subject: [PATCH 13/29] Imrpove logging --- src/daq_config_server/app/client.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index ba87d780..996e72e6 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -1,4 +1,3 @@ -import logging import operator from collections.abc import Callable from logging import Logger, getLogger @@ -21,13 +20,9 @@ ServerResponse, ) -LOGGER = logging.getLogger(__name__) - TModel = TypeVar("TModel", bound=ConfigModel) TNonModel = TypeVar("TNonModel", str, bytes, dict[str, Any]) -T = TypeVar("T", str, dict[str, Any], ConfigModel) - class TypeConversionError(Exception): ... @@ -78,7 +73,7 @@ def __init__( returns a transformed object (e.g. dict, ConfigModel, etc.). """ self._url = url.rstrip("/") - self._log = log if log else getLogger("daq_config_server.client") + self._log = log or getLogger("daq_config_server.client") self._cache: TTLCache[tuple[str, str, Path], Response] = TTLCache( maxsize=cache_size, ttl=cache_lifetime_s ) From 9ed6504150e4f73d52aabff55a7a36bd9d3e8800 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 15:14:11 +0000 Subject: [PATCH 14/29] Update docs in wrong place. Update configure_mock doc string --- src/daq_config_server/app/client.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index 996e72e6..f3509d34 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -58,6 +58,23 @@ def __init__( cache_size: int = 10, cache_lifetime_s: int = 3600, ) -> None: + """ + Args: + url: Base URL of the config server. Defaults to central service. + log: Optional logger instance. + cache_size: Size of the cache (maximum number of items can be stored). + cache_lifetime_s: Lifetime of the cache (in seconds). + """ + + self._url = url.rstrip("/") + self._log = log or getLogger("daq_config_server.client") + self._cache: TTLCache[tuple[str, str, Path], Response] = TTLCache( + maxsize=cache_size, ttl=cache_lifetime_s + ) + self._lock = RLock() + self._server: ServerResponse = RealServerResponse(self._url, self._log) + + def configure_mock(self, converters: MockPathToConverterDict | None = None) -> None: """Switch the client into mock mode using a local filesystem backend. This replaces the real HTTP server implementation with a mock @@ -72,15 +89,6 @@ def __init__( Each function receives raw file contents as a string and returns a transformed object (e.g. dict, ConfigModel, etc.). """ - self._url = url.rstrip("/") - self._log = log or getLogger("daq_config_server.client") - self._cache: TTLCache[tuple[str, str, Path], Response] = TTLCache( - maxsize=cache_size, ttl=cache_lifetime_s - ) - self._lock = RLock() - self._server: ServerResponse = RealServerResponse(self._url, self._log) - - def configure_mock(self, converters: MockPathToConverterDict | None = None) -> None: self._server = MockServerResponse(converters) @cachedmethod( From 320a17fb4c7970861211c6cb84c5a65e716dc99c Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Fri, 26 Jun 2026 15:24:27 +0000 Subject: [PATCH 15/29] Attempt to make docs happy --- src/daq_config_server/app/_server_response.py | 2 +- src/daq_config_server/app/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/daq_config_server/app/_server_response.py b/src/daq_config_server/app/_server_response.py index 01ff6165..6cc9125c 100644 --- a/src/daq_config_server/app/_server_response.py +++ b/src/daq_config_server/app/_server_response.py @@ -8,7 +8,7 @@ from requests import Response as RealResponse from requests.exceptions import HTTPError -from daq_config_server.models.base_model import ConfigModel +from daq_config_server.models import ConfigModel from ._routes import ValidAcceptHeaders diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/app/client.py index f3509d34..56d5e1d1 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/app/client.py @@ -9,7 +9,7 @@ from pydantic import TypeAdapter from requests import Response -from daq_config_server.models.base_model import ConfigModel +from daq_config_server.models import ConfigModel from ._routes import ENDPOINTS, ValidAcceptHeaders from ._server_response import ( From c6fb99d05f89be7e3c79a39cd575ddd861359f7f Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Mon, 29 Jun 2026 10:02:34 +0000 Subject: [PATCH 16/29] Move ConfigClient to client module --- README.md | 2 +- docs/how-to/config-server-guide.md | 2 +- src/daq_config_server/__init__.py | 8 ++++++-- src/daq_config_server/app/__init__.py | 3 +-- src/daq_config_server/app/_routes.py | 15 +-------------- src/daq_config_server/app/endpoints.py | 14 ++++++++++++++ src/daq_config_server/client/__init__.py | 3 +++ .../{app/client.py => client/_client.py} | 4 ++-- .../{app => client}/_server_response.py | 5 ++--- tests/system_tests/test_client.py | 2 +- tests/unit_tests/client/__init__.py | 0 tests/unit_tests/{app => client}/test_client.py | 4 ++-- .../{app => client}/test_server_response.py | 2 +- .../unit_tests/models/test_beamline_parameters.py | 2 +- 14 files changed, 36 insertions(+), 30 deletions(-) create mode 100644 src/daq_config_server/app/endpoints.py create mode 100644 src/daq_config_server/client/__init__.py rename src/daq_config_server/{app/client.py => client/_client.py} (98%) rename src/daq_config_server/{app => client}/_server_response.py (97%) create mode 100644 tests/unit_tests/client/__init__.py rename tests/unit_tests/{app => client}/test_client.py (98%) rename tests/unit_tests/{app => client}/test_server_response.py (84%) diff --git a/README.md b/README.md index c6426658..e518b69f 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ A service to read files on Diamond's filesystem from a BlueAPI container. Curren Here is a minimal example to read a file from the centrally hosted service after installing this package ```python -from daq_config_server import ConfigClient +from daq_config_server.client import ConfigClient config_client = ConfigClient("https://daq-config.diamond.ac.uk") diff --git a/docs/how-to/config-server-guide.md b/docs/how-to/config-server-guide.md index 4e620018..66dc8659 100644 --- a/docs/how-to/config-server-guide.md +++ b/docs/how-to/config-server-guide.md @@ -6,7 +6,7 @@ The server is centrally hosted on argus and is accessible anywhere within the Di This library provides a python client to easily make requests from Bluesky code. The client can use caching to prevent needlessly making time-consuming requests on data which won't have changed. You can choose the maximum number of items it can hold as well as the lifetime of an item upon instantiation. ```python -from daq_config_server import ConfigClient +from daq_config_server.client import ConfigClient config_client = ConfigClient("https://daq-config.diamond.ac.uk", cache_size = 10, cache_lifetime_s = 3600) ``` diff --git a/src/daq_config_server/__init__.py b/src/daq_config_server/__init__.py index 9a1b7a18..6878ddbf 100644 --- a/src/daq_config_server/__init__.py +++ b/src/daq_config_server/__init__.py @@ -7,6 +7,10 @@ """ from ._version import __version__ -from .app.client import ConfigClient -__all__ = ["__version__", "ConfigClient"] +# from .client._client import ConfigClient + +__all__ = [ + "__version__", + # "ConfigClient" +] diff --git a/src/daq_config_server/app/__init__.py b/src/daq_config_server/app/__init__.py index 8d823765..d6cd5f69 100644 --- a/src/daq_config_server/app/__init__.py +++ b/src/daq_config_server/app/__init__.py @@ -1,4 +1,3 @@ -from ._server_response import MockPathToConverterDict from .api import main -__all__ = ["MockPathToConverterDict", "main"] +__all__ = ["main"] diff --git a/src/daq_config_server/app/_routes.py b/src/daq_config_server/app/_routes.py index 7fa6dbe0..56f8bdba 100644 --- a/src/daq_config_server/app/_routes.py +++ b/src/daq_config_server/app/_routes.py @@ -1,7 +1,5 @@ import json import os -from dataclasses import dataclass -from enum import StrEnum from pathlib import Path from typing import Any @@ -9,6 +7,7 @@ from fastapi.responses import JSONResponse, Response from starlette import status +from daq_config_server.app.endpoints import ENDPOINTS, ValidAcceptHeaders from daq_config_server.models.base_model import ConfigModel from ._file_converter_map import get_converter @@ -38,18 +37,6 @@ def get_converted_file_contents(file_path: Path) -> dict[str, Any]: router = APIRouter() -class ValidAcceptHeaders(StrEnum): - JSON = "application/json" - PLAIN_TEXT = "text/plain" - RAW_BYTES = "application/octet-stream" - - -@dataclass(frozen=True) -class ENDPOINTS: - CONFIG = "/config" - HEALTH = "/healthz" - - @router.get( ENDPOINTS.CONFIG + "/{file_path:path}", responses={ diff --git a/src/daq_config_server/app/endpoints.py b/src/daq_config_server/app/endpoints.py new file mode 100644 index 00000000..fcd72a0a --- /dev/null +++ b/src/daq_config_server/app/endpoints.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from enum import StrEnum + + +class ValidAcceptHeaders(StrEnum): + JSON = "application/json" + PLAIN_TEXT = "text/plain" + RAW_BYTES = "application/octet-stream" + + +@dataclass(frozen=True) +class ENDPOINTS: + CONFIG = "/config" + HEALTH = "/healthz" diff --git a/src/daq_config_server/client/__init__.py b/src/daq_config_server/client/__init__.py new file mode 100644 index 00000000..cee7e011 --- /dev/null +++ b/src/daq_config_server/client/__init__.py @@ -0,0 +1,3 @@ +from ._client import ConfigClient + +__all__ = ["ConfigClient"] diff --git a/src/daq_config_server/app/client.py b/src/daq_config_server/client/_client.py similarity index 98% rename from src/daq_config_server/app/client.py rename to src/daq_config_server/client/_client.py index 56d5e1d1..6b91a1f9 100644 --- a/src/daq_config_server/app/client.py +++ b/src/daq_config_server/client/_client.py @@ -9,9 +9,9 @@ from pydantic import TypeAdapter from requests import Response -from daq_config_server.models import ConfigModel +from daq_config_server.app.endpoints import ENDPOINTS, ValidAcceptHeaders +from daq_config_server.models.base_model import ConfigModel -from ._routes import ENDPOINTS, ValidAcceptHeaders from ._server_response import ( MockPathToConverterDict, MockServerResponse, diff --git a/src/daq_config_server/app/_server_response.py b/src/daq_config_server/client/_server_response.py similarity index 97% rename from src/daq_config_server/app/_server_response.py rename to src/daq_config_server/client/_server_response.py index 6cc9125c..a851c39d 100644 --- a/src/daq_config_server/app/_server_response.py +++ b/src/daq_config_server/client/_server_response.py @@ -8,9 +8,8 @@ from requests import Response as RealResponse from requests.exceptions import HTTPError -from daq_config_server.models import ConfigModel - -from ._routes import ValidAcceptHeaders +from daq_config_server.app.endpoints import ValidAcceptHeaders +from daq_config_server.models.base_model import ConfigModel NonModel = str | bytes | dict[str, Any] MockPathToConverterDict = dict[Path, Callable[[str], ConfigModel | NonModel]] diff --git a/tests/system_tests/test_client.py b/tests/system_tests/test_client.py index be66c550..6364cdcc 100644 --- a/tests/system_tests/test_client.py +++ b/tests/system_tests/test_client.py @@ -11,7 +11,7 @@ from pydantic import ValidationError from daq_config_server.app._file_converter_map import CONVERTER_FUNCS -from daq_config_server.app.client import ConfigClient +from daq_config_server.client import ConfigClient from daq_config_server.models import ConfigModel, DisplayConfig from daq_config_server.models.lookup_tables import ( BeamlinePitchLookupTable, diff --git a/tests/unit_tests/client/__init__.py b/tests/unit_tests/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit_tests/app/test_client.py b/tests/unit_tests/client/test_client.py similarity index 98% rename from tests/unit_tests/app/test_client.py rename to tests/unit_tests/client/test_client.py index 1e144cc9..24f04ac7 100644 --- a/tests/unit_tests/app/test_client.py +++ b/tests/unit_tests/client/test_client.py @@ -10,7 +10,7 @@ from httpx import Response from daq_config_server.app._routes import ENDPOINTS, ValidAcceptHeaders -from daq_config_server.app.client import ( +from daq_config_server.client._client import ( ConfigClient, TModel, TNonModel, @@ -27,7 +27,7 @@ ) from daq_config_server.testing import make_test_response -REQUEST_PATCH = "daq_config_server.app._server_response.requests.get" +REQUEST_PATCH = "daq_config_server.client._server_response.requests.get" test_path = Path("test") diff --git a/tests/unit_tests/app/test_server_response.py b/tests/unit_tests/client/test_server_response.py similarity index 84% rename from tests/unit_tests/app/test_server_response.py rename to tests/unit_tests/client/test_server_response.py index 2e0c4324..77ac5183 100644 --- a/tests/unit_tests/app/test_server_response.py +++ b/tests/unit_tests/client/test_server_response.py @@ -1,7 +1,7 @@ import json from daq_config_server.app._routes import ValidAcceptHeaders -from daq_config_server.app._server_response import MockResponse +from daq_config_server.client._server_response import MockResponse def test_mock_response_using_bytes(): diff --git a/tests/unit_tests/models/test_beamline_parameters.py b/tests/unit_tests/models/test_beamline_parameters.py index d7cd029b..2aefe4ae 100644 --- a/tests/unit_tests/models/test_beamline_parameters.py +++ b/tests/unit_tests/models/test_beamline_parameters.py @@ -3,7 +3,7 @@ import pytest -from daq_config_server import ConfigClient +from daq_config_server.client import ConfigClient from daq_config_server.models.beamline_parameters import ( _parse_value, beamline_parameters_to_dict, From cc63b46bd3fe67b9bc55d30dac26d45bead8f778 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Mon, 29 Jun 2026 10:19:14 +0000 Subject: [PATCH 17/29] Update ENDPOINTS to an enum --- src/daq_config_server/app/_routes.py | 4 ++-- src/daq_config_server/app/endpoints.py | 4 +--- src/daq_config_server/client/_client.py | 4 ++-- tests/unit_tests/app/test_routes.py | 24 ++++++++++++------------ 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/daq_config_server/app/_routes.py b/src/daq_config_server/app/_routes.py index 56f8bdba..c8fd5af3 100644 --- a/src/daq_config_server/app/_routes.py +++ b/src/daq_config_server/app/_routes.py @@ -7,7 +7,7 @@ from fastapi.responses import JSONResponse, Response from starlette import status -from daq_config_server.app.endpoints import ENDPOINTS, ValidAcceptHeaders +from daq_config_server.app.endpoints import EndPoints, ValidAcceptHeaders from daq_config_server.models.base_model import ConfigModel from ._file_converter_map import get_converter @@ -38,7 +38,7 @@ def get_converted_file_contents(file_path: Path) -> dict[str, Any]: @router.get( - ENDPOINTS.CONFIG + "/{file_path:path}", + EndPoints.CONFIG + "/{file_path:path}", responses={ 200: { "description": "Returns JSON, plain text, or binary file.", diff --git a/src/daq_config_server/app/endpoints.py b/src/daq_config_server/app/endpoints.py index fcd72a0a..93d1ee04 100644 --- a/src/daq_config_server/app/endpoints.py +++ b/src/daq_config_server/app/endpoints.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass from enum import StrEnum @@ -8,7 +7,6 @@ class ValidAcceptHeaders(StrEnum): RAW_BYTES = "application/octet-stream" -@dataclass(frozen=True) -class ENDPOINTS: +class EndPoints(StrEnum): CONFIG = "/config" HEALTH = "/healthz" diff --git a/src/daq_config_server/client/_client.py b/src/daq_config_server/client/_client.py index 6b91a1f9..912b2fc2 100644 --- a/src/daq_config_server/client/_client.py +++ b/src/daq_config_server/client/_client.py @@ -9,7 +9,7 @@ from pydantic import TypeAdapter from requests import Response -from daq_config_server.app.endpoints import ENDPOINTS, ValidAcceptHeaders +from daq_config_server.app.endpoints import EndPoints, ValidAcceptHeaders from daq_config_server.models.base_model import ConfigModel from ._server_response import ( @@ -214,7 +214,7 @@ def get_file_contents( accept_header = _get_mime_type(desired_return_type) result = self._get( - ENDPOINTS.CONFIG, + EndPoints.CONFIG, accept_header=accept_header, file_path=file_path, reset_cached_result=reset_cached_result, diff --git a/tests/unit_tests/app/test_routes.py b/tests/unit_tests/app/test_routes.py index b230223c..f697de30 100644 --- a/tests/unit_tests/app/test_routes.py +++ b/tests/unit_tests/app/test_routes.py @@ -11,8 +11,8 @@ from fastapi.testclient import TestClient from daq_config_server.app._routes import ( - ENDPOINTS, ConverterParseError, + EndPoints, ValidAcceptHeaders, get_converted_file_contents, ) @@ -124,7 +124,7 @@ async def test_get_configuration_on_plain_text_file(mock_app: TestClient): ) await _assert_get_and_response( - mock_app, f"{ENDPOINTS.CONFIG}/{file_path}", expected_response + mock_app, f"{EndPoints.CONFIG}/{file_path}", expected_response ) @@ -141,7 +141,7 @@ async def test_get_configuration_raw_bytes(mock_app: TestClient): await _assert_get_and_response( mock_app, - f"{ENDPOINTS.CONFIG}/{file_path}", + f"{EndPoints.CONFIG}/{file_path}", expected_response, accept_header={"Accept": expected_type}, ) @@ -150,7 +150,7 @@ async def test_get_configuration_raw_bytes(mock_app: TestClient): def test_get_configuration_exception_on_invalid_file(mock_app: TestClient): file_path = TestDataPaths.TEST_INVALID_FILE_PATH response = mock_app.get( - f"{ENDPOINTS.CONFIG}/{file_path}", headers=ACCEPT_HEADER_DEFAULT + f"{EndPoints.CONFIG}/{file_path}", headers=ACCEPT_HEADER_DEFAULT ) assert response.status_code == status.HTTP_404_NOT_FOUND @@ -167,7 +167,7 @@ async def test_get_configuration_on_json_file(mock_app: TestClient): ) await _assert_get_and_response( mock_app, - f"{ENDPOINTS.CONFIG}/{file_path}", + f"{EndPoints.CONFIG}/{file_path}", expected_response, accept_header={"Accept": expected_type}, ) @@ -181,42 +181,42 @@ async def test_get_configuration_gives_http_422_on_failed_conversion( with open(file_path, "wb") as f: f.write(b"\x80\x81\xfe\xff") response = mock_app.get( - f"{ENDPOINTS.CONFIG}/{file_path}", headers=ACCEPT_HEADER_DEFAULT + f"{EndPoints.CONFIG}/{file_path}", headers=ACCEPT_HEADER_DEFAULT ) assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT def test_get_configuration_on_non_absolute_filepath(mock_app: TestClient): file_path = "relative_path" - response = mock_app.get(f"{ENDPOINTS.CONFIG}/{file_path}") + response = mock_app.get(f"{EndPoints.CONFIG}/{file_path}") assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT async def test_health_check_returns_code_200( mock_app: TestClient, ): - assert mock_app.get(ENDPOINTS.HEALTH).status_code == status.HTTP_200_OK + assert mock_app.get(EndPoints.HEALTH).status_code == status.HTTP_200_OK def test_validate_path_against_whitelist_on_valid_file(mock_app: TestClient): file_path = TestDataPaths.TEST_BEAMLINE_PARAMETERS_PATH - response = mock_app.get(f"{ENDPOINTS.CONFIG}/{file_path}") + response = mock_app.get(f"{EndPoints.CONFIG}/{file_path}") assert response.status_code == status.HTTP_200_OK def test_validate_path_against_whitelist_on_invalid_file(mock_app: TestClient): file_path = TestDataPaths.TEST_FILE_NOT_ON_WHITELIST_PATH - response = mock_app.get(f"{ENDPOINTS.CONFIG}/{file_path}") + response = mock_app.get(f"{EndPoints.CONFIG}/{file_path}") assert response.status_code == status.HTTP_403_FORBIDDEN def test_validate_path_against_whitelist_on_file_in_invalid_dir(mock_app: TestClient): file_path = TestDataPaths.TEST_FILE_IN_BAD_DIR - response = mock_app.get(f"{ENDPOINTS.CONFIG}/{file_path}") + response = mock_app.get(f"{EndPoints.CONFIG}/{file_path}") assert response.status_code == status.HTTP_403_FORBIDDEN def test_validate_path_against_whitelist_on_file_in_valid_dir(mock_app: TestClient): file_path = TestDataPaths.TEST_FILE_IN_GOOD_DIR - response = mock_app.get(f"{ENDPOINTS.CONFIG}/{file_path}") + response = mock_app.get(f"{EndPoints.CONFIG}/{file_path}") assert response.status_code == status.HTTP_200_OK From 2b7e325f7592d72603a38592ae4c8d72d009bf44 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Mon, 29 Jun 2026 10:19:57 +0000 Subject: [PATCH 18/29] fix test --- tests/unit_tests/client/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests/client/test_client.py b/tests/unit_tests/client/test_client.py index 24f04ac7..7b46520a 100644 --- a/tests/unit_tests/client/test_client.py +++ b/tests/unit_tests/client/test_client.py @@ -9,7 +9,7 @@ from fastapi import status from httpx import Response -from daq_config_server.app._routes import ENDPOINTS, ValidAcceptHeaders +from daq_config_server.app._routes import EndPoints, ValidAcceptHeaders from daq_config_server.client._client import ( ConfigClient, TModel, @@ -48,7 +48,7 @@ def test_config_client_get_file_contents_default_header( mock_request.return_value = make_test_response("test") assert client.get_file_contents(test_path) == "test" mock_request.assert_called_once_with( - client._url + ENDPOINTS.CONFIG + "/" + str(test_path), + client._url + EndPoints.CONFIG + "/" + str(test_path), headers={"Accept": ValidAcceptHeaders.PLAIN_TEXT}, ) From 37bb7cf1f01191b3751353db7f22f160f667b2a3 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Mon, 29 Jun 2026 10:54:14 +0000 Subject: [PATCH 19/29] Rename endpoints.py to constants.py --- src/daq_config_server/app/{endpoints.py => constants.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/daq_config_server/app/{endpoints.py => constants.py} (100%) diff --git a/src/daq_config_server/app/endpoints.py b/src/daq_config_server/app/constants.py similarity index 100% rename from src/daq_config_server/app/endpoints.py rename to src/daq_config_server/app/constants.py From ed122cff24cc755f90193509ac3ce6db662d433f Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Mon, 29 Jun 2026 10:56:33 +0000 Subject: [PATCH 20/29] Fix from renaming --- src/daq_config_server/app/_routes.py | 2 +- src/daq_config_server/client/_client.py | 2 +- src/daq_config_server/client/_server_response.py | 2 +- tests/unit_tests/app/test_routes.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/daq_config_server/app/_routes.py b/src/daq_config_server/app/_routes.py index c8fd5af3..c4418659 100644 --- a/src/daq_config_server/app/_routes.py +++ b/src/daq_config_server/app/_routes.py @@ -7,7 +7,7 @@ from fastapi.responses import JSONResponse, Response from starlette import status -from daq_config_server.app.endpoints import EndPoints, ValidAcceptHeaders +from daq_config_server.app.constants import EndPoints, ValidAcceptHeaders from daq_config_server.models.base_model import ConfigModel from ._file_converter_map import get_converter diff --git a/src/daq_config_server/client/_client.py b/src/daq_config_server/client/_client.py index 912b2fc2..ed83b3c5 100644 --- a/src/daq_config_server/client/_client.py +++ b/src/daq_config_server/client/_client.py @@ -9,7 +9,7 @@ from pydantic import TypeAdapter from requests import Response -from daq_config_server.app.endpoints import EndPoints, ValidAcceptHeaders +from daq_config_server.app.constants import EndPoints, ValidAcceptHeaders from daq_config_server.models.base_model import ConfigModel from ._server_response import ( diff --git a/src/daq_config_server/client/_server_response.py b/src/daq_config_server/client/_server_response.py index a851c39d..3274eca2 100644 --- a/src/daq_config_server/client/_server_response.py +++ b/src/daq_config_server/client/_server_response.py @@ -8,7 +8,7 @@ from requests import Response as RealResponse from requests.exceptions import HTTPError -from daq_config_server.app.endpoints import ValidAcceptHeaders +from daq_config_server.app.constants import ValidAcceptHeaders from daq_config_server.models.base_model import ConfigModel NonModel = str | bytes | dict[str, Any] diff --git a/tests/unit_tests/app/test_routes.py b/tests/unit_tests/app/test_routes.py index f697de30..8025d205 100644 --- a/tests/unit_tests/app/test_routes.py +++ b/tests/unit_tests/app/test_routes.py @@ -12,11 +12,11 @@ from daq_config_server.app._routes import ( ConverterParseError, - EndPoints, ValidAcceptHeaders, get_converted_file_contents, ) from daq_config_server.app.api import app +from daq_config_server.app.constants import EndPoints from daq_config_server.models.beamline_parameters import beamline_parameters_to_dict from daq_config_server.models.lookup_tables import GenericLookupTable from tests.constants import TestDataPaths From 98e7901490d0c1f1781bdbf107874d790a0336e8 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Mon, 29 Jun 2026 11:00:40 +0000 Subject: [PATCH 21/29] Remove unused code --- src/daq_config_server/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/daq_config_server/__init__.py b/src/daq_config_server/__init__.py index 6878ddbf..a2ffbf36 100644 --- a/src/daq_config_server/__init__.py +++ b/src/daq_config_server/__init__.py @@ -8,9 +8,4 @@ from ._version import __version__ -# from .client._client import ConfigClient - -__all__ = [ - "__version__", - # "ConfigClient" -] +__all__ = ["__version__"] From 39a9fe6b7a3d35d4f6e7ff8830f62eb752f34588 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Wed, 1 Jul 2026 13:58:47 +0000 Subject: [PATCH 22/29] Review feedback, remove converts from client. Instead return mock data directly --- src/daq_config_server/client/_client.py | 14 +++---- .../client/_server_response.py | 38 +++++++++---------- tests/unit_tests/client/test_client.py | 18 +++------ .../models/test_beamline_parameters.py | 18 ++------- 4 files changed, 34 insertions(+), 54 deletions(-) diff --git a/src/daq_config_server/client/_client.py b/src/daq_config_server/client/_client.py index ed83b3c5..d81a8b83 100644 --- a/src/daq_config_server/client/_client.py +++ b/src/daq_config_server/client/_client.py @@ -13,8 +13,8 @@ from daq_config_server.models.base_model import ConfigModel from ._server_response import ( - MockPathToConverterDict, MockServerResponse, + PathToMockDataDict, RealServerResponse, ResponseType, ServerResponse, @@ -74,7 +74,9 @@ def __init__( self._lock = RLock() self._server: ServerResponse = RealServerResponse(self._url, self._log) - def configure_mock(self, converters: MockPathToConverterDict | None = None) -> None: + def configure_mock( + self, path_to_mock_data: PathToMockDataDict | None = None + ) -> None: """Switch the client into mock mode using a local filesystem backend. This replaces the real HTTP server implementation with a mock @@ -84,12 +86,10 @@ def configure_mock(self, converters: MockPathToConverterDict | None = None) -> N or transformation logic on a per-file basis. Args: - converters: - Optional mapping of file paths to converter functions. - Each function receives raw file contents as a string and - returns a transformed object (e.g. dict, ConfigModel, etc.). + path_to_mock_data: + Optional mapping of file paths to mock data to return from the server. """ - self._server = MockServerResponse(converters) + self._server = MockServerResponse(path_to_mock_data) @cachedmethod( cache=operator.attrgetter("_cache"), lock=operator.attrgetter("_lock") diff --git a/src/daq_config_server/client/_server_response.py b/src/daq_config_server/client/_server_response.py index 3274eca2..dc30acbf 100644 --- a/src/daq_config_server/client/_server_response.py +++ b/src/daq_config_server/client/_server_response.py @@ -1,5 +1,4 @@ import json -from collections.abc import Callable from logging import Logger from pathlib import Path from typing import Any, Protocol @@ -12,7 +11,7 @@ from daq_config_server.models.base_model import ConfigModel NonModel = str | bytes | dict[str, Any] -MockPathToConverterDict = dict[Path, Callable[[str], ConfigModel | NonModel]] +PathToMockDataDict = dict[str, ConfigModel | NonModel] class MockResponse: @@ -69,31 +68,30 @@ def get_response( class MockServerResponse(ServerResponse): """Mock implementation of ServerResponse used for unit testing. - This class simulates a config server by reading local files instead of - performing HTTP requests. It supports optional per-file converter functions - that can transform raw file contents before they are returned. + This class simulates a config server by reading local files instead of performing + HTTP requests. Supports optional overrides for a specified path to the data you + want to return instead. """ - def __init__(self, mock_data_converters: MockPathToConverterDict | None = None): - self._mock_data_converters = mock_data_converters or {} + def __init__(self, path_to_mock_data: PathToMockDataDict | None = None): + self.path_to_mock_data = path_to_mock_data or {} def get_response( self, endpoint: str, accept_header: ValidAcceptHeaders, file_path: Path ) -> MockResponse: - raw = file_path.read_text() - # Apply optional converter hook - if file_path in self._mock_data_converters: - converted = self._mock_data_converters[file_path](raw) - # If it's a Pydantic model, serialize properly - if isinstance(converted, ConfigModel): - raw = converted.model_dump_json() - - elif isinstance(converted, dict): - raw = json.dumps(converted) - # otherwise assume already string-like + if file_path in self.path_to_mock_data: + mock_data = self.path_to_mock_data[str(file_path)] + if isinstance(mock_data, ConfigModel): + mock_response = mock_data.model_dump_json() + elif isinstance(mock_data, dict): + mock_response = json.dumps(mock_data) + elif isinstance(mock_data, bytes): + mock_response = mock_data.decode() else: - raw = str(converted) - return MockResponse(raw, accept_header) + mock_response = mock_data + else: + mock_response = file_path.read_text() + return MockResponse(mock_response, accept_header) class RealServerResponse(ServerResponse): diff --git a/tests/unit_tests/client/test_client.py b/tests/unit_tests/client/test_client.py index 7b46520a..5bc23f13 100644 --- a/tests/unit_tests/client/test_client.py +++ b/tests/unit_tests/client/test_client.py @@ -302,19 +302,11 @@ def test_mock_config_client_get_file_contents_as_config_model_gives_expected_res def test_mock_config_client_converter_table_to_json(tmp_path: Path): - file = tmp_path / "beamline.txt" - - # "table" format (header + row) - file.write_text("x|y\n1|test") - - def table_to_dict(contents: str) -> dict[str, Any]: - lines = contents.strip().splitlines() - headers = lines[0].split("|") - values = lines[1].split("|") - return dict(zip(headers, values, strict=True)) + file = Path("/path/to/data.txt") client = ConfigClient() - client.configure_mock({file: table_to_dict}) + expected_data = MyModel() + client.configure_mock({file: expected_data}) - result = client.get_file_contents(file, desired_return_type=dict) - assert result == {"x": "1", "y": "test"} + result = client.get_file_contents(file, desired_return_type=MyModel) + assert result == expected_data diff --git a/tests/unit_tests/models/test_beamline_parameters.py b/tests/unit_tests/models/test_beamline_parameters.py index 2aefe4ae..b433e027 100644 --- a/tests/unit_tests/models/test_beamline_parameters.py +++ b/tests/unit_tests/models/test_beamline_parameters.py @@ -3,7 +3,6 @@ import pytest -from daq_config_server.client import ConfigClient from daq_config_server.models.beamline_parameters import ( _parse_value, beamline_parameters_to_dict, @@ -11,21 +10,12 @@ from tests.constants import TestDataPaths -@pytest.fixture -def config_client() -> ConfigClient: - client = ConfigClient() - client.configure_mock( - {TestDataPaths.TEST_BEAMLINE_PARAMETERS_PATH: beamline_parameters_to_dict} - ) - return client - - -def test_beamline_parameters_to_dict_gives_expected_result(config_client: ConfigClient): +def test_beamline_parameters_to_dict_gives_expected_result(): + with open(TestDataPaths.TEST_BEAMLINE_PARAMETERS_PATH) as f: + contents = f.read() with open(TestDataPaths.EXPECTED_BEAMLINE_PARAMETERS_JSON_PATH) as f: expected = json.load(f) - result = config_client.get_file_contents( - TestDataPaths.TEST_BEAMLINE_PARAMETERS_PATH, desired_return_type=dict - ) + result = beamline_parameters_to_dict(contents) assert result == expected From 62653073efcaa26415cc6fa431e9cd9ce9a90cb3 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Wed, 1 Jul 2026 14:04:58 +0000 Subject: [PATCH 23/29] Update documentation --- docs/how-to/config-server-guide.md | 68 +++++++++++++++--------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/docs/how-to/config-server-guide.md b/docs/how-to/config-server-guide.md index 66dc8659..3d961dc6 100644 --- a/docs/how-to/config-server-guide.md +++ b/docs/how-to/config-server-guide.md @@ -78,57 +78,59 @@ config_client.configure_mock() ``` Once enabled, all file access goes through the mock layer instead of the real server. -### Mock converters +### Mock data -Mock converters allow you to simulate the server-side converter_map behaviour locally. +Mock mode allows you to override the contents returned for specific files without +running a real config server. -A converter is a function with the signature: +Mock data is registered as a mapping from `str` file path to the value that should be +returned. Supported values are: ```python -Callable[[str], ConfigModel | str | bytes | dict[str, Any]] +ConfigModel | str | bytes | dict[str, Any] ``` -You register converters keyed by Path: - ```python -from pathlib import Path -config_client.setup_mock( +config_client.configure_mock( { - Path("/tmp/my_file.txt"): my_converter_function + "/path/to/data.txt": {"key": "value"} } ) ``` -Example: converting a table to JSON -You can simulate a file containing a table-like format (e.g. whitespace-separated columns) and convert it into JSON. +### Example: returning a model + +You can configure the mock to return a `ConfigModel` instance directly. -### Example file content -``` -key value -x 1 -y test -``` -### Converter function ```python -import json -def table_to_json(contents: str) -> str: - lines = contents.strip().splitlines() - headers = lines[0].split() +expected = MyModel(field="value") - result = {} - for line in lines[1:]: - key, value = line.split() - # attempt numeric conversion - if value.isdigit(): - value = int(value) - result[key] = value +client.configure_mock({"/path/to/data.txt": expected}) - return json.dumps(result) +result = client.get_file_contents( + "/path/to/data.txt", + desired_return_type=MyModel, +) + +assert result == expected ``` -### Test setup + +### Example: returning a dictionary + +If the caller requests a dictionary, provide a dictionary as the mock data. + ```python -file_path = Path("/path/to/data.txt") -client.setup_mock({file_path: table_to_json}) + +expected = { + "x": 1, + "y": "test", +} + +client.configure_mock({"/path/to/data.txt": expected}) + +result = client.get_file_contents("/path/to/data.txt", desired_return_type=dict) + +assert result == expected ``` From 95584e51c6ee1e054acf00b48b9b503414cbea2c Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Wed, 1 Jul 2026 14:10:33 +0000 Subject: [PATCH 24/29] fix tests --- src/daq_config_server/client/_server_response.py | 2 +- tests/unit_tests/client/test_client.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/daq_config_server/client/_server_response.py b/src/daq_config_server/client/_server_response.py index dc30acbf..93e9a0a2 100644 --- a/src/daq_config_server/client/_server_response.py +++ b/src/daq_config_server/client/_server_response.py @@ -79,7 +79,7 @@ def __init__(self, path_to_mock_data: PathToMockDataDict | None = None): def get_response( self, endpoint: str, accept_header: ValidAcceptHeaders, file_path: Path ) -> MockResponse: - if file_path in self.path_to_mock_data: + if str(file_path) in self.path_to_mock_data: mock_data = self.path_to_mock_data[str(file_path)] if isinstance(mock_data, ConfigModel): mock_response = mock_data.model_dump_json() diff --git a/tests/unit_tests/client/test_client.py b/tests/unit_tests/client/test_client.py index 5bc23f13..79f6c949 100644 --- a/tests/unit_tests/client/test_client.py +++ b/tests/unit_tests/client/test_client.py @@ -301,8 +301,8 @@ def test_mock_config_client_get_file_contents_as_config_model_gives_expected_res assert result == expected_data -def test_mock_config_client_converter_table_to_json(tmp_path: Path): - file = Path("/path/to/data.txt") +def test_mock_config_client_converter_table_to_json(): + file = "/path/to/data.txt" client = ConfigClient() expected_data = MyModel() From e6d4618faeb1fb5d3960f42b56aa3d860409611e Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Wed, 1 Jul 2026 14:21:17 +0000 Subject: [PATCH 25/29] Add test coverage --- tests/unit_tests/client/test_client.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/client/test_client.py b/tests/unit_tests/client/test_client.py index 79f6c949..1118fe21 100644 --- a/tests/unit_tests/client/test_client.py +++ b/tests/unit_tests/client/test_client.py @@ -17,6 +17,7 @@ TypeConversionError, _get_mime_type, ) +from daq_config_server.client._server_response import NonModel from daq_config_server.models import ConfigModel, DisplayConfig from daq_config_server.models.lookup_tables import ( BeamlinePitchLookupTable, @@ -301,12 +302,26 @@ def test_mock_config_client_get_file_contents_as_config_model_gives_expected_res assert result == expected_data -def test_mock_config_client_converter_table_to_json(): +@pytest.mark.parametrize( + "expected_data, return_type", + ( + (MyModel(), MyModel), + ({"data1": 5, "data2": "value"}, dict), + ("My string data", str), + ), +) +def test_mock_config_client_with_path_to_data_override( + expected_data: ConfigModel | NonModel, return_type: type[ConfigModel | NonModel] +): file = "/path/to/data.txt" client = ConfigClient() - expected_data = MyModel() client.configure_mock({file: expected_data}) - result = client.get_file_contents(file, desired_return_type=MyModel) + result = client.get_file_contents(file, desired_return_type=return_type) assert result == expected_data + + with pytest.raises(FileNotFoundError): + client.get_file_contents( + "/file/not/configured/with/mock/data", desired_return_type=return_type + ) From b087289c444bd2dbadb90723fec82308cd37b0aa Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Wed, 1 Jul 2026 14:26:29 +0000 Subject: [PATCH 26/29] Add bytes test coverage --- tests/unit_tests/client/test_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit_tests/client/test_client.py b/tests/unit_tests/client/test_client.py index 1118fe21..c6ec8a14 100644 --- a/tests/unit_tests/client/test_client.py +++ b/tests/unit_tests/client/test_client.py @@ -308,6 +308,7 @@ def test_mock_config_client_get_file_contents_as_config_model_gives_expected_res (MyModel(), MyModel), ({"data1": 5, "data2": "value"}, dict), ("My string data", str), + (b"My string data", bytes), ), ) def test_mock_config_client_with_path_to_data_override( From 8adfa4589e5465b52c6a0d4c534fd729d5fe90eb Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Wed, 1 Jul 2026 14:28:38 +0000 Subject: [PATCH 27/29] Rename variables to be clearer on whats testing --- tests/unit_tests/client/test_client.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/unit_tests/client/test_client.py b/tests/unit_tests/client/test_client.py index c6ec8a14..593c4746 100644 --- a/tests/unit_tests/client/test_client.py +++ b/tests/unit_tests/client/test_client.py @@ -291,14 +291,14 @@ class MyModel(ConfigModel): def test_mock_config_client_get_file_contents_as_config_model_gives_expected_result( tmp_path: Path, ): - file = tmp_path / "beamline.json" + real_file = tmp_path / "beamline.json" expected_data = MyModel().model_dump_json() - file.write_text(expected_data) + real_file.write_text(expected_data) client = ConfigClient() client.configure_mock() - result = client.get_file_contents(file) + result = client.get_file_contents(real_file) assert result == expected_data @@ -312,14 +312,15 @@ def test_mock_config_client_get_file_contents_as_config_model_gives_expected_res ), ) def test_mock_config_client_with_path_to_data_override( - expected_data: ConfigModel | NonModel, return_type: type[ConfigModel | NonModel] + expected_data: ConfigModel | NonModel, + return_type: type[ConfigModel | NonModel], ): - file = "/path/to/data.txt" + mock_file = "/path/to/data.txt" client = ConfigClient() - client.configure_mock({file: expected_data}) + client.configure_mock({mock_file: expected_data}) - result = client.get_file_contents(file, desired_return_type=return_type) + result = client.get_file_contents(mock_file, desired_return_type=return_type) assert result == expected_data with pytest.raises(FileNotFoundError): From 0fce6d9c4e85a0e2001cba9e72486bedd9fb54dd Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Wed, 1 Jul 2026 14:37:06 +0000 Subject: [PATCH 28/29] Updated mock docs to remove reference to conversion --- docs/how-to/config-server-guide.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/how-to/config-server-guide.md b/docs/how-to/config-server-guide.md index 3d961dc6..f536f7df 100644 --- a/docs/how-to/config-server-guide.md +++ b/docs/how-to/config-server-guide.md @@ -67,16 +67,32 @@ If you need to read a file which contains sensitive information, or `dls-dasc` d # Mocking the Config Client (for tests and offline development) -The ConfigClient can be configured to run in a fully offline mode for unit tests and local development. In this mode, no HTTP requests are made. Instead, file reads are intercepted and optionally transformed using mock converters. +The `ConfigClient` can be configured to run in a fully offline mode for unit tests and local development. In this mode, no HTTP requests are made. Instead, responses for specific file paths can be overridden with mock data. -This allows you to simulate server-side conversion behaviour (e.g. JSON → dict, table → JSON, or custom Pydantic models) without requiring a running config service. +Mock data can be provided as a `ConfigModel`, `dict`, `str`, or `bytes`, allowing tests to simulate the responses that would normally be returned by the config server without requiring a running service or real configuration files. ## Enabling mock mode + ```python config_client = ConfigClient() config_client.configure_mock() ``` -Once enabled, all file access goes through the mock layer instead of the real server. + +With no mock data configured, the client reads directly from the local filesystem instead of contacting the config server. + +To override the response for specific files, pass a mapping from `Path` to the desired response: + +```python +from pathlib import Path + +config_client.configure_mock( + { + Path("/path/to/config"): {"enabled": True}, + } +) +``` + +When a mocked path is requested, the configured value is returned. For all other paths, the client reads the file contents from the local filesystem. ### Mock data From 2e25a9dec70de45e8d4f66c59726f72c64781153 Mon Sep 17 00:00:00 2001 From: Oli Wenman Date: Wed, 1 Jul 2026 14:41:49 +0000 Subject: [PATCH 29/29] More doc adjustments --- docs/how-to/config-server-guide.md | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/docs/how-to/config-server-guide.md b/docs/how-to/config-server-guide.md index f536f7df..2305f29e 100644 --- a/docs/how-to/config-server-guide.md +++ b/docs/how-to/config-server-guide.md @@ -80,27 +80,13 @@ config_client.configure_mock() With no mock data configured, the client reads directly from the local filesystem instead of contacting the config server. -To override the response for specific files, pass a mapping from `Path` to the desired response: +### Mock data -```python -from pathlib import Path - -config_client.configure_mock( - { - Path("/path/to/config"): {"enabled": True}, - } -) -``` +Mock mode allows you to override the contents returned for specific files without running a real config server. When a mocked path is requested, the configured value is returned. For all other paths, the client reads the file contents from the local filesystem. -### Mock data - -Mock mode allows you to override the contents returned for specific files without -running a real config server. - -Mock data is registered as a mapping from `str` file path to the value that should be -returned. Supported values are: +Mock data is registered as a mapping from `str` file path to the value that should be returned. Supported values are: ```python ConfigModel | str | bytes | dict[str, Any]