|
| 1 | +"""Shared utilities for reference source ETL modules. |
| 2 | +
|
| 3 | +Provides helpers used across multiple built-in sources, such as extracting |
| 4 | +user-configured extra fields from raw API responses. |
| 5 | +""" |
| 6 | + |
| 7 | +import logging |
| 8 | + |
| 9 | +from jsonpath_ng import parse as jsonpath_parse |
| 10 | +from jsonpath_ng.exceptions import JsonPathParserError |
| 11 | + |
| 12 | +logger = logging.getLogger(__name__) |
| 13 | + |
| 14 | + |
| 15 | +def extract_extra_fields(data: dict, field_map: dict[str, str]) -> dict[str, str]: |
| 16 | + """Extract extra fields from a raw API response using JSONPath expressions. |
| 17 | +
|
| 18 | + For each entry in *field_map*, the corresponding JSONPath expression is |
| 19 | + evaluated against *data*. Fields that produce no match, an empty value, |
| 20 | + or an invalid JSONPath expression are omitted from the result. |
| 21 | +
|
| 22 | + Args: |
| 23 | + data: Raw API response dictionary to extract from. |
| 24 | + field_map: Mapping of ``field_name`` → ``JSONPath expression``. |
| 25 | + Example: ``{"eligibility": "$.protocolSection.eligibilityModule.eligibilityCriteria"}``. |
| 26 | +
|
| 27 | + Returns: |
| 28 | + A dict mapping ``field_name`` → extracted text string for each field |
| 29 | + that had a non-empty value. Use :func:`format_extra_fields_for_content` |
| 30 | + to turn this into text to append to reference content, and |
| 31 | + ``list(result.keys())`` for ``extra_fields_captured`` metadata. |
| 32 | +
|
| 33 | + Examples: |
| 34 | + >>> extract_extra_fields({}, {}) |
| 35 | + {} |
| 36 | + >>> extract_extra_fields({"title": "My Paper"}, {}) |
| 37 | + {} |
| 38 | + >>> extract_extra_fields({}, {"eligibility": "$.eligibility"}) |
| 39 | + {} |
| 40 | + >>> extract_extra_fields({"foo": "bar"}, {"foo": "$.foo"}) |
| 41 | + {'foo': 'bar'} |
| 42 | + >>> result = extract_extra_fields( |
| 43 | + ... {"a": "alpha", "b": "beta"}, |
| 44 | + ... {"a": "$.a", "b": "$.b"}, |
| 45 | + ... ) |
| 46 | + >>> result == {"a": "alpha", "b": "beta"} |
| 47 | + True |
| 48 | + >>> extract_extra_fields({"other": "x"}, {"missing": "$.missing"}) |
| 49 | + {} |
| 50 | + >>> extract_extra_fields({"foo": "bar"}, {"bad": "not a valid $[[[jsonpath"}) |
| 51 | + {} |
| 52 | + """ |
| 53 | + if not field_map or not data: |
| 54 | + return {} |
| 55 | + |
| 56 | + result: dict[str, str] = {} |
| 57 | + |
| 58 | + for field_name, jsonpath_expr in field_map.items(): |
| 59 | + try: |
| 60 | + parsed = jsonpath_parse(jsonpath_expr) |
| 61 | + except JsonPathParserError as exc: |
| 62 | + logger.warning("Invalid JSONPath expression '%s' for field '%s': %s", jsonpath_expr, field_name, exc) |
| 63 | + continue |
| 64 | + |
| 65 | + matches = parsed.find(data) |
| 66 | + if not matches: |
| 67 | + continue |
| 68 | + |
| 69 | + raw_value = matches[0].value |
| 70 | + if raw_value is None: |
| 71 | + continue |
| 72 | + |
| 73 | + if isinstance(raw_value, list): |
| 74 | + text = " ".join(str(item) for item in raw_value if str(item).strip()) |
| 75 | + else: |
| 76 | + text = str(raw_value) |
| 77 | + |
| 78 | + if not text.strip(): |
| 79 | + continue |
| 80 | + |
| 81 | + result[field_name] = text |
| 82 | + |
| 83 | + return result |
| 84 | + |
| 85 | + |
| 86 | +def format_extra_fields_for_content(extra: dict[str, str]) -> str: |
| 87 | + """Format an extra-fields dict as markdown sections for appending to reference content. |
| 88 | +
|
| 89 | + Args: |
| 90 | + extra: Result of :func:`extract_extra_fields` (field_name → text). |
| 91 | +
|
| 92 | + Returns: |
| 93 | + String of ``### field_name\\n\\ntext`` sections joined by ``\\n\\n``, |
| 94 | + or empty string if *extra* is empty. |
| 95 | +
|
| 96 | + Examples: |
| 97 | + >>> format_extra_fields_for_content({}) |
| 98 | + '' |
| 99 | + >>> format_extra_fields_for_content({"foo": "bar content"}) |
| 100 | + '### foo\\n\\nbar content' |
| 101 | + >>> format_extra_fields_for_content({"a": "alpha", "b": "beta"}) |
| 102 | + '### a\\n\\nalpha\\n\\n### b\\n\\nbeta' |
| 103 | + """ |
| 104 | + if not extra: |
| 105 | + return "" |
| 106 | + return "\n\n".join(f"### {k}\n\n{v}" for k, v in extra.items()) |
0 commit comments