feat(harvester): add ROR assignment in mapping

Author: kpsherva

site/cds_rdm/inspire_harvester/load/files.py

@@ -45,7 +45,6 @@ def __init__(self, retry_config: RetryConfig = None):
 
     def compute_diff(self, existing_files, new_files) -> FileDiff:
         """Return the set difference between existing and new file checksums."""
-
         existing_checksums = [value["checksum"] for value in existing_files.values()]
         new_checksums = [value["checksum"] for value in new_files.values()]
 
@@ -102,6 +101,7 @@ def fetch(self, url, logger) -> BytesIO:
         )
 
     def check_files_should_update(self, record, incoming_record, logger):
+        """Check if files should be updated."""
         if not record:
             return True
         record_dict = record.to_dict()

site/cds_rdm/inspire_harvester/transform/config.py

@@ -7,8 +7,10 @@
 
 """INSPIRE to CDS harvester config module."""
 from cds_rdm.inspire_harvester.transform.mappers.article import (
+    ArticleDescriptionMapper,
     ArticleDOIMapper,
-    ArticleFilesMapper, ArticleTitleMapper, ArticleDescriptionMapper,
+    ArticleFilesMapper,
+    ArticleTitleMapper,
 )
 from cds_rdm.inspire_harvester.transform.mappers.basic_metadata import (
     AdditionalDescriptionsMapper,
@@ -38,8 +40,10 @@
     RelatedIdentifiersMapper,
 )
 from cds_rdm.inspire_harvester.transform.mappers.preprint import (
+    PreprintDescriptionMapper,
     PreprintDOIMapper,
-    PreprintFilesMapper, PreprintTitleMapper, PreprintDescriptionMapper,
+    PreprintFilesMapper,
+    PreprintTitleMapper,
 )
 from cds_rdm.inspire_harvester.transform.mappers.thesis import (
     ThesisContributorsMapper,

site/cds_rdm/inspire_harvester/transform/mappers/contributors.py

@@ -9,6 +9,8 @@
 
 from dataclasses import dataclass
 
+from idutils.normalizers import normalize_ror
+
 from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase
 
 
@@ -38,16 +40,31 @@ def _transform_author_identifiers(self, author):
         return processed_identifiers
 
     def _transform_author_affiliations(self, author):
-        """Transform affiliations."""
+        """Transform affiliations.
+
+        If affiliations_identifiers is present, ROR values are matched by index
+        to the affiliations list and used as vocabulary IDs. Affiliations without
+        a matching ROR fall back to free-text name.
+        """
         affiliations = author.get("affiliations", [])
+        affiliations_identifiers = author.get("affiliations_identifiers")
         mapped_affiliations = []
 
-        for affiliation in affiliations:
-            value = affiliation.get("value")
-
-            if value:
-                value = value.rstrip(".").strip()
-                mapped_affiliations.append({"name": value})
+        ror_ids = []
+        if affiliations_identifiers:
+            ror_ids = [
+                normalize_ror(ai["value"])
+                for ai in affiliations_identifiers
+                if ai.get("schema") == "ROR"
+            ]
+
+        for i, affiliation in enumerate(affiliations):
+            if i < len(ror_ids):
+                mapped_affiliations.append({"id": ror_ids[i]})
+            else:
+                value = affiliation.get("value")
+                if value:
+                    mapped_affiliations.append({"name": value.rstrip(".").strip()})
 
         return mapped_affiliations
 

site/cds_rdm/inspire_harvester/transform/splitter.py

@@ -33,6 +33,7 @@ class InspireVersionSplitter:
     """
 
     def __init__(self, inspire_record, ctx, cds_id, policy=mapper_policy):
+        """Constructor."""
         self.inspire_record = inspire_record
         self.inspire_id = self.inspire_record.get("id")
         self.policy = policy
@@ -42,6 +43,7 @@ def __init__(self, inspire_record, ctx, cds_id, policy=mapper_policy):
         self.logger = Logger(inspire_id=self.inspire_id)
 
     def needs_split(self) -> bool:
+        """Determine whether the record needs a split."""
         metadata = self.inspire_record.get("metadata", {})
         doc_types = metadata.get("document_type", [])
         if len(doc_types) <= 1:
@@ -58,7 +60,6 @@ def needs_split(self) -> bool:
 
     def split(self):
         """Return [preprint_record, publication_record], or None if split is not applicable."""
-
         versions = []
 
         if not self.needs_split():

site/cds_rdm/inspire_harvester/utils.py

@@ -16,6 +16,7 @@
 
 
 def compare_metadata(a, b):
+    """Compare metadata based on id key only."""
     # If both are dicts
     if isinstance(a, dict) and isinstance(b, dict):
         # If both have an id → compare only the id

site/cds_rdm/inspire_harvester/writer.py

@@ -58,8 +58,15 @@ def _process_entry(self, stream_entry):
             op_type = self._route(stream_entry)
         except WriterError as e:
             error_message = f"Error while processing entry : {str(e)}."
+            import traceback
+            traceback.print_exc()
         except ValidationError as e:
             error_message = f"Validation error while processing entry: {str(e)}."
+            import traceback
+            traceback.print_exc()
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
 
         if error_message:
             logger.error(error_message)

site/tests/conftest.py

@@ -325,6 +325,7 @@ def create_app(instance_path):
         "funders_v",
         "awards_v",
         "licenses_v",
+        "affiliations_v",
         "contributors_role_v",
         "description_type_v",
         "relation_type_v",
@@ -348,6 +349,7 @@ def running_app(
     funders_v,
     awards_v,
     licenses_v,
+    affiliations_v,
     contributors_role_v,
     description_type_v,
     relation_type_v,
@@ -372,6 +374,7 @@ def running_app(
         funders_v,
         awards_v,
         licenses_v,
+        affiliations_v,
         contributors_role_v,
         description_type_v,
         relation_type_v,
@@ -1249,6 +1252,40 @@ def licenses_v(app, licenses):
     return [cc_zero, cc_by]
 
 
+@pytest.fixture(scope="module")
+def affiliations_v(app):
+    """Affiliation records with ROR IDs from test data.
+
+    Affiliations are a dedicated record type (not a generic Vocabulary),
+    so they must be created via the affiliations service, not vocabulary_service.
+    The RDMDraft relation field validates against the Affiliation PID table.
+    """
+    from invenio_vocabularies.contrib.affiliations.api import Affiliation
+
+    affiliations_service = current_service_registry.get("affiliations")
+    entries = [
+        ("03gc1p724", "IJCLab, Orsay", "https://ror.org/03gc1p724"),
+        ("01ggx4157", "CERN", "https://ror.org/01ggx4157"),
+        ("013meh722", "University of Cambridge", "https://ror.org/013meh722"),
+        ("01a77tt86", "University of Warwick", "https://ror.org/01a77tt86"),
+        ("01v29qb04", "Durham University, IPPP", "https://ror.org/01v29qb04"),
+    ]
+    result = None
+    for ror_id, name, ror_url in entries:
+        result = affiliations_service.create(
+            system_identity,
+            {
+                "id": ror_id,
+                "name": name,
+                "identifiers": [{"identifier": ror_url, "scheme": "ror"}],
+            },
+        )
+
+    Affiliation.index.refresh()
+
+    return result
+
+
 @pytest.fixture(scope="module")
 def contributors_role_type(app):
     """Contributor role vocabulary type."""

site/tests/inspire_harvester/test_transformer.py

@@ -433,7 +433,7 @@ def test_transform_author_identifiers(running_app):
 
 
 def test_transform_author_affiliations(running_app):
-    """Test CreatibutorsMapper._transform_author_affiliations."""
+    """Test CreatibutorsMapper._transform_author_affiliations without ROR identifiers."""
     author = {
         "affiliations": [
             {"value": "CERN"},
@@ -450,6 +450,52 @@ def test_transform_author_affiliations(running_app):
     assert {"name": "MIT"} in result
 
 
+def test_transform_author_affiliations_with_ror(running_app):
+    """Test CreatibutorsMapper._transform_author_affiliations with ROR identifiers."""
+    author = {
+        "affiliations": [
+            {"value": "IJCLab, Orsay"},
+            {"value": "CERN"},
+        ],
+        "affiliations_identifiers": [
+            {"value": "grid.460789.4", "schema": "GRID"},
+            {"value": "grid.9132.9", "schema": "GRID"},
+            {"value": "https://ror.org/03gc1p724", "schema": "ROR"},
+            {"value": "https://ror.org/01ggx4157", "schema": "ROR"},
+        ],
+    }
+
+    mapper = CreatibutorsMapper()
+    result = mapper._transform_author_affiliations(author)
+
+    assert len(result) == 2
+    assert {"id": "03gc1p724"} in result
+    assert {"id": "01ggx4157"} in result
+
+
+def test_transform_author_affiliations_with_ror_partial(running_app):
+    """Test fallback to name when ROR count is less than affiliations count."""
+    author = {
+        "affiliations": [
+            {"value": "IJCLab, Orsay"},
+            {"value": "CERN"},
+            {"value": "Unknown Institute"},
+        ],
+        "affiliations_identifiers": [
+            {"value": "https://ror.org/03gc1p724", "schema": "ROR"},
+            {"value": "https://ror.org/01ggx4157", "schema": "ROR"},
+        ],
+    }
+
+    mapper = CreatibutorsMapper()
+    result = mapper._transform_author_affiliations(author)
+
+    assert len(result) == 3
+    assert {"id": "03gc1p724"} in result
+    assert {"id": "01ggx4157"} in result
+    assert {"name": "Unknown Institute"} in result
+
+
 def test_transform_copyrights_complete(running_app):
     """Test CopyrightMapper with complete copyright info."""
     src_metadata = {