Add utilities to facilitate the handling of the different I/O formats and related conversions

doc/MarlinWrapperIntroduction.md

@@ -227,6 +227,71 @@ unconditionally. Hence, if it's missing the conversion will fail. Make sure to
 read it in, or create it on the fly.
 ```
 
+## Tools for facilitating interoperability
+Mixed reconstruction and analysis chains (i.e. chains that have wrapped Marlin
+processors and native Gaudi algorithms) come with some challenges, e.g. the need
+to handle LCIO and EDM4hep inputs and outputs transparently and making sure that
+all the necessary conversions are done at the appropriate places. To facilitate
+this k4MarlinWrapper provides some python tools to automate parts of this.
+
+Most importantly the `IOHandlerHelper` class can be used to create and insert
+the correct readers and writers as well as injecting converters at the necessary
+places. We recommend using it if you need to support different input and output
+formats. The basic setup looks like this:
+
+```python
+from Configurables import EventDataSvc
+from k4FWCore import IOSvc
+from k4MarlinWrapper.io_helpers import IOHandlerHelper
+
+alg_list = []
+evt_svc = EventDataSvc("EventDataSvc")
+svc_list = [evt_svc]
+io_svc = IOSvc()
+
+io_handler = IOHandlerHelper(alg_list, io_svc)
+# Create an appropriate reader for the input files
+io_handler.add_reader(input_files)
+```
+
+This will either add an EDM4hep reader to the `IOSvc` (at the very beginning of
+the reconstruction chain) or add an `LcioEvent` algorithm at the current place
+in the `alg_list`. You can now simply add all the algorithms
+(`MarlinProcessorWrapper` or native Gaudi algorithms) as usual to the
+`alg_list`.
+
+For adding EDm4hep output simply do
+```python
+io_handler.add_edm4hep_writer("output.edm4hep.root")
+```
+
+which will create an output file `output.edm4hep.root` and use the `["keep *"]`
+as `IOSvc.outputCommands`. The latter can be changed by passing a second
+argument.
+
+For adding LCIO output you need to
+```python
+lcio_writer = io_handler.add_lcio_writer("LCIOWriter")
+lcio_writer.Parameters = {
+    "LCIOOutputFile": ["output.slcio"],
+    "LCIOWriteMode": ["WRITE_NEW"]
+}
+```
+
+Note that in this case the `lcio_writer` is not yet fully configured but simply
+added to the chain of algorithms to execute.
+
+Finally, to insert all the necessary converters simply call
+`finalize_converters` before handing off the algorithm list to the
+`ApplicationMgr`, i.e.
+
+```python
+io_handler.finalize_converters()
+
+from k4FWCore import ApplicationMgr
+ApplicationMgr(TopAlg=alg_list, EvtSel="NONE", ExtSvc=[evt_svc])
+```
+
 ## Potential pitfalls when using other Gaudi Algorithms
 
 Although mixing wrapped Marlin Processors with other Gaudi Algorithms is working

k4MarlinWrapper/examples/event_display.py

@@ -19,9 +19,10 @@
 #
 
 from Gaudi.Configuration import INFO
-from Configurables import MarlinProcessorWrapper, k4DataSvc, GeoSvc
+from Configurables import MarlinProcessorWrapper, EventDataSvc, GeoSvc
+from k4FWCore import ApplicationMgr, IOSvc
 from k4FWCore.parseArgs import parser
-from k4MarlinWrapper.inputReader import create_reader, attach_edm4hep2lcio_conversion
+from k4MarlinWrapper.io_helpers import IOHandlerHelper
 
 
 parser.add_argument(
@@ -42,25 +43,18 @@
 reco_args = parser.parse_known_args()[0]
 
 algList = []
-svcList = []
-
-evtsvc = k4DataSvc("EventDataSvc")
-svcList.append(evtsvc)
+svcList = [EventDataSvc("EventDataSvc")]
 
+iosvc = IOSvc()
 
 geoSvc = GeoSvc("GeoSvc")
 geoSvc.detectors = [reco_args.compactFile]
 geoSvc.OutputLevel = INFO
 geoSvc.EnableGeant4Geo = False
 svcList.append(geoSvc)
 
-
-if reco_args.inputFiles:
-    read = create_reader(reco_args.inputFiles, evtsvc)
-    read.OutputLevel = INFO
-    algList.append(read)
-else:
-    read = None
+io_handler = IOHandlerHelper(algList, iosvc)
+io_handler.add_reader(reco_args.inputFiles)
 
 MyCEDViewer = MarlinProcessorWrapper("MyCEDViewer")
 MyCEDViewer.OutputLevel = INFO
@@ -527,8 +521,6 @@
 algList.append(MyCEDViewer)
 
 # We need to convert the inputs in case we have EDM4hep input
-attach_edm4hep2lcio_conversion(algList, read)
-
-from Configurables import ApplicationMgr
+io_handler.finalize_converters()
 
 ApplicationMgr(TopAlg=algList, EvtSel="NONE", EvtMax=10, ExtSvc=svcList, OutputLevel=INFO)

k4MarlinWrapper/python/k4MarlinWrapper/inputReader.py

@@ -18,7 +18,21 @@
 #
 
 import sys
-from Configurables import LcioEvent, PodioInput, MarlinProcessorWrapper, EDM4hep2LcioTool
+import warnings
+
+warnings.warn(
+    "The 'k4MarlinWrapper.inputReader' module is deprecated and will be removed in a future version. "
+    "Please use the utilities in `k4MarlinWrapper.io_helpers` instead.",
+    category=DeprecationWarning,
+    stacklevel=2,
+)
+
+from Configurables import (
+    LcioEvent,
+    PodioInput,
+    MarlinProcessorWrapper,
+    EDM4hep2LcioTool,
+)
 
 
 def create_reader(input_files, evtSvc):

k4MarlinWrapper/python/k4MarlinWrapper/io_helpers.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2019-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import logging
+
+from Configurables import (
+    LcioEvent,
+    MarlinProcessorWrapper,
+    EDM4hep2LcioTool,
+    Lcio2EDM4hepTool,
+)
+
+logger = logging.getLogger()
+
+
+def _is_wrapped_proc_without_conv(alg, from_edm, to_edm):
+    """Check if this algorithm has a configured (i.e. named) converter attached
+    for the direction described by from_edm and to_edm"""
+    if isinstance(alg, MarlinProcessorWrapper):
+        if not getattr(alg, f"{from_edm}2{to_edm}Tool").getName():
+            return True
+
+    return False
+
+
+class IOHandlerHelper:
+    """Helper class to facilitate the transparent handling of LCIO or EDM4hep
+    inputs and outputs.
+
+    This class allows to
+    - add the correct reader as determined from the input file name(s)
+    - add (multiple) LCIO writers
+    - add an EDM4hep writer
+    - make sure that the necessary converters are introduced at the right places
+    """
+
+    def __init__(self, alg_list, io_svc):
+        """Create a IOHandlerHelper
+
+        Args:
+            alg_list (list): The algorithm list which is being populated
+            io_svc (IOSvc): The IOSvc that is being used for this run
+        """
+        self._alg_list = alg_list
+        self._io_svc = io_svc
+        self._lcio_input = False
+        self._edm4hep_output = False
+
+    def add_reader(self, input_files):
+        """Add a reader that is equipped to read the passed files
+
+        If the input is LCIO the necessary algorithm will be configured and
+        added to the list of algorithms at the current spot. If the input is
+        EDM4hep the file names will be passed to the IOSvc.Input
+
+        Args:
+            input_files (list): The input files that should be read
+        """
+        if input_files[0].endswith(".slcio"):
+            if any(not f.endswith(".slcio") for f in input_files):
+                logger.error("All input files need to have the same format (LCIO)")
+                sys.exit(1)
+
+            read = LcioEvent()
+            read.Files = input_files
+            self._alg_list.append(read)
+            self._lcio_input = True
+        else:
+            if any(not f.endswith(".root") for f in input_files):
+                logger.error("All input files need to have the same format (EDM4hep)")
+                sys.exit(1)
+
+            self._io_svc.Input = input_files
+
+    def add_lcio_writer(self, alg_name):
+        """Add a writer for LCIO output at the current spot in the algorithm list
+
+        Note:
+            This doesn't configure anything yet, that is still left to do outside
+
+        Args:
+            alg_name (str): The name this writer should have
+
+        Returns:
+            MarlinProcessorWrapper: The wrapped processor that has just been
+                inserted into the algorithm list and that now needs to be
+                further configured
+        """
+        writer = MarlinProcessorWrapper(alg_name, ProcessorType="LCIOOutputProcessor")
+        self._alg_list.append(writer)
+        return writer
+
+    def add_edm4hep_writer(self, output_file, output_cmds=["keep *"]):
+        """Add an EDM4hep writer at the very end of the algorithm execution
+
+        This will pass the output file name as well as the output commands to
+        the IOSvc.Ouptut and IOSvc.outputCommands respectively
+
+        Args:
+            output_file (str): The name of the output file
+
+            output_cmds (list, optional): The list of output commands that
+                should be applied. Defaults to ["keep *"]
+        """
+        self._io_svc.Output = output_file
+        self._io_svc.outputCommands = output_cmds
+        self._edm4hep_output = True
+
+    def finalize_converters(self):
+        """Attach the appropriate converters in all places they are necessary
+
+        Go through the algorithm list and determine where appropriate converters
+        need to be inserted such that the algorithms or wrapped processors always
+        see a consistent picture of the event in both formats. Basically what
+        this does is to go through the complete list of algorithms and introduce
+        an LCIO to EDM4hep converter at the start of every run of
+        MarlinProcessorWrappers and in the other direction at the end of every
+        such run
+
+        Note:
+            Call this just before you pass the algorithm list to the ApplicationMgr
+
+        Note:
+            This will not change existing converters on wrapped processors
+        """
+        for alg, next_alg in zip(self._alg_list, self._alg_list[1:]):
+            if not isinstance(next_alg, MarlinProcessorWrapper) and _is_wrapped_proc_without_conv(
+                alg, "Lcio", "EDM4hep"
+            ):
+                # We change from a run of wrapped processors to algorithms and
+                # we don't have a converter yet
+                output_conv = Lcio2EDM4hepTool(f"{alg.getName()}_OutputConverter")
+                output_conv.convertAll = True
+                output_conv.collNameMapping = {"MCParticle": "MCParticles"}
+                alg.Lcio2EDM4hepTool = output_conv
+                logger.info(
+                    f"Added an output converter (LCIO to EDM4hep) to the {alg.getName()} algorithm"
+                )
+
+            if not isinstance(
+                alg, (MarlinProcessorWrapper, LcioEvent)
+            ) and _is_wrapped_proc_without_conv(next_alg, "EDM4hep", "Lcio"):
+                # We change from a run of algorithms to wrapped processors and
+                # we do not have a converter yet
+                input_conv = EDM4hep2LcioTool(f"{next_alg.getName()}_InputConverter")
+                input_conv.convertAll = True
+                input_conv.collNameMapping = {"MCParticles": "MCParticle"}
+                next_alg.EDM4hep2LcioTool = input_conv
+                logger.info(
+                    f"Added an input converter (EDM4hep to LCIO) to the {next_alg.getName()} algorithm"
+                )
+
+        if not self._lcio_input:
+            # We need to convert the input to LCIO from EDM4hep. We attach this
+            # to the first wrapped processor that does NOT have another converter
+            # configured
+            for alg in self._alg_list:
+                if _is_wrapped_proc_without_conv(alg, "EDM4hep", "Lcio"):
+                    input_conv = EDM4hep2LcioTool("InputConversion")
+                    input_conv.convertAll = True
+                    input_conv.collNameMapping = {"MCParticles": "MCParticle"}
+                    alg.EDM4hep2LcioTool = input_conv
+                    logger.info(
+                        f"Added an input converter (EDM4hep to LCIO) to the {alg.getName()} algorithm"
+                    )
+                    break
+
+        if self._edm4hep_output:
+            # We need to convert to EDM4hep. We attach the converter to the last
+            # wrapped processor that does not have another converter attached
+            for alg in reversed(self._alg_list):
+                if _is_wrapped_proc_without_conv(alg, "Lcio", "EDM4hep"):
+                    output_conv = Lcio2EDM4hepTool("OutputConverter")
+                    output_conv.convertAll = True
+                    output_conv.collNameMapping = {"MCParticle": "MCParticles"}
+                    alg.Lcio2EDM4hepTool = output_conv
+                    logger.info(
+                        f"Added an output converter (LCIO to EDM4hep) to the {alg.getName()} algorithm"
+                    )
+                    break

test/CMakeLists.txt

@@ -43,6 +43,7 @@ find_package(Marlin REQUIRED)
 add_library(MarlinTestProcessors SHARED
   src/TrivialMCTruthLinkerProcessor.cc
   src/MarlinMCRecoLinkChecker.cc
+  src/PseudoRecoProcessor.cc
 )
 target_link_libraries(MarlinTestProcessors PUBLIC ${Marlin_LIBRARIES})
 target_include_directories(MarlinTestProcessors PUBLIC ${Marlin_INCLUDE_DIRS})
@@ -139,6 +140,42 @@ ExternalData_Add_Test( marlinwrapper_tests
   COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_link_conversion_edm4hep.py --use-gaudi-algorithm --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}
 )
 
+# ---- IO Helpers tests
+ExternalData_Add_Test( marlinwrapper_tests
+  NAME io_helpers_lcio_marlin_only
+  COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_io_utilities.py --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/muons.slcio} --output-type lcio --outputbase lcio_marlin_only
+)
+
+ExternalData_Add_Test( marlinwrapper_tests
+  NAME io_helpers_edm4hep_marlin_only
+  COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_io_utilities.py --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --output-type edm4hep --outputbase edm4hep_marlin_only
+)
+
+ExternalData_Add_Test( marlinwrapper_tests
+  NAME io_helpers_lcio_in_edm4hep_out_marlin_only
+  COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_io_utilities.py --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/muons.slcio} --output-type edm4hep --outputbase lcio_in_edm4hep_out_marlin_only
+)
+
+ExternalData_Add_Test( marlinwrapper_tests
+  NAME io_helpers_edm4hep_in_lcio_out_marlin_only
+  COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_io_utilities.py --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --output-type lcio --outputbase edm4hep_in_lcio_out_marlin_only
+)
+
+ExternalData_Add_Test( marlinwrapper_tests
+  NAME io_helpers_lcio_marlin_gaudi
+  COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_io_utilities.py --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/muons.slcio} --with-gaudi-algorithm --output-type lcio --outputbase lcio_marlin_gaudi
+)
+
+ExternalData_Add_Test( marlinwrapper_tests
+  NAME io_helpers_edm4hep_marlin_gaudi
+  COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_io_utilities.py --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --with-gaudi-algorithm --output-type edm4hep --outputbase edm4hep_marlin_gaudi
+)
+
+ExternalData_Add_Test( marlinwrapper_tests
+  NAME io_helpers_edm4hep_both_out_marlin_gaudi
+  COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_io_utilities.py --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --with-gaudi-algorithm --output-type both --outputbase edm4hep_both_out_marlin_gaudi
+)
+
 add_test( event_header_conversion bash -c "k4run ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/createEventHeader.py && anajob test.slcio | grep 'EVENT: 42'" )
 
 ExternalData_Add_Target(marlinwrapper_tests)