From 9acbf153f7f2c13a8ad47edc12ef83554d622a7e Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 13 Jan 2026 11:08:51 +0100 Subject: [PATCH 01/13] added static method to merge list of SubarrayDescription --- src/ctapipe/instrument/subarray.py | 75 +++++++++++++++++++ src/ctapipe/instrument/tests/test_subarray.py | 62 +++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/src/ctapipe/instrument/subarray.py b/src/ctapipe/instrument/subarray.py index 9e22fe97964..945182b9316 100644 --- a/src/ctapipe/instrument/subarray.py +++ b/src/ctapipe/instrument/subarray.py @@ -808,3 +808,78 @@ def check_matching_subarrays(subarray_list: list) -> bool: set(subarray.tel_ids) == set(subarray_list[0].tel_ids) for subarray in subarray_list ) + + @staticmethod + def merge_subarrays( + subarray_list: list, + name=None, + overwrite_tel_ids: bool = False, + ) -> "SubarrayDescription": + """Merge multiple subarrays into one + + Parameters + ---------- + subarray_list: list(SubarrayDescription) + list of subarrays to merge + name: str + name of new merged subarray + overwrite_tel_ids: bool + if True, telescope entries from later subarrays replace earlier ones; + if False (default), encountering a duplicate telescope id raises a + ValueError. + Returns + ------- + SubarrayDescription + + """ + + tel_positions, tel_descriptions = {}, {} + tel_ids = set() # To collect unique telescope IDs + reference_location = subarray_list[0].reference_location # Get the reference location from the first subarray + for s, subarray in enumerate(subarray_list): + if not isinstance(subarray, SubarrayDescription): + raise TypeError( + "All elements of subarray_list must be 'SubarrayDescription' " + f"instances, got '{type(subarray)}' for element '{s}'." + ) + for tid in subarray.tel_ids: + if tid in tel_ids: + if overwrite_tel_ids: + # Warn about overwriting telescope entry + msg = ( + f"Overwriting telescope id '{tid}' from subarray " + f"'{subarray.name}' into merged subarray." + ) + warnings.warn(msg, UserWarning) + else: + raise ValueError( + "Duplicate telescope id encountered while merging subarrays. " + f"Telescope '{tid}' already defined; set overwrite_tel_ids=True to " + "allow later subarrays to replace earlier entries." + ) + tel_ids.add(tid) + if subarray.reference_location != reference_location: + raise ValueError( + "All subarrays must have the same reference_location to be merged. " + f"Subarray '{subarray.name}' ({subarray.reference_location}) does not match " + f"the reference location of the first subarray '{subarray_list[0].name}' " + f"({reference_location})." + ) + + # Merge telescope positions and descriptions, optionally allowing later entries to overwrite + for subarray in subarray_list: + for tid in subarray.tel_ids: + # Copy/overwrite telescope position and description from the current subarray + tel_positions[tid] = subarray.positions[tid] + tel_descriptions[tid] = subarray.tels[tid] + + if not name: + name = "Merged_" + _range_extraction(tel_ids) + + newsub = SubarrayDescription( + name, + tel_positions=tel_positions, + tel_descriptions=tel_descriptions, + reference_location=reference_location, + ) + return newsub diff --git a/src/ctapipe/instrument/tests/test_subarray.py b/src/ctapipe/instrument/tests/test_subarray.py index 95df7f1fdee..3ddd927f426 100644 --- a/src/ctapipe/instrument/tests/test_subarray.py +++ b/src/ctapipe/instrument/tests/test_subarray.py @@ -313,3 +313,65 @@ def test_check_matchings_subarray(example_subarray, subarray_prod5_paranal): assert not SubarrayDescription.check_matching_subarrays( [example_subarray, subarray_prod5_paranal] ) + + +def test_merge_subarrays(example_subarray): + """Test SubarrayDescription.merge_subarrays static method""" + + sub1 = example_subarray.select_subarray([1, 2], name="s1") + sub2 = example_subarray.select_subarray([3, 4], name="s2") + expected_sub = example_subarray.select_subarray([1, 2, 3, 4], name="Merged_1-4") + merged_sub = SubarrayDescription.merge_subarrays([sub1, sub2]) + + assert expected_sub.__eq__(merged_sub) + + +def test_merge_subarrays_exceptions(example_subarray): + """Merging subarrays with invalid parameters should raise exceptions.""" + + sub1 = example_subarray.select_subarray([1, 2], name="s1") + sub2 = example_subarray.select_subarray([3, 4], name="s2") + + # Check that invalid inputs raise exceptions + with pytest.raises(TypeError, match="All elements of subarray_list must be 'SubarrayDescription'"): + SubarrayDescription.merge_subarrays([sub1, int(67)]) + + # Check that duplicate telescope ids without overwrite raises exception + with pytest.raises(ValueError, match="Duplicate telescope id encountered"): + SubarrayDescription.merge_subarrays([sub1, sub1], overwrite_tel_ids=False) + + # Check that different reference locations raises exception + shifted_location = EarthLocation( + lon=sub2.reference_location.lon, + lat=sub2.reference_location.lat, + height=sub2.reference_location.height + 1 * u.m, + ) + sub2_shifted = SubarrayDescription( + name="shifted", + tel_positions=sub2.positions, + tel_descriptions=sub2.tel, + reference_location=shifted_location, + ) + with pytest.raises(ValueError, match="All subarrays must have the same reference_location"): + SubarrayDescription.merge_subarrays([sub1, sub2_shifted], overwrite_tel_ids=True) + + +def test_merge_subarrays_overwrite_tel_ids(example_subarray): + """Later subarrays overwrite earlier telescope entries when enabled.""" + + sub1 = example_subarray.select_subarray([1, 2], name="s1") + sub2 = example_subarray.select_subarray([1, 2], name="s2") + + # Overwrite positions of the telescopes in sub2 + sub2.positions[1] = np.array([10, 0, 0]) * u.m + sub2.positions[2] = np.array([-10, 0, 0]) * u.m + + with pytest.warns(UserWarning, match="Overwriting telescope id"): + merged_sub = SubarrayDescription.merge_subarrays( + [sub1, sub2], overwrite_tel_ids=True + ) + + assert merged_sub.name == "Merged_1,2" + assert merged_sub.n_tels == 2 + np.testing.assert_allclose(merged_sub.positions[1], [10, 0, 0] * u.m) + np.testing.assert_allclose(merged_sub.positions[2], [-10, 0, 0] * u.m) \ No newline at end of file From 0316e7438de58f02d2301f4bfeab9fed3e82267e Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 13 Jan 2026 11:36:40 +0100 Subject: [PATCH 02/13] added some instrument paths to ctapipe.io.hdf5dataformat --- src/ctapipe/instrument/subarray.py | 29 ++++++++++++++++++++--------- src/ctapipe/io/hdf5dataformat.py | 11 +++++++++++ 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/ctapipe/instrument/subarray.py b/src/ctapipe/instrument/subarray.py index 945182b9316..9b8dce79472 100644 --- a/src/ctapipe/instrument/subarray.py +++ b/src/ctapipe/instrument/subarray.py @@ -614,12 +614,18 @@ def to_hdf(self, h5file, overwrite=False, mode="a"): """ # here to prevent circular import from ..io import write_table + from ..io.hdf5dataformat import ( + CONFIG_INSTRUMENT_SUBARRAY, + CONFIG_INSTRUMENT_SUBARRAY_LAYOUT, + CONFIG_INSTRUMENT_TEL_OPTICS, + CONFIG_INSTRUMENT_TEL_CAMERA, + ) with ExitStack() as stack: if not isinstance(h5file, tables.File): h5file = stack.enter_context(tables.open_file(h5file, mode=mode)) - if "/configuration/instrument/subarray" in h5file.root and not overwrite: + if CONFIG_INSTRUMENT_SUBARRAY in h5file.root and not overwrite: raise OSError( "File already contains a SubarrayDescription and overwrite=False" ) @@ -630,26 +636,26 @@ def to_hdf(self, h5file, overwrite=False, mode="a"): write_table( subarray_table, h5file, - path="/configuration/instrument/subarray/layout", + path=CONFIG_INSTRUMENT_SUBARRAY_LAYOUT, overwrite=overwrite, ) write_table( self.to_table(kind="optics"), h5file, - path="/configuration/instrument/telescope/optics", + path=CONFIG_INSTRUMENT_TEL_OPTICS, overwrite=overwrite, ) for i, camera in enumerate(self.camera_types): write_table( camera.geometry.to_table(), h5file, - path=f"/configuration/instrument/telescope/camera/geometry_{i}", + path=f"{CONFIG_INSTRUMENT_TEL_CAMERA}/geometry_{i}", overwrite=overwrite, ) write_table( camera.readout.to_table(), h5file, - path=f"/configuration/instrument/telescope/camera/readout_{i}", + path=f"{CONFIG_INSTRUMENT_TEL_CAMERA}/readout_{i}", overwrite=overwrite, ) @@ -657,12 +663,17 @@ def to_hdf(self, h5file, overwrite=False, mode="a"): def from_hdf(cls, path, focal_length_choice=FocalLengthKind.EFFECTIVE): # here to prevent circular import from ..io import read_table + from ..io.hdf5dataformat import ( + CONFIG_INSTRUMENT_SUBARRAY_LAYOUT, + CONFIG_INSTRUMENT_TEL_OPTICS, + CONFIG_INSTRUMENT_TEL_CAMERA, + ) if isinstance(focal_length_choice, str): focal_length_choice = FocalLengthKind[focal_length_choice.upper()] layout = read_table( - path, "/configuration/instrument/subarray/layout", table_cls=QTable + path, CONFIG_INSTRUMENT_SUBARRAY_LAYOUT, table_cls=QTable ) version = layout.meta.get("TAB_VER") @@ -678,12 +689,12 @@ def from_hdf(cls, path, focal_length_choice=FocalLengthKind.EFFECTIVE): for idx in set(layout["camera_index"]): geometry = CameraGeometry.from_table( read_table( - path, f"/configuration/instrument/telescope/camera/geometry_{idx}" + path, f"{CONFIG_INSTRUMENT_TEL_CAMERA}/geometry_{idx}" ) ) readout = CameraReadout.from_table( read_table( - path, f"/configuration/instrument/telescope/camera/readout_{idx}" + path, f"{CONFIG_INSTRUMENT_TEL_CAMERA}/readout_{idx}" ) ) cameras[idx] = CameraDescription( @@ -691,7 +702,7 @@ def from_hdf(cls, path, focal_length_choice=FocalLengthKind.EFFECTIVE): ) optics_table = read_table( - path, "/configuration/instrument/telescope/optics", table_cls=QTable + path, CONFIG_INSTRUMENT_TEL_OPTICS, table_cls=QTable ) optics_version = optics_table.meta.get("TAB_VER") diff --git a/src/ctapipe/io/hdf5dataformat.py b/src/ctapipe/io/hdf5dataformat.py index 0e24fc35372..f095abb8d10 100644 --- a/src/ctapipe/io/hdf5dataformat.py +++ b/src/ctapipe/io/hdf5dataformat.py @@ -9,6 +9,11 @@ "SIMULATION_GROUP", "SIMULATION_TEL_TABLE", "CONFIG_GROUP", + "CONFIG_INSTRUMENT_SUBARRAY", + "CONFIG_INSTRUMENT_SUBARRAY_LAYOUT", + "CONFIG_INSTRUMENT_TEL", + "CONFIG_INSTRUMENT_TEL_OPTICS", + "CONFIG_INSTRUMENT_TEL_CAMERA", "SCHEDULING_BLOCK_TABLE", "OBSERVATION_BLOCK_TABLE", "SIMULATION_RUN_TABLE", @@ -60,10 +65,16 @@ # Configuration, service, and simulation group CONFIG_GROUP = "/configuration" +CONFIG_INSTRUMENT_SUBARRAY = "/configuration/instrument/subarray" +CONFIG_INSTRUMENT_SUBARRAY_LAYOUT = "/configuration/instrument/subarray/layout" +CONFIG_INSTRUMENT_TEL = "/configuration/instrument/telescope" +CONFIG_INSTRUMENT_TEL_OPTICS = "/configuration/instrument/telescope/optics" +CONFIG_INSTRUMENT_TEL_CAMERA = "/configuration/instrument/telescope/camera" SCHEDULING_BLOCK_TABLE = "/configuration/observation/scheduling_block" OBSERVATION_BLOCK_TABLE = "/configuration/observation/observation_block" SIMULATION_RUN_TABLE = "/configuration/simulation/run" FIXED_POINTING_GROUP = "/configuration/telescope/pointing" + DL1_IMAGE_STATISTICS_TABLE = "/dl1/service/image_statistics" DL2_EVENT_STATISTICS_GROUP = "/dl2/service/tel_event_statistics" SIMULATION_GROUP = "/simulation" From ac7b9050eaeb70add499f2fed35c452febbb64a1 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 13 Jan 2026 15:53:32 +0100 Subject: [PATCH 03/13] added 'combine-telescope-events' merging strategy to combine telescope-wise data from the same OB --- src/ctapipe/instrument/subarray.py | 26 +-- src/ctapipe/instrument/tests/test_subarray.py | 14 +- src/ctapipe/io/hdf5merger.py | 175 +++++++++++++++--- src/ctapipe/tools/merge.py | 4 + src/ctapipe/tools/tests/test_merge.py | 69 +++++++ 5 files changed, 244 insertions(+), 44 deletions(-) diff --git a/src/ctapipe/instrument/subarray.py b/src/ctapipe/instrument/subarray.py index 9b8dce79472..550a6692b91 100644 --- a/src/ctapipe/instrument/subarray.py +++ b/src/ctapipe/instrument/subarray.py @@ -617,8 +617,8 @@ def to_hdf(self, h5file, overwrite=False, mode="a"): from ..io.hdf5dataformat import ( CONFIG_INSTRUMENT_SUBARRAY, CONFIG_INSTRUMENT_SUBARRAY_LAYOUT, - CONFIG_INSTRUMENT_TEL_OPTICS, CONFIG_INSTRUMENT_TEL_CAMERA, + CONFIG_INSTRUMENT_TEL_OPTICS, ) with ExitStack() as stack: @@ -665,16 +665,14 @@ def from_hdf(cls, path, focal_length_choice=FocalLengthKind.EFFECTIVE): from ..io import read_table from ..io.hdf5dataformat import ( CONFIG_INSTRUMENT_SUBARRAY_LAYOUT, - CONFIG_INSTRUMENT_TEL_OPTICS, CONFIG_INSTRUMENT_TEL_CAMERA, + CONFIG_INSTRUMENT_TEL_OPTICS, ) if isinstance(focal_length_choice, str): focal_length_choice = FocalLengthKind[focal_length_choice.upper()] - layout = read_table( - path, CONFIG_INSTRUMENT_SUBARRAY_LAYOUT, table_cls=QTable - ) + layout = read_table(path, CONFIG_INSTRUMENT_SUBARRAY_LAYOUT, table_cls=QTable) version = layout.meta.get("TAB_VER") if version not in cls.COMPATIBLE_VERSIONS: @@ -688,22 +686,16 @@ def from_hdf(cls, path, focal_length_choice=FocalLengthKind.EFFECTIVE): for idx in set(layout["camera_index"]): geometry = CameraGeometry.from_table( - read_table( - path, f"{CONFIG_INSTRUMENT_TEL_CAMERA}/geometry_{idx}" - ) + read_table(path, f"{CONFIG_INSTRUMENT_TEL_CAMERA}/geometry_{idx}") ) readout = CameraReadout.from_table( - read_table( - path, f"{CONFIG_INSTRUMENT_TEL_CAMERA}/readout_{idx}" - ) + read_table(path, f"{CONFIG_INSTRUMENT_TEL_CAMERA}/readout_{idx}") ) cameras[idx] = CameraDescription( name=geometry.name, readout=readout, geometry=geometry ) - optics_table = read_table( - path, CONFIG_INSTRUMENT_TEL_OPTICS, table_cls=QTable - ) + optics_table = read_table(path, CONFIG_INSTRUMENT_TEL_OPTICS, table_cls=QTable) optics_version = optics_table.meta.get("TAB_VER") if optics_version not in OpticsDescription.COMPATIBLE_VERSIONS: @@ -841,12 +833,14 @@ def merge_subarrays( Returns ------- SubarrayDescription - + """ tel_positions, tel_descriptions = {}, {} tel_ids = set() # To collect unique telescope IDs - reference_location = subarray_list[0].reference_location # Get the reference location from the first subarray + reference_location = subarray_list[ + 0 + ].reference_location # Get the reference location from the first subarray for s, subarray in enumerate(subarray_list): if not isinstance(subarray, SubarrayDescription): raise TypeError( diff --git a/src/ctapipe/instrument/tests/test_subarray.py b/src/ctapipe/instrument/tests/test_subarray.py index 3ddd927f426..a6cb63b8db7 100644 --- a/src/ctapipe/instrument/tests/test_subarray.py +++ b/src/ctapipe/instrument/tests/test_subarray.py @@ -333,7 +333,9 @@ def test_merge_subarrays_exceptions(example_subarray): sub2 = example_subarray.select_subarray([3, 4], name="s2") # Check that invalid inputs raise exceptions - with pytest.raises(TypeError, match="All elements of subarray_list must be 'SubarrayDescription'"): + with pytest.raises( + TypeError, match="All elements of subarray_list must be 'SubarrayDescription'" + ): SubarrayDescription.merge_subarrays([sub1, int(67)]) # Check that duplicate telescope ids without overwrite raises exception @@ -352,8 +354,12 @@ def test_merge_subarrays_exceptions(example_subarray): tel_descriptions=sub2.tel, reference_location=shifted_location, ) - with pytest.raises(ValueError, match="All subarrays must have the same reference_location"): - SubarrayDescription.merge_subarrays([sub1, sub2_shifted], overwrite_tel_ids=True) + with pytest.raises( + ValueError, match="All subarrays must have the same reference_location" + ): + SubarrayDescription.merge_subarrays( + [sub1, sub2_shifted], overwrite_tel_ids=True + ) def test_merge_subarrays_overwrite_tel_ids(example_subarray): @@ -374,4 +380,4 @@ def test_merge_subarrays_overwrite_tel_ids(example_subarray): assert merged_sub.name == "Merged_1,2" assert merged_sub.n_tels == 2 np.testing.assert_allclose(merged_sub.positions[1], [10, 0, 0] * u.m) - np.testing.assert_allclose(merged_sub.positions[2], [-10, 0, 0] * u.m) \ No newline at end of file + np.testing.assert_allclose(merged_sub.positions[2], [-10, 0, 0] * u.m) diff --git a/src/ctapipe/io/hdf5merger.py b/src/ctapipe/io/hdf5merger.py index 85d96f23eed..58684602874 100644 --- a/src/ctapipe/io/hdf5merger.py +++ b/src/ctapipe/io/hdf5merger.py @@ -4,7 +4,9 @@ from contextlib import ExitStack from pathlib import Path +import numpy as np import tables +from astropy.table import unique, vstack from astropy.time import Time from ..containers import EventType @@ -12,7 +14,7 @@ from ..instrument.optics import FocalLengthKind from ..instrument.subarray import SubarrayDescription from ..utils.arrays import recarray_drop_columns -from . import metadata +from . import metadata, read_table, write_table from .hdf5dataformat import ( DL0_TEL_POINTING_GROUP, DL1_CAMERA_COEFFICIENTS_GROUP, @@ -54,6 +56,8 @@ "v7.2.0", "v7.3.0", ] +SUBARRAY_EVENT_KEYS = ["obs_id", "event_id"] +TEL_EVENT_KEYS = ["obs_id", "event_id", "tel_id"] class NodeType(enum.Enum): @@ -181,13 +185,19 @@ class HDF5Merger(Component): ).tag(config=True) merge_strategy = traits.CaselessStrEnum( - ["events-multiple-obs", "events-single-ob", "monitoring-only"], + [ + "events-multiple-obs", + "events-single-ob", + "combine-telescope-events", + "monitoring-only", + ], default_value="events-multiple-obs", help=( "Strategy to handle different use cases when merging HDF5 files. " "'events-multiple-obs': allows merging event files (w and w/o monitoring data) from different observation blocks; " - "'events-single-ob': for merging events in consecutive chunks of the same OB." - "'monitoring-only': attaches horizontally monitoring data from the same observation block (requires monitoring=True)." + "'events-single-ob': for merging events in consecutive chunks of the same OB; " + "'combine-telescope-events': merges telescope-wise data from different files for the same OB (requires telescope_events=True); " + "'monitoring-only': attaches horizontally monitoring data from the same OB (requires monitoring=True)." ), ).tag(config=True) @@ -205,12 +215,20 @@ def __init__(self, output_path=None, **kwargs): self.single_ob = ( self.merge_strategy == "events-single-ob" or self.merge_strategy == "monitoring-only" + or self.merge_strategy == "combine-telescope-events" ) self.attach_monitoring = self.merge_strategy == "monitoring-only" if self.attach_monitoring and not self.monitoring: raise traits.TraitError( "Merge strategy 'monitoring-only' requires monitoring=True" ) + self.combine_telescope_events = ( + self.merge_strategy == "combine-telescope-events" + ) + if self.combine_telescope_events and not self.telescope_events: + raise traits.TraitError( + "Merge strategy 'combine-telescope-events' requires telescope_events=True" + ) output_exists = self.output_path.exists() appending = False @@ -232,6 +250,9 @@ def __init__(self, output_path=None, **kwargs): self.data_model_version = None self.data_category = None self.subarray = None + self.subarray_list = [] + self.tel_trigger_tables = [] + self.tel_ids_set = set() self.meta = None self._merged_obs_ids = set() self._n_merged = 0 @@ -249,12 +270,23 @@ def __init__(self, output_path=None, **kwargs): self.h5file, focal_length_choice=FocalLengthKind.EQUIVALENT, ) + self.subarray_list.append(self.subarray) + + # Update tel_ids_set with existing telescope IDs when appending + if self.combine_telescope_events: + self.tel_ids_set.update(self.subarray.tel_ids) # Get required nodes from existing output file self.required_nodes = self._get_required_nodes(self.h5file) # this will update _merged_obs_ids from existing input file self._check_obs_ids(self.h5file) + + # Append existing tel trigger tables if the merge strategy is 'combine-telescope-events' + if self.combine_telescope_events: + self.tel_trigger_tables.append( + read_table(self.h5file, DL1_TEL_TRIGGER_TABLE) + ) self._n_merged += 1 def __call__(self, other: str | Path | tables.File): @@ -376,8 +408,11 @@ def _get_required_nodes(self, h5file): if node not in h5file.root: continue - if node_type in (NodeType.TABLE, NodeType.TEL_GROUP): + if node_type is NodeType.TABLE: required_nodes.add(node) + elif node_type is NodeType.TEL_GROUP: + if not self.combine_telescope_events: + required_nodes.add(node) elif node_type is NodeType.ITER_GROUP: for kind_group in h5file.root[node]._f_iter_nodes("Group"): @@ -386,6 +421,9 @@ def _get_required_nodes(self, h5file): elif node_type is NodeType.ITER_TEL_GROUP: for kind_group in h5file.root[node]._f_iter_nodes("Group"): + if self.combine_telescope_events: + required_nodes.add(kind_group.name) + continue for iter_group in kind_group._f_iter_nodes("Group"): required_nodes.add(iter_group._v_pathname) else: @@ -402,7 +440,9 @@ def _append_simulation_data(self, other): ] for key in simulation_table_keys: if key in other.root: - self._append_table(other, other.root[key]) + self._append_table( + other, other.root[key], once=self.combine_telescope_events + ) if FIXED_POINTING_GROUP in other.root: self._append_table_group( @@ -437,15 +477,20 @@ def _append_waveform_data(self, other): def _append_dl1_data(self, other): """Append DL1 data (triggers, images, parameters, muon).""" - # DL1 subarray trigger table (always check) - if DL1_SUBARRAY_TRIGGER_TABLE in other.root: + if ( + DL1_SUBARRAY_TRIGGER_TABLE in other.root + and not self.combine_telescope_events + ): self._append_table(other, other.root[DL1_SUBARRAY_TRIGGER_TABLE]) if not self.telescope_events: return - if DL1_TEL_TRIGGER_TABLE in other.root: - self._append_table(other, other.root[DL1_TEL_TRIGGER_TABLE]) + if self.combine_telescope_events: + self.tel_trigger_tables.append(read_table(other, DL1_TEL_TRIGGER_TABLE)) + else: + if DL1_TEL_TRIGGER_TABLE in other.root: + self._append_table(other, other.root[DL1_TEL_TRIGGER_TABLE]) if self.dl1_images and DL1_TEL_IMAGES_GROUP in other.root: self._append_table_group(other, other.root[DL1_TEL_IMAGES_GROUP]) @@ -553,8 +598,72 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): + self._flush() self.close() + def _flush(self): + """Flush merged data when combining telescope events.""" + if not self.combine_telescope_events: + return + + # Merge all subarrays into one + self.log.info( + f"Merging {len(self.subarray_list)} subarrays for combine_telescope_events" + ) + merged_subarray = SubarrayDescription.merge_subarrays(self.subarray_list) + + # Write merged subarray to HDF5 (overwrite if existing) + merged_subarray.to_hdf(self.h5file, overwrite=True) + self.log.info(f"Wrote merged subarray with {merged_subarray.n_tels} telescopes") + + # Combine telescope trigger tables + self.log.info( + f"Combining {len(self.tel_trigger_tables)} telescope trigger tables" + ) + combined_tel_triggers = vstack(self.tel_trigger_tables) + combined_tel_triggers.sort(TEL_EVENT_KEYS) + write_table( + combined_tel_triggers, + self.output_path, + path=DL1_TEL_TRIGGER_TABLE, + overwrite=True, + ) + self.log.info( + f"Wrote combined telescope trigger table with {len(combined_tel_triggers)} rows" + ) + + # Create the subarray trigger table from the combined telescope triggers + subarray_trigger_table = combined_tel_triggers.copy() + subarray_trigger_table.keep_columns( + SUBARRAY_EVENT_KEYS + ["time", "event_type"] + ) + subarray_trigger_table = unique( + subarray_trigger_table, keys=SUBARRAY_EVENT_KEYS + ) + # Add tels_with_trigger column indicating which telescopes had a trigger for each event + tel_trigger_groups = combined_tel_triggers.group_by(SUBARRAY_EVENT_KEYS) + tel_with_trigger = [] + for tel_trigger in tel_trigger_groups.groups: + tel_with_trigger_mask = np.zeros(len(merged_subarray.tel_ids), dtype=bool) + tel_with_trigger_mask[ + merged_subarray.tel_ids_to_indices(tel_trigger["tel_id"]) + ] = True + tel_with_trigger.append(tel_with_trigger_mask) + + subarray_trigger_table.add_column( + tel_with_trigger, index=-2, name="tels_with_trigger" + ) + + write_table( + subarray_trigger_table, + self.output_path, + DL1_SUBARRAY_TRIGGER_TABLE, + overwrite=True, + ) + self.log.info( + f"Wrote combined subarray trigger table with {len(subarray_trigger_table)} rows" + ) + def close(self): if hasattr(self, "h5file"): self.h5file.close() @@ -576,22 +685,40 @@ def _append_subarray(self, other): other, focal_length_choice=FocalLengthKind.EQUIVALENT ) + # Check for duplicate telescope IDs when combining telescope events + if self.combine_telescope_events: + new_tel_ids = set(subarray.tel_ids) + duplicates = self.tel_ids_set.intersection(new_tel_ids) + if duplicates: + raise ValueError( + f"Duplicate telescope IDs found when merging file {other.filename}: {sorted(duplicates)}. " + "Each telescope ID must be unique across all input files when using " + "the merge strategy 'combine-telescope-events'." + ) + self.tel_ids_set.update(new_tel_ids) + + self.subarray_list.append(subarray) + if self.subarray is None: self.subarray = subarray - self.subarray.to_hdf(self.h5file) - - # Relax subarray matching requirements for attaching - # monitoring data of the same observation block. - if not self.single_ob or not self.attach_monitoring: - if self.subarray != subarray: - raise CannotMerge(f"Subarrays do not match for file: {other.filename}") - else: - if not SubarrayDescription.check_matching_subarrays( - [self.subarray, subarray] - ): - raise CannotMerge( - f"Subarrays are not compatible for file: {other.filename}" - ) + if not self.combine_telescope_events: + self.subarray.to_hdf(self.h5file) + + if not self.combine_telescope_events: + # Relax subarray matching requirements for attaching + # monitoring data of the same observation block. + if not self.single_ob or not self.attach_monitoring: + if self.subarray != subarray: + raise CannotMerge( + f"Subarrays do not match for file: {other.filename}" + ) + else: + if not SubarrayDescription.check_matching_subarrays( + [self.subarray, subarray] + ): + raise CannotMerge( + f"Subarrays are not compatible for file: {other.filename}" + ) def _append_table_group(self, file, input_group, filter_columns=None, once=False): """Add a group that has a number of child tables to outputfile""" diff --git a/src/ctapipe/tools/merge.py b/src/ctapipe/tools/merge.py index 7c08a3622ae..4848f6a9c72 100644 --- a/src/ctapipe/tools/merge.py +++ b/src/ctapipe/tools/merge.py @@ -94,6 +94,10 @@ class MergeTool(Tool): {"HDF5Merger": {"merge_strategy": "monitoring-only"}}, ("Attach monitoring data from the same observation block."), ), + "combine-telescope-events": ( + {"HDF5Merger": {"merge_strategy": "combine-telescope-events"}}, + ("Combine telescope-wise data from the same observation block."), + ), "progress": ( {"MergeTool": {"progress_bar": True}}, "Show a progress bar for all given input files", diff --git a/src/ctapipe/tools/tests/test_merge.py b/src/ctapipe/tools/tests/test_merge.py index 429ef121289..61bc66aa4d0 100644 --- a/src/ctapipe/tools/tests/test_merge.py +++ b/src/ctapipe/tools/tests/test_merge.py @@ -295,6 +295,75 @@ def test_merge_single_ob_append(tmp_path, dl1_file, dl1_chunks): assert_table_equal(merged_tel_events, initial_tel_events) +def test_merge_telescope_data(dl1_tmp_path, prod6_gamma_simtel_path, dl1_tel1_file): + """ + Test merging telescope events from different files produces same result + as processing all telescopes together. + """ + from ctapipe.tools.merge import MergeTool + from ctapipe.tools.process import ProcessorTool + + # Create DL1 file with telescopes 2-4 + dl1_tel2_file = dl1_tmp_path / "gamma_tel2-4.dl1.h5" + + few_tels = [f"--EventSource.allowed_tels={i}" for i in (2, 3, 4)] + argv = [ + f"--input={prod6_gamma_simtel_path}", + f"--output={dl1_tel2_file}", + "--write-images", + ] + few_tels + run_tool(ProcessorTool(), argv=argv, cwd=dl1_tmp_path) + + # Create reference DL1 file with all telescopes 1-4 + dl1_all_tels_file = dl1_tmp_path / "gamma_tel1-4_ref.dl1.h5" + all_tel_ids = [f"--EventSource.allowed_tels={i}" for i in (1, 2, 3, 4)] + argv = [ + f"--input={prod6_gamma_simtel_path}", + f"--output={dl1_all_tels_file}", + "--write-images", + ] + all_tel_ids + run_tool(ProcessorTool(), argv=argv, cwd=dl1_tmp_path) + + # Merge tel1 and tel2-4 files + merged_output = dl1_tmp_path / "gamma_merged_tel1-4.dl1.h5" + run_tool( + MergeTool(), + argv=[ + str(dl1_tel1_file), + str(dl1_tel2_file), + f"--output={merged_output}", + "--telescope-events", + "--combine-telescope-events", + ], + cwd=dl1_tmp_path, + raises=True, + ) + + # Compare merged result with reference + with TableLoader(merged_output) as loader: + merged_telescope_data = loader.read_telescope_events( + telescopes=[1, 2, 3, 4], dl1_images=True + ) + # TODO: check why one row is not matching for the timing columns + merged_telescope_data.remove_columns( + ["timing_intercept", "timing_deviation", "timing_slope"] + ) + merged_subarray_data = loader.read_subarray_events() + + with TableLoader(dl1_all_tels_file) as loader: + reference_telescope_data = loader.read_telescope_events( + telescopes=[1, 2, 3, 4], dl1_images=True + ) + # TODO: check why one row is not matching for the timing columns + reference_telescope_data.remove_columns( + ["timing_intercept", "timing_deviation", "timing_slope"] + ) + reference_subarray_data = loader.read_subarray_events() + + assert_table_equal(merged_telescope_data, reference_telescope_data) + assert_table_equal(merged_subarray_data, reference_subarray_data) + + def test_merge_exceptions( tmp_path, calibpipe_camcalib_sims_single_chunk, dl1_mon_pointing_file ): From 5a22dfedbf30360aac83c98d113d4d8b9411f538 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 13 Jan 2026 16:01:06 +0100 Subject: [PATCH 04/13] add changelog --- docs/changes/2916.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/2916.feature.rst diff --git a/docs/changes/2916.feature.rst b/docs/changes/2916.feature.rst new file mode 100644 index 00000000000..357c1c6a648 --- /dev/null +++ b/docs/changes/2916.feature.rst @@ -0,0 +1 @@ +Add a new merge_strategy option 'combine-telescope-events' to HDF5Merger component to support the merging of telescope-wise data from different files for the same observation block. From 921fd229253d062ad4b84ff90ec9a02e22aaf971 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Thu, 15 Jan 2026 09:20:03 +0100 Subject: [PATCH 05/13] polish subarray merging --- src/ctapipe/instrument/subarray.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/ctapipe/instrument/subarray.py b/src/ctapipe/instrument/subarray.py index 550a6692b91..21fad2b53a4 100644 --- a/src/ctapipe/instrument/subarray.py +++ b/src/ctapipe/instrument/subarray.py @@ -101,6 +101,11 @@ def __init__( if self.positions.keys() != self.tels.keys(): raise ValueError("Telescope ids in positions and descriptions do not match") + # Ensure sorted order of telescopes by tel_id + sorted_keys = sorted(self.positions.keys()) + self.positions = {k: self.positions[k] for k in sorted_keys} + self.tels = {k: self.tels[k] for k in sorted_keys} + def __str__(self): return self.name @@ -837,7 +842,7 @@ def merge_subarrays( """ tel_positions, tel_descriptions = {}, {} - tel_ids = set() # To collect unique telescope IDs + tel_ids, tid_to_subarray = set(), {} reference_location = subarray_list[ 0 ].reference_location # Get the reference location from the first subarray @@ -862,6 +867,7 @@ def merge_subarrays( f"Telescope '{tid}' already defined; set overwrite_tel_ids=True to " "allow later subarrays to replace earlier entries." ) + tid_to_subarray[tid] = subarray tel_ids.add(tid) if subarray.reference_location != reference_location: raise ValueError( @@ -872,14 +878,14 @@ def merge_subarrays( ) # Merge telescope positions and descriptions, optionally allowing later entries to overwrite - for subarray in subarray_list: - for tid in subarray.tel_ids: - # Copy/overwrite telescope position and description from the current subarray - tel_positions[tid] = subarray.positions[tid] - tel_descriptions[tid] = subarray.tels[tid] - - if not name: - name = "Merged_" + _range_extraction(tel_ids) + for tid in sorted(tel_ids): + # Copy/overwrite telescope position and description from the current subarray + subarray = tid_to_subarray[tid] + tel_positions[tid] = subarray.positions[tid] + tel_descriptions[tid] = subarray.tels[tid] + + if name is None: + name = "Merged_" + _range_extraction(sorted(tel_ids)) newsub = SubarrayDescription( name, From c97119c9c850ee52cd87b14620ba7765238a87a2 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Thu, 15 Jan 2026 09:36:33 +0100 Subject: [PATCH 06/13] fix tests and increase coverage by testing also exceptions --- src/ctapipe/io/hdf5merger.py | 134 ++++++++++---------- src/ctapipe/tools/merge.py | 4 +- src/ctapipe/tools/tests/test_merge.py | 169 ++++++++++++++++++-------- 3 files changed, 191 insertions(+), 116 deletions(-) diff --git a/src/ctapipe/io/hdf5merger.py b/src/ctapipe/io/hdf5merger.py index 58684602874..ddd2a616566 100644 --- a/src/ctapipe/io/hdf5merger.py +++ b/src/ctapipe/io/hdf5merger.py @@ -4,9 +4,8 @@ from contextlib import ExitStack from pathlib import Path -import numpy as np import tables -from astropy.table import unique, vstack +from astropy.table import join, unique, vstack from astropy.time import Time from ..containers import EventType @@ -188,7 +187,7 @@ class HDF5Merger(Component): [ "events-multiple-obs", "events-single-ob", - "combine-telescope-events", + "combine-telescope-data", "monitoring-only", ], default_value="events-multiple-obs", @@ -196,7 +195,7 @@ class HDF5Merger(Component): "Strategy to handle different use cases when merging HDF5 files. " "'events-multiple-obs': allows merging event files (w and w/o monitoring data) from different observation blocks; " "'events-single-ob': for merging events in consecutive chunks of the same OB; " - "'combine-telescope-events': merges telescope-wise data from different files for the same OB (requires telescope_events=True); " + "'combine-telescope-data': merges telescope-wise data from different files for the same OB (requires telescope_events=True); " "'monitoring-only': attaches horizontally monitoring data from the same OB (requires monitoring=True)." ), ).tag(config=True) @@ -215,19 +214,17 @@ def __init__(self, output_path=None, **kwargs): self.single_ob = ( self.merge_strategy == "events-single-ob" or self.merge_strategy == "monitoring-only" - or self.merge_strategy == "combine-telescope-events" + or self.merge_strategy == "combine-telescope-data" ) self.attach_monitoring = self.merge_strategy == "monitoring-only" if self.attach_monitoring and not self.monitoring: raise traits.TraitError( "Merge strategy 'monitoring-only' requires monitoring=True" ) - self.combine_telescope_events = ( - self.merge_strategy == "combine-telescope-events" - ) - if self.combine_telescope_events and not self.telescope_events: + self.combine_telescope_data = self.merge_strategy == "combine-telescope-data" + if self.combine_telescope_data and not self.telescope_events: raise traits.TraitError( - "Merge strategy 'combine-telescope-events' requires telescope_events=True" + "Merge strategy 'combine-telescope-data' requires telescope_events=True" ) output_exists = self.output_path.exists() @@ -251,7 +248,7 @@ def __init__(self, output_path=None, **kwargs): self.data_category = None self.subarray = None self.subarray_list = [] - self.tel_trigger_tables = [] + self.tel_trigger_tables, self.shower_tables = [], [] self.tel_ids_set = set() self.meta = None self._merged_obs_ids = set() @@ -273,7 +270,7 @@ def __init__(self, output_path=None, **kwargs): self.subarray_list.append(self.subarray) # Update tel_ids_set with existing telescope IDs when appending - if self.combine_telescope_events: + if self.combine_telescope_data: self.tel_ids_set.update(self.subarray.tel_ids) # Get required nodes from existing output file @@ -282,11 +279,15 @@ def __init__(self, output_path=None, **kwargs): # this will update _merged_obs_ids from existing input file self._check_obs_ids(self.h5file) - # Append existing tel trigger tables if the merge strategy is 'combine-telescope-events' - if self.combine_telescope_events: + # Append existing tel trigger tables and shower tables + # if the merge strategy is 'combine-telescope-data' + if self.combine_telescope_data: self.tel_trigger_tables.append( read_table(self.h5file, DL1_TEL_TRIGGER_TABLE) ) + self.shower_tables.append( + read_table(self.h5file, SIMULATION_SHOWER_TABLE) + ) self._n_merged += 1 def __call__(self, other: str | Path | tables.File): @@ -408,11 +409,8 @@ def _get_required_nodes(self, h5file): if node not in h5file.root: continue - if node_type is NodeType.TABLE: + if node_type in (NodeType.TABLE, NodeType.TEL_GROUP): required_nodes.add(node) - elif node_type is NodeType.TEL_GROUP: - if not self.combine_telescope_events: - required_nodes.add(node) elif node_type is NodeType.ITER_GROUP: for kind_group in h5file.root[node]._f_iter_nodes("Group"): @@ -421,9 +419,6 @@ def _get_required_nodes(self, h5file): elif node_type is NodeType.ITER_TEL_GROUP: for kind_group in h5file.root[node]._f_iter_nodes("Group"): - if self.combine_telescope_events: - required_nodes.add(kind_group.name) - continue for iter_group in kind_group._f_iter_nodes("Group"): required_nodes.add(iter_group._v_pathname) else: @@ -433,15 +428,20 @@ def _get_required_nodes(self, h5file): def _append_simulation_data(self, other): """Append simulation-related data (run, shower, impact, images, parameters).""" + + if SIMULATION_SHOWER_TABLE in other.root: + if self.combine_telescope_data: + self.shower_tables.append(read_table(other, SIMULATION_SHOWER_TABLE)) + else: + self._append_table(other, other.root[SIMULATION_SHOWER_TABLE]) simulation_table_keys = [ SIMULATION_RUN_TABLE, SHOWER_DISTRIBUTION_TABLE, - SIMULATION_SHOWER_TABLE, ] for key in simulation_table_keys: if key in other.root: self._append_table( - other, other.root[key], once=self.combine_telescope_events + other, other.root[key], once=self.combine_telescope_data ) if FIXED_POINTING_GROUP in other.root: @@ -477,19 +477,16 @@ def _append_waveform_data(self, other): def _append_dl1_data(self, other): """Append DL1 data (triggers, images, parameters, muon).""" - if ( - DL1_SUBARRAY_TRIGGER_TABLE in other.root - and not self.combine_telescope_events - ): + if DL1_SUBARRAY_TRIGGER_TABLE in other.root and not self.combine_telescope_data: self._append_table(other, other.root[DL1_SUBARRAY_TRIGGER_TABLE]) if not self.telescope_events: return - if self.combine_telescope_events: - self.tel_trigger_tables.append(read_table(other, DL1_TEL_TRIGGER_TABLE)) - else: - if DL1_TEL_TRIGGER_TABLE in other.root: + if DL1_TEL_TRIGGER_TABLE in other.root: + if self.combine_telescope_data: + self.tel_trigger_tables.append(read_table(other, DL1_TEL_TRIGGER_TABLE)) + else: self._append_table(other, other.root[DL1_TEL_TRIGGER_TABLE]) if self.dl1_images and DL1_TEL_IMAGES_GROUP in other.root: @@ -598,40 +595,34 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): - self._flush() self.close() + self._flush() def _flush(self): - """Flush merged data when combining telescope events.""" - if not self.combine_telescope_events: + """Flush any remaining data to the output file. + + This is relevant for the 'combine-telescope-data' merge strategy, + where subarray and tel_trigger tables are only written at the end + after all files have been processed. + """ + + if not self.combine_telescope_data or not self.tel_trigger_tables: return # Merge all subarrays into one - self.log.info( - f"Merging {len(self.subarray_list)} subarrays for combine_telescope_events" - ) merged_subarray = SubarrayDescription.merge_subarrays(self.subarray_list) - # Write merged subarray to HDF5 (overwrite if existing) - merged_subarray.to_hdf(self.h5file, overwrite=True) - self.log.info(f"Wrote merged subarray with {merged_subarray.n_tels} telescopes") - - # Combine telescope trigger tables - self.log.info( - f"Combining {len(self.tel_trigger_tables)} telescope trigger tables" - ) + merged_subarray.to_hdf(self.output_path, overwrite=True) + # Stack the tel_trigger tables vertically and sort by telescope event keys combined_tel_triggers = vstack(self.tel_trigger_tables) combined_tel_triggers.sort(TEL_EVENT_KEYS) + # Write combined telescope trigger table to HDF5 (overwrite if existing) write_table( combined_tel_triggers, self.output_path, path=DL1_TEL_TRIGGER_TABLE, overwrite=True, ) - self.log.info( - f"Wrote combined telescope trigger table with {len(combined_tel_triggers)} rows" - ) - # Create the subarray trigger table from the combined telescope triggers subarray_trigger_table = combined_tel_triggers.copy() subarray_trigger_table.keep_columns( @@ -644,25 +635,42 @@ def _flush(self): tel_trigger_groups = combined_tel_triggers.group_by(SUBARRAY_EVENT_KEYS) tel_with_trigger = [] for tel_trigger in tel_trigger_groups.groups: - tel_with_trigger_mask = np.zeros(len(merged_subarray.tel_ids), dtype=bool) - tel_with_trigger_mask[ - merged_subarray.tel_ids_to_indices(tel_trigger["tel_id"]) - ] = True - tel_with_trigger.append(tel_with_trigger_mask) - + tel_with_trigger.append( + merged_subarray.tel_ids_to_mask(tel_trigger["tel_id"]) + ) + # Add the new column to the table indicating which telescopes had a trigger for each event subarray_trigger_table.add_column( tel_with_trigger, index=-2, name="tels_with_trigger" ) - + # Write subarray trigger table to HDF5 (overwrite if existing) write_table( subarray_trigger_table, self.output_path, DL1_SUBARRAY_TRIGGER_TABLE, overwrite=True, ) - self.log.info( - f"Wrote combined subarray trigger table with {len(subarray_trigger_table)} rows" - ) + # Create and write the merged shower table with only unique events + # that are also in the subarray trigger table + if self.shower_tables: + # Stack the shower tables vertically and keep only unique events + shower_table_stacked = vstack(self.shower_tables) + shower_table_stacked = unique( + shower_table_stacked, keys=SUBARRAY_EVENT_KEYS + ) + # Join with subarray trigger table to keep only events that had a trigger + shower_table = join( + shower_table_stacked, + subarray_trigger_table[SUBARRAY_EVENT_KEYS], + join_type="inner", + ) + shower_table.sort(SUBARRAY_EVENT_KEYS) + # Write shower table to HDF5 (overwrite if existing) + write_table( + shower_table, + self.output_path, + SIMULATION_SHOWER_TABLE, + overwrite=True, + ) def close(self): if hasattr(self, "h5file"): @@ -686,14 +694,14 @@ def _append_subarray(self, other): ) # Check for duplicate telescope IDs when combining telescope events - if self.combine_telescope_events: + if self.combine_telescope_data: new_tel_ids = set(subarray.tel_ids) duplicates = self.tel_ids_set.intersection(new_tel_ids) if duplicates: raise ValueError( f"Duplicate telescope IDs found when merging file {other.filename}: {sorted(duplicates)}. " "Each telescope ID must be unique across all input files when using " - "the merge strategy 'combine-telescope-events'." + "the merge strategy 'combine-telescope-data'." ) self.tel_ids_set.update(new_tel_ids) @@ -701,10 +709,10 @@ def _append_subarray(self, other): if self.subarray is None: self.subarray = subarray - if not self.combine_telescope_events: + if not self.combine_telescope_data: self.subarray.to_hdf(self.h5file) - if not self.combine_telescope_events: + if not self.combine_telescope_data: # Relax subarray matching requirements for attaching # monitoring data of the same observation block. if not self.single_ob or not self.attach_monitoring: diff --git a/src/ctapipe/tools/merge.py b/src/ctapipe/tools/merge.py index 4848f6a9c72..4e119eaafe0 100644 --- a/src/ctapipe/tools/merge.py +++ b/src/ctapipe/tools/merge.py @@ -94,8 +94,8 @@ class MergeTool(Tool): {"HDF5Merger": {"merge_strategy": "monitoring-only"}}, ("Attach monitoring data from the same observation block."), ), - "combine-telescope-events": ( - {"HDF5Merger": {"merge_strategy": "combine-telescope-events"}}, + "combine-telescope-data": ( + {"HDF5Merger": {"merge_strategy": "combine-telescope-data"}}, ("Combine telescope-wise data from the same observation block."), ), "progress": ( diff --git a/src/ctapipe/tools/tests/test_merge.py b/src/ctapipe/tools/tests/test_merge.py index 61bc66aa4d0..2a1f42b06fe 100644 --- a/src/ctapipe/tools/tests/test_merge.py +++ b/src/ctapipe/tools/tests/test_merge.py @@ -295,73 +295,140 @@ def test_merge_single_ob_append(tmp_path, dl1_file, dl1_chunks): assert_table_equal(merged_tel_events, initial_tel_events) -def test_merge_telescope_data(dl1_tmp_path, prod6_gamma_simtel_path, dl1_tel1_file): +def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): """ Test merging telescope events from different files produces same result as processing all telescopes together. """ + + from ctapipe.io.hdf5merger import CannotMerge from ctapipe.tools.merge import MergeTool from ctapipe.tools.process import ProcessorTool - # Create DL1 file with telescopes 2-4 - dl1_tel2_file = dl1_tmp_path / "gamma_tel2-4.dl1.h5" - - few_tels = [f"--EventSource.allowed_tels={i}" for i in (2, 3, 4)] - argv = [ + common_argv = [ f"--input={prod6_gamma_simtel_path}", - f"--output={dl1_tel2_file}", "--write-images", - ] + few_tels - run_tool(ProcessorTool(), argv=argv, cwd=dl1_tmp_path) + ] + outputs = { + "ref": tmp_path / "gamma_ref.dl1.h5", + "sub1": tmp_path / "gamma_sub1.dl1.h5", + "sub2": tmp_path / "gamma_sub2.dl1.h5", + "sub2_dl1b": tmp_path / "gamma_sub2_noimages.dl1b.h5", + "merged": tmp_path / "gamma_merged.dl1.h5", + "merged_appendmode": tmp_path / "gamma_merged_appendmode.dl1.h5", + "tel_ids_invalid": tmp_path / "duplicated_tel_ids_invalid.dl1.h5", + "required_node_invalid": tmp_path / "required_node_invalid.dl1.h5", + } + allowed_tels = range(1, 26, 4) + allowed_tels_strings = [ + f"--EventSource.allowed_tels={tel_id}" for tel_id in allowed_tels + ] + tel_sets = [ + ("ref", allowed_tels_strings), + ("sub1", allowed_tels_strings[:4]), + ("sub2", allowed_tels_strings[4:]), + ] + # Run ProcessorTool for each subset + for name, tel_args in tel_sets: + run_tool( + ProcessorTool(), + argv=[ + *common_argv, + *tel_args, + f"--output={outputs[name]}", + ], + cwd=tmp_path, + ) - # Create reference DL1 file with all telescopes 1-4 - dl1_all_tels_file = dl1_tmp_path / "gamma_tel1-4_ref.dl1.h5" - all_tel_ids = [f"--EventSource.allowed_tels={i}" for i in (1, 2, 3, 4)] - argv = [ - f"--input={prod6_gamma_simtel_path}", - f"--output={dl1_all_tels_file}", - "--write-images", - ] + all_tel_ids - run_tool(ProcessorTool(), argv=argv, cwd=dl1_tmp_path) + tel_sets = [ + ("ref", allowed_tels_strings), + ("sub1", allowed_tels_strings[:4]), + ("sub2", allowed_tels_strings[4:]), + ] + # For append mode test, copy one of the subset files to start with + shutil.copy(outputs["sub1"], outputs["merged_appendmode"]) + # Merge subset files into single file which should match reference + # Test both normal merge and append mode + merger_mode_argv = { + "merged": [str(outputs["sub1"])], + "merged_appendmode": ["--append"], + } + for merged_mode_name in ["merged", "merged_appendmode"]: + run_tool( + MergeTool(), + argv=merger_mode_argv[merged_mode_name] + + [ + str(outputs["sub2"]), + f"--output={outputs[merged_mode_name]}", + "--telescope-events", + "--combine-telescope-data", + ], + cwd=tmp_path, + raises=True, + ) - # Merge tel1 and tel2-4 files - merged_output = dl1_tmp_path / "gamma_merged_tel1-4.dl1.h5" + # Compare merged result with reference + with ( + TableLoader(outputs[merged_mode_name]) as merged_loader, + TableLoader(outputs["ref"]) as ref_loader, + ): + # Compare telescope data for each telescope + for tel_id in allowed_tels: + merged_telescope_data = merged_loader.read_telescope_events( + telescopes=[tel_id], dl1_images=True + ) + reference_telescope_data = ref_loader.read_telescope_events( + telescopes=[tel_id], dl1_images=True + ) + assert_table_equal(merged_telescope_data, reference_telescope_data) + # Compare subarray data + merged_subarray_data = merged_loader.read_subarray_events() + reference_subarray_data = ref_loader.read_subarray_events() + assert_table_equal(merged_subarray_data, reference_subarray_data) + + # Check that merging files with overlapping telescope IDs raises an error + # When combining telescope data, telescope IDs must be unique. + with pytest.raises( + ValueError, match="Duplicate telescope IDs found when merging file" + ): + run_tool( + MergeTool(), + argv=[ + str(outputs["sub1"]), + str(outputs[merged_mode_name]), + "--telescope-events", + "--combine-telescope-data", + f"--output={outputs['tel_ids_invalid']}", + ], + cwd=tmp_path, + raises=True, + ) + + # Check that merging files. with different data levels raises an error + # When combining telescope data, data levels must match. run_tool( - MergeTool(), + ProcessorTool(), argv=[ - str(dl1_tel1_file), - str(dl1_tel2_file), - f"--output={merged_output}", - "--telescope-events", - "--combine-telescope-events", + f"--input={prod6_gamma_simtel_path}", + "--no-write-images", + *allowed_tels_strings[4:], + f"--output={outputs['sub2_dl1b']}", ], - cwd=dl1_tmp_path, - raises=True, + cwd=tmp_path, ) - - # Compare merged result with reference - with TableLoader(merged_output) as loader: - merged_telescope_data = loader.read_telescope_events( - telescopes=[1, 2, 3, 4], dl1_images=True - ) - # TODO: check why one row is not matching for the timing columns - merged_telescope_data.remove_columns( - ["timing_intercept", "timing_deviation", "timing_slope"] - ) - merged_subarray_data = loader.read_subarray_events() - - with TableLoader(dl1_all_tels_file) as loader: - reference_telescope_data = loader.read_telescope_events( - telescopes=[1, 2, 3, 4], dl1_images=True - ) - # TODO: check why one row is not matching for the timing columns - reference_telescope_data.remove_columns( - ["timing_intercept", "timing_deviation", "timing_slope"] + with pytest.raises(CannotMerge, match="Required node"): + run_tool( + MergeTool(), + argv=[ + str(outputs["sub1"]), + str(outputs["sub2_dl1b"]), + "--telescope-events", + "--combine-telescope-data", + f"--output={outputs['required_node_invalid']}", + ], + cwd=tmp_path, + raises=True, ) - reference_subarray_data = loader.read_subarray_events() - - assert_table_equal(merged_telescope_data, reference_telescope_data) - assert_table_equal(merged_subarray_data, reference_subarray_data) def test_merge_exceptions( From 2637fee2a1b5c30b440928858c2bf20a4dbc53d3 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Thu, 15 Jan 2026 10:10:11 +0100 Subject: [PATCH 07/13] do not sort keys in the SubarrayDescription --- src/ctapipe/instrument/subarray.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ctapipe/instrument/subarray.py b/src/ctapipe/instrument/subarray.py index 21fad2b53a4..3b858b01b68 100644 --- a/src/ctapipe/instrument/subarray.py +++ b/src/ctapipe/instrument/subarray.py @@ -101,11 +101,6 @@ def __init__( if self.positions.keys() != self.tels.keys(): raise ValueError("Telescope ids in positions and descriptions do not match") - # Ensure sorted order of telescopes by tel_id - sorted_keys = sorted(self.positions.keys()) - self.positions = {k: self.positions[k] for k in sorted_keys} - self.tels = {k: self.tels[k] for k in sorted_keys} - def __str__(self): return self.name From 833a49b231ba759962c1d5f76fc8d1ae04ea960d Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Thu, 15 Jan 2026 10:34:07 +0100 Subject: [PATCH 08/13] remove timing columns from comparison Extraction from timing parameters do not seems deterministic. --- src/ctapipe/tools/tests/test_merge.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/tools/tests/test_merge.py b/src/ctapipe/tools/tests/test_merge.py index 2a1f42b06fe..c5cec8b4153 100644 --- a/src/ctapipe/tools/tests/test_merge.py +++ b/src/ctapipe/tools/tests/test_merge.py @@ -305,6 +305,12 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): from ctapipe.tools.merge import MergeTool from ctapipe.tools.process import ProcessorTool + # To be dropped from comparison + TIMING_COLUMNS = [ + "timing_intercept", + "timing_deviation", + "timing_slope", + ] common_argv = [ f"--input={prod6_gamma_simtel_path}", "--write-images", @@ -380,7 +386,11 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): reference_telescope_data = ref_loader.read_telescope_events( telescopes=[tel_id], dl1_images=True ) - assert_table_equal(merged_telescope_data, reference_telescope_data) + # Assert equality of the two tables after removing timing columns + assert_table_equal( + merged_telescope_data.remove_columns(TIMING_COLUMNS), + reference_telescope_data.remove_columns(TIMING_COLUMNS), + ) # Compare subarray data merged_subarray_data = merged_loader.read_subarray_events() reference_subarray_data = ref_loader.read_subarray_events() From 8ac57d94e01e67e75e9602acf6a46fbf7606da84 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Thu, 15 Jan 2026 11:13:22 +0100 Subject: [PATCH 09/13] fix tests by restricting the selected tel ids (ensuring to have data) --- src/ctapipe/tools/tests/test_merge.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ctapipe/tools/tests/test_merge.py b/src/ctapipe/tools/tests/test_merge.py index c5cec8b4153..f1ebcc9d0b1 100644 --- a/src/ctapipe/tools/tests/test_merge.py +++ b/src/ctapipe/tools/tests/test_merge.py @@ -325,7 +325,9 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): "tel_ids_invalid": tmp_path / "duplicated_tel_ids_invalid.dl1.h5", "required_node_invalid": tmp_path / "required_node_invalid.dl1.h5", } - allowed_tels = range(1, 26, 4) + # Select a few telescopes that cover different telescope types + # and have at least one triggered event in the simulated file. + allowed_tels = [1, 4, 5, 9, 13, 17, 25] allowed_tels_strings = [ f"--EventSource.allowed_tels={tel_id}" for tel_id in allowed_tels ] @@ -387,10 +389,9 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): telescopes=[tel_id], dl1_images=True ) # Assert equality of the two tables after removing timing columns - assert_table_equal( - merged_telescope_data.remove_columns(TIMING_COLUMNS), - reference_telescope_data.remove_columns(TIMING_COLUMNS), - ) + merged_telescope_data.remove_columns(TIMING_COLUMNS) + reference_telescope_data.remove_columns(TIMING_COLUMNS) + assert_table_equal(merged_telescope_data, reference_telescope_data) # Compare subarray data merged_subarray_data = merged_loader.read_subarray_events() reference_subarray_data = ref_loader.read_subarray_events() From fdc5960e4744863c88fc86d2cda78f6ab53402e3 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Thu, 15 Jan 2026 11:35:36 +0100 Subject: [PATCH 10/13] remove duplicated variable --- src/ctapipe/tools/tests/test_merge.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ctapipe/tools/tests/test_merge.py b/src/ctapipe/tools/tests/test_merge.py index f1ebcc9d0b1..fdd44acc782 100644 --- a/src/ctapipe/tools/tests/test_merge.py +++ b/src/ctapipe/tools/tests/test_merge.py @@ -348,11 +348,6 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): cwd=tmp_path, ) - tel_sets = [ - ("ref", allowed_tels_strings), - ("sub1", allowed_tels_strings[:4]), - ("sub2", allowed_tels_strings[4:]), - ] # For append mode test, copy one of the subset files to start with shutil.copy(outputs["sub1"], outputs["merged_appendmode"]) # Merge subset files into single file which should match reference From 8e513321117a749b63d48d15037f20df93589a84 Mon Sep 17 00:00:00 2001 From: Tjark Miener <37835610+TjarkMiener@users.noreply.github.com> Date: Thu, 15 Jan 2026 12:44:33 +0100 Subject: [PATCH 11/13] Update src/ctapipe/instrument/tests/test_subarray.py Co-authored-by: Karl Kosack --- src/ctapipe/instrument/tests/test_subarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/instrument/tests/test_subarray.py b/src/ctapipe/instrument/tests/test_subarray.py index a6cb63b8db7..c01f8b36c42 100644 --- a/src/ctapipe/instrument/tests/test_subarray.py +++ b/src/ctapipe/instrument/tests/test_subarray.py @@ -323,7 +323,7 @@ def test_merge_subarrays(example_subarray): expected_sub = example_subarray.select_subarray([1, 2, 3, 4], name="Merged_1-4") merged_sub = SubarrayDescription.merge_subarrays([sub1, sub2]) - assert expected_sub.__eq__(merged_sub) + assert expected_sub == merged_sub def test_merge_subarrays_exceptions(example_subarray): From 5fe1a0b82ff1db01ef0ce7b6b3193ea040d009b2 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 16 Jan 2026 15:34:51 +0100 Subject: [PATCH 12/13] exclude DL2 subarray when merging telescope data --- src/ctapipe/io/hdf5merger.py | 9 ++++-- src/ctapipe/tools/tests/test_merge.py | 45 ++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/ctapipe/io/hdf5merger.py b/src/ctapipe/io/hdf5merger.py index ddd2a616566..36c555daf05 100644 --- a/src/ctapipe/io/hdf5merger.py +++ b/src/ctapipe/io/hdf5merger.py @@ -222,9 +222,12 @@ def __init__(self, output_path=None, **kwargs): "Merge strategy 'monitoring-only' requires monitoring=True" ) self.combine_telescope_data = self.merge_strategy == "combine-telescope-data" - if self.combine_telescope_data and not self.telescope_events: + if self.combine_telescope_data and ( + not self.telescope_events or self.dl2_subarray + ): raise traits.TraitError( - "Merge strategy 'combine-telescope-data' requires telescope_events=True" + "Merge strategy 'combine-telescope-data' requires telescope_events=True " + "and dl2_subarray=False" ) output_exists = self.output_path.exists() @@ -551,7 +554,7 @@ def _append_monitoring_dl2_groups(self, other): DL2_SUBARRAY_CROSS_CALIBRATION_GROUP, ] for key in monitoring_dl2_subarray_groups: - if key in other.root: + if self.dl2_subarray and key in other.root: self._append_table(other, other.root[key], once=self.single_ob) def _append_pixel_statistics(self, other): diff --git a/src/ctapipe/tools/tests/test_merge.py b/src/ctapipe/tools/tests/test_merge.py index fdd44acc782..d3a2af5faac 100644 --- a/src/ctapipe/tools/tests/test_merge.py +++ b/src/ctapipe/tools/tests/test_merge.py @@ -301,6 +301,7 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): as processing all telescopes together. """ + from ctapipe.core import traits from ctapipe.io.hdf5merger import CannotMerge from ctapipe.tools.merge import MergeTool from ctapipe.tools.process import ProcessorTool @@ -322,7 +323,7 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): "sub2_dl1b": tmp_path / "gamma_sub2_noimages.dl1b.h5", "merged": tmp_path / "gamma_merged.dl1.h5", "merged_appendmode": tmp_path / "gamma_merged_appendmode.dl1.h5", - "tel_ids_invalid": tmp_path / "duplicated_tel_ids_invalid.dl1.h5", + "invalid": tmp_path / "invalid.dl1.h5", "required_node_invalid": tmp_path / "required_node_invalid.dl1.h5", } # Select a few telescopes that cover different telescope types @@ -364,6 +365,7 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): str(outputs["sub2"]), f"--output={outputs[merged_mode_name]}", "--telescope-events", + "--no-dl2-subarray", "--combine-telescope-data", ], cwd=tmp_path, @@ -394,17 +396,49 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): # Check that merging files with overlapping telescope IDs raises an error # When combining telescope data, telescope IDs must be unique. + argv_options = [ + str(outputs["sub1"]), + str(outputs[merged_mode_name]), + f"--output={outputs['invalid']}", + "--combine-telescope-data", + ] with pytest.raises( ValueError, match="Duplicate telescope IDs found when merging file" ): run_tool( MergeTool(), argv=[ - str(outputs["sub1"]), - str(outputs[merged_mode_name]), + *argv_options, "--telescope-events", - "--combine-telescope-data", - f"--output={outputs['tel_ids_invalid']}", + "--no-dl2-subarray", + ], + cwd=tmp_path, + raises=True, + ) + + # Check that merging files with incompatible options raises a TraitError + with pytest.raises( + traits.TraitError, match="Merge strategy 'combine-telescope-data' requires" + ): + run_tool( + MergeTool(), + argv=[ + *argv_options, + "--no-telescope-events", + "--no-dl2-subarray", + ], + cwd=tmp_path, + raises=True, + ) + with pytest.raises( + traits.TraitError, match="Merge strategy 'combine-telescope-data' requires" + ): + run_tool( + MergeTool(), + argv=[ + *argv_options, + "--telescope-events", + "--dl2-subarray", ], cwd=tmp_path, raises=True, @@ -430,6 +464,7 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): str(outputs["sub2_dl1b"]), "--telescope-events", "--combine-telescope-data", + "--no-dl2-subarray", f"--output={outputs['required_node_invalid']}", ], cwd=tmp_path, From e9b8c4e7a59285396c2c405daf6c80ce88866c9e Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 21 Jan 2026 13:43:47 +0100 Subject: [PATCH 13/13] also exclude DL2 telescope merging when tel-wise merge is selected use data format defs in merge tool tests --- src/ctapipe/io/hdf5dataformat.py | 2 ++ src/ctapipe/io/hdf5merger.py | 4 +-- src/ctapipe/tools/tests/test_merge.py | 52 ++++++++++++++++++--------- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/ctapipe/io/hdf5dataformat.py b/src/ctapipe/io/hdf5dataformat.py index f095abb8d10..29bd2d58b95 100644 --- a/src/ctapipe/io/hdf5dataformat.py +++ b/src/ctapipe/io/hdf5dataformat.py @@ -43,6 +43,7 @@ "DL2_TEL_GEOMETRY_GROUP", "DL2_TEL_ENERGY_GROUP", "DL2_TEL_PARTICLETYPE_GROUP", + "DL2_TEL_IMPACT_GROUP", "DL0_TEL_POINTING_GROUP", "DL1_SUBARRAY_POINTING_GROUP", "DL1_TEL_POINTING_GROUP", @@ -104,6 +105,7 @@ DL2_TEL_GEOMETRY_GROUP = "/dl2/event/telescope/geometry" DL2_TEL_ENERGY_GROUP = "/dl2/event/telescope/energy" DL2_TEL_PARTICLETYPE_GROUP = "/dl2/event/telescope/particle_type" +DL2_TEL_IMPACT_GROUP = "/dl2/event/telescope/impact" DL2_GROUP = "/dl2" DL2_SUBARRAY_GROUP = "/dl2/event/subarray" diff --git a/src/ctapipe/io/hdf5merger.py b/src/ctapipe/io/hdf5merger.py index 36c555daf05..cd60515aef2 100644 --- a/src/ctapipe/io/hdf5merger.py +++ b/src/ctapipe/io/hdf5merger.py @@ -223,11 +223,11 @@ def __init__(self, output_path=None, **kwargs): ) self.combine_telescope_data = self.merge_strategy == "combine-telescope-data" if self.combine_telescope_data and ( - not self.telescope_events or self.dl2_subarray + not self.telescope_events or self.dl2_telescope or self.dl2_subarray ): raise traits.TraitError( "Merge strategy 'combine-telescope-data' requires telescope_events=True " - "and dl2_subarray=False" + "and dl2_telescope=False and dl2_subarray=False" ) output_exists = self.output_path.exists() diff --git a/src/ctapipe/tools/tests/test_merge.py b/src/ctapipe/tools/tests/test_merge.py index d3a2af5faac..d72c6f39c68 100644 --- a/src/ctapipe/tools/tests/test_merge.py +++ b/src/ctapipe/tools/tests/test_merge.py @@ -14,6 +14,14 @@ from ctapipe.core import ToolConfigurationError, run_tool from ctapipe.io import DataWriter, EventSource, TableLoader from ctapipe.io.astropy_helpers import read_table +from ctapipe.io.hdf5dataformat import ( + DL1_TEL_MUON_GROUP, + DL2_EVENT_STATISTICS_GROUP, + DL2_SUBARRAY_GEOMETRY_GROUP, + OBSERVATION_BLOCK_TABLE, + SCHEDULING_BLOCK_TABLE, + SIMULATION_IMAGES_GROUP, +) from ctapipe.io.tests.test_astropy_helpers import assert_table_equal from ctapipe.tools.process import ProcessorTool @@ -108,7 +116,7 @@ def test_skip_images(tmp_path, dl1_file, dl1_proton_file): assert "images" in f.root.simulation.event.telescope assert "parameters" in f.root.dl1.event.telescope - t = read_table(output, "/simulation/event/telescope/images/tel_001") + t = read_table(output, f"{SIMULATION_IMAGES_GROUP}/tel_001") assert "true_image" not in t.colnames assert "true_image_sum" in t.colnames @@ -128,13 +136,13 @@ def test_dl2(tmp_path, dl2_shower_geometry_file, dl2_proton_geometry_file): ) table1 = read_table( - dl2_shower_geometry_file, "/dl2/event/subarray/geometry/HillasReconstructor" + dl2_shower_geometry_file, f"{DL2_SUBARRAY_GEOMETRY_GROUP}/HillasReconstructor" ) table2 = read_table( - dl2_proton_geometry_file, "/dl2/event/subarray/geometry/HillasReconstructor" + dl2_proton_geometry_file, f"{DL2_SUBARRAY_GEOMETRY_GROUP}/HillasReconstructor" ) table_merged = read_table( - output, "/dl2/event/subarray/geometry/HillasReconstructor" + output, f"{DL2_SUBARRAY_GEOMETRY_GROUP}/HillasReconstructor" ) diff = StringIO() @@ -143,7 +151,7 @@ def test_dl2(tmp_path, dl2_shower_geometry_file, dl2_proton_geometry_file): f"Merged table not equal to individual tables. Diff:\n {diff.getvalue()}" ) - stats_key = "/dl2/service/tel_event_statistics/HillasReconstructor" + stats_key = f"{DL2_EVENT_STATISTICS_GROUP}/HillasReconstructor" merged_stats = read_table(output, stats_key) stats1 = read_table(dl2_shower_geometry_file, stats_key) stats2 = read_table(dl2_proton_geometry_file, stats_key) @@ -152,8 +160,8 @@ def test_dl2(tmp_path, dl2_shower_geometry_file, dl2_proton_geometry_file): assert np.all(merged_stats[col] == (stats1[col] + stats2[col])) # test reading configurations as well: - obs = read_table(output, "/configuration/observation/observation_block") - sbs = read_table(output, "/configuration/observation/scheduling_block") + obs = read_table(output, OBSERVATION_BLOCK_TABLE) + sbs = read_table(output, SCHEDULING_BLOCK_TABLE) assert len(obs) == 2, "should have two OB entries" assert len(sbs) == 2, "should have two SB entries" @@ -182,8 +190,8 @@ def test_muon(tmp_path, dl1_muon_output_file): raises=True, ) - table = read_table(output, "/dl1/event/telescope/muon/tel_001") - input_table = read_table(dl1_muon_output_file, "/dl1/event/telescope/muon/tel_001") + table = read_table(output, f"{DL1_TEL_MUON_GROUP}/tel_001") + input_table = read_table(dl1_muon_output_file, f"{DL1_TEL_MUON_GROUP}/tel_001") n_input = len(input_table) assert len(table) == n_input @@ -365,6 +373,7 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): str(outputs["sub2"]), f"--output={outputs[merged_mode_name]}", "--telescope-events", + "--no-dl2-telescope", "--no-dl2-subarray", "--combine-telescope-data", ], @@ -410,6 +419,7 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): argv=[ *argv_options, "--telescope-events", + "--no-dl2-telescope", "--no-dl2-subarray", ], cwd=tmp_path, @@ -417,9 +427,8 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): ) # Check that merging files with incompatible options raises a TraitError - with pytest.raises( - traits.TraitError, match="Merge strategy 'combine-telescope-data' requires" - ): + traits_error_msg = "Merge strategy 'combine-telescope-data' requires" + with pytest.raises(traits.TraitError, match=traits_error_msg): run_tool( MergeTool(), argv=[ @@ -430,20 +439,30 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): cwd=tmp_path, raises=True, ) - with pytest.raises( - traits.TraitError, match="Merge strategy 'combine-telescope-data' requires" - ): + with pytest.raises(traits.TraitError, match=traits_error_msg): run_tool( MergeTool(), argv=[ *argv_options, "--telescope-events", + "--no-dl2-telescope", "--dl2-subarray", ], cwd=tmp_path, raises=True, ) - + with pytest.raises(traits.TraitError, match=traits_error_msg): + run_tool( + MergeTool(), + argv=[ + *argv_options, + "--telescope-events", + "--dl2-telescope", + "--no-dl2-subarray", + ], + cwd=tmp_path, + raises=True, + ) # Check that merging files. with different data levels raises an error # When combining telescope data, data levels must match. run_tool( @@ -465,6 +484,7 @@ def test_merge_telescope_data(tmp_path, prod6_gamma_simtel_path): "--telescope-events", "--combine-telescope-data", "--no-dl2-subarray", + "--no-dl2-telescope", f"--output={outputs['required_node_invalid']}", ], cwd=tmp_path,