Add meta_block_size as an output trajectory parameter

MBartkowiakSTFC · MBartkowiakSTFC · commit 9dde6b7f81bd · 2026-06-29T12:00:01.000+01:00
diff --git a/MDANSE/Src/MDANSE/Framework/Configurators/OutputTrajectoryConfigurator.py b/MDANSE/Src/MDANSE/Framework/Configurators/OutputTrajectoryConfigurator.py
@@ -66,9 +66,14 @@ def __init__(self, name, format=None, **kwargs):
     def configure(self, value: tuple):
         self._original_input = value
 
-        root, dtype, chunk_size, compression, logs = value
+        root, dtype, chunk_size, compression, logs = value[:5]
         root = Path(root)
 
+        try:
+            meta_block_size = int(value[5])
+        except Exception:
+            meta_block_size = 4096
+
         if logs not in self.log_options:
             self.error_status = "log level option not recognised"
             return
@@ -112,4 +117,5 @@ def configure(self, value: tuple):
         self["chunk_size"] = self._chunk_limit
         self["log_level"] = logs
         self["write_logs"] = logs != "no logs"
+        self["meta_block_size"] = meta_block_size
         self.error_status = "OK"
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/ASE.py b/MDANSE/Src/MDANSE/Framework/Converters/ASE.py
@@ -147,6 +147,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
             initial_charges=self._initial_charges,
         )
 
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/CASTEP.py b/MDANSE/Src/MDANSE/Framework/Converters/CASTEP.py
@@ -97,6 +97,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
         )
 
     def run_step(self, index: int) -> tuple[int, None]:
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/CP2K.py b/MDANSE/Src/MDANSE/Framework/Converters/CP2K.py
@@ -154,6 +154,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
         )
 
         data_to_be_written = ["configuration", "time"]
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/DCD.py b/MDANSE/Src/MDANSE/Framework/Converters/DCD.py
@@ -104,6 +104,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
         )
 
     def run_step(self, index):
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/DL_POLY.py b/MDANSE/Src/MDANSE/Framework/Converters/DL_POLY.py
@@ -113,6 +113,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
             initial_charges=self.field_file.get_atom_charges(),
         )
 
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/Forcite.py b/MDANSE/Src/MDANSE/Framework/Converters/Forcite.py
@@ -148,6 +148,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
             initial_charges=charges,
         )
 
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/Gromacs.py b/MDANSE/Src/MDANSE/Framework/Converters/Gromacs.py
@@ -141,6 +141,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
         )
 
     def run_step(self, index):
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/LAMMPS.py b/MDANSE/Src/MDANSE/Framework/Converters/LAMMPS.py
@@ -212,6 +212,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
             initial_charges=charges,
         )
 
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/MDAnalysis.py b/MDANSE/Src/MDANSE/Framework/Converters/MDAnalysis.py
@@ -174,6 +174,7 @@ def initialize(self):
             "positions_dtype": self.configuration["output_files"]["dtype"],
             "chunking_limit": self.configuration["output_files"]["chunk_size"],
             "compression": self.configuration["output_files"]["compression"],
+            "meta_block_size": self.configuration["output_files"]["meta_block_size"],
         }
         if hasattr(self.u.atoms, "charges"):
             kwargs["initial_charges"] = self.u.atoms.charges
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/MDTraj.py b/MDANSE/Src/MDANSE/Framework/Converters/MDTraj.py
@@ -156,6 +156,7 @@ def initialize(self):
             "positions_dtype": self.configuration["output_files"]["dtype"],
             "chunking_limit": self.configuration["output_files"]["chunk_size"],
             "compression": self.configuration["output_files"]["compression"],
+            "meta_block_size": self.configuration["output_files"]["meta_block_size"],
         }
         self._trajectory = TrajectoryWriter(
             self.configuration["output_files"]["file"],
diff --git a/MDANSE/Src/MDANSE/Framework/Converters/VASP.py b/MDANSE/Src/MDANSE/Framework/Converters/VASP.py
@@ -123,6 +123,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
         )
 
     def run_step(self, index):
diff --git a/MDANSE/Src/MDANSE/Framework/Jobs/CenterOfMassesTrajectory.py b/MDANSE/Src/MDANSE/Framework/Jobs/CenterOfMassesTrajectory.py
@@ -111,6 +111,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
         )
         self._unique_atoms = np.unique(new_element_list)
         self._molecule_radii = {
diff --git a/MDANSE/Src/MDANSE/Framework/Jobs/TrajectoryEditor.py b/MDANSE/Src/MDANSE/Framework/Jobs/TrajectoryEditor.py
@@ -178,6 +178,7 @@ def initialize(self):
             positions_dtype=self.configuration["output_files"]["dtype"],
             chunking_limit=self.configuration["output_files"]["chunk_size"],
             compression=self.configuration["output_files"]["compression"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
         )
 
     def run_step(self, index):
diff --git a/MDANSE/Src/MDANSE/Framework/Jobs/TrajectoryFilter.py b/MDANSE/Src/MDANSE/Framework/Jobs/TrajectoryFilter.py
@@ -207,9 +207,11 @@ def finalize(self):
             self.configuration["output_files"]["file"],
             output_chemical_system,
             self.configuration["frames"]["number"],
-            None,
+            selected_atoms=None,
             positions_dtype=self.configuration["output_files"]["dtype"],
             compression=self.configuration["output_files"]["compression"],
+            chunking_limit=self.configuration["output_files"]["chunk_size"],
+            meta_block_size=self.configuration["output_files"]["meta_block_size"],
         )
 
         # Write trajectory
diff --git a/MDANSE/Src/MDANSE/MolecularDynamics/Trajectory.py b/MDANSE/Src/MDANSE/MolecularDynamics/Trajectory.py
@@ -1052,7 +1052,10 @@ def __init__(
         """
         self._h5_filename = Path(h5_filename)
         PLATFORM.create_directory(self._h5_filename.parent)
-        self._h5_file = h5py.File(self._h5_filename, "w", meta_block_size=meta_block_size)
+        self._h5_file = h5py.File(self._h5_filename,
+                                  "w",
+                                  meta_block_size=meta_block_size,
+                                  libver=('earliest', 'v114'),)
 
         self._chemical_system = chemical_system
         self._last_configuration = None
diff --git a/MDANSE_GUI/Src/MDANSE_GUI/InputWidgets/OutputTrajectoryWidget.py b/MDANSE_GUI/Src/MDANSE_GUI/InputWidgets/OutputTrajectoryWidget.py
@@ -16,6 +16,7 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import NamedTuple
 
 from qtpy.QtCore import Qt, Slot
 from qtpy.QtWidgets import (
@@ -37,6 +38,36 @@
 dtype_lookup = {"float16": 16, "float32": 32, "float64": 64}
 
 
+class SpinBoxDefaults(NamedTuple):
+    minimum: int = 1
+    maximum: int = 0xFFFFFF
+    step: int = 1
+    start_value: int = 1
+    tooltip: str = "Input spin box"
+
+
+SPIN_BOX_SETTINGS = {
+    "frames": SpinBoxDefaults(
+        tooltip="Specifies the number of frames in a single chunk of the HDF5 file. "
+        "Affects the performance of reading and writing the trajectory."
+    ),
+    "atoms": SpinBoxDefaults(
+        step=32,
+        start_value=128,
+        tooltip="Specifies the number of atoms in a single chunk of the HDF5 file. "
+        "Affects the performance of reading and writing the trajectory.",
+    ),
+    "meta_block": SpinBoxDefaults(
+        minimum=0,
+        maximum=0x80000,
+        step=2048,
+        start_value=4096,
+        tooltip="Size of a single metadata block in the HDF5 file. "
+        "Larger values (e.g. 32768) may work better on cloud-based virtual machines (Ada, VISA, etc.).",
+    ),
+}
+
+
 class OutputTrajectoryWidget(WidgetBase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, layout_type="QGridLayout", **kwargs)
@@ -68,19 +99,18 @@ def __init__(self, *args, **kwargs):
         self.dtype_box.setCurrentText("float64")
         self.chunk_atom_box = QSpinBox(self._base)
         self.chunk_frame_box = QSpinBox(self._base)
+        self.meta_block_box = QSpinBox(self._base)
         for typ, chunk_box in zip(
-            ("atoms", "frames"),
-            (self.chunk_atom_box, self.chunk_frame_box),
+            ("atoms", "frames", "meta_block"),
+            (self.chunk_atom_box, self.chunk_frame_box, self.meta_block_box),
             strict=True,
         ):
-            chunk_box.setMinimum(1)
-            chunk_box.setMaximum(0xFFFF)
-            chunk_box.setValue(1 if typ == "frames" else 128)
-            chunk_box.setSingleStep(1 if typ == "frames" else 32)
-            chunk_box.setToolTip(
-                f"Specifies the number of {typ} in a single chunk of the HDF5 file."
-                "Affects the performance of reading and writing the trajectory."
-            )
+            settings = SPIN_BOX_SETTINGS[typ]
+            chunk_box.setMinimum(settings.minimum)
+            chunk_box.setMaximum(settings.maximum)
+            chunk_box.setValue(settings.start_value)
+            chunk_box.setSingleStep(settings.step)
+            chunk_box.setToolTip(settings.tooltip)
         self.compression_box = QComboBox(self._base)
         self.compression_box.addItems(["none", "gzip"])
         self.compression_box.setCurrentText("gzip")
@@ -91,28 +121,27 @@ def __init__(self, *args, **kwargs):
         label.setAlignment(Qt.AlignRight | Qt.AlignVCenter)
         label2 = QLabel("Atoms per chunk")
         label2.setAlignment(Qt.AlignRight | Qt.AlignVCenter)
-        label2.setToolTip(
-            "Specifies the number of atoms in a single chunk of the HDF5 file."
-            "Affects the performance of reading and writing the trajectory."
-        )
+        label2.setToolTip(SPIN_BOX_SETTINGS["atoms"].tooltip)
         label3 = QLabel("Frames per chunk")
         label3.setAlignment(Qt.AlignRight | Qt.AlignVCenter)
-        label3.setToolTip(
-            "Specifies the number of frames in a single chunk of the HDF5 file."
-            "Affects the performance of reading and writing the trajectory."
-        )
+        label3.setToolTip(SPIN_BOX_SETTINGS["frames"].tooltip)
+        label4 = QLabel("HDF5 meta_block_size")
+        label4.setAlignment(Qt.AlignRight | Qt.AlignVCenter)
+        label4.setToolTip(SPIN_BOX_SETTINGS["meta_block"].tooltip)
         self.logs_combo = QComboBox(self._base)
         self.logs_combo.addItems(OutputTrajectoryConfigurator.log_options)
         self._layout.addWidget(self._field, 0, 0)
         self._layout.addWidget(self.dtype_box, 0, 1)
         self._layout.addWidget(self.compression_box, 0, 2)
         self._layout.addWidget(browse_button, 0, 3)
-        self._layout.addWidget(label, 1, 0)
-        self._layout.addWidget(self.logs_combo, 1, 1)
-        self._layout.addWidget(label2, 1, 2)
-        self._layout.addWidget(self.chunk_atom_box, 1, 3)
-        self._layout.addWidget(label3, 2, 2)
-        self._layout.addWidget(self.chunk_frame_box, 2, 3)
+        self._layout.addWidget(label2, 1, 0)
+        self._layout.addWidget(self.chunk_atom_box, 1, 1)
+        self._layout.addWidget(label3, 1, 2)
+        self._layout.addWidget(self.chunk_frame_box, 1, 3)
+        self._layout.addWidget(label4, 2, 0)
+        self._layout.addWidget(self.meta_block_box, 2, 1)
+        self._layout.addWidget(label, 2, 2)
+        self._layout.addWidget(self.logs_combo, 2, 3)
         self._default_value = default_value
         self._field.textChanged.connect(self.updateValue)
         self.default_labels()
@@ -169,4 +198,5 @@ def get_widget_value(self):
         chunk_size = (self.chunk_frame_box.value(), self.chunk_atom_box.value())
         compression = self.compression_box.currentText()
         logs = self.logs_combo.currentText()
-        return (filename, dtype, chunk_size, compression, logs)
+        meta_block_size = self.meta_block_box.value()
+        return (filename, dtype, chunk_size, compression, logs, meta_block_size)

Original file line number	Diff line number	Diff line change
`@@ -147,6 +147,7 @@ def initialize(self):`
`147`	`147`	`positions_dtype=self.configuration["output_files"]["dtype"],`
`148`	`148`	`chunking_limit=self.configuration["output_files"]["chunk_size"],`
`149`	`149`	`compression=self.configuration["output_files"]["compression"],`
	`150`	`+ meta_block_size=self.configuration["output_files"]["meta_block_size"],`
`150`	`151`	`initial_charges=self._initial_charges,`
`151`	`152`	`)`
`152`	`153`
Original file line number	Diff line number	Diff line change
`@@ -97,6 +97,7 @@ def initialize(self):`
`97`	`97`	`positions_dtype=self.configuration["output_files"]["dtype"],`
`98`	`98`	`chunking_limit=self.configuration["output_files"]["chunk_size"],`
`99`	`99`	`compression=self.configuration["output_files"]["compression"],`
	`100`	`+ meta_block_size=self.configuration["output_files"]["meta_block_size"],`
`100`	`101`	`)`
`101`	`102`
`102`	`103`	`def run_step(self, index: int) -> tuple[int, None]:`
Original file line number	Diff line number	Diff line change
`@@ -154,6 +154,7 @@ def initialize(self):`
`154`	`154`	`positions_dtype=self.configuration["output_files"]["dtype"],`
`155`	`155`	`chunking_limit=self.configuration["output_files"]["chunk_size"],`
`156`	`156`	`compression=self.configuration["output_files"]["compression"],`
	`157`	`+ meta_block_size=self.configuration["output_files"]["meta_block_size"],`
`157`	`158`	`)`
`158`	`159`
`159`	`160`	`data_to_be_written = ["configuration", "time"]`
Original file line number	Diff line number	Diff line change
`@@ -104,6 +104,7 @@ def initialize(self):`
`104`	`104`	`positions_dtype=self.configuration["output_files"]["dtype"],`
`105`	`105`	`chunking_limit=self.configuration["output_files"]["chunk_size"],`
`106`	`106`	`compression=self.configuration["output_files"]["compression"],`
	`107`	`+ meta_block_size=self.configuration["output_files"]["meta_block_size"],`
`107`	`108`	`)`
`108`	`109`
`109`	`110`	`def run_step(self, index):`
Original file line number	Diff line number	Diff line change
`@@ -113,6 +113,7 @@ def initialize(self):`
`113`	`113`	`positions_dtype=self.configuration["output_files"]["dtype"],`
`114`	`114`	`chunking_limit=self.configuration["output_files"]["chunk_size"],`
`115`	`115`	`compression=self.configuration["output_files"]["compression"],`
	`116`	`+ meta_block_size=self.configuration["output_files"]["meta_block_size"],`
`116`	`117`	`initial_charges=self.field_file.get_atom_charges(),`
`117`	`118`	`)`
`118`	`119`
Original file line number	Diff line number	Diff line change
`@@ -148,6 +148,7 @@ def initialize(self):`
`148`	`148`	`positions_dtype=self.configuration["output_files"]["dtype"],`
`149`	`149`	`chunking_limit=self.configuration["output_files"]["chunk_size"],`
`150`	`150`	`compression=self.configuration["output_files"]["compression"],`
	`151`	`+ meta_block_size=self.configuration["output_files"]["meta_block_size"],`
`151`	`152`	`initial_charges=charges,`
`152`	`153`	`)`
`153`	`154`
Original file line number	Diff line number	Diff line change
`@@ -141,6 +141,7 @@ def initialize(self):`
`141`	`141`	`positions_dtype=self.configuration["output_files"]["dtype"],`
`142`	`142`	`chunking_limit=self.configuration["output_files"]["chunk_size"],`
`143`	`143`	`compression=self.configuration["output_files"]["compression"],`
	`144`	`+ meta_block_size=self.configuration["output_files"]["meta_block_size"],`
`144`	`145`	`)`
`145`	`146`
`146`	`147`	`def run_step(self, index):`
Original file line number	Diff line number	Diff line change
`@@ -212,6 +212,7 @@ def initialize(self):`
`212`	`212`	`positions_dtype=self.configuration["output_files"]["dtype"],`
`213`	`213`	`chunking_limit=self.configuration["output_files"]["chunk_size"],`
`214`	`214`	`compression=self.configuration["output_files"]["compression"],`
	`215`	`+ meta_block_size=self.configuration["output_files"]["meta_block_size"],`
`215`	`216`	`initial_charges=charges,`
`216`	`217`	`)`
`217`	`218`
Original file line number	Diff line number	Diff line change
`@@ -174,6 +174,7 @@ def initialize(self):`
`174`	`174`	`"positions_dtype": self.configuration["output_files"]["dtype"],`
`175`	`175`	`"chunking_limit": self.configuration["output_files"]["chunk_size"],`
`176`	`176`	`"compression": self.configuration["output_files"]["compression"],`
	`177`	`+ "meta_block_size": self.configuration["output_files"]["meta_block_size"],`
`177`	`178`	`}`
`178`	`179`	`if hasattr(self.u.atoms, "charges"):`
`179`	`180`	`kwargs["initial_charges"] = self.u.atoms.charges`