Merge pull request #1063 from fractal-analytics-platform/1061_apply_registration_fix

Fix table copying in apply_registration_to_image task

Author: jluethi

CHANGELOG.md

@@ -1,6 +1,19 @@
 **Note**: Numbers like (\#123) point to closed Pull Requests on the fractal-tasks-core repository.
 
-# Unreleased
+# 2.1.0
+* Tasks:
+    * Change default table backends to better fit each task's data (\#1063):
+        * `measure_features`: default `table_backend` is now `"parquet"` (was `"anndata"`).
+        * `import_ome_zarr`: default `table_backend` for `image_ROI_table`/`grid_ROI_table` is now `"csv"` (was `"anndata"`).
+        * `threshold_segmentation`'s `CreateMaskingRoiTable`: default `table_backend` is now `"csv"` (was `"anndata"`).
+        * `projection`/`compute_projection_hcs`: confirmed (no code change needed) that the projected image's ROI tables already preserve the input image's table backend, and added regression tests for this.
+    * Refactor `import_ome_zarr` task interface (\#1063):
+        * `update_omero_metadata`, `add_image_roi_table`, and `add_grid_roi_table` now default to `False` (previously `True`), so ROI-table/metadata creation is now opt-in.
+        * All parameters after `zarr_name` are now grouped into a single `advanced_options: AdvancedOptions` argument.
+        * `add_grid_roi_table`/`grid_y_shape`/`grid_x_shape` are replaced by `advanced_options.grid_roi_table: GridRoiTable | None`; set it to build a `grid_ROI_table`, leave it `None` to skip it.
+    * Fix table copying in `apply_registration_to_image` task: Use the existing table backend for the newly written table, perform table writing outside the try/except loop to catch reading race conditions and provide reading exception in the Runtime Error (\#1063).
+* Chore:
+    * Handle deprecation warnings in ngio (\#1063).
 * Documentation:
     * Remove `extra_javascript` from docs (\#1055).
     * Move from `mkdocs` to `zensical` (\#1060).

fractal_tasks_core/__FRACTAL_MANIFEST__.json

@@ -418,8 +418,8 @@
                 "type": "string"
               },
               "table_backend": {
-                "default": "anndata",
-                "description": "Backend to use for storing the masking ROI table. Options are \"anndata\", \"json\",\n\"csv\", and \"parquet\".",
+                "default": "csv",
+                "description": "Backend to use for storing the masking ROI table. Options are \"anndata\", \"json\",\n\"csv\", and \"parquet\". Defaults to \"csv\".",
                 "enum": [
                   "anndata",
                   "json",
@@ -961,8 +961,8 @@
                 "type": "boolean"
               },
               "table_backend": {
-                "default": "anndata",
-                "description": "Table backend to use for the output table. Defaults to \"anndata\".",
+                "default": "parquet",
+                "description": "Table backend to use for the output table. Defaults to \"parquet\".",
                 "enum": [
                   "anndata",
                   "json",
@@ -1122,7 +1122,7 @@
               "level_path": null,
               "use_scaling": true,
               "use_cache": true,
-              "table_backend": "anndata"
+              "table_backend": "parquet"
             },
             "title": "Advanced Options",
             "description": "Advanced options for feature measurement."
@@ -1458,6 +1458,77 @@
       "type": "converter_non_parallel",
       "executable_non_parallel": "import_ome_zarr.py",
       "args_schema_non_parallel": {
+        "$defs": {
+          "AdvancedOptions": {
+            "description": "Advanced options for importing an OME-Zarr.",
+            "properties": {
+              "update_omero_metadata": {
+                "default": false,
+                "description": "Whether to update Omero-channels metadata with channel labels and\nwavelength ids to make them compatible with some downstream Fractal tasks.",
+                "title": "Update Omero Metadata",
+                "type": "boolean"
+              },
+              "add_image_roi_table": {
+                "default": false,
+                "description": "Whether to add an `image_ROI_table` table to each image, with a single ROI\ncovering the whole image.",
+                "title": "Add Image Roi Table",
+                "type": "boolean"
+              },
+              "grid_roi_table": {
+                "anyOf": [
+                  {
+                    "$ref": "#/$defs/GridRoiTable"
+                  },
+                  {
+                    "type": "null"
+                  }
+                ],
+                "default": null,
+                "description": "If set, add a `grid_ROI_table` table to each image, with the image split\ninto a rectangular grid of ROIs of the given shape. If `None`, no\n`grid_ROI_table` is created.",
+                "title": "Grid_Roi_Table"
+              },
+              "table_backend": {
+                "default": "csv",
+                "description": "Table backend to use for the new ROI tables. Defaults to \"csv\".",
+                "enum": [
+                  "anndata",
+                  "json",
+                  "csv",
+                  "parquet"
+                ],
+                "title": "Table Backend",
+                "type": "string"
+              },
+              "overwrite": {
+                "default": false,
+                "description": "Whether new ROI tables (added when `add_image_roi_table=True` and/or\n`grid_roi_table` is not `None`) can overwrite existing ones.",
+                "title": "Overwrite",
+                "type": "boolean"
+              }
+            },
+            "title": "AdvancedOptions",
+            "type": "object"
+          },
+          "GridRoiTable": {
+            "description": "Configuration for building a `grid_ROI_table`.",
+            "properties": {
+              "grid_y_shape": {
+                "default": 2,
+                "description": "Number of ROIs along the Y axis. The image is split into a `grid_y_shape` by\n`grid_x_shape` grid of ROIs (e.g. the default 2 by 2 produces 4 ROIs).",
+                "title": "Grid Y Shape",
+                "type": "integer"
+              },
+              "grid_x_shape": {
+                "default": 2,
+                "description": "Number of ROIs along the X axis. The image is split into a `grid_y_shape` by\n`grid_x_shape` grid of ROIs (e.g. the default 2 by 2 produces 4 ROIs).",
+                "title": "Grid X Shape",
+                "type": "integer"
+              }
+            },
+            "title": "GridRoiTable",
+            "type": "object"
+          }
+        },
         "additionalProperties": false,
         "properties": {
           "zarr_dir": {
@@ -1470,53 +1541,17 @@
             "type": "string",
             "description": "The OME-Zarr name, without its parent folder. The parent folder is provided by zarr_dir; e.g. `zarr_name=\"array.zarr\"`, if the OME-Zarr path is in `/zarr_dir/array.zarr`."
           },
-          "update_omero_metadata": {
-            "default": true,
-            "title": "Update Omero Metadata",
-            "type": "boolean",
-            "description": "Whether to update Omero-channels metadata, to make them Fractal-compatible."
-          },
-          "add_image_roi_table": {
-            "default": true,
-            "title": "Add Image Roi Table",
-            "type": "boolean",
-            "description": "Whether to add a `image_ROI_table` table to each image, with a single ROI covering the whole image."
-          },
-          "add_grid_roi_table": {
-            "default": true,
-            "title": "Add Grid Roi Table",
-            "type": "boolean",
-            "description": "Whether to add a `grid_ROI_table` table to each image, with the image split into a rectangular grid of ROIs."
-          },
-          "grid_y_shape": {
-            "default": 2,
-            "title": "Grid Y Shape",
-            "type": "integer",
-            "description": "Number of ROIs along the Y axis of `grid_ROI_table`. The image is split into a `grid_y_shape` by `grid_x_shape` grid of ROIs (e.g. the default 2 by 2 produces 4 ROIs)."
-          },
-          "grid_x_shape": {
-            "default": 2,
-            "title": "Grid X Shape",
-            "type": "integer",
-            "description": "Number of ROIs along the X axis of `grid_ROI_table`. The image is split into a `grid_y_shape` by `grid_x_shape` grid of ROIs (e.g. the default 2 by 2 produces 4 ROIs)."
-          },
-          "table_backend": {
-            "default": "anndata",
-            "enum": [
-              "anndata",
-              "json",
-              "csv",
-              "parquet"
-            ],
-            "title": "Table Backend",
-            "type": "string",
-            "description": "Backend to use for the new ROI tables. Defaults to \"anndata\"."
-          },
-          "overwrite": {
-            "default": false,
-            "title": "Overwrite",
-            "type": "boolean",
-            "description": "Whether new ROI tables (added when `add_image_roi_table` and/or `add_grid_roi_table` are `True`) can overwrite existing ones."
+          "advanced_options": {
+            "$ref": "#/$defs/AdvancedOptions",
+            "default": {
+              "update_omero_metadata": false,
+              "add_image_roi_table": false,
+              "grid_roi_table": null,
+              "table_backend": "csv",
+              "overwrite": false
+            },
+            "title": "Advanced Options",
+            "description": "Advanced options for importing an OME-Zarr, including whether to add `image_ROI_table`/`grid_ROI_table` tables, whether to update Omero-channels metadata, the table backend to use, and whether new tables can overwrite existing ones."
           }
         },
         "required": [

fractal_tasks_core/_import_ome_zarr_utils.py

@@ -0,0 +1,56 @@
+# Copyright 2022-2026 (C) BioVisionCenter, University of Zurich
+"""Pydantic models for the import_ome_zarr task."""
+
+from pydantic import BaseModel
+
+from fractal_tasks_core._utils import AVAILABLE_TABLE_BACKENDS
+
+
+class GridRoiTable(BaseModel):
+    """Configuration for building a `grid_ROI_table`."""
+
+    grid_y_shape: int = 2
+    """
+    Number of ROIs along the Y axis. The image is split into a `grid_y_shape` by
+    `grid_x_shape` grid of ROIs (e.g. the default 2 by 2 produces 4 ROIs).
+    """
+
+    grid_x_shape: int = 2
+    """
+    Number of ROIs along the X axis. The image is split into a `grid_y_shape` by
+    `grid_x_shape` grid of ROIs (e.g. the default 2 by 2 produces 4 ROIs).
+    """
+
+
+class AdvancedOptions(BaseModel):
+    """Advanced options for importing an OME-Zarr."""
+
+    update_omero_metadata: bool = False
+    """
+    Whether to update Omero-channels metadata with channel labels and
+    wavelength ids to make them compatible with some downstream Fractal tasks.
+    """
+
+    add_image_roi_table: bool = False
+    """
+    Whether to add an `image_ROI_table` table to each image, with a single ROI
+    covering the whole image.
+    """
+
+    grid_roi_table: GridRoiTable | None = None
+    """
+    If set, add a `grid_ROI_table` table to each image, with the image split
+    into a rectangular grid of ROIs of the given shape. If `None`, no
+    `grid_ROI_table` is created.
+    """
+
+    table_backend: AVAILABLE_TABLE_BACKENDS = "csv"
+    """
+    Table backend to use for the new ROI tables. Defaults to "csv".
+    """
+
+    overwrite: bool = False
+    """
+    Whether new ROI tables (added when `add_image_roi_table=True` and/or
+    `grid_roi_table` is not `None`) can overwrite existing ones.
+    """

fractal_tasks_core/_measure_features_utils.py

@@ -12,7 +12,7 @@
 from pydantic import BaseModel, Field
 from skimage import measure
 
-from fractal_tasks_core._utils import AVAILABLE_TABLE_BACKENDS, DEFAULT_TABLE_BACKEND
+from fractal_tasks_core._utils import AVAILABLE_TABLE_BACKENDS
 
 
 class ShapeFeatures(BaseModel):
@@ -155,9 +155,9 @@ class AdvancedOptions(BaseModel):
     but can also increase memory usage. Defaults to True.
     """
 
-    table_backend: AVAILABLE_TABLE_BACKENDS = DEFAULT_TABLE_BACKEND
+    table_backend: AVAILABLE_TABLE_BACKENDS = "parquet"
     """
-    Table backend to use for the output table. Defaults to "anndata".
+    Table backend to use for the output table. Defaults to "parquet".
     """
 
 

fractal_tasks_core/_threshold_segmentation_utils.py

@@ -10,7 +10,7 @@
 from skimage.filters import threshold_otsu
 from skimage.measure import label
 
-from fractal_tasks_core._utils import AVAILABLE_TABLE_BACKENDS, DEFAULT_TABLE_BACKEND
+from fractal_tasks_core._utils import AVAILABLE_TABLE_BACKENDS
 
 logger = logging.getLogger("threshold_segmentation_task_utils")
 
@@ -32,10 +32,10 @@ class CreateMaskingRoiTable(BaseModel):
     "{output_label_name}", which will be replaced by the name of the label image used
     for segmentation.
     """
-    table_backend: AVAILABLE_TABLE_BACKENDS = DEFAULT_TABLE_BACKEND
+    table_backend: AVAILABLE_TABLE_BACKENDS = "csv"
     """
     Backend to use for storing the masking ROI table. Options are "anndata", "json",
-    "csv", and "parquet".
+    "csv", and "parquet". Defaults to "csv".
     """
 
     def get_table_name(self, output_label_name: str) -> str:

fractal_tasks_core/_utils.py

@@ -5,7 +5,6 @@
 from typing import Literal, TypeAlias
 
 AVAILABLE_TABLE_BACKENDS: TypeAlias = Literal["anndata", "json", "csv", "parquet"]
-DEFAULT_TABLE_BACKEND: AVAILABLE_TABLE_BACKENDS = "anndata"
 
 
 @dataclass

fractal_tasks_core/apply_registration_to_image.py

@@ -254,23 +254,38 @@ def apply_registration_to_image(
         for table_name, source in tables_to_copy.items():
             logger.info(f"Copying table: {table_name}")
             # Retry loop to guard against race conditions (see issue #516)
+            last_exception: Exception | None = None
             for attempt in range(max_retries):
                 try:
                     table = source.get_table(table_name)
-                    new_ome_zarr.add_table(name=table_name, table=table, overwrite=True)
                     break
-                except Exception:
+                except Exception as e:
+                    last_exception = e
                     logger.debug(
                         f"Table {table_name} not found in attempt {attempt}. "
                         f"Waiting {sleep_time} seconds before trying again."
                     )
                     time.sleep(sleep_time)
             else:
                 raise RuntimeError(
-                    f"Table {table_name} not found after {max_retries} attempts. "
-                    "Check whether this table actually exists. If it does, "
-                    "this may be a race condition issue."
+                    f"Table {table_name} could not be loaded after "
+                    f"{max_retries} attempts. "
+                    f"The original error was: {last_exception}"
+                ) from last_exception
+            # Writing table outside of the retry loop intentionally. We only
+            # want to catch loading race conditions above, not real writing
+            # errors.
+            # Only overwrite the backend if it is set before (which should
+            # always be the case for loaded tables)
+            if table.backend_name:
+                new_ome_zarr.add_table(
+                    name=table_name,
+                    table=table,
+                    backend=table.backend_name,
+                    overwrite=True,
                 )
+            else:
+                new_ome_zarr.add_table(name=table_name, table=table, overwrite=True)
 
     ####################
     # Clean up Zarr file