Various YAML Tool Hardening and Progress toward Tool State Goals

lib/galaxy/app_unittest_utils/tools_support.py

@@ -27,12 +27,17 @@
 galaxy.model.set_datatypes_registry(datatypes_registry)
 
 
+def mock_app_for_tool_support() -> UniverseApplication:
+    app = cast(UniverseApplication, MockApp())
+    app.config.new_file_path = tempfile.mkdtemp()
+    app.config.admin_users = "mary@example.com"
+    return app
+
+
 class UsesApp:
     def setup_app(self):
         self.test_directory = tempfile.mkdtemp()
-        self.app = cast(UniverseApplication, MockApp())
-        self.app.config.new_file_path = os.path.join(self.test_directory, "new_files")
-        self.app.config.admin_users = "mary@example.com"
+        self.app = mock_app_for_tool_support()
 
     def tear_down_app(self):
         shutil.rmtree(self.test_directory)

lib/galaxy/celery/tasks.py

@@ -350,11 +350,13 @@ def fetch_data(
 
 @galaxy_task(action="queuing up submitted jobs")
 def queue_jobs(request: QueueJobs, app: MinimalManagerApp, job_submitter: JobSubmitter):
+    raw_tool_source = request.tool_source.raw_tool_source
+    tool_source_class = request.tool_source.tool_source_class
     tool = cached_create_tool_from_representation(
         app=app,
-        raw_tool_source=request.tool_source.raw_tool_source,
+        raw_tool_source=raw_tool_source,
         tool_dir=request.tool_source.tool_dir,
-        tool_source_class=request.tool_source.tool_source_class,
+        tool_source_class=tool_source_class,
     )
 
     job_submitter.queue_jobs(

lib/galaxy/model/__init__.py

@@ -1394,6 +1394,7 @@ class ToolSource(Base, Dictifiable, RepresentById):
     id: Mapped[int] = mapped_column(primary_key=True)
     hash: Mapped[Optional[str]] = mapped_column(Unicode(255))
     source: Mapped[dict] = mapped_column(JSONType)
+    source_class: Mapped[str] = mapped_column(TrimmedString(255))
 
 
 class ToolRequest(Base, Dictifiable, RepresentById):
@@ -1403,7 +1404,7 @@ class ToolRequest(Base, Dictifiable, RepresentById):
 
     id: Mapped[int] = mapped_column(primary_key=True)
     tool_source_id: Mapped[int] = mapped_column(ForeignKey("tool_source.id"), index=True)
-    history_id: Mapped[Optional[int]] = mapped_column(ForeignKey("history.id"), index=True)
+    history_id: Mapped[int] = mapped_column(ForeignKey("history.id"), index=True, nullable=False)
     request: Mapped[dict] = mapped_column(JSONType)
     state: Mapped[Optional[str]] = mapped_column(TrimmedString(32), index=True)
     state_message: Mapped[Optional[str]] = mapped_column(JSONType, index=True)
@@ -1619,6 +1620,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable):
     preferred_object_store_id: Mapped[Optional[str]] = mapped_column(String(255))
     object_store_id_overrides: Mapped[Optional[dict[str, Optional[str]]]] = mapped_column(JSONType)
     tool_request_id: Mapped[Optional[int]] = mapped_column(ForeignKey("tool_request.id"), index=True)
+    tool_state: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON().with_variant(JSONB, "postgresql"))
 
     dynamic_tool: Mapped[Optional["DynamicTool"]] = relationship()
     tool_request: Mapped[Optional["ToolRequest"]] = relationship(back_populates="jobs")
@@ -1764,6 +1766,7 @@ def copy_from_job(self, job: "Job", copy_outputs: bool = False):
         self.exit_code = job.exit_code
         self.job_runner_name = job.job_runner_name
         self.job_runner_external_id = job.job_runner_external_id
+        self.tool_state = job.tool_state
         if copy_outputs:
             assert self.history
             requires_addition_to_history = False
@@ -2205,6 +2208,7 @@ def _serialize(self, id_encoder, serialization_options):
         job_attrs["create_time"] = self.create_time.isoformat()
         job_attrs["update_time"] = self.update_time.isoformat()
         job_attrs["job_messages"] = self.job_messages
+        job_attrs["tool_state"] = self.tool_state
 
         # Get the job's parameters
         param_dict = self.raw_param_dict()

lib/galaxy/model/migrations/alembic/versions_gxy/566b691307a5_persist_validated_job_internal_tool_.py

@@ -0,0 +1,77 @@
+"""Persist validated job internal tool state and enforce tool request constraints.
+
+Revision ID: 566b691307a5
+Revises: 9930b68c85af
+Create Date: 2025-09-30 04:48:19.727414
+
+"""
+
+from sqlalchemy import (
+    Column,
+    Integer,
+    JSON,
+    String,
+)
+from sqlalchemy.dialects.postgresql import JSONB
+
+from galaxy.model.migrations.util import (
+    add_column,
+    alter_column,
+    drop_column,
+    transaction,
+)
+
+# revision identifiers, used by Alembic.
+revision = "566b691307a5"
+down_revision = "9930b68c85af"
+branch_labels = None
+depends_on = None
+
+table_name = "job"
+column_name = "tool_state"
+
+# additional database object names used in this revision
+tool_request_table_name = "tool_request"
+tool_request_history_id_column = "history_id"
+
+tool_source_table_name = "tool_source"
+tool_source_source_class_column = "source_class"
+
+
+def upgrade():
+    # Wrap statements for sqlite safety and to ensure atomicity where supported
+    with transaction():
+        # Persist validated job internal tool state
+        add_column(table_name, Column(column_name, JSON().with_variant(JSONB, "postgresql")))
+
+        # Ensure tool_request.history_id is NOT NULL
+        alter_column(
+            tool_request_table_name,
+            tool_request_history_id_column,
+            existing_type=Integer(),
+            nullable=False,
+        )
+
+        # Add tool_source.source_class as a new NOT NULL string column
+        add_column(
+            tool_source_table_name,
+            Column(tool_source_source_class_column, String(255), nullable=False),
+        )
+
+
+def downgrade():
+    # Reverse operations in a transaction
+    with transaction():
+        # Drop tool_source.source_class
+        drop_column(tool_source_table_name, tool_source_source_class_column)
+
+        # Revert tool_request.history_id to be nullable
+        alter_column(
+            tool_request_table_name,
+            tool_request_history_id_column,
+            existing_type=Integer(),
+            nullable=True,
+        )
+
+        # Drop job.tool_state
+        drop_column(table_name, column_name)

lib/galaxy/model/store/__init__.py

@@ -1684,6 +1684,7 @@ def _set_job_attributes(
             "job_stdout",
             "job_stderr",
             "galaxy_version",
+            "tool_state",
         )
         for attribute in ATTRIBUTES:
             value = job_attrs.get(attribute)

lib/galaxy/tool_util/parameters/__init__.py

@@ -65,10 +65,12 @@
     validate_internal_landing_request,
     validate_internal_request,
     validate_internal_request_dereferenced,
+    validate_job_runtime,
     validate_landing_request,
     validate_relaxed_request,
     validate_request,
     validate_test_case,
+    validate_test_case_json,
     validate_workflow_step,
     validate_workflow_step_linked,
     ValidationFunctionT,
@@ -82,6 +84,7 @@
     RequestInternalDereferencedToolState,
     RequestInternalToolState,
     RequestToolState,
+    TestCaseJsonToolState,
     TestCaseToolState,
     ToolState,
     WorkflowStepLinkedToolState,
@@ -142,18 +145,21 @@
     "validate_against_model",
     "validate_internal_job",
     "validate_internal_landing_request",
+    "validate_job_runtime",
     "validate_internal_request",
     "validate_internal_request_dereferenced",
     "validate_landing_request",
     "validate_relaxed_request",
     "validate_request",
     "validate_test_case",
+    "validate_test_case_json",
     "validate_workflow_step",
     "validate_workflow_step_linked",
     "validate_explicit_conditional_test_value",
     "is_optional",
     "ToolState",
     "TestCaseToolState",
+    "TestCaseJsonToolState",
     "ToolParameterT",
     "HasToolParameters",
     "to_json_schema_string",

lib/galaxy/tool_util/parameters/case.py

@@ -10,6 +10,7 @@
 )
 
 from packaging.version import Version
+from typing_extensions import Literal
 
 from galaxy.tool_util.parser.interface import (
     TestCollectionDef,
@@ -160,7 +161,10 @@ def test_case_state(
     unhandled_inputs = []
     state: Dict[str, Any] = {}
 
-    handled_inputs = _merge_level_into_state(tool_parameter_bundle, inputs, state, profile, warnings, None)
+    state_representation = test_dict.get("value_state_representation", "test_case_xml")
+    handled_inputs = _merge_level_into_state(
+        tool_parameter_bundle, inputs, state, profile, state_representation, warnings, None
+    )
 
     for test_input in inputs:
         input_name = test_input["name"]
@@ -200,12 +204,15 @@ def _merge_level_into_state(
     inputs: ToolSourceTestInputs,
     state_at_level: dict,
     profile: str,
+    state_representation: Literal["test_case_xml", "test_case_json"],
     warnings: List[str],
     prefix: Optional[str],
 ) -> Set[str]:
     handled_inputs: Set[str] = set()
     for tool_input in tool_inputs:
-        handled_inputs.update(_merge_into_state(tool_input, inputs, state_at_level, profile, warnings, prefix))
+        handled_inputs.update(
+            _merge_into_state(tool_input, inputs, state_at_level, profile, state_representation, warnings, prefix)
+        )
 
     return handled_inputs
 
@@ -223,6 +230,7 @@ def _merge_into_state(
     inputs: ToolSourceTestInputs,
     state_at_level: dict,
     profile: str,
+    state_representation: Literal["test_case_xml", "test_case_json"],
     warnings: List[str],
     prefix: Optional[str],
 ) -> Set[str]:
@@ -240,10 +248,14 @@ def _merge_into_state(
         when: ConditionalWhen = _select_which_when(tool_input, conditional_state, inputs, state_path)
         test_parameter = tool_input.test_parameter
         handled_inputs.update(
-            _merge_into_state(test_parameter, inputs, conditional_state, profile, warnings, state_path)
+            _merge_into_state(
+                test_parameter, inputs, conditional_state, profile, state_representation, warnings, state_path
+            )
         )
         handled_inputs.update(
-            _merge_level_into_state(when.parameters, inputs, conditional_state, profile, warnings, state_path)
+            _merge_level_into_state(
+                when.parameters, inputs, conditional_state, profile, state_representation, warnings, state_path
+            )
         )
     elif isinstance(tool_input, (RepeatParameterModel,)):
         repeat_state_array = state_at_level.get(input_name, [])
@@ -261,7 +273,13 @@ def _merge_into_state(
             repeat_instance_prefix = f"{state_path}_{i}"
             handled_inputs.update(
                 _merge_level_into_state(
-                    tool_input.parameters, inputs, repeat_state_array[i], profile, warnings, repeat_instance_prefix
+                    tool_input.parameters,
+                    inputs,
+                    repeat_state_array[i],
+                    profile,
+                    state_representation,
+                    warnings,
+                    repeat_instance_prefix,
                 )
             )
     elif isinstance(tool_input, (SectionParameterModel,)):
@@ -270,7 +288,9 @@ def _merge_into_state(
             state_at_level[input_name] = section_state
 
         handled_inputs.update(
-            _merge_level_into_state(tool_input.parameters, inputs, section_state, profile, warnings, state_path)
+            _merge_level_into_state(
+                tool_input.parameters, inputs, section_state, profile, state_representation, warnings, state_path
+            )
         )
     else:
         test_input = _input_for(state_path, inputs)
@@ -283,17 +303,24 @@ def _merge_into_state(
             elif isinstance(tool_input, (DataParameterModel,)):
                 if tool_input.multiple:
                     value = test_input["value"]
-                    input_value_list = []
+                    input_value_list: List[Any] = []
                     if value:
-                        test_input_values = cast(str, value).split(",")
-                        for test_input_value in test_input_values:
-                            instance_test_input = test_input.copy()
-                            instance_test_input["value"] = test_input_value
-                            input_value_json = xml_data_input_to_json(instance_test_input)
-                            input_value_list.append(input_value_json)
+                        if state_representation == "test_case_json":
+                            input_value_list = test_input["value"] if test_input["value"] is not None else []
+                        else:
+                            test_input_values = cast(str, value).split(",")
+                            for test_input_value in test_input_values:
+                                instance_test_input = test_input.copy()
+                                instance_test_input["value"] = test_input_value
+                                input_value_json = xml_data_input_to_json(instance_test_input)
+                                input_value_list.append(input_value_json)
+
                     input_value = input_value_list
                 else:
-                    input_value = xml_data_input_to_json(test_input)
+                    if state_representation == "test_case_json":
+                        input_value = test_input["value"]
+                    else:
+                        input_value = xml_data_input_to_json(test_input)
             else:
                 input_value = test_input["value"]
                 input_value = legacy_from_string(tool_input, input_value, warnings, profile)