secondaryFiles in outputs

JensKrumsieck · JensKrumsieck · commit 796e37cc0c8d · 2025-05-20T11:18:00.000+02:00
diff --git a/crates/cwl-execution/src/expression.rs b/crates/cwl-execution/src/expression.rs
@@ -217,9 +217,9 @@ pub(crate) fn process_expressions(tool: &mut CWLDocument, input_values: &mut Inp
         if input.secondary_files.is_empty() {
             continue;
         }
-        
-        if let DefaultValue::File(file) = &input_values.inputs[&tmp] {            
-            set_self(&file.snapshot())?;
+
+        if let Some(DefaultValue::File(file)) = input_values.inputs.get(&tmp) {
+            set_self(&file.preload())?;
             for sec_file in &mut input.secondary_files {
                 sec_file.pattern = replace_expressions(&sec_file.pattern)?;
             }
diff --git a/crates/cwl-execution/src/util.rs b/crates/cwl-execution/src/util.rs
@@ -1,5 +1,5 @@
 use crate::{
-    expression::{output_eval, set_self, unset_self},
+    expression::{output_eval, replace_expressions, set_self, unset_self},
     io::{copy_dir, copy_file, get_first_file_with_prefix},
 };
 use cwl::{
@@ -10,6 +10,7 @@ use cwl::{
     types::{CWLType, DefaultValue, Directory, File},
 };
 use glob::glob;
+use log::info;
 use serde_yaml::{Mapping, Value};
 use std::{
     collections::HashMap,
@@ -233,10 +234,31 @@ fn evaluate_output_impl(
 
 fn handle_file_output(entry: &PathBuf, initial_dir: &Path, output: &CommandOutputParameter) -> Result<DefaultValue, Box<dyn Error>> {
     let current_dir = env::temp_dir().to_string_lossy().into_owned();
-    let path = &initial_dir.join(entry.strip_prefix(current_dir).unwrap_or(entry));
+    let path = &initial_dir.join(entry.strip_prefix(&current_dir).unwrap_or(entry));
     fs::copy(entry, path).map_err(|e| format!("Failed to copy file from {entry:?} to {path:?}: {e}"))?;
-    eprintln!("📜 Wrote output file: {path:?}");
-    Ok(DefaultValue::File(get_file_metadata(path, output.format.clone())))
+    info!("📜 Wrote output file: {path:?}");
+
+    let mut file = get_file_metadata(path, output.format.clone());
+    if !output.secondary_files.is_empty() {
+        set_self(&file)?;
+        let folder = entry.parent().unwrap_or(Path::new("."));
+        let mut secondary_files = vec![];
+        for secondary in &output.secondary_files {
+            let pattern = replace_expressions(&secondary.pattern)?;
+            let pattern = format!("{}*{}", folder.to_string_lossy(), pattern);
+            for entry in glob(&pattern)? {
+                let entry = entry?;
+                let sec_path = initial_dir.join(&entry);
+                fs::copy(&entry, &sec_path).map_err(|e| format!("Failed to copy file from {entry:?} to {sec_path:?}: {e}"))?;
+                info!("📜 Wrote secondary file: {sec_path:?}");
+                secondary_files.push(DefaultValue::File(get_file_metadata(&sec_path, None)));
+            }
+        }
+        file.secondary_files = Some(secondary_files);
+        unset_self()?;
+    }
+
+    Ok(DefaultValue::File(file))
 }
 
 fn handle_dir_output(entry: &PathBuf, initial_dir: &Path) -> Result<DefaultValue, Box<dyn Error>> {
diff --git a/crates/cwl/src/clt.rs b/crates/cwl/src/clt.rs
@@ -300,8 +300,7 @@ baseCommand:
         let tool = CommandLineTool::default().with_outputs(vec![CommandOutputParameter {
             id: "stdout".to_string(),
             type_: CWLType::Stdout,
-            output_binding: None,
-            format: None,
+            ..Default::default()
         }]);
         assert!(tool.has_stdout_output());
     }
@@ -311,8 +310,7 @@ baseCommand:
         let tool = CommandLineTool::default().with_outputs(vec![CommandOutputParameter {
             id: "stderr".to_string(),
             type_: CWLType::Stderr,
-            output_binding: None,
-            format: None,
+            ..Default::default()
         }]);
         assert!(tool.has_stderr_output());
     }
diff --git a/crates/cwl/src/inputs.rs b/crates/cwl/src/inputs.rs
@@ -1,3 +1,5 @@
+use crate::types::SecondaryFileSchema;
+
 use super::{
     deserialize::Identifiable,
     types::{CWLType, DefaultValue},
@@ -121,48 +123,6 @@ where
     Ok(parameters)
 }
 
-#[derive(Serialize, Debug, Default, PartialEq, Clone)]
-pub struct SecondaryFileSchema {
-    pub pattern: String,
-    pub required: bool,
-}
-
-impl From<String> for SecondaryFileSchema {
-    fn from(pattern: String) -> Self {
-        if pattern.ends_with("?") {
-            let pattern = pattern.trim_end_matches('?').to_string();
-            SecondaryFileSchema { pattern, required: false }
-        } else {
-            SecondaryFileSchema { pattern, required: true }
-        }
-    }
-}
-
-impl<'de> Deserialize<'de> for SecondaryFileSchema {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let value: Value = Deserialize::deserialize(deserializer)?;
-        match value {
-            Value::String(pattern) => Ok(SecondaryFileSchema::from(pattern)),
-            Value::Mapping(map) => {
-                let pattern = map
-                    .get("pattern")
-                    .and_then(|v| v.as_str())
-                    .ok_or_else(|| serde::de::Error::custom("Expected string for pattern"))?
-                    .to_string();
-                let required = map
-                    .get("required")
-                    .and_then(|v| v.as_bool())
-                    .unwrap_or(true);
-                Ok(SecondaryFileSchema { pattern, required })
-            }
-            _ => Err(serde::de::Error::custom("Expected string or mapping for secondary file schema")),
-        }
-    }
-}
-
 #[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct WorkflowStepInputParameter {
diff --git a/crates/cwl/src/outputs.rs b/crates/cwl/src/outputs.rs
@@ -1,4 +1,5 @@
 use super::{deserialize::Identifiable, types::CWLType};
+use crate::types::SecondaryFileSchema;
 use serde::{Deserialize, Deserializer, Serialize};
 use serde_yaml::Value;
 
@@ -12,6 +13,8 @@ pub struct CommandOutputParameter {
     pub output_binding: Option<CommandOutputBinding>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub format: Option<String>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub secondary_files: Vec<SecondaryFileSchema>,
 }
 
 impl CommandOutputParameter {
diff --git a/crates/cwl/src/types.rs b/crates/cwl/src/types.rs
@@ -420,7 +420,7 @@ impl File {
         let current = env::current_dir().unwrap_or_default();
         let absolute_path = if path.is_absolute() { path } else { &current.join(path) };
         let absolute_str = absolute_path.display().to_string();
-        let metadata = fs::metadata(path).expect("Could not get metadata");
+        let metadata = fs::metadata(path).unwrap_or_else(|_| panic!("Could not get metadata for {absolute_str}"));
         let mut hasher = Sha1::new();
         let hash = fs::read(path).ok().map(|f| {
             hasher.update(&f);
@@ -442,6 +442,30 @@ impl File {
             ..Default::default()
         }
     }
+
+    pub fn preload(&self) -> Self {
+        let loc = self.location.clone().unwrap_or_default();
+        let path = Path::new(&loc);
+
+        let mut item = self.clone();
+        if item.path.is_none() {
+            item.path = Some(path.display().to_string());
+        }
+
+        if item.basename.is_none() {
+            item.basename = path.file_name().map(|f| f.to_string_lossy().into_owned());
+        }
+
+        if item.nameroot.is_none() {
+            item.nameroot = path.file_stem().map(|f| f.to_string_lossy().into_owned());
+        }
+
+        if item.nameext.is_none() {
+            item.nameext = path.extension().map(|f| format!(".{}", f.to_string_lossy()));
+        }
+
+        item
+    }
 }
 
 fn resolve_format(format: Option<String>) -> Option<String> {
@@ -467,6 +491,45 @@ impl PathItem for File {
     }
 }
 
+#[derive(Serialize, Debug, Default, PartialEq, Clone)]
+pub struct SecondaryFileSchema {
+    pub pattern: String,
+    pub required: bool,
+}
+
+impl From<String> for SecondaryFileSchema {
+    fn from(pattern: String) -> Self {
+        if pattern.ends_with("?") {
+            let pattern = pattern.trim_end_matches('?').to_string();
+            SecondaryFileSchema { pattern, required: false }
+        } else {
+            SecondaryFileSchema { pattern, required: true }
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for SecondaryFileSchema {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let value: Value = Deserialize::deserialize(deserializer)?;
+        match value {
+            Value::String(pattern) => Ok(SecondaryFileSchema::from(pattern)),
+            Value::Mapping(map) => {
+                let pattern = map
+                    .get("pattern")
+                    .and_then(|v| v.as_str())
+                    .ok_or_else(|| serde::de::Error::custom("Expected string for pattern"))?
+                    .to_string();
+                let required = map.get("required").and_then(|v| v.as_bool()).unwrap_or(true);
+                Ok(SecondaryFileSchema { pattern, required })
+            }
+            _ => Err(serde::de::Error::custom("Expected string or mapping for secondary file schema")),
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct Directory {

Original file line number	Diff line number	Diff line change
`@@ -217,9 +217,9 @@ pub(crate) fn process_expressions(tool: &mut CWLDocument, input_values: &mut Inp`
`217`	`217`	`if input.secondary_files.is_empty() {`
`218`	`218`	`continue;`
`219`	`219`	`}`
`220`		`-`
`221`		`- if let DefaultValue::File(file) = &input_values.inputs[&tmp] {`
`222`		`- set_self(&file.snapshot())?;`
	`220`	`+`
	`221`	`+ if let Some(DefaultValue::File(file)) = input_values.inputs.get(&tmp) {`
	`222`	`+ set_self(&file.preload())?;`
`223`	`223`	`for sec_file in &mut input.secondary_files {`
`224`	`224`	`sec_file.pattern = replace_expressions(&sec_file.pattern)?;`
`225`	`225`	`}`