Support InlineTools in Workflow step fixes #111

Author: JensKrumsieck

CHANGELOG.md

@@ -1,11 +1,12 @@
 # Unreleased
 ## 🚀 Features
 - added NetworkAccess Requirment via `--net/--enable-network` in `s4n tool create` #104
+- Support InlineTools in Workflow step #111
 
 ## 🐛 Bugfixes
 - fixed a bug with Dockerfile path resolution
 - handle NetworkAccess Requirment in runner
-- ramping up runner conformance from 160/378 to 164/378
+- ramping up runner conformance from 160/378 to 176/378
 
 # v0.5.2
 ## 🐛 Bugfixes

crates/cwl-execution/src/lib.rs

@@ -77,7 +77,7 @@ pub fn execute_cwlfile(cwlfile: impl AsRef<Path>, raw_inputs: &[String], outdir:
         }
     }
 
-    let output_values = execute(cwlfile, inputs, outdir)?;
+    let output_values = execute(cwlfile, inputs, outdir, None)?;
     let json = serde_json::to_string_pretty(&output_values)?;
     println!("{json}");
 
@@ -88,11 +88,16 @@ pub fn execute(
     cwlfile: impl AsRef<Path>,
     inputs: HashMap<String, DefaultValue>,
     outdir: Option<impl AsRef<Path>>,
+    cwl_doc: Option<&CWLDocument>,
 ) -> Result<HashMap<String, DefaultValue>, Box<dyn Error>> {
     //load cwl
-    let contents = fs::read_to_string(&cwlfile).map_err(|e| format!("Could not read CWL File {:?}: {e}", cwlfile.as_ref()))?;
-    let contents = preprocess_cwl(&contents, &cwlfile)?;
-    let mut doc: CWLDocument = serde_yaml::from_str(&contents).map_err(|e| format!("Could not parse CWL File {:?}: {e}", cwlfile.as_ref()))?;
+    let mut doc: CWLDocument = if let Some(doc) = cwl_doc {
+        doc.clone()
+    } else {
+        let contents = fs::read_to_string(&cwlfile).map_err(|e| format!("Could not read CWL File {:?}: {e}", cwlfile.as_ref()))?;
+        let contents = preprocess_cwl(&contents, &cwlfile)?;
+        serde_yaml::from_str(&contents).map_err(|e| format!("Could not parse CWL File {:?}: {e}", cwlfile.as_ref()))?
+    };
 
     match doc {
         CWLDocument::CommandLineTool(_) | CWLDocument::ExpressionTool(_) => run_tool(

crates/cwl-execution/src/runner.rs

@@ -21,7 +21,7 @@ use cwl::{
     inputs::{CommandLineBinding, WorkflowStepInput},
     requirements::{DockerRequirement, InlineJavascriptRequirement, StringOrInclude},
     types::{CWLType, DefaultValue, Directory, Entry, File, PathItem},
-    wf::Workflow,
+    wf::{StringOrDocument, Workflow},
     CWLDocument, StringOrNumber,
 };
 use log::{info, warn};
@@ -66,7 +66,11 @@ pub fn run_workflow(
     let mut outputs: HashMap<String, DefaultValue> = HashMap::new();
     for step_id in sorted_step_ids {
         if let Some(step) = workflow.get_step(&step_id) {
-            let path = workflow_folder.join(step.run.clone());
+            let path = if let StringOrDocument::String(run) = &step.run {
+                Some(workflow_folder.join(run))
+            } else {
+                None
+            };
 
             //map inputs to correct fields
             let mut step_inputs = HashMap::new();
@@ -113,8 +117,13 @@ pub fn run_workflow(
                     }
                 }
             }
-
-            let step_outputs = execute(&path, step_inputs, Some(tmp_path.clone()))?;
+            let step_outputs = if let Some(path) = path {
+                execute(&path, step_inputs, Some(tmp_path.clone()), None)?
+            } else if let StringOrDocument::Document(doc) = &step.run {
+                execute(workflow_folder, step_inputs, Some(tmp_path.clone()), Some(doc))?
+            } else {
+                unreachable!()
+            };
             for (key, value) in step_outputs {
                 outputs.insert(format!("{}/{}", step.id, key), value);
             }

crates/cwl/src/clt.rs

@@ -59,7 +59,7 @@ impl Default for CommandLineTool {
                 label: Option::default(),
                 doc: Option::default(),
                 class: String::from("CommandLineTool"),
-                cwl_version: String::from("v1.2"),
+                cwl_version: Some(String::from("v1.2")),
                 inputs: Vec::default(),
                 requirements: Option::default(),
                 hints: Option::default(),

crates/cwl/src/et.rs

@@ -22,7 +22,7 @@ impl Default for ExpressionTool {
     fn default() -> Self {
         Self {
             base: DocumentBase {
-                cwl_version: String::from("v1.2"),
+                cwl_version: Some(String::from("v1.2")),
                 class: String::from("ExpressionTool"),
                 id: Option::default(),
                 label: Option::default(),

crates/cwl/src/lib.rs

@@ -47,7 +47,7 @@ pub mod wf;
 /// let document: CWLDocument = serde_yaml::from_str(yaml).unwrap();
 /// assert!(matches!(document, CWLDocument::CommandLineTool(_)));
 /// ```
-#[derive(Serialize, Debug, PartialEq)]
+#[derive(Serialize, Debug, PartialEq, Clone)]
 #[serde(untagged)]
 pub enum CWLDocument {
     CommandLineTool(CommandLineTool),
@@ -107,7 +107,7 @@ impl<'de> Deserialize<'de> for CWLDocument {
 #[serde(rename_all = "camelCase")]
 pub struct DocumentBase {
     pub class: String,
-    pub cwl_version: String,
+    pub cwl_version: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub id: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]

crates/cwl/src/wf.rs

@@ -3,7 +3,7 @@ use super::{
     inputs::WorkflowStepInput,
     outputs::WorkflowOutputParameter,
 };
-use crate::DocumentBase;
+use crate::{CWLDocument, DocumentBase};
 use serde::{Deserialize, Serialize};
 use std::{
     collections::{HashMap, VecDeque},
@@ -15,7 +15,7 @@ use std::{
 /// directed acyclic graph, and independent steps may run concurrently.
 ///
 /// Reference: <https://www.commonwl.org/v1.2/Workflow.html>
-#[derive(Serialize, Deserialize, Debug, PartialEq)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct Workflow {
     #[serde(flatten)]
@@ -34,7 +34,7 @@ impl Default for Workflow {
                 label: None,
                 doc: None,
                 class: String::from("Workflow"),
-                cwl_version: String::from("v1.2"),
+                cwl_version: Some(String::from("v1.2")),
                 requirements: Default::default(),
                 hints: Default::default(),
                 inputs: Default::default(),
@@ -102,7 +102,6 @@ impl Workflow {
         step.unwrap().out.iter().any(|output| output == parts[1])
     }
 
-    
     /// Returns the `Workflow`'s `WorkflowStep` with id `id`
     pub fn get_step(&self, id: &str) -> Option<&WorkflowStep> {
         self.steps.iter().find(|s| s.id == *id)
@@ -166,7 +165,7 @@ impl Workflow {
 pub struct WorkflowStep {
     #[serde(default)]
     pub id: String,
-    pub run: String,
+    pub run: StringOrDocument,
     pub in_: HashMap<String, WorkflowStepInput>,
     pub out: Vec<String>,
 }
@@ -181,6 +180,19 @@ impl Identifiable for WorkflowStep {
     }
 }
 
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[serde(untagged)]
+pub enum StringOrDocument {
+    String(String),
+    Document(Box<CWLDocument>),
+}
+
+impl Default for StringOrDocument {
+    fn default() -> Self {
+        Self::String(String::default())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/commands/workflow.rs

@@ -8,8 +8,9 @@ use colored::Colorize;
 use cwl::{
     format::format_cwl,
     inputs::WorkflowStepInput,
-    wf::Workflow,
-    {load_tool, load_workflow},
+    load_tool, load_workflow,
+    wf::{StringOrDocument, Workflow},
+    CWLDocument,
 };
 use cwl_execution::io::create_and_write_file;
 use git2::Repository;
@@ -208,8 +209,13 @@ pub fn get_workflow_status(args: &CreateWorkflowArgs) -> Result<(), Box<dyn Erro
     table.add_row(row![b -> "Steps:"]);
 
     for step in &workflow.steps {
-        let tool = load_tool(path.join(&step.run))?;
-
+        let tool = match &step.run {
+            StringOrDocument::String(run) => load_tool(path.join(run))?,
+            StringOrDocument::Document(boxed_doc) => match &**boxed_doc {
+                CWLDocument::CommandLineTool(doc) => doc.clone(),
+                _ => unreachable!(), //see #95
+            },
+        };
         let input_status = tool
             .inputs
             .iter()
@@ -246,7 +252,12 @@ pub fn get_workflow_status(args: &CreateWorkflowArgs) -> Result<(), Box<dyn Erro
             })
             .collect::<Vec<_>>()
             .join("\n");
-        table.add_row(row![b -> &step.run, &input_status, &output_status]);
+        let run = if let StringOrDocument::String(run) = &step.run {
+            run
+        } else {
+            &String::from("Inline Document")
+        };
+        table.add_row(row![b -> run, &input_status, &output_status]);
     }
 
     table.printstd();

src/cwl.rs

@@ -6,7 +6,7 @@ use cwl::{
     outputs::WorkflowOutputParameter,
     requirements::Requirement,
     types::{DefaultValue, Entry, PathItem},
-    wf::{Workflow, WorkflowStep},
+    wf::{StringOrDocument, Workflow, WorkflowStep},
 };
 use log::{info, warn};
 use std::{collections::HashMap, error::Error};
@@ -67,7 +67,7 @@ impl Connectable for Workflow {
         if !self.has_step(name) {
             let workflow_step = WorkflowStep {
                 id: name.to_string(),
-                run: format!("../{name}/{name}.cwl"),
+                run: StringOrDocument::String(format!("../{name}/{name}.cwl")),
                 in_: HashMap::new(),
                 out: tool.get_output_ids(),
             };
@@ -271,7 +271,7 @@ mod tests {
         clt::Command,
         inputs::CommandLineBinding,
         requirements::{DockerRequirement, InitialWorkDirRequirement},
-        types::{CWLType, File, Dirent},
+        types::{CWLType, Dirent, File},
     };
     use serde_yaml::Value;
     use std::path::Path;