Automatic config generation for automatic delay cal selection

pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "LINC @ git+https://git.astron.nl/RD/LINC",
     "lofar_facet_selfcal @ git+https://github.com/rvweeren/lofar_facet_selfcal",
     "lofar_helpers @ git+https://github.com/jurjen93/lofar_helpers",
+    "lofar-vlbi-plot",
     "losoto",
     "numpy",
     "pandas",

steps/delay_cal_model.cwl

@@ -29,9 +29,6 @@ inputs:
         position: 2
         prefix: --model-image
         separate: true
-        valueFrom: $(self.basename)
-
-
     - id: process_all
       type: boolean?
       default: false

steps/lofar_vlbi_plot.cwl

@@ -22,6 +22,11 @@ outputs:
     doc: Delay Calibrator CSV
     outputBinding:
       glob: delay_calibrators.csv
+  - id: image_catalogue_pf
+    type: File
+    doc: Image catalogue CSV
+    outputBinding:
+      glob: image_catalogue.csv
 
 arguments:
   - --continue_no_lotss

workflows/delay-calibration.cwl

@@ -43,7 +43,7 @@ inputs:
         If not provided, lofar-vlbi-plot will be run to generate the CSV (requires internet access).
 
     - id: image_catalogue
-      type: File
+      type: File?
       doc: An image catalogue file in CSV format.
 
     - id: Ateam_skymodel
@@ -71,10 +71,6 @@ inputs:
       default: TGSSphase
       doc: The name of the target solution table to use from the solset input.
 
-    - id: configfile
-      type: File
-      doc: Settings for the delay calibration in delay_solve.
-
     - id: reference_stationSB
       type: int?
       default: 104
@@ -158,10 +154,11 @@ inputs:
 
     - id: starting_skymodel
       type:
-        - File?
         - File[]?
       doc: |
-        Optional starting model(s) in BBS-compatible text format used to kickstart the delay calibration. If given and `do_auto_delay_selection` is enabled, the number of skymodels must be equal to `select_best_n_delay_calibrators`. Additionally, they should be named in such a way that when sorted by name, the delay calibrator MSes and skymodels end up in the same order.
+        Optional starting model(s) in FITS format used to kickstart the delay calibration when automatic selection from multiple delay candidates is used.
+        If given and `do_auto_delay_selection` is enabled, the number of skymodels must be equal to `select_best_n_delay_calibrators`.
+        Additionally, they should be named in such a way that when sorted by name, the delay calibrator MSes and skymodels end up in the same order.
 
     - id: do_auto_delay_selection
       type: boolean?
@@ -206,6 +203,7 @@ steps:
           source: delay_calibrator
       out:
         - id: delay_calibrator_pf
+        - id: image_catalogue_pf
       run: ../steps/lofar_vlbi_plot.cwl
       when: $(inputs.delay_calibrator == null)
 
@@ -320,14 +318,18 @@ steps:
           linkMerge: merge_nested
           pickValue: first_non_null
           valueFrom: $(self)
-        - id: configfile
-          source: configfile
         - id: delay_calibrator
           source:
             - delay_calibrator
             - lofar_vlbi_plot/delay_calibrator_pf
           pickValue: first_non_null
           valueFrom: $(self)
+        - id: image_catalogue
+          source:
+            - image_catalogue
+            - lofar_vlbi_plot/image_catalogue_pf
+          pickValue: first_non_null
+          valueFrom: $(self)
         - id: select_best_n_delay_calibrators
           source: select_best_n_delay_calibrators
         - id: do_auto_delay_selection
@@ -461,8 +463,7 @@ outputs:
       - phaseup/starting_skymodel
       - select_best_delay_cal/starting_skymodels
     type: File[]
-    pickValue: all_non_null
-    linkMerge: merge_flattened
+    pickValue: first_non_null
     doc: |
         The starting model(s) that were used to kick start the delay calibration.
 

workflows/subworkflows/find-best-delay-calibrator.cwl

@@ -19,14 +19,14 @@ inputs:
     - id: delay_calibrator
       type: File
       doc: A delay calibrator catalogue in CSV format.
-    - id: configfile
+    - id: image_catalogue
       type: File
-      doc: Settings for the delay calibration in delay_solve.
+      doc: A catalogue with other sources in the field (e.g. a LoTSS catalogue).
     - id: starting_skymodel
       type:
         - File[]?
       doc: |
-        Optional starting models in BBS-compatible text format used to kickstart the delay calibration. If given, the number of skymodels must be equal to `select_best_n_delay_calibrators`. Additionally, they should be named in such a way that when sorted by name, the delay calibrator MSes and skymodels end up in the same order.
+        Optional starting models in FITS format used to kickstart the delay calibration. If given, the number of skymodels must be equal to `select_best_n_delay_calibrators`. Additionally, they should be named in such a way that when sorted by name, the delay calibrator MSes and skymodels end up in the same order.
     - id: select_best_n_delay_calibrators
       type: int?
       default: 1
@@ -43,34 +43,18 @@ inputs:
         Time resolution to average the split off delay calibrators to.
 
 steps:
-    - id: generate_skymodels
-      in:
-        - id: msin
-          source: msin
-          valueFrom: $(self[0])
-        - id: delay_calibrator
-          source: delay_calibrator
-        - id: process_all
-          default: true
-      out: 
-        - id: skymodel
-        - id: logfile
-      run: ../../steps/delay_cal_model.cwl
-
     - id: select_best_delay_cal
       in:
         - id: msin
           source: msin
         - id: dd_selection
           default: true
         - id: do_selfcal
-          default: true
+          default: false
         - id: image_cat
           source: delay_calibrator
         - id: select_best_n
           source: select_best_n_delay_calibrators
-        - id: configfile
-          source: configfile
         - id: frequency_resolution
           source: frequency_resolution
         - id: time_resolution
@@ -86,9 +70,8 @@ steps:
     - id: sort_skymodels
       in:
         - id: input_entry
-          source: 
+          source:
             - starting_skymodel
-            - generate_skymodels/skymodel
           pickValue: all_non_null
           linkMerge: merge_flattened
       out:
@@ -108,26 +91,40 @@ steps:
       in:
         - id: msin
           source: sort_ms/sorted_entries
-        - id: configfile
-          source: configfile
-        - id: skymodel
+        - id: delay_calibrator
+          source: delay_calibrator
+        - id: image_catalogue
+          source: image_catalogue
+        - id: model_image
           source: sort_skymodels/sorted_entries
       out:
-        - id: images
-        - id: h5parm
-      run: ../../steps/facet_selfcal.cwl
-      scatter: [msin, skymodel]
+        - id: solutions
+        - id: starting_skymodel
+        - id: config
+        - id: pictures
+        - id: logfile
+      run: ./delay_cal_run.cwl
+      scatter: [msin, model_image]
       scatterMethod: dotproduct
 
     - id: flatten_delay_images
       in:
         - id: nestedarray
-          source: delay_selfcal/images
+          source: delay_selfcal/pictures
       out:
         - id: flattenedarray
       run: ../../steps/flatten.cwl
       label: flatten_delay_images
 
+    - id: flatten_delay_models
+      in:
+        - id: nestedarray
+          source: delay_selfcal/starting_skymodel
+      out:
+        - id: flattenedarray
+      run: ../../steps/flatten.cwl
+      label: flatten_delay_models
+
 outputs:
   - id: msout
     outputSource:
@@ -139,9 +136,11 @@ outputs:
         format, phase-shifted to the delay calibrator.
 
   - id: starting_skymodels
-    outputSource: generate_skymodels/skymodel
+    outputSource: 
+      - flatten_delay_models/flattenedarray
+    pickValue: first_non_null
     type: File[]
-    doc: Starting models that were used to kickstart the delay calibration.
+    doc: Starting skymodels that were used to kickstart the delay calibration.
 
   - id: pictures
     outputSource: 
@@ -151,6 +150,14 @@ outputs:
     linkMerge: merge_flattened
     doc: Inspection plots generated by lofar_facet_selfcal.
 
+  - id: configs
+    outputSource: 
+      - delay_selfcal/config
+    type: File[]
+    pickValue: all_non_null
+    linkMerge: merge_flattened
+    doc: Facetselfcal config files used for the calibration.
+
   - id: phasediff_score_csv
     outputSource: select_best_delay_cal/phasediff_score_csv
     type: 
@@ -159,7 +166,7 @@ outputs:
       A CSV file containing the phasediff scores for each of the calibrators that were split out.
 
   - id: solutions
-    outputSource: delay_selfcal/h5parm
+    outputSource: delay_selfcal/solutions
     type:
       - File
       - File[]