feat: annotate access patterns (#369)

* feat: annotate access patterns

* naming

* polish

* min snakemake version

* fmt

* update super linter

* minor

* use mega-linter

Author: johanneskoester

.github/workflows/main.yml

@@ -13,40 +13,8 @@ concurrency:
 
 
 jobs:
-  Formatting:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          # Full git history is needed to get a proper
-          # list of changed files within `super-linter`
-          fetch-depth: 0
-      - name: Formatting
-        uses: github/super-linter@v5
-        env:
-          VALIDATE_ALL_CODEBASE: false
-          DEFAULT_BRANCH: master
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          VALIDATE_SNAKEMAKE_SNAKEFMT: true
-
-  Linting:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Lint workflow
-      uses: snakemake/snakemake-github-action@v2
-      with:
-        directory: .
-        snakefile: workflow/Snakefile
-        args: "--lint"
-
   Testing:
     runs-on: ubuntu-latest
-    needs: 
-      - Linting
-      - Formatting
     strategy:
       fail-fast: true
       matrix:

workflow/Snakefile

@@ -1,6 +1,6 @@
 from snakemake.utils import min_version
 
-min_version("8.13.0")
+min_version("9.1.0")
 
 
 configfile: "config/config.yaml"
@@ -13,6 +13,10 @@ scattergather:
     calling=16,
 
 
+inputflags:
+    access.sequential,
+
+
 # Allow users to fix the underlying OS via singularity.
 container: "docker://continuumio/miniconda3"
 

workflow/rules/annotation.smk

@@ -1,9 +1,9 @@
 rule annotate_candidate_variants:
     input:
         calls="results/candidate-calls/{group}.{caller}.{scatteritem}.bcf",
-        cache="resources/vep/cache",
-        plugins="resources/vep/plugins",
-        fasta=genome,
+        cache=access.random("resources/vep/cache"),
+        plugins=access.random("resources/vep/plugins"),
+        fasta=access.random(genome),
         fai=genome_fai,
     output:
         calls="results/candidate-calls/{group}.{caller}.{scatteritem}.annotated.bcf",
@@ -25,11 +25,11 @@ rule annotate_candidate_variants:
 rule annotate_variants:
     input:
         calls="results/calls/{group}.{calling_type}.{scatteritem}.bcf",
-        cache="resources/vep/cache",
-        plugins="resources/vep/plugins",
+        cache=access.random("resources/vep/cache"),
+        plugins=access.random("resources/vep/plugins"),
         revel=lambda wc: get_plugin_aux("REVEL"),
         revel_tbi=lambda wc: get_plugin_aux("REVEL", True),
-        fasta=genome,
+        fasta=access.random(genome),
         fai=genome_fai,
     output:
         calls="results/calls/{group}.{calling_type}.{scatteritem}.annotated.bcf",

workflow/rules/calling.smk

@@ -44,7 +44,7 @@ rule varlociraptor_preprocess:
     input:
         ref=genome,
         ref_idx=genome_fai,
-        candidates=get_candidate_calls,
+        candidates=access.multi(get_candidate_calls),
         bam="results/recal/{sample}.bam",
         bai="results/recal/{sample}.bai",
         alignment_props=get_alignment_props,

workflow/rules/candidate_calling.smk

@@ -1,10 +1,10 @@
 rule freebayes:
     input:
-        ref=genome,
+        ref=access.random(genome),
         ref_idx=genome_fai,
         regions="results/regions/{group}.expanded_regions.filtered.bed",
         # you can have a list of samples here
-        alns=lambda w: get_group_bams(w),
+        alns=access.random(lambda w: get_group_bams(w)),
         idxs=lambda w: get_group_bams(w, bai=True),
     output:
         "results/candidate-calls/{group}.freebayes.bcf",
@@ -25,9 +25,9 @@ rule freebayes:
 
 rule delly:
     input:
-        ref=genome,
+        ref=access.random(genome),
         ref_idx=genome_fai,
-        alns=lambda w: get_group_bams(w),
+        alns=access.random(lambda w: get_group_bams(w)),
         index=lambda w: get_group_bams(w, bai=True),
         exclude=get_delly_excluded_regions(),
     output:

workflow/rules/common.smk

@@ -1479,7 +1479,7 @@ def get_primer_extra(wc, input):
     min_primer_len = get_shortest_primer_length(input.reads)
     # Check if shortest primer is below default values
     if min_primer_len < 32:
-        extra += f" -T {min_primer_len - 2}"
+        extra += f" -T {min_primer_len-2}"
     if min_primer_len < 19:
         extra += f" -k {min_primer_len}"
     return extra

workflow/rules/fusion_calling.smk

@@ -62,7 +62,7 @@ rule annotate_exons:
         """
 
 
-# Use script provide by arriba once new version is released
+# Use script provided by arriba once new version is released
 rule convert_fusions:
     input:
         fasta=rules.get_genome.output,

workflow/rules/maf.smk

@@ -33,12 +33,12 @@ rule group_vcf_to_maf:
             dpath="maf/primary_alias", within=config, default="tumor"
         ),
         vcf_control_alias_option=(
-            f'--vcf-normal-id {lookup(dpath = "maf/control_alias", within = config , default = "")}'
+            f'--vcf-normal-id {lookup(dpath= "maf/control_alias", within= config, default= "")}'
             if lookup(dpath="maf/control_alias", within=config, default=False)
             else ""
         ),
         normal_id=lambda wc: (
-            f'--normal-id {wc.group}_{lookup(dpath = "maf/control_alias", within = config , default = "")}'
+            f'--normal-id {wc.group}_{lookup(dpath= "maf/control_alias", within= config, default= "")}'
             if lookup(dpath="maf/control_alias", within=config, default=False)
             else ""
         ),

workflow/rules/mapping.smk

@@ -1,7 +1,7 @@
 rule map_reads_bwa:
     input:
         reads=get_map_reads_input,
-        idx=rules.bwa_index.output,
+        idx=access.random(rules.bwa_index.output),
     output:
         temp("results/mapped/bwa/{sample}.bam"),
     log:
@@ -53,10 +53,10 @@ rule create_reference_paths:
 rule map_reads_vg:
     input:
         reads=get_map_reads_input,
-        graph=f"{pangenome_prefix}.gbz",
-        kmers="results/kmers/{sample}.kff",
-        hapl=f"{pangenome_prefix}.hapl",
-        paths="resources/reference_paths.txt",
+        graph=access.random(f"{pangenome_prefix}.gbz"),
+        kmers=access.random("results/kmers/{sample}.kff"),
+        hapl=access.random(f"{pangenome_prefix}.hapl"),
+        paths=access.random("resources/reference_paths.txt"),
     output:
         bam=temp("results/mapped/vg/{sample}.preprocessed.bam"),
         indexes=temp(
@@ -229,7 +229,7 @@ rule calc_consensus_reads:
 rule map_consensus_reads:
     input:
         reads=get_processed_consensus_input,
-        idx=rules.bwa_index.output,
+        idx=access.random(rules.bwa_index.output),
     output:
         temp("results/consensus/{sample}.consensus.{read_type}.mapped.bam"),
     params:

workflow/rules/primers.smk

@@ -47,7 +47,7 @@ rule trim_primers:
 rule map_primers:
     input:
         reads=lambda wc: get_panel_primer_input(wc.panel),
-        idx=rules.bwa_index.output,
+        idx=access.random(rules.bwa_index.output),
     output:
         "results/primers/{panel}_primers.bam",
     log: