2.0.1

CHANGELOG.md

@@ -1,3 +1,13 @@
+# 2.0.1
+
+## Features
+
+* Resources reservation
+
+## Fixes
+
+* Samtools configuration
+
 # 2.0.0
 
 Breaking change: Non-canonical chromosomes removed by defaults

config/config.yaml

@@ -2,20 +2,39 @@ samples: config/samples.csv
 
 # Optional parameters
 params:
-  # Optional parameters for pyfaidx
+  # Optional parameters for pyfaidx (filter/correct fasta format)
   pyfaidx:
     # Filter-out non canonical chromosomes
     dna: '--regex "^[[0-9]+|X|Y|MT]"'
     # Keep all cdna sequences
-    cdna: '--regex "^.+\|.+\|.+\|[[0-9]+|X|Y|MT]"'
-  # Optional parameters for FastQC
-  fastqc: ""
-  # Optional parameters for FastP
+    cdna: ""
+  # Optional parameters for agat (filter/correct GTF format)
+  agat:
+    # Optional parameters for agat_convert_sp_gff2gtf.pl
+    gff2gtf: ""
+    # Optional parameters for agat_sq_filter_feature_from_fasta.pl
+    filter_features: ""
+    # Optional parameters for agat_sq_select_feature_by_attribute_value.pl
+    select_feature_by_attribute_value: "--attribute 'transcript_support_level' --value '\"NA\"' --test '='"
+    # Optional parameters for agat_convert_sp_gff2tsv
+    agat_convert_sp_gff2tsv: ""
+  # Optional parameters for GFFRead
+  gffread: ""
+  # Optional parameters for bedtools
+  bedtools:
+    # Optional parameters for filtering non-canonical chromosomes over dbSNP
+    filter_non_canonical_chrom: ""
+  # Optional parameters for tabix index VCF
+  tabix: "-p vcf"
+  # Optional parameters for fastp
   fastp:
-    # Optional command line adapters
+    # Optional adapters to remove
     adapters: ""
-    # Optional command line parameters
+    # Optional command line arguments for fastp
     extra: ""
+  # Optional parameters for fastqc
+  fastqc: ""
+  # Optional parameters for bowtie2
   bowtie2:
     # Optional parameters for bowtie2-build
     build: ""
@@ -25,12 +44,18 @@ params:
     # Optional parameters for sambamba view
     view: "--format 'bam' --filter 'mapping_quality >= 30 and not (unmapped or mate_is_unmapped)' "
     # Optional parameters for sambamba markdup
-    markdup: "--remove-duplicates --overflow-size 500000"
+    markdup: "--remove-duplicates --overflow-list-size=500000"
   picard:
     # Mapping QC optional parameters
     metrics: ""
+    # Optional parameters for picard create sequence dictionary
+    createsequencedictionary: ""
   # Optional parameters for samtools stats
-  samtools: ""
+  samtools:
+    # Optional parameters for samtools fasta index
+    faidx: ""
+    # Optional parameters for samtools stats
+    stats: ""
   macs2:
     # Optional parameters for Macs2 Callpeaks
     callpeak: ""

workflow/rules/bedtools_merge_peaks.smk

@@ -9,6 +9,10 @@ rule xsv_cat_macs2_peaks:
         temp(
             "tmp/csv/cat_rows/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.csv"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * 1024,
+        runtime=lambda wildcards, attempt: attempt * 10,
+        tmpdir="tmp",
     group:
         "concat_beds"
     log:
@@ -29,6 +33,10 @@ rule xsv_sort_macs2_concat_peaks:
         temp(
             "tmp/csv/sort/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.csv"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * 1024,
+        runtime=lambda wildcards, attempt: attempt * 10,
+        tmpdir="tmp",
     group:
         "concat_beds"
     log:
@@ -47,6 +55,10 @@ rule xsv_fmt_macs2_sorted_peaks:
         table="tmp/csv/sort/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.csv",
     output:
         temp("tmp/csv/fmt/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.bed"),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * 1024,
+        runtime=lambda wildcards, attempt: attempt * 10,
+        tmpdir="tmp",
     group:
         "concat_beds"
     log:
@@ -67,6 +79,10 @@ rule bedtools_merge_macs2_sorted_peaks:
         temp(
             "tmp/bedtools/merge/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.merged.bed"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 4),
+        runtime=lambda wildcards, attempt: attempt * 35,
+        tmpdir="tmp",
     log:
         "logs/bedtools/merge/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.log",
     benchmark:

workflow/rules/datavzrd.smk

@@ -5,6 +5,10 @@ rule datavzrd_homer_yaml:
         yaml=temp(
             "tmp/datavzrd/{species}.{build}.{release}.{datatype}/homer/{macs2_peak_type}/{sample}.yaml"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * 512,
+        runtime=lambda wildcards, attempt: attempt * 5,
+        tmpdir="tmp",
     log:
         "logs/datavzrd/{species}.{build}.{release}.{datatype}/homer/{macs2_peak_type}/{sample}.config.log",
     benchmark:
@@ -35,6 +39,10 @@ rule datavzrd_homer_render:
                 "sample": "{sample}",
             },
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 2),
+        runtime=lambda wildcards, attempt: attempt * 25,
+        tmpdir="tmp",
     log:
         "logs/datavzrd/{species}.{build}.{release}.{datatype}/homer/{macs2_peak_type}/{sample}.render.log",
     benchmark:

workflow/rules/deeptools_bam_qc.smk

@@ -5,6 +5,10 @@ rule deeptools_bamcoverage:
         protected(
             "results/{species}.{build}.{release}.{datatype}/Coverage/{sample}.bw",
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 45) + (1024 * 20),
+        runtime=lambda wildcards, attempt: attempt * 120 + 60,
+        tmpdir="tmp",
     log:
         "logs/deeptools/bamcoverage/{species}.{build}.{release}.{datatype}/{sample}.log",
     benchmark:
@@ -39,6 +43,10 @@ rule deeptools_plotcoverage:
         metrics=temp(
             "tmp/deeptools/plot_coverage/{species}.{build}.{release}.{datatype}/Coverage.metrics"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 45) + (1024 * 20),
+        runtime=lambda wildcards, attempt: attempt * 120 + 60,
+        tmpdir="tmp",
     log:
         "logs/deeptools/plot_coverage/{species}.{build}.{release}.{datatype}.log",
     benchmark:
@@ -69,6 +77,10 @@ rule deeptools_fingerprint:
         qc_metrics=temp(
             "tmp/deeptools/plot_fingerprint/{species}.{build}.{release}.{datatype}/qc_metrics.txt"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 45) + (1024 * 20),
+        runtime=lambda wildcards, attempt: attempt * 120 + 60,
+        tmpdir="tmp",
     log:
         "logs/deeptools/plot_fingerprint/{species}.{build}.{release}.{datatype}.log",
     benchmark:

workflow/rules/deeptools_correlations.smk

@@ -5,6 +5,10 @@ rule deeptools_multibigwig_summary:
         npz=temp(
             "tmp/deeptools/multibigwig_summary/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.npz"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 45) + (1024 * 20),
+        runtime=lambda wildcards, attempt: attempt * 120 + 60,
+        tmpdir="tmp",
     log:
         "logs/deeptools/multibigwig_summary/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.log",
     benchmark:
@@ -36,6 +40,10 @@ rule deeptools_plot_pca:
         tab=temp(
             "tmp/deeptools/plot_pca/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.tab"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 45) + (1024 * 20),
+        runtime=lambda wildcards, attempt: attempt * 120 + 60,
+        tmpdir="tmp",
     log:
         "logs/deeptools/plot_pca/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.log",
     benchmark:
@@ -65,6 +73,10 @@ rule deeptools_plot_correlation:
         tab=temp(
             "tmp/deeptools/plot_correlation/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.tab"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 45) + (1024 * 20),
+        runtime=lambda wildcards, attempt: attempt * 120 + 60,
+        tmpdir="tmp",
     log:
         "logs/deeptools/plot_correlation/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.log",
     benchmark:
@@ -96,6 +108,10 @@ rule deeptools_plot_enrichment:
         out_raw_counts=temp(
             "tmp/deeptools/plot_enrichment/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.tab"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 45) + (1024 * 20),
+        runtime=lambda wildcards, attempt: attempt * 120 + 60,
+        tmpdir="tmp",
     log:
         "logs/deeptools/plot_enrichment/{species}.{build}.{release}.{datatype}/{macs2_peak_type}.log",
     benchmark:

workflow/rules/homer.smk

@@ -8,6 +8,10 @@ rule homer_annotate_peaks:
                 non_empty=True,
             ),
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 3),
+        runtime=lambda wildcards, attempt: attempt * 45,
+        tmpdir="tmp",
     log:
         "logs/homer/annotatepeaks/{species}.{build}.{release}.{datatype}/{sample}.{macs2_peak_type}.log",
     benchmark:

workflow/rules/in_house_homer_post_process.smk

@@ -8,6 +8,10 @@ rule summarize_homer:
         tsv=temp(
             "tmp/summarize_homer/{species}.{build}.{release}.{datatype}/{sample}.{macs2_peak_type}.tsv"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 2),
+        runtime=lambda wildcards, attempt: attempt * 25,
+        tmpdir="tmp",
     log:
         "logs/summarize_homer/{species}.{build}.{release}.{datatype}/{sample}.{macs2_peak_type}.log",
     benchmark:
@@ -31,6 +35,10 @@ rule merge_homer_summaries:
         genes=temp(
             "tmp/summarize_homer/{species}.{build}.{release}.{datatype}.{macs2_peak_type}/GeneTypes.tsv"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 2),
+        runtime=lambda wildcards, attempt: attempt * 25,
+        tmpdir="tmp",
     log:
         "logs/summarize_homer/{species}.{build}.{release}.{datatype}.{macs2_peak_type}.log",
     benchmark:
@@ -56,6 +64,10 @@ rule plot_homer_summaries_for_regions:
                 "sample": "all",
             },
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 2),
+        runtime=lambda wildcards, attempt: attempt * 25,
+        tmpdir="tmp",
     log:
         "logs/plot_homer_summaries/{species}.{build}.{release}.{datatype}.{macs2_peak_type}/{content}.log",
     benchmark:

workflow/rules/macs2.smk

@@ -14,6 +14,10 @@ rule macs2_callpeak_narrow:
         temp(
             "tmp/macs2/{species}.{build}.{release}.{datatype}/narrowPeak/{sample}_summits.bed"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 4),
+        runtime=lambda wildcards, attempt: attempt * 45,
+        tmpdir="tmp",
     log:
         "logs/macs2/{species}.{build}.{release}.{datatype}/{sample}.narrow.log",
     benchmark:
@@ -38,6 +42,10 @@ use rule macs2_callpeak_narrow as macs2_callpeak_broad with:
         temp(
             "tmp/macs2/{species}.{build}.{release}.{datatype}/broadPeak/{sample}_peaks.gappedPeak"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 4),
+        runtime=lambda wildcards, attempt: attempt * 45,
+        tmpdir="tmp",
     log:
         "logs/macs2/{species}.{build}.{release}.{datatype}/{sample}.broad.log",
     benchmark:
@@ -55,6 +63,10 @@ rule macs2_peaks_to_csv:
         "logs/xsv/select/{species}.{build}.{release}.{datatype}/{sample}.{macs2_peak_type}.log",
     benchmark:
         "benchmark/xsv/select/{species}.{build}.{release}.{datatype}/{sample}.{macs2_peak_type}.tsv"
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * 1024,
+        runtime=lambda wildcards, attempt: attempt * 10,
+        tmpdir="tmp",
     group:
         "macs2_reformat"
     params:
@@ -71,6 +83,10 @@ rule macs2_csv_to_bed:
         protected(
             "results/{species}.{build}.{release}.{datatype}/PeakCalling/{macs2_peak_type}/{sample}.{macs2_peak_type}.bed"
         ),
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * 1024,
+        runtime=lambda wildcards, attempt: attempt * 10,
+        tmpdir="tmp",
     log:
         "logs/xsv/select/{species}.{build}.{release}.{datatype}/{sample}.{macs2_peak_type}.log",
     benchmark:

workflow/rules/multiqc.smk

@@ -13,6 +13,10 @@ rule multiqc_report:
             },
         ),
         "results/QC/MultiQC_PeakCalling_data.zip",
+    resources:
+        mem_mb=lambda wildcards, attempt: attempt * (1024 * 2),
+        runtime=lambda wildcards, attempt: attempt * 45,
+        tmpdir="tmp",
     params:
         extra=config.get("params", {}).get(
             "multiqc", "--zip-data-dir --verbose --no-megaqc-upload --no-ansi --force"