From 4b2bf8934b9254b09777e3a9d0bceb6b3c484031 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Fri, 26 Jan 2024 10:38:38 +0100 Subject: [PATCH 1/6] Remove redundant fastq option --- conf/modules.config | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 127e0a34..835677be 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -45,7 +45,7 @@ process { process { withName: 'CAT_FASTQ' { publishDir = [ - path: { "${params.outdir}/fastq" }, + path: { "${params.outdir}/cat_fastq" }, mode: params.publish_dir_mode, pattern: '*.fastq', // enabled: params.save_merged_fastq //TODO ? implement save_merged_fastq param @@ -88,14 +88,6 @@ process { } } -if (!(params.skip_fastqc)) { - process { - withName: '.*:FASTQC_UMITOOLS_FASTP:FASTQC_.*' { - ext.args = '--quiet' - } - } -} - if (!params.skip_fastp) { process { withName: '.*:FASTQC_UMITOOLS_FASTP:FASTP' { From 17f2ae53f6788f51de59a04839d5634cea032a8a Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Fri, 26 Jan 2024 10:47:15 +0100 Subject: [PATCH 2/6] Clean up config Replicating the if/else logic from the main workflow is not necessary, since now, by default nextflow.enable.configProcessNamesValidation = false --- conf/modules.config | 329 ++++++++++++++++++++------------------------ 1 file changed, 151 insertions(+), 178 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 835677be..6e3f17dd 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -10,17 +10,22 @@ ---------------------------------------------------------------------------------------- */ -// -// General configuration options -// + process { + + // + // General configuration options + // publishDir = [ path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] + // + // Input preparation + // withName: 'NFCORE_SMRNASEQ:SMRNASEQ:INPUT_CHECK:SAMPLESHEET_CHECK' { publishDir = [ path: { "${params.outdir}/pipeline_info" }, @@ -28,21 +33,6 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - - withName: 'CUSTOM_DUMPSOFTWAREVERSIONS' { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - pattern: '*_versions.yml' - ] - } -} - -// -// Genome preparation options -// - -process { withName: 'CAT_FASTQ' { publishDir = [ path: { "${params.outdir}/cat_fastq" }, @@ -51,6 +41,10 @@ process { // enabled: params.save_merged_fastq //TODO ? implement save_merged_fastq param ] } + + // + // Genome preparation options + // withName: 'INDEX_GENOME' { publishDir = [ path: { "${params.outdir}/index" }, @@ -72,13 +66,10 @@ process { enabled: true ] } -} -// -// Read QC and trimming options -// - -process { + // + // Read QC and trimming options + // withName: 'MIRTRACE_RUN' { publishDir = [ path: { "${params.outdir}/mirtrace" }, @@ -86,145 +77,128 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } -} -if (!params.skip_fastp) { - process { - withName: '.*:FASTQC_UMITOOLS_FASTP:FASTP' { - ext.args = [ "", - params.trim_fastq ? "" : "--disable_adapter_trimming", - params.clip_r1 > 0 ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1. - params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. - params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", - params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter == null ? '' : "--adapter_sequence ${params.three_prime_adapter}" - ].join(" ").trim() - publishDir = [ - [ - path: { "${params.outdir}/fastp" }, - mode: params.publish_dir_mode, - pattern: "*.{json,html}" - ], - [ - path: { "${params.outdir}/fastp/log" }, - mode: params.publish_dir_mode, - pattern: "*.log" - ], - [ - path: { "${params.outdir}/fastp" }, - mode: params.publish_dir_mode, - pattern: "*.fail.fastq.gz", - enabled: params.save_trimmed_fail - ] + withName: '.*:FASTQC_UMITOOLS_FASTP:FASTP' { + ext.args = [ "", + params.trim_fastq ? "" : "--disable_adapter_trimming", + params.clip_r1 > 0 ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1. + params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. + params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", + params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", + params.three_prime_adapter == null ? '' : "--adapter_sequence ${params.three_prime_adapter}" + ].join(" ").trim() + publishDir = [ + [ + path: { "${params.outdir}/fastp" }, + mode: params.publish_dir_mode, + pattern: "*.{json,html}" + ], + [ + path: { "${params.outdir}/fastp/log" }, + mode: params.publish_dir_mode, + pattern: "*.log" + ], + [ + path: { "${params.outdir}/fastp" }, + mode: params.publish_dir_mode, + pattern: "*.fail.fastq.gz", + enabled: params.save_trimmed_fail ] - } - + ] } - if (!params.skip_fastqc) { - process { - withName: '.*:.*:FASTQC_UMITOOLS_FASTP:FASTQC_RAW' { - ext.args = '--quiet' - publishDir = [ - path: { "${params.outdir}/fastqc/raw" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: '.*:.*:FASTQC_UMITOOLS_FASTP:FASTQC_TRIM' { - ext.args = '--quiet' - publishDir = [ - path: { "${params.outdir}/fastqc/trim" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - } - } -} -if (params.with_umi && !params.skip_umi_extract) { - process { - withName: '.*:FASTQ_FASTQC_UMITOOLS_FASTP:UMITOOLS_EXTRACT' { - ext.args = [ - params.umitools_extract_method ? "--extract-method=${params.umitools_extract_method}" : '', - params.umitools_bc_pattern ? "--bc-pattern='${params.umitools_bc_pattern}'" : '', - ].join(' ').trim() - publishDir = [ - [ - path: { "${params.outdir}/umitools" }, - mode: params.publish_dir_mode, - pattern: "*.log" - ], - [ - path: { "${params.outdir}/umitools" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz", - enabled: params.save_umi_intermeds - ] - ] - } + withName: '.*:.*:FASTQC_UMITOOLS_FASTP:FASTQC_RAW' { + ext.args = '--quiet' + publishDir = [ + path: { "${params.outdir}/fastqc/raw" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: '.*:.*:FASTQC_UMITOOLS_FASTP:FASTQC_TRIM' { + ext.args = '--quiet' + publishDir = [ + path: { "${params.outdir}/fastqc/trim" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] } -} -// -// UMI tools deduplication -// -if (params.with_umi) { - process { - withName: '.*:DEDUPLICATE_UMIS:UMICOLLAPSE' { - ext.args = { meta.single_end ? "--algo ${params.umitools_method} --two-pass" : "--method ${params.umitools_method} --two-pass --paired --remove-unpaired --remove-chimeric" } - ext.prefix = { "${meta.id}.umi_dedup.sorted" } - publishDir = [ - path: { "${params.outdir}/umi_dedup" }, + withName: '.*:FASTQ_FASTQC_UMITOOLS_FASTP:UMITOOLS_EXTRACT' { + ext.args = [ + params.umitools_extract_method ? "--extract-method=${params.umitools_extract_method}" : '', + params.umitools_bc_pattern ? "--bc-pattern='${params.umitools_bc_pattern}'" : '', + ].join(' ').trim() + publishDir = [ + [ + path: { "${params.outdir}/umitools" }, mode: params.publish_dir_mode, - pattern: '*.bam', - enabled: ( - params.save_umi_intermeds - ) - ] - } - - withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' { - ext.prefix = { "${meta.id}.sorted" } - publishDir = [ - path: { "${params.outdir}/umi_dedup" }, + pattern: "*.log" + ], + [ + path: { "${params.outdir}/umitools" }, mode: params.publish_dir_mode, - pattern: '*.{bam}', - enabled: ( - params.save_umi_intermeds - ) + pattern: "*.fastq.gz", + enabled: params.save_umi_intermeds ] - } + ] + } - withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX' { - ext.prefix = { "${meta.id}.sorted" } - publishDir = [ - path: { "${params.outdir}/umi_dedup" }, - mode: params.publish_dir_mode, - pattern: '*.{bai,csi}', - enabled: ( - params.save_umi_intermeds - ) - ] - } + // + // UMI tools deduplication + // - withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:.*' { - publishDir = [ - path: { "${params.outdir}/umi_dedup/samtools_stats" }, - mode: params.publish_dir_mode, - pattern: '*.{stats,flagstat,idxstats}' - ] - } + withName: '.*:DEDUPLICATE_UMIS:UMICOLLAPSE' { + ext.args = { meta.single_end ? "--algo ${params.umitools_method} --two-pass" : "--method ${params.umitools_method} --two-pass --paired --remove-unpaired --remove-chimeric" } + ext.prefix = { "${meta.id}.umi_dedup.sorted" } + publishDir = [ + path: { "${params.outdir}/umi_dedup" }, + mode: params.publish_dir_mode, + pattern: '*.bam', + enabled: ( + params.save_umi_intermeds + ) + ] } -} -// -// Quantification -// + withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' { + ext.prefix = { "${meta.id}.sorted" } + publishDir = [ + path: { "${params.outdir}/umi_dedup" }, + mode: params.publish_dir_mode, + pattern: '*.{bam}', + enabled: ( + params.save_umi_intermeds + ) + ] + } + + withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX' { + ext.prefix = { "${meta.id}.sorted" } + publishDir = [ + path: { "${params.outdir}/umi_dedup" }, + mode: params.publish_dir_mode, + pattern: '*.{bai,csi}', + enabled: ( + params.save_umi_intermeds + ) + ] + } + + withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:.*' { + publishDir = [ + path: { "${params.outdir}/umi_dedup/samtools_stats" }, + mode: params.publish_dir_mode, + pattern: '*.{stats,flagstat,idxstats}' + ] + } + + // + // Quantification + // -process { withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:PARSE_.*' { publishDir = [ path: { "${params.outdir}/genome" }, @@ -282,10 +256,8 @@ process { enabled: true ] } -} -process { withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_.*' { ext.prefix = { "${meta.id}.sorted" } publishDir = [ @@ -310,37 +282,38 @@ process { pattern: "unmapped/*.gz" ] } -} -if (!params.skip_mirdeep) { - process { - withName: 'MIRDEEP2_MAPPER' { - publishDir = [ - path: { "${params.outdir}/mirdeep2/mapper" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: 'MIRDEEP2_RUN' { - publishDir = [ - path: { "${params.outdir}/mirdeep2/run" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } + withName: 'MIRDEEP2_MAPPER' { + publishDir = [ + path: { "${params.outdir}/mirdeep2/mapper" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] } -} - -if (!params.skip_multiqc) { - process { - withName: 'MULTIQC' { - ext.args = params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' - publishDir = [ - path: { "${params.outdir}/multiqc" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } + withName: 'MIRDEEP2_RUN' { + publishDir = [ + path: { "${params.outdir}/mirdeep2/run" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] } + // + // reports + // + withName: 'CUSTOM_DUMPSOFTWAREVERSIONS' { + publishDir = [ + path: { "${params.outdir}/pipeline_info" }, + mode: params.publish_dir_mode, + pattern: '*_versions.yml' + ] + } + withName: 'MULTIQC' { + ext.args = params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' + publishDir = [ + path: { "${params.outdir}/multiqc" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } } From 306cd4e072cba60111196f01b6556b584d7f4d6b Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Fri, 26 Jan 2024 14:06:05 +0100 Subject: [PATCH 3/6] WIP reorganize config --- conf/modules.config | 133 ++++++++++++++++++++++++-------------------- 1 file changed, 74 insertions(+), 59 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 6e3f17dd..bcc0f018 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -43,50 +43,17 @@ process { } // - // Genome preparation options + // FASTQ_FASTQC_UMITOOLS_FASTP // - withName: 'INDEX_GENOME' { - publishDir = [ - path: { "${params.outdir}/index" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:FORMAT_.*' { - publishDir = [ - path: { "${params.outdir}/genome" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: 'EDGER_QC' { - publishDir = [ - path: { "${params.outdir}/edger" }, - mode: params.publish_dir_mode, - enabled: true - ] - } - - // - // Read QC and trimming options - // - withName: 'MIRTRACE_RUN' { - publishDir = [ - path: { "${params.outdir}/mirtrace" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: '.*:FASTQC_UMITOOLS_FASTP:FASTP' { - ext.args = [ "", - params.trim_fastq ? "" : "--disable_adapter_trimming", - params.clip_r1 > 0 ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1. - params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. - params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", - params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter == null ? '' : "--adapter_sequence ${params.three_prime_adapter}" - ].join(" ").trim() + ext.args = [ "", + params.trim_fastq ? "" : "--disable_adapter_trimming", + params.clip_r1 > 0 ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1. + params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. + params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", + params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", + params.three_prime_adapter == null ? '' : "--adapter_sequence ${params.three_prime_adapter}" + ].join(" ").trim() publishDir = [ [ path: { "${params.outdir}/fastp" }, @@ -106,8 +73,6 @@ process { ] ] } - - withName: '.*:.*:FASTQC_UMITOOLS_FASTP:FASTQC_RAW' { ext.args = '--quiet' publishDir = [ @@ -119,18 +84,16 @@ process { withName: '.*:.*:FASTQC_UMITOOLS_FASTP:FASTQC_TRIM' { ext.args = '--quiet' publishDir = [ - path: { "${params.outdir}/fastqc/trim" }, + path: { "${params.outdir}/fastqc/trimmed" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - - withName: '.*:FASTQ_FASTQC_UMITOOLS_FASTP:UMITOOLS_EXTRACT' { - ext.args = [ - params.umitools_extract_method ? "--extract-method=${params.umitools_extract_method}" : '', - params.umitools_bc_pattern ? "--bc-pattern='${params.umitools_bc_pattern}'" : '', - ].join(' ').trim() + ext.args = [ + params.umitools_extract_method ? "--extract-method=${params.umitools_extract_method}" : '', + params.umitools_bc_pattern ? "--bc-pattern='${params.umitools_bc_pattern}'" : '', + ].join(' ').trim() publishDir = [ [ path: { "${params.outdir}/umitools" }, @@ -147,9 +110,19 @@ process { } // - // UMI tools deduplication + // Make bowtie index // + withName: 'INDEX_GENOME' { + publishDir = [ + path: { "${params.outdir}/bowtie_index" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + // + // UMI deduplication + // withName: '.*:DEDUPLICATE_UMIS:UMICOLLAPSE' { ext.args = { meta.single_end ? "--algo ${params.umitools_method} --two-pass" : "--method ${params.umitools_method} --two-pass --paired --remove-unpaired --remove-chimeric" } ext.prefix = { "${meta.id}.umi_dedup.sorted" } @@ -163,6 +136,54 @@ process { ] } + // + // MIRTRACE QC + // + withName: 'MIRTRACE_RUN' { + publishDir = [ + path: { "${params.outdir}/mirtrace" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + // + // CONTAMINANT_FILTER + // + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:CONTAMINANT_FILTER:.*' { + publishDir = [ + path: { "${params.outdir}/contaminant_filter/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + + + //---- + + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:FORMAT_.*' { + publishDir = [ + path: { "${params.outdir}/genome" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: 'EDGER_QC' { + publishDir = [ + path: { "${params.outdir}/edger" }, + mode: params.publish_dir_mode, + enabled: true + ] + } + + + // + // UMI tools deduplication + // + + + withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' { ext.prefix = { "${meta.id}.sorted" } publishDir = [ @@ -249,13 +270,7 @@ process { pattern: "*.tsv" ] } - withName: 'EDGER_QC' { - publishDir = [ - path: { "${params.outdir}/edger" }, - mode: params.publish_dir_mode, - enabled: true - ] - } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_.*' { From ede1ee9ca78f710033bea6ba10ce13ccc90aa011 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Fri, 26 Jan 2024 14:32:17 +0100 Subject: [PATCH 4/6] More config updates --- conf/modules.config | 162 ++++++++++++++++++++++++-------------------- 1 file changed, 88 insertions(+), 74 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index bcc0f018..b6edc20c 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -96,12 +96,12 @@ process { ].join(' ').trim() publishDir = [ [ - path: { "${params.outdir}/umitools" }, + path: { "${params.outdir}/umi_dedup/fastq_extracted_umi" }, mode: params.publish_dir_mode, pattern: "*.log" ], [ - path: { "${params.outdir}/umitools" }, + path: { "${params.outdir}/umi_dedup/fastq_extracted_umi" }, mode: params.publish_dir_mode, pattern: "*.fastq.gz", enabled: params.save_umi_intermeds @@ -114,7 +114,7 @@ process { // withName: 'INDEX_GENOME' { publishDir = [ - path: { "${params.outdir}/bowtie_index" }, + path: { "${params.outdir}/bowtie_index/genome" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] @@ -123,11 +123,50 @@ process { // // UMI deduplication // + withName: '.*:DEDUPLICATE_UMIS:UMI_MAP_GENOME' { + publishDir = [ + path: { "${params.outdir}/umi_dedup/bam_mapped" }, + mode: params.publish_dir_mode, + pattern: '*.bam', + enabled: ( + params.save_umi_intermeds + ) + ] + } + withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' { + ext.prefix = { "${meta.id}.sorted" } + publishDir = [ + path: { "${params.outdir}/umi_dedup/bam_mapped_sorted" }, + mode: params.publish_dir_mode, + pattern: '*.{bam}', + enabled: ( + params.save_umi_intermeds + ) + ] + } + withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX' { + ext.prefix = { "${meta.id}.sorted" } + publishDir = [ + path: { "${params.outdir}/umi_dedup/bam_mapped_sorted" }, + mode: params.publish_dir_mode, + pattern: '*.{bai,csi}', + enabled: ( + params.save_umi_intermeds + ) + ] + } + withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:.*' { + publishDir = [ + path: { "${params.outdir}/umi_dedup/bam_mapped_sorted" }, + mode: params.publish_dir_mode, + pattern: '*.{stats,flagstat,idxstats}' + ] + } withName: '.*:DEDUPLICATE_UMIS:UMICOLLAPSE' { ext.args = { meta.single_end ? "--algo ${params.umitools_method} --two-pass" : "--method ${params.umitools_method} --two-pass --paired --remove-unpaired --remove-chimeric" } ext.prefix = { "${meta.id}.umi_dedup.sorted" } publishDir = [ - path: { "${params.outdir}/umi_dedup" }, + path: { "${params.outdir}/bam_deduplicated" }, mode: params.publish_dir_mode, pattern: '*.bam', enabled: ( @@ -135,6 +174,15 @@ process { ) ] } + withName: '.*:DEDUPLICATE_UMIS:SAMTOOLS_BAM2FQ' { + publishDir = [ + path: { "${params.outdir}/umi_dedup/fastq_deduplicated" }, + mode: params.publish_dir_mode, + enabled: ( + params.save_umi_intermeds + ) + ] + } // // MIRTRACE QC @@ -158,146 +206,112 @@ process { ] } - - - //---- - + // + // MIRNA_QUANT + // withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:FORMAT_.*' { publishDir = [ - path: { "${params.outdir}/genome" }, + path: { "${params.outdir}/mirna_quant/reference" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: 'EDGER_QC' { - publishDir = [ - path: { "${params.outdir}/edger" }, - mode: params.publish_dir_mode, - enabled: true - ] - } - - - // - // UMI tools deduplication - // - - - - withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' { - ext.prefix = { "${meta.id}.sorted" } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:PARSE_.*' { publishDir = [ - path: { "${params.outdir}/umi_dedup" }, + path: { "${params.outdir}/mirna_quant/reference" }, mode: params.publish_dir_mode, - pattern: '*.{bam}', - enabled: ( - params.save_umi_intermeds - ) + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - - withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX' { - ext.prefix = { "${meta.id}.sorted" } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:INDEX_MATURE' { publishDir = [ - path: { "${params.outdir}/umi_dedup" }, + path: { "${params.outdir}/bowtie_index/mirna_mature" }, mode: params.publish_dir_mode, - pattern: '*.{bai,csi}', - enabled: ( - params.save_umi_intermeds - ) + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - - withName: '.*:DEDUPLICATE_UMIS:BAM_SORT_STATS_SAMTOOLS:.*' { + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:INDEX_HAIRPIN' { publishDir = [ - path: { "${params.outdir}/umi_dedup/samtools_stats" }, - mode: params.publish_dir_mode, - pattern: '*.{stats,flagstat,idxstats}' - ] - } - - // - // Quantification - // - - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:PARSE_.*' { - publishDir = [ - path: { "${params.outdir}/genome" }, + path: { "${params.outdir}/bowtie_index/mirna_hairpin" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:BAM_STATS_.*:SAMTOOLS_SORT' { - ext.prefix = { "${meta.id}.sorted" } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:(BOWTIE_MAP_MATURE|BAM_STATS_MATURE).*' { publishDir = [ - path: { "${params.outdir}/samtools" }, + path: { "${params.outdir}/mirna_quant/bam/mature" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:BAM_STATS_.*:BAM_STATS_SAMTOOLS:.*' { - ext.prefix = { "${meta.id}.sorted" } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:(BOWTIE_MAP_HAIRPIN|BAM_STATS_HAIRPIN).*' { publishDir = [ - path: { "${params.outdir}/samtools/samtools_stats" }, + path: { "${params.outdir}/mirna_quant/bam/hairpin" }, mode: params.publish_dir_mode, - pattern: "*.{stats,flagstat,idxstats}" + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:BOWTIE_MAP_.*' { + withName: 'EDGER_QC' { publishDir = [ - path: { "${params.outdir}/unmapped/fastq" }, + path: { "${params.outdir}/mirna_quant/edger_qc" }, mode: params.publish_dir_mode, - pattern: "unmapped/*.gz" + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } withName: 'SEQCLUSTER_SEQUENCES' { publishDir = [ - path: { "${params.outdir}/seqcluster" }, - enabled: false + path: { "${params.outdir}/mirna_quant/seqcluster" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } withName: 'MIRTOP_QUANT' { publishDir = [ - path: { "${params.outdir}" }, + path: { "${params.outdir}/mirna_quant/mirtop" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:TABLE_MERGE' { publishDir = [ - path: { "${params.outdir}/mirtop" }, + path: { "${params.outdir}/mirna_quant/mirtop" }, mode: params.publish_dir_mode, pattern: "*.tsv" ] } - + // + // GENOME_QUANT + // withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_.*' { ext.prefix = { "${meta.id}.sorted" } publishDir = [ - path: { "${params.outdir}/samtools" }, + path: { "${params.outdir}/genome_quant/bam" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' { ext.prefix = { "${meta.id}.sorted" } publishDir = [ - path: { "${params.outdir}/samtools/samtools_stats" }, + path: { "${params.outdir}/genome_quant/bam" }, mode: params.publish_dir_mode, pattern: "*.{stats,flagstat,idxstats}" ] } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BOWTIE_MAP_.*' { + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BOWTIE_MAP_GENOME' { publishDir = [ - path: { "${params.outdir}/unmapped/fastq" }, + path: { "${params.outdir}/unmapped/bam" }, mode: params.publish_dir_mode, pattern: "unmapped/*.gz" ] } + + // + // MIRDEEP + // withName: 'MIRDEEP2_MAPPER' { publishDir = [ path: { "${params.outdir}/mirdeep2/mapper" }, From e707e28a4ad0c55acf1b95a76ff3ad9144e29a0c Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Fri, 26 Jan 2024 14:49:14 +0100 Subject: [PATCH 5/6] Final outputpath fixes --- conf/modules.config | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index b6edc20c..b57e4f92 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -73,7 +73,7 @@ process { ] ] } - withName: '.*:.*:FASTQC_UMITOOLS_FASTP:FASTQC_RAW' { + withName: '.*:FASTQ_FASTQC_UMITOOLS_FASTP:FASTQC_RAW' { ext.args = '--quiet' publishDir = [ path: { "${params.outdir}/fastqc/raw" }, @@ -81,7 +81,7 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: '.*:.*:FASTQC_UMITOOLS_FASTP:FASTQC_TRIM' { + withName: '.*:FASTQ_FASTQC_UMITOOLS_FASTP:FASTQC_TRIM' { ext.args = '--quiet' publishDir = [ path: { "${params.outdir}/fastqc/trimmed" }, @@ -166,7 +166,7 @@ process { ext.args = { meta.single_end ? "--algo ${params.umitools_method} --two-pass" : "--method ${params.umitools_method} --two-pass --paired --remove-unpaired --remove-chimeric" } ext.prefix = { "${meta.id}.umi_dedup.sorted" } publishDir = [ - path: { "${params.outdir}/bam_deduplicated" }, + path: { "${params.outdir}/umi_dedup/bam_deduplicated" }, mode: params.publish_dir_mode, pattern: '*.bam', enabled: ( @@ -178,6 +178,7 @@ process { publishDir = [ path: { "${params.outdir}/umi_dedup/fastq_deduplicated" }, mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename }, enabled: ( params.save_umi_intermeds ) @@ -189,7 +190,8 @@ process { // withName: 'MIRTRACE_RUN' { publishDir = [ - path: { "${params.outdir}/mirtrace" }, + //"mirtrace" already part of the published folder + path: { "${params.outdir}" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] @@ -237,14 +239,30 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:(BOWTIE_MAP_MATURE|BAM_STATS_MATURE).*' { + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:BOWTIE_MAP_MATURE' { publishDir = [ path: { "${params.outdir}/mirna_quant/bam/mature" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:(BOWTIE_MAP_HAIRPIN|BAM_STATS_HAIRPIN).*' { + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:BAM_STATS_MATURE:.*' { + ext.prefix = { "${meta.id}.sorted" } + publishDir = [ + path: { "${params.outdir}/mirna_quant/bam/mature" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:BOWTIE_MAP_HAIRPIN' { + publishDir = [ + path: { "${params.outdir}/mirna_quant/bam/hairpin" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:BAM_STATS_HAIRPIN:.*' { + ext.prefix = { "${meta.id}.sorted" } publishDir = [ path: { "${params.outdir}/mirna_quant/bam/hairpin" }, mode: params.publish_dir_mode, @@ -265,9 +283,17 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:MIRNA_QUANT:BOWTIE_MAP_SEQCLUSTER' { + publishDir = [ + path: { "${params.outdir}/mirna_quant/bam/seqcluster" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } withName: 'MIRTOP_QUANT' { publishDir = [ - path: { "${params.outdir}/mirna_quant/mirtop" }, + //mirtop already part of the output folder + path: { "${params.outdir}/mirna_quant/" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] @@ -302,9 +328,9 @@ process { } withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BOWTIE_MAP_GENOME' { publishDir = [ - path: { "${params.outdir}/unmapped/bam" }, + path: { "${params.outdir}/genome_quant/bam" }, mode: params.publish_dir_mode, - pattern: "unmapped/*.gz" + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } From ec22f390cd9a534ad760f1320f9de565a602eea7 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Fri, 26 Jan 2024 15:47:54 +0100 Subject: [PATCH 6/6] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75b6e128..be1d0138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## v2.3.0 - 2024-01-25 - Gray Zinc Dalmatian +- [[#307]](https://github.com/nf-core/smrnaseq/pull/307) - Clean up config file and improve output folder structure - [[#299]](https://github.com/nf-core/smrnaseq/issues/299) - Bugfix for missing inputs in BAM stats (`genome_quant.r`) - [[#164]](https://github.com/nf-core/smrnaseq/pull/164) - UMI Handling Feature implemented in the pipeline - [[#302]](https://github.com/nf-core/smrnaseq/pull/302) - Merged in nf-core template v2.11.1