From 3aa0d1ae7a3990b69e175fb003ea51191eea4c59 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 7 Jul 2020 14:24:59 +0100 Subject: [PATCH 1/8] error handling --- modules/spades.nf | 6 ------ nextflow.config | 14 +++++++++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/modules/spades.nf b/modules/spades.nf index 9afdafa..7b7dd31 100644 --- a/modules/spades.nf +++ b/modules/spades.nf @@ -2,12 +2,6 @@ process spades { label 'spades' publishDir "${params.output}/${name}/assembly/", mode: 'copy', pattern: "${name}_raw_assembly.fasta" - errorStrategy { 'retry' } - cpus { 36 } - memory { 400.GB * task.attempt } - clusterOptions { '-P bigmem' } - maxRetries 3 - input: tuple val(name), file(ont), file(illumina) output: diff --git a/nextflow.config b/nextflow.config index dc75882..dd6cc2e 100755 --- a/nextflow.config +++ b/nextflow.config @@ -174,11 +174,15 @@ profiles { } includeConfig 'configs/container.config' + /* - process { + process { withLabel: spades { cpus = 32; - memory = '200 GB'; - clusterOptions = '-P bigmem' } + memory = 90.GB * task.attempt; + clusterOptions = '-P bigmem'; + errorStrategy { task.exitStatus in 130..140 ? 'retry' : 'terminate' }; + maxRetries 4 } + withLabel: flye { cpus = 30; memory { 120.GB * task.attempt }; clusterOptions = '-P bigmem'; @@ -208,6 +212,10 @@ profiles { cacheDir = params.cachedir } includeConfig 'configs/container.config' + + process { + withLabel: spades { cpus = 32; memory = {90.GB * task.attempt}; clusterOptions = '-P bigmem'; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; maxRetries = 4; } + } } ara { From 05b60f123f52aea07a74a11e6f32af000abdab66 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 7 Jul 2020 15:26:25 +0200 Subject: [PATCH 2/8] reduce prodigal cores --- configs/node.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/node.config b/configs/node.config index d4078e7..82e034a 100644 --- a/configs/node.config +++ b/configs/node.config @@ -2,7 +2,7 @@ process { withLabel: basics { cpus = 2 ; memory = '16 GB' } withLabel: bbmap { cpus = 24 ; memory = '16 GB' } withLabel: fastp { cpus = 12 ; memory = '16 GB' } - withLabel: prodigal { cpus = 8 ; memory = '16 GB' } + withLabel: prodigal { cpus = 1 ; memory = '4 GB' } withLabel: diamond { cpus = 12 ; memory = '16 GB' } withLabel: ggplot2 { cpus = 2 ; memory = '4 GB' } withLabel: nanoplot { cpus = 8 ; memory = '14 GB' } From b37061fcd550c72c465fc5b6d88d37028b57288a Mon Sep 17 00:00:00 2001 From: hoelzer Date: Fri, 10 Jul 2020 10:31:29 +0100 Subject: [PATCH 3/8] increase default mem spades --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 8454373..cb5bc10 100755 --- a/nextflow.config +++ b/nextflow.config @@ -215,7 +215,7 @@ profiles { includeConfig 'configs/container.config' process { - withLabel: spades { cpus = 32; memory = {90.GB * task.attempt}; clusterOptions = '-P bigmem'; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; maxRetries = 4; } + withLabel: spades { cpus = 32; memory = {250.GB * task.attempt}; clusterOptions = '-P bigmem'; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; maxRetries = 2; } } } From 836b554daffaf7a6710957b6c476c63c6c8b0995 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Mon, 13 Jul 2020 16:50:00 +0100 Subject: [PATCH 4/8] increase spades mem --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index cb5bc10..1ab00c1 100755 --- a/nextflow.config +++ b/nextflow.config @@ -215,7 +215,7 @@ profiles { includeConfig 'configs/container.config' process { - withLabel: spades { cpus = 32; memory = {250.GB * task.attempt}; clusterOptions = '-P bigmem'; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; maxRetries = 2; } + withLabel: spades { cpus = 32; memory = {400.GB * task.attempt}; clusterOptions = '-P bigmem'; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; maxRetries = 2; } } } From f43f7fc00b4a91835b77d3a8e22f6b9864c40655 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Fri, 17 Jul 2020 09:55:16 +0100 Subject: [PATCH 5/8] diamond cpus --- configs/node.config | 2 +- nextflow.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/node.config b/configs/node.config index 82e034a..d044c25 100644 --- a/configs/node.config +++ b/configs/node.config @@ -3,7 +3,7 @@ process { withLabel: bbmap { cpus = 24 ; memory = '16 GB' } withLabel: fastp { cpus = 12 ; memory = '16 GB' } withLabel: prodigal { cpus = 1 ; memory = '4 GB' } - withLabel: diamond { cpus = 12 ; memory = '16 GB' } + withLabel: diamond { cpus = 24 ; memory = '16 GB' } withLabel: ggplot2 { cpus = 2 ; memory = '4 GB' } withLabel: nanoplot { cpus = 8 ; memory = '14 GB' } withLabel: spades { cpus = 32 ; memory = '80 GB' } diff --git a/nextflow.config b/nextflow.config index 1ab00c1..774e050 100755 --- a/nextflow.config +++ b/nextflow.config @@ -215,7 +215,7 @@ profiles { includeConfig 'configs/container.config' process { - withLabel: spades { cpus = 32; memory = {400.GB * task.attempt}; clusterOptions = '-P bigmem'; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; maxRetries = 2; } + withLabel: spades { cpus = 32; memory = {400.GB * task.attempt}; clusterOptions = '-P bigmem'; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; maxRetries = 4; } } } From 85b3300e9076d74ecb0c26bd8a9cb7f36bf91271 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Sat, 25 Jul 2020 08:34:18 +0100 Subject: [PATCH 6/8] mem handling spades --- modules/spades.nf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/spades.nf b/modules/spades.nf index 7b7dd31..f5bcddb 100644 --- a/modules/spades.nf +++ b/modules/spades.nf @@ -8,7 +8,13 @@ process spades { tuple val(name), file("${name}_raw_assembly.fasta") script: """ - MEM=\$(echo ${task.memory} | sed 's/ GB//g') + SIZE=\$(echo ${task.memory} | awk 'BEGIN{FS=" "};{print \$2}') + if [ \$SIZE == "TB" ]; then + MEM=\$(echo ${task.memory} | sed 's/ TB//g') + MEM=\$(echo \$MEM*1000 | bc | awk 'BEGIN{FS="."};{print \$1}') + else + MEM=\$(echo ${task.memory} | sed 's/ GB//g') + fi spades.py --only-assembler -1 ${illumina[0]} -2 ${illumina[1]} --nanopore ${ont} --meta -o spades_output -t ${task.cpus} -m \${MEM} mv spades_output/contigs.fasta ${name}_raw_assembly.fasta """ From 26ff9efb9376cafb2f945e722a20521d8cf07378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=B6lzer?= Date: Thu, 10 Sep 2020 17:05:15 +0200 Subject: [PATCH 7/8] test push new system --- main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.nf b/main.nf index ab33aa5..d87ca04 100755 --- a/main.nf +++ b/main.nf @@ -93,6 +93,7 @@ if (params.host) { .view() } + /************************** * MODULES **************************/ From 47eaf9afdb8116d772e3a63be1185df9bdbdd21a Mon Sep 17 00:00:00 2001 From: hoelzer Date: Wed, 30 Sep 2020 09:38:44 +0100 Subject: [PATCH 8/8] update hpc changes --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 774e050..2686cbf 100755 --- a/nextflow.config +++ b/nextflow.config @@ -215,7 +215,7 @@ profiles { includeConfig 'configs/container.config' process { - withLabel: spades { cpus = 32; memory = {400.GB * task.attempt}; clusterOptions = '-P bigmem'; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; maxRetries = 4; } + withLabel: spades { cpus = 32; memory = {1600.GB * task.attempt}; clusterOptions = '-P bigmem'; errorStrategy = { task.exitStatus in 1 || 130..140 ? 'retry' : 'terminate' }; maxRetries = 4; } } }