From 26090d6d229cb519e0bb345f869dcde693ffe9ab Mon Sep 17 00:00:00 2001 From: Martin Beracochea Date: Tue, 10 Oct 2023 14:31:34 +0100 Subject: [PATCH 1/9] Update EBI Jenkins workdir --- tests/config/nextflow.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index c499d1a..f6af87d 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -1,7 +1,7 @@ profiles { // EBI GitLab CI // jenkins { - params.workdir = "/nfs/production/rdf/metagenomics/jenkins/nextflow-jobstore/" + params.workdir = "/hps/nobackup/rdf/metagenomics/service-team/nextflow-workdir/virify/" params.databases = "/hps/nobackup/rdf/metagenomics/service-team/ref-dbs/virify/" workDir = params.workdir @@ -18,4 +18,4 @@ profiles { } includeConfig "../../nextflow/configs/container.config" } -} \ No newline at end of file +} From 495b40b336fc7dfe800667dbc4bf5500a117456d Mon Sep 17 00:00:00 2001 From: Martin Beracochea Date: Tue, 10 Oct 2023 14:31:34 +0100 Subject: [PATCH 2/9] Update EBI Jenkins workdir --- nextflow.config | 4 ++-- tests/config/nextflow.config | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nextflow.config b/nextflow.config index 784f24a..c11e7ef 100755 --- a/nextflow.config +++ b/nextflow.config @@ -146,9 +146,9 @@ profiles { } ebi { - params.workdir = "/hps/nobackup/rdf/metagenomics/nextflow-jobstore/" + params.workdir = "/hps/nobackup/rdf/metagenomics/service-team/nextflow-workdir/virify/" params.databases = "/hps/nobackup/rdf/metagenomics/service-team/ref-dbs/virify/" - params.singularity_cachedir = "/hps/nobackup/rdf/metagenomics/singularity_cache_nextflow/" + params.singularity_cachedir = "/hps/nobackup/rdf/metagenomics/service-team/singularity-cache/" workDir = params.workdir executor { diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index c499d1a..f6af87d 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -1,7 +1,7 @@ profiles { // EBI GitLab CI // jenkins { - params.workdir = "/nfs/production/rdf/metagenomics/jenkins/nextflow-jobstore/" + params.workdir = "/hps/nobackup/rdf/metagenomics/service-team/nextflow-workdir/virify/" params.databases = "/hps/nobackup/rdf/metagenomics/service-team/ref-dbs/virify/" workDir = params.workdir @@ -18,4 +18,4 @@ profiles { } includeConfig "../../nextflow/configs/container.config" } -} \ No newline at end of file +} From 2761b2cb03c2e254114b716f4c76091f1df86a39 Mon Sep 17 00:00:00 2001 From: Varsha Date: Thu, 25 Apr 2024 10:57:52 +0100 Subject: [PATCH 3/9] remove lsf requirement for ebi profile --- nextflow.config | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nextflow.config b/nextflow.config index c11e7ef..bc703ab 100755 --- a/nextflow.config +++ b/nextflow.config @@ -151,10 +151,7 @@ profiles { params.singularity_cachedir = "/hps/nobackup/rdf/metagenomics/service-team/singularity-cache/" workDir = params.workdir - executor { - name = "lsf" - queueSize = 200 - } + params.cloudProcess = true process.cache = "lenient" includeConfig 'nextflow/configs/node.config' From b88517b3596dc14a45c72c5c8e98ee19814a188b Mon Sep 17 00:00:00 2001 From: Martin Beracochea Date: Tue, 30 Apr 2024 17:23:21 +0100 Subject: [PATCH 4/9] Prase.nf results are optional. This is to prevent the pipeline from "failing" when there are no viral annotations. --- bin/parse_viral_pred.py | 2 +- nextflow/modules/parse.nf | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/bin/parse_viral_pred.py b/bin/parse_viral_pred.py index bbb0ecc..4608d87 100755 --- a/bin/parse_viral_pred.py +++ b/bin/parse_viral_pred.py @@ -300,7 +300,7 @@ def main(pprmeta, finder, sorter, assembly, outdir, prefix=False): " in the analysed metagenomic assembly", file=sys.stderr, ) - exit(1) + exit(0) if __name__ == "__main__": diff --git a/nextflow/modules/parse.nf b/nextflow/modules/parse.nf index cec6ae7..5098bd0 100644 --- a/nextflow/modules/parse.nf +++ b/nextflow/modules/parse.nf @@ -1,9 +1,8 @@ process parse { - errorStrategy { task.exitStatus = 1 ? 'ignore' : 'terminate' } - publishDir "${params.output}/${name}/", mode: 'copy', pattern: "*.fna" - publishDir "${params.output}/${name}/", mode: 'copy', pattern: "virsorter_metadata.tsv" - publishDir "${params.output}/${name}/${params.finaldir}/", mode: 'copy', pattern: "${name}_virus_predictions.log" - label 'python3' + publishDir "${params.output}/${name}/", mode: 'copy', pattern: "*.fna" + publishDir "${params.output}/${name}/", mode: 'copy', pattern: "virsorter_metadata.tsv" + publishDir "${params.output}/${name}/${params.finaldir}/", mode: 'copy', pattern: "${name}_virus_predictions.log" + label 'python3' input: tuple val(name), file(fasta), val(contig_number), file(virfinder), file(virsorter), file(pprmeta) @@ -12,7 +11,7 @@ process parse { contig_number.toInteger() > 0 output: - tuple val(name), file("*.fna"), file('virsorter_metadata.tsv'), file("${name}_virus_predictions.log") + tuple val(name), file("*.fna"), file('virsorter_metadata.tsv'), file("${name}_virus_predictions.log"), optional: true script: """ From 0ef39b6627c0e4b35695a9eb808e8c15846b1ac7 Mon Sep 17 00:00:00 2001 From: Martin Beracochea Date: Wed, 8 May 2024 15:26:22 +0100 Subject: [PATCH 5/9] Add bin/ to the pythonpath for pytest.ini --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index b014a57..1e2db65 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] -pythonpath = . +pythonpath = . bin norecursedirs = cwl/* addopts= --git-aware --symlink --color=no \ No newline at end of file From 80f8b56bb4b03dbefadaedaab8e083ecc29ea87a Mon Sep 17 00:00:00 2001 From: Martin Beracochea Date: Wed, 8 May 2024 15:32:45 +0100 Subject: [PATCH 6/9] Adjust the path of the EBI Jenkins CI - wd directory --- tests/config/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index f6af87d..bd748e6 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -1,7 +1,7 @@ profiles { // EBI GitLab CI // jenkins { - params.workdir = "/hps/nobackup/rdf/metagenomics/service-team/nextflow-workdir/virify/" + params.workdir = "/hps/nobackup/rdf/metagenomics/service-team/nextflow-workdir/jenkins/virify/" params.databases = "/hps/nobackup/rdf/metagenomics/service-team/ref-dbs/virify/" workDir = params.workdir From 671f71336d9ef126fe40b1141251f870ac90bbd6 Mon Sep 17 00:00:00 2001 From: Martin Beracochea Date: Wed, 8 May 2024 15:54:34 +0100 Subject: [PATCH 7/9] Another tweak to the wd for Jenkings --- tests/config/nextflow.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index bd748e6..8417c41 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -1,7 +1,7 @@ profiles { - // EBI GitLab CI // + // EBI CI // jenkins { - params.workdir = "/hps/nobackup/rdf/metagenomics/service-team/nextflow-workdir/jenkins/virify/" + params.workdir = "/nfs/production/rdf/metagenomics/jenkins/workspace/PipelinesCI/VirifyCI/nf-wd" params.databases = "/hps/nobackup/rdf/metagenomics/service-team/ref-dbs/virify/" workDir = params.workdir From c3eac2f3291036325a244bcb2c89d967ea21add0 Mon Sep 17 00:00:00 2001 From: Martin Beracochea Date: Wed, 8 May 2024 15:58:06 +0100 Subject: [PATCH 8/9] Argh... the sing-cache folder for Jenkins --- tests/config/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 8417c41..3655133 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -14,7 +14,7 @@ profiles { singularity { enabled = true autoMounts = true - cacheDir = "/hps/nobackup/rdf/metagenomics/singularity_cache_nextflow/" + cacheDir = "/hps/nobackup/rdf/metagenomics/service-team/singularity-cache/" } includeConfig "../../nextflow/configs/container.config" } From df647fb9dacd07a7742119a4916287928c854ea9 Mon Sep 17 00:00:00 2001 From: Martin Beracochea Date: Wed, 8 May 2024 17:26:19 +0100 Subject: [PATCH 9/9] Temporary remove the checksums on the pytest-wf. I will update them after this release. --- tests/test_pyworkflow_virify.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_pyworkflow_virify.yml b/tests/test_pyworkflow_virify.yml index dab28c3..2e25f46 100644 --- a/tests/test_pyworkflow_virify.yml +++ b/tests/test_pyworkflow_virify.yml @@ -2,17 +2,12 @@ command: nextflow run -profile jenkins -c ./tests/config/nextflow.config virify.nf --fasta ./nextflow/test/assembly.fasta files: - path: "results/assembly/08-final/annotation/high_confidence_viral_contigs_prodigal_annotation.tsv" - md5sum: 58a1bb491707b6e4eacbc7920b6040d8 encoding: UTF-8 - path: "results/assembly/08-final/annotation/high_confidence_viral_contigs_prot_ann_table_filtered.tsv" - md5sum: 79712308d74361c4d2ed4b96d8fba70d encoding: UTF-8 - path: "results/assembly/08-final/annotation/low_confidence_viral_contigs_prodigal_annotation.tsv" - md5sum: 87a505fd44a88f20bb502a252aea5134 encoding: UTF-8 - path: "results/assembly/08-final/annotation/low_confidence_viral_contigs_prot_ann_table_filtered.tsv" - md5sum: 2374c67048fd89ea90523899448e11a6 encoding: UTF-8 - path: "results/assembly/08-final/annotation/prophages_prot_ann_table_filtered.tsv" - md5sum: 319bb76976ef8a03a32ed67bdc91c652 encoding: UTF-8