Remove --versioned_outdir param (#52)

* Remove --versioned_outdir param * Add test to check for expected outputs
BCCDC-PHL · May 16, 2024 · c8b63aa · c8b63aa
1 parent d014d32
commit c8b63aa
Show file tree

Hide file tree

Showing 9 changed files with 46 additions and 11 deletions.
diff --git a/.github/scripts/check_outputs.py b/.github/scripts/check_outputs.py
@@ -27,6 +27,36 @@ def check_provenance_format_valid(provenance_files, schema):
 
     return True
 
+def check_expected_files_exist(output_dir, sample_ids):
+    """
+    Check that the expected files exist in the output directory.
+
+    :param output_dir: Path to the output directory
+    :param sample_ids: List of sample IDs
+    :return: True if all expected files exist, False otherwise
+    :rtype: bool
+    """
+    for sample_id in sample_ids:
+        expected_files = [
+            f"{sample_id}/{sample_id}_fastp.csv",
+            f"{sample_id}/{sample_id}_fastp.json",
+            f"{sample_id}/{sample_id}_unicycler_short.fa",
+            f"{sample_id}/{sample_id}_unicycler_short.gfa",
+            f"{sample_id}/{sample_id}_unicycler_short.log",
+            f"{sample_id}/{sample_id}_unicycler_short_bandage.png",
+            f"{sample_id}/{sample_id}_unicycler_short_prokka.gbk",
+            f"{sample_id}/{sample_id}_unicycler_short_prokka.gff",
+            f"{sample_id}/{sample_id}_unicycler_short_quast.csv",
+        ]
+
+        for expected_file in expected_files:
+            expected_file_path = os.path.join(output_dir, expected_file)
+            if not os.path.exists(expected_file_path):
+                print(f"Expected file {expected_file_path} not found")
+                return False
+
+    return True
+
 
 def main(args):
 
@@ -44,12 +74,18 @@ def main(args):
     provenace_files_glob = f"{args.pipeline_outdir}/**/*_provenance.yml"
     provenance_files = glob.glob(provenace_files_glob, recursive=True)
 
-    # TODO: add more tests
+    sample_ids = [os.path.basename(provenance_file).split("_")[0] for provenance_file in provenance_files]
+
+    # TODO: Add more tests
     tests = [
         {
             "test_name": "provenance_format_valid",
             "test_passed": check_provenance_format_valid(provenance_files, provenance_schema),
         },
+        {
+            "test_name": "all_expected_files_exist",
+            "test_passed": check_expected_files_exist(args.pipeline_outdir, sample_ids),
+        },
     ]
 
     output_fields = [

diff --git a/modules/bakta.nf b/modules/bakta.nf
@@ -4,7 +4,7 @@ process bakta {
 
     errorStrategy 'ignore'
 
-    publishDir params.versioned_outdir ? "${params.outdir}/${sample_id}/${params.pipeline_short_name}-v${params.pipeline_minor_version}-output" : "${params.outdir}/${sample_id}", pattern: "${sample_id}*.{gbk,gff,json,log}", mode: 'copy'
+    publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}*.{gbk,gff,json,log}", mode: 'copy'
 
     input:
     tuple val(sample_id), path(assembly), val(assembler), val(assembly_mode)

diff --git a/modules/fastp.nf b/modules/fastp.nf
@@ -2,7 +2,7 @@ process fastp {
 
     tag { sample_id }
 
-    publishDir params.versioned_outdir ? "${params.outdir}/${sample_id}/${params.pipeline_short_name}-v${params.minor_version}-output" : "${params.outdir}/${sample_id}", pattern: "${sample_id}_fastp.json", mode: 'copy'
+    publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_fastp.json", mode: 'copy'
 
     input:
     tuple val(sample_id), path(reads)
@@ -40,7 +40,7 @@ process fastp_json_to_csv {
 
     executor 'local'
 
-    publishDir params.versioned_outdir ? "${params.outdir}/${sample_id}/${params.pipeline_short_name}-v${params.pipeline_minor_version}-output" : "${params.outdir}/${sample_id}", pattern: "${sample_id}_fastp.csv", mode: 'copy'
+    publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_fastp.csv", mode: 'copy'
 
     input:
     tuple val(sample_id), path(fastp_json)

diff --git a/modules/long_read_qc.nf b/modules/long_read_qc.nf
@@ -60,7 +60,7 @@ process merge_nanoq_reports {
 
     executor 'local'
 
-    publishDir params.versioned_outdir ? "${params.outdir}/${sample_id}/${params.pipeline_short_name}-v${params.minor_version}-output" : "${params.outdir}/${sample_id}", pattern: "${sample_id}_nanoq.csv", mode: 'copy'
+    publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_nanoq.csv", mode: 'copy'
 
     input:
     tuple val(sample_id), path(nanoq_pre_filter), path(nanoq_post_filter)
@@ -82,7 +82,7 @@ process bandage {
     executor 'local'
     errorStrategy 'ignore'
 
-    publishDir params.versioned_outdir ? "${params.outdir}/${sample_id}/${params.pipeline_short_name}-v${params.minor_version}-output" : "${params.outdir}/${sample_id}", pattern: "${sample_id}_${assembler}_${assembly_mode}_bandage.png", mode: 'copy'
+    publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_${assembler}_${assembly_mode}_bandage.png", mode: 'copy'
 
     input:
     tuple val(sample_id), path(assembly_graph), val(assembler), val(assembly_mode)

diff --git a/modules/prokka.nf b/modules/prokka.nf
@@ -4,7 +4,7 @@ process prokka {
 
     errorStrategy 'ignore'
 
-    publishDir params.versioned_outdir ? "${params.outdir}/${sample_id}/${params.pipeline_short_name}-v${params.pipeline_minor_version}-output" : "${params.outdir}/${sample_id}", pattern: "${sample_id}_${assembler}_${assembly_mode}_prokka.{gbk,gff}", mode: 'copy'
+    publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_${assembler}_${assembly_mode}_prokka.{gbk,gff}", mode: 'copy'
 
     input:
     tuple val(sample_id), path(assembly), val(assembler), val(assembly_mode)

diff --git a/modules/provenance.nf b/modules/provenance.nf
@@ -4,7 +4,7 @@ process collect_provenance {
 
     executor 'local'
 
-    publishDir params.versioned_outdir ? "${params.outdir}/${sample_id}/${params.pipeline_short_name}-v${params.pipeline_minor_version}-output" : "${params.outdir}/${sample_id}", pattern: "${sample_id}_*_provenance.yml", mode: 'copy'
+    publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_*_provenance.yml", mode: 'copy'
 
     input:
     tuple val(sample_id), path(provenance_files)

diff --git a/modules/quast.nf b/modules/quast.nf
@@ -41,7 +41,7 @@ process parse_quast_report {
 
     executor 'local'
 
-    publishDir params.versioned_outdir ? "${params.outdir}/${sample_id}/${params.pipeline_short_name}-v${params.pipeline_minor_version}-output" : "${params.outdir}/${sample_id}", pattern: "${sample_id}_${assembler}_${assembly_mode}_quast.csv", mode: 'copy'
+    publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_${assembler}_${assembly_mode}_quast.csv", mode: 'copy'
 
     input:
     tuple val(sample_id), path(quast_report), val(assembler), val(assembly_mode)

diff --git a/modules/unicycler.nf b/modules/unicycler.nf
@@ -2,7 +2,7 @@ process unicycler {
 
     tag { sample_id + ' / ' + assembly_mode }
 
-    publishDir params.versioned_outdir ? "${params.outdir}/${sample_id}/${params.pipeline_short_name}-v${params.pipeline_minor_version}-output" : "${params.outdir}/${sample_id}", pattern: "${sample_id}_unicycler_${assembly_mode}.{fa,gfa,log}", mode: 'copy'
+    publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_unicycler_${assembly_mode}.{fa,gfa,log}", mode: 'copy'
 
     input:
     tuple val(sample_id), path(reads), val(assembly_mode)

diff --git a/nextflow.config b/nextflow.config
@@ -23,7 +23,6 @@ params {
     filtlong_min_length = 1000
     filtlong_keep_percent = 90
     samplesheet_input = 'NO_FILE'
-    versioned_outdir = false
     pipeline_short_name = parsePipelineName(manifest.toMap().get('name'))
     pipeline_minor_version = parseMinorVersion(manifest.toMap().get('version'))
 }