Merge pull request #195 from adamrtalbot/inherit_configs_recursively

Recursively inherit configs

.github/tags.yml

@@ -14,74 +14,66 @@ sra_to_samplesheet:
 
 # workflow: sra
 sra_custom_ena_metadata_fields:
-  - conf/modules/sra.config
-  - modules/local/sra_ids_to_runinfo/main.nf
-  - modules/local/sra_runinfo_to_ftp/main.nf
-  - modules/local/sra_fastq_ftp/main.nf
-  - modules/local/sra_to_samplesheet/main.nf
-  - modules/local/sra_merge_samplesheet/main.nf
-  - workflows/sra/main.nf
+  - modules/local/sra_ids_to_runinfo/*
+  - modules/local/sra_runinfo_to_ftp/*
+  - modules/local/sra_fastq_ftp/*
+  - modules/local/sra_to_samplesheet/*
+  - modules/local/sra_merge_samplesheet/*
+  - workflows/sra/*
   - workflows/sra/tests/sra_custom_ena_metadata_fields.nf.test
 sra_default_parameters:
-  - conf/modules/sra.config
-  - modules/local/sra_ids_to_runinfo/main.nf
-  - modules/local/sra_runinfo_to_ftp/main.nf
-  - modules/local/sra_fastq_ftp/main.nf
-  - modules/local/sra_to_samplesheet/main.nf
-  - modules/local/sra_merge_samplesheet/main.nf
-  - workflows/sra/main.nf
+  - modules/local/sra_ids_to_runinfo/*
+  - modules/local/sra_runinfo_to_ftp/*
+  - modules/local/sra_fastq_ftp/*
+  - modules/local/sra_to_samplesheet/*
+  - modules/local/sra_merge_samplesheet/*
+  - workflows/sra/*
   - workflows/sra/tests/sra_default_parameters.nf.test
 sra_force_sratools_download:
-  - conf/modules/sra.config
-  - modules/local/sra_ids_to_runinfo/main.nf
-  - modules/local/sra_runinfo_to_ftp/main.nf
-  - modules/local/sra_fastq_ftp/main.nf
-  - modules/local/sra_to_samplesheet/main.nf
-  - modules/local/sra_merge_samplesheet/main.nf
-  - workflows/sra/main.nf
+  - modules/local/sra_ids_to_runinfo/*
+  - modules/local/sra_runinfo_to_ftp/*
+  - modules/local/sra_fastq_ftp/*
+  - modules/local/sra_to_samplesheet/*
+  - modules/local/sra_merge_samplesheet/*
+  - workflows/sra/*
   - workflows/sra/tests/sra_force_sratools_download.nf.test
 sra_nf_core_pipeline_atacseq:
-  - conf/modules/sra.config
-  - modules/local/sra_ids_to_runinfo/main.nf
-  - modules/local/sra_runinfo_to_ftp/main.nf
-  - modules/local/sra_fastq_ftp/main.nf
-  - modules/local/sra_to_samplesheet/main.nf
-  - modules/local/sra_merge_samplesheet/main.nf
-  - workflows/sra/main.nf
+  - modules/local/sra_ids_to_runinfo/*
+  - modules/local/sra_runinfo_to_ftp/*
+  - modules/local/sra_fastq_ftp/*
+  - modules/local/sra_to_samplesheet/*
+  - modules/local/sra_merge_samplesheet/*
+  - workflows/sra/*
   - workflows/sra/tests/sra_nf_core_pipeline_atacseq.nf.test
 sra_nf_core_pipeline_rnaseq:
-  - conf/modules/sra.config
-  - modules/local/sra_ids_to_runinfo/main.nf
-  - modules/local/sra_runinfo_to_ftp/main.nf
-  - modules/local/sra_fastq_ftp/main.nf
-  - modules/local/sra_to_samplesheet/main.nf
-  - modules/local/sra_merge_samplesheet/main.nf
-  - workflows/sra/main.nf
+  - modules/local/sra_ids_to_runinfo/*
+  - modules/local/sra_runinfo_to_ftp/*
+  - modules/local/sra_fastq_ftp/*
+  - modules/local/sra_to_samplesheet/*
+  - modules/local/sra_merge_samplesheet/*
+  - workflows/sra/*
   - workflows/sra/tests/sra_nf_core_pipeline_rnaseq.nf.test
 sra_nf_core_pipeline_taxprofiler:
-  - conf/modules/sra.config
-  - modules/local/sra_ids_to_runinfo/main.nf
-  - modules/local/sra_runinfo_to_ftp/main.nf
-  - modules/local/sra_fastq_ftp/main.nf
-  - modules/local/sra_to_samplesheet/main.nf
-  - modules/local/sra_merge_samplesheet/main.nf
-  - workflows/sra/main.nf
+  - modules/local/sra_ids_to_runinfo/*
+  - modules/local/sra_runinfo_to_ftp/*
+  - modules/local/sra_fastq_ftp/*
+  - modules/local/sra_to_samplesheet/*
+  - modules/local/sra_merge_samplesheet/*
+  - workflows/sra/*
   - workflows/sra/tests/sra_nf_core_pipeline_taxprofiler.nf.test
 sra_nf_core_pipeline_viralrecon:
-  - conf/modules/sra.config
-  - modules/local/sra_ids_to_runinfo/main.nf
-  - modules/local/sra_runinfo_to_ftp/main.nf
-  - modules/local/sra_fastq_ftp/main.nf
-  - modules/local/sra_to_samplesheet/main.nf
-  - modules/local/sra_merge_samplesheet/main.nf
-  - workflows/sra/main.nf
+  - modules/local/sra_ids_to_runinfo/*
+  - modules/local/sra_runinfo_to_ftp/*
+  - modules/local/sra_fastq_ftp/*
+  - modules/local/sra_to_samplesheet/*
+  - modules/local/sra_merge_samplesheet/*
+  - workflows/sra/*
   - workflows/sra/tests/sra_nf_core_pipeline_viralrecon.nf.test
 sra_skip_fastq_download:
-  - conf/modules/sra.config
-  - modules/local/sra_ids_to_runinfo/main.nf
-  - modules/local/sra_runinfo_to_ftp/main.nf
-  - modules/local/sra_fastq_ftp/main.nf
-  - modules/local/sra_to_samplesheet/main.nf
-  - modules/local/sra_merge_samplesheet/main.nf
-  - workflows/sra/main.nf
+  - modules/local/sra_ids_to_runinfo/*
+  - modules/local/sra_runinfo_to_ftp/*
+  - modules/local/sra_fastq_ftp/*
+  - modules/local/sra_to_samplesheet/*
+  - modules/local/sra_merge_samplesheet/*
+  - workflows/sra/*
   - workflows/sra/tests/sra_skip_fastq_download.nf.test

.github/workflows/cloud_tests_full.yml

@@ -27,7 +27,7 @@ jobs:
           compute_env: ${{ secrets.TOWER_CE_AWS_CPU }}
           workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/fetchngs/work-${{ github.sha }}"
           run_name: "aws_fetchngs_full"
-          profiles: test_full,public_aws_ecr
+          profiles: test_full
           parameters: |
             {
                 "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",

.github/workflows/cloud_tests_small.yml

@@ -25,7 +25,7 @@ jobs:
           compute_env: ${{ secrets.TOWER_CE_AWS_CPU }}
           workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/fetchngs/work-${{ github.sha }}"
           run_name: "aws_fetchngs_small"
-          profiles: test,public_aws_ecr
+          profiles: test
           parameters: |
             {
                 "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/fetchngs/results-test-${{ github.sha }}"

CHANGELOG.md

@@ -18,7 +18,8 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements
 ### Enhancements & fixes
 
 - [#166](https://github.com/nf-core/fetchngs/issues/175) - Add NF-TEST pipeline end-to-end tests for existing CI tests
-- [#184](https://github.com/nf-core/fetchngs/pull/184/files) Emits all files in workflow for testing and enabling workflow to be imported (not supported by Nextflow)
+- [#184](https://github.com/nf-core/fetchngs/pull/184) Emits all files in workflow for testing and enabling workflow to be imported (not supported by Nextflow)
+- [#195](https://github.com/nf-core/fetchngs/pull/195) Separate configuration files for all modules
 
 ## [[1.10.0](https://github.com/nf-core/fetchngs/releases/tag/1.10.0)] - 2023-05-16
 

conf/base.config

@@ -14,6 +14,12 @@ process {
     memory = { check_max( 6.GB * task.attempt, 'memory' ) }
     time   = { check_max( 4.h  * task.attempt, 'time'   ) }
 
+    publishDir = [
+        path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" },
+        mode: params.publish_dir_mode,
+        saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+    ]
+
     errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
     maxRetries    = 1
     maxErrors     = '-1'
@@ -57,7 +63,4 @@ process {
         errorStrategy = 'retry'
         maxRetries    = 2
     }
-    withName:CUSTOM_DUMPSOFTWAREVERSIONS {
-        cache = false
-    }
 }

conf/modules/base.config

@@ -22,6 +22,7 @@ process {
     ]
 
     withName: CUSTOM_DUMPSOFTWAREVERSIONS {
+        cache = false
         publishDir = [
             path: { "${params.outdir}/pipeline_info" },
             mode: params.publish_dir_mode,

modules.json

@@ -7,7 +7,7 @@
                 "nf-core": {
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
-                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
+                        "git_sha": "05c280924b6c768d484c7c443dad5e605c4ff4b4",
                         "installed_by": ["modules"]
                     },
                     "custom/sratoolsncbisettings": {

modules/local/multiqc_mappings_config/nextflow.config

@@ -0,0 +1,9 @@
+process {
+    withName: 'MULTIQC_MAPPINGS_CONFIG' {
+        publishDir = [
+            path: { "${params.outdir}/samplesheet" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ]
+    }
+}

modules/local/sra_fastq_ftp/nextflow.config

@@ -0,0 +1,17 @@
+process {
+    withName: 'SRA_FASTQ_FTP' {
+        ext.args = '--retry 5 --continue-at - --max-time 1200'
+        publishDir = [
+            [
+                path: { "${params.outdir}/fastq" },
+                mode: params.publish_dir_mode,
+                pattern: "*.fastq.gz"
+            ],
+            [
+                path: { "${params.outdir}/fastq/md5" },
+                mode: params.publish_dir_mode,
+                pattern: "*.md5"
+            ]
+        ]
+    }
+}

modules/local/sra_ids_to_runinfo/nextflow.config

@@ -0,0 +1,8 @@
+process {
+    withName: 'SRA_IDS_TO_RUNINFO' {
+        publishDir = [
+            path: { "${params.outdir}/metadata" },
+            enabled: false
+        ]
+    }
+}

modules/local/sra_merge_samplesheet/nextflow.config

@@ -0,0 +1,9 @@
+process {
+    withName: 'SRA_MERGE_SAMPLESHEET' {
+        publishDir = [
+            path: { "${params.outdir}/samplesheet" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ]
+    }
+}