Merge pull request #30 from aertslab/publish_directory_updates

Updates to outdir publishing Former-commit-id: 373c97a
vib-singlecell-nf · Nov 7, 2019 · 8cf1d5d · 8cf1d5d
2 parents c13bdfd + 5419c45
commit 8cf1d5d
Show file tree

Hide file tree

Showing 24 changed files with 79 additions and 36 deletions.
diff --git a/src/cellranger/processes/count.nf b/src/cellranger/processes/count.nf
@@ -2,7 +2,7 @@ nextflow.preview.dsl=2
 
 process SC__CELLRANGER__COUNT {
 
-  publishDir "${params.global.outdir}/counts", mode: 'symlink'
+  publishDir "${params.global.outdir}/counts", mode: 'link', overwrite: true
   container params.sc.cellranger.container
 
   input:
@@ -36,3 +36,4 @@ process SC__CELLRANGER__COUNT {
         ${(params.sc.cellranger.count.containsKey('indicies')) ? '--indicies ' + params.sc.cellranger.count.indicies: ''} 
     """
 }
+
diff --git a/src/cellranger/processes/mkfastq.nf b/src/cellranger/processes/mkfastq.nf
@@ -2,7 +2,7 @@ nextflow.preview.dsl=2
 
 process SC__CELLRANGER__MKFASTQ {
 
-  publishDir "${params.global.outdir}/fastqs", saveAs: { filename -> dirname = filename =~ /(.*)_fastqOut/; "${dirname[0][1]}" }, mode: 'symlink'
+  publishDir "${params.global.outdir}/fastqs", saveAs: { filename -> dirname = filename =~ /(.*)_fastqOut/; "${dirname[0][1]}" }, mode: 'link', overwrite: true
   container params.sc.cellranger.container
 
   input:
@@ -35,3 +35,4 @@ process SC__CELLRANGER__MKFASTQ {
     done
     """
 }
+
diff --git a/src/scanpy/processes/adjust.nf b/src/scanpy/processes/adjust.nf
@@ -8,7 +8,7 @@ normalizationVariablesToRegressOutAsArguments = params.normalizationVariablesToR
 process SC__SCANPY__ADJUSTMENT {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -23,3 +23,4 @@ process SC__SCANPY__ADJUSTMENT {
          "${getBaseName(f)}.SC__SCANPY__ADJUSTMENT.${params.off}" 
     """
 }
+
diff --git a/src/scanpy/processes/batch_effect_correct.nf b/src/scanpy/processes/batch_effect_correct.nf
@@ -5,7 +5,7 @@ include getBaseName from '../../utils/processes/files.nf'
 process SC__SCANPY__BATCH_EFFECT_CORRECTION {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -28,3 +28,4 @@ process SC__SCANPY__BATCH_EFFECT_CORRECTION {
         $f
     """
 }
+
diff --git a/src/scanpy/processes/cluster.nf b/src/scanpy/processes/cluster.nf
@@ -5,7 +5,7 @@ include getBaseName from '../../utils/processes/files.nf'
 process SC__SCANPY__CLUSTERING {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -20,3 +20,4 @@ process SC__SCANPY__CLUSTERING {
          "${getBaseName(f)}.SC__SCANPY__CLUSTERING.${params.off}"
     """
 }
+
diff --git a/src/scanpy/processes/dim_reduction.nf b/src/scanpy/processes/dim_reduction.nf
@@ -5,7 +5,7 @@ include getBaseName from '../../utils/processes/files.nf'
 process SC__SCANPY__DIM_REDUCTION {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -26,3 +26,4 @@ process SC__SCANPY__DIM_REDUCTION {
          "${getBaseName(f)}.SC__SCANPY__DIM_REDUCTION_${method}.${params.off}"
     """
 }
+
diff --git a/src/scanpy/processes/feature_selection.nf b/src/scanpy/processes/feature_selection.nf
@@ -5,7 +5,7 @@ include getBaseName from '../../utils/processes/files.nf'
 process SC__SCANPY__FEATURE_SELECTION {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -23,3 +23,4 @@ process SC__SCANPY__FEATURE_SELECTION {
         "${getBaseName(f)}.SC__SCANPY__FEATURE_SELECTION.${params.off}"
     """
 }
+
diff --git a/src/scanpy/processes/filter.nf b/src/scanpy/processes/filter.nf
@@ -30,7 +30,7 @@ process SC__SCANPY__COMPUTE_QC_STATS {
 process SC__SCANPY__GENE_FILTER {
 
     container params.sc.scanpy.container
-    publishDir "${params.outdir}/data", mode: 'symlink'
+    publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
     input:
         file(f)
@@ -50,7 +50,7 @@ process SC__SCANPY__GENE_FILTER {
 process SC__SCANPY__CELL_FILTER {
 
     container params.sc.scanpy.container
-    publishDir "${params.outdir}/data", mode: 'symlink'
+    publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
     input:
         file(f)

diff --git a/src/scanpy/processes/marker_genes.nf b/src/scanpy/processes/marker_genes.nf
@@ -5,7 +5,7 @@ include getBaseName from '../../utils/processes/files.nf'
 process SC__SCANPY__MARKER_GENES {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -21,3 +21,4 @@ process SC__SCANPY__MARKER_GENES {
          "${getBaseName(f)}.SC__SCANPY__MARKER_GENES.${params.off}"
     """
 }
+
diff --git a/src/scanpy/processes/reports.nf b/src/scanpy/processes/reports.nf
@@ -9,7 +9,7 @@ outputs ipynb named by the value in ${report_title}
 process SC__SCANPY__GENERATE_REPORT {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/notebooks", mode: 'symlink'
+  publishDir "${params.outdir}/notebooks/intermediate", mode: 'link', overwrite: true
 
   input:
     file ipynb
@@ -30,7 +30,7 @@ process SC__SCANPY__GENERATE_REPORT {
 process SC__SCANPY__FILTER_QC_REPORT {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/notebooks", mode: 'symlink'
+  publishDir "${params.outdir}/notebooks/intermediate", mode: 'link', overwrite: true
 
   input:
     file(ipynb)
@@ -51,7 +51,9 @@ process SC__SCANPY__FILTER_QC_REPORT {
 process SC__SCANPY__REPORT_TO_HTML {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/notebooks", mode: 'symlink'
+  publishDir "${params.outdir}/notebooks/intermediate", mode: 'link', overwrite: true
+  // copy final "merged_report" to notbooks root:
+  publishDir "${params.outdir}/notebooks", pattern: '*merged_report*', mode: 'link', overwrite: true
 
   input:
     file ipynb
@@ -67,7 +69,9 @@ process SC__SCANPY__REPORT_TO_HTML {
 process SC__SCANPY__MERGE_REPORTS {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/notebooks", mode: 'symlink'
+  publishDir "${params.outdir}/notebooks/intermediate", mode: 'link', overwrite: true
+  // copy final "merged_report" to notbooks root:
+  publishDir "${params.outdir}/notebooks", pattern: '*merged_report*', mode: 'link', overwrite: true
 
   input:
     file(ipynbs)

diff --git a/src/scanpy/processes/transform.nf b/src/scanpy/processes/transform.nf
@@ -5,7 +5,7 @@ include getBaseName from '../../utils/processes/files.nf'
 process SC__SCANPY__NORMALIZATION {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -24,7 +24,7 @@ process SC__SCANPY__NORMALIZATION {
 process SC__SCANPY__DATA_TRANSFORMATION {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -42,7 +42,7 @@ process SC__SCANPY__DATA_TRANSFORMATION {
 process SC__SCANPY__FEATURE_SCALING {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -57,3 +57,4 @@ process SC__SCANPY__FEATURE_SCALING {
        "${getBaseName(f)}.SC__SCANPY__FEATURE_SCALING.${params.off}"
     """
 }
+
diff --git a/src/scanpy/workflows/bec_bbknn.nf b/src/scanpy/workflows/bec_bbknn.nf
@@ -30,6 +30,7 @@ include SC__SCANPY__CLUSTERING from '../processes/cluster.nf' params(params.sc.s
 include SC__SCANPY__DIM_REDUCTION as SC__SCANPY__DIM_REDUCTION__UMAP from '../processes/dim_reduction.nf' params(params.sc.scanpy.dim_reduction.umap + params.global + params)
 include SC__H5AD_TO_LOOM from '../../utils/processes/h5ad_to_loom.nf' params(params.global + params)
 include CLUSTER_IDENTIFICATION from './cluster_identification.nf' params(params + params.global)
+include SC__PUBLISH_H5AD from '../src/utils/processes/utils.nf' params(params + params.global)
 
 //////////////////////////////////////////////////////
 //  Define the workflow 
@@ -41,6 +42,8 @@ workflow BEC_BBKNN {
         SC__SCANPY__BATCH_EFFECT_CORRECTION( data )
         CLUSTER_IDENTIFICATION( SC__SCANPY__BATCH_EFFECT_CORRECTION.out )
         SC__SCANPY__DIM_REDUCTION__UMAP( CLUSTER_IDENTIFICATION.out.marker_genes )
+        SC__PUBLISH_H5AD( SC__SCANPY__DIM_REDUCTION__UMAP.out,
+            params.global.project_name+".BEC_BBKNN.output")
         SC__H5AD_TO_LOOM(SC__SCANPY__DIM_REDUCTION__UMAP.out )
         // Not using t-SNE as it does not use the neighbour graph (which BBKNN alters) when constructing its dimensionality reduction
     emit:

diff --git a/src/scenic/processes/aggregateMultiRunsFeatures.nf b/src/scenic/processes/aggregateMultiRunsFeatures.nf
@@ -9,7 +9,7 @@ if(!params.containsKey("test")) {
 process SC__SCENIC__AGGR_MULTI_RUNS_FEATURES {
     cache 'deep'
     container params.sc.scenic.container
-    publishDir "${params.sc.scenic.scenicoutdir}/multi_runs_cistarget/", mode: 'copy'
+    publishDir "${params.sc.scenic.scenicoutdir}/multi_runs_cistarget/", mode: 'link', overwrite: true
     // This process requires a large amount of memory especially for big datasets
     // This process is quite slow (could take more than 1h for big datasets, so keep 24h for now)
     clusterOptions "-l nodes=1:ppn=${params.sc.scenic.numWorkers} -l pmem=6gb -l walltime=24:00:00 -A ${params.global.qsubaccount}"

diff --git a/src/scenic/processes/aggregateMultiRunsRegulons.nf b/src/scenic/processes/aggregateMultiRunsRegulons.nf
@@ -9,7 +9,7 @@ if(!params.containsKey("test")) {
 process SC__SCENIC__AGGR_MULTI_RUNS_REGULONS {
     cache 'deep'
     container params.sc.scenic.container
-    publishDir "${params.sc.scenic.scenicoutdir}", mode: 'copy'
+    publishDir "${params.sc.scenic.scenicoutdir}", mode: 'link', overwrite: true
     clusterOptions "-l nodes=1:ppn=${params.sc.scenic.numWorkers} -l pmem=2gb -l walltime=24:00:00 -A ${params.global.qsubaccount}"
 
     input:

diff --git a/src/scenic/processes/aucell.nf b/src/scenic/processes/aucell.nf
@@ -5,7 +5,7 @@ nextflow.preview.dsl=2
 process SC__SCENIC__AUCELL {
     cache 'deep'
     container params.sc.scenic.container
-    publishDir "${params.sc.scenic.scenicoutdir}/aucell/${params.sc.scenic.numRuns > 1 ? "run_" + runId : ""}", mode: 'symlink'
+    publishDir "${params.sc.scenic.scenicoutdir}/aucell/${params.sc.scenic.numRuns > 1 ? "run_" + runId : ""}", mode: 'link', overwrite: true
     clusterOptions "-l nodes=1:ppn=${params.sc.scenic.numWorkers} -l pmem=2gb -l walltime=24:00:00 -A ${params.global.qsubaccount}"
     maxForks params.sc.scenic.maxForks
 
@@ -29,3 +29,4 @@ process SC__SCENIC__AUCELL {
         --num_workers ${params.sc.scenic.numWorkers}
     """
 }
+
diff --git a/src/scenic/processes/aucellGeneSigsFromFolder.nf b/src/scenic/processes/aucellGeneSigsFromFolder.nf
@@ -11,7 +11,7 @@ if(!params.containsKey("test")) {
 process SC__SCENIC__AUCELL_GENESIGS_FROM_FOLDER {
     cache 'deep'
     container params.sc.scenic.container
-    publishDir "${params.sc.scenic.scenicoutdir}/multi_runs_aucell/", mode: 'copy'
+    publishDir "${params.sc.scenic.scenicoutdir}/multi_runs_aucell/", mode: 'link', overwrite: true
     clusterOptions "-l nodes=1:ppn=${params.sc.scenic.numWorkers} -l pmem=2gb -l walltime=24:00:00 -A ${params.global.qsubaccount}"
 
     input:
@@ -37,3 +37,4 @@ process SC__SCENIC__AUCELL_GENESIGS_FROM_FOLDER {
         --gene-attribute ${params.sc.scenic.gene_attribute}
     """
 }
+
diff --git a/src/scenic/processes/cistarget.nf b/src/scenic/processes/cistarget.nf
@@ -5,7 +5,7 @@ nextflow.preview.dsl=2
 process SC__SCENIC__CISTARGET {
     cache 'deep'
     container params.sc.scenic.container
-    publishDir "${params.sc.scenic.scenicoutdir}/cistarget/${params.sc.scenic.numRuns > 1 ? "run_" + runId : ""}", mode: 'symlink'
+    publishDir "${params.sc.scenic.scenicoutdir}/cistarget/${params.sc.scenic.numRuns > 1 ? "run_" + runId : ""}", mode: 'link', overwrite: true
     clusterOptions "-l nodes=1:ppn=${params.sc.scenic.numWorkers} -l pmem=2gb -l walltime=24:00:00 -A ${params.global.qsubaccount}"
     maxForks params.sc.scenic.maxForks
 
@@ -54,3 +54,4 @@ process SC__SCENIC__CISTARGET {
         --min_genes MIN_GENES
         --expression_mtx_fname EXPRESSION_MTX_FNAME
 */
+
diff --git a/src/scenic/processes/grnboost2withoutDask.nf b/src/scenic/processes/grnboost2withoutDask.nf
@@ -11,7 +11,7 @@ if(!params.containsKey("test")) {
 process SC__SCENIC__GRNBOOST2_WITHOUT_DASK {
     cache 'deep'
     container params.sc.scenic.container
-    publishDir "${params.sc.scenic.scenicoutdir}/grnboost2withoutDask/${params.sc.scenic.numRuns > 1 ? "run_" + runId : ""}", mode: 'symlink'
+    publishDir "${params.sc.scenic.scenicoutdir}/grnboost2withoutDask/${params.sc.scenic.numRuns > 1 ? "run_" + runId : ""}", mode: 'link', overwrite: true
     clusterOptions "-l nodes=1:ppn=${params.sc.scenic.numWorkers} -l pmem=${params.sc.scenic.grn.pmem} -l walltime=24:00:00 -A ${params.global.qsubaccount}"
     maxForks params.sc.scenic.maxForks
 

diff --git a/src/scenic/processes/saveScenicMultiRunsToLoom.nf b/src/scenic/processes/saveScenicMultiRunsToLoom.nf
@@ -11,7 +11,7 @@ if(!params.containsKey("test")) {
 process SC__SCENIC__SAVE_SCENIC_MULTI_RUNS_TO_LOOM {
     cache 'deep'
     container params.sc.scenic.container
-    publishDir "${params.sc.scenic.scenicoutdir}/multi_runs_looms/", mode: 'copy'
+    publishDir "${params.sc.scenic.scenicoutdir}/multi_runs_looms/", mode: 'link', overwrite: true
     // This process requires a large amount of memory especially for big datasets (force to use bigmem node)
     // This process is quite slow (could take more than 1h for big datasets, so keep 24h for now)
     clusterOptions "-l nodes=1:ppn=${params.sc.scenic.numWorkers} -l pmem=6gb -l walltime=24:00:00 -A ${params.global.qsubaccount}"
@@ -45,3 +45,4 @@ process SC__SCENIC__SAVE_SCENIC_MULTI_RUNS_TO_LOOM {
         --scope-tree-level-3 "${params.sc.scope.tree.level_3}"
     """
 }
+
diff --git a/src/scenic/processes/scenicLoomHandler.nf b/src/scenic/processes/scenicLoomHandler.nf
@@ -44,7 +44,7 @@ process SC__SCENIC__VISUALIZE {
 process SC__SCENIC__MERGE_MOTIF_TRACK_LOOMS {
     cache 'deep'
     container params.sc.scenic.container
-    publishDir "${params.sc.scenic.scenicoutdir}", mode: 'copy'
+    publishDir "${params.sc.scenic.scenicoutdir}", mode: 'link', overwrite: true
     clusterOptions "-l nodes=1:ppn=${params.sc.scenic.numWorkers} -l pmem=2gb -l walltime=24:00:00 -A ${params.global.qsubaccount}"
 
     input:
@@ -68,7 +68,7 @@ process SC__SCENIC__MERGE_MOTIF_TRACK_LOOMS {
 process SC__SCENIC__APPEND_SCENIC_LOOM {
     cache 'deep'
     container params.sc.scenic.container
-    publishDir "${params.sc.scenic.scenicoutdir}", mode: 'copy'
+    publishDir "${params.sc.scenic.scenicoutdir}", mode: 'link', overwrite: true
 
     input:
     file scopeloom

diff --git a/src/utils/processes/h5ad_to_loom.nf b/src/utils/processes/h5ad_to_loom.nf
@@ -5,7 +5,7 @@ include getBaseName from './files.nf'
 process SC__H5AD_TO_LOOM {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/loom", mode: 'symlink'
+  publishDir "${params.outdir}/loom", mode: 'link', overwrite: true
 
   input:
     file(f)
@@ -22,7 +22,7 @@ process SC__H5AD_TO_LOOM {
 process SC__H5AD_TO_FILTERED_LOOM {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/loom", mode: 'symlink'
+  publishDir "${params.outdir}/loom", mode: 'link', overwrite: true
 
   input:
     file(f)

diff --git a/src/utils/processes/utils.nf b/src/utils/processes/utils.nf
@@ -12,7 +12,7 @@ process SC__FILE_CONVERTER {
 
   cache 'deep'
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     set id, file(f)
@@ -64,7 +64,7 @@ process SC__FILE_CONVERTER_HELP {
 process SC__FILE_CONCATENATOR() {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -82,7 +82,7 @@ process SC__FILE_CONCATENATOR() {
 process SC__STAR_CONCATENATOR() {
 
   container "aertslab/sctx-scanpy:0.5.0"
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -102,7 +102,7 @@ include getBaseName from './files.nf'
 process SC__FILE_ANNOTATOR() {
 
   container params.sc.scanpy.container
-  publishDir "${params.outdir}/data", mode: 'symlink'
+  publishDir "${params.outdir}/data/intermediate", mode: 'symlink', overwrite: true
 
   input:
     file(f)
@@ -118,3 +118,19 @@ process SC__FILE_ANNOTATOR() {
       "${getBaseName(f)}.SC__FILE_ANNOTATOR.${params.off}"
     """
 }
+
+process SC__PUBLISH_H5AD {
+
+    publishDir "${params.outdir}/data", mode: 'link', overwrite: true
+
+    input:
+        file f_in
+        val f_out
+    output:
+        file "${f_out}.h5ad"
+    script:
+    """
+    ln -s ${f_in} ${f_out}.h5ad
+    """
+}
+