Merge pull request #1 from ICGC-TCGA-PanCancer/my-wf-1@0.1.0

ready to [release]

my-wf-1/.gitignore

@@ -0,0 +1,69 @@
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+.eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+venv*/
+pyvenv*/
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+.coverage.*
+nosetests.xml
+coverage.xml
+htmlcov
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+.idea
+*.iml
+*.komodoproject
+
+# Complexity
+output/*.html
+output/*/index.html
+
+# Sphinx
+docs/_build
+
+.DS_Store
+*~
+.*.sw[po]
+.build
+.ve
+.env
+.cache
+.pytest
+.bootstrap
+.appveyor.token
+*.bak
+*.log
+.vscode
+.python-version
+.nextflow*
+work
+outdir

my-wf-1/my-wf-1.nf

@@ -0,0 +1,31 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+version = '0.1.0'  // tool version
+
+// universal params go here, change default value as needed
+params.container_version = ""
+params.cpus = 1
+params.mem = 1  // GB
+params.publish_dir = ""  // set to empty string will disable publishDir
+
+// tool specific parmas go here, add / change as needed
+params.input_file = ""
+params.output_pattern = "*.html"  // fastqc output html report
+
+include { fastqc } from "./wfpr_modules/github.com/icgc-tcga-pancancer/awesome-wfpkgs1/fastqc@0.1.0/fastqc"
+include { cleanupWorkdir as cleanup } from "./wfpr_modules/github.com/icgc-argo/demo-wfpkgs/demo-utils@1.1.0/main"
+
+
+workflow MyWf1 {
+  take:  // input, make update as needed
+    input_file
+
+  main:
+    fastqc(input_file)
+
+    cleanup(fastqc.out, true)
+
+  emit:
+    output_file = fastqc.out.output
+}

my-wf-1/nextflow.config

@@ -0,0 +1,4 @@
+docker {
+    enabled = true
+    runOptions = '-u \$(id -u):\$(id -g)'
+}

my-wf-1/pkg.json

@@ -0,0 +1,33 @@
+{
+    "name": "my-wf-1",
+    "version": "0.1.0",
+    "description": "FastQC workflow",
+    "main": "my-wf-1",
+    "scripts": {
+        "test": "wfpm test"
+    },
+    "deprecated": false,
+    "keywords": [
+        "bioinformatics",
+        "seq",
+        "qc metrics"
+    ],
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/icgc-tcga-pancancer/demo-wfs.git"
+    },
+    "dependencies": [
+        "github.com/icgc-tcga-pancancer/awesome-wfpkgs1/fastqc@0.1.0",
+        "github.com/icgc-argo/demo-wfpkgs/demo-utils@1.1.0"
+    ],
+    "devDependencies": [],
+    "contributors": [
+        {
+            "name": "Junjun Zhang",
+            "email": "junjun.zhang@oicr.on.ca"
+        }
+    ],
+    "license": "MIT",
+    "bugReport": "https://github.com/icgc-tcga-pancancer/demo-wfs/issues",
+    "homepage": "https://github.com/icgc-tcga-pancancer/demo-wfs#readme"
+}

my-wf-1/tests/checker.nf

@@ -0,0 +1,67 @@
+#!/usr/bin/env nextflow
+
+/*
+ This is an auto-generated checker workflow, please update as needed
+*/
+
+nextflow.enable.dsl = 2
+version = '0.1.0'  // tool version
+
+// universal params
+params.publish_dir = ""
+params.container_version = ""
+
+// tool specific parmas go here, add / change as needed
+params.input_file = ""
+params.expected_output = ""
+
+include { MyWf1 } from '../my-wf-1'
+
+Channel
+  .fromPath(params.input_file, checkIfExists: true)
+  .set { input_file }
+
+
+process file_diff {
+  input:
+    path output_file
+    path expected_gzip
+
+  output:
+    stdout()
+
+  script:
+    """
+    # remove date field before comparison eg, <div id="header_filename">Tue 19 Jan 2021<br/>test_rg_3.bam</div>
+    # sed -e 's#"header_filename">.*<br/>test_rg_3.bam#"header_filename"><br/>test_rg_3.bam</div>#'
+
+    diff <( cat ${output_file} | sed -e 's#"header_filename">.*<br/>test_rg_3.bam#"header_filename"><br/>test_rg_3.bam</div>#' ) \
+         <( gunzip -c ${expected_gzip} | sed -e 's#"header_filename">.*<br/>test_rg_3.bam#"header_filename"><br/>test_rg_3.bam</div>#' ) \
+    && ( echo "Test PASSED" && exit 0 ) || ( echo "Test FAILED, output file mismatch." && exit 1 )
+    """
+}
+
+
+workflow checker {
+  take:
+    input_file
+    expected_output
+
+  main:
+    MyWf1(
+      input_file
+    )
+
+    file_diff(
+      MyWf1.out.output_file,
+      expected_output
+    )
+}
+
+
+workflow {
+  checker(
+    file(params.input_file),
+    file(params.expected_output)
+  )
+}

my-wf-1/tests/expected/expected.input_file_name.txt

@@ -0,0 +1 @@
+The input file name is README.md

my-wf-1/tests/input/README.md

@@ -0,0 +1 @@
+This folder contains tiny data files for testing.

my-wf-1/tests/nextflow.config

@@ -0,0 +1 @@
+includeConfig '../nextflow.config'

my-wf-1/tests/test-job-1.json

@@ -0,0 +1,7 @@
+{
+    "input_file": "input/test_rg_3.bam",
+    "expected_output": "expected/expected.test_rg_3_fastqc.out.gz",
+    "publish_dir": "outdir",
+    "cpus": 1,
+    "mem": 0.5
+}

my-wf-1/tests/wfpr_modules

@@ -0,0 +1 @@
+../wfpr_modules

my-wf-1/wfpr_modules

@@ -0,0 +1 @@
+../wfpr_modules

wfpr_modules/github.com/icgc-argo/demo-wfpkgs/demo-utils@1.1.0/.dockerignore

@@ -0,0 +1,5 @@
+.nextflow*
+.gitignore
+work
+outdir
+tests

wfpr_modules/github.com/icgc-argo/demo-wfpkgs/demo-utils@1.1.0/Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu:20.04
+
+LABEL org.opencontainers.image.source https://github.com/icgc-argo/demo-wfpkgs
+
+RUN groupadd -g 1000 ubuntu && \
+    useradd -l -u 1000 -g ubuntu ubuntu && \
+    install -d -m 0755 -o ubuntu -g ubuntu /home/ubuntu
+
+USER ubuntu
+
+CMD ["/bin/bash"]

wfpr_modules/github.com/icgc-argo/demo-wfpkgs/demo-utils@1.1.0/local_modules/cleanup-workdir.nf

@@ -0,0 +1,75 @@
+#!/usr/bin/env nextflow
+
+/*
+ * Copyright (c) 2019-2021, Ontario Institute for Cancer Research (OICR).
+ *                                                                                                               
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Contributors:
+ *   Junjun Zhang <junjun.zhang@oicr.on.ca>
+ */
+
+/********************************************************************/
+/* this block is auto-generated based on info from pkg.json where   */
+/* changes can be made if needed, do NOT modify this block manually */
+nextflow.enable.dsl = 2
+version = '1.1.0'
+container = [
+    'ghcr.io': 'ghcr.io/icgc-argo/demo-wfpkgs.demo-utils',
+    'quay.io': 'quay.io/icgc-argo/demo-wfpkgs.demo-utils'
+]
+default_container_registry = 'quay.io'
+/********************************************************************/
+
+params.cpus = 1
+params.mem = 1
+params.files_to_delete = 'NO_FILE'
+params.container_version = ''
+params.container_registry = default_container_registry
+
+
+process cleanupWorkdir {
+    container "${container[params.container_registry]}:${params.container_version ?: version}"
+    cpus params.cpus
+    memory "${params.mem} GB"
+
+    input:
+        path files_to_delete  // more accurately, other non-hidden files in the same folder will be deleted as well
+        val virtual_dep_flag  // for specifying steps do not produce output files but produce values, set those values here
+
+    output:
+        stdout
+
+    script:
+        """
+        set -euxo pipefail
+
+        IFS=" "
+        read -a files <<< "${files_to_delete}"
+        for f in "\${files[@]}"
+        do
+            dir_to_rm=\$(dirname \$(readlink -f \$f))
+
+            if [[ \$dir_to_rm != ${workflow.workDir}/* ]]; then  # skip dir not under workdir, like from input file dir
+                echo "Not delete: \$dir_to_rm/*\"
+                continue
+            fi
+
+            rm -fr \$dir_to_rm/*  # delete all files and subdirs but not hidden ones
+            echo "Deleted: \$dir_to_rm/*"
+        done
+        """
+}

wfpr_modules/github.com/icgc-argo/demo-wfpkgs/demo-utils@1.1.0/main.nf

@@ -0,0 +1,66 @@
+#!/usr/bin/env nextflow
+
+/*
+ * Copyright (c) 2019-2021, Ontario Institute for Cancer Research (OICR).
+ *                                                                                                               
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Contributors:
+ *   Junjun Zhang <junjun.zhang@oicr.on.ca>
+ */
+
+/********************************************************************/
+/* this block is auto-generated based on info from pkg.json where   */
+/* changes can be made if needed, do NOT modify this block manually */
+nextflow.enable.dsl = 2
+name = 'demo-utils'
+version = '1.1.0'
+/********************************************************************/
+
+
+// load local process (module)
+include { cleanupWorkdir } from './local_modules/cleanup-workdir.nf'
+
+
+// this is kind of like CWL's secondary files
+def getSecondaryFiles(main_file, exts){
+  if (!(main_file instanceof String)) {
+    exit 1, "[getSecondaryFiles] param: main_file must be a string"
+  }
+
+  if (!(exts instanceof List)) {
+    exit 1, "[getSecondaryFiles] param: exts must be a list of strings"
+  }
+
+  def secondaryFiles = []
+  for (ext in exts) {
+    if (ext.startsWith("^")) {
+      ext = ext.replace("^", "")
+      parts = main_file.split("\\.").toList()
+      parts.removeLast()
+      secondaryFiles.add((parts + [ext]).join("."))
+    } else {
+      secondaryFiles.add(main_file + '.' + ext)
+    }
+  }
+  return secondaryFiles
+}
+
+
+// get specific secondary files for BWA alignment, ensure none is missing
+def getBwaSecondaryFiles(main_file){
+  return getSecondaryFiles(main_file, ['fai', 'sa', 'bwt', 'ann', 'amb', 'pac', 'alt'])
+}

wfpr_modules/github.com/icgc-argo/demo-wfpkgs/demo-utils@1.1.0/nextflow.config

@@ -0,0 +1,4 @@
+docker {
+    enabled = true
+    runOptions = '-u \$(id -u):\$(id -g)'
+}