Merge pull request #18 from icgc-argo-workflows/sanger-wxs-variant-ca…

…lling@3.1.6-3.7.0 [release]
icgc-argo-workflows · Jun 29, 2023 · 657a804 · 657a804
2 parents 7b77703 + 3805c67
commit 657a804
Show file tree

Hide file tree

Showing 25 changed files with 179 additions and 85 deletions.
diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml
@@ -19,10 +19,13 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
       with:
-        python-version: 3.6
+        python-version: "3.10"
+        architecture: "x64"
+      env:
+        AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache
 
     - name: Extract package name and version from branch name
       id: get_pkg_info
@@ -80,10 +83,13 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
       with:
-        python-version: 3.6
+        python-version: "3.10"
+        architecture: "x64"
+      env:
+        AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache
 
     - name: Install dependencies
       run: |
@@ -177,6 +183,8 @@ jobs:
         body: |
           * Release `${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}` (${{ github.sha }})
           * Package `${{ steps.prep_assets.outputs.pkg_tar }}` (sha256: `${{ steps.prep_assets.outputs.pkg_tar_sha }}`)
+          * Package URI `github.com/${{ needs.build.outputs.repo_lowercase }}/${{ steps.get_pkg_info.outputs.pkg_name }}@${{ steps.get_pkg_info.outputs.pkg_ver }}`
+          * Run the package: `nextflow run ${{ needs.build.outputs.repo_lowercase }}/${{ steps.get_pkg_info.outputs.pkg_name }}/main.nf -r ${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }} -params-file <params-json-file>`
         draft: false
         prerelease: false
 

diff --git a/.wfpm b/.wfpm
@@ -2,4 +2,4 @@ project_name: sanger-wxs-variant-calling
 license: GNU Affero General Public License v3
 repo_type: git
 repo_server: github.com
-repo_account: icgc-argo
+repo_account: icgc-argo-workflows
diff --git a/sanger-wxs-variant-calling/main.nf b/sanger-wxs-variant-calling/main.nf
@@ -2,7 +2,7 @@
 nextflow.enable.dsl = 2
 name = 'sanger-wxs-variant-calling'
 short_name = 'sanger-wxs'
-version = '3.1.6-3.6.0'
+version = '3.1.6-3.7.0'
 
 
 /*
@@ -119,17 +119,24 @@ params.normal_extra_info = "NO_FILE6"
 
 // dir for outputs, must be set when running in local mode
 params.publish_dir = ""
-
-params.api_token = ""
-params.song_url = ""
-params.score_url = ""
 params.cleanup = true
-
 params.cpus = 1
 params.mem = 1
 params.max_retries = 5  // set to 0 will disable retry
 params.first_retry_wait_time = 1  // in seconds
 
+// song/score setting
+params.song_url = ""
+params.song_container = "ghcr.io/overture-stack/song-client"
+params.song_container_version = "5.0.2"
+params.score_url = ""
+params.score_container = "ghcr.io/overture-stack/score"
+params.score_container_version = "5.9.0"
+params.score_mem = 20
+params.score_cpus = 8
+params.score_force = false
+params.api_token = ""
+
 params.download = [:]
 params.generateBas = [:]
 params.sangerWxsVariantCaller = [:]
@@ -145,7 +152,16 @@ download_params = [
     'max_retries': params.max_retries,
     'first_retry_wait_time': params.first_retry_wait_time,
     'song_url': params.song_url,
+    'song_container': params.song_container,
+    'song_container_version': params.song_container_version,
+    'song_cpus': params.cpus,
+    'song_mem': params.mem,
     'score_url': params.score_url,
+    'score_container': params.score_container,
+    'score_container_version': params.score_container_version,
+    'score_cpus' : params.score_cpus,
+    'score_mem' : params.score_mem,
+    'score_transport_mem' : params.score_mem, 
     'api_token': params.api_token,
     *:(params.download ?: [:])
 ]
@@ -205,29 +221,36 @@ payloadGenVariantCall_params = [
 ]
 
 upload_params = [
-    'cpus': params.cpus,
-    'mem': params.mem,
     'max_retries': params.max_retries,
-    'publish_dir': params.publish_dir,
     'first_retry_wait_time': params.first_retry_wait_time,
     'song_url': params.song_url,
+    'song_container': params.song_container,
+    'song_container_version': params.song_container_version,
+    'song_cpus': params.cpus,
+    'song_mem': params.mem,
     'score_url': params.score_url,
+    'score_container': params.score_container,
+    'score_container_version': params.score_container_version,
+    'score_force' : params.score_force,
+    'score_cpus' : params.score_cpus,
+    'score_mem' : params.score_mem,
+    'score_transport_mem' : params.score_mem,
     'api_token': params.api_token,
-    *:(params.uploadVariant ?: [:])
+    *:(params.upload ?: [:])
 ]
 
 
 // Include all modules and pass params
-include { SongScoreDownload as dnldT; SongScoreDownload as dnldN } from './wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/main.nf' params(download_params)
+include { SongScoreDownload as dnldT; SongScoreDownload as dnldN } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/main.nf' params(download_params)
 include { generateBas as basT; generateBas as basN; } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/generate-bas.0.2.1.0/tools/generate-bas/generate-bas' params(generateBas_params)
 include { sangerWxsVariantCaller as sangerWxs } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/sanger-wxs-variant-caller.3.1.6-3/tools/sanger-wxs-variant-caller/sanger-wxs-variant-caller.nf' params(sangerWxsVariantCaller_params)
 include { repackSangerResults as repack } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/repack-sanger-results.0.2.0.0/tools/repack-sanger-results/repack-sanger-results' params(repackSangerResults_params)
 include { cavemanVcfFix as cavemanFix } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/caveman-vcf-fix.0.1.0.0/tools/caveman-vcf-fix/caveman-vcf-fix' params(cavemanVcfFix_params)
 include { prepSangerSupplement as prepSupp } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/prep-sanger-supplement.0.1.2.0/tools/prep-sanger-supplement/prep-sanger-supplement' params(prepSangerSupplement_params)
 include { prepSangerQc as prepQc } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/prep-sanger-qc.0.1.3.0/tools/prep-sanger-qc/prep-sanger-qc' params(prepSangerQc_params)
 include { extractFilesFromTarball as extractVarSnv; extractFilesFromTarball as extractVarIndel; extractFilesFromTarball as extractQC } from './modules/raw.githubusercontent.com/icgc-argo/data-processing-utility-tools/extract-files-from-tarball.0.2.0.0/tools/extract-files-from-tarball/extract-files-from-tarball' params(extractSangerCall_params)
-include { payloadGenVariantCalling as pGenVarSnv; payloadGenVariantCalling as pGenVarIndel; payloadGenVariantCalling as pGenVarSupp; payloadGenVariantCalling as pGenQc } from "./wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/payload-gen-variant-calling@0.6.0/main" params(payloadGenVariantCall_params)
-include { SongScoreUpload as upSnv; SongScoreUpload as upIndel; SongScoreUpload as upQc; SongScoreUpload as upSupp} from './wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-upload@2.6.1/main.nf' params(upload_params)
+include { payloadGenVariantCalling as pGenVarSnv; payloadGenVariantCalling as pGenVarIndel; payloadGenVariantCalling as pGenVarSupp; payloadGenVariantCalling as pGenQc } from "./wfpr_modules/github.com/icgc-argo-workflows/data-processing-utility-tools/payload-gen-variant-calling@0.7.0/main" params(payloadGenVariantCall_params)
+include { SongScoreUpload as upSnv; SongScoreUpload as upIndel; SongScoreUpload as upQc; SongScoreUpload as upSupp} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf' params(upload_params)
 include { cleanupWorkdir as cleanup } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/cleanup-workdir@1.0.0/main'
 include { getSecondaryFiles } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/helper-functions@1.0.0/main'
 include { payloadAddUniformIds as pAddIdT; payloadAddUniformIds as pAddIdN } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/payload-add-uniform-ids@0.1.1/main'
@@ -358,10 +381,10 @@ workflow SangerWxs {
 
         // upload variant results in paralllel
         if (!local_mode) {
-            upSnv(study_id, pGenVarSnv.out.payload, pGenVarSnv.out.files_to_upload)
-            upIndel(study_id, pGenVarIndel.out.payload, pGenVarIndel.out.files_to_upload)
-            upSupp(study_id, pGenVarSupp.out.payload, pGenVarSupp.out.files_to_upload)
-            upQc(study_id, pGenQc.out.payload, pGenQc.out.files_to_upload)
+            upSnv(study_id, pGenVarSnv.out.payload, pGenVarSnv.out.files_to_upload, '')
+            upIndel(study_id, pGenVarIndel.out.payload, pGenVarIndel.out.files_to_upload, '')
+            upSupp(study_id, pGenVarSupp.out.payload, pGenVarSupp.out.files_to_upload, '')
+            upQc(study_id, pGenQc.out.payload, pGenQc.out.files_to_upload, '')
         }
 
         if (params.cleanup) {

diff --git a/sanger-wxs-variant-calling/pkg.json b/sanger-wxs-variant-calling/pkg.json
@@ -1,6 +1,6 @@
 {
     "name": "sanger-wxs-variant-calling",
-    "version": "3.1.6-3.6.0",
+    "version": "3.1.6-3.7.0",
     "description": "ICGC ARGO Sanger WXS Variant Calling Workflow",
     "main": "main.nf",
     "deprecated": false,
@@ -11,15 +11,15 @@
     ],
     "repository": {
         "type": "git",
-        "url": "https://github.com/icgc-argo/sanger-wxs-variant-calling.git"
+        "url": "https://github.com/icgc-argo-workflows/sanger-wxs-variant-calling.git"
     },
     "dependencies": [
-        "github.com/icgc-argo/data-processing-utility-tools/payload-gen-variant-calling@0.6.0",
+        "github.com/icgc-argo-workflows/data-processing-utility-tools/payload-gen-variant-calling@0.7.0",
         "github.com/icgc-argo/data-processing-utility-tools/payload-add-uniform-ids@0.1.1",
         "github.com/icgc-argo/data-processing-utility-tools/helper-functions@1.0.0",
         "github.com/icgc-argo/data-processing-utility-tools/cleanup-workdir@1.0.0",
-        "github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1",
-        "github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-upload@2.6.1"
+        "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0",
+        "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3"
     ],
     "devDependencies": [],
     "contributors": [
@@ -31,6 +31,6 @@
         }
     ],
     "license": "GNU Affero General Public License v3",
-    "bugReport": "https://github.com/icgc-argo/sanger-wxs-variant-calling/issues",
-    "homepage": "https://github.com/icgc-argo/sanger-wxs-variant-calling#readme"
+    "bugReport": "https://github.com/icgc-argo-workflows/sanger-wxs-variant-calling/issues",
+    "homepage": "https://github.com/icgc-argo-workflows/sanger-wxs-variant-calling#readme"
 }
diff --git a/...d-gen-variant-calling@0.6.0/.dockerignore → ...d-gen-variant-calling@0.7.0/.dockerignore b/...d-gen-variant-calling@0.6.0/.dockerignore → ...d-gen-variant-calling@0.7.0/.dockerignore
diff --git a/...load-gen-variant-calling@0.6.0/Dockerfile → ...load-gen-variant-calling@0.7.0/Dockerfile b/...load-gen-variant-calling@0.6.0/Dockerfile → ...load-gen-variant-calling@0.7.0/Dockerfile
@@ -1,6 +1,6 @@
 FROM python:3.7.9
 
-LABEL org.opencontainers.image.source https://github.com/icgc-argo/data-processing-utility-tools
+LABEL org.opencontainers.image.source https://github.com/icgc-argo-workflows/data-processing-utility-tools
 
 RUN groupadd -g 1000 ubuntu && \
     useradd -l -u 1000 -g ubuntu ubuntu && \

diff --git a/...payload-gen-variant-calling@0.6.0/main.nf → ...payload-gen-variant-calling@0.7.0/main.nf b/...payload-gen-variant-calling@0.6.0/main.nf → ...payload-gen-variant-calling@0.7.0/main.nf
@@ -25,10 +25,10 @@
 /* this block is auto-generated based on info from pkg.json where   */
 /* changes can be made if needed, do NOT modify this block manually */
 nextflow.enable.dsl = 2
-version = '0.6.0'
+version = '0.7.0'
 
 container = [
-    'ghcr.io': 'ghcr.io/icgc-argo/data-processing-utility-tools.payload-gen-variant-calling'
+    'ghcr.io': 'ghcr.io/icgc-argo-workflows/data-processing-utility-tools.payload-gen-variant-calling'
 ]
 default_container_registry = 'ghcr.io'
 /********************************************************************/

diff --git a/...payload-gen-variant-calling@0.6.0/main.py → ...payload-gen-variant-calling@0.7.0/main.py b/...payload-gen-variant-calling@0.6.0/main.py → ...payload-gen-variant-calling@0.7.0/main.py
@@ -170,11 +170,13 @@ def get_files_info(file_to_upload, wf_short_name,  wf_version, somatic_or_germli
         else:
             sys.exit('Error: unknown file type "%s"' % file_to_upload)
 
+        ## Disable the population of qc_metrics into payload to avoid the exposure of sensitive info
         tar = tarfile.open(file_to_upload)
         for member in tar.getmembers():
             if member.name.endswith('.extra_info.json'):
                 f = tar.extractfile(member)
                 extra_info = json.load(f)
+                extra_info.pop('metrics', None)
                 break
     else:
         sys.exit('Error: unknown file type "%s"' % file_to_upload)

diff --git a/...gen-variant-calling@0.6.0/nextflow.config → ...gen-variant-calling@0.7.0/nextflow.config b/...gen-variant-calling@0.6.0/nextflow.config → ...gen-variant-calling@0.7.0/nextflow.config
diff --git a/...ayload-gen-variant-calling@0.6.0/pkg.json → ...ayload-gen-variant-calling@0.7.0/pkg.json b/...ayload-gen-variant-calling@0.6.0/pkg.json → ...ayload-gen-variant-calling@0.7.0/pkg.json
@@ -1,6 +1,6 @@
 {
     "name": "payload-gen-variant-calling",
-    "version": "0.6.0",
+    "version": "0.7.0",
     "description": "A tool to generate SONG payloads for variant calling workflows",
     "main": "main.nf",
     "deprecated": false,
@@ -11,14 +11,14 @@
     ],
     "repository": {
         "type": "git",
-        "url": "https://github.com/icgc-argo/data-processing-utility-tools.git"
+        "url": "https://github.com/icgc-argo-workflows/data-processing-utility-tools.git"
     },
     "container": {
         "registries": [
             {
                 "registry": "ghcr.io",
                 "type": "docker",
-                "org": "icgc-argo",
+                "org": "icgc-argo-workflows",
                 "default": true
             }
         ]
@@ -31,6 +31,6 @@
         }
     ],
     "license": "GNU Affero General Public License v3",
-    "bugReport": "https://github.com/icgc-argo/data-processing-utility-tools/issues",
-    "homepage": "https://github.com/icgc-argo/data-processing-utility-tools#readme"
+    "bugReport": "https://github.com/icgc-argo-workflows/data-processing-utility-tools/issues",
+    "homepage": "https://github.com/icgc-argo-workflows/data-processing-utility-tools#readme"
 }
diff --git a/...ad-gen-variant-calling@0.6.0/wfpr_modules → ...ad-gen-variant-calling@0.7.0/wfpr_modules b/...ad-gen-variant-calling@0.6.0/wfpr_modules → ...ad-gen-variant-calling@0.7.0/wfpr_modules
diff --git a/...oad@2.6.1/local_modules/score-download.nf → ...oad@2.9.0/local_modules/score-download.nf b/...oad@2.6.1/local_modules/score-download.nf → ...oad@2.9.0/local_modules/score-download.nf
@@ -11,7 +11,8 @@ params.max_retries = 5  // set to 0 will disable retry
 params.first_retry_wait_time = 1  // in seconds
 
 // required params w/ default
-params.container_version = "5.0.0"
+params.container = "ghcr.io/overture-stack/score"
+params.container_version = "5.8.1"
 params.transport_mem = 2 // Transport memory is in number of GBs
 
 // optional if secret mounted from pod else required
@@ -34,12 +35,18 @@ process scoreDownload {
     cpus params.cpus
     memory "${params.mem} GB"
 
-    container "overture/score:${params.container_version}"
-    publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
+    container "${ params.score_container ?: params.container}:${params.score_container_version ?: params.container_version}"
+    publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir ? true : false
 
     label "scoreDownload"
     tag "${analysis_id}"
 
+    if (workflow.containerEngine == "singularity") {
+        containerOptions "--bind \$(pwd):/score-client/logs"
+    } else if (workflow.containerEngine == "docker") {
+        containerOptions "-v \$(pwd):/score-client/logs"
+    }
+
     input:
         path analysis
         val study_id

diff --git a/...@2.6.1/local_modules/song-get-analysis.nf → ...@2.9.0/local_modules/song-get-analysis.nf b/...@2.6.1/local_modules/song-get-analysis.nf → ...@2.9.0/local_modules/song-get-analysis.nf
@@ -11,7 +11,8 @@ params.max_retries = 5  // set to 0 will disable retry
 params.first_retry_wait_time = 1  // in seconds
 
 // required params w/ default
-params.container_version = "4.2.1"
+params.container = "ghcr.io/overture-stack/song-client"
+params.container_version = "5.0.2"
 
 // optional if secret mounted from pod else required
 params.api_token = "" // song/score API token for download process
@@ -31,12 +32,18 @@ process songGetAnalysis {
 
     cpus params.cpus
     memory "${params.mem} GB"
-
-    container "overture/song-client:${params.container_version}"
-    publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
+    
+    container "${ params.song_container ?: params.container}:${params.song_container_version ?: params.container_version}"
+    publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir ? true : false
 
     tag "${analysis_id}"
 
+    if (workflow.containerEngine == "singularity") {
+        containerOptions "--bind \$(pwd):/song-client/logs"
+    } else if (workflow.containerEngine == "docker") {
+        containerOptions "-v \$(pwd):/song-client/logs"
+    }
+
     input:
         val study_id
         val analysis_id

diff --git a/...y-tools/song-score-download@2.6.1/main.nf → ...y-tools/song-score-download@2.9.0/main.nf b/...y-tools/song-score-download@2.6.1/main.nf → ...y-tools/song-score-download@2.9.0/main.nf
@@ -21,7 +21,7 @@
 */
 
 nextflow.enable.dsl = 2
-version = '2.6.1'
+version = '2.9.0'
 
 // universal params go here, change default value as needed
 params.publish_dir = ""  // set to empty string will disable publishDir
@@ -30,33 +30,35 @@ params.max_retries = 5  // set to 0 will disable retry
 params.first_retry_wait_time = 1  // in seconds
 
 // tool specific parmas go here, add / change as needed
-params.study_id = "TEST-PR"
-params.analysis_id = "9940db0f-c100-496a-80db-0fc100d96ac1"
+params.study_id = ""
+params.analysis_id = ""
 
 params.api_token = ""
 
 params.song_cpus = 1
 params.song_mem = 1  // GB
 params.song_url = "https://song.rdpc-qa.cancercollaboratory.org"
 params.song_api_token = ""
-params.song_container_version = "4.2.1"
+params.song_container = "ghcr.io/overture-stack/song-client"
+params.song_container_version = "5.0.2"
 
 params.score_cpus = 1
 params.score_mem = 1  // GB
 params.score_transport_mem = 1  // GB
 params.score_url = "https://score.rdpc-qa.cancercollaboratory.org"
 params.score_api_token = ""
-params.score_container_version = "5.0.0"
+params.score_container = "ghcr.io/overture-stack/score"
+params.score_container_version = "5.8.1"
 
 
 song_params = [
     *:params,
     'cpus': params.song_cpus,
     'mem': params.song_mem,
     'song_url': params.song_url,
+    'song_container': params.song_container,
     'song_container_version': params.song_container_version,
-    'api_token': params.song_api_token ?: params.api_token,
-    'publish_dir': ''
+    'api_token': params.song_api_token ?: params.api_token
 ]
 
 score_params = [
@@ -66,6 +68,7 @@ score_params = [
     'transport_mem': params.score_transport_mem,
     'song_url': params.song_url,
     'score_url': params.score_url,
+    'score_container': params.score_container,
     'score_container_version': params.score_container_version,
     'api_token': params.score_api_token ?: params.api_token
 ]

diff --git a/...song-score-download@2.6.1/nextflow.config → ...song-score-download@2.9.0/nextflow.config b/...song-score-download@2.6.1/nextflow.config → ...song-score-download@2.9.0/nextflow.config