From ba87d010ee911c144f71bd70261a894e04d68e1e Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Fri, 4 Oct 2019 13:12:56 -0700 Subject: [PATCH 01/18] fix for issue #91 and issue #63 --- src/encode_task_merge_fastq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/encode_task_merge_fastq.py b/src/encode_task_merge_fastq.py index 2ee0add2..8d82cd94 100755 --- a/src/encode_task_merge_fastq.py +++ b/src/encode_task_merge_fastq.py @@ -7,7 +7,7 @@ import os import argparse from encode_lib_common import ( - hard_link, log, ls_l, mkdir_p, read_tsv, run_shell_cmd, + soft_link, log, ls_l, mkdir_p, read_tsv, run_shell_cmd, strip_ext_fastq) @@ -69,7 +69,7 @@ def merge_fastqs(fastqs, end, out_dir): run_shell_cmd(cmd) return merged else: - return hard_link(fastqs[0], merged) + return soft_link(fastqs[0], merged) def main(): From 76fb6df2bf44400274cf2070ae40c91ec3e027bb Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Fri, 4 Oct 2019 13:30:02 -0700 Subject: [PATCH 02/18] add caper and croo to Conda env --- scripts/install_conda_env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/install_conda_env.sh b/scripts/install_conda_env.sh index c8da3233..22ab0bd6 100755 --- a/scripts/install_conda_env.sh +++ b/scripts/install_conda_env.sh @@ -82,6 +82,9 @@ ${CONDA_BIN}/pip install --no-dependencies SAMstats==0.2.1 # install pyhocon for caper ${CONDA_BIN}/pip install pyhocon +# install caper and croo +${CONDA_BIN}/pip install caper croo + echo "=== Updating pipeline's Conda environments ===" cd ${CONDA_BIN} chmod u+rx ${SRC_DIR}/*.py From 00ea6e3665d7062f903ea3df17e8dda903eace86 Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Fri, 4 Oct 2019 13:30:14 -0700 Subject: [PATCH 03/18] add ghostscript to Conda env (fix for gs error on sherlock) --- scripts/requirements.txt | 1 + scripts/requirements_py2.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index d74186c7..572dcc88 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -44,6 +44,7 @@ libgcc requests ncurses gnuplot +ghostscript numpy scipy diff --git a/scripts/requirements_py2.txt b/scripts/requirements_py2.txt index e1ab89c4..512701df 100644 --- a/scripts/requirements_py2.txt +++ b/scripts/requirements_py2.txt @@ -5,3 +5,5 @@ python ==2.7.16 macs2 ==2.1.3.3 metaseq ==0.5.6 python-dateutil ==2.8.0 + +ghostscript From 15a6e220249fe1873df0e3f182bfc04a022b450e Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Wed, 9 Oct 2019 22:07:05 -0700 Subject: [PATCH 04/18] description for xcor in report is more accurate --- chip.wdl | 8 +++++- src/encode_task_qc_report.py | 48 +++++++++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/chip.wdl b/chip.wdl index 5142a7c7..5a205340 100644 --- a/chip.wdl +++ b/chip.wdl @@ -1147,6 +1147,8 @@ workflow chip { cap_num_peak = cap_num_peak_, idr_thresh = idr_thresh, pval_thresh = pval_thresh, + xcor_pe_trim_bp = xcor_pe_trim_bp, + xcor_subsample_reads = xcor_subsample_reads, samstat_qcs = align.samstat_qc, nodup_samstat_qcs = filter.samstat_qc, @@ -1844,7 +1846,9 @@ task qc_report { String peak_caller Int cap_num_peak Float idr_thresh - Float pval_thresh + Float pval_thresh + Int xcor_pe_trim_bp + Int xcor_subsample_reads # QCs Array[File?] samstat_qcs Array[File?] nodup_samstat_qcs @@ -1907,6 +1911,8 @@ task qc_report { ${'--cap-num-peak ' + cap_num_peak} \ --idr-thresh ${idr_thresh} \ --pval-thresh ${pval_thresh} \ + --xcor-pe-trim-bp ${xcor_pe_trim_bp} \ + --xcor-subsample-reads ${xcor_subsample_reads} \ --samstat-qcs ${sep='_:_' samstat_qcs} \ --nodup-samstat-qcs ${sep='_:_' nodup_samstat_qcs} \ --dup-qcs ${sep='_:_' dup_qcs} \ diff --git a/src/encode_task_qc_report.py b/src/encode_task_qc_report.py index 7e63045b..df608876 100755 --- a/src/encode_task_qc_report.py +++ b/src/encode_task_qc_report.py @@ -58,6 +58,12 @@ def parse_arguments(): help='IDR threshold.') parser.add_argument('--pval-thresh', type=float, help='pValue threshold for MACS2 peak caller.') + parser.add_argument('--xcor-pe-trim-bp', type=int, + help='FASTQs are trimmed to this for cross-correlation ' + 'analysis only.') + parser.add_argument('--xcor-subsample-reads', type=int, + help='Subsampled TAG-ALIGNs to this depth for cross-correlation ' + 'analysis only.') parser.add_argument('--samstat-qcs', type=str, nargs='*', help='List of samstat QC (raw BAM) files per replicate.') parser.add_argument('--nodup-samstat-qcs', type=str, nargs='*', @@ -268,7 +274,8 @@ def make_cat_root(args): ('aligner', args.aligner), ('peak_caller', args.peak_caller), ]) - if args.ctl_paired_ends: + if args.ctl_paired_ends \ + and args.pipeline_type not in ('atac', 'dnase'): d_general['ctl_paired_end'] = args.ctl_paired_ends cat_root.add_log(d_general, key='general') @@ -626,20 +633,37 @@ def make_cat_align_enrich(args, cat_root): parent=cat_root ) + if args.pipeline_type in ('tf', 'histone'): + html_foot_xcor = """ +

Performed on subsampled ({xcor_subsample_reads}) reads mapped from FASTQs that are trimmed to {xcor_pe_trim_bp}. + Such FASTQ trimming and subsampling reads are for cross-corrleation analysis only. + Untrimmed FASTQs are used for all the other analyses.

+

+ NOTE1: For SE datasets, reads from replicates are randomly subsampled to {xcor_subsample_reads}.
+ NOTE2: For PE datasets, the first end (R1) of each read-pair is selected and trimmed to {xcor_pe_trim_bp} the reads are then randomly subsampled to {xcor_subsample_reads}.
+ """.format( + xcor_subsample_reads=args.xcor_subsample_reads, + xcor_pe_trim_bp=args.xcor_pe_trim_bp, + ) + else: + html_foot_xcor = """ +

Performed on subsampled ({xcor_subsample_reads}) reads. + Such FASTQ trimming is for cross-corrleation analysis only.

+

+ """.format( + xcor_subsample_reads=args.xcor_subsample_reads + ) + html_foot_xcor += """

    +
  • Normalized strand cross-correlation coefficient (NSC) = col9 in outFile
    • +
  • Relative strand cross-correlation coefficient (RSC) = col10 in outFile
    • +
  • Estimated fragment length = col3 in outFile, take the top value
    • +


+ """ + cat_xcor = QCCategory( 'xcor_score', html_head='

Strand cross-correlation measures

', - html_foot=""" -

Performed on subsampled reads

-

- NOTE1: For SE datasets, reads from replicates are randomly subsampled.
- NOTE2: For PE datasets, the first end of each read-pair is selected and the reads are then randomly subsampled.
-

    -
  • Normalized strand cross-correlation coefficient (NSC) = col9 in outFile
    • -
  • Relative strand cross-correlation coefficient (RSC) = col10 in outFile
    • -
  • Estimated fragment length = col3 in outFile, take the top value
    • -


- """, + html_foot=html_foot_xcor, parser=parse_xcor_score, map_key_desc=MAP_KEY_DESC_XCOR_SCORE, parent=cat_align_enrich, From abdee18f9a02ad348ff97f3404cf062cef1ac814 Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Wed, 9 Oct 2019 22:14:54 -0700 Subject: [PATCH 05/18] conda/docker: added latest MACS2==2.2.4 to env/docker --- dev/docker_image/Dockerfile | 20 +++++++++++++------- scripts/install_conda_env.sh | 17 ++--------------- scripts/requirements.txt | 13 ++++++++++--- scripts/requirements_py2.txt | 1 - 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/dev/docker_image/Dockerfile b/dev/docker_image/Dockerfile index 63a5b86f..2d3f1704 100644 --- a/dev/docker_image/Dockerfile +++ b/dev/docker_image/Dockerfile @@ -94,8 +94,11 @@ RUN git clone --branch 2.0.4.2 --single-branch https://github.com/kundajelab/idr RUN pip2 install --no-cache-dir numpy matplotlib==2.2.4 # Install genomic python packages (python2) -RUN pip2 install --no-cache-dir Cython -RUN pip2 install --no-cache-dir macs2==2.1.3.3 metaseq==0.5.6 +RUN pip2 install --no-cache-dir metaseq==0.5.6 + +# Install MACS2 (python3) +RUN pip3 install --no-cache-dir Cython +RUN pip3 install --no-cache-dir macs2==2.2.4 # Install UCSC tools (v377) RUN git clone https://github.com/ENCODE-DCC/kentUtils_bin_v377 @@ -109,15 +112,18 @@ ENV PYTHONNOUSERSITE=True ENV OPENBLAS_NUM_THREADS=1 ENV MKL_NUM_THREADS=1 +# make some temporary directories +RUN mkdir -p /mnt/ext_{0..9} + +# make pipeline src directory to store py's +RUN mkdir -p chip-seq-pipeline/src +ENV PATH="/software/chip-seq-pipeline:/software/chip-seq-pipeline/src:${PATH}" + # Get ENCODE chip-seq-pipeline container repository # This COPY assumes the build context is the root of the chip-seq-pipeline repo # and it gets whatever is checked out plus local modifications -# so the buildling command should: +# so the buildling command should be: # cd [GIT_REPO_DIR] && docker build -f dev/docker_image/Dockerfile . -RUN mkdir -p chip-seq-pipeline/src COPY src chip-seq-pipeline/src/ COPY chip.wdl chip-seq-pipeline/ -ENV PATH="/software/chip-seq-pipeline:/software/chip-seq-pipeline/src:${PATH}" -# make some temporary directories -RUN mkdir -p /mnt/ext_{0..9} diff --git a/scripts/install_conda_env.sh b/scripts/install_conda_env.sh index 22ab0bd6..42d582b9 100755 --- a/scripts/install_conda_env.sh +++ b/scripts/install_conda_env.sh @@ -12,8 +12,8 @@ SRC_DIR=${SH_SCRIPT_DIR}/../src conda --version # check if conda exists echo "=== Installing pipeline's Conda environments ===" -conda create -n ${CONDA_ENV_PY3} --file ${REQ_TXT_PY3} -y -c bioconda -c r -conda create -n ${CONDA_ENV_PY2} --file ${REQ_TXT_PY2} -y -c bioconda -c conda-forge +conda create -n ${CONDA_ENV_PY3} --file ${REQ_TXT_PY3} -y -c defaults -c r -c bioconda -c conda-forge +conda create -n ${CONDA_ENV_PY2} --file ${REQ_TXT_PY2} -y -c defaults -c r -c bioconda -c conda-forge echo "=== Configuring for pipeline's Conda environments ===" CONDA_PREFIX_PY3=$(conda env list | grep -P "\b${CONDA_ENV_PY3}\s" | awk '{if (NF==3) print $3; else print $2}') @@ -69,22 +69,9 @@ echo "unset OLD_R_HOME" >> ${CONDA_DEACTIVATE_SH} echo "unset OLD_R_LIBS" >> ${CONDA_DEACTIVATE_SH} # hack around the need for both python2 and python3 in the same environment -# we will keep this workaround until MACS2 and metaseq are ported to python3. cd ${CONDA_BIN} -rm -f macs2 python2 -ln -s ../../${CONDA_ENV_PY2}/bin/macs2 ln -s ../../${CONDA_ENV_PY2}/bin/python2 -# install SAMstats 0.2.1 -# we will keep this until SAMstats is added to bioconda -${CONDA_BIN}/pip install --no-dependencies SAMstats==0.2.1 - -# install pyhocon for caper -${CONDA_BIN}/pip install pyhocon - -# install caper and croo -${CONDA_BIN}/pip install caper croo - echo "=== Updating pipeline's Conda environments ===" cd ${CONDA_BIN} chmod u+rx ${SRC_DIR}/*.py diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 572dcc88..5cc0112f 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,12 +1,14 @@ -# repos: bioconda, r +# for python3 +# conda repos: defaults, r, bioconda, conda-forge -python ==3.6.6 # 3.7.4 didn't work with Conda >= 4.7 +nomkl # using MKL can change MACS2 output randomly on different platforms idr ==2.0.4.2 tabix samtools ==1.9 htslib ==1.9 sambamba ==0.6.6 +samstats ==0.2.1 bedtools ==2.29.0 picard ==2.20.7 @@ -33,12 +35,14 @@ pysam ==0.15.3 pybedtools ==0.8.0 phantompeakqualtools ==1.2.1 pybigwig ==0.3.13 -openssl ==1.0.2t # important to get the same random value +openssl ==1.0.2t # important to get the same random seed for "shuf" deeptools ==3.3.1 cutadapt ==2.5 preseq ==2.0.3 pyfaidx ==0.5.5.2 +macs2 ==2.2.4 + jsondiff ==1.1.1 libgcc requests @@ -54,3 +58,6 @@ gsl matplotlib java-jdk + +caper +#croo diff --git a/scripts/requirements_py2.txt b/scripts/requirements_py2.txt index 512701df..ea7883a5 100644 --- a/scripts/requirements_py2.txt +++ b/scripts/requirements_py2.txt @@ -2,7 +2,6 @@ python ==2.7.16 -macs2 ==2.1.3.3 metaseq ==0.5.6 python-dateutil ==2.8.0 From 7d09dee46d2c16adf098b382cbdb8bdae411b972 Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Wed, 9 Oct 2019 22:22:16 -0700 Subject: [PATCH 06/18] sync with atac (encode_task_fraglen_stat_pe.py) --- src/encode_task_fraglen_stat_pe.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/encode_task_fraglen_stat_pe.py b/src/encode_task_fraglen_stat_pe.py index 3998d9cd..fcecec82 100755 --- a/src/encode_task_fraglen_stat_pe.py +++ b/src/encode_task_fraglen_stat_pe.py @@ -12,7 +12,7 @@ import os import argparse from encode_lib_common import ( - strip_ext_bam, ls_l, log, rm_f) + strip_ext_bam, ls_l, log, rm_f, pdf2png) from encode_lib_genomic import ( remove_read_group, locate_picard) import matplotlib as mpl @@ -194,8 +194,12 @@ def fragment_length_plot(data_file, prefix, peaks=None): # plot_img = BytesIO() # fig.savefig(plot_img, format='png') + plot_pdf = prefix + '.fraglen_dist.pdf' plot_png = prefix + '.fraglen_dist.png' - fig.savefig(plot_png, format='png') + + fig.savefig(plot_pdf, format='pdf') + pdf2png(plot_pdf, os.path.dirname(plot_pdf)) + rm_f(plot_pdf) return plot_png From 6b1f611407193705fa3b28984619e59afa9c49a1 Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Wed, 9 Oct 2019 23:57:32 -0700 Subject: [PATCH 07/18] use py3 for macs2 --- chip.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chip.wdl b/chip.wdl index 5a205340..cedb662f 100644 --- a/chip.wdl +++ b/chip.wdl @@ -1582,7 +1582,7 @@ task call_peak { command { if [ '${peak_caller}' == 'macs2' ]; then - python2 $(which encode_task_macs2_chip.py) \ + python3 $(which encode_task_macs2_chip.py) \ ${sep=' ' tas} \ ${'--gensz '+ gensz} \ ${'--chrsz ' + chrsz} \ @@ -1651,7 +1651,7 @@ task macs2_signal_track { String disks command { - python2 $(which encode_task_macs2_signal_track_chip.py) \ + python3 $(which encode_task_macs2_signal_track_chip.py) \ ${sep=' ' tas} \ ${'--gensz '+ gensz} \ ${'--chrsz ' + chrsz} \ From ec42635d2350b4356f345b7ffe9affde7208f080 Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Thu, 10 Oct 2019 09:33:39 -0700 Subject: [PATCH 08/18] conda: add croo to env --- scripts/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 5cc0112f..9eba191b 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -60,4 +60,4 @@ matplotlib java-jdk caper -#croo +croo From b36ddf352ff4eae620f552b0378d0db0bb42e9b3 Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Thu, 10 Oct 2019 09:44:42 -0700 Subject: [PATCH 09/18] ver: v1.3.0 -> dev-v1.3.1 (no output change) --- chip.wdl | 6 ++--- dev/dev.md | 6 ++--- dev/test/test_task/test.sh | 2 +- dev/test/test_workflow/test_chip.sh | 2 +- dev/workflow_opts/docker.json | 2 +- dev/workflow_opts/scg.json | 2 +- dev/workflow_opts/sge.json | 2 +- dev/workflow_opts/sherlock.json | 2 +- dev/workflow_opts/singularity.json | 2 +- dev/workflow_opts/slurm.json | 2 +- docs/deprecated/tutorial_local_singularity.md | 4 ++-- docs/deprecated/tutorial_scg.md | 2 +- docs/deprecated/tutorial_scg_backend.md | 4 ++-- docs/deprecated/tutorial_sge.md | 4 ++-- docs/deprecated/tutorial_sge_backend.md | 2 +- docs/deprecated/tutorial_sherlock.md | 2 +- docs/deprecated/tutorial_sherlock_backend.md | 4 ++-- docs/deprecated/tutorial_slurm.md | 4 ++-- docs/deprecated/tutorial_slurm_backend.md | 4 ++-- docs/tutorial_dx_web.md | 24 +++++++++---------- 20 files changed, 41 insertions(+), 41 deletions(-) diff --git a/chip.wdl b/chip.wdl index cedb662f..c30d88e5 100644 --- a/chip.wdl +++ b/chip.wdl @@ -1,12 +1,12 @@ # ENCODE TF/Histone ChIP-Seq pipeline # Author: Jin Lee (leepc12@gmail.com) -#CAPER docker quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 -#CAPER singularity docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 +#CAPER docker quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 +#CAPER singularity docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 #CROO out_def https://storage.googleapis.com/encode-pipeline-output-definition/chip.croo.json workflow chip { - String pipeline_ver = 'v1.3.0' + String pipeline_ver = 'dev-v1.3.1' ### sample name, description String title = 'Untitled' String description = 'No description' diff --git a/dev/dev.md b/dev/dev.md index df42070c..21b232a4 100644 --- a/dev/dev.md +++ b/dev/dev.md @@ -2,8 +2,8 @@ ## Command line for version change ```bash -PREV_VER=v1.3.0 -NEW_VER=v1.3.0 +PREV_VER=dev-v1.3.1 +NEW_VER=v1.3.1 for f in $(grep -rl ${PREV_VER} --include=*.{wdl,md,sh}) do sed -i "s/${PREV_VER}/${NEW_VER}/g" ${f} @@ -24,7 +24,7 @@ Run the following command line locally to build out DX workflows for this pipeli ```bash # version -VER=v1.3.0 +VER=dev-v1.3.1 DOCKER=quay.io/encode-dcc/chip-seq-pipeline:$VER # general diff --git a/dev/test/test_task/test.sh b/dev/test/test_task/test.sh index e5edeed0..57bcb22c 100755 --- a/dev/test/test_task/test.sh +++ b/dev/test/test_task/test.sh @@ -12,7 +12,7 @@ INPUT=$2 if [ $# -gt 2 ]; then DOCKER_IMAGE=$3 else - DOCKER_IMAGE=quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 + DOCKER_IMAGE=quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 fi if [ $# -gt 3 ]; then NUM_TASK=$4 diff --git a/dev/test/test_workflow/test_chip.sh b/dev/test/test_workflow/test_chip.sh index 277878ce..335f15a5 100755 --- a/dev/test/test_workflow/test_chip.sh +++ b/dev/test/test_workflow/test_chip.sh @@ -8,7 +8,7 @@ fi if [ $# -gt 2 ]; then DOCKER_IMAGE=$3 else - DOCKER_IMAGE=quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 + DOCKER_IMAGE=quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 fi INPUT=$1 GCLOUD_SERVICE_ACCOUNT_SECRET_JSON_FILE=$2 diff --git a/dev/workflow_opts/docker.json b/dev/workflow_opts/docker.json index 84743551..80739e71 100644 --- a/dev/workflow_opts/docker.json +++ b/dev/workflow_opts/docker.json @@ -1,6 +1,6 @@ { "default_runtime_attributes" : { - "docker" : "quay.io/encode-dcc/chip-seq-pipeline:v1.3.0", + "docker" : "quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1", "zones": "us-west1-a us-west1-b us-west1-c us-central1-c us-central1-b", "failOnStderr" : false, "continueOnReturnCode" : 0, diff --git a/dev/workflow_opts/scg.json b/dev/workflow_opts/scg.json index 8f3a645d..9e7bbf92 100644 --- a/dev/workflow_opts/scg.json +++ b/dev/workflow_opts/scg.json @@ -1,7 +1,7 @@ { "default_runtime_attributes" : { "slurm_account" : "YOUR_SLURM_ACCOUNT", - "singularity_container" : "/reference/ENCODE/pipeline_singularity_images/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "/reference/ENCODE/pipeline_singularity_images/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/reference/ENCODE,/scratch,/srv/gsfs0" } } diff --git a/dev/workflow_opts/sge.json b/dev/workflow_opts/sge.json index d6edf792..d2e74cb1 100644 --- a/dev/workflow_opts/sge.json +++ b/dev/workflow_opts/sge.json @@ -1,6 +1,6 @@ { "default_runtime_attributes" : { "sge_pe" : "shm", - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg" + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg" } } diff --git a/dev/workflow_opts/sherlock.json b/dev/workflow_opts/sherlock.json index 06853a92..ed8ffec8 100644 --- a/dev/workflow_opts/sherlock.json +++ b/dev/workflow_opts/sherlock.json @@ -1,7 +1,7 @@ { "default_runtime_attributes" : { "slurm_partition" : "normal", - "singularity_container" : "/home/groups/cherry/encode/pipeline_singularity_images/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "/home/groups/cherry/encode/pipeline_singularity_images/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/scratch,/lscratch,/oak/stanford,/home/groups/cherry/encode" } } diff --git a/dev/workflow_opts/singularity.json b/dev/workflow_opts/singularity.json index 824177a5..287f2d9e 100644 --- a/dev/workflow_opts/singularity.json +++ b/dev/workflow_opts/singularity.json @@ -1,5 +1,5 @@ { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg" + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg" } } diff --git a/dev/workflow_opts/slurm.json b/dev/workflow_opts/slurm.json index e9f34bd6..f45d7d90 100644 --- a/dev/workflow_opts/slurm.json +++ b/dev/workflow_opts/slurm.json @@ -2,6 +2,6 @@ "default_runtime_attributes" : { "slurm_partition" : "YOUR_SLURM_PARTITION", "slurm_account" : "YOUR_SLURM_ACCOUNT", - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg" + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg" } } diff --git a/docs/deprecated/tutorial_local_singularity.md b/docs/deprecated/tutorial_local_singularity.md index 072f9f09..fd46593c 100644 --- a/docs/deprecated/tutorial_local_singularity.md +++ b/docs/deprecated/tutorial_local_singularity.md @@ -33,7 +33,7 @@ 6. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.0.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 ``` 7. Run a pipeline for the test sample. @@ -53,7 +53,7 @@ ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR2,..." } } diff --git a/docs/deprecated/tutorial_scg.md b/docs/deprecated/tutorial_scg.md index d48826fb..569b7848 100644 --- a/docs/deprecated/tutorial_scg.md +++ b/docs/deprecated/tutorial_scg.md @@ -63,7 +63,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/reference/ENCODE,/scratch,/srv/gsfs0,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR1,..." } } diff --git a/docs/deprecated/tutorial_scg_backend.md b/docs/deprecated/tutorial_scg_backend.md index 63706cd5..ed991779 100644 --- a/docs/deprecated/tutorial_scg_backend.md +++ b/docs/deprecated/tutorial_scg_backend.md @@ -58,7 +58,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 5. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash $ sdev # SCG cluster does not allow building a container on login node - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.0.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 $ exit ``` @@ -77,7 +77,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/scratch/users,/srv/gsfs0,/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR1,..." } } diff --git a/docs/deprecated/tutorial_sge.md b/docs/deprecated/tutorial_sge.md index e5f261a1..1a31835e 100644 --- a/docs/deprecated/tutorial_sge.md +++ b/docs/deprecated/tutorial_sge.md @@ -61,7 +61,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 7. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.0.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 ``` 8. Run a pipeline for the test sample. If your parallel environment (PE) found from step 5) has a different name from `shm` then edit the following shell script to change the PE name. @@ -83,7 +83,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR2,..." } } diff --git a/docs/deprecated/tutorial_sge_backend.md b/docs/deprecated/tutorial_sge_backend.md index ffca7e49..2a3f3696 100644 --- a/docs/deprecated/tutorial_sge_backend.md +++ b/docs/deprecated/tutorial_sge_backend.md @@ -68,7 +68,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 7. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.0.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.1 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.1 ``` 8. Run a pipeline for the test sample. diff --git a/docs/deprecated/tutorial_sherlock.md b/docs/deprecated/tutorial_sherlock.md index 8dc65609..b877bf0d 100644 --- a/docs/deprecated/tutorial_sherlock.md +++ b/docs/deprecated/tutorial_sherlock.md @@ -68,7 +68,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/scratch,/lscratch,/oak/stanford,/home/groups/cherry/encode,/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR1,..." } } diff --git a/docs/deprecated/tutorial_sherlock_backend.md b/docs/deprecated/tutorial_sherlock_backend.md index aa94d9af..6eed70df 100644 --- a/docs/deprecated/tutorial_sherlock_backend.md +++ b/docs/deprecated/tutorial_sherlock_backend.md @@ -63,7 +63,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 6. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. Stanford Sherlock does not allow building a container on login nodes. Wait until you get a command prompt after `sdev`. ```bash $ sdev # sherlock cluster does not allow building a container on login node - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.0.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 $ exit # exit from an interactive node ``` @@ -82,7 +82,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/scratch,/oak/stanford,/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR1,..." } } diff --git a/docs/deprecated/tutorial_slurm.md b/docs/deprecated/tutorial_slurm.md index 6f75f5b7..c72b5c37 100644 --- a/docs/deprecated/tutorial_slurm.md +++ b/docs/deprecated/tutorial_slurm.md @@ -56,7 +56,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 7. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.0.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 ``` 8. Run a pipeline for the test sample. If your cluster requires to specify any of them then add one to the command line. @@ -78,7 +78,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR2,..." } } diff --git a/docs/deprecated/tutorial_slurm_backend.md b/docs/deprecated/tutorial_slurm_backend.md index c0804e53..e7625ac4 100644 --- a/docs/deprecated/tutorial_slurm_backend.md +++ b/docs/deprecated/tutorial_slurm_backend.md @@ -68,7 +68,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 7. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.0.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.0 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 ``` 8. Run a pipeline for the test sample. @@ -86,7 +86,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.0.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", "singularity_bindpath" : "/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR2,..." } } diff --git a/docs/tutorial_dx_web.md b/docs/tutorial_dx_web.md index 0c01fb23..61e9bb87 100644 --- a/docs/tutorial_dx_web.md +++ b/docs/tutorial_dx_web.md @@ -15,8 +15,8 @@ This document describes instruction for the item 2). 3. Move to one of the following workflow directories according to the platform you have chosen for your project (AWS or Azure). These DX workflows are pre-built with all parameters defined. -* [AWS test workflow](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.0/test_ENCSR936XTK_subsampled_chr19_only) -* [Azure test workflow](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.0/test_ENCSR936XTK_subsampled_chr19_only) +* [AWS test workflow](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/test_ENCSR936XTK_subsampled_chr19_only) +* [Azure test workflow](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/test_ENCSR936XTK_subsampled_chr19_only) 4. Copy it to your project by right-clicking on the DX workflow `chip` and choose "Copy". @@ -40,16 +40,16 @@ This document describes instruction for the item 2). 1. DNAnexus allows only one copy of a workflow per project. The example workflow in the previous section is pre-built for the subsampled test sample [ENCSR936XTK](https://www.encodeproject.org/experiments/ENCSR936XTK/) with all parameters defined already. 2. Copy one of the following workflows according to the platform you have chosen for your project (AWS or Azure). -* [AWS general](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.0/general) without pre-defined reference genome. -* [AWS hg38](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.0/hg38) with pre-defined hg38 reference genome. -* [AWS hg19](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.0/hg19) with pre-defined hg19 reference genome. -* [AWS mm10](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.0/mm10) with pre-defined mm10 reference genome. -* [AWS mm9](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.0/mm9) with pre-defined mm9 reference genome. -* [Azure general](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.0/general) without pre-defined reference genome. -* [Azure hg38](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.0/hg38) with pre-defined hg38 reference genome. -* [Azure hg19](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.0/hg19) with pre-defined hg19 reference genome. -* [Azure mm10](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.0/mm10) with pre-defined mm10 reference genome. -* [Azure mm9](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.0/mm9) with pre-defined mm9 reference genome. +* [AWS general](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/general) without pre-defined reference genome. +* [AWS hg38](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/hg38) with pre-defined hg38 reference genome. +* [AWS hg19](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/hg19) with pre-defined hg19 reference genome. +* [AWS mm10](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/mm10) with pre-defined mm10 reference genome. +* [AWS mm9](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/mm9) with pre-defined mm9 reference genome. +* [Azure general](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/general) without pre-defined reference genome. +* [Azure hg38](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/hg38) with pre-defined hg38 reference genome. +* [Azure hg19](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/hg19) with pre-defined hg19 reference genome. +* [Azure mm10](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/mm10) with pre-defined mm10 reference genome. +* [Azure mm9](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/mm9) with pre-defined mm9 reference genome. 3. Click on the DX workflow `chip`. From 2aa9f0db56643caf80035fcb38375829e076d01c Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Thu, 10 Oct 2019 09:55:07 -0700 Subject: [PATCH 10/18] dx: organize example input JSONs --- dev/dev.md | 34 +++++++++--------- example_input_json/dx/ENCSR000DYI_dx.json | 19 ++++++++++ .../ENCSR000DYI_subsampled_chr19_only_dx.json | 18 ++++++++++ ...R000DYI_subsampled_chr19_only_rep1_dx.json | 14 ++++++++ example_input_json/dx/ENCSR936XTK_dx.json | 35 +++++++++++++++++++ example_input_json/dx/template_general.json | 3 ++ example_input_json/dx/template_hg19.json | 4 +++ example_input_json/dx/template_hg38.json | 4 +++ example_input_json/dx/template_mm10.json | 4 +++ example_input_json/dx/template_mm9.json | 4 +++ .../dx_azure/ENCSR000DYI_dx_azure.json | 19 ++++++++++ ...000DYI_subsampled_chr19_only_dx_azure.json | 18 ++++++++++ .../dx_azure/ENCSR936XTK_dx_azure.json | 35 +++++++++++++++++++ .../dx_azure/template_general.json | 3 ++ .../dx_azure/template_hg19.json | 4 +++ .../dx_azure/template_hg38.json | 4 +++ .../dx_azure/template_mm10.json | 4 +++ example_input_json/dx_azure/template_mm9.json | 4 +++ 18 files changed, 213 insertions(+), 17 deletions(-) create mode 100644 example_input_json/dx/ENCSR000DYI_dx.json create mode 100644 example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_dx.json create mode 100644 example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_rep1_dx.json create mode 100644 example_input_json/dx/ENCSR936XTK_dx.json create mode 100644 example_input_json/dx/template_general.json create mode 100644 example_input_json/dx/template_hg19.json create mode 100644 example_input_json/dx/template_hg38.json create mode 100644 example_input_json/dx/template_mm10.json create mode 100644 example_input_json/dx/template_mm9.json create mode 100644 example_input_json/dx_azure/ENCSR000DYI_dx_azure.json create mode 100644 example_input_json/dx_azure/ENCSR000DYI_subsampled_chr19_only_dx_azure.json create mode 100644 example_input_json/dx_azure/ENCSR936XTK_dx_azure.json create mode 100644 example_input_json/dx_azure/template_general.json create mode 100644 example_input_json/dx_azure/template_hg19.json create mode 100644 example_input_json/dx_azure/template_hg38.json create mode 100644 example_input_json/dx_azure/template_mm10.json create mode 100644 example_input_json/dx_azure/template_mm9.json diff --git a/dev/dev.md b/dev/dev.md index 21b232a4..86c79be8 100644 --- a/dev/dev.md +++ b/dev/dev.md @@ -28,55 +28,55 @@ VER=dev-v1.3.1 DOCKER=quay.io/encode-dcc/chip-seq-pipeline:$VER # general -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/general -defaults dev/examples/dx/template_general.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/general -defaults example_input_json/dx/template_general.json # hg38 -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/hg38 -defaults dev/examples/dx/template_hg38.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/hg38 -defaults example_input_json/dx/template_hg38.json # hg19 -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/hg19 -defaults dev/examples/dx/template_hg19.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/hg19 -defaults example_input_json/dx/template_hg19.json # mm10 -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/mm10 -defaults dev/examples/dx/template_mm10.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/mm10 -defaults example_input_json/dx/template_mm10.json # mm9 -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/mm9 -defaults dev/examples/dx/template_mm9.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/mm9 -defaults example_input_json/dx/template_mm9.json # test sample PE ENCSR936XTK (full) -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR936XTK -defaults dev/examples/dx/ENCSR936XTK_dx.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR936XTK -defaults example_input_json/dx/ENCSR936XTK_dx.json # test sample SE ENCSR000DYI (full) -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI -defaults dev/examples/dx/ENCSR000DYI_dx.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI -defaults example_input_json/dx/ENCSR000DYI_dx.json # test sample SE ENCSR000DYI (subsampled, chr19/chrM only) -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI_subsampled_chr19_only -defaults dev/examples/dx/ENCSR000DYI_subsampled_chr19_only_dx.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI_subsampled_chr19_only -defaults example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_dx.json # test sample SE ENCSR000DYI (subsampled, chr19/chrM only, rep1) -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI_subsampled_chr19_only_rep1 -defaults dev/examples/dx/ENCSR000DYI_subsampled_chr19_only_rep1_dx.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI_subsampled_chr19_only_rep1 -defaults example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_rep1_dx.json ## DX Azure # general -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/general -defaults dev/examples/dx_azure/template_general.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/general -defaults example_input_json/dx_azure/template_general.json # hg38 -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/hg38 -defaults dev/examples/dx_azure/template_hg38.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/hg38 -defaults example_input_json/dx_azure/template_hg38.json # hg19 -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/hg19 -defaults dev/examples/dx_azure/template_hg19.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/hg19 -defaults example_input_json/dx_azure/template_hg19.json # mm10 -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/mm10 -defaults dev/examples/dx_azure/template_mm10.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/mm10 -defaults example_input_json/dx_azure/template_mm10.json # mm9 -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/mm9 -defaults dev/examples/dx_azure/template_mm9.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/mm9 -defaults example_input_json/dx_azure/template_mm9.json # test sample PE ENCSR936XTK (full) -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR936XTK -defaults dev/examples/dx_azure/ENCSR936XTK_dx_azure.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR936XTK -defaults example_input_json/dx_azure/ENCSR936XTK_dx_azure.json # test sample SE ENCSR000DYI (full) -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI -defaults dev/examples/dx_azure/ENCSR000DYI_dx_azure.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI -defaults example_input_json/dx_azure/ENCSR000DYI_dx_azure.json # test sample SE ENCSR000DYI (subsampled, chr19/chrM only) -java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI_subsampled_chr19_only -defaults dev/examples/dx_azure/ENCSR000DYI_subsampled_chr19_only_dx_azure.json +java -jar ~/dxWDL-0.79.1.jar compile chip.wdl -project "ENCODE Uniform Processing Pipelines Azure" -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") -f -folder /ChIP-seq2/workflows/$VER/test_ENCSR000DYI_subsampled_chr19_only -defaults example_input_json/dx_azure/ENCSR000DYI_subsampled_chr19_only_dx_azure.json ``` diff --git a/example_input_json/dx/ENCSR000DYI_dx.json b/example_input_json/dx/ENCSR000DYI_dx.json new file mode 100644 index 00000000..05220eaa --- /dev/null +++ b/example_input_json/dx/ENCSR000DYI_dx.json @@ -0,0 +1,19 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-genome-data/hg38_dx.tsv", + "chip.fastqs_rep1_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq/rep1.fastq.gz" + ], + "chip.fastqs_rep2_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq/rep2.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq/ctl1.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq/ctl2.fastq.gz" + ], + + "chip.paired_end" : false, + "chip.always_use_pooled_ctl" : true, + "chip.spp_cpu" : 2, + + "chip.title" : "ENCSR000DYI", + "chip.description" : "CEBPB ChIP-seq on human A549 produced by the Snyder lab" +} diff --git a/example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_dx.json b/example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_dx.json new file mode 100644 index 00000000..11829311 --- /dev/null +++ b/example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_dx.json @@ -0,0 +1,18 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-genome-data/hg38_chr19_chrM_dx.tsv", + "chip.fastqs_rep1_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/rep1.subsampled.25.fastq.gz" + ], + "chip.fastqs_rep2_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/rep2.subsampled.20.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/ctl1.subsampled.25.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/ctl2.subsampled.25.fastq.gz" + ], + + "chip.paired_end" : false, + "chip.always_use_pooled_ctl" : true, + + "chip.title" : "ENCSR000DYI (subsampled 1/25, chr19 and chrM only)", + "chip.description" : "CEBPB ChIP-seq on human A549 produced by the Snyder lab" +} diff --git a/example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_rep1_dx.json b/example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_rep1_dx.json new file mode 100644 index 00000000..9d06d40e --- /dev/null +++ b/example_input_json/dx/ENCSR000DYI_subsampled_chr19_only_rep1_dx.json @@ -0,0 +1,14 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-genome-data/hg38_chr19_chrM_dx.tsv", + "chip.fastqs_rep1_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/rep1.subsampled.25.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R1" : ["dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/ctl1.subsampled.25.fastq.gz" + ], + + "chip.paired_end" : false, + "chip.always_use_pooled_ctl" : true, + + "chip.title" : "ENCSR000DYI (subsampled 1/25, chr19 and chrM only)", + "chip.description" : "CEBPB ChIP-seq on human A549 produced by the Snyder lab" +} diff --git a/example_input_json/dx/ENCSR936XTK_dx.json b/example_input_json/dx/ENCSR936XTK_dx.json new file mode 100644 index 00000000..c6745f97 --- /dev/null +++ b/example_input_json/dx/ENCSR936XTK_dx.json @@ -0,0 +1,35 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-genome-data/hg38_dx.tsv", + + "chip.fastqs_rep1_R1" : [ + "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/rep1-R1.fastq.gz" + ], + "chip.fastqs_rep1_R2" : [ + "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/rep1-R2.fastq.gz" + ], + "chip.fastqs_rep2_R1" : [ + "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/rep2-R1.fastq.gz" + ], + "chip.fastqs_rep2_R2" : [ + "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/rep2-R2.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R1" : [ + "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/ctl1-R1.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R2" : [ + "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/ctl1-R2.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R1" : [ + "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/ctl2-R1.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R2" : [ + "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/ctl2-R2.fastq.gz" + ], + + "chip.paired_end" : true, + + "chip.always_use_pooled_ctl" : true, + "chip.title" : "ENCSR936XTK", + "chip.description" : "ZNF143 ChIP-seq on human GM12878" +} diff --git a/example_input_json/dx/template_general.json b/example_input_json/dx/template_general.json new file mode 100644 index 00000000..dd070109 --- /dev/null +++ b/example_input_json/dx/template_general.json @@ -0,0 +1,3 @@ +{ + "chip.pipeline_type" : "tf" +} diff --git a/example_input_json/dx/template_hg19.json b/example_input_json/dx/template_hg19.json new file mode 100644 index 00000000..b0a157c5 --- /dev/null +++ b/example_input_json/dx/template_hg19.json @@ -0,0 +1,4 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-genome-data/hg19_dx.tsv" +} diff --git a/example_input_json/dx/template_hg38.json b/example_input_json/dx/template_hg38.json new file mode 100644 index 00000000..84e98a32 --- /dev/null +++ b/example_input_json/dx/template_hg38.json @@ -0,0 +1,4 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-genome-data/hg38_dx.tsv" +} diff --git a/example_input_json/dx/template_mm10.json b/example_input_json/dx/template_mm10.json new file mode 100644 index 00000000..5e57f32e --- /dev/null +++ b/example_input_json/dx/template_mm10.json @@ -0,0 +1,4 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-genome-data/mm10_dx.tsv" +} diff --git a/example_input_json/dx/template_mm9.json b/example_input_json/dx/template_mm9.json new file mode 100644 index 00000000..9930dd31 --- /dev/null +++ b/example_input_json/dx/template_mm9.json @@ -0,0 +1,4 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-BKpvFg00VBPV975PgJ6Q03v6:pipeline-genome-data/mm9_dx.tsv" +} diff --git a/example_input_json/dx_azure/ENCSR000DYI_dx_azure.json b/example_input_json/dx_azure/ENCSR000DYI_dx_azure.json new file mode 100644 index 00000000..9113fe48 --- /dev/null +++ b/example_input_json/dx_azure/ENCSR000DYI_dx_azure.json @@ -0,0 +1,19 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-genome-data/hg38_dx_azure.tsv", + "chip.fastqs_rep1_R1" : ["dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq/rep1.fastq.gz" + ], + "chip.fastqs_rep2_R1" : ["dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq/rep2.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R1" : ["dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq/ctl1.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R1" : ["dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq/ctl2.fastq.gz" + ], + + "chip.paired_end" : false, + "chip.always_use_pooled_ctl" : true, + "chip.spp_cpu" : 2, + + "chip.title" : "ENCSR000DYI", + "chip.description" : "CEBPB ChIP-seq on human A549 produced by the Snyder lab" +} diff --git a/example_input_json/dx_azure/ENCSR000DYI_subsampled_chr19_only_dx_azure.json b/example_input_json/dx_azure/ENCSR000DYI_subsampled_chr19_only_dx_azure.json new file mode 100644 index 00000000..1767227e --- /dev/null +++ b/example_input_json/dx_azure/ENCSR000DYI_subsampled_chr19_only_dx_azure.json @@ -0,0 +1,18 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-genome-data/hg38_chr19_chrM_dx_azure.tsv", + "chip.fastqs_rep1_R1" : ["dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/rep1.subsampled.25.fastq.gz" + ], + "chip.fastqs_rep2_R1" : ["dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/rep2.subsampled.15.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R1" : ["dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/ctl1.subsampled.25.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R1" : ["dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/ctl2.subsampled.25.fastq.gz" + ], + + "chip.paired_end" : false, + "chip.always_use_pooled_ctl" : true, + + "chip.title" : "ENCSR000DYI (subsampled 1/25, chr19/chrM only)", + "chip.description" : "CEBPB ChIP-seq on human A549 produced by the Snyder lab" +} diff --git a/example_input_json/dx_azure/ENCSR936XTK_dx_azure.json b/example_input_json/dx_azure/ENCSR936XTK_dx_azure.json new file mode 100644 index 00000000..d77bde21 --- /dev/null +++ b/example_input_json/dx_azure/ENCSR936XTK_dx_azure.json @@ -0,0 +1,35 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-genome-data/hg38_dx_azure.tsv", + + "chip.fastqs_rep1_R1" : [ + "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/rep1-R1.fastq.gz" + ], + "chip.fastqs_rep1_R2" : [ + "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/rep1-R2.fastq.gz" + ], + "chip.fastqs_rep2_R1" : [ + "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/rep2-R1.fastq.gz" + ], + "chip.fastqs_rep2_R2" : [ + "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/rep2-R2.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R1" : [ + "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/ctl1-R1.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R2" : [ + "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/ctl1-R2.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R1" : [ + "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/ctl2-R1.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R2" : [ + "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq/ctl2-R2.fastq.gz" + ], + + "chip.paired_end" : true, + + "chip.always_use_pooled_ctl" : true, + "chip.title" : "ENCSR936XTK", + "chip.description" : "ZNF143 ChIP-seq on human GM12878" +} diff --git a/example_input_json/dx_azure/template_general.json b/example_input_json/dx_azure/template_general.json new file mode 100644 index 00000000..dd070109 --- /dev/null +++ b/example_input_json/dx_azure/template_general.json @@ -0,0 +1,3 @@ +{ + "chip.pipeline_type" : "tf" +} diff --git a/example_input_json/dx_azure/template_hg19.json b/example_input_json/dx_azure/template_hg19.json new file mode 100644 index 00000000..ed6de33c --- /dev/null +++ b/example_input_json/dx_azure/template_hg19.json @@ -0,0 +1,4 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-genome-data/hg19_dx_azure.tsv" +} diff --git a/example_input_json/dx_azure/template_hg38.json b/example_input_json/dx_azure/template_hg38.json new file mode 100644 index 00000000..4b97929f --- /dev/null +++ b/example_input_json/dx_azure/template_hg38.json @@ -0,0 +1,4 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-genome-data/hg38_dx_azure.tsv" +} diff --git a/example_input_json/dx_azure/template_mm10.json b/example_input_json/dx_azure/template_mm10.json new file mode 100644 index 00000000..c285bfb2 --- /dev/null +++ b/example_input_json/dx_azure/template_mm10.json @@ -0,0 +1,4 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-genome-data/mm10_dx_azure.tsv" +} diff --git a/example_input_json/dx_azure/template_mm9.json b/example_input_json/dx_azure/template_mm9.json new file mode 100644 index 00000000..af735323 --- /dev/null +++ b/example_input_json/dx_azure/template_mm9.json @@ -0,0 +1,4 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "dx://project-F6K911Q9xyfgJ36JFzv03Z5J:pipeline-genome-data/mm9_dx_azure.tsv" +} From 362275e62abf4364cfb82de004cc839bab232a33 Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Thu, 10 Oct 2019 11:20:28 -0700 Subject: [PATCH 11/18] doc: fix dx path --- docs/tutorial_dx_cli.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/tutorial_dx_cli.md b/docs/tutorial_dx_cli.md index 2784eb5b..f1e0f2fe 100644 --- a/docs/tutorial_dx_cli.md +++ b/docs/tutorial_dx_cli.md @@ -45,8 +45,9 @@ This document describes instruction for the item 1). ```bash $ PROJECT=[YOUR_PROJECT_NAME] $ OUT_FOLDER=/test_sample_chip_ENCSR936XTK_subsampled_chr19_only + $ DOCKER=quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 - $ java -jar dxWDL-0.77.jar compile chip.wdl -project ${PROJECT} -f -folder ${OUT_FOLDER} -defaults ${INPUT} -extras dev/workflow_opts/docker.json + $ java -jar dxWDL-0.77.jar compile chip.wdl -project ${PROJECT} -f -folder ${OUT_FOLDER} -defaults ${INPUT} -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") ``` 8. Go to DNAnexus [project page](https://platform.DNAnexus.com/projects) and click on your project. From 367febf10d8e59462378177a2b54e28defaefdc1 Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Thu, 10 Oct 2019 11:20:58 -0700 Subject: [PATCH 12/18] circleci: update docker cache --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9669266..0f5cdcf3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,7 +51,7 @@ jobs: name: build image command: | source ${BASH_ENV} - export DOCKER_CACHE_TAG=new-test-v1.3.0 + export DOCKER_CACHE_TAG=dev-v1.3.1 echo "pulling ${DOCKER_CACHE_TAG}!" docker pull quay.io/encode-dcc/chip-seq-pipeline:${DOCKER_CACHE_TAG} docker login -u=${QUAY_ROBOT_USER} -p=${QUAY_ROBOT_USER_TOKEN} quay.io From c4c49171056f999e6fdcc8d9e36a532babcbd5fa Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Thu, 10 Oct 2019 19:32:00 -0700 Subject: [PATCH 13/18] align: can use .tar.gz as aligner index --- src/encode_lib_common.py | 13 ++++++++++--- src/encode_task_bowtie2.py | 3 ++- src/encode_task_bwa.py | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/encode_lib_common.py b/src/encode_lib_common.py index 3af6a263..bd7d412d 100755 --- a/src/encode_lib_common.py +++ b/src/encode_lib_common.py @@ -34,7 +34,7 @@ def strip_ext_bam(bam): def strip_ext_tar(tar): - return re.sub(r'\.tar$', '', str(tar)) + return re.sub(r'\.(tar|tar\.gz)$', '', str(tar)) def strip_ext_ta(ta): @@ -157,9 +157,16 @@ def mkdir_p(dirname): def untar(tar, out_dir): - cmd = 'tar xvf {} --no-same-owner -C {}'.format( + if tar.endswith('.gz'): + cmd = 'tar zxvf {} --no-same-owner -C {}'.format( + tar, + out_dir if out_dir else '.') + else: + cmd = 'tar xvf {} --no-same-owner -C {}' + cmd = cmd.format( tar, - out_dir if out_dir else '.') + out_dir if out_dir else '.' + ) run_shell_cmd(cmd) diff --git a/src/encode_task_bowtie2.py b/src/encode_task_bowtie2.py index aa0f56de..d46ae662 100755 --- a/src/encode_task_bowtie2.py +++ b/src/encode_task_bowtie2.py @@ -124,7 +124,8 @@ def main(): temp_files = [] # files to deleted later at the end # if bowtie2 index is tarball then unpack it - if args.bowtie2_index_prefix_or_tar.endswith('.tar'): + if args.bowtie2_index_prefix_or_tar.endswith('.tar') or \ + args.bowtie2_index_prefix_or_tar.endswith('.tar.gz'): log.info('Unpacking bowtie2 index tar...') tar = args.bowtie2_index_prefix_or_tar # untar diff --git a/src/encode_task_bwa.py b/src/encode_task_bwa.py index b9119fdd..18a36a4e 100755 --- a/src/encode_task_bwa.py +++ b/src/encode_task_bwa.py @@ -166,7 +166,8 @@ def main(): temp_files = [] # files to deleted later at the end # if bwa index is tarball then unpack it - if args.bwa_index_prefix_or_tar.endswith('.tar'): + if args.bwa_index_prefix_or_tar.endswith('.tar') or \ + args.bwa_index_prefix_or_tar.endswith('.tar.gz'): log.info('Unpacking bwa index tar...') tar = args.bwa_index_prefix_or_tar # untar From f3b98c0f533af058743eff87fdb9d985edcce0ba Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Fri, 11 Oct 2019 13:55:57 -0700 Subject: [PATCH 14/18] bug fix: completely remove file-linkg (soft/hard) from pipeline. fixes issue #91 --- src/encode_task_merge_fastq.py | 8 ++++---- src/encode_task_reproducibility.py | 8 ++++---- src/encode_task_trim_adapter.py | 25 ++++++++----------------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/encode_task_merge_fastq.py b/src/encode_task_merge_fastq.py index 8d82cd94..6a8ec315 100755 --- a/src/encode_task_merge_fastq.py +++ b/src/encode_task_merge_fastq.py @@ -7,7 +7,7 @@ import os import argparse from encode_lib_common import ( - soft_link, log, ls_l, mkdir_p, read_tsv, run_shell_cmd, + copy_f_to_f, log, ls_l, mkdir_p, read_tsv, run_shell_cmd, strip_ext_fastq) @@ -52,10 +52,10 @@ def parse_arguments(debug=False): log.info(sys.argv) return args -# make merged fastqs on $out_dir/R1, $out_dir/R2 - def merge_fastqs(fastqs, end, out_dir): + """make merged fastqs on $out_dir/R1, $out_dir/R2 + """ out_dir = os.path.join(out_dir, end) mkdir_p(out_dir) prefix = os.path.join(out_dir, @@ -69,7 +69,7 @@ def merge_fastqs(fastqs, end, out_dir): run_shell_cmd(cmd) return merged else: - return soft_link(fastqs[0], merged) + return copy_f_to_f(fastqs[0], merged) def main(): diff --git a/src/encode_task_reproducibility.py b/src/encode_task_reproducibility.py index 243da8e5..ea4f911d 100755 --- a/src/encode_task_reproducibility.py +++ b/src/encode_task_reproducibility.py @@ -8,7 +8,7 @@ import argparse from encode_lib_common import ( copy_f_to_f, get_num_lines, infer_n_from_nC2, - infer_pair_label_from_idx, log, make_soft_link, mkdir_p) + infer_pair_label_from_idx, log, mkdir_p) from encode_lib_genomic import ( peak_to_bigbed, peak_to_hammock, get_region_size_metrics, get_num_peaks) @@ -82,7 +82,7 @@ def main(): label_tr = infer_pair_label_from_idx(num_rep, Nt_idx) conservative_set = label_tr - conservative_peak = make_soft_link(args.peaks[Nt_idx], args.out_dir) + conservative_peak = args.peaks[Nt_idx] N_conservative = Nt if Nt > Np: optimal_set = conservative_set @@ -90,7 +90,7 @@ def main(): N_optimal = N_conservative else: optimal_set = "pooled-pr1_vs_pooled-pr2" - optimal_peak = make_soft_link(args.peak_ppr, args.out_dir) + optimal_peak = args.peak_ppr N_optimal = Np else: # single replicate case @@ -102,7 +102,7 @@ def main(): self_consistency_ratio = 1.0 conservative_set = 'rep1-pr1_vs_rep1-pr2' - conservative_peak = make_soft_link(args.peaks_pr[0], args.out_dir) + conservative_peak = args.peaks_pr[0] N_conservative = N[0] optimal_set = conservative_set optimal_peak = conservative_peak diff --git a/src/encode_task_trim_adapter.py b/src/encode_task_trim_adapter.py index 8638e284..96812407 100755 --- a/src/encode_task_trim_adapter.py +++ b/src/encode_task_trim_adapter.py @@ -9,7 +9,7 @@ import copy from detect_adapter import detect_most_likely_adapter from encode_lib_common import ( - hard_link, log, ls_l, mkdir_p, read_tsv, rm_f, + copy_f_to_dir, copy_f_to_f, log, ls_l, mkdir_p, read_tsv, rm_f, run_shell_cmd, strip_ext_fastq) @@ -106,11 +106,7 @@ def trim_adapter_se(fastq, adapter, adapter_for_all, cutadapt_param, out_dir): run_shell_cmd(cmd) return trimmed else: - # make hard link - linked = os.path.join(out_dir, - os.path.basename(fastq)) - os.link(fastq, linked) - return linked + return copy_f_to_dir(fastq, out_dir) def trim_adapter_pe(fastq1, fastq2, adapter1, adapter2, adapter_for_all, @@ -132,19 +128,14 @@ def trim_adapter_pe(fastq1, fastq2, adapter1, adapter2, adapter_for_all, run_shell_cmd(cmd) return [trimmed1, trimmed2] else: - # make hard link - linked1 = os.path.join(out_dir, - os.path.basename(fastq1)) - linked2 = os.path.join(out_dir, - os.path.basename(fastq2)) - os.link(fastq1, linked1) - os.link(fastq2, linked2) - return [linked1, linked2] - -# make merged fastqs on $out_dir/R1, $out_dir/R2 + fq1 = copy_f_to_dir(fastq1, out_dir) + fq2 = copy_f_to_dir(fastq2, out_dir) + return [fq1, fq2] def merge_fastqs(fastqs, end, out_dir): + """make merged fastqs on $out_dir/R1, $out_dir/R2 + """ out_dir = os.path.join(out_dir, end) mkdir_p(out_dir) prefix = os.path.join(out_dir, @@ -158,7 +149,7 @@ def merge_fastqs(fastqs, end, out_dir): run_shell_cmd(cmd) return merged else: - return hard_link(fastqs[0], merged) + return copy_f_to_f(fastqs[0], merged) def main(): From 52159c541323046d888be9398bae203ea573061b Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Fri, 11 Oct 2019 17:47:07 -0700 Subject: [PATCH 15/18] combine tasks (merge_fastq + trim_fastq + align) not to generate unnecessary intermediate outputs --- chip.wdl | 170 ++++++++--------------- dev/test/test_task/test_bowtie2.json | 7 +- dev/test/test_task/test_bowtie2.wdl | 12 +- dev/test/test_task/test_bwa.json | 7 +- dev/test/test_task/test_bwa.wdl | 12 +- dev/test/test_task/test_merge_fastq.json | 17 --- dev/test/test_task/test_merge_fastq.wdl | 44 ------ dev/test/test_task/test_trim_fastq.json | 5 - dev/test/test_task/test_trim_fastq.wdl | 28 ---- docs/input.md | 2 +- 10 files changed, 82 insertions(+), 222 deletions(-) delete mode 100644 dev/test/test_task/test_merge_fastq.json delete mode 100644 dev/test/test_task/test_merge_fastq.wdl delete mode 100644 dev/test/test_task/test_trim_fastq.json delete mode 100644 dev/test/test_task/test_trim_fastq.wdl diff --git a/chip.wdl b/chip.wdl index c30d88e5..eefa0363 100644 --- a/chip.wdl +++ b/chip.wdl @@ -180,10 +180,6 @@ workflow chip { Array[File] ctl_fastqs_rep10_R2 = [] ### other input types (bam, nodup_bam, ta) - Array[File?] merged_fastqs_R1 = [] - Array[File?] merged_fastqs_R2 = [] - Array[File?] ctl_merged_fastqs_R1 = [] - Array[File?] ctl_merged_fastqs_R2 = [] Array[File?] bams = [] # [rep_id] Array[File?] ctl_bams = [] # [rep_id] Array[File?] nodup_bams = [] # [rep_id] @@ -343,9 +339,7 @@ workflow chip { # temporary variables to get number of replicates # WDLic implementation of max(A,B,C,...) Int num_rep_fastq = length(fastqs_R1) - Int num_rep_merged_fastq = if length(merged_fastqs_R1)0 - Boolean has_output_of_merge_fastq = i0 Boolean has_output_of_align = i0 - Boolean has_output_of_merge_fastq_ctl = i0 Boolean has_output_of_align_ctl = i /dev/null || true)" ]] then echo -e "\n* Error: pipeline dependencies not found." 1>&2 - echo 'Conda users: Did you install Conda and environment correctly (scripts/install_conda_env.sh)?' 1>&2 + echo 'Conda users: Did you activate Conda environment (conda activate encode-chip-seq-pipeline)?' 1>&2 + echo ' Or did you install Conda and environment correctly (bash scripts/install_conda_env.sh)?' 1>&2 echo 'GCP/AWS/Docker users: Did you add --docker flag to Caper command line arg?' 1>&2 echo 'Singularity users: Did you add --singularity flag to Caper command line arg?' 1>&2 echo -e "\n" 1>&2 EXCEPTION_RAISED fi - python3 $(which encode_task_merge_fastq.py) \ ${write_tsv(tmp_fastqs)} \ ${if paired_end then '--paired-end' else ''} \ ${'--nth ' + 1} - } - output { - File merged_fastq_R1 = glob('R1/*.fastq.gz')[0] - File? merged_fastq_R2 = if paired_end then glob('R2/*.fastq.gz')[0] else null_f - } - runtime { - cpu : 1 - memory : '8000 MB' - time : 2 - disks : 'local-disk 100 HDD' - } -} -task trim_fastq { # trim fastq (for PE R1 only) - File fastq - Int trim_bp - - command { - python3 $(which encode_task_trim_fastq.py) \ - ${fastq} \ - --trim-bp ${trim_bp} - } - output { - File trimmed_fastq = glob('*.fastq.gz')[0] - } - runtime { - cpu : 1 - memory : '8000 MB' - time : 1 - disks : 'local-disk 50 HDD' - } -} - -task align { - String aligner - String mito_chr_name - Int? multimapping - File? custom_align_py - File? idx_tar # reference index tar - File? fastq_R1 # [read_end_id] - File? fastq_R2 - Boolean paired_end - Boolean use_bwa_mem_for_pe - - Int cpu - Int mem_mb - Int time_hr - String disks + if [ -z '${trim_bp}' ]; then + SUFFIX= + else + SUFFIX=_trimmed + python3 $(which encode_task_trim_fastq.py) \ + R1/*.fastq.gz \ + --trim-bp ${trim_bp} \ + --out-dir R1$SUFFIX + if [ '${paired_end}' == 'true' ]; then + python3 $(which encode_task_trim_fastq.py) \ + R2/*.fastq.gz \ + --trim-bp ${trim_bp} \ + --out-dir R2$SUFFIX + fi + fi - command { if [ '${aligner}' == 'bwa' ]; then python3 $(which encode_task_bwa.py) \ ${idx_tar} \ - ${fastq_R1} ${fastq_R2} \ + R1$SUFFIX/*.fastq.gz \ + ${if paired_end then 'R2$SUFFIX/*.fastq.gz' else ''} \ ${if paired_end then '--paired-end' else ''} \ ${if use_bwa_mem_for_pe then '--use-bwa-mem-for-pe' else ''} \ ${'--nth ' + cpu} @@ -1291,22 +1232,25 @@ task align { elif [ '${aligner}' == 'bowtie2' ]; then python3 $(which encode_task_bowtie2.py) \ ${idx_tar} \ - ${fastq_R1} ${fastq_R2} \ + R1$SUFFIX/*.fastq.gz \ + ${if paired_end then 'R2$SUFFIX/*.fastq.gz' else ''} \ ${'--multimapping ' + multimapping} \ ${if paired_end then '--paired-end' else ''} \ ${'--nth ' + cpu} else python3 ${custom_align_py} \ ${idx_tar} \ - ${fastq_R1} ${fastq_R2} \ + R1$SUFFIX/*.fastq.gz \ + ${if paired_end then 'R2$SUFFIX/*.fastq.gz' else ''} \ ${if paired_end then '--paired-end' else ''} \ ${'--nth ' + cpu} fi python3 $(which encode_task_post_align.py) \ - ${fastq_R1} $(ls *.bam) \ + R1$SUFFIX/*.fastq.gz $(ls *.bam) \ ${'--mito-chr-name ' + mito_chr_name} \ ${'--nth ' + cpu} + rm -rf R1 R2 R1$SUFFIX R2$SUFFIX } output { File bam = glob('*.bam')[0] diff --git a/dev/test/test_task/test_bowtie2.json b/dev/test/test_task/test_bowtie2.json index 60567735..71880476 100644 --- a/dev/test/test_task/test_bowtie2.json +++ b/dev/test/test_task/test_bowtie2.json @@ -2,13 +2,16 @@ "test_bowtie2.pe_bowtie2_idx_tar" : "chip-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/bowtie2_index/GRCh38_no_alt_analysis_set_GCA_000001405.15.chr19_chrM.fasta.tar", "test_bowtie2.se_bowtie2_idx_tar" : "chip-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/bowtie2_index/GRCh38_no_alt_analysis_set_GCA_000001405.15.chr19_chrM.fasta.tar", - "test_bowtie2.pe_fastqs" : [ - "chip-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/rep1-R1.subsampled.67.fastq.gz", + "test_bowtie2.pe_fastqs_R1" : [ + "chip-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/rep1-R1.subsampled.67.fastq.gz" + ], + "test_bowtie2.pe_fastqs_R2" : [ "chip-seq-pipeline-test-data/input/pe/fastqs/rep1/pair2/rep1-R2.subsampled.67.fastq.gz" ], "test_bowtie2.se_fastqs" : [ "chip-seq-pipeline-test-data/input/se/fastqs/rep1/rep1.subsampled.25.fastq.gz" ], + "test_bowtie2.ref_pe_flagstat" : "chip-seq-pipeline-test-data/ref_output/test_bowtie2/pe/rep1-R1.subsampled.67.samstats.qc", "test_bowtie2.ref_se_flagstat" : "chip-seq-pipeline-test-data/ref_output/test_bowtie2/se/rep1.subsampled.25.samstats.qc" } diff --git a/dev/test/test_task/test_bowtie2.wdl b/dev/test/test_task/test_bowtie2.wdl index 24526265..5cf7bc0c 100644 --- a/dev/test/test_task/test_bowtie2.wdl +++ b/dev/test/test_task/test_bowtie2.wdl @@ -4,8 +4,9 @@ import '../../../chip.wdl' as chip import 'compare_md5sum.wdl' as compare_md5sum workflow test_bowtie2 { - Array[String] pe_fastqs - Array[String] se_fastqs + Array[File] pe_fastqs_R1 + Array[File] pe_fastqs_R2 + Array[File] se_fastqs # we don't compare BAM because BAM's header includes date # hence md5sums don't match all the time @@ -24,8 +25,8 @@ workflow test_bowtie2 { aligner = 'bowtie2', idx_tar = pe_bowtie2_idx_tar, mito_chr_name = 'chrM', - fastq_R1 = pe_fastqs[0], - fastq_R2 = pe_fastqs[1], + fastqs_R1 = pe_fastqs_R1, + fastqs_R2 = pe_fastqs_R2, paired_end = true, use_bwa_mem_for_pe = false, @@ -38,7 +39,8 @@ workflow test_bowtie2 { aligner = 'bowtie2', idx_tar = se_bowtie2_idx_tar, mito_chr_name = 'chrM', - fastq_R1 = se_fastqs[0], + fastqs_R1 = se_fastqs, + fastqs_R2 = [], paired_end = false, use_bwa_mem_for_pe = false, diff --git a/dev/test/test_task/test_bwa.json b/dev/test/test_task/test_bwa.json index 25345e35..b5f15a2f 100644 --- a/dev/test/test_task/test_bwa.json +++ b/dev/test/test_task/test_bwa.json @@ -2,13 +2,16 @@ "test_bwa.pe_bwa_idx_tar" : "chip-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/bwa_index/GRCh38_no_alt_analysis_set_GCA_000001405.15.chr19_chrM.fasta.tar", "test_bwa.se_bwa_idx_tar" : "chip-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/bwa_index/GRCh38_no_alt_analysis_set_GCA_000001405.15.chr19_chrM.fasta.tar", - "test_bwa.pe_fastqs" : [ - "chip-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/rep1-R1.subsampled.67.fastq.gz", + "test_bwa.pe_fastqs_R1" : [ + "chip-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/rep1-R1.subsampled.67.fastq.gz" + ], + "test_bwa.pe_fastqs_R2" : [ "chip-seq-pipeline-test-data/input/pe/fastqs/rep1/pair2/rep1-R2.subsampled.67.fastq.gz" ], "test_bwa.se_fastqs" : [ "chip-seq-pipeline-test-data/input/se/fastqs/rep1/rep1.subsampled.25.fastq.gz" ], + "test_bwa.ref_pe_flagstat" : "chip-seq-pipeline-test-data/ref_output/test_bwa/pe/rep1-R1.subsampled.67.samstats.qc", "test_bwa.ref_se_flagstat" : "chip-seq-pipeline-test-data/ref_output/test_bwa/se/rep1.subsampled.25.samstats.qc" } diff --git a/dev/test/test_task/test_bwa.wdl b/dev/test/test_task/test_bwa.wdl index 0ba37a1f..5e0ecf52 100644 --- a/dev/test/test_task/test_bwa.wdl +++ b/dev/test/test_task/test_bwa.wdl @@ -4,8 +4,9 @@ import '../../../chip.wdl' as chip import 'compare_md5sum.wdl' as compare_md5sum workflow test_bwa { - Array[String] pe_fastqs - Array[String] se_fastqs + Array[File] pe_fastqs_R1 + Array[File] pe_fastqs_R2 + Array[File] se_fastqs # we don't compare BAM because BAM's header includes date # hence md5sums don't match all the time @@ -24,8 +25,8 @@ workflow test_bwa { aligner = 'bwa', idx_tar = pe_bwa_idx_tar, mito_chr_name = 'chrM', - fastq_R1 = pe_fastqs[0], - fastq_R2 = pe_fastqs[1], + fastqs_R1 = pe_fastqs_R1, + fastqs_R2 = pe_fastqs_R2, paired_end = true, use_bwa_mem_for_pe = false, @@ -38,7 +39,8 @@ workflow test_bwa { aligner = 'bwa', idx_tar = se_bwa_idx_tar, mito_chr_name = 'chrM', - fastq_R1 = se_fastqs[0], + fastqs_R1 = se_fastqs, + fastqs_R2 = [], paired_end = false, use_bwa_mem_for_pe = false, diff --git a/dev/test/test_task/test_merge_fastq.json b/dev/test/test_task/test_merge_fastq.json deleted file mode 100644 index 0859620e..00000000 --- a/dev/test/test_task/test_merge_fastq.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "test_merge_fastq.pe_fastqs_R1" : [ - "chip-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/rep1-R1.subsampled.67.fastq.gz", - "chip-seq-pipeline-test-data/input/pe/fastqs/rep2/pair1/rep2-R1.subsampled.67.fastq.gz" - ], - "test_merge_fastq.pe_fastqs_R2" : [ - "chip-seq-pipeline-test-data/input/pe/fastqs/rep1/pair2/rep1-R2.subsampled.67.fastq.gz", - "chip-seq-pipeline-test-data/input/pe/fastqs/rep2/pair2/rep2-R2.subsampled.67.fastq.gz" - ], - "test_merge_fastq.se_fastqs" : [ - "chip-seq-pipeline-test-data/input/se/fastqs/rep1/rep1.subsampled.25.fastq.gz", - "chip-seq-pipeline-test-data/input/se/fastqs/rep2/rep2.subsampled.20.fastq.gz" - ], - "test_merge_fastq.ref_pe_merged_fastq_R1" : "chip-seq-pipeline-test-data/ref_output/test_merge_fastq/pe_R1_merged.fastq.gz", - "test_merge_fastq.ref_pe_merged_fastq_R2" : "chip-seq-pipeline-test-data/ref_output/test_merge_fastq/pe_R2_merged.fastq.gz", - "test_merge_fastq.ref_se_merged_fastq" : "chip-seq-pipeline-test-data/ref_output/test_merge_fastq/se_merged.fastq.gz" -} diff --git a/dev/test/test_task/test_merge_fastq.wdl b/dev/test/test_task/test_merge_fastq.wdl deleted file mode 100644 index 7b85157e..00000000 --- a/dev/test/test_task/test_merge_fastq.wdl +++ /dev/null @@ -1,44 +0,0 @@ -# ENCODE DCC ChIP-Seq pipeline tester for task bowtie2 -# Author: Jin Lee (leepc12@gmail.com) -import '../../../chip.wdl' as chip -import 'compare_md5sum.wdl' as compare_md5sum - -workflow test_merge_fastq { - # test merging rep1 and rep2 - Array[File] pe_fastqs_R1 - Array[File] pe_fastqs_R2 - Array[File] se_fastqs - - File ref_pe_merged_fastq_R1 - File ref_pe_merged_fastq_R2 - File ref_se_merged_fastq - - call chip.merge_fastq as pe_merge_fastq { input : - fastqs_R1 = pe_fastqs_R1, - fastqs_R2 = pe_fastqs_R2, - paired_end = true, - } - call chip.merge_fastq as se_merge_fastq { input : - fastqs_R1 = se_fastqs, - fastqs_R2 = [], - paired_end = false, - } - - call compare_md5sum.compare_md5sum { input : - labels = [ - 'pe_merge_fastq_R1', - 'pe_merge_fastq_R2', - 'se_merge_fastq', - ], - files = select_all([ - pe_merge_fastq.merged_fastq_R1, - pe_merge_fastq.merged_fastq_R2, - se_merge_fastq.merged_fastq_R1, - ]), - ref_files = [ - ref_pe_merged_fastq_R1, - ref_pe_merged_fastq_R2, - ref_se_merged_fastq, - ], - } -} diff --git a/dev/test/test_task/test_trim_fastq.json b/dev/test/test_task/test_trim_fastq.json deleted file mode 100644 index 6a44b4f6..00000000 --- a/dev/test/test_task/test_trim_fastq.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "test_trim_fastq.pe_fastq_rep1_R1" : "chip-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/rep1-R1.subsampled.67.fastq.gz", - - "test_trim_fastq.ref_pe_trimmed_fastq_rep1_R1" : "chip-seq-pipeline-test-data/ref_output/test_trim_fastq/merge_fastqs_R1_rep1-R1.subsampled.67.merged.trim_50bp.fastq.gz" -} diff --git a/dev/test/test_task/test_trim_fastq.wdl b/dev/test/test_task/test_trim_fastq.wdl deleted file mode 100644 index 82002f81..00000000 --- a/dev/test/test_task/test_trim_fastq.wdl +++ /dev/null @@ -1,28 +0,0 @@ -# ENCODE DCC ChIP-Seq pipeline tester for task bowtie2 -# Author: Jin Lee (leepc12@gmail.com) -import '../../../chip.wdl' as chip -import 'compare_md5sum.wdl' as compare_md5sum - -workflow test_trim_fastq { - Int xcor_pe_trim_bp = 50 # for cross-correlation analysis only - String pe_fastq_rep1_R1 - - String ref_pe_trimmed_fastq_rep1_R1 - - call chip.trim_fastq as pe_trim_fastq { input : - fastq = pe_fastq_rep1_R1, - trim_bp = xcor_pe_trim_bp, - } - - call compare_md5sum.compare_md5sum { input : - labels = [ - 'pe_trim_fastq', - ], - files = [ - pe_trim_fastq.trimmed_fastq, - ], - ref_files = [ - ref_pe_trimmed_fastq_rep1_R1, - ], - } -} diff --git a/docs/input.md b/docs/input.md index 86cea617..6df12c0e 100644 --- a/docs/input.md +++ b/docs/input.md @@ -117,7 +117,7 @@ You can mix up different data types for individual replicate/control replicate. "chip.fastqs_rep1_R1" : ["rep1.fastq.gz"], "chip.fastqs_rep3_R1" : ["rep3.fastq.gz"], "chip.bams" : [null, "rep2.bam", null, null, null], - "chip.nodup_bams" : [null, "rep2.bam", null, "rep4.nodup.bam", null], + "chip.nodup_bams" : [null, null, null, "rep4.nodup.bam", null], "chip.tas" : [null, null, null, null, "rep5.tagAlign.gz"] } ``` From 5364982f66ec9821d45bf9ccb6bb8bbe49ab98c2 Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Fri, 11 Oct 2019 19:15:49 -0700 Subject: [PATCH 16/18] ver: dev-v1.3.1 -> v1.3.1, dx: small fix for dx --- chip.wdl | 5 ++-- dev/dev.md | 6 ++--- dev/test/test_task/test.sh | 2 +- dev/test/test_workflow/test_chip.sh | 2 +- dev/workflow_opts/docker.json | 2 +- dev/workflow_opts/scg.json | 2 +- dev/workflow_opts/sge.json | 2 +- dev/workflow_opts/sherlock.json | 2 +- dev/workflow_opts/singularity.json | 2 +- dev/workflow_opts/slurm.json | 2 +- docs/deprecated/tutorial_local_singularity.md | 4 ++-- docs/deprecated/tutorial_scg.md | 2 +- docs/deprecated/tutorial_scg_backend.md | 4 ++-- docs/deprecated/tutorial_sge.md | 4 ++-- docs/deprecated/tutorial_sge_backend.md | 2 +- docs/deprecated/tutorial_sherlock.md | 2 +- docs/deprecated/tutorial_sherlock_backend.md | 4 ++-- docs/deprecated/tutorial_slurm.md | 4 ++-- docs/deprecated/tutorial_slurm_backend.md | 4 ++-- docs/tutorial_dx_cli.md | 2 +- docs/tutorial_dx_web.md | 24 +++++++++---------- 21 files changed, 42 insertions(+), 41 deletions(-) diff --git a/chip.wdl b/chip.wdl index eefa0363..a5bb9c00 100644 --- a/chip.wdl +++ b/chip.wdl @@ -520,8 +520,9 @@ workflow chip { } # special trimming/mapping for xcor (when starting from BAMs) - Boolean has_input_of_bam2ta_no_dedup = has_output_of_align || defined(align.bam) - if ( has_input_of_bam2ta_no_dedup && !defined(bam2ta_no_dedup_R1.ta) ) { + Boolean has_input_of_bam2ta_no_dedup = (has_output_of_align || defined(align.bam)) + && !defined(bam2ta_no_dedup_R1.ta) + if ( has_input_of_bam2ta_no_dedup ) { call filter as filter_no_dedup { input : bam = bam_, paired_end = paired_end_, diff --git a/dev/dev.md b/dev/dev.md index 86c79be8..55a277cc 100644 --- a/dev/dev.md +++ b/dev/dev.md @@ -2,8 +2,8 @@ ## Command line for version change ```bash -PREV_VER=dev-v1.3.1 -NEW_VER=v1.3.1 +PREV_VER=v1.3.1 +NEW_VER=dev-v1.3.2 for f in $(grep -rl ${PREV_VER} --include=*.{wdl,md,sh}) do sed -i "s/${PREV_VER}/${NEW_VER}/g" ${f} @@ -24,7 +24,7 @@ Run the following command line locally to build out DX workflows for this pipeli ```bash # version -VER=dev-v1.3.1 +VER=v1.3.1 DOCKER=quay.io/encode-dcc/chip-seq-pipeline:$VER # general diff --git a/dev/test/test_task/test.sh b/dev/test/test_task/test.sh index 57bcb22c..d052655d 100755 --- a/dev/test/test_task/test.sh +++ b/dev/test/test_task/test.sh @@ -12,7 +12,7 @@ INPUT=$2 if [ $# -gt 2 ]; then DOCKER_IMAGE=$3 else - DOCKER_IMAGE=quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 + DOCKER_IMAGE=quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 fi if [ $# -gt 3 ]; then NUM_TASK=$4 diff --git a/dev/test/test_workflow/test_chip.sh b/dev/test/test_workflow/test_chip.sh index 335f15a5..1b7872c1 100755 --- a/dev/test/test_workflow/test_chip.sh +++ b/dev/test/test_workflow/test_chip.sh @@ -8,7 +8,7 @@ fi if [ $# -gt 2 ]; then DOCKER_IMAGE=$3 else - DOCKER_IMAGE=quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 + DOCKER_IMAGE=quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 fi INPUT=$1 GCLOUD_SERVICE_ACCOUNT_SECRET_JSON_FILE=$2 diff --git a/dev/workflow_opts/docker.json b/dev/workflow_opts/docker.json index 80739e71..35e87cae 100644 --- a/dev/workflow_opts/docker.json +++ b/dev/workflow_opts/docker.json @@ -1,6 +1,6 @@ { "default_runtime_attributes" : { - "docker" : "quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1", + "docker" : "quay.io/encode-dcc/chip-seq-pipeline:v1.3.1", "zones": "us-west1-a us-west1-b us-west1-c us-central1-c us-central1-b", "failOnStderr" : false, "continueOnReturnCode" : 0, diff --git a/dev/workflow_opts/scg.json b/dev/workflow_opts/scg.json index 9e7bbf92..3ca2147d 100644 --- a/dev/workflow_opts/scg.json +++ b/dev/workflow_opts/scg.json @@ -1,7 +1,7 @@ { "default_runtime_attributes" : { "slurm_account" : "YOUR_SLURM_ACCOUNT", - "singularity_container" : "/reference/ENCODE/pipeline_singularity_images/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "/reference/ENCODE/pipeline_singularity_images/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/reference/ENCODE,/scratch,/srv/gsfs0" } } diff --git a/dev/workflow_opts/sge.json b/dev/workflow_opts/sge.json index d2e74cb1..e6765d7a 100644 --- a/dev/workflow_opts/sge.json +++ b/dev/workflow_opts/sge.json @@ -1,6 +1,6 @@ { "default_runtime_attributes" : { "sge_pe" : "shm", - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg" + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg" } } diff --git a/dev/workflow_opts/sherlock.json b/dev/workflow_opts/sherlock.json index ed8ffec8..53cfd464 100644 --- a/dev/workflow_opts/sherlock.json +++ b/dev/workflow_opts/sherlock.json @@ -1,7 +1,7 @@ { "default_runtime_attributes" : { "slurm_partition" : "normal", - "singularity_container" : "/home/groups/cherry/encode/pipeline_singularity_images/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "/home/groups/cherry/encode/pipeline_singularity_images/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/scratch,/lscratch,/oak/stanford,/home/groups/cherry/encode" } } diff --git a/dev/workflow_opts/singularity.json b/dev/workflow_opts/singularity.json index 287f2d9e..710c5e23 100644 --- a/dev/workflow_opts/singularity.json +++ b/dev/workflow_opts/singularity.json @@ -1,5 +1,5 @@ { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg" + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg" } } diff --git a/dev/workflow_opts/slurm.json b/dev/workflow_opts/slurm.json index f45d7d90..7b09a1cc 100644 --- a/dev/workflow_opts/slurm.json +++ b/dev/workflow_opts/slurm.json @@ -2,6 +2,6 @@ "default_runtime_attributes" : { "slurm_partition" : "YOUR_SLURM_PARTITION", "slurm_account" : "YOUR_SLURM_ACCOUNT", - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg" + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg" } } diff --git a/docs/deprecated/tutorial_local_singularity.md b/docs/deprecated/tutorial_local_singularity.md index fd46593c..0a093bed 100644 --- a/docs/deprecated/tutorial_local_singularity.md +++ b/docs/deprecated/tutorial_local_singularity.md @@ -33,7 +33,7 @@ 6. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 ``` 7. Run a pipeline for the test sample. @@ -53,7 +53,7 @@ ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR2,..." } } diff --git a/docs/deprecated/tutorial_scg.md b/docs/deprecated/tutorial_scg.md index 569b7848..f95223e6 100644 --- a/docs/deprecated/tutorial_scg.md +++ b/docs/deprecated/tutorial_scg.md @@ -63,7 +63,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/reference/ENCODE,/scratch,/srv/gsfs0,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR1,..." } } diff --git a/docs/deprecated/tutorial_scg_backend.md b/docs/deprecated/tutorial_scg_backend.md index ed991779..43fda507 100644 --- a/docs/deprecated/tutorial_scg_backend.md +++ b/docs/deprecated/tutorial_scg_backend.md @@ -58,7 +58,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 5. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash $ sdev # SCG cluster does not allow building a container on login node - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 $ exit ``` @@ -77,7 +77,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/scratch/users,/srv/gsfs0,/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR1,..." } } diff --git a/docs/deprecated/tutorial_sge.md b/docs/deprecated/tutorial_sge.md index 1a31835e..2c4d0e6f 100644 --- a/docs/deprecated/tutorial_sge.md +++ b/docs/deprecated/tutorial_sge.md @@ -61,7 +61,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 7. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 ``` 8. Run a pipeline for the test sample. If your parallel environment (PE) found from step 5) has a different name from `shm` then edit the following shell script to change the PE name. @@ -83,7 +83,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR2,..." } } diff --git a/docs/deprecated/tutorial_sge_backend.md b/docs/deprecated/tutorial_sge_backend.md index 2a3f3696..1c3b83e8 100644 --- a/docs/deprecated/tutorial_sge_backend.md +++ b/docs/deprecated/tutorial_sge_backend.md @@ -68,7 +68,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 7. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.1 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.1 ``` 8. Run a pipeline for the test sample. diff --git a/docs/deprecated/tutorial_sherlock.md b/docs/deprecated/tutorial_sherlock.md index b877bf0d..e1d020db 100644 --- a/docs/deprecated/tutorial_sherlock.md +++ b/docs/deprecated/tutorial_sherlock.md @@ -68,7 +68,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/scratch,/lscratch,/oak/stanford,/home/groups/cherry/encode,/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR1,..." } } diff --git a/docs/deprecated/tutorial_sherlock_backend.md b/docs/deprecated/tutorial_sherlock_backend.md index 6eed70df..13a200d1 100644 --- a/docs/deprecated/tutorial_sherlock_backend.md +++ b/docs/deprecated/tutorial_sherlock_backend.md @@ -63,7 +63,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 6. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. Stanford Sherlock does not allow building a container on login nodes. Wait until you get a command prompt after `sdev`. ```bash $ sdev # sherlock cluster does not allow building a container on login node - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 $ exit # exit from an interactive node ``` @@ -82,7 +82,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/scratch,/oak/stanford,/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR1,..." } } diff --git a/docs/deprecated/tutorial_slurm.md b/docs/deprecated/tutorial_slurm.md index c72b5c37..00e3aa6f 100644 --- a/docs/deprecated/tutorial_slurm.md +++ b/docs/deprecated/tutorial_slurm.md @@ -56,7 +56,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 7. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 ``` 8. Run a pipeline for the test sample. If your cluster requires to specify any of them then add one to the command line. @@ -78,7 +78,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR2,..." } } diff --git a/docs/deprecated/tutorial_slurm_backend.md b/docs/deprecated/tutorial_slurm_backend.md index e7625ac4..83412710 100644 --- a/docs/deprecated/tutorial_slurm_backend.md +++ b/docs/deprecated/tutorial_slurm_backend.md @@ -68,7 +68,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt 7. Pull a singularity container for the pipeline. This will pull pipeline's docker container first and build a singularity one on `~/.singularity`. ```bash - $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-dev-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 + $ mkdir -p ~/.singularity && cd ~/.singularity && SINGULARITY_CACHEDIR=~/.singularity SINGULARITY_PULLFOLDER=~/.singularity singularity pull --name chip-seq-pipeline-v1.3.1.simg -F docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 ``` 8. Run a pipeline for the test sample. @@ -86,7 +86,7 @@ Our pipeline supports both [Conda](https://conda.io/docs/) and [Singularity](htt ```javascript { "default_runtime_attributes" : { - "singularity_container" : "~/.singularity/chip-seq-pipeline-dev-v1.3.1.simg", + "singularity_container" : "~/.singularity/chip-seq-pipeline-v1.3.1.simg", "singularity_bindpath" : "/your/,YOUR_OWN_DATA_DIR1,YOUR_OWN_DATA_DIR2,..." } } diff --git a/docs/tutorial_dx_cli.md b/docs/tutorial_dx_cli.md index f1e0f2fe..40536257 100644 --- a/docs/tutorial_dx_cli.md +++ b/docs/tutorial_dx_cli.md @@ -45,7 +45,7 @@ This document describes instruction for the item 1). ```bash $ PROJECT=[YOUR_PROJECT_NAME] $ OUT_FOLDER=/test_sample_chip_ENCSR936XTK_subsampled_chr19_only - $ DOCKER=quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 + $ DOCKER=quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 $ java -jar dxWDL-0.77.jar compile chip.wdl -project ${PROJECT} -f -folder ${OUT_FOLDER} -defaults ${INPUT} -extras <(echo "{\"default_runtime_attributes\":{\"docker\":\"${DOCKER}\"}}") ``` diff --git a/docs/tutorial_dx_web.md b/docs/tutorial_dx_web.md index 61e9bb87..792b5c23 100644 --- a/docs/tutorial_dx_web.md +++ b/docs/tutorial_dx_web.md @@ -15,8 +15,8 @@ This document describes instruction for the item 2). 3. Move to one of the following workflow directories according to the platform you have chosen for your project (AWS or Azure). These DX workflows are pre-built with all parameters defined. -* [AWS test workflow](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/test_ENCSR936XTK_subsampled_chr19_only) -* [Azure test workflow](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/test_ENCSR936XTK_subsampled_chr19_only) +* [AWS test workflow](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.1/test_ENCSR936XTK_subsampled_chr19_only) +* [Azure test workflow](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.1/test_ENCSR936XTK_subsampled_chr19_only) 4. Copy it to your project by right-clicking on the DX workflow `chip` and choose "Copy". @@ -40,16 +40,16 @@ This document describes instruction for the item 2). 1. DNAnexus allows only one copy of a workflow per project. The example workflow in the previous section is pre-built for the subsampled test sample [ENCSR936XTK](https://www.encodeproject.org/experiments/ENCSR936XTK/) with all parameters defined already. 2. Copy one of the following workflows according to the platform you have chosen for your project (AWS or Azure). -* [AWS general](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/general) without pre-defined reference genome. -* [AWS hg38](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/hg38) with pre-defined hg38 reference genome. -* [AWS hg19](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/hg19) with pre-defined hg19 reference genome. -* [AWS mm10](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/mm10) with pre-defined mm10 reference genome. -* [AWS mm9](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/dev-v1.3.1/mm9) with pre-defined mm9 reference genome. -* [Azure general](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/general) without pre-defined reference genome. -* [Azure hg38](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/hg38) with pre-defined hg38 reference genome. -* [Azure hg19](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/hg19) with pre-defined hg19 reference genome. -* [Azure mm10](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/mm10) with pre-defined mm10 reference genome. -* [Azure mm9](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/dev-v1.3.1/mm9) with pre-defined mm9 reference genome. +* [AWS general](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.1/general) without pre-defined reference genome. +* [AWS hg38](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.1/hg38) with pre-defined hg38 reference genome. +* [AWS hg19](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.1/hg19) with pre-defined hg19 reference genome. +* [AWS mm10](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.1/mm10) with pre-defined mm10 reference genome. +* [AWS mm9](https://platform.DNAnexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/ChIP-seq2/workflows/v1.3.1/mm9) with pre-defined mm9 reference genome. +* [Azure general](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.1/general) without pre-defined reference genome. +* [Azure hg38](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.1/hg38) with pre-defined hg38 reference genome. +* [Azure hg19](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.1/hg19) with pre-defined hg19 reference genome. +* [Azure mm10](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.1/mm10) with pre-defined mm10 reference genome. +* [Azure mm9](https://platform.DNAnexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/ChIP-seq2/workflows/v1.3.1/mm9) with pre-defined mm9 reference genome. 3. Click on the DX workflow `chip`. From dd91a2ee14e6b3e1d952a1b7b2e06a08cad36dba Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Fri, 11 Oct 2019 19:54:32 -0700 Subject: [PATCH 17/18] dx: fix old param in dx example --- example_input_json/dx/ENCSR000DYI_dx.json | 1 - example_input_json/dx_azure/ENCSR000DYI_dx_azure.json | 1 - 2 files changed, 2 deletions(-) diff --git a/example_input_json/dx/ENCSR000DYI_dx.json b/example_input_json/dx/ENCSR000DYI_dx.json index 05220eaa..d0109376 100644 --- a/example_input_json/dx/ENCSR000DYI_dx.json +++ b/example_input_json/dx/ENCSR000DYI_dx.json @@ -12,7 +12,6 @@ "chip.paired_end" : false, "chip.always_use_pooled_ctl" : true, - "chip.spp_cpu" : 2, "chip.title" : "ENCSR000DYI", "chip.description" : "CEBPB ChIP-seq on human A549 produced by the Snyder lab" diff --git a/example_input_json/dx_azure/ENCSR000DYI_dx_azure.json b/example_input_json/dx_azure/ENCSR000DYI_dx_azure.json index 9113fe48..a961628e 100644 --- a/example_input_json/dx_azure/ENCSR000DYI_dx_azure.json +++ b/example_input_json/dx_azure/ENCSR000DYI_dx_azure.json @@ -12,7 +12,6 @@ "chip.paired_end" : false, "chip.always_use_pooled_ctl" : true, - "chip.spp_cpu" : 2, "chip.title" : "ENCSR000DYI", "chip.description" : "CEBPB ChIP-seq on human A549 produced by the Snyder lab" From 92449d8d09c19506a694ec0b18071a4d292f522f Mon Sep 17 00:00:00 2001 From: Jin Lee Date: Fri, 11 Oct 2019 19:56:35 -0700 Subject: [PATCH 18/18] ver: v1.3.1 --- chip.wdl | 6 +++--- dev/dev.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/chip.wdl b/chip.wdl index a5bb9c00..fb59ae33 100644 --- a/chip.wdl +++ b/chip.wdl @@ -1,12 +1,12 @@ # ENCODE TF/Histone ChIP-Seq pipeline # Author: Jin Lee (leepc12@gmail.com) -#CAPER docker quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 -#CAPER singularity docker://quay.io/encode-dcc/chip-seq-pipeline:dev-v1.3.1 +#CAPER docker quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 +#CAPER singularity docker://quay.io/encode-dcc/chip-seq-pipeline:v1.3.1 #CROO out_def https://storage.googleapis.com/encode-pipeline-output-definition/chip.croo.json workflow chip { - String pipeline_ver = 'dev-v1.3.1' + String pipeline_ver = 'v1.3.1' ### sample name, description String title = 'Untitled' String description = 'No description' diff --git a/dev/dev.md b/dev/dev.md index 55a277cc..f7d9aa5c 100644 --- a/dev/dev.md +++ b/dev/dev.md @@ -3,7 +3,7 @@ ## Command line for version change ```bash PREV_VER=v1.3.1 -NEW_VER=dev-v1.3.2 +NEW_VER=v1.3.1 for f in $(grep -rl ${PREV_VER} --include=*.{wdl,md,sh}) do sed -i "s/${PREV_VER}/${NEW_VER}/g" ${f}