diff --git a/.github/workflows/lintr.yaml b/.github/workflows/lintr.yaml index ab839dc..03da921 100644 --- a/.github/workflows/lintr.yaml +++ b/.github/workflows/lintr.yaml @@ -4,6 +4,11 @@ on: branches: - master - dev + pull_request: + branches: + - master + - dev + jobs: Lintr: runs-on: ubuntu-latest @@ -22,4 +27,4 @@ jobs: touch hg38.fa genes.gtf hg38.bed hg38.tss.bed hg38_refseq.ucsc Ecoli_GCF_000005845.2_ASM584v2_genomic.fna adapters.fa && \ snakemake --lint -s /opt2/workflow/Snakefile \ -d /opt2/output_carlisle --configfile /opt2/.test/config_lint.yaml || \ - echo 'There may have been a few warnings or errors. Please read through the log to determine if its harmless.'" \ No newline at end of file + echo 'There may have been a few warnings or errors. Please read through the log to determine if its harmless.'" diff --git a/.github/workflows/projects.yml b/.github/workflows/projects.yml new file mode 100644 index 0000000..61a2816 --- /dev/null +++ b/.github/workflows/projects.yml @@ -0,0 +1,14 @@ +name: Add issues/PRs to user projects + +on: + issues: + types: + - assigned + pull_request: + types: + - assigned + +jobs: + add-to-project: + uses: CCBR/.github/.github/workflows/auto-add-user-project.yml@v0.1.0 + secrets: inherit diff --git a/.github/workflows/test_dev.yml b/.github/workflows/test_dev.yml index ed83aa9..33a6a87 100644 --- a/.github/workflows/test_dev.yml +++ b/.github/workflows/test_dev.yml @@ -1,25 +1,32 @@ -name: DevTesting +name: test on: push: branches: + - master - dev + pull_request: + branches: + - master + - dev + jobs: deploy: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4.1.0 - uses: docker://snakemake/snakemake:v7.19.1 with: directory: '.test' - - name: Dev Testing Workflow - continue-on-error: true + - name: dryrun + shell: bash {0} run: | - docker run -v $PWD:/opt2 snakemake/snakemake:v7.19.1 /bin/bash -c \ - "mkdir -p /opt2/output_carlisle/config /opt2/output_carlisle/annotation && \ - cp -r /opt2/workflow/scripts/ /opt2/output_carlisle/ && \ - cp /opt2/resources/cluster_biowulf.yaml /opt2/output_carlisle/config/cluster.yaml && \ - cp /opt2/resources/tools_biowulf.yaml /opt2/output_carlisle/config/tools.yaml && \ - cd /opt2/output_carlisle/annotation && \ - touch hg38.fa genes.gtf hg38.bed hg38.tss.bed hg38_refseq.ucsc Ecoli_GCF_000005845.2_ASM584v2_genomic.fna adapters.fa && \ - snakemake --lint -s /opt2/workflow/Snakefile \ - -d /opt2/output_carlisle --configfile /opt2/.test/config_lint.yaml || \ - echo 'There may have been a few warnings or errors. Please read through the log to determine if its harmless.'" + bash ./install.sh ./test-workdir/carlisle-v9999.9999.9999-dev + ./test-workdir/carlisle-v9999.9999.9999-dev/bin/carlisle --runmode=init --workdir=./test-workdir/output_carlisle + cp ./test-workdir/carlisle-v9999.9999.9999-dev/bin/.test/config_lint.yaml ./test-workdir/output_carlisle/config/config.yaml + # TODO: use `carlisle run --mode dryrun` + docker run -v ./test-workdir/:/opt2 snakemake/snakemake:v7.19.1 /bin/bash -c \ + "cd /opt2 && snakemake \ + -s ./carlisle-v9999.9999.9999-dev/bin/workflow/Snakefile \ + --dryrun \ + --configfile carlisle-v9999.9999.9999-dev/bin/.test/config_lint.yaml \ + --directory output_carlisle" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..12dc373 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,48 @@ +default_install_hook_types: [pre-commit, commit-msg] +default_stages: [pre-commit] +exclude: | + (?x)( + ^assets/| + ^docs/.*.html| + ^_extensions/ + ) +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v1.2.3 + hooks: + - id: check-added-large-files + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-json + # spell check + - repo: https://github.com/codespell-project/codespell + rev: v2.2.4 + hooks: + - id: codespell + # https://github.com/codespell-project/codespell/issues/1498 + # Python formatting + - repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + # R formatting + - repo: https://github.com/lorenzwalthert/precommit + rev: v0.1.2 + hooks: + - id: style-files + # general linting + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.7.1 + hooks: + - id: prettier + # enforce commit format + - repo: https://github.com/compilerla/conventional-pre-commit + rev: v2.3.0 + hooks: + - id: conventional-pre-commit + stages: [commit-msg] + args: [] + - repo: https://github.com/citation-file-format/cffconvert + rev: 054bda51dbe278b3e86f27c890e3f3ac877d616c + hooks: + - id: validate-cff diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..f0a6037 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,8 @@ +# gitignore +.nextflow* +work/ +data/ +results/ +.DS_Store +*.code-workspace +assets/*.html diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..81ee40a --- /dev/null +++ b/.prettierrc @@ -0,0 +1,7 @@ +overrides: + - files: + - "*.md" + - "*.cff" + - ".prettierrc" + options: + tabWidth: 2 diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..b940de0 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,29 @@ +cff-version: 1.2.0 +message: "If you use CARLISLE, please cite it as below." +authors: + - family-names: "Sevilla" + given-names: "Samantha" + orcid: "https://orcid.org/0000-0002-8734-9875" + affiliation: Advanced Biomedical Computational Science, Frederick National Laboratory for Cancer Research, Frederick, MD 21702, USA + - family-names: "Koparde" + given-names: "Vishal" + orcid: "https://orcid.org/0000-0001-8978-8495" + affiliation: Advanced Biomedical Computational Science, Frederick National Laboratory for Cancer Research, Frederick, MD 21702, USA + - family-names: "Chou" + given-names: "Hsien-chao" + orcid: "https://orcid.org/0000-0002-4870-9663" + - family-names: "Kim" + given-names: "Sohyoung" + - family-names: "Hu" + given-names: "Yue" + - family-names: "Saloura" + given-names: "Vassiliki" +title: "Cut And Run anaLysIS pipeLinE (CARLISLE)" +url: https://ccbr.github.io/CARLISLE/ +repository-code: https://github.com/CCBR/CARLISLE +identifiers: + - description: Archived snapshots of all versions + type: doi + value: 10.5281/zenodo.10483876 +version: 2.4.1 +date-released: 2024-01-10 diff --git a/README.md b/README.md index 70fea14..6b2a25c 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,11 @@
- - [![GitHub releases](https://img.shields.io/github/release/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/releases) [![GitHub issues](https://img.shields.io/github/issues/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/issues) [![GitHub license](https://img.shields.io/github/license/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/blob/master/LICENSE) + +[![GitHub releases](https://img.shields.io/github/release/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/releases) +[![GitHub issues](https://img.shields.io/github/issues/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/issues) +[![GitHub license](https://img.shields.io/github/license/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/blob/master/LICENSE) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10483876.svg)](https://doi.org/10.5281/zenodo.10483876)
diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..005119b --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +2.4.1 diff --git a/carlisle b/carlisle index 7872439..c24238f 100755 --- a/carlisle +++ b/carlisle @@ -14,7 +14,6 @@ SNAKEMAKE_VERSION="snakemake/7.19.1" SINGULARITY_VERSION="singularity/3.10.5" set -eo pipefail -module purge SCRIPTNAME="$0" SCRIPTBASENAME=$(readlink -f $(basename $0)) @@ -28,29 +27,27 @@ tools_specific_yaml="tools_biowulf.yaml" # essential files # these are relative to the workflows' base folder # these are copied into the WORKDIR -ESSENTIAL_FILES="config/config.yaml config/samples.tsv config/contrasts.tsv config/fqscreen_config.conf config/multiqc_config.yaml resources/cluster_* resources/tools_*" -ESSENTIAL_FOLDERS="workflow/scripts" +ESSENTIAL_FILES="config/config.yaml config/samples.tsv config/contrasts.tsv config/fqscreen_config.conf config/multiqc_config.yaml" +ESSENTIAL_FOLDERS="workflow/scripts annotation" # set extra singularity bindings EXTRA_SINGULARITY_BINDS="-B /data/CCBR_Pipeliner/,/lscratch" -function get_git_commitid_tag() { - # This function gets the latest git commit id and tag - # Input is PIPELINE_HOME folder which is a git folder - cd $1 - gid=$(git rev-parse HEAD) - tag=$(git describe --tags $gid 2>/dev/null) - echo -ne "$gid\t$tag" -} - # ## setting PIPELINE_HOME PIPELINE_HOME=$(readlink -f $(dirname "$0")) echo "Pipeline Dir: $PIPELINE_HOME" # set snakefile SNAKEFILE="${PIPELINE_HOME}/workflow/Snakefile" -# get github commit tag -GIT_COMMIT_TAG=$(get_git_commitid_tag $PIPELINE_HOME) -echo "Git Commit/Tag: $GIT_COMMIT_TAG" +function get_version_from_path() { + PIPELINE_HOME=$1 + VERSION="$(echo $PIPELINE_HOME | grep -oP 'v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?' - )" + echo -ne $VERSION +} + +# get version from directory name +# regex source: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string +#echo "Version: $VERSION" +echo "Version: $(get_version_from_path $PIPELINE_HOME)" function usage() { # This function prints generic usage of the wrapper script. @@ -104,8 +101,8 @@ function init() { done # rename config dependent on partition used - cp $WORKDIR/config/$cluster_specific_yaml $WORKDIR/config/cluster.yaml - cp $WORKDIR/config/$tools_specific_yaml $WORKDIR/config/tools.yaml + cp ${PIPELINE_HOME}/resources/$cluster_specific_yaml $WORKDIR/config/cluster.yaml + cp ${PIPELINE_HOME}/resources/$tools_specific_yaml $WORKDIR/config/tools.yaml # copy essential folders for f in $ESSENTIAL_FOLDERS;do @@ -194,8 +191,9 @@ function controlcheck(){ check2=`awk '{print $2}' ${WORKDIR}/config/contrasts.tsv` for sample_id in ${control_list[@]}; do - if [[ $check1 =~ $sample_id || $check2 =~ $sample_id ]]; then + if [[ $check1 == $sample_id || $check2 == $sample_id ]]; then echo "Controls ($sample_id) cannot be listed in contrast.csv - update and re-run" + echo "$check1 okkk $check2" exit 0 fi done @@ -442,4 +440,4 @@ function main(){ } # call the main function -main "$@" \ No newline at end of file +main "$@" diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..59e7a29 --- /dev/null +++ b/install.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# usage: +# ./install.sh new/path/to/install +# examples +# ./install.sh .v2.4.0 +# /data/CCBR_Pipeliner/Pipelines/CARLISLE/dev/install.sh /data/CCBR_Pipeliner/Pipelines/CARLISLE/.v2.5.0-a +set -euo pipefail + +VERSION=$1 +mkdir -p ${VERSION}/bin +INSTALL_PATH=$(readlink -f ${VERSION}/bin) +DIRNAME=$(readlink -f $(dirname $0)) + +if [ -n "$(ls -A $INSTALL_PATH 2>/dev/null)" ] +then + echo "ERROR: directory not empty: ${INSTALL_PATH}" + echo $(ls $INSTALL_PATH) + exit 1 +fi + +# copy essential files & directories + +## carlisle CLI script +cp $DIRNAME/carlisle $INSTALL_PATH/ + +## all config & workflow scripts; +for subdir in annotation config workflow resources .test +do + mkdir -p ${INSTALL_PATH}/$subdir + cp -r ${DIRNAME}/$subdir/* ${INSTALL_PATH}/$subdir +done + +# export path +if [[ ":$PATH:" != *":${INSTALL_PATH}:"* ]];then + export PATH="${PATH}:${INSTALL_PATH}" +fi + +echo "${INSTALL_PATH}" diff --git a/workflow/scripts/_spooker b/workflow/scripts/_spooker index d772359..9d2fd8a 100644 --- a/workflow/scripts/_spooker +++ b/workflow/scripts/_spooker @@ -35,10 +35,10 @@ if [[ $ISBIOWULF == true || $ISFRCE == true ]];then fi fi echo "spook is now in PATH:$SPOOK" - SPOOK_COPY2DIR="/scratch/carlisle" + SPOOK_COPY2DIR="/scratch/ccbrpipeliner" fi if [[ $ISFRCE == true ]];then - SPOOK_COPY2DIR="/mnt/projects/CCBR-Pipelines/pipelines/userdata/carlisle" + SPOOK_COPY2DIR="/mnt/projects/CCBR-Pipelines/pipelines/userdata/ccbrpipeliner" fi DT=$(date +%y%m%d%H%M%S) @@ -63,8 +63,8 @@ if [[ $ISBIOWULF == true || $ISFRCE == true ]];then tree $PIPELINE_OUTDIR >> $treefile cmd="$cmd $treefile" - if [[ -d "${PIPELINE_OUTDIR}/logfiles" ]];then - logdir="${PIPELINE_OUTDIR}/logfiles" + if [[ -d "${PIPELINE_OUTDIR}/logs" ]];then + logdir="${PIPELINE_OUTDIR}/logs" for thisfile in "snakemake.log" "snakemake.log.jobby" "master.log" "runtime_statistics.json";do absthisfile="${logdir}/${thisfile}" if [[ -f "$absthisfile" ]];then