Merge pull request #114 from CCBR/master

v2.4.1 changes from main

.github/workflows/lintr.yaml

@@ -4,6 +4,11 @@ on:
     branches:
       - master
       - dev
+  pull_request:
+    branches:
+      - master
+      - dev
+
 jobs:
   Lintr:
     runs-on: ubuntu-latest
@@ -22,4 +27,4 @@ jobs:
         touch hg38.fa genes.gtf hg38.bed hg38.tss.bed hg38_refseq.ucsc Ecoli_GCF_000005845.2_ASM584v2_genomic.fna adapters.fa && \
         snakemake --lint -s /opt2/workflow/Snakefile \
         -d /opt2/output_carlisle --configfile /opt2/.test/config_lint.yaml || \
-        echo 'There may have been a few warnings or errors. Please read through the log to determine if its harmless.'"
+        echo 'There may have been a few warnings or errors. Please read through the log to determine if its harmless.'"

.github/workflows/projects.yml

@@ -0,0 +1,14 @@
+name: Add issues/PRs to user projects
+
+on:
+  issues:
+    types:
+      - assigned
+  pull_request:
+    types:
+      - assigned
+
+jobs:
+  add-to-project:
+    uses: CCBR/.github/.github/workflows/auto-add-user-project.yml@v0.1.0
+    secrets: inherit

.github/workflows/test_dev.yml

@@ -1,25 +1,32 @@
-name: DevTesting
+name: test
 on:
   push:
     branches:
+      - master
       - dev
+  pull_request:
+    branches:
+      - master
+      - dev
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+    - uses: actions/checkout@v4.1.0
     - uses: docker://snakemake/snakemake:v7.19.1
       with:
         directory: '.test'
-    - name: Dev Testing Workflow
-      continue-on-error: true
+    - name: dryrun
+      shell: bash {0}
       run: |
-        docker run -v $PWD:/opt2 snakemake/snakemake:v7.19.1 /bin/bash -c \
-          "mkdir -p /opt2/output_carlisle/config /opt2/output_carlisle/annotation && \
-          cp -r /opt2/workflow/scripts/ /opt2/output_carlisle/ && \
-          cp /opt2/resources/cluster_biowulf.yaml /opt2/output_carlisle/config/cluster.yaml && \
-          cp /opt2/resources/tools_biowulf.yaml /opt2/output_carlisle/config/tools.yaml && \
-          cd /opt2/output_carlisle/annotation && \
-          touch hg38.fa genes.gtf hg38.bed hg38.tss.bed hg38_refseq.ucsc Ecoli_GCF_000005845.2_ASM584v2_genomic.fna adapters.fa && \
-          snakemake --lint -s /opt2/workflow/Snakefile \
-          -d /opt2/output_carlisle --configfile /opt2/.test/config_lint.yaml || \
-          echo 'There may have been a few warnings or errors. Please read through the log to determine if its harmless.'"
+        bash ./install.sh ./test-workdir/carlisle-v9999.9999.9999-dev
+        ./test-workdir/carlisle-v9999.9999.9999-dev/bin/carlisle --runmode=init --workdir=./test-workdir/output_carlisle
+        cp ./test-workdir/carlisle-v9999.9999.9999-dev/bin/.test/config_lint.yaml ./test-workdir/output_carlisle/config/config.yaml
+        # TODO: use `carlisle run --mode dryrun`
+        docker run -v ./test-workdir/:/opt2 snakemake/snakemake:v7.19.1 /bin/bash -c \
+          "cd /opt2 && snakemake \
+            -s ./carlisle-v9999.9999.9999-dev/bin/workflow/Snakefile \
+            --dryrun \
+            --configfile carlisle-v9999.9999.9999-dev/bin/.test/config_lint.yaml \
+            --directory output_carlisle"

.pre-commit-config.yaml

@@ -0,0 +1,48 @@
+default_install_hook_types: [pre-commit, commit-msg]
+default_stages: [pre-commit]
+exclude: |
+  (?x)(
+      ^assets/|
+      ^docs/.*.html|
+      ^_extensions/
+   )
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.2.3
+    hooks:
+      - id: check-added-large-files
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-json
+  # spell check
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.4
+    hooks:
+      - id: codespell
+        # https://github.com/codespell-project/codespell/issues/1498
+  # Python formatting
+  - repo: https://github.com/psf/black
+    rev: 23.7.0
+    hooks:
+      - id: black
+  # R formatting
+  - repo: https://github.com/lorenzwalthert/precommit
+    rev: v0.1.2
+    hooks:
+      - id: style-files
+  # general linting
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.7.1
+    hooks:
+      - id: prettier
+  # enforce commit format
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v2.3.0
+    hooks:
+      - id: conventional-pre-commit
+        stages: [commit-msg]
+        args: []
+  - repo: https://github.com/citation-file-format/cffconvert
+    rev: 054bda51dbe278b3e86f27c890e3f3ac877d616c
+    hooks:
+      - id: validate-cff

.prettierignore

@@ -0,0 +1,8 @@
+# gitignore
+.nextflow*
+work/
+data/
+results/
+.DS_Store
+*.code-workspace
+assets/*.html

.prettierrc

@@ -0,0 +1,7 @@
+overrides:
+  - files:
+      - "*.md"
+      - "*.cff"
+      - ".prettierrc"
+    options:
+      tabWidth: 2

CITATION.cff

@@ -0,0 +1,29 @@
+cff-version: 1.2.0
+message: "If you use CARLISLE, please cite it as below."
+authors:
+  - family-names: "Sevilla"
+    given-names: "Samantha"
+    orcid: "https://orcid.org/0000-0002-8734-9875"
+    affiliation: Advanced Biomedical Computational Science, Frederick National Laboratory for Cancer Research, Frederick, MD 21702, USA
+  - family-names: "Koparde"
+    given-names: "Vishal"
+    orcid: "https://orcid.org/0000-0001-8978-8495"
+    affiliation: Advanced Biomedical Computational Science, Frederick National Laboratory for Cancer Research, Frederick, MD 21702, USA
+  - family-names: "Chou"
+    given-names: "Hsien-chao"
+    orcid: "https://orcid.org/0000-0002-4870-9663"
+  - family-names: "Kim"
+    given-names: "Sohyoung"
+  - family-names: "Hu"
+    given-names: "Yue"
+  - family-names: "Saloura"
+    given-names: "Vassiliki"
+title: "Cut And Run anaLysIS pipeLinE (CARLISLE)"
+url: https://ccbr.github.io/CARLISLE/
+repository-code: https://github.com/CCBR/CARLISLE
+identifiers:
+  - description: Archived snapshots of all versions
+    type: doi
+    value: 10.5281/zenodo.10483876
+version: 2.4.1
+date-released: 2024-01-10

README.md

@@ -2,8 +2,11 @@
 
 <div align="center">
   <img src="resources/img/CUTandRUN_Workflow.jpeg" width="85%" height="85%">
-
-  [![GitHub releases](https://img.shields.io/github/release/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/releases)  [![GitHub issues](https://img.shields.io/github/issues/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/issues) [![GitHub license](https://img.shields.io/github/license/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/blob/master/LICENSE)  
+
+[![GitHub releases](https://img.shields.io/github/release/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/releases)
+[![GitHub issues](https://img.shields.io/github/issues/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/issues)
+[![GitHub license](https://img.shields.io/github/license/CCBR/CARLISLE)](https://github.com/CCBR/CARLISLE/blob/master/LICENSE)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10483876.svg)](https://doi.org/10.5281/zenodo.10483876)
 
 </div>
 

VERSION

@@ -0,0 +1 @@
+2.4.1

carlisle

@@ -14,7 +14,6 @@ SNAKEMAKE_VERSION="snakemake/7.19.1"
 SINGULARITY_VERSION="singularity/3.10.5"
 
 set -eo pipefail
-module purge
 
 SCRIPTNAME="$0"
 SCRIPTBASENAME=$(readlink -f $(basename $0))
@@ -28,29 +27,27 @@ tools_specific_yaml="tools_biowulf.yaml"
 # essential files
 # these are relative to the workflows' base folder
 # these are copied into the WORKDIR
-ESSENTIAL_FILES="config/config.yaml config/samples.tsv config/contrasts.tsv config/fqscreen_config.conf config/multiqc_config.yaml resources/cluster_* resources/tools_*"
-ESSENTIAL_FOLDERS="workflow/scripts"
+ESSENTIAL_FILES="config/config.yaml config/samples.tsv config/contrasts.tsv config/fqscreen_config.conf config/multiqc_config.yaml"
+ESSENTIAL_FOLDERS="workflow/scripts annotation"
 # set extra singularity bindings
 EXTRA_SINGULARITY_BINDS="-B /data/CCBR_Pipeliner/,/lscratch"
 
-function get_git_commitid_tag() {
-  # This function gets the latest git commit id and tag
-  # Input is PIPELINE_HOME folder which is a git folder
-    cd $1
-    gid=$(git rev-parse HEAD)
-    tag=$(git describe --tags $gid 2>/dev/null)
-    echo -ne "$gid\t$tag"
-}
-
 # ## setting PIPELINE_HOME
 PIPELINE_HOME=$(readlink -f $(dirname "$0"))
 echo "Pipeline Dir: $PIPELINE_HOME"
 # set snakefile
 SNAKEFILE="${PIPELINE_HOME}/workflow/Snakefile"
 
-# get github commit tag
-GIT_COMMIT_TAG=$(get_git_commitid_tag $PIPELINE_HOME)
-echo "Git Commit/Tag: $GIT_COMMIT_TAG"
+function get_version_from_path() {
+  PIPELINE_HOME=$1
+  VERSION="$(echo $PIPELINE_HOME | grep -oP 'v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?' - )"
+  echo -ne $VERSION
+}
+
+# get version from directory name
+# regex source: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+#echo "Version: $VERSION"
+echo "Version: $(get_version_from_path $PIPELINE_HOME)"
 
 function usage() { 
 # This function prints generic usage of the wrapper script.
@@ -104,8 +101,8 @@ function init() {
   done
 
   # rename config dependent on partition used
-  cp $WORKDIR/config/$cluster_specific_yaml $WORKDIR/config/cluster.yaml
-  cp $WORKDIR/config/$tools_specific_yaml $WORKDIR/config/tools.yaml
+  cp ${PIPELINE_HOME}/resources/$cluster_specific_yaml $WORKDIR/config/cluster.yaml
+  cp ${PIPELINE_HOME}/resources/$tools_specific_yaml $WORKDIR/config/tools.yaml
 
   # copy essential folders
   for f in $ESSENTIAL_FOLDERS;do
@@ -194,8 +191,9 @@ function controlcheck(){
   check2=`awk '{print $2}' ${WORKDIR}/config/contrasts.tsv`
 
   for sample_id in ${control_list[@]}; do
-    if [[ $check1 =~ $sample_id || $check2 =~ $sample_id ]]; then 
+    if [[ $check1 == $sample_id || $check2 == $sample_id ]]; then 
       echo "Controls ($sample_id) cannot be listed in contrast.csv - update and re-run"
+      echo "$check1 okkk $check2"
       exit 0
     fi
   done
@@ -442,4 +440,4 @@ function main(){
 }
 
 # call the main function
-main "$@"
+main "$@"

install.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# usage:
+#   ./install.sh new/path/to/install
+# examples
+#   ./install.sh .v2.4.0
+#   /data/CCBR_Pipeliner/Pipelines/CARLISLE/dev/install.sh /data/CCBR_Pipeliner/Pipelines/CARLISLE/.v2.5.0-a
+set -euo pipefail
+
+VERSION=$1
+mkdir -p ${VERSION}/bin
+INSTALL_PATH=$(readlink -f ${VERSION}/bin)
+DIRNAME=$(readlink -f $(dirname $0))
+
+if [ -n "$(ls -A $INSTALL_PATH 2>/dev/null)" ]
+then
+    echo "ERROR: directory not empty: ${INSTALL_PATH}"
+    echo $(ls $INSTALL_PATH)
+    exit 1
+fi
+
+# copy essential files & directories
+
+## carlisle CLI script
+cp $DIRNAME/carlisle $INSTALL_PATH/
+
+## all config & workflow scripts;
+for subdir in annotation config workflow resources .test
+do  
+    mkdir -p ${INSTALL_PATH}/$subdir
+    cp -r ${DIRNAME}/$subdir/* ${INSTALL_PATH}/$subdir
+done
+
+# export path
+if [[ ":$PATH:" != *":${INSTALL_PATH}:"* ]];then
+    export PATH="${PATH}:${INSTALL_PATH}"
+fi
+
+echo "${INSTALL_PATH}"

workflow/scripts/_spooker

@@ -35,10 +35,10 @@ if [[ $ISBIOWULF == true || $ISFRCE == true ]];then
 	    fi
         fi
         echo "spook is now in PATH:$SPOOK"
-        SPOOK_COPY2DIR="/scratch/carlisle"
+        SPOOK_COPY2DIR="/scratch/ccbrpipeliner"
     fi
     if [[ $ISFRCE == true ]];then
-        SPOOK_COPY2DIR="/mnt/projects/CCBR-Pipelines/pipelines/userdata/carlisle"
+        SPOOK_COPY2DIR="/mnt/projects/CCBR-Pipelines/pipelines/userdata/ccbrpipeliner"
     fi
 
     DT=$(date +%y%m%d%H%M%S)
@@ -63,8 +63,8 @@ if [[ $ISBIOWULF == true || $ISFRCE == true ]];then
         tree $PIPELINE_OUTDIR >> $treefile
         cmd="$cmd $treefile"
 
-        if [[ -d "${PIPELINE_OUTDIR}/logfiles" ]];then
-            logdir="${PIPELINE_OUTDIR}/logfiles"
+        if [[ -d "${PIPELINE_OUTDIR}/logs" ]];then
+            logdir="${PIPELINE_OUTDIR}/logs"
             for thisfile in "snakemake.log" "snakemake.log.jobby" "master.log" "runtime_statistics.json";do
                 absthisfile="${logdir}/${thisfile}"
                 if [[ -f "$absthisfile" ]];then