From 1005fa03f6ba406e6016288166540d6faf25f716 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Tue, 17 Oct 2023 09:00:09 -0400 Subject: [PATCH 01/10] feat: create 'install' script --- install.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 install.sh diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..55b4aa9 --- /dev/null +++ b/install.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# usage: +# ./install.sh new/path/to/install +# examples +# ./install.sh .v2.4.0 +# /data/CCBR_Pipeliner/Pipelines/CARLISLE/dev/install.sh /data/CCBR_Pipeliner/Pipelines/CARLISLE/.v2.5.0-a +set -euo pipefail + +VERSION=$1 +mkdir -p ${VERSION}/bin +INSTALL_PATH=$(readlink -f ${VERSION}/bin) +DIRNAME=$(readlink -f $(dirname $0)) + +if [ -n "$(ls -A $INSTALL_PATH 2>/dev/null)" ] +then + echo "ERROR: directory not empty: ${INSTALL_PATH}" + echo $(ls $INSTALL_PATH) + exit 1 +fi + +# copy essential files & directories + +## carlisle CLI script +cp $DIRNAME/carlisle $INSTALL_PATH/ + +## all config & workflow scripts; +for subdir in config workflow/scripts +do + mkdir -p ${INSTALL_PATH}/$subdir + cp ${DIRNAME}/$subdir/* ${INSTALL_PATH}/$subdir +done +cp ${DIRNAME}/workflow/Snakefile ${INSTALL_PATH}/workflow + +## selected resources +mkdir -p ${INSTALL_PATH}/resources/ +cp ${DIRNAME}/resources/*.yaml ${INSTALL_PATH}/resources/ + +# export path +if [[ ":$PATH:" != *":${INSTALL_PATH}:"* ]];then + export PATH="${PATH}:${INSTALL_PATH}" +fi + +echo "${INSTALL_PATH}" From e2360c7853beedf732a48e34262c9affead528b3 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 18 Oct 2023 14:28:12 -0400 Subject: [PATCH 02/10] fix: don't call module purge or else it unloads ccbrpipeliner --- carlisle | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/carlisle b/carlisle index 175df12..35b523a 100755 --- a/carlisle +++ b/carlisle @@ -14,7 +14,6 @@ SNAKEMAKE_VERSION="snakemake/7.19.1" SINGULARITY_VERSION="singularity/3.10.5" set -eo pipefail -module purge SCRIPTNAME="$0" SCRIPTBASENAME=$(readlink -f $(basename $0)) @@ -443,4 +442,4 @@ function main(){ } # call the main function -main "$@" \ No newline at end of file +main "$@" From af8c132c2c877e6e77f82619442157359d27b729 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 18 Oct 2023 14:29:26 -0400 Subject: [PATCH 03/10] fix: copy annotation dir --- carlisle | 2 +- install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/carlisle b/carlisle index 35b523a..6e0b37c 100755 --- a/carlisle +++ b/carlisle @@ -28,7 +28,7 @@ tools_specific_yaml="tools_biowulf.yaml" # these are relative to the workflows' base folder # these are copied into the WORKDIR ESSENTIAL_FILES="config/config.yaml config/samples.tsv config/contrasts.tsv config/fqscreen_config.conf config/multiqc_config.yaml resources/cluster_* resources/tools_*" -ESSENTIAL_FOLDERS="workflow/scripts" +ESSENTIAL_FOLDERS="workflow/scripts annotation" # set extra singularity bindings EXTRA_SINGULARITY_BINDS="-B /data/CCBR_Pipeliner/,/lscratch" diff --git a/install.sh b/install.sh index 55b4aa9..431a869 100755 --- a/install.sh +++ b/install.sh @@ -24,7 +24,7 @@ fi cp $DIRNAME/carlisle $INSTALL_PATH/ ## all config & workflow scripts; -for subdir in config workflow/scripts +for subdir in config workflow/scripts annotation do mkdir -p ${INSTALL_PATH}/$subdir cp ${DIRNAME}/$subdir/* ${INSTALL_PATH}/$subdir From 57f7d79ff4bd5062ff776134d69346250328d148 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 18 Oct 2023 14:31:32 -0400 Subject: [PATCH 04/10] fix: copy .test dir --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 431a869..976c916 100755 --- a/install.sh +++ b/install.sh @@ -24,7 +24,7 @@ fi cp $DIRNAME/carlisle $INSTALL_PATH/ ## all config & workflow scripts; -for subdir in config workflow/scripts annotation +for subdir in config workflow/scripts annotation .test do mkdir -p ${INSTALL_PATH}/$subdir cp ${DIRNAME}/$subdir/* ${INSTALL_PATH}/$subdir From 6df550e59ab8986faea56c81d6dac8b85a318c04 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 18 Oct 2023 14:42:38 -0400 Subject: [PATCH 05/10] fix: just copy the entire git repo dir during 'installation' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤷🏻‍♀️ --- install.sh | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/install.sh b/install.sh index 976c916..810e05b 100755 --- a/install.sh +++ b/install.sh @@ -18,22 +18,8 @@ then exit 1 fi -# copy essential files & directories - -## carlisle CLI script -cp $DIRNAME/carlisle $INSTALL_PATH/ - -## all config & workflow scripts; -for subdir in config workflow/scripts annotation .test -do - mkdir -p ${INSTALL_PATH}/$subdir - cp ${DIRNAME}/$subdir/* ${INSTALL_PATH}/$subdir -done -cp ${DIRNAME}/workflow/Snakefile ${INSTALL_PATH}/workflow - -## selected resources -mkdir -p ${INSTALL_PATH}/resources/ -cp ${DIRNAME}/resources/*.yaml ${INSTALL_PATH}/resources/ +# copy entire repo, including dotfiles +cp -r ${DIRNAME}/. ${INSTALL_PATH} # export path if [[ ":$PATH:" != *":${INSTALL_PATH}:"* ]];then From 0041346c678010a79188eb0064d6f3289044c40e Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 18 Oct 2023 14:50:43 -0400 Subject: [PATCH 06/10] fix: copy cluster-specific config & tool yaml files without wildcards --- carlisle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/carlisle b/carlisle index 6e0b37c..b5f8d30 100755 --- a/carlisle +++ b/carlisle @@ -27,7 +27,7 @@ tools_specific_yaml="tools_biowulf.yaml" # essential files # these are relative to the workflows' base folder # these are copied into the WORKDIR -ESSENTIAL_FILES="config/config.yaml config/samples.tsv config/contrasts.tsv config/fqscreen_config.conf config/multiqc_config.yaml resources/cluster_* resources/tools_*" +ESSENTIAL_FILES="config/config.yaml config/samples.tsv config/contrasts.tsv config/fqscreen_config.conf config/multiqc_config.yaml" ESSENTIAL_FOLDERS="workflow/scripts annotation" # set extra singularity bindings EXTRA_SINGULARITY_BINDS="-B /data/CCBR_Pipeliner/,/lscratch" @@ -103,8 +103,8 @@ function init() { done # rename config dependent on partition used - cp $WORKDIR/config/$cluster_specific_yaml $WORKDIR/config/cluster.yaml - cp $WORKDIR/config/$tools_specific_yaml $WORKDIR/config/tools.yaml + cp ${PIPELINE_HOME}/config/$cluster_specific_yaml $WORKDIR/config/cluster.yaml + cp ${PIPELINE_HOME}/config/$tools_specific_yaml $WORKDIR/config/tools.yaml # copy essential folders for f in $ESSENTIAL_FOLDERS;do From 25c0b1af20e6301b5bdebd64c994c3b6c1e6eeaa Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 19 Oct 2023 08:27:03 -0400 Subject: [PATCH 07/10] fix: copy cluster/tool conf from resources --- carlisle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/carlisle b/carlisle index b5f8d30..37a50e0 100755 --- a/carlisle +++ b/carlisle @@ -103,8 +103,8 @@ function init() { done # rename config dependent on partition used - cp ${PIPELINE_HOME}/config/$cluster_specific_yaml $WORKDIR/config/cluster.yaml - cp ${PIPELINE_HOME}/config/$tools_specific_yaml $WORKDIR/config/tools.yaml + cp ${PIPELINE_HOME}/resources/$cluster_specific_yaml $WORKDIR/config/cluster.yaml + cp ${PIPELINE_HOME}/resources/$tools_specific_yaml $WORKDIR/config/tools.yaml # copy essential folders for f in $ESSENTIAL_FOLDERS;do From 01230a86dd928fd3fe1f07dc9182400295826776 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 19 Oct 2023 08:27:19 -0400 Subject: [PATCH 08/10] Revert "fix: just copy the entire git repo dir during 'installation'" This reverts commit 6df550e59ab8986faea56c81d6dac8b85a318c04. --- install.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 810e05b..976c916 100755 --- a/install.sh +++ b/install.sh @@ -18,8 +18,22 @@ then exit 1 fi -# copy entire repo, including dotfiles -cp -r ${DIRNAME}/. ${INSTALL_PATH} +# copy essential files & directories + +## carlisle CLI script +cp $DIRNAME/carlisle $INSTALL_PATH/ + +## all config & workflow scripts; +for subdir in config workflow/scripts annotation .test +do + mkdir -p ${INSTALL_PATH}/$subdir + cp ${DIRNAME}/$subdir/* ${INSTALL_PATH}/$subdir +done +cp ${DIRNAME}/workflow/Snakefile ${INSTALL_PATH}/workflow + +## selected resources +mkdir -p ${INSTALL_PATH}/resources/ +cp ${DIRNAME}/resources/*.yaml ${INSTALL_PATH}/resources/ # export path if [[ ":$PATH:" != *":${INSTALL_PATH}:"* ]];then From d4952ee7cd05c86cc9bf0320ab12ca348d0b1f60 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 19 Oct 2023 09:10:04 -0400 Subject: [PATCH 09/10] refactor!: get version from installation path VERSION will be blank if it doesn't match the semantic versioning format https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string --- carlisle | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/carlisle b/carlisle index 37a50e0..c24238f 100755 --- a/carlisle +++ b/carlisle @@ -32,24 +32,22 @@ ESSENTIAL_FOLDERS="workflow/scripts annotation" # set extra singularity bindings EXTRA_SINGULARITY_BINDS="-B /data/CCBR_Pipeliner/,/lscratch" -function get_git_commitid_tag() { - # This function gets the latest git commit id and tag - # Input is PIPELINE_HOME folder which is a git folder - cd $1 - gid=$(git rev-parse HEAD) - tag=$(git describe --tags $gid 2>/dev/null) - echo -ne "$gid\t$tag" -} - # ## setting PIPELINE_HOME PIPELINE_HOME=$(readlink -f $(dirname "$0")) echo "Pipeline Dir: $PIPELINE_HOME" # set snakefile SNAKEFILE="${PIPELINE_HOME}/workflow/Snakefile" -# get github commit tag -GIT_COMMIT_TAG=$(get_git_commitid_tag $PIPELINE_HOME) -echo "Git Commit/Tag: $GIT_COMMIT_TAG" +function get_version_from_path() { + PIPELINE_HOME=$1 + VERSION="$(echo $PIPELINE_HOME | grep -oP 'v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?' - )" + echo -ne $VERSION +} + +# get version from directory name +# regex source: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string +#echo "Version: $VERSION" +echo "Version: $(get_version_from_path $PIPELINE_HOME)" function usage() { # This function prints generic usage of the wrapper script. From 852c79f40e0a013a3d7ed4cf7cf5ed5c455f28a1 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 19 Oct 2023 09:15:30 -0400 Subject: [PATCH 10/10] fix: recursively copy most directories --- install.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/install.sh b/install.sh index 976c916..59e7a29 100755 --- a/install.sh +++ b/install.sh @@ -24,16 +24,11 @@ fi cp $DIRNAME/carlisle $INSTALL_PATH/ ## all config & workflow scripts; -for subdir in config workflow/scripts annotation .test +for subdir in annotation config workflow resources .test do mkdir -p ${INSTALL_PATH}/$subdir - cp ${DIRNAME}/$subdir/* ${INSTALL_PATH}/$subdir + cp -r ${DIRNAME}/$subdir/* ${INSTALL_PATH}/$subdir done -cp ${DIRNAME}/workflow/Snakefile ${INSTALL_PATH}/workflow - -## selected resources -mkdir -p ${INSTALL_PATH}/resources/ -cp ${DIRNAME}/resources/*.yaml ${INSTALL_PATH}/resources/ # export path if [[ ":$PATH:" != *":${INSTALL_PATH}:"* ]];then