From 5d15e037ece9cd1846724d1587f440737fbd11c6 Mon Sep 17 00:00:00 2001 From: Lael Barlow Date: Wed, 5 Jan 2022 14:46:22 -0700 Subject: [PATCH] Remove unnecessary Makefile and associated scripts. --- Makefile | 49 ------ scripts/activate_python_env.sh | 7 - scripts/archive_workflow.sh | 14 -- scripts/clean_virtualenvs.sh | 25 --- scripts/deactivate_python_env.sh | 11 -- scripts/determine_snakemake_env_options.sh | 17 -- scripts/determine_snakemake_profile.sh | 24 --- scripts/dry_run.sh | 29 ---- scripts/install.sh | 179 --------------------- scripts/quantify_test_coverage.py | 42 ----- scripts/run_data_setup.sh | 22 --- scripts/run_fwd_only.sh | 22 --- scripts/run_workflow.sh | 23 --- scripts/uninstall.sh | 19 --- scripts/unlock_workflow.sh | 10 -- scripts/workflow_python_env_definition.sh | 131 --------------- 16 files changed, 624 deletions(-) delete mode 100644 Makefile delete mode 100644 scripts/activate_python_env.sh delete mode 100644 scripts/archive_workflow.sh delete mode 100644 scripts/clean_virtualenvs.sh delete mode 100644 scripts/deactivate_python_env.sh delete mode 100644 scripts/determine_snakemake_env_options.sh delete mode 100644 scripts/determine_snakemake_profile.sh delete mode 100644 scripts/dry_run.sh delete mode 100644 scripts/install.sh delete mode 100644 scripts/quantify_test_coverage.py delete mode 100644 scripts/run_data_setup.sh delete mode 100644 scripts/run_fwd_only.sh delete mode 100644 scripts/run_workflow.sh delete mode 100644 scripts/uninstall.sh delete mode 100644 scripts/unlock_workflow.sh delete mode 100644 scripts/workflow_python_env_definition.sh diff --git a/Makefile b/Makefile deleted file mode 100644 index 7da5462..0000000 --- a/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -# Makefile for the AMOEBAE Example Workflow repository. - -SHELL=/bin/bash - -install: - bash scripts/install.sh - -dry_run: - bash scripts/dry_run.sh - -run_data_setup: - nohup bash scripts/run_data_setup.sh & echo $$! > pid_nohup.txt - -run_fwd_only: - nohup bash scripts/run_fwd_only.sh & echo $$! > pid_nohup.txt - -run: - nohup bash scripts/run_workflow.sh & echo $$! > pid_nohup.txt - -archive: - nohup bash scripts/archive_workflow.sh & echo $$! > pid_nohup.txt - -killall: - kill -9 `cat pid_nohup.txt` - rm pid_nohup.txt - killall -TERM snakemake - echo Jobs may still be running on the cluster. Cancel those manually, if necessary. - -clean: - -rm nohup.out - -rm pid_nohup.txt - -clean_results: - rm -rf results/* - -clean_virtualenvs: - bash scripts/clean_virtualenvs.sh - -unlock: - bash scripts/unlock_workflow.sh - -uninstall: - bash scripts/uninstall.sh - -.PHONY: list -list: - @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' - - diff --git a/scripts/activate_python_env.sh b/scripts/activate_python_env.sh deleted file mode 100644 index caea597..0000000 --- a/scripts/activate_python_env.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -printf "\nActivating Python environment. Enter 'deactivate' when finished.\n\n" - -# Activate python virtual environment. -source scripts/workflow_python_env_definition.sh - diff --git a/scripts/archive_workflow.sh b/scripts/archive_workflow.sh deleted file mode 100644 index e705538..0000000 --- a/scripts/archive_workflow.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Activate python virtual environment. -source scripts/workflow_python_env_definition.sh - -# Define a timestamp for the archive file. -TIMESTAMP=`date +%Y-%m-%d_%H-%M-%S` - -# Run snakemake in python virtual environment. -snakemake --archive amoebae_workflow_$TIMESTAMP.tar.gz --cores 1 - -# Deactivate python virtual environment. -source scripts/deactivate_python_env.sh - diff --git a/scripts/clean_virtualenvs.sh b/scripts/clean_virtualenvs.sh deleted file mode 100644 index f1931c4..0000000 --- a/scripts/clean_virtualenvs.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Script for removing all files associated with amoebae_example_workflow -# virtual environments. This is so that they can be re-generated when the code -# is modified. - -# Define a function to confirm with the user before deleting files. -confirm() { - echo -n "Do you want to run $*? [N/y] " - read -N 1 REPLY - echo - if test "$REPLY" = "y" -o "$REPLY" = "Y"; then - "$@" - else - echo "Cancelled by user" - fi -} - -# Remove files associated with virtual environments. -if test "$(command -v conda)"; then - confirm conda env remove --name conda_env_amoebae_snakemake_workflow -else - confirm rm -rf ~/env_amoebae_snakemake_workflow -fi - - diff --git a/scripts/deactivate_python_env.sh b/scripts/deactivate_python_env.sh deleted file mode 100644 index 2874691..0000000 --- a/scripts/deactivate_python_env.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -# Deactivate python virtual environment. -if test "$(command -v conda)"; then - # Deactivate Conda environment. - conda deactivate -else - # Assume a Python venv environment was used instead of Conda. - deactivate -fi - diff --git a/scripts/determine_snakemake_env_options.sh b/scripts/determine_snakemake_env_options.sh deleted file mode 100644 index 7980a6a..0000000 --- a/scripts/determine_snakemake_env_options.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# Determine which environment management options to use. -env_options="" - -if test "$(command -v conda)"; then - env_options="--use-conda" - -elif test "$(command -v module)"; then - #env_options="--use-envmodules" - env_options="" - -else - env_options="--use-conda --use-singularity" - -fi - diff --git a/scripts/determine_snakemake_profile.sh b/scripts/determine_snakemake_profile.sh deleted file mode 100644 index a14947e..0000000 --- a/scripts/determine_snakemake_profile.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -# Determine what string to use to define Snakemake profile in current -# environment. - -snakemake_profile="" - -if test "$(command -v sbatch)"; then - echo SLURM job scheduler detected. - snakemake_profile="slurm_amoebae" - -elif qstat -help; then - printf "(SUN/Univa) Grid Engine job scheduler detected." - snakemake_profile="sge_amoebae" - -elif pbs-config --version; then - printf "PBS-TORQUE job scheduler detected." - snakemake_profile="pbs-torque_amoebae" - -else - echo Unable to detect a job scheduler. - #exit 1 - -fi diff --git a/scripts/dry_run.sh b/scripts/dry_run.sh deleted file mode 100644 index 9fc0917..0000000 --- a/scripts/dry_run.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -# Determine what Snakemake profile to use. -source scripts/determine_snakemake_profile.sh - -# Determine whether to use the --profile option at all. -profile_option="--profile" -if [ "$snakemake_profile" == "" ]; then - profile_option="--cores 1" -fi - -# Determine which environment management options to use. -source scripts/determine_snakemake_env_options.sh - -# Activate python virtual environment. -source scripts/workflow_python_env_definition.sh - -# Run snakemake command. -echo "Python version:" && \ -python3 --version && \ -echo "Snakemake version:" && \ -snakemake --version && \ -snakemake -n --snakefile workflow/Snakefile && \ -snakemake plot_workflow -j 100 $env_options $profile_option $snakemake_profile \ ---snakefile workflow/Snakefile --verbose - -# Deactivate python virtual environment. -source scripts/deactivate_python_env.sh - diff --git a/scripts/install.sh b/scripts/install.sh deleted file mode 100644 index a63b136..0000000 --- a/scripts/install.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash - -# Script for setting up amoebae_example_workflow. - -# Start timing this script. -SECONDS=0 - -# Set to exit immediately if any commands return an error. -set -e - -# Detect job scheduler. -source scripts/determine_snakemake_profile.sh - -# Determine whether packages can be loaded using the 'module' command. -if test "$(command -v module)"; then - # Use the 'module' command to load Python module. - - # Load appropriate version of Python. - module load python/3.7 - -else - # Print the version. - echo "" - echo Using this version of Python3: - python3 --version - -fi - -# Check that Python can be used. -if test "$(command -v python3)"; then - echo "" -else - echo Unable to access Python3. - exit 1 -fi - -# Define path to python environment directory. -PYENVDIR=~/env_amoebae_workflow_setup_py - -# Set up a python environment with cookiecutter. -if [ ! -d "$PYENVDIR" ]; then - #virtualenv --no-download $PYENVDIR - python3 -m venv $PYENVDIR - source $PYENVDIR/bin/activate - pip install --upgrade pip - pip install cookiecutter - pip install -U pytest -else - source $PYENVDIR/bin/activate -fi - - -if [ "$(uname)" == "Darwin" ]; then - printf "\nDetected MacOS." - # Do something under Mac OS X platform - echo "No further setup required." - -elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then - printf "\nDetected Linux.\n\n" - # Do something under GNU/Linux platform - - # Determine what job scheduler is in use, and set up the appropriate - # SnakeMake profile. - if [ "$snakemake_profile" == "slurm_amoebae" ]; then - printf "\nSetting up config files for running SnakeMake with the SLURM job scheduler.\n\n" - # Install configuration files for running on a SLURM cluster - # (https://github.com/Snakemake-Profiles/slurm). - - # Removing existing directory, if present. - if [ -d ~/.config/snakemake/slurm_amoebae ]; then - rm -rf ~/.config/snakemake/slurm_amoebae - fi - - mkdir -p ~/.config/snakemake - cd ~/.config/snakemake - cookiecutter https://github.com/Snakemake-Profiles/slurm.git \ - profile_name="slurm_amoebae" \ - cluster_config="cluster_config.yaml" - cd - - # Now you can run snakemake as: snakemake --profile slurm_amoebae ... - - # Copy amoebae_cluster_config.yaml example file from resources directory to - # profile files. - cp resources/example_slurm_cluster_config.yaml \ - ~/.config/snakemake/slurm_amoebae/cluster_config.yaml - - ## Update value of the cluster_config variable in the submission - ## python file with the name of the cluster_config.yaml file. - #sed -i 's/CLUSTER_CONFIG = \"\"/CLUSTER_CONFIG = \"cluster_config.yaml\"/g' \ - # ~/.config/snakemake/slurm_amoebae/slurm-submit.py - - # Copy over slurm-jobscript.sh file in the profile directory (this - # replaces the default script with a script that loads modules and - # activates a python virtual environment for every job). - cp resources/example_slurm-jobscript.sh \ - ~/.config/snakemake/slurm_amoebae/slurm-jobscript.sh - - elif [ "$snakemake_profile" == "sge_amoebae" ]; then - printf "\nSetting up config files for running SnakeMake with the (SUN/Univa) Grid Engine job scheduler.\n\n" - - # Install configuration files for running on a cluster with the Grid - # Engine scheduler. - - # Removing existing directory, if present. - if [ -d ~/.config/snakemake/sge_amoebae ]; then - rm -rf ~/.config/snakemake/sge_amoebae - fi - - mkdir -p ~/.config/snakemake - cd ~/.config/snakemake - # Make profile directory using cookiecutter to install from GitHub. - cookiecutter https://github.com/Snakemake-Profiles/sge.git \ - profile_name="sge_amoebae" - # Make Python scripts in profile executable. - chmod a+x ~/.config/snakemake/sge_amoebae/*.py - # Change back to previous directory. - cd - - # Now you can run snakemake as: snakemake --profile sge_amoebae ... - - - elif [ "$snakemake_profile" == "pbs-torque" ]; then - # Install configuration files for running on a cluster with the - # PBS-TORQUE scheduler. - - printf "\nSetting up config files for running SnakeMake with the - PBS-TORQUE job scheduler.\n\n" - - # Removing existing directory, if present. - if [ -d ~/.config/snakemake/pbs-torque_amoebae ]; then - rm -rf ~/.config/snakemake/pbs-torque_amoebae - fi - - mkdir -p ~/.config/snakemake - cd ~/.config/snakemake - # Make profile directory using cookiecutter to install from GitHub. - cookiecutter https://github.com/Snakemake-Profiles/pbs-torque.git \ - profile_name="pbs-torque_amoebae" - # Make Python scripts in profile executable. - chmod a+x ~/.config/snakemake/pbs-torque_amoebae/*.py - # Change back to previous directory. - cd - - # Now you can run snakemake as: snakemake --profile pbs-torque_amoebae ... - - else - printf "\nUnable to automatically set up config files for submitting - jobs through the job scheduler on this system. You must do this - manually.\n\n" - fi - - -elif [ "$(expr substr $(uname -s) 1 10)" == "MINGW32_NT" ]; then - printf "\nDetected Windows." - # Do something under 32 bits Windows NT platform - echo "This software has not been tested on Windows, as it has been designed to run on linux clusters." - exit 1 - -elif [ "$(expr substr $(uname -s) 1 10)" == "MINGW64_NT" ]; then - printf "\nDetected Windows." - # Do something under 64 bits Windows NT platform - echo "This software has not been tested on Windows, as it has been designed to run on linux clusters." - exit 1 -fi - - - -####################################################### - -# Deactivate and remove Python environment. -deactivate -rm -rf $PYENVDIR - - -####################################################### -# Report how much time it took for this script to run. -echo "" -ELAPSED="Setting up this workflow took the following amount of time: $(($SECONDS / 3600))hrs $((($SECONDS / 60) % 60))min $(($SECONDS % 60))sec" -echo $ELAPSED - - diff --git a/scripts/quantify_test_coverage.py b/scripts/quantify_test_coverage.py deleted file mode 100644 index 2c7202c..0000000 --- a/scripts/quantify_test_coverage.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 - -import glob -import os -import matplotlib.pyplot as plt; plt.rcdefaults() -import numpy as np -import matplotlib.pyplot as plt - - -# Count how many function definitions there are. -funct_def_count = 0 -for d in ['amoebaelib', 'misc_scripts']: - for f in glob.glob(os.path.join(d, '*.py')): - with open(f) as infh: - for i in infh: - if i.startswith("def "): - #print(i.strip()) - funct_def_count += 1 -print("Total number of function definitions: " + str(funct_def_count)) - -# Count how many functions have tests in files in the tests directory. -tested_def_count = 0 -for d in ['tests']: - for f in glob.glob(os.path.join(d, '*.py')): - with open(f) as infh: - contents = infh.read() - split_on_defs = contents.split() - for i in infh: - if i.startswith("def "): - #print(i.strip()) - funct_def_count += 1 - -objects = ('Total Defs', 'Tested Defs') -y_pos = np.arange(len(objects)) -performance = [funct_def_count, 0] - -plt.bar(y_pos, performance, align='center', alpha=0.5) -plt.xticks(y_pos, objects) -plt.ylabel('Def count') -plt.title('Test coverage') - -plt.show() diff --git a/scripts/run_data_setup.sh b/scripts/run_data_setup.sh deleted file mode 100644 index 852fc7a..0000000 --- a/scripts/run_data_setup.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Determine what Snakemake profile to use. -source scripts/determine_snakemake_profile.sh - -# Determine whether to use the --profile option at all. -profile_option="--profile" -if [ "$snakemake_profile" == "" ]; then - profile_option="--cores 1" -fi - -# Determine which environment management options to use. -source scripts/determine_snakemake_env_options.sh - -# Activate python virtual environment. -source scripts/workflow_python_env_definition.sh - -# Run snakemake command. -snakemake get_ref_seqs -j 100 $env_options $profile_option $snakemake_profile --verbose - -# Deactivate python virtual environment. -source scripts/deactivate_python_env.sh diff --git a/scripts/run_fwd_only.sh b/scripts/run_fwd_only.sh deleted file mode 100644 index b37d5f8..0000000 --- a/scripts/run_fwd_only.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Determine what Snakemake profile to use. -source scripts/determine_snakemake_profile.sh - -# Determine whether to use the --profile option at all. -profile_option="--profile" -if [ "$snakemake_profile" == "" ]; then - profile_option="--cores 1" -fi - -# Determine which environment management options to use. -source scripts/determine_snakemake_env_options.sh - -# Activate python virtual environment. -source scripts/workflow_python_env_definition.sh - -# Run snakemake command. -snakemake get_fwd_hit_seqs -j 100 $env_options $profile_option $snakemake_profile --verbose - -# Deactivate python virtual environment. -source scripts/deactivate_python_env.sh diff --git a/scripts/run_workflow.sh b/scripts/run_workflow.sh deleted file mode 100644 index 03bf9a1..0000000 --- a/scripts/run_workflow.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# Determine what Snakemake profile to use. -source scripts/determine_snakemake_profile.sh - -# Determine whether to use the --profile option at all. -profile_option="--profile" -if [ "$snakemake_profile" == "" ]; then - profile_option="--cores 1" -fi - -# Determine which environment management options to use. -source scripts/determine_snakemake_env_options.sh - -# Activate python virtual environment. -source scripts/workflow_python_env_definition.sh - -# Run snakemake command. -snakemake -j 100 $env_options $profile_option $snakemake_profile --verbose - -# Deactivate python virtual environment. -source scripts/deactivate_python_env.sh - diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh deleted file mode 100644 index 4e0bc19..0000000 --- a/scripts/uninstall.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Script for removing all files associated with amoebae. - -# Define a function to confirm with the user before deleting files. -confirm() { - echo -n "Do you want to run $*? [N/y] " - read -N 1 REPLY - echo - if test "$REPLY" = "y" -o "$REPLY" = "Y"; then - "$@" - else - echo "Cancelled by user" - fi -} - -# Remove files. - -confirm rm -rf ../amoebae - diff --git a/scripts/unlock_workflow.sh b/scripts/unlock_workflow.sh deleted file mode 100644 index e63f7e9..0000000 --- a/scripts/unlock_workflow.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -# Activate python virtual environment. -source scripts/workflow_python_env_definition.sh - -# Run snakemake in python virtual environment. -snakemake --unlock --cores 1 - -# Deactivate python virtual environment. -source scripts/deactivate_python_env.sh diff --git a/scripts/workflow_python_env_definition.sh b/scripts/workflow_python_env_definition.sh deleted file mode 100644 index 241678c..0000000 --- a/scripts/workflow_python_env_definition.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash - -# Script for building and accessing virtual environments for running snakemake. -# If you are running this on a cluster that uses the Conda package manager, -# make sure that you have activated a Conda environment using an appropriate -# version of Python. - -# Determine snakemake profile name. -source scripts/determine_snakemake_profile.sh - -# Determine whether packages can be loaded using the 'module' command. -if test "$(command -v module)"; then - # Use the 'module' command to load Python and Singularity modules. - - # Load appropriate version of Python. - # (tested using 3.7.4). - module load python/3.7 - - # Load an appropriate version of Singularity. - module load singularity/3.6 - -else - # Print the versions. - echo "" - echo Using this version of Python: - python3 --version - echo "" - echo Using this version of Singularity: - singularity --version - -fi - -# Check that Python can be used. -if test "$(command -v python3)"; then - echo "" -else - echo Unable to access Python3. - exit 1 -fi - -# Check that Singularity can be used. -if test "$(command -v singularity)"; then - echo "" -else - echo Unable to access Singularity. - #exit 1 -fi - -# Define path to directory to contain python virtual environment files -# (installation of dependencies) -DIR=~/env_amoebae_snakemake_workflow - -# Build the virtual environment, if necessary. - -if test "$(command -v conda)"; then - echo Conda is available. - - # Install dependencies in virtual environment using conda. - if ! test "$(conda info --envs | grep "conda_env_amoebae_snakemake_workflow")"; then - #conda create --name conda_env_amoebae_snakemake_workflow python=3.7.4 - #source activate conda_env_amoebae_snakemake_workflow - - # Check that mamba is installed. - if ! test "$(command -v mamba)"; then - echo Installing mamba. - # Install Mamba for making snakemake install easier. - conda install -c conda-forge mamba - fi - - echo "" - echo Setting up environment for snakemake with mamba. - echo "" - - - # Create conda environment for snakemake with mamba. - mamba create -c conda-forge -c bioconda \ - -n conda_env_amoebae_snakemake_workflow \ - snakemake \ - matplotlib \ - numpy \ - graphviz \ - requests \ - pulp \ - - echo Environment set up. Re-run to activate. - exit 1 - - else - # Otherwise, just activate the environment. - echo Attempting to activate conda environment from script. - - # Get the path to the conda base environment. - conda_base_env=$(conda info | grep -i 'base environment' | tr -s ' ' | cut -d' ' -f5) - # Source conda from the base environment so that it can be used in a - # subshell. - source $conda_base_env/etc/profile.d/conda.sh - # Activate the conda environment. - conda activate conda_env_amoebae_snakemake_workflow - - # Add profile path to $PATH. - export PATH=$PATH:~/.config/snakemake/$snakemake_profile - - fi - -else - # Install dependencies in virtual environment using pip. - if [ ! -d "$DIR" ]; then - python3 -m venv $DIR - source $DIR/bin/activate - pip install --upgrade pip - pip install setuptools wheel - pip install requests - pip install \ - numpy \ - biopython==1.77 \ - gffutils==0.10.1 \ - pypdf2==1.26.0 \ - reportlab==3.5.51 \ - pandoc==2.0a4 \ - pandas==1.1.3 \ - matplotlib==3.3.2 \ - graphviz==0.14.2 \ - snakemake==5.26.1 - - else - # Otherwise, just activate the environment. - source $DIR/bin/activate - fi -fi - -