From 84f82fedb7bda46652dba164373bf983e233a9a3 Mon Sep 17 00:00:00 2001 From: pdimens Date: Thu, 21 Nov 2024 12:49:06 -0500 Subject: [PATCH 01/10] update test for new name --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 15306f65d..6e5c8d329 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -822,9 +822,9 @@ jobs: resources/buildforCI.sh - name: Clear Space uses: jlumbroso/free-disk-space@main - - name: harpy stitchparams + - name: harpy imputeparams shell: micromamba-shell {0} - run: harpy stitchparams -o params.file + run: harpy imputeparams -o params.file - name: harpy popgroup shell: micromamba-shell {0} run: harpy popgroup test/fastq From c470d1bb639f4b5229f0e609fbd3eb4b6a318eeb Mon Sep 17 00:00:00 2001 From: pdimens Date: Thu, 21 Nov 2024 13:06:20 -0500 Subject: [PATCH 02/10] fix tests --- .github/filters.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/filters.yml b/.github/filters.yml index ffee38109..24d98825d 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -164,7 +164,8 @@ assembly: &assembly - 'harpy/snakefiles/assembly.smk' - 'harpy/snakefiles/metassembly.smk' other: &other - - 'harpy/stitchparams.py' + - *common + - 'harpy/imputeparams.py' - 'harpy/popgroup.py' - 'harpy/hpc.py' modules: From 632f85337700828d3cafd26fa88bc0da6d061b79 Mon Sep 17 00:00:00 2001 From: pdimens Date: Thu, 21 Nov 2024 15:07:02 -0500 Subject: [PATCH 03/10] improve specificity of installation script --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4f8a5e2df..64cb03db8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,8 @@ readme = {file = "README.md", content-type = "text/markdown"} license = {text = "GPL-3.0"} [tool.setuptools.packages.find] -where = ["."] +include = ["harpy"] +exclude = ["test"] [tool.setuptools.package-data] "*" = ["*.*"] From 8eb7c93053ddf6af0cb436c64f6069816adf2f1a Mon Sep 17 00:00:00 2001 From: pdimens Date: Fri, 22 Nov 2024 10:34:07 -0500 Subject: [PATCH 04/10] fix name --- harpy/__main__.py | 2 +- harpy/{popgroups.py => popgroup.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename harpy/{popgroups.py => popgroup.py} (100%) diff --git a/harpy/__main__.py b/harpy/__main__.py index d7c89baa2..d8a805a6d 100644 --- a/harpy/__main__.py +++ b/harpy/__main__.py @@ -16,7 +16,7 @@ from . import simulate from . import snp from . import sv -from .popgroups import popgroup +from .popgroup import popgroup from .imputeparams import imputeparams click.rich_click.USE_MARKDOWN = True diff --git a/harpy/popgroups.py b/harpy/popgroup.py similarity index 100% rename from harpy/popgroups.py rename to harpy/popgroup.py From f2e03ee1e05041f3c4ebce45e00a771ebebd94ab Mon Sep 17 00:00:00 2001 From: pdimens Date: Fri, 22 Nov 2024 11:39:13 -0500 Subject: [PATCH 05/10] slean up --- pyproject.toml | 2 -- resources/buildlocal.sh | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 64cb03db8..610f9ba8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,5 @@ exclude = ["test"] [project.scripts] harpy = "harpy.__main__:cli" -[tool.setuptools_scm] - [project.urls] Homepage = "https://github.com/pdimens/harpy" diff --git a/resources/buildlocal.sh b/resources/buildlocal.sh index 3eadbd8a2..c05dab567 100755 --- a/resources/buildlocal.sh +++ b/resources/buildlocal.sh @@ -12,8 +12,8 @@ mkdir -p ${CONDA_PREFIX}/bin g++ harpy/bin/extractReads.cpp -O3 -o ${CONDA_PREFIX}/bin/extractReads # install harpy proper -pip install . --no-deps && \ - rm -r build harpy.egg-info +pip install --no-deps --disable-pip-version-check -e . && \ + rm -r build #harpy.egg-info # associated scripts chmod +x harpy/bin/* From 335b68331de62e145ae71561fa64bf055b413996 Mon Sep 17 00:00:00 2001 From: pdimens Date: Fri, 22 Nov 2024 11:39:17 -0500 Subject: [PATCH 06/10] add view --- harpy/__main__.py | 4 ++- harpy/view.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 harpy/view.py diff --git a/harpy/__main__.py b/harpy/__main__.py index d8a805a6d..1ef421d33 100644 --- a/harpy/__main__.py +++ b/harpy/__main__.py @@ -18,6 +18,7 @@ from . import sv from .popgroup import popgroup from .imputeparams import imputeparams +from . import view click.rich_click.USE_MARKDOWN = True click.rich_click.SHOW_ARGUMENTS = False @@ -44,6 +45,7 @@ def cli(): # main program cli.add_command(popgroup) cli.add_command(imputeparams) +cli.add_command(view.view) cli.add_command(preflight.preflight) cli.add_command(demultiplex.demultiplex) cli.add_command(qc.qc) @@ -70,7 +72,7 @@ def cli(): }, { "name": "Other Commands", - "commands": sorted(["deconvolve", "hpc", "imputeparams", "popgroup","preflight","resume"]) + "commands": sorted(["deconvolve", "hpc", "imputeparams", "popgroup","preflight","resume", "view"]) } ], } | simulate.commandstring | hpc.docstring diff --git a/harpy/view.py b/harpy/view.py new file mode 100644 index 000000000..1f5bdc6cd --- /dev/null +++ b/harpy/view.py @@ -0,0 +1,66 @@ +"""View the latest log or snakefile of a workflow""" + +import os +import sys +import glob +import subprocess +import rich_click as click +from ._printing import print_error +from ._validations import is_gzip + +@click.command(no_args_is_help = True, context_settings=dict(allow_interspersed_args=False)) +@click.option('-s', '--snakefile', is_flag = True, show_default = True, default = False, help = "View the snakefile, not the log file") +@click.argument('inputdir', required=True, type=click.Path(exists=True, file_okay=False)) +def view(inputdir, snakefile): + """ + View a workflow log file or snakefile + + This convenience command lets you view the latest workflow log file + of a workflow directory. You can use `--snakefile` to view the workflow + snakefile instead. The output will be printed to your screen via the `less` command, + so use standard `less` keyboard shortcuts to navigate the output, e.g.: + + | key | function | + |:---:|:---| + |up/down arrow | scroll up/down | + |page up/down | scroll up/down, but faster | + |`q` | exit | + """ + # check if there is a workflow or log folder + # and whether the expected files are in there + err = 0 + err_text = "" + if snakefile: + files = [i for i in glob.iglob(f"{inputdir}/workflow/*.smk")] + if not os.path.exists(f"{inputdir}/workflow"): + err = 1 + err_text += f"{inputdir}/workflow/" + elif not files: + err = 2 + err_text += "snakefiles" + else: + files = [i for i in glob.iglob(f"{inputdir}/logs/snakemake/*.log*")] + if not os.path.exists(f"{inputdir}/logs/snakemake"): + err = 1 + err_text += f"{inputdir}/logs/snakemake/" + elif not files: + err = 2 + err_text += "log files" + if err == 1: + print_error( + "Directory not found", + f"The file you are trying to view is expected to be in [blue]{err_text}[/blue], but that directory was not found. Please check that this is the correct folder." + ) + sys.exit(1) + elif err == 2: + print_error( + "File not found", + f"There are no {err_text} in the directory provided. Please check that this is the correct folder." + ) + sys.exit(1) + # sort and pull only the most recent file (based on modification time) + file = sorted(files, key = os.path.getmtime)[-1] + cat_cmd = "zcat" if is_gzip(file) else "cat" + stream = subprocess.Popen([cat_cmd, file], stdout=subprocess.PIPE) + pygment = subprocess.Popen(["pygmentize", "-l", "yaml"], stdin = stream.stdout, stdout = subprocess.PIPE) + subprocess.run(["less", "-R"], stdin = pygment.stdout) From c4259f6a31768958189c266e7ebf80e5a2afd4ca Mon Sep 17 00:00:00 2001 From: pdimens Date: Fri, 22 Nov 2024 13:47:34 -0500 Subject: [PATCH 07/10] much more convenient build --- resources/buildlocal.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/buildlocal.sh b/resources/buildlocal.sh index c05dab567..042e1aeeb 100755 --- a/resources/buildlocal.sh +++ b/resources/buildlocal.sh @@ -13,7 +13,7 @@ g++ harpy/bin/extractReads.cpp -O3 -o ${CONDA_PREFIX}/bin/extractReads # install harpy proper pip install --no-deps --disable-pip-version-check -e . && \ - rm -r build #harpy.egg-info + rm -rf build #harpy.egg-info # associated scripts chmod +x harpy/bin/* From 111121ab536d874bdd4b7431bac53716e7b2941c Mon Sep 17 00:00:00 2001 From: pdimens Date: Fri, 22 Nov 2024 13:47:41 -0500 Subject: [PATCH 08/10] add harpy view --- harpy/view.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/harpy/view.py b/harpy/view.py index 1f5bdc6cd..0edaba7e8 100644 --- a/harpy/view.py +++ b/harpy/view.py @@ -10,39 +10,39 @@ @click.command(no_args_is_help = True, context_settings=dict(allow_interspersed_args=False)) @click.option('-s', '--snakefile', is_flag = True, show_default = True, default = False, help = "View the snakefile, not the log file") -@click.argument('inputdir', required=True, type=click.Path(exists=True, file_okay=False)) -def view(inputdir, snakefile): +@click.argument('directory', required=True, type=click.Path(exists=True, file_okay=False)) +def view(directory, snakefile): """ View a workflow log file or snakefile This convenience command lets you view the latest workflow log file - of a workflow directory. You can use `--snakefile` to view the workflow + of a Harpy output directory. You can use `--snakefile` to view the workflow snakefile instead. The output will be printed to your screen via the `less` command, so use standard `less` keyboard shortcuts to navigate the output, e.g.: - | key | function | - |:---:|:---| - |up/down arrow | scroll up/down | - |page up/down | scroll up/down, but faster | - |`q` | exit | + | key | function | + | :---------------------- | :------------------------- | + | `up arrow`/`down arrow` | scroll up/down | + | `Page Up`/`Page Down` | scroll up/down, but faster | + | `q` | exit | """ # check if there is a workflow or log folder # and whether the expected files are in there err = 0 err_text = "" if snakefile: - files = [i for i in glob.iglob(f"{inputdir}/workflow/*.smk")] - if not os.path.exists(f"{inputdir}/workflow"): + files = [i for i in glob.iglob(f"{directory}/workflow/*.smk")] + if not os.path.exists(f"{directory}/workflow"): err = 1 - err_text += f"{inputdir}/workflow/" + err_text += f"{directory}/workflow/" elif not files: err = 2 err_text += "snakefiles" else: - files = [i for i in glob.iglob(f"{inputdir}/logs/snakemake/*.log*")] - if not os.path.exists(f"{inputdir}/logs/snakemake"): + files = [i for i in glob.iglob(f"{directory}/logs/snakemake/*.log*")] + if not os.path.exists(f"{directory}/logs/snakemake"): err = 1 - err_text += f"{inputdir}/logs/snakemake/" + err_text += f"{directory}/logs/snakemake/" elif not files: err = 2 err_text += "log files" From 84e26001e65a7523643d83b8057bb52853eadfb0 Mon Sep 17 00:00:00 2001 From: pdimens Date: Fri, 22 Nov 2024 13:54:08 -0500 Subject: [PATCH 09/10] add read access check --- harpy/view.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/harpy/view.py b/harpy/view.py index 0edaba7e8..2de704fae 100644 --- a/harpy/view.py +++ b/harpy/view.py @@ -29,37 +29,43 @@ def view(directory, snakefile): # check if there is a workflow or log folder # and whether the expected files are in there err = 0 - err_text = "" if snakefile: files = [i for i in glob.iglob(f"{directory}/workflow/*.smk")] + err_dir = f"{directory}/workflow/" + err_file = "snakefiles" if not os.path.exists(f"{directory}/workflow"): err = 1 - err_text += f"{directory}/workflow/" elif not files: err = 2 - err_text += "snakefiles" else: files = [i for i in glob.iglob(f"{directory}/logs/snakemake/*.log*")] + err_dir = f"{directory}/logs/snakemake/" + err_file = "log files" if not os.path.exists(f"{directory}/logs/snakemake"): err = 1 - err_text += f"{directory}/logs/snakemake/" elif not files: err = 2 - err_text += "log files" if err == 1: print_error( "Directory not found", - f"The file you are trying to view is expected to be in [blue]{err_text}[/blue], but that directory was not found. Please check that this is the correct folder." + f"The file you are trying to view is expected to be in [blue]{err_dir}[/blue], but that directory was not found. Please check that this is the correct folder." ) sys.exit(1) elif err == 2: print_error( "File not found", - f"There are no {err_text} in the directory provided. Please check that this is the correct folder." + f"There are no {err_file} in [blue]{err_dir}[/blue]. Please check that this is the correct folder." ) sys.exit(1) # sort and pull only the most recent file (based on modification time) file = sorted(files, key = os.path.getmtime)[-1] + if not os.access(file, os.R_OK): + print_error( + "Incorrect permissions", + f"[blue]{file}[/blue] does not have read access. Please check the file permissions." + ) + sys.exit(1) + cat_cmd = "zcat" if is_gzip(file) else "cat" stream = subprocess.Popen([cat_cmd, file], stdout=subprocess.PIPE) pygment = subprocess.Popen(["pygmentize", "-l", "yaml"], stdin = stream.stdout, stdout = subprocess.PIPE) From 1475a937d1b47ae516cbfe4cec4addad51879d98 Mon Sep 17 00:00:00 2001 From: pdimens Date: Fri, 22 Nov 2024 14:00:45 -0500 Subject: [PATCH 10/10] improve phrasing --- harpy/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/harpy/view.py b/harpy/view.py index 2de704fae..48b2f1c8e 100644 --- a/harpy/view.py +++ b/harpy/view.py @@ -17,8 +17,8 @@ def view(directory, snakefile): This convenience command lets you view the latest workflow log file of a Harpy output directory. You can use `--snakefile` to view the workflow - snakefile instead. The output will be printed to your screen via the `less` command, - so use standard `less` keyboard shortcuts to navigate the output, e.g.: + snakefile instead. Output is printed to the screen via `less` and + accepts the typical keyboard shortcuts to navigate the output, e.g.: | key | function | | :---------------------- | :------------------------- |