goodbye Rmarkdown, hello Quarto (#180)

* move static snakemake cli calls to outside

* add .conda here b/c that's the new name of the conda dir folder

* make sure prefix is respected in container

* migrate to qmd

* simplify

* lets make it materia

* touchups

* update logo

* finally update formatting

* replace preflight_bam with quarto

* clearer

* update qc to qmd

* updates

* new mol_cov on the block and it's way more efficient

* update to qmd

* improve text

* update to qmd

* migrate to quarto

* qmd migration

* migrate to fixed SM call

* add bai

* nicer ending text with time

* report to qmd

* leviathan to qmd

* update formatting

* update naibr to qmd

* update

* update to qmd

* migrate to qmd,  remove info footer from DT tables

* legend secretly table

* update

* better quarto rendering

* correct the log

* move quarto config to external yml

* touchups

* update to qmd, make borders dark

* start qmd migration

* update

* add pixi and conda install scripts

* add pixi

* add pixi global as preferred

* rm lines, they seem dumb now

* clarification

* swap to qmd

* replace

* complete transition

* update

* add hapcut to the quarto party

* looks like we've ran out of reasons keeping you around

* final touches

* better reort logo

* rm deprecated

* rm in favor of new dev scripts

* suggestions and fixes

* fix sql logic

* rm prefix (it's redundant)

* sleeker printing

* fix typo

* fix call

* fix quarto call

* better printing

* rm redundant F strings

.github/filters.yml

@@ -25,8 +25,8 @@ preflight: &preflight
   - 'test/bam/**'
   - 'harpy/bin/check_bam.py'
   - 'harpy/bin/check_fastq.py'
-  - 'harpy/reports/preflight_fastq.Rmd'
-  - 'harpy/reports/preflight_bam.Rmd'
+  - 'harpy/reports/preflight_fastq.qmd'
+  - 'harpy/reports/preflight_bam.qmd'
 deconvolve: &deconvolve
   - *common
   - *container
@@ -51,8 +51,8 @@ bwa: &bwa
   - *container
   - 'harpy/align.py'
   - 'harpy/snakefiles/align_bwa.smk'
-  - 'harpy/reports/align_stats.Rmd'
-  - 'harpy/reports/align_bxstats.Rmd'
+  - 'harpy/reports/align_stats.qmd'
+  - 'harpy/reports/align_bxstats.qmd'
   - 'harpy/bin/bx_stats.py'
   - 'harpy/bin/count_bx.py'
   - 'harpy/bin/assign_mi.py'
@@ -64,8 +64,8 @@ ema: &ema
   - *container
   - 'harpy/align.py'
   - 'harpy/snakefiles/align_ema.smk'
-  - 'harpy/reports/align_stats.Rmd'
-  - 'harpy/reports/align_bxstats.Rmd'
+  - 'harpy/reports/align_stats.qmd'
+  - 'harpy/reports/align_bxstats.qmd'
   - 'harpy/bin/bx_stats.py'
   - 'harpy/bin/count_bx.py'
   - 'harpy/bin/make_windows.py'
@@ -76,8 +76,8 @@ strobealign: &strobealign
   - *container
   - 'harpy/align.py'
   - 'harpy/snakefiles/align_strobealign.smk'
-  - 'harpy/reports/align_stats.Rmd'
-  - 'harpy/reports/align_bxstats.Rmd'
+  - 'harpy/reports/align_stats.qmd'
+  - 'harpy/reports/align_bxstats.qmd'
   - 'harpy/bin/assign_mi.py'
   - 'harpy/bin/bx_stats.py'
   - 'harpy/bin/count_bx.py'
@@ -89,15 +89,15 @@ mpileup: &mpileup
   - *container
   - 'harpy/snp.py'
   - 'harpy/snakefiles/snp_mpileup.smk'
-  - 'harpy/reports/bcftools_stats.Rmd'
+  - 'harpy/reports/bcftools_stats.qmd'
   - 'harpy/bin/make_windows.py'
   - 'test/bam/**'
 freebayes: &freebayes
   - *common
   - *container
   - 'harpy/snp.py'
   - 'harpy/snakefiles/snp_freebayes.smk'
-  - 'harpy/reports/bcftools_stats.Rmd'
+  - 'harpy/reports/bcftools_stats.qmd'
   - 'harpy/bin/make_windows.py'
   - 'test/bam/**'
 impute: &impute
@@ -107,8 +107,8 @@ impute: &impute
   - 'harpy/snakefiles/impute.smk'
   - 'test/bam/**'
   - 'test/vcf/test.bcf'
-  - 'harpy/reports/impute.Rmd'
-  - 'harpy/reports/stitch_collate.Rmd'
+  - 'harpy/reports/impute.qmd'
+  - 'harpy/reports/stitch_collate.qmd'
   - 'harpy/scripts/stitch_impute.R'
 leviathan: &leviathan
   - *common
@@ -117,7 +117,7 @@ leviathan: &leviathan
   - 'harpy/snakefiles/sv_leviathan**.smk'
   - 'test/bam/**'
   - 'harpy/bin/concatenate_bam.py'
-  - 'harpy/reports/leviathan**.Rmd'
+  - 'harpy/reports/leviathan**.qmd'
 naibr: &naibr
   - *common
   - *container
@@ -127,7 +127,7 @@ naibr: &naibr
   - 'test/bam/**'
   - 'test/bam_phased/**'
   - 'test/vcf/test.phased.bcf'
-  - 'harpy/reports/naibr**.Rmd'
+  - 'harpy/reports/naibr**.qmd'
   - 'harpy/bin/infer_sv.py'
 phase: &phase
   - *common
@@ -136,7 +136,7 @@ phase: &phase
   - 'harpy/snakefiles/phase.smk'
   - 'test/bam/**'
   - 'test/vcf/test.bcf'
-  - 'harpy/reports/hapcut.Rmd'
+  - 'harpy/reports/hapcut.qmd'
   - 'harpy/bin/parse_phaseblocks.py'
 simvars: &simvars
   - *common

.github/workflows/tests.yml

@@ -79,7 +79,7 @@ jobs:
           pip install dist/*.whl
           resources/buildforCI.sh
       - name: Clear Space
-        run: rm -rf /opt/hostedtoolcache
+        uses: jlumbroso/free-disk-space@main
       - name: Rebuild Dockerfile
         id: rebuild
         if: ${{ needs.changes.outputs.container == 'true' }}

.gitignore

@@ -1,6 +1,7 @@
 .snakemake/
 .vscode/
 .condarc
+.conda
 .cache/
 hpc/
 QC/
@@ -20,11 +21,21 @@ Metassembly/
 samples.populations
 *.fasta
 *.bcf
+*.bai
 genome.fasta
 build/
 harpy.egg-info/
 __pycache__/
 .Benchmark/
 extractReads
 haplotag.bc
-_Inline
+_Inline
+# pixi environments
+.pixi
+pixi*
+.gitattributes
+*.egg-info
+
+# pixi environments
+.pixi
+*.egg-info

README.md

@@ -9,16 +9,33 @@
 
 
 ## 📥 Install 
-To avoid dependency conflicts with an existing environment, it is best to create a new environment for a harpy installation. The code below creates a new conda/mamba environment called `harpy` (via `-n harpy`) and installs harpy into it. You can name this environment whatever you like using the `-n somename` argument. 
+### 🐍 Conda
+It's best to create a new environment for a harpy installation. The code below creates a new conda/mamba environment called `harpy` (via `-n harpy`) and installs harpy into it. You can name this environment whatever you like using the `-n somename` argument. 
 ```bash
 conda create -n harpy -c bioconda -c conda-forge harpy
 ```
 
+Once conda/mamba finishes, activate the harpy conda/mamba environment with:
+```bash
+conda activate env_name
+```
+where `env_name` is the name of that environment. After doing so, the `harpy` executable should be callable from your path.
+
 <details>
-  <summary>⚪️ install into an existing conda environment ⚪️</summary>
+  <summary>⬇️ install as local conda environment </summary>
+
+Alternatively, you can create the environment locally within a specific project folder, just swap `-n harpy` for
+`-p path/to/workdir/harpy`, which creates the environment in that specific folder (e.g. `potato_blight/harpy`).
+```
+# for local project directory
+conda create -p path/to/workdir/harpy -c bioconda -c conda-forge harpy
+```
+
+</details>
 
-  ---
-
+<details>
+  <summary>⬇️ install into existing conda environment </summary>
+
 If you wish to install harpy and its dependencies into an existing environment, activate that environment (`conda activate env_name`) and execute this installation code:
 ```bash
 conda install -c conda-forge bioconda::harpy
@@ -28,22 +45,65 @@ Or provide `-n envname` to install it into an existing environment named `envnam
 conda install -n envname -c conda-forge bioconda::harpy
 ```
 
----
-
 </details>
 
-## Update
+<details>
+  <summary>⬆️ updating harpy </summary>
+
+If installed via conda, you can update Harpy by activating the environment
+and running `conda update` like so:
+
 ```bash
 conda update -c conda-forge bioconda::harpy
 ```
 
-## 🌟 Activate the harpy environment
-Once conda/mamba finishes, activate the conda/mamba environment you installed harpy into with
+</details>
+
+### 🌟 Pixi
+If you prefer [Pixi](https://pixi.sh/latest/) (it's pretty good, you should try it), you can
+install Harpy to be accessible in your PATH-- just make sure `~/.pixi/bin` is in your PATH:
+```
+# ~/.zshrc or ~/.bashrc (or equivalent) 
+export PATH=~/.pixi/bin:$PATH
+```
 ```bash
-conda activate env_name
+pixi global install -c conda-forge -c bioconda harpy
 ```
-where `env_name` is the name of that environment. After doing so, the `harpy` executable should be callable from your path.
 
+<details>
+  <summary>⬇️ install as local environment </summary>
+
+Likewise, you can do an installation into a local project directory:
+
+```bash
+pixi init -c bioconda projectname && cd projectname
+pixi add harpy
+```
+After that finishes, you can activate the environment with:
+```bash
+pixi shell
+```
+Or run `harpy` by prefixing it with `pixi run`:
+```bash
+pixi run harpy
+```
+</details>
+
+<details>
+  <summary>⬆️ updating harpy </summary>
+
+If installed via Pixi, you can update Harpy with `pixi update`:
+```bash
+# global install
+pixi global update harpy
+
+# local install
+# project dir has the pixi.toml file
+cd path/to/projectdir
+pixi update harpy
+```
+
+</details>
 
 ## ⚡ Usage
 Just call `harpy` or `harpy --help` on the command line to get started!

harpy/_conda.py

@@ -50,11 +50,10 @@ def create_conda_recipes(outdir: str, envs: list=None) -> None:
             "conda-forge::pandoc",
             "conda-forge::r-dt",
             "conda-forge::r-dplyr",
-            "conda-forge::r-flexdashboard",
-            "conda-forge::r-ggplot2",
             "conda-forge::r-highcharter",
             "conda-forge::r-magrittr",
             "conda-forge::r-plotly",
+            "conda-forge::r-quarto",
             "conda-forge::r-scales",
             "conda-forge::r-stringi",
             "conda-forge::r-tidyr",

harpy/_launch.py

@@ -5,6 +5,7 @@
 import sys
 import glob
 import subprocess
+from datetime import datetime
 from rich import print as rprint
 from rich.console import Console
 from ._misc import gzip_file, harpy_progressbar, harpy_pulsebar
@@ -14,6 +15,7 @@
 EXIT_CODE_GENERIC_ERROR = 1
 EXIT_CODE_CONDA_ERROR = 2
 EXIT_CODE_RUNTIME_ERROR = 3
+SNAKEMAKE_CMD = "snakemake --rerun-incomplete --show-failed-logs --rerun-triggers input mtime params --nolock --conda-prefix .conda --conda-cleanup-pkgs cache --directory ."
 
 def iserror(text):
     """logical check for erroring trigger words in snakemake output"""
@@ -30,6 +32,7 @@ def purge_empty_logs(target_dir):
 
 def launch_snakemake(sm_args, workflow, starttext, outdir, sm_logfile, quiet, summaryfile = None):
     """launch snakemake with the given commands"""
+    start_time = datetime.now()
     if not quiet:
         print_onstart(starttext, workflow.replace("_", " "))
     exitcode = None
@@ -153,14 +156,16 @@ def launch_snakemake(sm_args, workflow, starttext, outdir, sm_logfile, quiet, su
             gzip_file(sm_logfile)
             purge_empty_logs(outdir)
             if not quiet:
-                print_onsuccess(outdir, summaryfile)
+                end_time = datetime.now()
+                elapsed_time = end_time - start_time
+                print_onsuccess(outdir, summaryfile, elapsed_time)
             sys.exit(0)
         else:
             if exitcode in (1,2):
                 print_setup_error(exitcode)
             elif exitcode == 3:
                 print_onerror(sm_logfile)
-            while output and not output.endswith("]") and not output.startswith("Shutting down"):
+            while output and not output.endswith("]") and not output.startswith("Shutting down"):                   
                 if "Exception" in output or "Error" in output:
                     rprint("[yellow bold]" + output.rstrip(), file = sys.stderr)
                     output = process.stderr.readline()
@@ -172,6 +177,8 @@ def launch_snakemake(sm_args, workflow, starttext, outdir, sm_logfile, quiet, su
                 if output:
                     if not output.startswith("Complete log"):
                         rprint("[red]" + output.replace("\t","    ").rstrip(), file = sys.stderr)
+                    if output.startswith("Removing output files of failed job"):
+                        break
                 output = process.stderr.readline()
             gzip_file(sm_logfile)
             purge_empty_logs(outdir)

harpy/_misc.py

@@ -85,6 +85,15 @@ def fetch_report(workdir, target):
             print_solution("There may be an issue with your Harpy installation, which would require reinstalling Harpy. Alternatively, there may be in a issue with your conda/mamba environment or configuration.")
             sys.exit(1)
 
+    with open(f"{workdir}/report/_quarto.yml", "w", encoding="utf-8") as yml:
+        if os.path.isfile(files(harpy.reports).joinpath("_quarto.yml")):
+            yml.write(files(harpy.reports).joinpath("_quarto.yml").read_text())
+        else:
+            print_error("report configuration missing", f"The required quarto configuration file [blue bold]_quarto.yml[/blue bold] was not found within the Harpy installation.")
+            print_solution("There may be an issue with your Harpy installation, which would require reinstalling Harpy. Alternatively, there may be in a issue with your conda/mamba environment or configuration.")
+            sys.exit(1)
+
+
 def snakemake_log(outdir, workflow):
     """Return a snakemake logfile name. Iterates logfile run number if one exists."""
     attempts = glob.glob(f"{outdir}/logs/snakemake/*.log*")