WIP: Update_packaging (#732)

* Do not try to set usetex=False or resave
if savefig fails. Instead just return error msg.

* Replace check_call with check_out in sh()
Allows collection of outputs.
Added redirect_error arg so stderr can be
redirected to stdout and user in error logging.

* Collect stderr msgs from lastal for use in
informative logging.

* Log suggestion to use --notex if savefig fails

* Add cleanup empty pdf before exit if savefig fails

* Add logging to clarify why latex is
deactivated.

* ignore version and cython files

* mv exclude to pyproject

* use hatch for build

* use hatch-vcs version

* log error when no cblast

* mv pkg into src

* ignore version

* init env yml

* rm version, switch to dynamic vcs version

* WIP: migrating to hatch

* rm cblast.c from tracking

* ignore dev files

* Use setuptools for cython modules

* automate run setup.py for cython build

* setup_magick_home() assumes ImageMagick always installed with homebrew on MacOS, change error to warning if homebrew path not found.

* install non-pip deps using conda. Install libmagic without homebrew.

* fix bug - incorrect selection of wget on MocOS without wget

* version to _version

* bump min Python version.

* conda env instructions

* Do not try to set usetex=False or resave
if savefig fails. Instead just return error msg.

* Replace check_call with check_out in sh()
Allows collection of outputs.
Added redirect_error arg so stderr can be
redirected to stdout and user in error logging.

* Move files back to jcvi/

* Move jcvi to src/ directory with git mv

* optparse not used

* use logger instead of logging

* fix conda env to py 3.12

* init pytest action

* black fmt

* use miniforge

* switch run order

* handle testing in build action

* black fmt

* Add cython to deps

* typo

* Add cli entrypoint to check version

* update readme

.github/workflows/build.yml

@@ -9,7 +9,7 @@ jobs:
       max-parallel: 4
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ["3.8", "3.10"]
+        python-version: ["3.8", "3.10", "3.12"]
 
     steps:
       - uses: actions/checkout@v4
@@ -31,9 +31,8 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e .
+          pip install -e '.[tests]'
 
       - name: Test with pytest
         run: |
-          pip install PyYAML pytest pytest-cov pytest-benchmark mock
           pytest --cov=jcvi tests

.gitignore

@@ -1,4 +1,15 @@
-# Created by http://www.gitignore.io
+# Mac stuff
+.DS_Store
+
+# Development
+dev/
+
+# Versioning
+src/jcvi/_version.py
+
+# Ignore Cython generated C files
+src/jcvi/assembly/chic.c
+src/jcvi/formats/cblast.c
 
 # Dev testing
 dev
@@ -7,6 +18,7 @@ dev
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
+*$py.class
 
 # C extensions
 *.so
@@ -17,12 +29,15 @@ env/
 build/
 develop-eggs/
 dist/
+downloads/
 eggs/
+.eggs/
 lib/
 lib64/
 parts/
 sdist/
 var/
+wheels/
 *.egg-info/
 .installed.cfg
 *.egg
@@ -35,9 +50,12 @@ pip-delete-this-directory.txt
 htmlcov/
 .tox/
 .coverage
+.coverage.*
 .cache
 nosetests.xml
 coverage.xml
+*.cover
+.hypothesis/
 
 # Translations
 *.mo
@@ -51,3 +69,4 @@ docs/_build/
 
 # gffutils temp DB file
 *.db
+src/jcvi/version.py

README.md

@@ -104,49 +104,77 @@ full-fledged applications.
 
 ## Dependencies
 
-Following are a list of third-party python packages that are used by
-some routines in the library. These dependencies are _not_ mandatory
-since they are only used by a few modules.
+JCVI requires Python3 between v3.8 and v3.12.
 
-- [Biopython](http://www.biopython.org)
-- [numpy](http://numpy.scipy.org)
-- [matplotlib](http://matplotlib.org/)
+A few modules may ask for locations of external programs,
+if the executable cannot be found in your `PATH`. 
 
-There are other Python modules here and there in various scripts. The
-best way is to install them via `pip install` when you see
-`ImportError`.
+The external programs that are often used are:
+
+- [Kent tools](http://hgdownload.cse.ucsc.edu/admin/jksrc.zip)
+- [BEDTOOLS](http://code.google.com/p/bedtools/)
+- [EMBOSS](http://emboss.sourceforge.net/)
 
 ## Installation
 
-The easiest way is to install it via PyPI:
+**Installing JCVI in a Conda environment:**
 
-```console
-pip install jcvi
+You can create a Conda environment with Python 3.12 and basic dependencies for JCVI using the YAML files in this repo.
+
+If you are new to Conda, we recommend the [Miniforge](https://conda-forge.org/download/) distribution.
+
+
+```bash
+conda env create -f environment.yml
+
+conda activate jcvi
 ```
 
-To install the development version:
+Note: If you are using a Mac with an ARM64 (Apple Silicon) processor, some dependencies are not currently available from Bioconda for this architecture. 
 
-```console
+You can instead create a virtual OSX64 (intel) env like this:
+
+```bash
+conda env create -f env_osx64.yml
+
+conda activate jcvi-osx64
+```
+
+After activating the Conda environment install JCVI using one of the following options.
+
+
+**Installation options:**  
+
+1) Use pip to install the latest development version directly from this repo.
+
+```bash
 pip install git+git://github.com/tanghaibao/jcvi.git
 ```
 
-Alternatively, if you want to install manually:
+2) Install latest release from PyPi.
 
-```console
-cd ~/code  # or any directory of your choice
-git clone git://github.com/tanghaibao/jcvi.git
-pip install -e .
+```bash
+pip install jcvi
 ```
 
-In addition, a few module might ask for locations of external programs,
-if the extended cannot be found in your `PATH`. The external programs
-that are often used are:
+3) Alternatively, if you want to install in development mode.
 
-- [Kent tools](http://hgdownload.cse.ucsc.edu/admin/jksrc.zip)
-- [BEDTOOLS](http://code.google.com/p/bedtools/)
-- [EMBOSS](http://emboss.sourceforge.net/)
+```bash
+git clone git://github.com/tanghaibao/jcvi.git && cd jcvi
+pip install -e '.[tests]'
+```
+
+**Test Installation:**  
+
+If installed successfully, you can check the version with:
+
+```bash
+jcvi --version
+```
+
+Use `python -m` to call any of the modules installed with JCVI.
 
-Most of the scripts in this package contains multiple actions. To use
+Most of the modules in this package contains multiple actions. To use
 the `fasta` example:
 
 ```console

build.py

@@ -0,0 +1,9 @@
+import subprocess
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+
+
+class CustomBuildHook(BuildHookInterface):
+    def initialize(self, version, build_data):
+        # Run setup.py build_ext before main build
+        subprocess.check_call(["python", "setup.py", "build_ext", "--inplace"])
+        return super().initialize(version, build_data)

env_osx64.yml

@@ -0,0 +1,14 @@
+name: jcvi-osx64
+channels:
+  - conda-forge/osx-64
+  - bioconda/osx-64
+dependencies:
+  - python 3.12
+  - bedtools
+  - imagemagick
+  - libmagic  # System-level magic library
+  - wand  # Python bindings for ImageMagick
+  - pip
+  - pip:
+      - hatch
+      - pytest

environment.yml

@@ -0,0 +1,14 @@
+name: jcvi
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  - python 3.12
+  - bedtools
+  - imagemagick
+  - libmagic  # System-level magic library
+  - wand  # Python bindings for ImageMagick
+  - pip
+  - pip:
+      - pytest
+      - hatch

pyproject.toml

@@ -1,15 +1,110 @@
+# Specifies the build system requirements and backend
 [build-system]
 requires = [
-    "Cython",
-    "numpy",
-    "setuptools",
-    "setuptools_scm[toml]",
-    "setuptools_scm_git_archive",
-    "wheel",
-]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools_scm]
-write_to = "jcvi/version.py"
-git_describe_command = "git describe --dirty --tags --long --match v* --first-parent"
-version_scheme = "no-guess-dev"
+    "hatchling",  # Build backend
+    "hatch-vcs",  # Version control system plugin for dynamic versioning
+    "setuptools",  # Setuptools for compiling C extensions
+    "cython",  # Cython for compiling C extensions
+    "numpy",      # NumPy for numerical operations and C extension includes
+]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.hooks.custom]
+path = "build.py"
+
+# Project metadata and configuration
+[project]
+name = "jcvi"
+description = "Python utility libraries on genome assembly, annotation and comparative genomics"
+readme = "README.md"
+requires-python = ">=3.8"
+license = {text = "BSD"}
+authors = [
+    {name = "Haibao Tang", email = "tanghaibao@gmail.com"},
+    {name = "Vivek Krishnakumar"},
+    {name = "Jingping Li"}
+]
+
+classifiers = [
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: BSD License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Bio-Informatics"
+]
+
+dependencies = [
+    "biopython",
+    "boto3",
+    "brewer2mpl",
+    "CrossMap",
+    "cython",
+    "deap",
+    "ete3",
+    "ftpretty",
+    "genomepy",
+    "gffutils",
+    "goatools",
+    "graphviz",
+    "jinja2",
+    "matplotlib",
+    "more-itertools",
+    "natsort",
+    "networkx",
+    "numpy<2",
+    "ortools",
+    "pybedtools",
+    "pyefd",
+    "pypdf",
+    "pytesseract",
+    "rich",
+    "scikit-image",
+    "scipy",
+    "seaborn",
+    "Wand",
+    "webcolors"
+]
+
+dynamic = ["version"]
+
+[project.optional-dependencies]
+tests = [
+    "mock",
+    "pytest-benchmark",
+    "pytest-cov",
+    "pytest",
+    "PyYAML",
+]
+
+[project.urls]
+homepage = "http://github.com/tanghaibao/jcvi"
+
+# Command-line script entry point
+[project.scripts]
+jcvi = "jcvi.cli:main"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build]
+packages = ["src/jcvi"]
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/jcvi/_version.py"
+
+[tool.hatch.version.vcs]
+tag-pattern = "v*"
+fallback-version = "0.0.0"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/**/*.py",
+    "src/**/*.pyx",
+    "README.md",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/jcvi"]