From d6571d8cdfbdae8844e20cb322ad13b997a14c3d Mon Sep 17 00:00:00 2001 From: viktorht Date: Thu, 16 May 2024 14:34:31 +0200 Subject: [PATCH 1/7] WIP REFACTOR Current version installs only non stan dependent part Created a separate module for the error_propagation. Files paths and imports are altered to fit new setup. This version was tested by locally install the package in fresh venv and run test_integration and test_unit --- pseudobatch/__init__.py | 51 +------------------ pseudobatch/error_propagation/__init__.py | 49 ++++++++++++++++++ .../error_propagation.py | 12 ++--- .../{ => error_propagation}/stan/__init__.py | 0 .../stan/error_propagation.stan | 0 .../stan/functions.stan | 0 setup.cfg | 7 +-- setup.py | 4 +- tests/test_error_propagation.py | 2 +- 9 files changed, 63 insertions(+), 62 deletions(-) create mode 100644 pseudobatch/error_propagation/__init__.py rename pseudobatch/{ => error_propagation}/error_propagation.py (95%) rename pseudobatch/{ => error_propagation}/stan/__init__.py (100%) rename pseudobatch/{ => error_propagation}/stan/error_propagation.stan (100%) rename pseudobatch/{ => error_propagation}/stan/functions.stan (100%) diff --git a/pseudobatch/__init__.py b/pseudobatch/__init__.py index 985cb65..05e5404 100644 --- a/pseudobatch/__init__.py +++ b/pseudobatch/__init__.py @@ -1,10 +1,3 @@ -import shutil -import warnings -from pathlib import Path -from .import_from_excel import process_excel_template -# from .data_correction import pseudobatch_transform_multiple, pseudobatch_transform, pseudobatch_transform_pandas, accumulated_dilution_factor, convert_volumetric_rates_from_pseudo_to_real, pseudobatch_transform_pandas_by_group -import cmdstanpy - from pseudobatch.data_correction import ( pseudobatch_transform, pseudobatch_transform_multiple, @@ -12,46 +5,4 @@ hypothetical_concentration, metabolised_amount, ) -from pseudobatch.error_propagation import run_error_propagation - -STAN_FILES_FOLDER = Path(__file__).parent / "stan" -CMDSTAN_VERSION = "2.31.0" - - -# on Windows specifically, we should point cmdstanpy to the repackaged -# CmdStan if it exists. This lets cmdstanpy handle the TBB path for us. -local_cmdstan = STAN_FILES_FOLDER / f"cmdstan-{CMDSTAN_VERSION}" -if local_cmdstan.exists(): - cmdstanpy.set_cmdstan_path(str(local_cmdstan.resolve())) - - -def load_stan_model(name: str) -> cmdstanpy.CmdStanModel: - """ - Try to load precompiled Stan models. If that fails, - compile them. - """ - try: - model = cmdstanpy.CmdStanModel( - exe_file=STAN_FILES_FOLDER / f"{name}.exe", - stan_file=STAN_FILES_FOLDER / f"{name}.stan", - compile=False, - ) - except ValueError: - warnings.warn(f"Failed to load pre-built model '{name}.exe', compiling") - model = cmdstanpy.CmdStanModel( - stan_file=STAN_FILES_FOLDER / f"{name}.stan", - stanc_options={"O1": True}, - ) - shutil.copy( - model.exe_file, # type: ignore - STAN_FILES_FOLDER / f"{name}.exe", - ) - - return model - - -ERROR_PROPAGATION = load_stan_model("error_propagation") - - -# example: just print the info of the model -print(ERROR_PROPAGATION.exe_info()) +from .import_from_excel import process_excel_template diff --git a/pseudobatch/error_propagation/__init__.py b/pseudobatch/error_propagation/__init__.py new file mode 100644 index 0000000..1c98896 --- /dev/null +++ b/pseudobatch/error_propagation/__init__.py @@ -0,0 +1,49 @@ +import shutil +import warnings +from pathlib import Path +import cmdstanpy +from pseudobatch.error_propagation.error_propagation import ( + run_error_propagation, +) + +STAN_FILES_FOLDER = Path(__file__).parent / "stan" +CMDSTAN_VERSION = "2.31.0" + + +# on Windows specifically, we should point cmdstanpy to the repackaged +# CmdStan if it exists. This lets cmdstanpy handle the TBB path for us. +local_cmdstan = STAN_FILES_FOLDER / f"cmdstan-{CMDSTAN_VERSION}" +if local_cmdstan.exists(): + cmdstanpy.set_cmdstan_path(str(local_cmdstan.resolve())) + + +def load_stan_model(name: str) -> cmdstanpy.CmdStanModel: + """ + Try to load precompiled Stan models. If that fails, + compile them. + """ + try: + model = cmdstanpy.CmdStanModel( + exe_file=STAN_FILES_FOLDER / f"{name}.exe", + stan_file=STAN_FILES_FOLDER / f"{name}.stan", + compile=False, + ) + except ValueError: + warnings.warn(f"Failed to load pre-built model '{name}.exe', compiling") + model = cmdstanpy.CmdStanModel( + stan_file=STAN_FILES_FOLDER / f"{name}.stan", + stanc_options={"O1": True}, + ) + shutil.copy( + model.exe_file, # type: ignore + STAN_FILES_FOLDER / f"{name}.exe", + ) + + return model + + +ERROR_PROPAGATION = load_stan_model("error_propagation") + + +# example: just print the info of the model +print(ERROR_PROPAGATION.exe_info()) diff --git a/pseudobatch/error_propagation.py b/pseudobatch/error_propagation/error_propagation.py similarity index 95% rename from pseudobatch/error_propagation.py rename to pseudobatch/error_propagation/error_propagation.py index de4096d..f0cca53 100644 --- a/pseudobatch/error_propagation.py +++ b/pseudobatch/error_propagation/error_propagation.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field from pydantic.functional_validators import model_validator -from pseudobatch import stan +from pseudobatch.error_propagation import stan from pseudobatch.util import ( get_lognormal_params_from_quantiles, get_normal_params_from_quantiles, @@ -114,7 +114,7 @@ def run_error_propagation( sd_sample_volume : Sample volume measurement error. - sd_concentration_in_feed : Error for concentration in feed measurements. + sd_concentration_in_feed : Error for concentration in feed measurements. prior_input : Dictionary that can be used to load a PriorInput object. @@ -134,11 +134,11 @@ def run_error_propagation( "then prior_cfeed must not be None." ) assert all(y == 0 for y in y_concentration_in_feed), msg - prior_cfeed = [[0. for _ in range(S)], [1. for _ in range(S)]] + prior_cfeed = [[0.0 for _ in range(S)], [1.0 for _ in range(S)]] else: prior_cfeed = [ - [p.loc for p in pi.prior_cfeed], - [p.scale for p in pi.prior_cfeed] + [p.loc for p in pi.prior_cfeed], + [p.scale for p in pi.prior_cfeed], ] prior_m = [Prior0dLogNormal(pct1=1e-9, pct99=1e9) for _ in range(S)] prior_f = Prior0dLogNormal(pct1=1e-6, pct99=1e6) @@ -173,7 +173,7 @@ def run_error_propagation( "c": ["sample", "species"], "v": ["sample"], "cfeed": ["species"], - "pseudobatch_c": ["sample", "species"] + "pseudobatch_c": ["sample", "species"], } data_prior = {**data, **{"likelihood": 0}} data_posterior = {**data, **{"likelihood": 1}} diff --git a/pseudobatch/stan/__init__.py b/pseudobatch/error_propagation/stan/__init__.py similarity index 100% rename from pseudobatch/stan/__init__.py rename to pseudobatch/error_propagation/stan/__init__.py diff --git a/pseudobatch/stan/error_propagation.stan b/pseudobatch/error_propagation/stan/error_propagation.stan similarity index 100% rename from pseudobatch/stan/error_propagation.stan rename to pseudobatch/error_propagation/stan/error_propagation.stan diff --git a/pseudobatch/stan/functions.stan b/pseudobatch/error_propagation/stan/functions.stan similarity index 100% rename from pseudobatch/stan/functions.stan rename to pseudobatch/error_propagation/stan/functions.stan diff --git a/setup.cfg b/setup.cfg index 98239c5..a3d846c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,11 +13,8 @@ packages = find: include_package_data = True requires_python = >=3.9 install_requires = - arviz - cmdstanpy>=1.0.7 numpy pandas - pydantic==2.4.2 [options.package_data] * = @@ -37,3 +34,7 @@ documentation = sphinx nbsphinx pydata-sphinx-theme +error_propagation = + arviz + cmdstanpy>=1.0.7 + pydantic==2.4.2 \ No newline at end of file diff --git a/setup.py b/setup.py index 6a6ba30..94aa0d1 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ from distutils.command.clean import clean from wheel.bdist_wheel import bdist_wheel -MODEL_DIR = "pseudobatch/stan" +MODEL_DIR = "pseudobatch/error_propagation/stan" MODELS = [ "error_propagation", @@ -168,7 +168,7 @@ def get_tag(self) -> Tuple[str, str, str]: setup( # Extension marks this as platform-specific - ext_modules=[Extension("pseudobatch.stan", [])], + ext_modules=[Extension("pseudobatch.error_propagation.stan", [])], # override the build and bdist commands cmdclass={ "build_ext": BuildModels, diff --git a/tests/test_error_propagation.py b/tests/test_error_propagation.py index b93ca8f..eb9614a 100644 --- a/tests/test_error_propagation.py +++ b/tests/test_error_propagation.py @@ -4,7 +4,7 @@ import pandas as pd from scipy.special import logit -from pseudobatch import run_error_propagation +from pseudobatch.error_propagation import run_error_propagation from pseudobatch.datasets import load_standard_fedbatch From 0e8b171b5cca34b61ea73475f1892806f1a5edb1 Mon Sep 17 00:00:00 2001 From: viktorht Date: Thu, 16 May 2024 16:50:51 +0200 Subject: [PATCH 2/7] WIP installs without cmdstanpy however also does not fail when using pseudobatch[error_propagation] --- build_testing.sh | 19 ++ pseudobatch/error_propagation/__init__.py | 100 +++---- pyproject.toml | 2 +- setup.py | 320 +++++++++++----------- 4 files changed, 236 insertions(+), 205 deletions(-) create mode 100755 build_testing.sh diff --git a/build_testing.sh b/build_testing.sh new file mode 100755 index 0000000..c44eec0 --- /dev/null +++ b/build_testing.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# remove the old virtual environment if it exists +rm -rf .venv-docker + +# Create a new virtual environment +python -m venv .venv-docker + +# Activate the virtual environment +source .venv-docker/bin/activate + +# Install the current directory +pip install -e '.[development]' + +# Run the tests +pytest + +# install error_propagation +pip install '.[error_propagation]' \ No newline at end of file diff --git a/pseudobatch/error_propagation/__init__.py b/pseudobatch/error_propagation/__init__.py index 1c98896..bd64715 100644 --- a/pseudobatch/error_propagation/__init__.py +++ b/pseudobatch/error_propagation/__init__.py @@ -1,49 +1,57 @@ import shutil import warnings from pathlib import Path -import cmdstanpy -from pseudobatch.error_propagation.error_propagation import ( - run_error_propagation, -) - -STAN_FILES_FOLDER = Path(__file__).parent / "stan" -CMDSTAN_VERSION = "2.31.0" - - -# on Windows specifically, we should point cmdstanpy to the repackaged -# CmdStan if it exists. This lets cmdstanpy handle the TBB path for us. -local_cmdstan = STAN_FILES_FOLDER / f"cmdstan-{CMDSTAN_VERSION}" -if local_cmdstan.exists(): - cmdstanpy.set_cmdstan_path(str(local_cmdstan.resolve())) - - -def load_stan_model(name: str) -> cmdstanpy.CmdStanModel: - """ - Try to load precompiled Stan models. If that fails, - compile them. - """ - try: - model = cmdstanpy.CmdStanModel( - exe_file=STAN_FILES_FOLDER / f"{name}.exe", - stan_file=STAN_FILES_FOLDER / f"{name}.stan", - compile=False, - ) - except ValueError: - warnings.warn(f"Failed to load pre-built model '{name}.exe', compiling") - model = cmdstanpy.CmdStanModel( - stan_file=STAN_FILES_FOLDER / f"{name}.stan", - stanc_options={"O1": True}, - ) - shutil.copy( - model.exe_file, # type: ignore - STAN_FILES_FOLDER / f"{name}.exe", - ) - - return model - - -ERROR_PROPAGATION = load_stan_model("error_propagation") - - -# example: just print the info of the model -print(ERROR_PROPAGATION.exe_info()) +from importlib.metadata import distribution + +try: + distribution("cmdstanpy") + import cmdstanpy + + cmdstanpy_installed = True +except: + cmdstanpy_installed = False + +if cmdstanpy_installed: + from pseudobatch.error_propagation.error_propagation import ( + run_error_propagation, + ) + + STAN_FILES_FOLDER = Path(__file__).parent / "stan" + CMDSTAN_VERSION = "2.31.0" + + # on Windows specifically, we should point cmdstanpy to the repackaged + # CmdStan if it exists. This lets cmdstanpy handle the TBB path for us. + local_cmdstan = STAN_FILES_FOLDER / f"cmdstan-{CMDSTAN_VERSION}" + if local_cmdstan.exists(): + cmdstanpy.set_cmdstan_path(str(local_cmdstan.resolve())) + + def load_stan_model(name: str) -> cmdstanpy.CmdStanModel: + """ + Try to load precompiled Stan models. If that fails, + compile them. + """ + try: + model = cmdstanpy.CmdStanModel( + exe_file=STAN_FILES_FOLDER / f"{name}.exe", + stan_file=STAN_FILES_FOLDER / f"{name}.stan", + compile=False, + ) + except ValueError: + warnings.warn( + f"Failed to load pre-built model '{name}.exe', compiling" + ) + model = cmdstanpy.CmdStanModel( + stan_file=STAN_FILES_FOLDER / f"{name}.stan", + stanc_options={"O1": True}, + ) + shutil.copy( + model.exe_file, # type: ignore + STAN_FILES_FOLDER / f"{name}.exe", + ) + + return model + + ERROR_PROPAGATION = load_stan_model("error_propagation") + + # example: just print the info of the model + print(ERROR_PROPAGATION.exe_info()) diff --git a/pyproject.toml b/pyproject.toml index 46d6963..05d4e50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools", "wheel", "cmdstanpy>=1.0.7"] +requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" [tool.isort] diff --git a/setup.py b/setup.py index 94aa0d1..1bf674e 100644 --- a/setup.py +++ b/setup.py @@ -8,171 +8,175 @@ from pathlib import Path from shutil import copy, copytree, rmtree from typing import Tuple - -import cmdstanpy +from importlib.metadata import distribution from setuptools import Extension, setup -from setuptools.command.build_ext import build_ext -from distutils.command.clean import clean -from wheel.bdist_wheel import bdist_wheel - -MODEL_DIR = "pseudobatch/error_propagation/stan" - -MODELS = [ - "error_propagation", -] - - -CMDSTAN_VERSION = "2.31.0" -BINARIES_DIR = "bin" -BINARIES = ["diagnose", "print", "stanc", "stansummary"] -MATH_LIB = "stan/lib/stan_math/lib" -TBB_DIRS = ["tbb", "tbb_2020.3"] - - -def prune_cmdstan(cmdstan_dir: os.PathLike) -> None: - """ - Keep only the cmdstan executables and tbb files - (minimum required to run a cmdstanpy commands on a pre-compiled model). - """ - original_dir = Path(cmdstan_dir).resolve() - parent_dir = original_dir.parent - temp_dir = parent_dir / "temp" - if temp_dir.is_dir(): - rmtree(temp_dir) - temp_dir.mkdir() - - print("Copying ", original_dir, " to ", temp_dir, " for pruning") - copytree(original_dir / BINARIES_DIR, temp_dir / BINARIES_DIR) - copy(original_dir / "makefile", temp_dir / "makefile") - for f in (temp_dir / BINARIES_DIR).iterdir(): - if f.is_dir(): - rmtree(f) - elif f.is_file() and f.stem not in BINARIES: - os.remove(f) - for tbb_dir in TBB_DIRS: - copytree( - original_dir / MATH_LIB / tbb_dir, temp_dir / MATH_LIB / tbb_dir - ) - - rmtree(original_dir) - temp_dir.rename(original_dir) - - -def repackage_cmdstan() -> bool: - return os.environ.get("PSEUDOBATCH_REPACKAGE_CMDSTAN", "").lower() in [ - "true", - "1", - ] - -def maybe_install_cmdstan_toolchain() -> None: - """Install C++ compilers required to build stan models on Windows machines.""" - try: - cmdstanpy.utils.cxx_toolchain_path() - except Exception: - from cmdstanpy.install_cxx_toolchain import run_rtools_install +try: + distribution("cmdstanpy") + import cmdstanpy - run_rtools_install({"version": None, "dir": None, "verbose": True}) - cmdstanpy.utils.cxx_toolchain_path() + cmdstanpy_installed = True +except: + print("cmdstanpy not found, skipping model compilation.") + cmdstanpy_installed = False +if not cmdstanpy_installed: + setup() +else: + from setuptools.command.build_ext import build_ext + from distutils.command.clean import clean + from wheel.bdist_wheel import bdist_wheel -def install_cmdstan_deps(cmdstan_dir: Path) -> None: - from multiprocessing import cpu_count + MODEL_DIR = "pseudobatch/error_propagation/stan" - if repackage_cmdstan(): - if platform.platform().startswith("Win"): - maybe_install_cmdstan_toolchain() - print("Installing cmdstan to", cmdstan_dir) - if os.path.isdir(cmdstan_dir): - print("Removing existing dir", cmdstan_dir) - rmtree(cmdstan_dir) + MODELS = [ + "error_propagation", + ] - if not cmdstanpy.install_cmdstan( - version=CMDSTAN_VERSION, - dir=cmdstan_dir.parent, - overwrite=True, - verbose=True, - cores=cpu_count(), - ): - raise RuntimeError( - "CmdStan failed to install in repackaged directory" + CMDSTAN_VERSION = "2.31.0" + BINARIES_DIR = "bin" + BINARIES = ["diagnose", "print", "stanc", "stansummary"] + MATH_LIB = "stan/lib/stan_math/lib" + TBB_DIRS = ["tbb", "tbb_2020.3"] + + def prune_cmdstan(cmdstan_dir: os.PathLike) -> None: + """ + Keep only the cmdstan executables and tbb files + (minimum required to run a cmdstanpy commands on a pre-compiled model). + """ + original_dir = Path(cmdstan_dir).resolve() + parent_dir = original_dir.parent + temp_dir = parent_dir / "temp" + if temp_dir.is_dir(): + rmtree(temp_dir) + temp_dir.mkdir() + + print("Copying ", original_dir, " to ", temp_dir, " for pruning") + copytree(original_dir / BINARIES_DIR, temp_dir / BINARIES_DIR) + copy(original_dir / "makefile", temp_dir / "makefile") + for f in (temp_dir / BINARIES_DIR).iterdir(): + if f.is_dir(): + rmtree(f) + elif f.is_file() and f.stem not in BINARIES: + os.remove(f) + for tbb_dir in TBB_DIRS: + copytree( + original_dir / MATH_LIB / tbb_dir, temp_dir / MATH_LIB / tbb_dir ) - else: + + rmtree(original_dir) + temp_dir.rename(original_dir) + + def repackage_cmdstan() -> bool: + return os.environ.get("PSEUDOBATCH_REPACKAGE_CMDSTAN", "").lower() in [ + "true", + "1", + ] + + def maybe_install_cmdstan_toolchain() -> None: + """Install C++ compilers required to build stan models on Windows machines.""" + try: - cmdstanpy.cmdstan_path() - except ValueError as e: - raise SystemExit( - "CmdStan not installed, but the package is building from source" - ) from e - - -def build_models(target_dir: str) -> None: - cmdstan_dir = (Path(target_dir) / f"cmdstan-{CMDSTAN_VERSION}").resolve() - install_cmdstan_deps(cmdstan_dir) - for model in MODELS: - sm = cmdstanpy.CmdStanModel( - stan_file=os.path.join(MODEL_DIR, model + ".stan"), - stanc_options={"O1": True}, - ) - copy(sm.exe_file, os.path.join(target_dir, model + ".exe")) - - if repackage_cmdstan(): - prune_cmdstan(cmdstan_dir) - - -class BuildModels(build_ext): - """Custom build command to pre-compile Stan models.""" - - def run(self) -> None: - if not self.dry_run: - target_dir = os.path.join(self.build_lib, MODEL_DIR) - self.mkpath(target_dir) - build_models(target_dir) - # don't call build_ext.run, since we're not really building c files - - -def clean_models(target_dir: str) -> None: - # Remove compiled stan files - for model in MODELS: - for filename in [model, f"{model}.hpp", f"{model}.exe"]: - stan_file = Path(target_dir) / filename - if stan_file.exists(): - stan_file.unlink() - - -class CleanModels(clean): - """Custom clean command to remove pre-compile Stan models.""" - - def run(self) -> None: - if not self.dry_run: - target_dir = os.path.join(self.build_lib, MODEL_DIR) - clean_models(target_dir) - clean_models(MODEL_DIR) - super().run() - - -# this is taken from the cibuildwheel example https://github.com/joerick/python-ctypes-package-sample -# it marks the wheel as not specific to the Python API version. -# This means the wheel will only be built once per platform, rather than per-Python-per-platform. -# If you are combining with any actual C extensions, you will most likely want to remove this. -class WheelABINone(bdist_wheel): - def finalize_options(self) -> None: - bdist_wheel.finalize_options(self) - self.root_is_pure = False - - def get_tag(self) -> Tuple[str, str, str]: - _, _, plat = bdist_wheel.get_tag(self) - return "py3", "none", plat - - -setup( - # Extension marks this as platform-specific - ext_modules=[Extension("pseudobatch.error_propagation.stan", [])], - # override the build and bdist commands - cmdclass={ - "build_ext": BuildModels, - "bdist_wheel": WheelABINone, - "clean": CleanModels, - }, -) + cmdstanpy.utils.cxx_toolchain_path() + except Exception: + from cmdstanpy.install_cxx_toolchain import run_rtools_install + + run_rtools_install({"version": None, "dir": None, "verbose": True}) + cmdstanpy.utils.cxx_toolchain_path() + + def install_cmdstan_deps(cmdstan_dir: Path) -> None: + from multiprocessing import cpu_count + + if repackage_cmdstan(): + if platform.platform().startswith("Win"): + maybe_install_cmdstan_toolchain() + print("Installing cmdstan to", cmdstan_dir) + if os.path.isdir(cmdstan_dir): + print("Removing existing dir", cmdstan_dir) + rmtree(cmdstan_dir) + + if not cmdstanpy.install_cmdstan( + version=CMDSTAN_VERSION, + dir=cmdstan_dir.parent, + overwrite=True, + verbose=True, + cores=cpu_count(), + ): + raise RuntimeError( + "CmdStan failed to install in repackaged directory" + ) + else: + try: + cmdstanpy.cmdstan_path() + except ValueError as e: + raise SystemExit( + "CmdStan not installed, but the package is building from source" + ) from e + + def build_models(target_dir: str) -> None: + cmdstan_dir = ( + Path(target_dir) / f"cmdstan-{CMDSTAN_VERSION}" + ).resolve() + install_cmdstan_deps(cmdstan_dir) + for model in MODELS: + sm = cmdstanpy.CmdStanModel( + stan_file=os.path.join(MODEL_DIR, model + ".stan"), + stanc_options={"O1": True}, + ) + copy(sm.exe_file, os.path.join(target_dir, model + ".exe")) + + if repackage_cmdstan(): + prune_cmdstan(cmdstan_dir) + + class BuildModels(build_ext): + """Custom build command to pre-compile Stan models.""" + + def run(self) -> None: + if not self.dry_run: + target_dir = os.path.join(self.build_lib, MODEL_DIR) + self.mkpath(target_dir) + build_models(target_dir) + # don't call build_ext.run, since we're not really building c files + + def clean_models(target_dir: str) -> None: + # Remove compiled stan files + for model in MODELS: + for filename in [model, f"{model}.hpp", f"{model}.exe"]: + stan_file = Path(target_dir) / filename + if stan_file.exists(): + stan_file.unlink() + + class CleanModels(clean): + """Custom clean command to remove pre-compile Stan models.""" + + def run(self) -> None: + if not self.dry_run: + target_dir = os.path.join(self.build_lib, MODEL_DIR) + clean_models(target_dir) + clean_models(MODEL_DIR) + super().run() + + # this is taken from the cibuildwheel example https://github.com/joerick/python-ctypes-package-sample + # it marks the wheel as not specific to the Python API version. + # This means the wheel will only be built once per platform, rather than per-Python-per-platform. + # If you are combining with any actual C extensions, you will most likely want to remove this. + class WheelABINone(bdist_wheel): + def finalize_options(self) -> None: + bdist_wheel.finalize_options(self) + self.root_is_pure = False + + def get_tag(self) -> Tuple[str, str, str]: + _, _, plat = bdist_wheel.get_tag(self) + return "py3", "none", plat + + setup( + # Extension marks this as platform-specific + ext_modules=[Extension("pseudobatch.error_propagation.stan", [])], + # override the build and bdist commands + cmdclass={ + "build_ext": BuildModels, + "bdist_wheel": WheelABINone, + "clean": CleanModels, + }, + ) From 91f227054fe9daf3104c1ee1eb7d419c4e74a914 Mon Sep 17 00:00:00 2001 From: viktorht Date: Fri, 17 May 2024 11:50:30 +0200 Subject: [PATCH 3/7] ignores any .venv folders --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index af283ff..f2aaa4b 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ ipython_config.py # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version +.venv* # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. From bb03862e8582cd4e592f26e2948330f2543160a1 Mon Sep 17 00:00:00 2001 From: viktorht Date: Sat, 18 May 2024 14:32:15 +0200 Subject: [PATCH 4/7] Raises ImportError if user tries to use the error_propagation module without having cmdstanpy installed --- pseudobatch/error_propagation/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pseudobatch/error_propagation/__init__.py b/pseudobatch/error_propagation/__init__.py index bd64715..6a0f90d 100644 --- a/pseudobatch/error_propagation/__init__.py +++ b/pseudobatch/error_propagation/__init__.py @@ -3,6 +3,10 @@ from pathlib import Path from importlib.metadata import distribution +# The error propagation module requires to be installed separately. +# Cmdstanpy is here used as an indication of whether the error +# propagation module is installed or not. If cmdstanpy is installed, +# the error propagation module will be loaded. try: distribution("cmdstanpy") import cmdstanpy @@ -10,6 +14,13 @@ cmdstanpy_installed = True except: cmdstanpy_installed = False + raise ImportError( + "cmdstanpy is not installed. To use the error propagation module, " + "please install cmdstanpy by running 'pip install cmdstanpy'. " + "Then install the CmdStan binaries by running 'cmdstanpy.install_cmdstan()'. " + "Finally, install the remaining dependencies for the error propagation module " + "by running 'pip install pseudobatch[error_propagation]'." + ) if cmdstanpy_installed: from pseudobatch.error_propagation.error_propagation import ( From 029e446ab84e597208187f017b05030c2d5fea7c Mon Sep 17 00:00:00 2001 From: viktorht Date: Sat, 18 May 2024 14:33:37 +0200 Subject: [PATCH 5/7] Separately installs the error_propagation module before running tests --- .github/workflows/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index fa6fd41..cc44edf 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -56,6 +56,7 @@ jobs: - name: Install package run: | pip install .[development] + pip install .[error_propagation] - name: Run tests run: | From 26c8cbaca2fedb14cfb6dc61d8a4b2bc73237beb Mon Sep 17 00:00:00 2001 From: viktorht Date: Sat, 18 May 2024 16:44:42 +0200 Subject: [PATCH 6/7] Explains installation of the error propagation module --- README.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d6c3c1d..ee4e106 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,33 @@ Download the excel template from the [excel-pseudobatch folder](./excel-pseudoba The Python package holds functions which apply the pseudobatch transformation to data either in the form of `Numpy` Arrays or `Pandas` dataframe. Please visit [our documentation]() to how to use the Python package. ## How to install? -The Pseudobatch Python package can be install through PYPI using pip. -``` +The Pseudobatch Python package can be install through PYPI using pip. Most of the functionality can be installed simply be calling + +```shell pip install pseudobatch ``` +The error propagation functionality requires installation of cmdstanpy and CmdStan. Thus, installing the error propagation functionality takes a few steps. First install cmdstanpy in the prefered virtual environment. + +```shell +pip install cmdstanpy +``` + +Second, use cmdstanpy to install CmdStan. To due this you need to call use the function `cmdstanpy.install_cmdstan()`. This can for example be done by opening a python session in the terminal and run the following two Python commands + +```python +import cmdstanpy +cmdstanpy.install_cmdstan() +``` + +Now exit the Python session and install the remaining dependencies of the error propagation module through pip. + +```shell +pip install pseudobatch[error_propagation] +``` + +Now the error propagation module is installed and ready to use. + ## How to cite If you use the pseudobatch transformation please cite the original article XXX. From 6f830b6aaffa6b4c4e6075ba8c075440fc1cabd8 Mon Sep 17 00:00:00 2001 From: viktorht Date: Sat, 18 May 2024 16:54:24 +0200 Subject: [PATCH 7/7] Dockerfile installs the pseudobatch package with error propagation module --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c785e08..5edb083 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN pip install --no-cache-dir "cmdstanpy==1.0.4" # Install cmdstan RUN python -m cmdstanpy.install_cmdstan --version "${CMDSTAN_VERSION}" --cores 2 -RUN pip install --no-cache-dir -e "." && \ +RUN pip install --no-cache-dir -e ".[error_propagation]" && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}"