diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01085ef..4d34bf6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,16 +8,15 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - config_file: [./test/setup.cfg, ./test/pyproject.toml] steps: - uses: actions/checkout@v4 - name: run local action code uses: ./ with: - file: ${{ matrix.config_file }} + files: ./test/pyproject.toml[doc] ./test/setup.cfg[pip_only] ./test/environment.yaml ./test/requirements.txt output: environment_test.yml channels: conda-forge defaults - extras: test pip_only + extras: test setup_requires: include pip: bidict - uses: mamba-org/setup-micromamba@v1.8.0 @@ -34,8 +33,6 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - config_file: [./test/setup.cfg, ./test/pyproject.toml] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -44,4 +41,4 @@ jobs: - name: pip install package run: pip install . - name: test cmd script - run: pydeps2env ${{ matrix.config_file }} output.yaml -c defaults --extras test -b include --pip pandas + run: pydeps2env ./test/setup.cfg ./test/pyproject.toml[doc] ./test/environment.yaml ./test/requirements.txt -o output.yaml -c defaults --extras test -b include --pip pandas diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f9e3c92..befa9ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.5.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] @@ -8,7 +8,7 @@ repos: - id: check-yaml # ----- Python formatting ----- - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.1.13 + rev: v0.3.2 hooks: # Run ruff linter. - id: ruff @@ -24,10 +24,10 @@ repos: - id: pretty-format-yaml args: [--autofix, --indent, '2'] - repo: https://github.com/tox-dev/pyproject-fmt - rev: 1.6.0 + rev: 1.7.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.15 + rev: v0.16 hooks: - id: validate-pyproject diff --git a/README.md b/README.md index 1cb130b..0448512 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,29 @@ An easy way to create conda environment files from you python project dependencies. Creates a conda `environment.yml` file from python package dependencies listed in a `pyproject.toml` or `setup.cfg` file. -## basic usage +The project contains +- GitHub action +- python package +- command line script + +```mermaid +flowchart LR + pyproject.toml --> pydeps2env + setup.cfg --> pydeps2env + environment.yaml --> pydeps2env + requirements.txt --> pydeps2env + pydeps2env --> E2[environment.yaml] + pydeps2env --> R2[requirements.txt] +``` + +## basic usage (GitHub action) By default, the action will parse a `pyproject.toml` file in your root directory into `environment.yml`. Here is an example of a simple setup: ```yaml steps: - - uses: CagtayFabry/pydeps2env@v0.3.0 + - uses: CagtayFabry/pydeps2env@v1.0.0 ``` ```toml @@ -40,7 +55,7 @@ dependencies: - pandas>=1.0 ``` -A full output with options `--setup_requires include --extras test pip_only --pip bidict` +A full output with options `--build_system include --extras test pip_only --pip bidict` ```yaml channels: @@ -63,9 +78,10 @@ dependencies: To customize the output the input options are available to the action: -### file +### files -Specify the location of the `'setup.cfg'` or `'pyproject.toml'` file to parse. (defaults to `'pyproject.toml'`) +Specify the location of the `'setup.cfg'` or `'pyproject.toml'` files to parse. (defaults to `'pyproject.toml'`) +Multiple files can be listed. This will result in a combined environment file. ### output: @@ -78,8 +94,9 @@ Separate a list of multiple channels by spaces (e.g. `'conda-forge defaults'`). ### extras: -Specify one or more optional `[extras_require]` sections to add to the environment (e.g. `'test'` to include package that -you would normally install with `pip install pkg[test]`) +Specify one or more optional `[extras_require]` sections to add to all the environments (e.g. `'test'` to include package that +you would normally install with `pip install pkg[test]`). +Note that for individual packages, the [extra]` syntax is also possible. ### setup_requires: @@ -94,9 +111,9 @@ The dependencies will be listet under the `pip:` section in the environment file ```yaml steps: - - uses: CagtayFabry/pydeps2env@main + - uses: CagtayFabry/pydeps2env@v1.0.0 with: - file: './test/setup.cfg' # or ./test/pyproject.toml + files: ./test/pyproject.toml[doc] ./test/setup.cfg # comine both files, add [doc] only for pyproject.toml output: 'environment_test.yml' channels: 'conda-forge defaults' extras: 'test' @@ -131,6 +148,7 @@ dependencies: - pytest - setuptools>=40.9.0 - setuptools_scm + - sphinx - wheel - pip: - bidict diff --git a/action.yml b/action.yml index 0b9c2b3..03c5cf1 100644 --- a/action.yml +++ b/action.yml @@ -1,11 +1,11 @@ name: pydeps2env description: create conda environment file from python project dependencies inputs: - file: + files: description: >- - Specify the location of the 'setup.cfg' file to parse. (defaults to 'setup.cfg') + Specify the location of the dependencies files to parse. (defaults to 'pyproject.toml') required: true - default: setup.cfg + default: pyproject.toml output: description: >- Specify the location and name of the conda environment file to generate. (defaults to 'environment.yml') @@ -39,7 +39,8 @@ runs: - name: create environment file run: > pip3 install tomli packaging pyyaml && - python3 $GITHUB_ACTION_PATH/pydeps2env/generate_environment.py ${{ inputs.file }} ${{ inputs.output }} + python3 $GITHUB_ACTION_PATH/pydeps2env/generate_environment.py ${{ inputs.files }} + --output ${{ inputs.output }} --channels ${{ inputs.channels }} --extras ${{ inputs.extras }} --setup_requires ${{ inputs.setup_requires }} diff --git a/pydeps2env/environment.py b/pydeps2env/environment.py index 216a3b3..2d3ae71 100644 --- a/pydeps2env/environment.py +++ b/pydeps2env/environment.py @@ -10,6 +10,21 @@ import warnings +def clean_list(item: list, sort: bool = True) -> list: + """Remove duplicate entries from a list.""" + pass + + +def split_extras(filename: str) -> tuple[str, set]: + """Split extras requirements indicated in [].""" + if "[" in filename: + filename, extras = filename.split("[", 1) + extras = set(extras.split("]", 1)[0].split(",")) + else: + extras = {} + return filename, extras + + def add_requirement( req: Requirement | str, requirements: dict[str, Requirement], @@ -45,16 +60,29 @@ def combine_requirements( class Environment: filename: str | Path channels: list[str] = field(default_factory=lambda: ["conda-forge"]) - extras: list[str] = field(default_factory=list) + extras: set[str] | list[str] = field(default_factory=set) pip_packages: set[str] = field(default_factory=set) # install via pip requirements: dict[str, Requirement] = field(default_factory=dict, init=False) build_system: dict[str, Requirement] = field(default_factory=dict, init=False) def __post_init__(self): + # cleanup duplicates etc. + self.extras = set(self.extras) + self.channels = list(dict.fromkeys(self.channels)) + self.pip_packages = set(self.pip_packages) + + if isinstance(self.filename, str): + self.filename, extras = split_extras(self.filename) + self.extras |= set(extras) + if Path(self.filename).suffix == ".toml": self.load_pyproject() elif Path(self.filename).suffix == ".cfg": self.load_config() + elif Path(self.filename).suffix in [".yaml", ".yml"]: + self.load_yaml() + elif Path(self.filename).suffix in [".txt"]: + self.load_txt() else: raise ValueError(f"Unsupported input {self.filename}") @@ -104,7 +132,35 @@ def load_config(self): for dep in extra_deps: add_requirement(dep, self.requirements) - def _get_dependencies(self, include_build_system: bool = True): + def load_yaml(self): + """Load a conda-style environment.yaml file.""" + with open(self.filename, "r") as f: + env = yaml.load(f.read(), yaml.SafeLoader) + + self.channels += env.get("channels", []) + self.channels = list(dict.fromkeys(self.channels)) + + for dep in env.get("dependencies"): + if isinstance(dep, str): + add_requirement(dep, self.requirements) + elif isinstance(dep, dict) and "pip" in dep: + add_requirement("pip", self.requirements) + for pip_dep in dep["pip"]: + req = Requirement(pip_dep) + self.pip_packages |= {req.name} + add_requirement(req, self.requirements) + + def load_txt(self): + """Load simple list of requirements from txt file.""" + with open(self.filename, "r") as f: + deps = f.readlines() + + for dep in deps: + add_requirement(dep, self.requirements) + + def _get_dependencies( + self, include_build_system: bool = True + ) -> tuple[list[str], list[str]]: """Get the default conda environment entries.""" reqs = self.requirements.copy() @@ -132,7 +188,9 @@ def export( conda_env = {"channels": self.channels, "dependencies": deps.copy()} if pip: - conda_env["dependencies"] += ["pip", {"pip": pip}] + if "pip" not in self.requirements: + conda_env["dependencies"] += ["pip"] + conda_env["dependencies"] += [{"pip": pip}] if outfile is None: return conda_env @@ -150,3 +208,9 @@ def export( ) with open(p, "w") as outfile: yaml.dump(conda_env, outfile, default_flow_style=False) + + def combine(self, other: Environment): + """Merge other Environment requirements into this Environment.""" + self.requirements = combine_requirements(self.requirements, other.requirements) + self.build_system = combine_requirements(self.build_system, other.build_system) + self.pip_packages = self.pip_packages | other.pip_packages diff --git a/pydeps2env/generate_environment.py b/pydeps2env/generate_environment.py index 7d6ea40..07a6154 100644 --- a/pydeps2env/generate_environment.py +++ b/pydeps2env/generate_environment.py @@ -2,21 +2,24 @@ from pathlib import Path +try: + from pydeps2env.environment import Environment, split_extras +except ModuleNotFoundError: # try local file if not installed + from environment import Environment, split_extras + def create_environment_file( - filename: str, + filename: list[str], output_file: str, channels: list[str], extras: list[str], - pip: list[str], + pip: set[str], include_build_system: bool = False, ): - try: - from pydeps2env.environment import Environment - except ModuleNotFoundError: # try local file if not installed - from environment import Environment - - env = Environment(filename, pip_packages=pip, extras=extras, channels=channels) + pip = set(pip) + env = Environment(filename[0], pip_packages=pip, extras=extras, channels=channels) + for f in filename[1:]: + env.combine(Environment(f, pip_packages=pip, extras=extras, channels=channels)) _include = include_build_system == "include" env.export(output_file, include_build_system=_include) @@ -27,9 +30,11 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument( - "setup", type=str, default="pyproject.toml", help="dependency file" + "setup", type=str, nargs="*", default="pyproject.toml", help="dependency file" + ) + parser.add_argument( + "-o", "--output", type=str, default="environment.yml", help="output file" ) - parser.add_argument("env", type=str, default="environment.yml", help="output file") parser.add_argument("-c", "--channels", type=str, nargs="*", default=["defaults"]) parser.add_argument("-e", "--extras", type=str, nargs="*", default=[]) parser.add_argument( @@ -42,12 +47,14 @@ def main(): parser.add_argument("-p", "--pip", type=str, nargs="*", default=[]) args = parser.parse_args() - if not Path(args.setup).is_file(): - raise FileNotFoundError(f"Could not find file {args.setup}") + for file in args.setup: + filename, _ = split_extras(file) + if not Path(filename).is_file(): + raise FileNotFoundError(f"Could not find file {filename}") create_environment_file( filename=args.setup, - output_file=args.env, + output_file=args.output, channels=args.channels, extras=args.extras, pip=args.pip, diff --git a/test/environment.yaml b/test/environment.yaml new file mode 100644 index 0000000..2601a87 --- /dev/null +++ b/test/environment.yaml @@ -0,0 +1,8 @@ +channels: +- conda-forge +dependencies: +- python>3.7 +- yaml +- pip +- pip: + - requests diff --git a/test/pyproject.toml b/test/pyproject.toml index a27d8b8..d54f4fa 100644 --- a/test/pyproject.toml +++ b/test/pyproject.toml @@ -13,6 +13,9 @@ dependencies = [ "pandas>=1", ] [project.optional-dependencies] +doc = [ + "sphinx", +] pip_only = [ "bidict", ] diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000..8a60482 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,2 @@ +setuptools +urllib3 diff --git a/test/setup.cfg b/test/setup.cfg index ed4bbb1..a3112ac 100644 --- a/test/setup.cfg +++ b/test/setup.cfg @@ -11,10 +11,12 @@ setup_requires = setuptools_scm install_requires = numpy >=1.20 - pandas >=1.0 + pandas <2.0 + boltons >=1.0 [options.extras_require] test = pytest + pytest-cov pip_only = bidict