From 195f36449aa21ec06c6f591c6e990a6efa54d420 Mon Sep 17 00:00:00 2001 From: antazoey Date: Tue, 17 Sep 2024 16:44:23 -0500 Subject: [PATCH] feat: PyPI dependencies, downloading + extracting archives from PyPI (#2289) --- docs/userguides/dependencies.md | 23 +++- src/ape/managers/project.py | 9 +- src/ape/utils/__init__.py | 2 + src/ape/utils/os.py | 50 ++++++++ src/ape_pm/__init__.py | 2 +- src/ape_pm/dependency.py | 166 ++++++++++++++++++++++---- tests/functional/test_dependencies.py | 18 +-- 7 files changed, 233 insertions(+), 37 deletions(-) diff --git a/docs/userguides/dependencies.md b/docs/userguides/dependencies.md index 9509d59c07..406eace59a 100644 --- a/docs/userguides/dependencies.md +++ b/docs/userguides/dependencies.md @@ -59,9 +59,20 @@ Bypass the original failing attempt by including a `v` in your dependency config **By knowing if the release is from the version API or only available via tag, and whether the version is v-prefixed or not, you save Ape some time and complexity when installing dependencies.** -### Python +### PyPI -You can use dependencies to PyPI by using the `python:` keyed dependency type. +You can use dependencies from [PyPI](https://pypi.org/) by using the `pypi:` key. + +```yaml +dependencies: + - pypi: snekmate + config_override: + contracts_folder: src +``` + +When using the `pypi:` key, dependencies are downloaded and extracted from PyPI using an HTTP requests library. + +You can also specify the `python:` key for already-installed dependencies: ```yaml dependencies: @@ -70,6 +81,14 @@ dependencies: contracts_folder: . ``` +Using `python:` requires the package to be installed in your `sys.path` (site-packages) folder, generally via `pip` or some other tool. +The `contracts_folder` override, in this case, is often needed because the site-package does not have the root source-folder included. +Additionally, `python:` specified dependencies may also be lacking project-configuration files, such as the `ape-config.yaml`. +Compilers such as `vyper` encourage users to use `pip` to publish and install smart-contract dependencies (other vyper files), but some features in Ape may be limited if the dependency is not also specified in your config somewhere. + +If wanting to use a dependency from `PyPI`, we recommend using the `pypi:` key instead of the `python:` key. +However, the `python:` key works great if you already used `pip` to install the dependency, especially if the dependency is not available on `PyPI`. + ### Local You can use already-downloaded projects as dependencies by referencing them as local dependencies. diff --git a/src/ape/managers/project.py b/src/ape/managers/project.py index a5f58b6763..0db1019141 100644 --- a/src/ape/managers/project.py +++ b/src/ape/managers/project.py @@ -1162,7 +1162,12 @@ def types(self) -> dict[str, type[DependencyAPI]]: for _, (config_key, dependency_class) in self.plugin_manager.dependencies: assert issubclass(dependency_class, DependencyAPI) # For mypy - dependency_classes[config_key] = dependency_class + if isinstance(config_key, tuple): + for sub_key in config_key: + dependency_classes[sub_key] = dependency_class + else: + # Single str-given. + dependency_classes[config_key] = dependency_class return dependency_classes @@ -1442,7 +1447,7 @@ def decode_dependency(self, **item: Any) -> DependencyAPI: if key in item: return cls.model_validate(item) - name = item.get("name") or json.dumps(item) # NOTE: Using 'or' for short-circuit eval + name = item.get("name") or f"{item}" # NOTE: Using 'or' for short-circuit eval raise ProjectError( f"No installed dependency API that supports '{name}'. " f"Keys={', '.join([x for x in item.keys()])}" diff --git a/src/ape/utils/__init__.py b/src/ape/utils/__init__.py index ac4733b542..979e32b84e 100644 --- a/src/ape/utils/__init__.py +++ b/src/ape/utils/__init__.py @@ -48,6 +48,7 @@ clean_path, create_tempdir, expand_environment_variables, + extract_archive, get_all_files_in_directory, get_full_extension, get_package_path, @@ -91,6 +92,7 @@ "EMPTY_BYTES32", "ExtraAttributesMixin", "expand_environment_variables", + "extract_archive", "extract_nested_value", "ExtraModelAttributes", "get_relative_path", diff --git a/src/ape/utils/os.py b/src/ape/utils/os.py index fb6fcd5eb6..822ea2387b 100644 --- a/src/ape/utils/os.py +++ b/src/ape/utils/os.py @@ -1,6 +1,8 @@ import os import re import sys +import tarfile +import zipfile from collections.abc import Callable, Iterator from contextlib import contextmanager from fnmatch import fnmatch @@ -324,3 +326,51 @@ def get_package_path(package_name: str) -> Path: raise ValueError(f"Package '{package_name}' not found in site-packages.") return package_path + + +def extract_archive(archive_file: Path, destination: Optional[Path] = None): + """ + Extract an archive file. Supports ``.zip`` or ``.tar.gz``. + + Args: + archive_file (Path): The file-path to the archive. + destination (Optional[Path]): Optionally provide a destination. + Defaults to the parent directory of the archive file. + """ + destination = destination or archive_file.parent + if archive_file.suffix == ".zip": + with zipfile.ZipFile(archive_file, "r") as zip_ref: + zip_members = zip_ref.namelist() + if top_level_dir := os.path.commonpath(zip_members): + for zip_member in zip_members: + # Modify the member name to remove the top-level directory. + member_path = Path(zip_member) + relative_path = ( + member_path.relative_to(top_level_dir) if top_level_dir else member_path + ) + target_path = destination / relative_path + + if member_path.is_dir(): + target_path.mkdir(parents=True, exist_ok=True) + else: + target_path.parent.mkdir(parents=True, exist_ok=True) + with zip_ref.open(member_path.as_posix()) as source: + target_path.write_bytes(source.read()) + + else: + zip_ref.extractall(f"{destination}") + + elif archive_file.name.endswith(".tar.gz"): + with tarfile.open(archive_file, "r:gz") as tar_ref: + tar_members = tar_ref.getmembers() + if top_level_dir := os.path.commonpath([m.name for m in tar_members]): + for tar_member in tar_members: + # Modify the member name to remove the top-level directory. + tar_member.name = os.path.relpath(tar_member.name, top_level_dir) + tar_ref.extract(tar_member, path=destination) + + else: + tar_ref.extractall(path=f"{destination}") + + else: + raise ValueError(f"Unsupported zip format: '{archive_file.suffix}'.") diff --git a/src/ape_pm/__init__.py b/src/ape_pm/__init__.py index 08823d9635..ef9bc44f66 100644 --- a/src/ape_pm/__init__.py +++ b/src/ape_pm/__init__.py @@ -15,7 +15,7 @@ def dependencies(): yield "github", GithubDependency yield "local", LocalDependency yield "npm", NpmDependency - yield "python", PythonDependency + yield ("python", "pypi"), PythonDependency @plugins.register(plugins.ProjectPlugin) diff --git a/src/ape_pm/dependency.py b/src/ape_pm/dependency.py index a8bd6485f7..90e9143a59 100644 --- a/src/ape_pm/dependency.py +++ b/src/ape_pm/dependency.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Optional, Union +import requests from pydantic import model_validator from ape.api.projects import DependencyAPI @@ -15,6 +16,7 @@ from ape.managers.project import _version_to_options from ape.utils import ManagerAccessMixin, clean_path, get_package_path, in_tempdir from ape.utils._github import _GithubClient, github_client +from ape.utils.os import extract_archive def _fetch_local(src: Path, destination: Path, config_override: Optional[dict] = None): @@ -393,14 +395,25 @@ def _get_version_from_package_json( return data.get("version") +# TODO: Rename to `PyPIDependency` in 0.9. class PythonDependency(DependencyAPI): """ - A dependency installed from Python, such as files published to PyPI. + A dependency installed from Python tooling, such as `pip`. """ - python: str + # TODO: Rename this `site_package_name` in 0.9. + python: Optional[str] = None """ - The Python site-package name. + The Python site-package name, such as ``"snekmate"``. Cannot use + with ``pypi:``. Requires the dependency to have been installed + either via ``pip`` or something alike. + """ + + pypi: Optional[str] = None + """ + The ``pypi`` reference, such as ``"snekmate"``. Cannot use with + ``python:``. When set, downloads the dependency from ``pypi`` + using HTTP directly (not ``pip``). """ version: Optional[str] = None @@ -411,41 +424,148 @@ class PythonDependency(DependencyAPI): @model_validator(mode="before") @classmethod def validate_model(cls, values): - if "name" not in values and "python" in values: - values["name"] = values["python"] + if "name" not in values: + if name := values.get("python") or values.get("pypi"): + values["name"] = name + else: + raise ValueError( + "Must set either 'pypi:' or 'python': when using Python dependencies" + ) return values @cached_property - def path(self) -> Path: - try: - return get_package_path(self.python) - except ValueError as err: - raise ProjectError(str(err)) from err + def path(self) -> Optional[Path]: + if self.pypi: + # Is pypi: specified; has no special path. + return None + + elif python := self.python: + try: + return get_package_path(python) + except ValueError as err: + raise ProjectError(str(err)) from err + + return None @property def package_id(self) -> str: - return self.python + if pkg_id := (self.pypi or self.python): + return pkg_id + + raise ProjectError("Must provide either 'pypi:' or 'python:' for python-base dependencies.") @property def version_id(self) -> str: - try: - vers = f"{metadata.version(self.python)}" - except metadata.PackageNotFoundError as err: - raise ProjectError(f"Dependency '{self.python}' not installed.") from err + if self.pypi: + # Version available in package data. + if not (vers := self.version_from_package_data or ""): + # I doubt this is a possible condition, but just in case. + raise ProjectError(f"Missing version from PyPI for package '{self.package_id}'.") - if spec_vers := self.version: - if spec_vers != vers: - raise ProjectError( - "Dependency installed with mismatched version. " - f"Expecting '{self.version}' but has '{vers}'" - ) + elif self.python: + try: + vers = f"{metadata.version(self.package_id)}" + except metadata.PackageNotFoundError as err: + raise ProjectError(f"Dependency '{self.package_id}' not installed.") from err + + if spec_vers := self.version: + if spec_vers != vers: + raise ProjectError( + "Dependency installed with mismatched version. " + f"Expecting '{self.version}' but has '{vers}'" + ) + + else: + raise ProjectError( + "Must provide either 'pypi:' or 'python:' for python-base dependencies." + ) return vers @property def uri(self) -> str: - return self.path.as_uri() + if self.pypi: + return self.download_archive_url + + elif self.python and (path := self.path): + # Local site-package path. + return path.as_uri() + + else: + raise ProjectError( + "Must provide either 'pypi:' or 'python:' for python-base dependencies." + ) + + @cached_property + def package_data(self) -> dict: + url = f"https://pypi.org/pypi/{self.package_id}/json" + response = requests.get(url) + + try: + response.raise_for_status() + except requests.HTTPError as err: + if err.response.status_code == 404: + raise ProjectError( + f"Unknown dependency '{self.package_id}'. " + "Is it spelled correctly and available on PyPI? " + "For local Python packages, use the `python:` key." + ) + else: + raise ProjectError( + f"Problem downloading package data for '{self.package_id}': {err}" + ) + + return response.json() + + @cached_property + def version_from_package_data(self) -> Optional[str]: + return self.package_data.get("info", {}).get("version") + + @cached_property + def download_archive_url(self) -> str: + if not (version := self.version): + if not (version := self.version_from_package_data): + # Not sure this is possible, but just in case API data changes or something. + raise ProjectError(f"Unable to find version for package '{self.package_id}'.") + + releases = self.package_data.get("releases", {}) + if version not in releases: + raise ProjectError(f"Version '{version}' not found for package '{self.package_id}'.") + + # Find the first zip file in the specified release. + for file_info in releases[version]: + if file_info.get("packagetype") != "sdist": + continue + + return file_info["url"] + + raise ProjectError( + f"No zip file found for package '{self.package_id}' with version '{version}' on PyPI." + ) def fetch(self, destination: Path): - _fetch_local(self.path, destination, config_override=self.config_override) + if self.pypi: + self._fetch_from_pypi(destination) + elif path := self.path: + # 'python:' key. + _fetch_local(path, destination, config_override=self.config_override) + + def _fetch_from_pypi(self, destination: Path): + archive_path = self._fetch_archive_file(destination) + extract_archive(archive_path) + archive_path.unlink(missing_ok=True) + + def _fetch_archive_file(self, destination) -> Path: + logger.info(f"Fetching python dependency '{self.package_id}' from 'pypi.") + download_url = self.download_archive_url + filename = download_url.split("/")[-1] + destination.mkdir(exist_ok=True, parents=True) + archive_destination = destination / filename + with requests.get(download_url, stream=True) as response: + response.raise_for_status() + with open(archive_destination, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): # 8 KB + file.write(chunk) + + return archive_destination diff --git a/tests/functional/test_dependencies.py b/tests/functional/test_dependencies.py index bfb2a3a0fc..d2f63d0fee 100644 --- a/tests/functional/test_dependencies.py +++ b/tests/functional/test_dependencies.py @@ -588,15 +588,15 @@ def test_fetch_ref(self, mock_client): class TestPythonDependency: - @pytest.fixture(scope="class") - def web3_dependency(self): - return PythonDependency.model_validate({"python": "web3"}) + @pytest.fixture(scope="class", params=("python", "pypi")) + def python_dependency(self, request): + return PythonDependency.model_validate({request.param: "web3"}) - def test_name(self, web3_dependency): - assert web3_dependency.name == "web3" + def test_name(self, python_dependency): + assert python_dependency.name == "web3" - def test_version_id(self, web3_dependency): - actual = web3_dependency.version_id + def test_version_id(self, python_dependency): + actual = python_dependency.version_id assert isinstance(actual, str) assert len(actual) > 0 assert actual[0].isnumeric() @@ -609,9 +609,9 @@ def test_version_id_not_found(self): with pytest.raises(ProjectError, match=expected): _ = dependency.version_id - def test_fetch(self, web3_dependency): + def test_fetch(self, python_dependency): with create_tempdir() as temp_dir: - web3_dependency.fetch(temp_dir) + python_dependency.fetch(temp_dir) files = [x for x in temp_dir.iterdir()] assert len(files) > 0