feat: PyPI dependencies, downloading + extracting archives from PyPI (#…

…2289)

docs/userguides/dependencies.md

@@ -59,9 +59,20 @@ Bypass the original failing attempt by including a `v` in your dependency config
 
 **By knowing if the release is from the version API or only available via tag, and whether the version is v-prefixed or not, you save Ape some time and complexity when installing dependencies.**
 
-### Python
+### PyPI
 
-You can use dependencies to PyPI by using the `python:` keyed dependency type.
+You can use dependencies from [PyPI](https://pypi.org/) by using the `pypi:` key.
+
+```yaml
+dependencies:
+   - pypi: snekmate
+     config_override:
+       contracts_folder: src
+```
+
+When using the `pypi:` key, dependencies are downloaded and extracted from PyPI using an HTTP requests library.
+
+You can also specify the `python:` key for already-installed dependencies:
 
 ```yaml
 dependencies:
@@ -70,6 +81,14 @@ dependencies:
        contracts_folder: .
 ```
 
+Using `python:` requires the package to be installed in your `sys.path` (site-packages) folder, generally via `pip` or some other tool.
+The `contracts_folder` override, in this case, is often needed because the site-package does not have the root source-folder included.
+Additionally, `python:` specified dependencies may also be lacking project-configuration files, such as the `ape-config.yaml`.
+Compilers such as `vyper` encourage users to use `pip` to publish and install smart-contract dependencies (other vyper files), but some features in Ape may be limited if the dependency is not also specified in your config somewhere.
+
+If wanting to use a dependency from `PyPI`, we recommend using the `pypi:` key instead of the `python:` key.
+However, the `python:` key works great if you already used `pip` to install the dependency, especially if the dependency is not available on `PyPI`.
+
 ### Local
 
 You can use already-downloaded projects as dependencies by referencing them as local dependencies.

src/ape/managers/project.py

@@ -1162,7 +1162,12 @@ def types(self) -> dict[str, type[DependencyAPI]]:
 
         for _, (config_key, dependency_class) in self.plugin_manager.dependencies:
             assert issubclass(dependency_class, DependencyAPI)  # For mypy
-            dependency_classes[config_key] = dependency_class
+            if isinstance(config_key, tuple):
+                for sub_key in config_key:
+                    dependency_classes[sub_key] = dependency_class
+            else:
+                # Single str-given.
+                dependency_classes[config_key] = dependency_class
 
         return dependency_classes
 
@@ -1442,7 +1447,7 @@ def decode_dependency(self, **item: Any) -> DependencyAPI:
             if key in item:
                 return cls.model_validate(item)
 
-        name = item.get("name") or json.dumps(item)  # NOTE: Using 'or' for short-circuit eval
+        name = item.get("name") or f"{item}"  # NOTE: Using 'or' for short-circuit eval
         raise ProjectError(
             f"No installed dependency API that supports '{name}'. "
             f"Keys={', '.join([x for x in item.keys()])}"

src/ape/utils/__init__.py

@@ -48,6 +48,7 @@
     clean_path,
     create_tempdir,
     expand_environment_variables,
+    extract_archive,
     get_all_files_in_directory,
     get_full_extension,
     get_package_path,
@@ -91,6 +92,7 @@
     "EMPTY_BYTES32",
     "ExtraAttributesMixin",
     "expand_environment_variables",
+    "extract_archive",
     "extract_nested_value",
     "ExtraModelAttributes",
     "get_relative_path",

src/ape/utils/os.py

@@ -1,6 +1,8 @@
 import os
 import re
 import sys
+import tarfile
+import zipfile
 from collections.abc import Callable, Iterator
 from contextlib import contextmanager
 from fnmatch import fnmatch
@@ -324,3 +326,51 @@ def get_package_path(package_name: str) -> Path:
         raise ValueError(f"Package '{package_name}' not found in site-packages.")
 
     return package_path
+
+
+def extract_archive(archive_file: Path, destination: Optional[Path] = None):
+    """
+    Extract an archive file. Supports ``.zip`` or ``.tar.gz``.
+
+    Args:
+        archive_file (Path): The file-path to the archive.
+        destination (Optional[Path]): Optionally provide a destination.
+          Defaults to the parent directory of the archive file.
+    """
+    destination = destination or archive_file.parent
+    if archive_file.suffix == ".zip":
+        with zipfile.ZipFile(archive_file, "r") as zip_ref:
+            zip_members = zip_ref.namelist()
+            if top_level_dir := os.path.commonpath(zip_members):
+                for zip_member in zip_members:
+                    # Modify the member name to remove the top-level directory.
+                    member_path = Path(zip_member)
+                    relative_path = (
+                        member_path.relative_to(top_level_dir) if top_level_dir else member_path
+                    )
+                    target_path = destination / relative_path
+
+                    if member_path.is_dir():
+                        target_path.mkdir(parents=True, exist_ok=True)
+                    else:
+                        target_path.parent.mkdir(parents=True, exist_ok=True)
+                        with zip_ref.open(member_path.as_posix()) as source:
+                            target_path.write_bytes(source.read())
+
+            else:
+                zip_ref.extractall(f"{destination}")
+
+    elif archive_file.name.endswith(".tar.gz"):
+        with tarfile.open(archive_file, "r:gz") as tar_ref:
+            tar_members = tar_ref.getmembers()
+            if top_level_dir := os.path.commonpath([m.name for m in tar_members]):
+                for tar_member in tar_members:
+                    # Modify the member name to remove the top-level directory.
+                    tar_member.name = os.path.relpath(tar_member.name, top_level_dir)
+                    tar_ref.extract(tar_member, path=destination)
+
+            else:
+                tar_ref.extractall(path=f"{destination}")
+
+    else:
+        raise ValueError(f"Unsupported zip format: '{archive_file.suffix}'.")

src/ape_pm/__init__.py

@@ -15,7 +15,7 @@ def dependencies():
     yield "github", GithubDependency
     yield "local", LocalDependency
     yield "npm", NpmDependency
-    yield "python", PythonDependency
+    yield ("python", "pypi"), PythonDependency
 
 
 @plugins.register(plugins.ProjectPlugin)

src/ape_pm/dependency.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from typing import Optional, Union
 
+import requests
 from pydantic import model_validator
 
 from ape.api.projects import DependencyAPI
@@ -15,6 +16,7 @@
 from ape.managers.project import _version_to_options
 from ape.utils import ManagerAccessMixin, clean_path, get_package_path, in_tempdir
 from ape.utils._github import _GithubClient, github_client
+from ape.utils.os import extract_archive
 
 
 def _fetch_local(src: Path, destination: Path, config_override: Optional[dict] = None):
@@ -393,14 +395,25 @@ def _get_version_from_package_json(
     return data.get("version")
 
 
+# TODO: Rename to `PyPIDependency` in 0.9.
 class PythonDependency(DependencyAPI):
     """
-    A dependency installed from Python, such as files published to PyPI.
+    A dependency installed from Python tooling, such as `pip`.
     """
 
-    python: str
+    # TODO: Rename this `site_package_name` in 0.9.
+    python: Optional[str] = None
     """
-    The Python site-package name.
+    The Python site-package name, such as ``"snekmate"``. Cannot use
+    with ``pypi:``. Requires the dependency to have been installed
+    either via ``pip`` or something alike.
+    """
+
+    pypi: Optional[str] = None
+    """
+    The ``pypi`` reference, such as ``"snekmate"``. Cannot use with
+    ``python:``. When set, downloads the dependency from ``pypi``
+    using HTTP directly (not ``pip``).
     """
 
     version: Optional[str] = None
@@ -411,41 +424,148 @@ class PythonDependency(DependencyAPI):
     @model_validator(mode="before")
     @classmethod
     def validate_model(cls, values):
-        if "name" not in values and "python" in values:
-            values["name"] = values["python"]
+        if "name" not in values:
+            if name := values.get("python") or values.get("pypi"):
+                values["name"] = name
+            else:
+                raise ValueError(
+                    "Must set either 'pypi:' or 'python': when using Python dependencies"
+                )
 
         return values
 
     @cached_property
-    def path(self) -> Path:
-        try:
-            return get_package_path(self.python)
-        except ValueError as err:
-            raise ProjectError(str(err)) from err
+    def path(self) -> Optional[Path]:
+        if self.pypi:
+            # Is pypi: specified; has no special path.
+            return None
+
+        elif python := self.python:
+            try:
+                return get_package_path(python)
+            except ValueError as err:
+                raise ProjectError(str(err)) from err
+
+        return None
 
     @property
     def package_id(self) -> str:
-        return self.python
+        if pkg_id := (self.pypi or self.python):
+            return pkg_id
+
+        raise ProjectError("Must provide either 'pypi:' or 'python:' for python-base dependencies.")
 
     @property
     def version_id(self) -> str:
-        try:
-            vers = f"{metadata.version(self.python)}"
-        except metadata.PackageNotFoundError as err:
-            raise ProjectError(f"Dependency '{self.python}' not installed.") from err
+        if self.pypi:
+            # Version available in package data.
+            if not (vers := self.version_from_package_data or ""):
+                # I doubt this is a possible condition, but just in case.
+                raise ProjectError(f"Missing version from PyPI for package '{self.package_id}'.")
 
-        if spec_vers := self.version:
-            if spec_vers != vers:
-                raise ProjectError(
-                    "Dependency installed with mismatched version. "
-                    f"Expecting '{self.version}' but has '{vers}'"
-                )
+        elif self.python:
+            try:
+                vers = f"{metadata.version(self.package_id)}"
+            except metadata.PackageNotFoundError as err:
+                raise ProjectError(f"Dependency '{self.package_id}' not installed.") from err
+
+            if spec_vers := self.version:
+                if spec_vers != vers:
+                    raise ProjectError(
+                        "Dependency installed with mismatched version. "
+                        f"Expecting '{self.version}' but has '{vers}'"
+                    )
+
+        else:
+            raise ProjectError(
+                "Must provide either 'pypi:' or 'python:' for python-base dependencies."
+            )
 
         return vers
 
     @property
     def uri(self) -> str:
-        return self.path.as_uri()
+        if self.pypi:
+            return self.download_archive_url
+
+        elif self.python and (path := self.path):
+            # Local site-package path.
+            return path.as_uri()
+
+        else:
+            raise ProjectError(
+                "Must provide either 'pypi:' or 'python:' for python-base dependencies."
+            )
+
+    @cached_property
+    def package_data(self) -> dict:
+        url = f"https://pypi.org/pypi/{self.package_id}/json"
+        response = requests.get(url)
+
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as err:
+            if err.response.status_code == 404:
+                raise ProjectError(
+                    f"Unknown dependency '{self.package_id}'. "
+                    "Is it spelled correctly and available on PyPI? "
+                    "For local Python packages, use the `python:` key."
+                )
+            else:
+                raise ProjectError(
+                    f"Problem downloading package data for '{self.package_id}': {err}"
+                )
+
+        return response.json()
+
+    @cached_property
+    def version_from_package_data(self) -> Optional[str]:
+        return self.package_data.get("info", {}).get("version")
+
+    @cached_property
+    def download_archive_url(self) -> str:
+        if not (version := self.version):
+            if not (version := self.version_from_package_data):
+                # Not sure this is possible, but just in case API data changes or something.
+                raise ProjectError(f"Unable to find version for package '{self.package_id}'.")
+
+        releases = self.package_data.get("releases", {})
+        if version not in releases:
+            raise ProjectError(f"Version '{version}' not found for package '{self.package_id}'.")
+
+        # Find the first zip file in the specified release.
+        for file_info in releases[version]:
+            if file_info.get("packagetype") != "sdist":
+                continue
+
+            return file_info["url"]
+
+        raise ProjectError(
+            f"No zip file found for package '{self.package_id}' with version '{version}' on PyPI."
+        )
 
     def fetch(self, destination: Path):
-        _fetch_local(self.path, destination, config_override=self.config_override)
+        if self.pypi:
+            self._fetch_from_pypi(destination)
+        elif path := self.path:
+            # 'python:' key.
+            _fetch_local(path, destination, config_override=self.config_override)
+
+    def _fetch_from_pypi(self, destination: Path):
+        archive_path = self._fetch_archive_file(destination)
+        extract_archive(archive_path)
+        archive_path.unlink(missing_ok=True)
+
+    def _fetch_archive_file(self, destination) -> Path:
+        logger.info(f"Fetching python dependency '{self.package_id}' from 'pypi.")
+        download_url = self.download_archive_url
+        filename = download_url.split("/")[-1]
+        destination.mkdir(exist_ok=True, parents=True)
+        archive_destination = destination / filename
+        with requests.get(download_url, stream=True) as response:
+            response.raise_for_status()
+            with open(archive_destination, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):  # 8 KB
+                    file.write(chunk)
+
+        return archive_destination

tests/functional/test_dependencies.py

@@ -588,15 +588,15 @@ def test_fetch_ref(self, mock_client):
 
 
 class TestPythonDependency:
-    @pytest.fixture(scope="class")
-    def web3_dependency(self):
-        return PythonDependency.model_validate({"python": "web3"})
+    @pytest.fixture(scope="class", params=("python", "pypi"))
+    def python_dependency(self, request):
+        return PythonDependency.model_validate({request.param: "web3"})
 
-    def test_name(self, web3_dependency):
-        assert web3_dependency.name == "web3"
+    def test_name(self, python_dependency):
+        assert python_dependency.name == "web3"
 
-    def test_version_id(self, web3_dependency):
-        actual = web3_dependency.version_id
+    def test_version_id(self, python_dependency):
+        actual = python_dependency.version_id
         assert isinstance(actual, str)
         assert len(actual) > 0
         assert actual[0].isnumeric()
@@ -609,9 +609,9 @@ def test_version_id_not_found(self):
         with pytest.raises(ProjectError, match=expected):
             _ = dependency.version_id
 
-    def test_fetch(self, web3_dependency):
+    def test_fetch(self, python_dependency):
         with create_tempdir() as temp_dir:
-            web3_dependency.fetch(temp_dir)
+            python_dependency.fetch(temp_dir)
             files = [x for x in temp_dir.iterdir()]
             assert len(files) > 0