Skip to content

MAINT Inject compat layer to Transaction #213

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 27, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions micropip/package_index.py
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@
from typing import Any
from urllib.parse import urljoin, urlparse, urlunparse

from ._compat import fetch_string_and_headers
from ._compat import CompatibilityLayer
from ._utils import is_package_compatible, parse_version
from ._vendored.mousebender.simple import from_project_details_html
from ._vendored.packaging.src.packaging.utils import InvalidWheelFilename
@@ -271,6 +271,9 @@ def _select_parser(
async def query_package(
name: str,
index_urls: list[str] | str,
*,
# TODO: instead of passing this as a parameter, it should be a class attribute
compat_layer: type[CompatibilityLayer],
Comment on lines +274 to +276
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree with the comment. Could you please open a issue to track it?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. I opened an issue to track the progress (#237). Not just for this comment but for overall refactoring for the compatibility layer.

fetch_kwargs: dict[str, Any] | None = None,
) -> ProjectInfo:
"""
@@ -312,7 +315,9 @@ async def query_package(
url = f"{url}/{name}/"
logger.debug("Url has no placeholder, appending package name : %r", url)
try:
metadata, headers = await fetch_string_and_headers(url, _fetch_kwargs)
metadata, headers = await compat_layer.fetch_string_and_headers(
url, _fetch_kwargs
)
except Exception as e:
logger.debug(
"Error fetching metadata for the package %r from (%r): %r, trying next index.",
1 change: 1 addition & 0 deletions micropip/package_manager.py
Original file line number Diff line number Diff line change
@@ -170,6 +170,7 @@ async def install(
wheel_base = Path(getsitepackages()[0])

transaction = Transaction(
_compat_layer=self.compat_layer,
ctx=ctx, # type: ignore[arg-type]
ctx_extras=[],
keep_going=keep_going,
11 changes: 7 additions & 4 deletions micropip/transaction.py
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@
from urllib.parse import urlparse

from . import package_index
from ._compat import LOCKFILE_PACKAGES
from ._compat import CompatibilityLayer
from ._utils import (
best_compatible_tag_index,
check_compatible,
@@ -30,6 +30,8 @@

@dataclass
class Transaction:
_compat_layer: type[CompatibilityLayer]

ctx: dict[str, str]
ctx_extras: list[dict[str, str]]
keep_going: bool
@@ -208,9 +210,9 @@ async def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool:
Find requirement from pyodide-lock.json. If the requirement is found,
add it to the package list and return True. Otherwise, return False.
"""
locked_package = LOCKFILE_PACKAGES.get(req.name)
locked_package = self._compat_layer.lockfile_packages.get(req.name)
if locked_package and req.specifier.contains(
LOCKFILE_PACKAGES[req.name]["version"], prereleases=True
self._compat_layer.lockfile_packages[req.name]["version"], prereleases=True
):
version = locked_package["version"]
self.pyodide_packages.append(
@@ -230,7 +232,8 @@ async def _add_requirement_from_package_index(self, req: Requirement):
metadata = await package_index.query_package(
req.name,
self.index_urls,
self.fetch_kwargs,
compat_layer=self._compat_layer,
fetch_kwargs=self.fetch_kwargs,
)

logger.debug("Transaction: got metadata %r for requirement %r", metadata, req)
12 changes: 11 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
@@ -304,7 +304,7 @@ def add_pkg_version(
self.metadata_map[filename] = metadata
self.top_level_map[filename] = top_level

async def query_package(self, pkgname, index_urls, kwargs):
async def query_package(self, pkgname, index_urls, *, compat_layer, fetch_kwargs):
from micropip.package_index import ProjectInfo

try:
@@ -453,3 +453,13 @@ def _run(*lines, error_match=None):
selenium_standalone_micropip.run_js(js)

return _run


@pytest.fixture
def host_compat_layer():
"""
Fixture to provide the compatibility layer for the host environment.
"""
from micropip._compat._compat_not_in_pyodide import CompatibilityNotInPyodide

yield CompatibilityNotInPyodide
10 changes: 7 additions & 3 deletions tests/test_package_index.py
Original file line number Diff line number Diff line change
@@ -133,7 +133,7 @@ def test_contain_placeholder():
("numpy", "black"),
],
)
async def test_query_package(mock_fixture, pkg1, pkg2, request):
async def test_query_package(mock_fixture, pkg1, pkg2, request, host_compat_layer):
gen_mock_server = request.getfixturevalue(mock_fixture)
pkg1_index_url = gen_mock_server(pkgs=[pkg1], pkgs_not_found=[pkg2])
pkg2_index_url = gen_mock_server(pkgs=[pkg2], pkgs_not_found=[pkg1])
@@ -143,10 +143,14 @@ async def test_query_package(mock_fixture, pkg1, pkg2, request):
[pkg1_index_url],
[pkg2_index_url, pkg1_index_url],
):
project_info = await package_index.query_package(pkg1, index_urls=_index_urls)
project_info = await package_index.query_package(
pkg1, index_urls=_index_urls, compat_layer=host_compat_layer
)

assert project_info.name == pkg1
assert project_info.releases

with pytest.raises(ValueError, match="Can't fetch metadata"):
await package_index.query_package(pkg1, index_urls=[pkg2_index_url])
await package_index.query_package(
pkg1, index_urls=[pkg2_index_url], compat_layer=host_compat_layer
)
32 changes: 21 additions & 11 deletions tests/test_transaction.py
Original file line number Diff line number Diff line change
@@ -52,10 +52,11 @@ def test_parse_wheel_url3():
assert wheel.tags == frozenset({Tag("cp35", "cp35m", "macosx_10_9_intel")})


def create_transaction(Transaction):
def create_transaction(Transaction, compat_layer):
from micropip.package_index import DEFAULT_INDEX_URLS

return Transaction(
_compat_layer=compat_layer,
wheels=[],
locked={},
keep_going=True,
@@ -71,12 +72,12 @@ def create_transaction(Transaction):


@pytest.mark.asyncio
async def test_add_requirement(wheel_catalog):
async def test_add_requirement(wheel_catalog, host_compat_layer):
from micropip.transaction import Transaction

snowballstemmer_wheel = wheel_catalog.get("snowballstemmer")

transaction = create_transaction(Transaction)
transaction = create_transaction(Transaction, host_compat_layer)
await transaction.add_requirement(snowballstemmer_wheel.url)

wheel = transaction.wheels[0]
@@ -90,10 +91,10 @@ async def test_add_requirement(wheel_catalog):


@pytest.mark.asyncio
async def test_add_requirement_marker(mock_importlib, wheel_base):
async def test_add_requirement_marker(mock_importlib, wheel_base, host_compat_layer):
from micropip.transaction import Transaction

transaction = create_transaction(Transaction)
transaction = create_transaction(Transaction, host_compat_layer)

await transaction.gather_requirements(
[
@@ -125,29 +126,31 @@ async def test_add_requirement_marker(mock_importlib, wheel_base):


@pytest.mark.asyncio
async def test_add_requirement_query_url(mock_importlib, wheel_base, monkeypatch):
async def test_add_requirement_query_url(
mock_importlib, wheel_base, monkeypatch, host_compat_layer
):
from micropip.transaction import Transaction

async def mock_add_wheel(self, wheel, extras, *, specifier=""):
self.mock_wheel = wheel

monkeypatch.setattr(Transaction, "add_wheel", mock_add_wheel)

transaction = create_transaction(Transaction)
transaction = create_transaction(Transaction, host_compat_layer)
await transaction.add_requirement(f"{SNOWBALL_WHEEL}?b=1")
wheel = transaction.mock_wheel
assert wheel.name == "snowballstemmer"
assert wheel.filename == SNOWBALL_WHEEL # without the query params


@pytest.mark.asyncio
async def test_install_non_pure_python_wheel():
async def test_install_non_pure_python_wheel(host_compat_layer):
from micropip.transaction import Transaction

msg = "Wheel platform 'macosx_10_9_intel' is not compatible with Pyodide's platform"
with pytest.raises(ValueError, match=msg):
url = "http://a/scikit_learn-0.22.2.post1-cp35-cp35m-macosx_10_9_intel.whl"
transaction = create_transaction(Transaction)
transaction = create_transaction(Transaction, host_compat_layer)
await transaction.add_requirement(url)


@@ -324,11 +327,12 @@ def test_last_version_and_best_tag_from_pypi(
assert str(wheel.version) == new_version


def test_search_pyodide_lock_first():
def test_search_pyodide_lock_first(host_compat_layer):
from micropip import package_index
from micropip.transaction import Transaction

t = Transaction(
_compat_layer=host_compat_layer,
ctx={},
ctx_extras=[],
keep_going=True,
@@ -341,6 +345,7 @@ def test_search_pyodide_lock_first():
assert t.search_pyodide_lock_first is True

t = Transaction(
_compat_layer=host_compat_layer,
ctx={},
ctx_extras=[],
keep_going=True,
@@ -355,7 +360,11 @@ def test_search_pyodide_lock_first():

@pytest.mark.asyncio
async def test_index_url_priority(
mock_importlib, wheel_base, monkeypatch, mock_package_index_simple_json_api
mock_importlib,
wheel_base,
monkeypatch,
mock_package_index_simple_json_api,
host_compat_layer,
):
# Test that if the index_urls are provided, package should be searched in
# the index_urls first before searching in Pyodide lock file.
@@ -373,6 +382,7 @@ async def mock_add_wheel(self, wheel, extras, *, specifier=""):
mock_index_url = mock_package_index_simple_json_api(pkgs=["black"])

t = Transaction(
_compat_layer=host_compat_layer,
keep_going=True,
deps=False,
pre=False,