From 2ae389d670ec22b9a84e4a9d785a94d2ea03a4bc Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 02:17:57 +0530 Subject: [PATCH 01/45] =?UTF-8?q?Fix=20a=20typo:=20pyoodide=20=E2=9E=A1?= =?UTF-8?q?=EF=B8=8F=20pyodide?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_emscripten.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_emscripten.py b/test/test_emscripten.py index 97ceb7f9f..aa7750bc5 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -20,7 +20,7 @@ def check_node(): # cibuildwheel adds a pinned node version to the PATH - # check it's in the PATH then, check it's the one that runs pyoodide + # check it's in the PATH then, check it's the one that runs pyodide cibw_cache_path = Path(sys.argv[1]).resolve(strict=True) # find the node executable in PATH node = shutil.which("node") From 0e3b3f1b8ae6d19cb3a2a4383da31b78d3e51228 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:12:32 +0530 Subject: [PATCH 02/45] Add `pyodide_build_version` attribute --- cibuildwheel/pyodide.py | 13 ++++++++++--- cibuildwheel/resources/build-platforms.toml | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 3b721a10a..eba3546e1 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -41,6 +41,7 @@ class PythonConfiguration: version: str identifier: str pyodide_version: str + pyodide_build_version: str emscripten_version: str node_version: str @@ -65,10 +66,14 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str: +def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: + # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are + # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall + # specify the pyodide-build version, which will set up the xbuildenv for the requested + # Pyodide version. pyodide_root = ( CIBW_CACHE_PATH - / f".pyodide-xbuildenv-{pyodide_version}/{pyodide_version}/xbuildenv/pyodide-root" + / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" ) with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): @@ -171,7 +176,9 @@ def setup_python( env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) log.step("Installing Pyodide xbuildenv...") - env["PYODIDE_ROOT"] = install_xbuildenv(env, python_configuration.pyodide_version) + env["PYODIDE_ROOT"] = install_xbuildenv( + env, python_configuration.pyodide_build_version, python_configuration.pyodide_version + ) return env diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index f96e6897a..1c967c9f5 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,5 +172,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.28.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From 394459fe359b26afab787f026dda7772b998c9ee Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:28:26 +0530 Subject: [PATCH 03/45] Add version to xbuildenv log step --- cibuildwheel/pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index eba3546e1..133ece0a0 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -175,7 +175,7 @@ def setup_python( env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) - log.step("Installing Pyodide xbuildenv...") + log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...") env["PYODIDE_ROOT"] = install_xbuildenv( env, python_configuration.pyodide_build_version, python_configuration.pyodide_version ) From dff7bf265ac0fb894c6cf35f4851daff3adc1c65 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:39:41 +0530 Subject: [PATCH 04/45] Add version to Emscripten log step --- cibuildwheel/pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 133ece0a0..4b8bef432 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -170,7 +170,7 @@ def setup_python( env=env, ) - log.step("Installing emscripten...") + log.step(f"Installing Emscripten version: {python_configuration.emscripten_version} ...") emcc_path = install_emscripten(tmp, python_configuration.emscripten_version) env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) From 16057bca6e211cd282f6981871252110d342f373 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:39:04 +0530 Subject: [PATCH 05/45] Use `pyodide-build`'s version for updating constraints --- noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 0c2057b6d..4f59a98d8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -102,10 +102,10 @@ def update_constraints(session: nox.Session) -> None: pyodides = build_platforms["pyodide"]["python_configurations"] for pyodide in pyodides: python_version = ".".join(pyodide["version"].split(".")[:2]) - pyodide_version = pyodide["pyodide_version"] + pyodide_build_version = pyodide["pyodide_build_version"] output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt" tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in" - tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_version}") + tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_build_version}") session.run( "uv", "pip", From f167c50cf570790a11dd4e608978321bb7f742cb Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:38:23 +0530 Subject: [PATCH 06/45] Bump Pyodide constraints by updating `pyodide-build` --- .../resources/constraints-pyodide312.txt | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 890a0736e..57bcb2c89 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -6,7 +6,7 @@ anyio==4.4.0 # via httpx auditwheel-emscripten==0.0.16 # via pyodide-build -build==1.2.1 +build==1.2.2 # via # -r .nox/update_constraints/tmp/constraints-pyodide.in # pyodide-build @@ -19,13 +19,11 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via typer -cloudpickle==3.0.0 - # via loky -cmake==3.30.2 +cmake==3.30.3 # via pyodide-build distlib==0.3.8 # via virtualenv -filelock==3.15.4 +filelock==3.16.0 # via virtualenv h11==0.14.0 # via httpcore @@ -40,8 +38,6 @@ idna==3.8 # requests leb128==1.0.8 # via auditwheel-emscripten -loky==3.4.1 - # via pyodide-build markdown-it-py==3.0.0 # via rich mdurl==0.1.2 @@ -54,33 +50,31 @@ packaging==24.1 # unearth pip==24.2 # via -r .nox/update_constraints/tmp/constraints-pyodide.in -platformdirs==4.2.2 +platformdirs==4.3.2 # via virtualenv -pydantic==2.8.2 +pydantic==2.9.1 # via # pyodide-build # pyodide-lock -pydantic-core==2.20.1 +pydantic-core==2.23.3 # via pydantic pygments==2.18.0 # via rich -pyodide-build==0.26.1 +pyodide-build==0.28.0 # via -r .nox/update_constraints/tmp/constraints-pyodide.in pyodide-cli==0.2.4 # via # auditwheel-emscripten # pyodide-build -pyodide-lock==0.1.0a6 +pyodide-lock==0.1.0a7 # via pyodide-build pyproject-hooks==1.1.0 # via build -pyyaml==6.0.2 - # via pyodide-build requests==2.32.3 # via pyodide-build resolvelib==1.0.1 # via pyodide-build -rich==13.8.0 +rich==13.8.1 # via # pyodide-build # pyodide-cli @@ -100,8 +94,6 @@ typer==0.12.5 # auditwheel-emscripten # pyodide-build # pyodide-cli -types-requests==2.32.0.20240712 - # via pyodide-build typing-extensions==4.12.2 # via # pydantic @@ -110,10 +102,8 @@ typing-extensions==4.12.2 unearth==0.17.2 # via pyodide-build urllib3==2.2.2 - # via - # requests - # types-requests -virtualenv==20.26.3 + # via requests +virtualenv==20.26.4 # via # build # pyodide-build From 31a6be9557e91bac8c1ad7ac8090f139017c47dd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Sep 2024 18:49:36 +0530 Subject: [PATCH 07/45] Add a schema for `pyodide-version` --- cibuildwheel/resources/cibuildwheel.schema.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 976751a55..714c73067 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -814,6 +814,11 @@ }, "test-requires": { "$ref": "#/properties/test-requires" + }, + "pyodide-version": { + "description": "Specify the Pyodide xbuildenv version to use for building", + "type": "string", + "title": "CIBW_PYODIDE_VERSION" } } } From 90030679658b4e761f8a2f0c66332db7f9c72d85 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 13 Sep 2024 05:09:43 +0530 Subject: [PATCH 08/45] Update Pyodide constraints --- .../resources/constraints-pyodide312.txt | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index a86e06e9a..25a4835c0 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -19,8 +19,6 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via typer -cloudpickle==3.0.0 - # via loky cmake==3.30.3 # via pyodide-build distlib==0.3.8 @@ -54,11 +52,11 @@ pip==24.2 # via -r .nox/update_constraints/tmp/constraints-pyodide.in platformdirs==4.3.2 # via virtualenv -pydantic==2.9.0 +pydantic==2.9.1 # via # pyodide-build # pyodide-lock -pydantic-core==2.23.2 +pydantic-core==2.23.3 # via pydantic pygments==2.18.0 # via rich @@ -96,21 +94,15 @@ typer==0.12.5 # auditwheel-emscripten # pyodide-build # pyodide-cli -types-requests==2.32.0.20240907 - # via pyodide-build typing-extensions==4.12.2 # via # pydantic # pydantic-core # typer -tzdata==2024.1 - # via pydantic unearth==0.17.2 # via pyodide-build -urllib3==2.2.2 - # via - # requests - # types-requests +urllib3==2.2.3 + # via requests virtualenv==20.26.4 # via # build From d8a8d5e8d73267042e8e0743779029516947fca6 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 21:40:47 +0530 Subject: [PATCH 09/45] Bump `pyodide-build` to new 0.29.0 --- .../resources/constraints-pyodide312.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 25a4835c0..2c80f2f9c 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -2,7 +2,7 @@ # nox -s update_constraints annotated-types==0.7.0 # via pydantic -anyio==4.4.0 +anyio==4.6.0 # via httpx auditwheel-emscripten==0.0.16 # via pyodide-build @@ -23,7 +23,7 @@ cmake==3.30.3 # via pyodide-build distlib==0.3.8 # via virtualenv -filelock==3.16.0 +filelock==3.16.1 # via virtualenv h11==0.14.0 # via httpcore @@ -31,7 +31,7 @@ httpcore==1.0.5 # via httpx httpx==0.27.2 # via unearth -idna==3.8 +idna==3.10 # via # anyio # httpx @@ -50,17 +50,17 @@ packaging==24.1 # unearth pip==24.2 # via -r .nox/update_constraints/tmp/constraints-pyodide.in -platformdirs==4.3.2 +platformdirs==4.3.6 # via virtualenv -pydantic==2.9.1 +pydantic==2.9.2 # via # pyodide-build # pyodide-lock -pydantic-core==2.23.3 +pydantic-core==2.23.4 # via pydantic pygments==2.18.0 # via rich -pyodide-build==0.28.0 +pyodide-build==0.29.0 # via -r .nox/update_constraints/tmp/constraints-pyodide.in pyodide-cli==0.2.4 # via @@ -103,7 +103,7 @@ unearth==0.17.2 # via pyodide-build urllib3==2.2.3 # via requests -virtualenv==20.26.4 +virtualenv==20.26.5 # via # build # pyodide-build From cf5dd3ec9ac65c6e7121ea6d8d4d58158df634bd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 21:41:00 +0530 Subject: [PATCH 10/45] Test out another Pyodide identifier --- cibuildwheel/resources/build-platforms.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 03fccc9d9..30d5f5a3c 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,5 +172,6 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.28.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.27.0a2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From 4fe86e0bdc90aaa9bb4afc6d66f53f1b0daec814 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 21:47:34 +0530 Subject: [PATCH 11/45] Update outdated Pyodide constraints --- cibuildwheel/resources/constraints-pyodide312.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index ebb1c6305..2c80f2f9c 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -94,8 +94,6 @@ typer==0.12.5 # auditwheel-emscripten # pyodide-build # pyodide-cli -types-requests==2.32.0.20240914 - # via pyodide-build typing-extensions==4.12.2 # via # pydantic @@ -104,9 +102,7 @@ typing-extensions==4.12.2 unearth==0.17.2 # via pyodide-build urllib3==2.2.3 - # via - # requests - # types-requests + # via requests virtualenv==20.26.5 # via # build From b6830ee4dad18274d60e071655142b70444c971d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:25:47 +0530 Subject: [PATCH 12/45] Add Pyodide version to temp directory name --- cibuildwheel/pyodide.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 4b8bef432..7253202a5 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -226,7 +226,8 @@ def build(options: Options, tmp_path: Path) -> None: log.build_start(config.identifier) - identifier_tmp_dir = tmp_path / config.identifier + # Include both the identifier and the Pyodide version in the temp directory name + identifier_tmp_dir = tmp_path / f"{config.identifier}_{config.pyodide_version}" built_wheel_dir = identifier_tmp_dir / "built_wheel" repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" identifier_tmp_dir.mkdir() From aaf32e594d44c1259bc227cbd5e14ac014e8d8c5 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:34:58 +0530 Subject: [PATCH 13/45] Remove Pyodide 0.26.1 from build configurations --- cibuildwheel/resources/build-platforms.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 30d5f5a3c..15a711f54 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,6 +172,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.27.0a2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From bb6e0d67f85f464ab2ffcf0c3c743b3855b67b4d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:57:25 +0530 Subject: [PATCH 14/45] Retrieve + validate + install specific xbuildenvs --- cibuildwheel/pyodide.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 7253202a5..50e0be661 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os import shutil import sys @@ -66,6 +67,36 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path +def retrieve_compatible_xbuildenvs() -> list[str]: + """Search for compatible xbuildenvs for the current pyodide-build version.""" + xbuildenvs = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + capture_stdout=True, + ).strip() + + xbuildenvs_dict = json.loads(xbuildenvs) + compatible_xbuildenvs = [ + env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + ] + + return compatible_xbuildenvs + + +def validate_xbuildenv_version(pyodide_build_version: str, xbuildenv_version: str) -> None: + """Validate that the requested xbuildenv version is compatible with the pyodide-build version.""" + xbuildenvs = retrieve_compatible_xbuildenvs() + if xbuildenv_version not in xbuildenvs: + msg = ( + f"The xbuildenv version {xbuildenv_version} is not compatible with the pyodide-build version {pyodide_build_version}." + f" The compatible versions are: {xbuildenvs}. Please use the 'pyodide xbuildenv search' command to find the compatible versions." + ) + raise errors.FatalError(msg) + + def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall @@ -75,6 +106,16 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v CIBW_CACHE_PATH / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" ) + + # Validate that the requested xbuildenv version is compatible with the pyodide-build version. + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this function really only checks for the case where the + # version is supplied manually through CIBW_PYODIDE_VERSION environment variable. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION") + if cibw_pyodide_version: + validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version) + pyodide_version = cibw_pyodide_version + with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): return str(pyodide_root) From 735d5bb100705a6225ae90e959a2b031750ec55b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:58:38 +0530 Subject: [PATCH 15/45] Test wheel builds with Pyodide 0.26.2 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 54e950e80..68a92cabf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -194,6 +194,7 @@ jobs: output-dir: wheelhouse env: CIBW_PLATFORM: pyodide + CIBW_PYODIDE_VERSION: 0.26.2 - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | From b8ac6c015fff7cf3312d57683874eda238dd3f5d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 23:09:51 +0530 Subject: [PATCH 16/45] Add correct Pyodide version to identifier temp dir --- cibuildwheel/pyodide.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 50e0be661..f019c0202 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -268,7 +268,9 @@ def build(options: Options, tmp_path: Path) -> None: log.build_start(config.identifier) # Include both the identifier and the Pyodide version in the temp directory name - identifier_tmp_dir = tmp_path / f"{config.identifier}_{config.pyodide_version}" + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", config.pyodide_version) + identifier_tmp_dir = tmp_path / f"{config.identifier}_{cibw_pyodide_version}" + built_wheel_dir = identifier_tmp_dir / "built_wheel" repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" identifier_tmp_dir.mkdir() From 7796311dc62ec42461534404cb9dbffc5b77a05a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 23:19:59 +0530 Subject: [PATCH 17/45] Don't pre-call Pyodide xbuildenv search --- cibuildwheel/pyodide.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index f019c0202..3d3778305 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -107,19 +107,19 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" ) - # Validate that the requested xbuildenv version is compatible with the pyodide-build version. - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this function really only checks for the case where the - # version is supplied manually through CIBW_PYODIDE_VERSION environment variable. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION") - if cibw_pyodide_version: - validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version) - pyodide_version = cibw_pyodide_version - with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): return str(pyodide_root) + # Validate that the requested xbuildenv version is compatible with the pyodide-build version. + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this function really only checks for the case where the + # version is supplied manually through CIBW_PYODIDE_VERSION environment variable. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION") + if cibw_pyodide_version: + validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version) + pyodide_version = cibw_pyodide_version + # We don't want to mutate env but we need to delete any existing # PYODIDE_ROOT so copy it first. env = dict(env) From 97e22c882534dec8ee686e51b2813735e16d9ee9 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 23:20:31 +0530 Subject: [PATCH 18/45] Fetch just the stable Pyodide versions --- cibuildwheel/pyodide.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 3d3778305..68b3a529e 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -83,7 +83,12 @@ def retrieve_compatible_xbuildenvs() -> list[str]: env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] ] - return compatible_xbuildenvs + # Fetch just the "stable" versions + compatible_xbuildenvs_filtered = [ + version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") + ] + + return compatible_xbuildenvs_filtered def validate_xbuildenv_version(pyodide_build_version: str, xbuildenv_version: str) -> None: From d30eb6aa882ff0ea3d3061fb8f2193bf4b5f6097 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:04:09 +0530 Subject: [PATCH 19/45] Refactor search + validation + install into one step --- cibuildwheel/pyodide.py | 61 ++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 68b3a529e..4c9f24ba2 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -67,8 +67,20 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def retrieve_compatible_xbuildenvs() -> list[str]: - """Search for compatible xbuildenvs for the current pyodide-build version.""" +def search_and_install_xbuildenv( + env: dict[str, str], pyodide_build_version: str, pyodide_version: str +) -> str: + """Searches, validates, and installs a particular Pyodide xbuildenv version.""" + # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are + # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall + # specify the pyodide-build version in the root path, which will set up the xbuildenv for + # the requested Pyodide version. + pyodide_root = ( + CIBW_CACHE_PATH + / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" + ) + + # Search for compatible xbuildenvs for the current pyodide-build version xbuildenvs = call( "pyodide", "xbuildenv", @@ -77,54 +89,33 @@ def retrieve_compatible_xbuildenvs() -> list[str]: "--all", capture_stdout=True, ).strip() - xbuildenvs_dict = json.loads(xbuildenvs) compatible_xbuildenvs = [ env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] ] - # Fetch just the "stable" versions compatible_xbuildenvs_filtered = [ version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") ] - return compatible_xbuildenvs_filtered - - -def validate_xbuildenv_version(pyodide_build_version: str, xbuildenv_version: str) -> None: - """Validate that the requested xbuildenv version is compatible with the pyodide-build version.""" - xbuildenvs = retrieve_compatible_xbuildenvs() - if xbuildenv_version not in xbuildenvs: + # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this condition really only checks for the case where the + # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) + if cibw_pyodide_version not in compatible_xbuildenvs_filtered: msg = ( - f"The xbuildenv version {xbuildenv_version} is not compatible with the pyodide-build version {pyodide_build_version}." - f" The compatible versions are: {xbuildenvs}. Please use the 'pyodide xbuildenv search' command to find the compatible versions." + f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" + f" version {pyodide_build_version}. The compatible versions available to download are:" + f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to" + f" find the compatible versions for {pyodide_build_version}" ) raise errors.FatalError(msg) - -def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: - # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are - # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall - # specify the pyodide-build version, which will set up the xbuildenv for the requested - # Pyodide version. - pyodide_root = ( - CIBW_CACHE_PATH - / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" - ) - with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): return str(pyodide_root) - # Validate that the requested xbuildenv version is compatible with the pyodide-build version. - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this function really only checks for the case where the - # version is supplied manually through CIBW_PYODIDE_VERSION environment variable. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION") - if cibw_pyodide_version: - validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version) - pyodide_version = cibw_pyodide_version - # We don't want to mutate env but we need to delete any existing # PYODIDE_ROOT so copy it first. env = dict(env) @@ -133,7 +124,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v "pyodide", "xbuildenv", "install", - pyodide_version, + cibw_pyodide_version, env=env, cwd=CIBW_CACHE_PATH, ) @@ -222,7 +213,7 @@ def setup_python( env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...") - env["PYODIDE_ROOT"] = install_xbuildenv( + env["PYODIDE_ROOT"] = search_and_install_xbuildenv( env, python_configuration.pyodide_build_version, python_configuration.pyodide_version ) From ce2a3f08b9725c7ddebea67f86a6efbe72124f6d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:20:46 +0530 Subject: [PATCH 20/45] Move all of it under a lock --- cibuildwheel/pyodide.py | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 4c9f24ba2..ff46e5fd4 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -80,42 +80,42 @@ def search_and_install_xbuildenv( / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" ) - # Search for compatible xbuildenvs for the current pyodide-build version - xbuildenvs = call( - "pyodide", - "xbuildenv", - "search", - "--json", - "--all", - capture_stdout=True, - ).strip() - xbuildenvs_dict = json.loads(xbuildenvs) - compatible_xbuildenvs = [ - env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] - ] - # Fetch just the "stable" versions - compatible_xbuildenvs_filtered = [ - version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") - ] - - # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this condition really only checks for the case where the - # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) - if cibw_pyodide_version not in compatible_xbuildenvs_filtered: - msg = ( - f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" - f" version {pyodide_build_version}. The compatible versions available to download are:" - f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to" - f" find the compatible versions for {pyodide_build_version}" - ) - raise errors.FatalError(msg) - with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): return str(pyodide_root) + # Search for compatible xbuildenvs for the current pyodide-build version + xbuildenvs = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + capture_stdout=True, + ).strip() + xbuildenvs_dict = json.loads(xbuildenvs) + compatible_xbuildenvs = [ + env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + ] + # Fetch just the "stable" versions + compatible_xbuildenvs_filtered = [ + version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") + ] + + # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this condition really only checks for the case where the + # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) + if cibw_pyodide_version not in compatible_xbuildenvs_filtered: + msg = ( + f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" + f" version {pyodide_build_version}. The compatible versions available to download are:" + f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to" + f" find the compatible versions for {pyodide_build_version}" + ) + raise errors.FatalError(msg) + # We don't want to mutate env but we need to delete any existing # PYODIDE_ROOT so copy it first. env = dict(env) From 13fbf66873c7e1a270d03da2b1824ff70b4f0eb4 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:28:48 +0530 Subject: [PATCH 21/45] Reorder xbuildenv installation --- cibuildwheel/pyodide.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index ff46e5fd4..353d9b71b 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -84,7 +84,12 @@ def search_and_install_xbuildenv( if pyodide_root.exists(): return str(pyodide_root) - # Search for compatible xbuildenvs for the current pyodide-build version + # We don't want to mutate env but we need to delete any existing + # PYODIDE_ROOT so copy it first. + env = dict(env) + env.pop("PYODIDE_ROOT", None) + + # 1. Search for compatible xbuildenvs for the current pyodide-build version xbuildenvs = call( "pyodide", "xbuildenv", @@ -102,7 +107,7 @@ def search_and_install_xbuildenv( version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") ] - # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. + # 2. Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, # which will always be compatible. Hence, this condition really only checks for the case where the # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. @@ -116,10 +121,7 @@ def search_and_install_xbuildenv( ) raise errors.FatalError(msg) - # We don't want to mutate env but we need to delete any existing - # PYODIDE_ROOT so copy it first. - env = dict(env) - env.pop("PYODIDE_ROOT", None) + # 3. Install the xbuildenv call( "pyodide", "xbuildenv", From 14ec0712667dcca54e18efb83012cb7bd3ee0564 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:34:49 +0530 Subject: [PATCH 22/45] Add env and cwd to xbuildenv search call --- cibuildwheel/pyodide.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 353d9b71b..04dcfe732 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -96,6 +96,8 @@ def search_and_install_xbuildenv( "search", "--json", "--all", + env=env, + cwd=CIBW_CACHE_PATH, capture_stdout=True, ).strip() xbuildenvs_dict = json.loads(xbuildenvs) From aae64bbc1e2631156ee03cf21fd259f09c54e63f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:39:33 +0530 Subject: [PATCH 23/45] Temporarily lower to 0.26.2 target --- cibuildwheel/resources/build-platforms.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 15a711f54..f2ca53ce5 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,5 +172,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.27.0a2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From 6956121375229028d0d10b74c2689153097eba89 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:34:00 +0530 Subject: [PATCH 24/45] Separate out search, validate, install; again --- cibuildwheel/pyodide.py | 97 +++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 04dcfe732..862df5603 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -67,10 +67,49 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def search_and_install_xbuildenv( - env: dict[str, str], pyodide_build_version: str, pyodide_version: str -) -> str: - """Searches, validates, and installs a particular Pyodide xbuildenv version.""" +def search_xbuildenv() -> list[str]: + """Searches for the compatible xbuildenvs for the current pyodide-build version""" + xbuildenvs = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + capture_stdout=True, + ).strip() + xbuildenvs_dict = json.loads(xbuildenvs) + compatible_xbuildenvs = [ + env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + ] + # Fetch just the "stable" versions + compatible_xbuildenvs_filtered = [ + version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") + ] + + return compatible_xbuildenvs_filtered + + +def validate_xbuildenv(pyodide_version: str, pyodide_build_version: str) -> None: + """Validate the Pyodide version if set manually for the current pyodide-build version""" + pyodide_versions = search_xbuildenv() + + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this condition really checks only for the case where the + # version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an + # error as appropriate. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) + if cibw_pyodide_version not in pyodide_versions: + msg = ( + f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" + f" version {pyodide_build_version}. The compatible versions available to download are:" + f" {pyodide_versions}. Please use the 'pyodide xbuildenv search' command to" + f" find the compatible versions for {pyodide_build_version}" + ) + raise errors.FatalError(msg) + + +def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: + """Installs a particular Pyodide xbuildenv version.""" # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall # specify the pyodide-build version in the root path, which will set up the xbuildenv for @@ -89,46 +128,12 @@ def search_and_install_xbuildenv( env = dict(env) env.pop("PYODIDE_ROOT", None) - # 1. Search for compatible xbuildenvs for the current pyodide-build version - xbuildenvs = call( - "pyodide", - "xbuildenv", - "search", - "--json", - "--all", - env=env, - cwd=CIBW_CACHE_PATH, - capture_stdout=True, - ).strip() - xbuildenvs_dict = json.loads(xbuildenvs) - compatible_xbuildenvs = [ - env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] - ] - # Fetch just the "stable" versions - compatible_xbuildenvs_filtered = [ - version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") - ] - - # 2. Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this condition really only checks for the case where the - # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) - if cibw_pyodide_version not in compatible_xbuildenvs_filtered: - msg = ( - f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" - f" version {pyodide_build_version}. The compatible versions available to download are:" - f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to" - f" find the compatible versions for {pyodide_build_version}" - ) - raise errors.FatalError(msg) - # 3. Install the xbuildenv call( "pyodide", "xbuildenv", "install", - cibw_pyodide_version, + pyodide_version, env=env, cwd=CIBW_CACHE_PATH, ) @@ -216,9 +221,17 @@ def setup_python( env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) - log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...") - env["PYODIDE_ROOT"] = search_and_install_xbuildenv( - env, python_configuration.pyodide_build_version, python_configuration.pyodide_version + # Allow overriding the xbuildenv version with an environment variable. This allows + # testing new Pyodide xbuildenv versions before they are officially released, or for + # using a different Pyodide version other than the one that is listed to be compatible + # with the pyodide-build version in the build-platforms.toml file. + cibw_pyodide_version = os.environ.get( + "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version + ) + log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") + validate_xbuildenv(cibw_pyodide_version, cibw_pyodide_version) + env["PYODIDE_ROOT"] = install_xbuildenv( + env, python_configuration.pyodide_build_version, cibw_pyodide_version ) return env From e5d8443586839839109c3972c5ee4bb05462b332 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:38:46 +0530 Subject: [PATCH 25/45] Run xbuildenv search in `CIBW_CACHE_PATH` --- cibuildwheel/pyodide.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 862df5603..590856176 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -75,6 +75,7 @@ def search_xbuildenv() -> list[str]: "search", "--json", "--all", + cwd=CIBW_CACHE_PATH, capture_stdout=True, ).strip() xbuildenvs_dict = json.loads(xbuildenvs) From 95c368116aad3b32a87b33169dda7251e6f1a396 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:54:57 +0530 Subject: [PATCH 26/45] Remove prior `PYODIDE_ROOT` env vars, copy envs --- cibuildwheel/pyodide.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 590856176..079f2e0d8 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -69,12 +69,17 @@ def install_emscripten(tmp: Path, version: str) -> Path: def search_xbuildenv() -> list[str]: """Searches for the compatible xbuildenvs for the current pyodide-build version""" + + env = dict(os.environ) + env.pop("PYODIDE_ROOT", None) + xbuildenvs = call( "pyodide", "xbuildenv", "search", "--json", "--all", + env=env, cwd=CIBW_CACHE_PATH, capture_stdout=True, ).strip() @@ -86,6 +91,7 @@ def search_xbuildenv() -> list[str]: compatible_xbuildenvs_filtered = [ version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") ] + # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions return compatible_xbuildenvs_filtered @@ -110,7 +116,7 @@ def validate_xbuildenv(pyodide_version: str, pyodide_build_version: str) -> None def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: - """Installs a particular Pyodide xbuildenv version.""" + """Install a particular Pyodide xbuildenv version and set a path to the Pyodide root.""" # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall # specify the pyodide-build version in the root path, which will set up the xbuildenv for From 09af46f14c9dede60160727878f74f7fb292c029 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 02:08:03 +0530 Subject: [PATCH 27/45] Validate doesn't need to depend on searching --- cibuildwheel/pyodide.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 079f2e0d8..b0ac81dcb 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -67,12 +67,8 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def search_xbuildenv() -> list[str]: +def search_xbuildenv(env: dict[str, str]) -> list[str]: """Searches for the compatible xbuildenvs for the current pyodide-build version""" - - env = dict(os.environ) - env.pop("PYODIDE_ROOT", None) - xbuildenvs = call( "pyodide", "xbuildenv", @@ -96,20 +92,21 @@ def search_xbuildenv() -> list[str]: return compatible_xbuildenvs_filtered -def validate_xbuildenv(pyodide_version: str, pyodide_build_version: str) -> None: - """Validate the Pyodide version if set manually for the current pyodide-build version""" - pyodide_versions = search_xbuildenv() +# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, +# which will always be compatible. Hence, this condition really checks only for the case where the +# version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an +# error as appropriate. +def validate_xbuildenv( + cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str] +) -> None: + """Validate the Pyodide version if set manually for the current pyodide-build version + against a list of compatible versions.""" - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this condition really checks only for the case where the - # version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an - # error as appropriate. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) - if cibw_pyodide_version not in pyodide_versions: + if cibw_pyodide_version not in compatible_versions: msg = ( f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" f" version {pyodide_build_version}. The compatible versions available to download are:" - f" {pyodide_versions}. Please use the 'pyodide xbuildenv search' command to" + f" {compatible_versions}. Please use the 'pyodide xbuildenv search' command to" f" find the compatible versions for {pyodide_build_version}" ) raise errors.FatalError(msg) @@ -236,7 +233,12 @@ def setup_python( "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version ) log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") - validate_xbuildenv(cibw_pyodide_version, cibw_pyodide_version) + # Search for compatible xbuildenv versions + compatible_versions = search_xbuildenv(env) + # and then validate the xbuildenv version + validate_xbuildenv( + cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions + ) env["PYODIDE_ROOT"] = install_xbuildenv( env, python_configuration.pyodide_build_version, cibw_pyodide_version ) From 66999cbdc45e6734ff0d69fe3f8593cac4f69b47 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 02:08:47 +0530 Subject: [PATCH 28/45] Add file lock when searching xbuildenvs --- cibuildwheel/pyodide.py | 43 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index b0ac81dcb..bc463da04 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -69,27 +69,28 @@ def install_emscripten(tmp: Path, version: str) -> Path: def search_xbuildenv(env: dict[str, str]) -> list[str]: """Searches for the compatible xbuildenvs for the current pyodide-build version""" - xbuildenvs = call( - "pyodide", - "xbuildenv", - "search", - "--json", - "--all", - env=env, - cwd=CIBW_CACHE_PATH, - capture_stdout=True, - ).strip() - xbuildenvs_dict = json.loads(xbuildenvs) - compatible_xbuildenvs = [ - env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] - ] - # Fetch just the "stable" versions - compatible_xbuildenvs_filtered = [ - version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") - ] - # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions - - return compatible_xbuildenvs_filtered + with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): + xbuildenvs = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + env=env, + cwd=CIBW_CACHE_PATH, + capture_stdout=True, + ).strip() + xbuildenvs_dict = json.loads(xbuildenvs) + compatible_xbuildenvs = [ + env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + ] + # Fetch just the "stable" versions + compatible_xbuildenvs_filtered = [ + version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") + ] + # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions + + return compatible_xbuildenvs_filtered # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, From 1af1e5dd8464bd27dfff8134b4d73cb5b9a0b704 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 03:06:25 +0530 Subject: [PATCH 29/45] Test the original version: 0.26.1 --- .github/workflows/test.yml | 1 - cibuildwheel/resources/build-platforms.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68a92cabf..54e950e80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -194,7 +194,6 @@ jobs: output-dir: wheelhouse env: CIBW_PLATFORM: pyodide - CIBW_PYODIDE_VERSION: 0.26.2 - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index f2ca53ce5..9c4cc2a4c 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,5 +172,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From d3445483bc5ffde485485afb8f0980462084cb7a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:22:32 +0530 Subject: [PATCH 30/45] Update Pyodide constraints --- cibuildwheel/resources/constraints-pyodide312.txt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 394806f20..f575ad707 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -19,8 +19,6 @@ charset-normalizer==3.4.0 # via requests click==8.1.7 # via typer -cloudpickle==3.1.0 - # via loky cmake==3.30.5 # via pyodide-build distlib==0.3.9 @@ -96,8 +94,6 @@ typer==0.12.5 # auditwheel-emscripten # pyodide-build # pyodide-cli -types-requests==2.32.0.20241016 - # via pyodide-build typing-extensions==4.12.2 # via # pydantic @@ -106,9 +102,7 @@ typing-extensions==4.12.2 unearth==0.17.2 # via pyodide-build urllib3==2.2.3 - # via - # requests - # types-requests + # via requests virtualenv==20.27.0 # via # build From 738ed1f5e5a18b7d5a67c19d43820eeef789dd1d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:03:58 +0530 Subject: [PATCH 31/45] Update constraints for `pyodide-build` 0.29.0 again --- cibuildwheel/resources/constraints-pyodide312.txt | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 1920c59da..a0fcc3bee 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -19,8 +19,6 @@ charset-normalizer==3.4.0 # via requests click==8.1.7 # via typer -cloudpickle==3.1.0 - # via loky cmake==3.31.0.1 # via pyodide-build distlib==0.3.9 @@ -54,11 +52,11 @@ pip==24.3.1 # via -r .nox/update_constraints/tmp/constraints-pyodide.in platformdirs==4.3.6 # via virtualenv -pydantic==2.9.2 +pydantic==2.10.0 # via # pyodide-build # pyodide-lock -pydantic-core==2.23.4 +pydantic-core==2.27.0 # via pydantic pygments==2.18.0 # via rich @@ -91,7 +89,7 @@ sniffio==1.3.1 # via # anyio # httpx -typer==0.13.0 +typer==0.13.1 # via # auditwheel-emscripten # pyodide-build @@ -104,9 +102,7 @@ typing-extensions==4.12.2 unearth==0.17.2 # via pyodide-build urllib3==2.2.3 - # via - # requests - # types-requests + # via requests virtualenv==20.27.1 # via # build From 41e466a487b3d411b6dbd4847ad82fab404fb94e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:12:22 +0530 Subject: [PATCH 32/45] =?UTF-8?q?Bump=20Pyodide=20from=20version=200.26.1?= =?UTF-8?q?=20=E2=9E=A1=EF=B8=8F=20version=200.26.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cibuildwheel/resources/build-platforms.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 8cfa436e6..6e2d296b2 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -190,5 +190,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.4", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From 2a66ddd0a8e6ed531c5e033a2992ad8435b627ee Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:15:44 +0530 Subject: [PATCH 33/45] Add note on compatibility for macOS + other archs --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 327bfe1f8..3970c3cb2 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `pyodide` you need to be on an x86-64 linux runner and `python3.12` must be available in `PATH`. +- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may work, too. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. From dff72caa48423f85c50d30483c2e13421a6436e9 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:15:58 +0530 Subject: [PATCH 34/45] Note Pyodide version for Pyodide identifier --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 3970c3cb2..4ce75774c 100644 --- a/docs/options.md +++ b/docs/options.md @@ -321,7 +321,7 @@ If you set the value lower, cibuildwheel will cap it to the lowest supported val Windows arm64 platform support is experimental. For an experimental WebAssembly build with `--platform pyodide`, -`cp312-pyodide_wasm32` is the only platform identifier. +`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.26.4`](https://github.com/pyodide/pyodide/releases/tag/0.26.4). See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for past end-of-life versions of Python, and PyPy2.7. From 9a78845d3d7d8852f495a6db66f369ac54f124d4 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:16:14 +0530 Subject: [PATCH 35/45] Docs about `CIBW_PYODIDE_VERSION` --- docs/setup.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/setup.md b/docs/setup.md index 32e0d01c5..e900eef4b 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -113,7 +113,14 @@ Pre-requisite: you need to have a matching host version of Python (unlike all other cibuildwheel platforms). Linux host highly recommended; macOS hosts may work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work. -You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). +You must target Pyodide with `--platform pyodide` (or use `--only` on the identifier). + +!!!tip + + It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` environment variable to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json) or can be + referenced using the `pyodide xbuildenv search` command. + + This option is **not available** in the `pyproject.toml` config. ## Configure a CI service From e551021a1bf71d8966baa6ace43adf162a6f7e23 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:47:09 +0530 Subject: [PATCH 36/45] Don't fetch just the stable versions --- cibuildwheel/pyodide.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 6c547124d..843e5339f 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -81,16 +81,11 @@ def search_xbuildenv(env: dict[str, str]) -> list[str]: capture_stdout=True, ).strip() xbuildenvs_dict = json.loads(xbuildenvs) - compatible_xbuildenvs = [ + compatible_xbuildenv_versions = [ env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] ] - # Fetch just the "stable" versions - compatible_xbuildenvs_filtered = [ - version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") - ] - # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions - return compatible_xbuildenvs_filtered + return compatible_xbuildenv_versions # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, From 1fe8a04d0a7c72b55697685dd358d47756aac54d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:26:16 +0530 Subject: [PATCH 37/45] Discard a variable that's not used later --- cibuildwheel/pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 843e5339f..44c5ccbb9 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -82,7 +82,7 @@ def search_xbuildenv(env: dict[str, str]) -> list[str]: ).strip() xbuildenvs_dict = json.loads(xbuildenvs) compatible_xbuildenv_versions = [ - env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + _["version"] for _ in xbuildenvs_dict["environments"] if _["compatible"] ] return compatible_xbuildenv_versions From 8a8177c646f365bb8654f9f5560446acadf246bd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:29:02 +0530 Subject: [PATCH 38/45] =?UTF-8?q?Rename=20`search=5Fxbuildenv`=20=E2=9E=A1?= =?UTF-8?q?=EF=B8=8F=20`get=5Fxbuildenv=5Fversions`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cibuildwheel/pyodide.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 44c5ccbb9..b6cb1a427 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -67,7 +67,7 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def search_xbuildenv(env: dict[str, str]) -> list[str]: +def get_xbuildenv_versions(env: dict[str, str]) -> list[str]: """Searches for the compatible xbuildenvs for the current pyodide-build version""" with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): xbuildenvs = call( @@ -230,7 +230,7 @@ def setup_python( ) log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") # Search for compatible xbuildenv versions - compatible_versions = search_xbuildenv(env) + compatible_versions = get_xbuildenv_versions(env) # and then validate the xbuildenv version validate_xbuildenv( cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions From fffb7050aa3eb15147b9409f2ca30f6f01092a8a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:29:32 +0530 Subject: [PATCH 39/45] =?UTF-8?q?`validate=5Fxbuildenv`=20=E2=9E=A1?= =?UTF-8?q?=EF=B8=8F=20`validate=5Fxbuildenv=5Fversion`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cibuildwheel/pyodide.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index b6cb1a427..2f403d499 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -92,7 +92,7 @@ def get_xbuildenv_versions(env: dict[str, str]) -> list[str]: # which will always be compatible. Hence, this condition really checks only for the case where the # version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an # error as appropriate. -def validate_xbuildenv( +def validate_xbuildenv_version( cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str] ) -> None: """Validate the Pyodide version if set manually for the current pyodide-build version @@ -232,7 +232,7 @@ def setup_python( # Search for compatible xbuildenv versions compatible_versions = get_xbuildenv_versions(env) # and then validate the xbuildenv version - validate_xbuildenv( + validate_xbuildenv_version( cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions ) env["PYODIDE_ROOT"] = install_xbuildenv( From 0df3c45f28761a16f56bee9441805f8e8f2ab23b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:36:58 +0530 Subject: [PATCH 40/45] Replace ordered comment, add newline --- cibuildwheel/pyodide.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 2f403d499..6fb4447ae 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -101,9 +101,9 @@ def validate_xbuildenv_version( if cibw_pyodide_version not in compatible_versions: msg = ( f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" - f" version {pyodide_build_version}. The compatible versions available to download are:" - f" {compatible_versions}. Please use the 'pyodide xbuildenv search' command to" - f" find the compatible versions for {pyodide_build_version}" + f" version {pyodide_build_version}. The compatible versions available to download are:\n" + f"{compatible_versions}. Please use the 'pyodide xbuildenv search' command to" + f" find the compatible versions for {pyodide_build_version}." ) raise errors.FatalError(msg) @@ -128,7 +128,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v env = dict(env) env.pop("PYODIDE_ROOT", None) - # 3. Install the xbuildenv + # Install the xbuildenv call( "pyodide", "xbuildenv", From 057e542fa3a670dd4e92773e0e33a12b806dbc6a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:37:53 +0530 Subject: [PATCH 41/45] Replace sentence on macOS support Co-Authored-By: Hood Chatham --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 4ce75774c..8281836ef 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may work, too. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. +- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts work, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. From 29b5effd8c586d5a2705c43304ba4c4922dbc87c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:40:31 +0530 Subject: [PATCH 42/45] =?UTF-8?q?Capitalise:=20"pyodide"=20=E2=9E=A1?= =?UTF-8?q?=EF=B8=8F=20"Pyodide"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Hood Chatham --- test/test_emscripten.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_emscripten.py b/test/test_emscripten.py index aa7750bc5..7a8fd9953 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -20,7 +20,7 @@ def check_node(): # cibuildwheel adds a pinned node version to the PATH - # check it's in the PATH then, check it's the one that runs pyodide + # check it's in the PATH then, check it's the one that runs Pyodide cibw_cache_path = Path(sys.argv[1]).resolve(strict=True) # find the node executable in PATH node = shutil.which("node") From 321e0dee3b7facc42fa8e913c56a96a810c9817a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:21:26 +0530 Subject: [PATCH 43/45] =?UTF-8?q?"work"=20=E2=9E=A1=EF=B8=8F=20"may=20succ?= =?UTF-8?q?eed"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Hood Chatham --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 8281836ef..d8cff279c 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts work, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. +- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. From 92babdb76184f1b5b29b310ea223c7cbef49ef36 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:28:00 +0530 Subject: [PATCH 44/45] Add another job to test a custom Pyodide version --- .github/workflows/test.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3757be634..45ba25b09 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -171,9 +171,13 @@ jobs: run: uv run pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py test-pyodide: - name: Test cibuildwheel building pyodide wheels + name: Test cibuildwheel building Pyodide wheels (${{ matrix.pyodide-version }} version) needs: lint runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + pyodide-version: ["default", "custom"] timeout-minutes: 180 steps: - uses: actions/checkout@v4 @@ -191,12 +195,26 @@ jobs: uv run -m test.test_projects test.test_0_basic.basic_project sample_proj - name: Run a sample build (GitHub Action) + if: matrix.pyodide-version == 'default' + uses: ./ + with: + package-dir: sample_proj + output-dir: wheelhouse + env: + CIBW_PLATFORM: pyodide + + - name: Run a sample build (GitHub Action) for an overridden Pyodide version + if: matrix.pyodide-version == 'custom' uses: ./ with: package-dir: sample_proj output-dir: wheelhouse + # In case this breaks at any point in time, switch to using the latest version + # available or any other version that is not the same as the default one set + # in cibuildwheel/resources/build-platforms.toml. env: CIBW_PLATFORM: pyodide + CIBW_PYODIDE_VERSION: "v0.27.0a2" - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | From d73f71f8c812971d98205e76dae40325157d7baa Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:32:02 +0530 Subject: [PATCH 45/45] Handle "v"-prefixed + non-prefixed versions --- .github/workflows/test.yml | 2 +- cibuildwheel/pyodide.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 45ba25b09..6320821a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -214,7 +214,7 @@ jobs: # in cibuildwheel/resources/build-platforms.toml. env: CIBW_PLATFORM: pyodide - CIBW_PYODIDE_VERSION: "v0.27.0a2" + CIBW_PYODIDE_VERSION: "0.27.0a2" - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 6fb4447ae..b8f688094 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -103,7 +103,7 @@ def validate_xbuildenv_version( f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" f" version {pyodide_build_version}. The compatible versions available to download are:\n" f"{compatible_versions}. Please use the 'pyodide xbuildenv search' command to" - f" find the compatible versions for {pyodide_build_version}." + f" find the compatible versions for 'pyodide-build' {pyodide_build_version}." ) raise errors.FatalError(msg) @@ -228,6 +228,8 @@ def setup_python( cibw_pyodide_version = os.environ.get( "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version ) + # If there's a "v" prefix, remove it: both would be equally valid + cibw_pyodide_version = cibw_pyodide_version.lstrip("v") log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") # Search for compatible xbuildenv versions compatible_versions = get_xbuildenv_versions(env)