Updates for Pyodide builds after pyodide-build was unvendored (#2090)

* Fix typos and capitalise

* Add docstring for `install_xbuildenv` function

* Include Emscripten, Pyodide version in logger messages

* Mark separation of pyodide-build and Pyodide

* Bump to Pyodide 0.26.4 (py3.12.7) + pyodide-build 0.29.0

* Use pyodide-build version as the source of truth

* Update Pyodide constraints

* Use 3.12 as the Python version instead of 3.12.7

Co-authored-by: Henry Schreiner <HenrySchreinerIII@gmail.com>

* chore: appease the formatter

---------

Co-authored-by: Henry Schreiner <HenrySchreinerIII@gmail.com>

Author: agriyakhetarpal

.github/workflows/test.yml

@@ -171,7 +171,7 @@ jobs:
       run: uv run pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py
 
   test-pyodide:
-    name: Test cibuildwheel building pyodide wheels
+    name: Test cibuildwheel building Pyodide wheels
     needs: lint
     runs-on: ubuntu-24.04
     timeout-minutes: 180

cibuildwheel/pyodide.py

@@ -41,6 +41,7 @@ class PythonConfiguration:
     version: str
     identifier: str
     pyodide_version: str
+    pyodide_build_version: str
     emscripten_version: str
     node_version: str
 
@@ -65,10 +66,15 @@ def install_emscripten(tmp: Path, version: str) -> Path:
     return emcc_path
 
 
-def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str:
+def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str:
+    """Install a particular Pyodide xbuildenv version and set a path to the Pyodide root."""
+    # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are
+    # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall
+    # specify the pyodide-build version in the root path, which will set up the xbuildenv for
+    # the requested Pyodide version.
     pyodide_root = (
         CIBW_CACHE_PATH
-        / f".pyodide-xbuildenv-{pyodide_version}/{pyodide_version}/xbuildenv/pyodide-root"
+        / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root"
     )
     with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
         if pyodide_root.exists():
@@ -165,13 +171,15 @@ def setup_python(
         env=env,
     )
 
-    log.step("Installing emscripten...")
+    log.step(f"Installing Emscripten version: {python_configuration.emscripten_version} ...")
     emcc_path = install_emscripten(tmp, python_configuration.emscripten_version)
 
     env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
 
-    log.step("Installing Pyodide xbuildenv...")
-    env["PYODIDE_ROOT"] = install_xbuildenv(env, python_configuration.pyodide_version)
+    log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...")
+    env["PYODIDE_ROOT"] = install_xbuildenv(
+        env, python_configuration.pyodide_build_version, python_configuration.pyodide_version
+    )
 
     return env
 

cibuildwheel/resources/build-platforms.toml

@@ -190,5 +190,5 @@ python_configurations = [
 
 [pyodide]
 python_configurations = [
-  { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", emscripten_version = "3.1.58", node_version = "v20" },
+  { identifier = "cp312-pyodide_wasm32", version = "3.12", pyodide_version = "0.26.4", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
 ]

cibuildwheel/resources/constraints-pyodide312.txt

@@ -19,8 +19,6 @@ charset-normalizer==3.4.0
     # via requests
 click==8.1.7
     # via typer
-cloudpickle==3.1.0
-    # via loky
 cmake==3.31.0.1
     # via pyodide-build
 distlib==0.3.9
@@ -40,8 +38,6 @@ idna==3.10
     #   requests
 leb128==1.0.8
     # via auditwheel-emscripten
-loky==3.4.1
-    # via pyodide-build
 markdown-it-py==3.0.0
     # via rich
 mdurl==0.1.2
@@ -56,26 +52,24 @@ pip==24.3.1
     # via -r .nox/update_constraints/tmp/constraints-pyodide.in
 platformdirs==4.3.6
     # via virtualenv
-pydantic==2.9.2
+pydantic==2.10.0
     # via
     #   pyodide-build
     #   pyodide-lock
-pydantic-core==2.23.4
+pydantic-core==2.27.0
     # via pydantic
 pygments==2.18.0
     # via rich
-pyodide-build==0.26.1
+pyodide-build==0.29.0
     # via -r .nox/update_constraints/tmp/constraints-pyodide.in
 pyodide-cli==0.2.4
     # via
     #   auditwheel-emscripten
     #   pyodide-build
-pyodide-lock==0.1.0a6
+pyodide-lock==0.1.0a7
     # via pyodide-build
 pyproject-hooks==1.2.0
     # via build
-pyyaml==6.0.2
-    # via pyodide-build
 requests==2.32.3
     # via pyodide-build
 resolvelib==1.1.0
@@ -95,13 +89,11 @@ sniffio==1.3.1
     # via
     #   anyio
     #   httpx
-typer==0.13.0
+typer==0.13.1
     # via
     #   auditwheel-emscripten
     #   pyodide-build
     #   pyodide-cli
-types-requests==2.32.0.20241016
-    # via pyodide-build
 typing-extensions==4.12.2
     # via
     #   pydantic
@@ -110,9 +102,7 @@ typing-extensions==4.12.2
 unearth==0.17.2
     # via pyodide-build
 urllib3==2.2.3
-    # via
-    #   requests
-    #   types-requests
+    # via requests
 virtualenv==20.27.1
     # via
     #   build

noxfile.py

@@ -106,10 +106,10 @@ def update_constraints(session: nox.Session) -> None:
     pyodides = build_platforms["pyodide"]["python_configurations"]
     for pyodide in pyodides:
         python_version = ".".join(pyodide["version"].split(".")[:2])
-        pyodide_version = pyodide["pyodide_version"]
+        pyodide_build_version = pyodide["pyodide_build_version"]
         output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt"
         tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in"
-        tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_version}")
+        tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_build_version}")
         session.run(
             "uv",
             "pip",

test/test_emscripten.py

@@ -20,7 +20,7 @@
 
 def check_node():
     # cibuildwheel adds a pinned node version to the PATH
-    # check it's in the PATH then, check it's the one that runs pyoodide
+    # check it's in the PATH then, check it's the one that runs Pyodide
     cibw_cache_path = Path(sys.argv[1]).resolve(strict=True)
     # find the node executable in PATH
     node = shutil.which("node")