diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f3ae2f604..6320821a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -171,9 +171,13 @@ jobs: run: uv run pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py test-pyodide: - name: Test cibuildwheel building Pyodide wheels + name: Test cibuildwheel building Pyodide wheels (${{ matrix.pyodide-version }} version) needs: lint runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + pyodide-version: ["default", "custom"] timeout-minutes: 180 steps: - uses: actions/checkout@v4 @@ -191,12 +195,26 @@ jobs: uv run -m test.test_projects test.test_0_basic.basic_project sample_proj - name: Run a sample build (GitHub Action) + if: matrix.pyodide-version == 'default' + uses: ./ + with: + package-dir: sample_proj + output-dir: wheelhouse + env: + CIBW_PLATFORM: pyodide + + - name: Run a sample build (GitHub Action) for an overridden Pyodide version + if: matrix.pyodide-version == 'custom' uses: ./ with: package-dir: sample_proj output-dir: wheelhouse + # In case this breaks at any point in time, switch to using the latest version + # available or any other version that is not the same as the default one set + # in cibuildwheel/resources/build-platforms.toml. env: CIBW_PLATFORM: pyodide + CIBW_PYODIDE_VERSION: "0.27.0a2" - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 19d47e5ed..b8f688094 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os import shutil import sys @@ -66,6 +67,47 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path +def get_xbuildenv_versions(env: dict[str, str]) -> list[str]: + """Searches for the compatible xbuildenvs for the current pyodide-build version""" + with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): + xbuildenvs = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + env=env, + cwd=CIBW_CACHE_PATH, + capture_stdout=True, + ).strip() + xbuildenvs_dict = json.loads(xbuildenvs) + compatible_xbuildenv_versions = [ + _["version"] for _ in xbuildenvs_dict["environments"] if _["compatible"] + ] + + return compatible_xbuildenv_versions + + +# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, +# which will always be compatible. Hence, this condition really checks only for the case where the +# version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an +# error as appropriate. +def validate_xbuildenv_version( + cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str] +) -> None: + """Validate the Pyodide version if set manually for the current pyodide-build version + against a list of compatible versions.""" + + if cibw_pyodide_version not in compatible_versions: + msg = ( + f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" + f" version {pyodide_build_version}. The compatible versions available to download are:\n" + f"{compatible_versions}. Please use the 'pyodide xbuildenv search' command to" + f" find the compatible versions for 'pyodide-build' {pyodide_build_version}." + ) + raise errors.FatalError(msg) + + def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: """Install a particular Pyodide xbuildenv version and set a path to the Pyodide root.""" # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are @@ -76,6 +118,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v CIBW_CACHE_PATH / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" ) + with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): return str(pyodide_root) @@ -84,6 +127,8 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v # PYODIDE_ROOT so copy it first. env = dict(env) env.pop("PYODIDE_ROOT", None) + + # Install the xbuildenv call( "pyodide", "xbuildenv", @@ -176,9 +221,24 @@ def setup_python( env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) - log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...") + # Allow overriding the xbuildenv version with an environment variable. This allows + # testing new Pyodide xbuildenv versions before they are officially released, or for + # using a different Pyodide version other than the one that is listed to be compatible + # with the pyodide-build version in the build-platforms.toml file. + cibw_pyodide_version = os.environ.get( + "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version + ) + # If there's a "v" prefix, remove it: both would be equally valid + cibw_pyodide_version = cibw_pyodide_version.lstrip("v") + log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") + # Search for compatible xbuildenv versions + compatible_versions = get_xbuildenv_versions(env) + # and then validate the xbuildenv version + validate_xbuildenv_version( + cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions + ) env["PYODIDE_ROOT"] = install_xbuildenv( - env, python_configuration.pyodide_build_version, python_configuration.pyodide_version + env, python_configuration.pyodide_build_version, cibw_pyodide_version ) return env @@ -227,7 +287,10 @@ def build(options: Options, tmp_path: Path) -> None: log.build_start(config.identifier) - identifier_tmp_dir = tmp_path / config.identifier + # Include both the identifier and the Pyodide version in the temp directory name + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", config.pyodide_version) + identifier_tmp_dir = tmp_path / f"{config.identifier}_{cibw_pyodide_version}" + built_wheel_dir = identifier_tmp_dir / "built_wheel" repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" identifier_tmp_dir.mkdir() diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 68c195ab2..0db7b0326 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -890,6 +890,11 @@ }, "test-requires": { "$ref": "#/properties/test-requires" + }, + "pyodide-version": { + "description": "Specify the Pyodide xbuildenv version to use for building", + "type": "string", + "title": "CIBW_PYODIDE_VERSION" } } } diff --git a/docs/options.md b/docs/options.md index 327bfe1f8..d8cff279c 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `pyodide` you need to be on an x86-64 linux runner and `python3.12` must be available in `PATH`. +- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. @@ -321,7 +321,7 @@ If you set the value lower, cibuildwheel will cap it to the lowest supported val Windows arm64 platform support is experimental. For an experimental WebAssembly build with `--platform pyodide`, -`cp312-pyodide_wasm32` is the only platform identifier. +`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.26.4`](https://github.com/pyodide/pyodide/releases/tag/0.26.4). See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for past end-of-life versions of Python, and PyPy2.7. diff --git a/docs/setup.md b/docs/setup.md index 32e0d01c5..e900eef4b 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -113,7 +113,14 @@ Pre-requisite: you need to have a matching host version of Python (unlike all other cibuildwheel platforms). Linux host highly recommended; macOS hosts may work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work. -You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). +You must target Pyodide with `--platform pyodide` (or use `--only` on the identifier). + +!!!tip + + It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` environment variable to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json) or can be + referenced using the `pyodide xbuildenv search` command. + + This option is **not available** in the `pyproject.toml` config. ## Configure a CI service