Skip to content

Commit

Permalink
chore: better determinism for the test virtual environment (pypa#1838)
Browse files Browse the repository at this point in the history 
* chore: better determinism for the test virtual environment

* review: apply suggestions
  • Loading branch information
@mayeut
mayeut authored May 27, 2024
1 parent 8f574d1 commit b713086
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 17 deletions.
9 changes: 8 additions & 1 deletion cibuildwheel/linux.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
from pathlib import Path, PurePath, PurePosixPath
from typing import OrderedDict, Tuple

from packaging.version import Version

from ._compat.typing import assert_never
from .architecture import Architecture
from .logger import log
Expand Down Expand Up @@ -332,7 +334,12 @@ def build_in_container(
)
venv_dir = testing_temp_dir / "venv"

container.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env)
# Use embedded dependencies from virtualenv to ensure determinism
venv_args = ["--no-periodic-update", "--pip=embed"]
# In Python<3.12, setuptools & wheel are installed as well
if Version(config.version) < Version("3.12"):
venv_args.extend(("--setuptools=embed", "--wheel=embed"))
container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env)

virtualenv_env = env.copy()
virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
Expand Down
17 changes: 9 additions & 8 deletions cibuildwheel/macos.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from typing import Literal, Tuple

from filelock import FileLock
from packaging.version import Version

from ._compat.typing import assert_never
from .architecture import Architecture
Expand Down Expand Up @@ -359,6 +360,7 @@ def build(options: Options, tmp_path: Path) -> None:
build_options.environment,
build_frontend.name,
)
pip_version = get_pip_version(env)

compatible_wheel = find_compatible_wheel(built_wheels, config.identifier)
if compatible_wheel:
Expand All @@ -384,7 +386,7 @@ def build(options: Options, tmp_path: Path) -> None:
extra_flags += build_frontend.args

build_env = env.copy()
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
build_env["VIRTUALENV_PIP"] = pip_version
if build_options.dependency_constraints:
constraint_path = build_options.dependency_constraints.get_for_python_version(
config.version
Expand Down Expand Up @@ -560,9 +562,12 @@ def build(options: Options, tmp_path: Path) -> None:
call_with_arch = functools.partial(call, *arch_prefix)
shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c")

# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
call_with_arch("python", "-m", "virtualenv", "--no-download", venv_dir, env=env)
# Use pip version from the initial env to ensure determinism
venv_args = ["--no-periodic-update", f"--pip={pip_version}"]
# In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones
if Version(config.version) < Version("3.12"):
venv_args.extend(("--setuptools=embed", "--wheel=embed"))
call_with_arch("python", "-m", "virtualenv", *venv_args, venv_dir, env=env)

virtualenv_env = env.copy()
virtualenv_env["PATH"] = os.pathsep.join(
Expand All @@ -575,10 +580,6 @@ def build(options: Options, tmp_path: Path) -> None:
# check that we are using the Python from the virtual environment
call_with_arch("which", "python", env=virtualenv_env)

# TODO remove me once virtualenv provides pip>=24.1b1
if config.version == "3.13":
call("python", "-m", "pip", "install", "pip>=24.1b1", env=virtualenv_env)

if build_options.before_test:
before_test_prepared = prepare_command(
build_options.before_test,
Expand Down
16 changes: 8 additions & 8 deletions cibuildwheel/windows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,7 @@ def build(options: Options, tmp_path: Path) -> None:
build_options.environment,
build_frontend.name,
)
pip_version = get_pip_version(env)

compatible_wheel = find_compatible_wheel(built_wheels, config.identifier)
if compatible_wheel:
Expand Down Expand Up @@ -415,7 +416,7 @@ def build(options: Options, tmp_path: Path) -> None:
extra_flags += build_frontend.args

build_env = env.copy()
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
build_env["VIRTUALENV_PIP"] = pip_version
if build_options.dependency_constraints:
constraints_path = build_options.dependency_constraints.get_for_python_version(
config.version
Expand Down Expand Up @@ -511,9 +512,12 @@ def build(options: Options, tmp_path: Path) -> None:
call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
venv_dir = identifier_tmp_dir / "venv-test"

# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
call("python", "-m", "virtualenv", "--no-download", venv_dir, env=env)
# Use pip version from the initial env to ensure determinism
venv_args = ["--no-periodic-update", f"--pip={pip_version}"]
# In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones
if Version(config.version) < Version("3.12"):
venv_args.extend(("--setuptools=embed", "--wheel=embed"))
call("python", "-m", "virtualenv", *venv_args, venv_dir, env=env)

virtualenv_env = env.copy()
virtualenv_env["PATH"] = os.pathsep.join(
Expand All @@ -526,10 +530,6 @@ def build(options: Options, tmp_path: Path) -> None:
# check that we are using the Python from the virtual environment
call("where", "python", env=virtualenv_env)

# TODO remove me once virtualenv provides pip>=24.1b1
if config.version.startswith("3.13."):
call("python", "-m", "pip", "install", "--pre", "-U", "pip", env=virtualenv_env)

if build_options.before_test:
before_test_prepared = prepare_command(
build_options.before_test,
Expand Down

0 comments on commit b713086

Please sign in to comment.