chore: refactor cibuildwheel.utils (pypa#2252)

* chore: refactor cibuildwheel.utils

* rework build_frontend extra flags

* ci(fix): use tonistiigi/binfmt:qemu-v8.1.5 image for qemu

Author: mayeut

.gitignore

@@ -84,9 +84,7 @@ celerybeat-schedule
 
 # virtualenv
 .venv
-venv/
-venv3/
-venv2/
+venv*/
 ENV/
 env/
 env2/
@@ -112,8 +110,5 @@ all_known_setup.yaml
 # mkdocs
 site/
 
-# Virtual environments
-venv*
-
 # PyCharm
 .idea/

cibuildwheel/__main__.py

@@ -8,12 +8,13 @@
 import sys
 import tarfile
 import textwrap
+import time
 import traceback
 import typing
-from collections.abc import Iterable, Sequence, Set
+from collections.abc import Generator, Iterable, Sequence, Set
 from pathlib import Path
 from tempfile import mkdtemp
-from typing import Protocol, assert_never
+from typing import Any, Protocol, TextIO, assert_never
 
 import cibuildwheel
 import cibuildwheel.linux
@@ -23,26 +24,43 @@
 import cibuildwheel.windows
 from cibuildwheel import errors
 from cibuildwheel.architecture import Architecture, allowed_architectures_check
+from cibuildwheel.ci import CIProvider, detect_ci_provider, fix_ansi_codes_for_github_actions
 from cibuildwheel.logger import log
 from cibuildwheel.options import CommandLineArguments, Options, compute_options
+from cibuildwheel.selector import BuildSelector, EnableGroup
 from cibuildwheel.typing import PLATFORMS, GenericPythonConfiguration, PlatformName
-from cibuildwheel.util import (
-    CIBW_CACHE_PATH,
-    BuildSelector,
-    CIProvider,
-    EnableGroup,
-    Unbuffered,
-    detect_ci_provider,
-    fix_ansi_codes_for_github_actions,
-    strtobool,
-)
+from cibuildwheel.util.file import CIBW_CACHE_PATH
+from cibuildwheel.util.helpers import strtobool
 
 
 @dataclasses.dataclass
 class GlobalOptions:
     print_traceback_on_error: bool = True  # decides what happens when errors are hit.
 
 
+@dataclasses.dataclass(frozen=True)
+class FileReport:
+    name: str
+    size: str
+
+
+# Taken from https://stackoverflow.com/a/107717
+class Unbuffered:
+    def __init__(self, stream: TextIO) -> None:
+        self.stream = stream
+
+    def write(self, data: str) -> None:
+        self.stream.write(data)
+        self.stream.flush()
+
+    def writelines(self, data: Iterable[str]) -> None:
+        self.stream.writelines(data)
+        self.stream.flush()
+
+    def __getattr__(self, attr: str) -> Any:
+        return getattr(self.stream, attr)
+
+
 def main() -> None:
     global_options = GlobalOptions()
     try:
@@ -288,6 +306,42 @@ def get_platform_module(platform: PlatformName) -> PlatformModule:
     assert_never(platform)
 
 
+@contextlib.contextmanager
+def print_new_wheels(msg: str, output_dir: Path) -> Generator[None, None, None]:
+    """
+    Prints the new items in a directory upon exiting. The message to display
+    can include {n} for number of wheels, {s} for total number of seconds,
+    and/or {m} for total number of minutes. Does not print anything if this
+    exits via exception.
+    """
+
+    start_time = time.time()
+    existing_contents = set(output_dir.iterdir())
+    yield
+    final_contents = set(output_dir.iterdir())
+
+    new_contents = [
+        FileReport(wheel.name, f"{(wheel.stat().st_size + 1023) // 1024:,d}")
+        for wheel in final_contents - existing_contents
+    ]
+
+    if not new_contents:
+        return
+
+    max_name_len = max(len(f.name) for f in new_contents)
+    max_size_len = max(len(f.size) for f in new_contents)
+    n = len(new_contents)
+    s = time.time() - start_time
+    m = s / 60
+    print(
+        msg.format(n=n, s=s, m=m),
+        *sorted(
+            f"  {f.name:<{max_name_len}s}   {f.size:>{max_size_len}s} kB" for f in new_contents
+        ),
+        sep="\n",
+    )
+
+
 def build_in_directory(args: CommandLineArguments) -> None:
     platform: PlatformName = _compute_platform(args)
     if platform == "pyodide" and sys.platform == "win32":
@@ -350,9 +404,7 @@ def build_in_directory(args: CommandLineArguments) -> None:
 
     tmp_path = Path(mkdtemp(prefix="cibw-run-")).resolve(strict=True)
     try:
-        with cibuildwheel.util.print_new_wheels(
-            "\n{n} wheels produced in {m:.0f} minutes:", output_dir
-        ):
+        with print_new_wheels("\n{n} wheels produced in {m:.0f} minutes:", output_dir):
             platform_module.build(options, tmp_path)
     finally:
         # avoid https://github.com/python/cpython/issues/86962 by performing

cibuildwheel/ci.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import os
+import re
+from enum import Enum
+
+from .util.helpers import strtobool
+
+
+class CIProvider(Enum):
+    travis_ci = "travis"
+    appveyor = "appveyor"
+    circle_ci = "circle_ci"
+    azure_pipelines = "azure_pipelines"
+    github_actions = "github_actions"
+    gitlab = "gitlab"
+    cirrus_ci = "cirrus_ci"
+    other = "other"
+
+
+def detect_ci_provider() -> CIProvider | None:
+    if "TRAVIS" in os.environ:
+        return CIProvider.travis_ci
+    elif "APPVEYOR" in os.environ:
+        return CIProvider.appveyor
+    elif "CIRCLECI" in os.environ:
+        return CIProvider.circle_ci
+    elif "AZURE_HTTP_USER_AGENT" in os.environ:
+        return CIProvider.azure_pipelines
+    elif "GITHUB_ACTIONS" in os.environ:
+        return CIProvider.github_actions
+    elif "GITLAB_CI" in os.environ:
+        return CIProvider.gitlab
+    elif "CIRRUS_CI" in os.environ:
+        return CIProvider.cirrus_ci
+    elif strtobool(os.environ.get("CI", "false")):
+        return CIProvider.other
+    else:
+        return None
+
+
+def fix_ansi_codes_for_github_actions(text: str) -> str:
+    """
+    Github Actions forgets the current ANSI style on every new line. This
+    function repeats the current ANSI style on every new line.
+    """
+    ansi_code_regex = re.compile(r"(\033\[[0-9;]*m)")
+    ansi_codes: list[str] = []
+    output = ""
+
+    for line in text.splitlines(keepends=True):
+        # add the current ANSI codes to the beginning of the line
+        output += "".join(ansi_codes) + line
+
+        # split the line at each ANSI code
+        parts = ansi_code_regex.split(line)
+        # if there are any ANSI codes, save them
+        if len(parts) > 1:
+            # iterate over the ANSI codes in this line
+            for code in parts[1::2]:
+                if code == "\033[0m":
+                    # reset the list of ANSI codes when the clear code is found
+                    ansi_codes = []
+                else:
+                    ansi_codes.append(code)
+
+    return output

cibuildwheel/frontend.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import shlex
+import typing
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Literal
+
+from .logger import log
+from .util.helpers import parse_key_value_string
+
+BuildFrontendName = Literal["pip", "build", "build[uv]"]
+
+
+@dataclass(frozen=True)
+class BuildFrontendConfig:
+    name: BuildFrontendName
+    args: Sequence[str] = ()
+
+    @staticmethod
+    def from_config_string(config_string: str) -> BuildFrontendConfig:
+        config_dict = parse_key_value_string(config_string, ["name"], ["args"])
+        name = " ".join(config_dict["name"])
+        if name not in {"pip", "build", "build[uv]"}:
+            msg = f"Unrecognised build frontend {name!r}, only 'pip', 'build', and 'build[uv]' are supported"
+            raise ValueError(msg)
+
+        name = typing.cast(BuildFrontendName, name)
+
+        args = config_dict.get("args") or []
+        return BuildFrontendConfig(name=name, args=args)
+
+    def options_summary(self) -> str | dict[str, str]:
+        if not self.args:
+            return self.name
+        else:
+            return {"name": self.name, "args": repr(self.args)}
+
+
+def _get_verbosity_flags(level: int, frontend: BuildFrontendName) -> list[str]:
+    if frontend == "pip":
+        if level > 0:
+            return ["-" + level * "v"]
+        if level < 0:
+            return ["-" + -level * "q"]
+    elif not 0 <= level < 2:
+        msg = f"build_verbosity {level} is not supported for build frontend. Ignoring."
+        log.warning(msg)
+    return []
+
+
+def _split_config_settings(config_settings: str, frontend: BuildFrontendName) -> list[str]:
+    config_settings_list = shlex.split(config_settings)
+    s = "s" if frontend == "pip" else ""
+    return [f"--config-setting{s}={setting}" for setting in config_settings_list]
+
+
+def get_build_frontend_extra_flags(
+    build_frontend: BuildFrontendConfig, verbosity_level: int, config_settings: str
+) -> list[str]:
+    return [
+        *_split_config_settings(config_settings, build_frontend.name),
+        *build_frontend.args,
+        *_get_verbosity_flags(verbosity_level, build_frontend.name),
+    ]

cibuildwheel/linux.py

@@ -14,21 +14,16 @@
 
 from . import errors
 from .architecture import Architecture
+from .frontend import BuildFrontendConfig, get_build_frontend_extra_flags
 from .logger import log
 from .oci_container import OCIContainer, OCIContainerEngineConfig, OCIPlatform
 from .options import BuildOptions, Options
+from .selector import BuildSelector
 from .typing import PathOrStr
-from .util import (
-    BuildFrontendConfig,
-    BuildSelector,
-    copy_test_sources,
-    find_compatible_wheel,
-    get_build_verbosity_extra_flags,
-    prepare_command,
-    read_python_configs,
-    split_config_settings,
-    unwrap,
-)
+from .util import resources
+from .util.file import copy_test_sources
+from .util.helpers import prepare_command, unwrap
+from .util.packaging import find_compatible_wheel
 
 ARCHITECTURE_OCI_PLATFORM_MAP = {
     Architecture.x86_64: OCIPlatform.AMD64,
@@ -63,7 +58,7 @@ def get_python_configurations(
     build_selector: BuildSelector,
     architectures: Set[Architecture],
 ) -> list[PythonConfiguration]:
-    full_python_configs = read_python_configs("linux")
+    full_python_configs = resources.read_python_configs("linux")
 
     python_configurations = [PythonConfiguration(**item) for item in full_python_configs]
 
@@ -275,11 +270,11 @@ def build_in_container(
             container.call(["rm", "-rf", built_wheel_dir])
             container.call(["mkdir", "-p", built_wheel_dir])
 
-            extra_flags = split_config_settings(build_options.config_settings, build_frontend.name)
-            extra_flags += build_frontend.args
+            extra_flags = get_build_frontend_extra_flags(
+                build_frontend, build_options.build_verbosity, build_options.config_settings
+            )
 
             if build_frontend.name == "pip":
-                extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity)
                 container.call(
                     [
                         "python",
@@ -294,9 +289,6 @@ def build_in_container(
                     env=env,
                 )
             elif build_frontend.name == "build" or build_frontend.name == "build[uv]":
-                if not 0 <= build_options.build_verbosity < 2:
-                    msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring."
-                    log.warning(msg)
                 if use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags:
                     extra_flags += ["--installer=uv"]
                 container.call(

cibuildwheel/logger.py

@@ -7,7 +7,7 @@
 import time
 from typing import IO, AnyStr, Final
 
-from .util import CIProvider, detect_ci_provider
+from .ci import CIProvider, detect_ci_provider
 
 FoldPattern = tuple[str, str]
 DEFAULT_FOLD_PATTERN: Final[FoldPattern] = ("{name}", "")