chore: rework single python tests (pypa#1835)

* chore: rework single python tests

* chore(tests): limit number of builds in test_abi3

* chore(tests): limit number of builds in test_before_all

* apply review suggestion

use `os.path.samefile` rather than comparing `os.stat` directly.

test/test_abi_variants.py

@@ -20,7 +20,7 @@
         CAN_USE_ABI3 = IS_CPYTHON and not Py_GIL_DISABLED
         cmdclass = {}
         extension_kwargs = {}
-        if CAN_USE_ABI3 and sys.version_info[:2] >= (3, 8):
+        if CAN_USE_ABI3 and sys.version_info[:2] >= (3, 10):
             from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
 
             class bdist_wheel_abi3(_bdist_wheel):
@@ -33,7 +33,7 @@ def get_tag(self):
                     return python, "abi3", plat
 
             cmdclass["bdist_wheel"] = bdist_wheel_abi3
-            extension_kwargs["define_macros"] = [("Py_LIMITED_API", "0x03080000")]
+            extension_kwargs["define_macros"] = [("Py_LIMITED_API", "0x030A0000")]
             extension_kwargs["py_limited_api"] = True
     """
     ),
@@ -53,17 +53,16 @@ def test_abi3(tmp_path):
         project_dir,
         add_env={
             # free_threaded and PyPy do not have a Py_LIMITED_API equivalent, just build one of those
-            "CIBW_BUILD": "cp3?-* cp31?-* cp313t-* pp310-*"
+            # also limit the number of builds for test performance reasons
+            "CIBW_BUILD": "cp39-* cp310-* pp310-* cp311-* cp313t-*"
         },
     )
 
     # check that the expected wheels are produced
     expected_wheels = [
-        w.replace("cp38-cp38", "cp38-abi3")
+        w.replace("cp310-cp310", "cp310-abi3")
         for w in utils.expected_wheels("spam", "0.1.0")
-        if ("-pp310" in w or "-pp" not in w)
-        and "-cp39" not in w
-        and ("-cp313t" in w or "-cp31" not in w)
+        if "-cp39" in w or "-cp310" in w or "-pp310" in w or "-cp313t" in w
     ]
     assert set(actual_wheels) == set(expected_wheels)
 

test/test_before_all.py

@@ -10,13 +10,22 @@
 project_with_before_build_asserts = test_projects.new_c_project(
     setup_py_add=textwrap.dedent(
         r"""
-        # assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step
-        # is the same one as is currently running.
+        import os
+
         with open("text_info.txt") as f:
             stored_text = f.read()
-
         print("## stored text: " + stored_text)
         assert stored_text == "sample text 123"
+
+        # assert that the Python version as written to python_prefix.txt in the CIBW_BEFORE_ALL step
+        # is not the same one as is currently running.
+        with open('python_prefix.txt') as f:
+            stored_prefix = f.read()
+        print('stored_prefix', stored_prefix)
+        print('sys.prefix', sys.prefix)
+        #  Works around path-comparison bugs caused by short-paths on Windows e.g.
+        #  vssadm~1 instead of vssadministrator
+        assert not os.path.samefile(stored_prefix, sys.prefix)
         """
     )
 )
@@ -30,21 +39,24 @@ def test(tmp_path):
         print("dummy text", file=ff)
 
     # build the wheels
-    before_all_command = '''python -c "import os;open('{project}/text_info.txt', 'w').write('sample text '+os.environ.get('TEST_VAL', ''))"'''
+    before_all_command = (
+        """python -c "import os, sys;open('{project}/text_info.txt', 'w').write('sample text '+os.environ.get('TEST_VAL', ''))" && """
+        '''python -c "import sys; open('{project}/python_prefix.txt', 'w').write(sys.prefix)"'''
+    )
     actual_wheels = utils.cibuildwheel_run(
         project_dir,
         add_env={
             # write python version information to a temporary file, this is
             # checked in setup.py
             "CIBW_BEFORE_ALL": before_all_command,
-            "CIBW_BEFORE_ALL_LINUX": f'{before_all_command} && python -c "import sys; assert sys.version_info >= (3, 6)"',
+            "CIBW_BEFORE_ALL_LINUX": f'{before_all_command} && python -c "import sys; assert sys.version_info >= (3, 8)"',
             "CIBW_ENVIRONMENT": "TEST_VAL='123'",
         },
+        single_python=True,
     )
 
     # also check that we got the right wheels
-    (project_dir / "text_info.txt").unlink()
-    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True)
     assert set(actual_wheels) == set(expected_wheels)
 
 
@@ -72,7 +84,8 @@ def test_cwd(tmp_path):
             "CIBW_BEFORE_ALL": f'''python -c "import os; assert os.getcwd() == {str(project_dir)!r}"''',
             "CIBW_BEFORE_ALL_LINUX": '''python -c "import os; assert os.getcwd() == '/project'"''',
         },
+        single_python=True,
     )
 
-    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True)
     assert set(actual_wheels) == set(expected_wheels)

test/test_before_build.py

@@ -82,7 +82,8 @@ def test_cwd(tmp_path):
             "CIBW_BEFORE_BUILD": f'''python -c "import os; assert os.getcwd() == {str(project_dir)!r}"''',
             "CIBW_BEFORE_BUILD_LINUX": '''python -c "import os; assert os.getcwd() == '/project'"''',
         },
+        single_python=True,
     )
 
-    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True)
     assert set(actual_wheels) == set(expected_wheels)

test/test_build_frontend_args.py

@@ -16,10 +16,8 @@ def test_build_frontend_args(tmp_path, capfd, frontend_name):
     with pytest.raises(subprocess.CalledProcessError):
         utils.cibuildwheel_run(
             project_dir,
-            add_env={
-                "CIBW_BUILD": "cp311-*",
-                "CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h",
-            },
+            add_env={"CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h"},
+            single_python=True,
         )
 
     captured = capfd.readouterr()

test/test_build_skip.py

@@ -6,12 +6,12 @@
 
 project_with_skip_asserts = test_projects.new_c_project(
     setup_py_add=textwrap.dedent(
-        r"""
-        # explode if run on PyPyor Python 3.7 (these should be skipped)
+        rf"""
         if sys.implementation.name != "cpython":
             raise Exception("Only CPython shall be built")
-        if sys.version_info[0:2] == (3, 7):
-            raise Exception("CPython 3.7 should be skipped")
+        expected_version = {"({}, {})".format(*utils.SINGLE_PYTHON_VERSION)}
+        if sys.version_info[0:2] != expected_version:
+            raise Exception("CPython {{}}.{{}} should be skipped".format(*sys.version_info[0:2]))
         """
     )
 )
@@ -21,17 +21,19 @@ def test(tmp_path):
     project_dir = tmp_path / "project"
     project_with_skip_asserts.generate(project_dir)
 
+    skip = " ".join(
+        f"cp3{minor}-*" for minor in range(6, 30) if (3, minor) != utils.SINGLE_PYTHON_VERSION
+    )
+
     # build the wheels
     actual_wheels = utils.cibuildwheel_run(
         project_dir,
         add_env={
             "CIBW_BUILD": "cp3*-*",
-            "CIBW_SKIP": "cp37-*",
+            "CIBW_SKIP": f"*t-* {skip}",
         },
     )
 
-    # check that we got the right wheels. There should be no PyPy or 3.7.
-    expected_wheels = [
-        w for w in utils.expected_wheels("spam", "0.1.0") if ("-cp3" in w) and ("-cp37" not in w)
-    ]
+    # check that we got the right wheels. There should be a single version of CPython.
+    expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True)
     assert set(actual_wheels) == set(expected_wheels)

test/test_container_engine.py

@@ -21,17 +21,16 @@ def test_podman(tmp_path, capfd, request):
     actual_wheels = utils.cibuildwheel_run(
         project_dir,
         add_env={
-            "CIBW_BUILD": "cp310-*{manylinux,musllinux}_x86_64",
+            "CIBW_ARCHS": "x86_64",
             "CIBW_BEFORE_ALL": "echo 'test log statement from before-all'",
             "CIBW_CONTAINER_ENGINE": "podman",
         },
+        single_python=True,
     )
 
     # check that the expected wheels are produced
     expected_wheels = [
-        w
-        for w in utils.expected_wheels("spam", "0.1.0")
-        if ("-cp310-" in w) and ("x86_64" in w) and ("manylinux" in w or "musllinux" in w)
+        w for w in utils.expected_wheels("spam", "0.1.0", single_python=True) if "x86_64" in w
     ]
     assert set(actual_wheels) == set(expected_wheels)
 
@@ -51,15 +50,16 @@ def test_create_args(tmp_path, capfd):
     actual_wheels = utils.cibuildwheel_run(
         project_dir,
         add_env={
-            "CIBW_BUILD": "cp310-manylinux_*",
+            "CIBW_SKIP": "*-musllinux_*",
             "CIBW_BEFORE_ALL": "echo TEST_CREATE_ARGS is set to $TEST_CREATE_ARGS",
             "CIBW_CONTAINER_ENGINE": "docker; create_args: --env=TEST_CREATE_ARGS=itworks",
         },
+        single_python=True,
     )
 
-    expected_wheels = [
-        w for w in utils.expected_wheels("spam", "0.1.0") if ("cp310-manylinux" in w)
-    ]
+    expected_wheels = utils.expected_wheels(
+        "spam", "0.1.0", musllinux_versions=[], single_python=True
+    )
     assert set(actual_wheels) == set(expected_wheels)
 
     captured = capfd.readouterr()

test/test_cpp_standards.py

@@ -56,76 +56,62 @@
 cpp_test_project.files["setup.py"] = jinja2.Template(setup_py_template)
 cpp_test_project.files["spam.cpp"] = jinja2.Template(spam_cpp_template)
 
-cpp11_project = cpp_test_project.copy()
-cpp11_project.template_context["extra_compile_args"] = (
-    ["/std:c++11"] if utils.platform == "windows" else ["-std=c++11"]
-)
-cpp11_project.template_context["spam_cpp_top_level_add"] = "#include <array>"
-
 
 def test_cpp11(tmp_path):
     # This test checks that the C++11 standard is supported
     project_dir = tmp_path / "project"
-
+    cpp11_project = cpp_test_project.copy()
+    cpp11_project.template_context["extra_compile_args"] = (
+        ["/std:c++11"] if utils.platform == "windows" else ["-std=c++11"]
+    )
+    cpp11_project.template_context["spam_cpp_top_level_add"] = "#include <array>"
     cpp11_project.generate(project_dir)
 
-    actual_wheels = utils.cibuildwheel_run(project_dir)
-    expected_wheels = list(utils.expected_wheels("spam", "0.1.0"))
+    actual_wheels = utils.cibuildwheel_run(project_dir, single_python=True)
+    expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True)
 
     assert set(actual_wheels) == set(expected_wheels)
 
 
-cpp14_project = cpp_test_project.copy()
-cpp14_project.template_context["extra_compile_args"] = (
-    ["/std:c++14"] if utils.platform == "windows" else ["-std=c++14"]
-)
-cpp14_project.template_context["spam_cpp_top_level_add"] = "int a = 100'000;"
-
-
 def test_cpp14(tmp_path):
     # This test checks that the C++14 standard is supported
     project_dir = tmp_path / "project"
-
+    cpp14_project = cpp_test_project.copy()
+    cpp14_project.template_context["extra_compile_args"] = (
+        ["/std:c++14"] if utils.platform == "windows" else ["-std=c++14"]
+    )
+    cpp14_project.template_context["spam_cpp_top_level_add"] = "int a = 100'000;"
     cpp14_project.generate(project_dir)
 
-    actual_wheels = utils.cibuildwheel_run(project_dir)
-    expected_wheels = list(utils.expected_wheels("spam", "0.1.0"))
+    actual_wheels = utils.cibuildwheel_run(project_dir, single_python=True)
+    expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True)
 
     assert set(actual_wheels) == set(expected_wheels)
 
 
-cpp17_project = cpp_test_project.copy()
-
-cpp17_project.template_context["extra_compile_args"] = [
-    "/std:c++17" if utils.platform == "windows" else "-std=c++17"
-]
-cpp17_project.template_context["spam_cpp_top_level_add"] = r"""
-#include <utility>
-auto a = std::pair(5.0, false);
-"""
-
-
 def test_cpp17(tmp_path):
     # This test checks that the C++17 standard is supported
     project_dir = tmp_path / "project"
-
+    cpp17_project = cpp_test_project.copy()
+    cpp17_project.template_context["extra_compile_args"] = [
+        "/std:c++17" if utils.platform == "windows" else "-std=c++17"
+    ]
+    cpp17_project.template_context["spam_cpp_top_level_add"] = r"""
+    #include <utility>
+    auto a = std::pair(5.0, false);
+    """
     cpp17_project.generate(project_dir)
 
     if os.environ.get("APPVEYOR_BUILD_WORKER_IMAGE", "") == "Visual Studio 2015":
         pytest.skip("Visual Studio 2015 does not support C++17")
 
-    # Pypy's distutils sets the default compiler to 'msvc9compiler', which
-    # is too old to support cpp17.
-    add_env = {"CIBW_SKIP": "pp*"}
-
+    add_env = {}
     if utils.platform == "macos":
         add_env["MACOSX_DEPLOYMENT_TARGET"] = "10.13"
 
-    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
-    expected_wheels = [
-        w
-        for w in utils.expected_wheels("spam", "0.1.0", macosx_deployment_target="10.13")
-        if "-pp" not in w
-    ]
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env, single_python=True)
+    expected_wheels = utils.expected_wheels(
+        "spam", "0.1.0", macosx_deployment_target="10.13", single_python=True
+    )
 
     assert set(actual_wheels) == set(expected_wheels)

test/test_environment.py

@@ -49,10 +49,11 @@ def test(tmp_path):
             "CIBW_ENVIRONMENT": """CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH=$PATH:/opt/cibw_test_path""",
             "CIBW_ENVIRONMENT_WINDOWS": f'''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$({python_echo} 'test string 3')" PATH="$PATH;/opt/cibw_test_path"''',
         },
+        single_python=True,
     )
 
     # also check that we got the right wheels built
-    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True)
     assert set(actual_wheels) == set(expected_wheels)
 
 
@@ -133,12 +134,12 @@ def test_overridden_pip_constraint(tmp_path, build_frontend):
     actual_wheels = utils.cibuildwheel_run(
         project_dir,
         add_env={
-            "CIBW_BUILD": "cp312-*",
             "CIBW_BUILD_FRONTEND": build_frontend,
             "PIP_CONSTRAINT": str(constraints_file),
             "CIBW_ENVIRONMENT_LINUX": "PIP_CONSTRAINT=./constraints.txt",
         },
+        single_python=True,
     )
 
-    expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp312" in w]
+    expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True)
     assert set(actual_wheels) == set(expected_wheels)