From 758350d98e50581a3f2bb090b588dbb882c4523b Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Martinez Date: Tue, 14 Oct 2025 09:20:22 +0200 Subject: [PATCH 1/7] fix: cleanup_loaded_modules: dont clean up modules that have not finished loading (#14840) Sometimes, when pytest imported a module that used `import ddtrace.auto` and gevent was installed, the module would be deleted from the cache too soon, producing a `KeyError`. This fixes the problem ensuring we don't delete unloaded modules from the cache. --------- Signed-off-by: Juanjo Alvarez (cherry picked from commit bbb2a7aec59d3204d76c4a76ccfc5b751da2d1bb) --- ddtrace/bootstrap/sitecustomize.py | 14 +++ .../pytest-ddtrace-auto-e6c09074a4b0af3d.yaml | 4 + tests/internal/test_auto.py | 93 ++++++++++++++++++- 3 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/pytest-ddtrace-auto-e6c09074a4b0af3d.yaml diff --git a/ddtrace/bootstrap/sitecustomize.py b/ddtrace/bootstrap/sitecustomize.py index d7537d0e3ca..a09c3545d06 100644 --- a/ddtrace/bootstrap/sitecustomize.py +++ b/ddtrace/bootstrap/sitecustomize.py @@ -45,6 +45,20 @@ def cleanup_loaded_modules(): def drop(module_name): # type: (str) -> None + module = sys.modules.get(module_name) + # Don't delete modules that are currently being imported (they might be None or incomplete) + # or that don't exist. This can happen when pytest's assertion rewriter is importing modules + # that themselves import ddtrace.auto, which triggers this cleanup during the import process. + if module is None: + return + # Skip modules that don't have a __spec__ attribute yet (still being imported) + if not hasattr(module, "__spec__"): + return + # Check if the module is currently being initialized + # During import, __spec__._initializing is True + spec = getattr(module, "__spec__", None) + if spec is not None and getattr(spec, "_initializing", False): + return del sys.modules[module_name] MODULES_REQUIRING_CLEANUP = ("gevent",) diff --git a/releasenotes/notes/pytest-ddtrace-auto-e6c09074a4b0af3d.yaml b/releasenotes/notes/pytest-ddtrace-auto-e6c09074a4b0af3d.yaml new file mode 100644 index 00000000000..4f959c122b5 --- /dev/null +++ b/releasenotes/notes/pytest-ddtrace-auto-e6c09074a4b0af3d.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fix some modules being unloaded too soon when using pytest + ddtrace + gevent. diff --git a/tests/internal/test_auto.py b/tests/internal/test_auto.py index 069733ea494..ad2be4a1c52 100644 --- a/tests/internal/test_auto.py +++ b/tests/internal/test_auto.py @@ -1,18 +1,105 @@ +from pathlib import Path +import subprocess import sys +import tempfile import pytest -@pytest.mark.skipif(sys.version_info < (3, 12, 5), reason="Python < 3.12.5 eagerly loads the threading module") +# DEV: This test must pass ALWAYS. If this test fails, it means that something +# needs to be fixed somewhere. Please DO NOT skip this test under any +# circumstance! @pytest.mark.subprocess( env=dict( DD_UNLOAD_MODULES_FROM_SITECUSTOMIZE="true", - DD_REMOTE_CONFIGURATION_ENABLED="1", - ) + ), + parametrize={ + "DD_REMOTE_CONFIGURATION_ENABLED": ("1", "0"), + }, ) def test_auto(): + import os import sys import ddtrace.auto # noqa:F401 assert "threading" not in sys.modules + assert "socket" not in sys.modules + assert "subprocess" not in sys.modules + + if os.getenv("DD_REMOTE_CONFIGURATION_ENABLED") == "0": + # When unloading modules we must have the HTTP clients imported already + assert "ddtrace.internal.http" in sys.modules + + # emulate socket patching (e.g. by gevent) + import socket # noqa:F401 + + socket.create_connection = None + socket.socket = None + + from ddtrace.internal.http import HTTPConnection # noqa:F401 + import ddtrace.internal.uds as uds + + assert HTTPConnection("localhost")._create_connection is not None + assert uds.socket.socket is not None + + +@pytest.mark.skipif(sys.version_info < (3, 8), reason="Test requires Python 3.8+") +def test_pytest_with_gevent_and_ddtrace_auto(): + """ + Test that pytest works when a module imports ddtrace.auto and gevent is installed. + + This is a regression test for an issue where cleanup_loaded_modules() would delete + modules from sys.modules while they were still being imported by pytest's assertion + rewriter, causing a KeyError during import. + + The fix checks if a module's __spec__._initializing is True before deleting it. + """ + pytest.importorskip("pytest") + pytest.importorskip("gevent") + + # Create a temporary directory with test files + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + + # Create a module that imports ddtrace.auto + foo_module = tmpdir_path / "foo.py" + foo_module.write_text( + """import ddtrace.auto +A = 1 +""" + ) + + # Create a test file that imports the module + test_file = tmpdir_path / "test_foo.py" + test_file.write_text( + """from foo import A + +def test_foo(): + assert A == 1 +""" + ) + + # Run pytest as a subprocess with the current ddtrace on PYTHONPATH + import os + + import ddtrace + + ddtrace_path = str(Path(ddtrace.__file__).parent.parent) + env = os.environ.copy() + # Prepend ddtrace path to PYTHONPATH to ensure we use the local version + env["PYTHONPATH"] = ddtrace_path + os.pathsep + env.get("PYTHONPATH", "") + + result = subprocess.run( + [sys.executable, "-m", "pytest", str(test_file), "-v"], + cwd=tmpdir, + capture_output=True, + text=True, + env=env, + ) + + # The test should pass without KeyError + assert result.returncode == 0, f"pytest failed:\nSTDOUT:\n{result.stdout}\n\nSTDERR:\n{result.stderr}" + assert "KeyError" not in result.stdout + assert "KeyError" not in result.stderr + assert "1 passed" in result.stdout From 754a8e76537e4e27ec4eb91cea39fa806159d5fe Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Thu, 16 Oct 2025 11:04:27 +0200 Subject: [PATCH 2/7] remove test in backport Signed-off-by: Juanjo Alvarez --- tests/internal/test_auto.py | 93 ++----------------------------------- 1 file changed, 3 insertions(+), 90 deletions(-) diff --git a/tests/internal/test_auto.py b/tests/internal/test_auto.py index ad2be4a1c52..069733ea494 100644 --- a/tests/internal/test_auto.py +++ b/tests/internal/test_auto.py @@ -1,105 +1,18 @@ -from pathlib import Path -import subprocess import sys -import tempfile import pytest -# DEV: This test must pass ALWAYS. If this test fails, it means that something -# needs to be fixed somewhere. Please DO NOT skip this test under any -# circumstance! +@pytest.mark.skipif(sys.version_info < (3, 12, 5), reason="Python < 3.12.5 eagerly loads the threading module") @pytest.mark.subprocess( env=dict( DD_UNLOAD_MODULES_FROM_SITECUSTOMIZE="true", - ), - parametrize={ - "DD_REMOTE_CONFIGURATION_ENABLED": ("1", "0"), - }, + DD_REMOTE_CONFIGURATION_ENABLED="1", + ) ) def test_auto(): - import os import sys import ddtrace.auto # noqa:F401 assert "threading" not in sys.modules - assert "socket" not in sys.modules - assert "subprocess" not in sys.modules - - if os.getenv("DD_REMOTE_CONFIGURATION_ENABLED") == "0": - # When unloading modules we must have the HTTP clients imported already - assert "ddtrace.internal.http" in sys.modules - - # emulate socket patching (e.g. by gevent) - import socket # noqa:F401 - - socket.create_connection = None - socket.socket = None - - from ddtrace.internal.http import HTTPConnection # noqa:F401 - import ddtrace.internal.uds as uds - - assert HTTPConnection("localhost")._create_connection is not None - assert uds.socket.socket is not None - - -@pytest.mark.skipif(sys.version_info < (3, 8), reason="Test requires Python 3.8+") -def test_pytest_with_gevent_and_ddtrace_auto(): - """ - Test that pytest works when a module imports ddtrace.auto and gevent is installed. - - This is a regression test for an issue where cleanup_loaded_modules() would delete - modules from sys.modules while they were still being imported by pytest's assertion - rewriter, causing a KeyError during import. - - The fix checks if a module's __spec__._initializing is True before deleting it. - """ - pytest.importorskip("pytest") - pytest.importorskip("gevent") - - # Create a temporary directory with test files - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir_path = Path(tmpdir) - - # Create a module that imports ddtrace.auto - foo_module = tmpdir_path / "foo.py" - foo_module.write_text( - """import ddtrace.auto -A = 1 -""" - ) - - # Create a test file that imports the module - test_file = tmpdir_path / "test_foo.py" - test_file.write_text( - """from foo import A - -def test_foo(): - assert A == 1 -""" - ) - - # Run pytest as a subprocess with the current ddtrace on PYTHONPATH - import os - - import ddtrace - - ddtrace_path = str(Path(ddtrace.__file__).parent.parent) - env = os.environ.copy() - # Prepend ddtrace path to PYTHONPATH to ensure we use the local version - env["PYTHONPATH"] = ddtrace_path + os.pathsep + env.get("PYTHONPATH", "") - - result = subprocess.run( - [sys.executable, "-m", "pytest", str(test_file), "-v"], - cwd=tmpdir, - capture_output=True, - text=True, - env=env, - ) - - # The test should pass without KeyError - assert result.returncode == 0, f"pytest failed:\nSTDOUT:\n{result.stdout}\n\nSTDERR:\n{result.stderr}" - assert "KeyError" not in result.stdout - assert "KeyError" not in result.stderr - assert "1 passed" in result.stdout From 9b5eb1c4928a003e7e50ac5b6b7b660ca2890369 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Thu, 16 Oct 2025 13:45:26 +0200 Subject: [PATCH 3/7] Update system tests versions Signed-off-by: Juanjo Alvarez --- .github/workflows/system-tests.yml | 6 +++--- .gitlab-ci.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 0b478747552..97a3a8e4b3c 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -47,7 +47,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: '04070bed4dd644284690650abe291e632eb27cd9' + ref: 'e2ed3829e49bdeda8dad48ec3dce42d8c0fc01d2' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -94,7 +94,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: '04070bed4dd644284690650abe291e632eb27cd9' + ref: 'e2ed3829e49bdeda8dad48ec3dce42d8c0fc01d2' - name: Build runner uses: ./.github/actions/install_runner @@ -279,7 +279,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: '04070bed4dd644284690650abe291e632eb27cd9' + ref: 'e2ed3829e49bdeda8dad48ec3dce42d8c0fc01d2' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f7bb2630377..0ff3c04f05e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap" # CI_DEBUG_SERVICES: "true" # Automatically managed, use scripts/update-system-tests-version to update - SYSTEM_TESTS_REF: "04070bed4dd644284690650abe291e632eb27cd9" + SYSTEM_TESTS_REF: "e2ed3829e49bdeda8dad48ec3dce42d8c0fc01d2" default: interruptible: true From 3a12e8c803e65355217ef17ff28b90b1195bf66d Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Thu, 16 Oct 2025 15:08:50 +0200 Subject: [PATCH 4/7] Update system tests versions Signed-off-by: Juanjo Alvarez --- .github/workflows/system-tests.yml | 6 +++--- .gitlab-ci.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 97a3a8e4b3c..2159070929c 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -47,7 +47,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: 'e2ed3829e49bdeda8dad48ec3dce42d8c0fc01d2' + ref: 'ec9667c41847cf5e65e99bb82a2d03efcea3205b' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -94,7 +94,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: 'e2ed3829e49bdeda8dad48ec3dce42d8c0fc01d2' + ref: 'ec9667c41847cf5e65e99bb82a2d03efcea3205b' - name: Build runner uses: ./.github/actions/install_runner @@ -279,7 +279,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: 'e2ed3829e49bdeda8dad48ec3dce42d8c0fc01d2' + ref: 'ec9667c41847cf5e65e99bb82a2d03efcea3205b' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ff3c04f05e..1f1b121a125 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap" # CI_DEBUG_SERVICES: "true" # Automatically managed, use scripts/update-system-tests-version to update - SYSTEM_TESTS_REF: "e2ed3829e49bdeda8dad48ec3dce42d8c0fc01d2" + SYSTEM_TESTS_REF: "ec9667c41847cf5e65e99bb82a2d03efcea3205b" default: interruptible: true From 8552019dd9f0126f8de0f2554cdeed553314285e Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Thu, 16 Oct 2025 15:58:20 +0200 Subject: [PATCH 5/7] Update system tests versions Signed-off-by: Juanjo Alvarez --- .github/workflows/system-tests.yml | 6 +++--- .gitlab-ci.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 2159070929c..d347bab0343 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -47,7 +47,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: 'ec9667c41847cf5e65e99bb82a2d03efcea3205b' + ref: '5a29727d63edca94b0459180b6c9405050b1ec84' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -94,7 +94,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: 'ec9667c41847cf5e65e99bb82a2d03efcea3205b' + ref: '5a29727d63edca94b0459180b6c9405050b1ec84' - name: Build runner uses: ./.github/actions/install_runner @@ -279,7 +279,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: 'ec9667c41847cf5e65e99bb82a2d03efcea3205b' + ref: '5a29727d63edca94b0459180b6c9405050b1ec84' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1f1b121a125..4e6dc10d080 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap" # CI_DEBUG_SERVICES: "true" # Automatically managed, use scripts/update-system-tests-version to update - SYSTEM_TESTS_REF: "ec9667c41847cf5e65e99bb82a2d03efcea3205b" + SYSTEM_TESTS_REF: "5a29727d63edca94b0459180b6c9405050b1ec84" default: interruptible: true From 2580aafcee9f58a6091379f839d58c3b2740f146 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Fri, 17 Oct 2025 09:39:57 +0200 Subject: [PATCH 6/7] Update system tests versions Signed-off-by: Juanjo Alvarez --- .github/workflows/system-tests.yml | 6 +++--- .gitlab-ci.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index d347bab0343..0412013a762 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -47,7 +47,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: '5a29727d63edca94b0459180b6c9405050b1ec84' + ref: 'ba54f3c826c91547081eb7423bcb94acf790042a' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -94,7 +94,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: '5a29727d63edca94b0459180b6c9405050b1ec84' + ref: 'ba54f3c826c91547081eb7423bcb94acf790042a' - name: Build runner uses: ./.github/actions/install_runner @@ -279,7 +279,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: '5a29727d63edca94b0459180b6c9405050b1ec84' + ref: 'ba54f3c826c91547081eb7423bcb94acf790042a' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4e6dc10d080..e6e22d7eb94 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap" # CI_DEBUG_SERVICES: "true" # Automatically managed, use scripts/update-system-tests-version to update - SYSTEM_TESTS_REF: "5a29727d63edca94b0459180b6c9405050b1ec84" + SYSTEM_TESTS_REF: "ba54f3c826c91547081eb7423bcb94acf790042a" default: interruptible: true From 96cf03c6ecad18673088ab04e5631da9e7568730 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 20 Oct 2025 17:05:37 +0200 Subject: [PATCH 7/7] Update system tests versions Signed-off-by: Juanjo Alvarez --- .github/workflows/system-tests.yml | 6 +++--- .gitlab-ci.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 0412013a762..2352a700869 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -47,7 +47,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: 'ba54f3c826c91547081eb7423bcb94acf790042a' + ref: '5a20302459c95e215b6f418e4b296142995baf2a' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -94,7 +94,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: 'ba54f3c826c91547081eb7423bcb94acf790042a' + ref: '5a20302459c95e215b6f418e4b296142995baf2a' - name: Build runner uses: ./.github/actions/install_runner @@ -279,7 +279,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: 'ba54f3c826c91547081eb7423bcb94acf790042a' + ref: '5a20302459c95e215b6f418e4b296142995baf2a' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e6e22d7eb94..6b1d6d6e829 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap" # CI_DEBUG_SERVICES: "true" # Automatically managed, use scripts/update-system-tests-version to update - SYSTEM_TESTS_REF: "ba54f3c826c91547081eb7423bcb94acf790042a" + SYSTEM_TESTS_REF: "5a20302459c95e215b6f418e4b296142995baf2a" default: interruptible: true