From 272d59cbd9f18d86d613a3f36e757ebbbe2b4676 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 13 Oct 2024 12:16:46 +0100 Subject: [PATCH 01/17] avoid refcycles in happy eyeballs --- pyproject.toml | 3 ++- src/anyio/_core/_sockets.py | 31 +++++++++++++++++-------------- tests/test_sockets.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9fb5d07b..a60ac2fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,8 @@ test = [ """\ uvloop >= 0.21.0b1; platform_python_implementation == 'CPython' \ and platform_system != 'Windows'\ - """ + """, + "ephemeral-port-reserve >= 1.1.4", ] doc = [ "packaging", diff --git a/src/anyio/_core/_sockets.py b/src/anyio/_core/_sockets.py index 6070c647..ac134abd 100644 --- a/src/anyio/_core/_sockets.py +++ b/src/anyio/_core/_sockets.py @@ -211,20 +211,23 @@ async def try_connect(remote_host: str, event: Event) -> None: target_addrs = [(socket.AF_INET, addr_obj.compressed)] oserrors: list[OSError] = [] - async with create_task_group() as tg: - for i, (af, addr) in enumerate(target_addrs): - event = Event() - tg.start_soon(try_connect, addr, event) - with move_on_after(happy_eyeballs_delay): - await event.wait() - - if connected_stream is None: - cause = ( - oserrors[0] - if len(oserrors) == 1 - else ExceptionGroup("multiple connection attempts failed", oserrors) - ) - raise OSError("All connection attempts failed") from cause + try: + async with create_task_group() as tg: + for i, (af, addr) in enumerate(target_addrs): + event = Event() + tg.start_soon(try_connect, addr, event) + with move_on_after(happy_eyeballs_delay): + await event.wait() + + if connected_stream is None: + cause = ( + oserrors[0] + if len(oserrors) == 1 + else ExceptionGroup("multiple connection attempts failed", oserrors) + ) + raise OSError("All connection attempts failed") from cause + finally: + oserrors.clear() if tls or tls_hostname or ssl_context: try: diff --git a/tests/test_sockets.py b/tests/test_sockets.py index 42937a36..4ecb5d8f 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -18,6 +18,7 @@ from threading import Thread from typing import Any, NoReturn, TypeVar, cast +import ephemeral_port_reserve import psutil import pytest from _pytest.fixtures import SubRequest @@ -125,6 +126,16 @@ def check_asyncio_bug(anyio_backend_name: str, family: AnyIPAddressFamily) -> No pytest.skip("Does not work due to a known bug (39148)") +if sys.version_info <= (3, 11): + + def no_other_refs() -> list[object]: + return [sys._getframe(1)] +else: + + def no_other_refs() -> list[object]: + return [] + + _T = TypeVar("_T") @@ -307,6 +318,28 @@ def serve() -> None: server_sock.close() assert client_addr[0] == expected_client_addr + @pytest.mark.skipif( + sys.implementation.name == "pypy", + reason=( + "gc.get_referrers is broken on PyPy see " + "https://github.com/pypy/pypy/issues/5075" + ), + ) + async def test_happy_eyeballs_refcycles(self) -> None: + """ + Test derived from https://github.com/python/cpython/pull/124859 + """ + port = ephemeral_port_reserve.reserve() + exc = None + try: + async with await connect_tcp("localhost", port): + pass + except OSError as e: + exc = e.__cause__ + + assert isinstance(exc, OSError) + assert gc.get_referrers(exc) == no_other_refs() + @pytest.mark.parametrize( "target, exception_class", [ From c217d95ed2652e1d3180d7a4f9bea83058ef0261 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 13 Oct 2024 12:22:02 +0100 Subject: [PATCH 02/17] update history --- docs/versionhistory.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/versionhistory.rst b/docs/versionhistory.rst index 8145e61e..ed2778c9 100644 --- a/docs/versionhistory.rst +++ b/docs/versionhistory.rst @@ -20,6 +20,10 @@ This library adheres to `Semantic Versioning 2.0 `_. current Python version on several of its methods, and made the ``is_junction`` method unavailable on Python versions earlier than 3.12 (`#794 `_) +- Fixed connect_tcp producing cyclic references in tracebacks + when raising exceptions (`#809 `_) + (PR by @graingert) + **4.6.0** From 9b46a182b3bd99e0cb745a865440ab1d42299efe Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 13 Oct 2024 12:41:35 +0100 Subject: [PATCH 03/17] can't clear oserrors --- src/anyio/_core/_sockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/anyio/_core/_sockets.py b/src/anyio/_core/_sockets.py index ac134abd..adcbd7f9 100644 --- a/src/anyio/_core/_sockets.py +++ b/src/anyio/_core/_sockets.py @@ -227,7 +227,7 @@ async def try_connect(remote_host: str, event: Event) -> None: ) raise OSError("All connection attempts failed") from cause finally: - oserrors.clear() + oserrors = [] if tls or tls_hostname or ssl_context: try: From 222c6cdadf771cb469d79fe9c3791fac22a05ad0 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 13 Oct 2024 12:46:54 +0100 Subject: [PATCH 04/17] specify only v4 --- tests/test_sockets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_sockets.py b/tests/test_sockets.py index 4ecb5d8f..3ab542e5 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -329,10 +329,11 @@ async def test_happy_eyeballs_refcycles(self) -> None: """ Test derived from https://github.com/python/cpython/pull/124859 """ - port = ephemeral_port_reserve.reserve() + ip = "127.0.0.1" + port = ephemeral_port_reserve.reserve(ip=ip) exc = None try: - async with await connect_tcp("localhost", port): + async with await connect_tcp(ip, port): pass except OSError as e: exc = e.__cause__ From a2baf53cbaada5448efa9d6b62f6448def75b497 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 13 Oct 2024 13:18:39 +0100 Subject: [PATCH 05/17] fix on trio on py < 3.13 --- src/anyio/_core/_sockets.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/anyio/_core/_sockets.py b/src/anyio/_core/_sockets.py index adcbd7f9..a225e6b9 100644 --- a/src/anyio/_core/_sockets.py +++ b/src/anyio/_core/_sockets.py @@ -223,11 +223,13 @@ async def try_connect(remote_host: str, event: Event) -> None: cause = ( oserrors[0] if len(oserrors) == 1 - else ExceptionGroup("multiple connection attempts failed", oserrors) + else ExceptionGroup( + "multiple connection attempts failed", oserrors.copy() + ) ) raise OSError("All connection attempts failed") from cause finally: - oserrors = [] + oserrors.clear() if tls or tls_hostname or ssl_context: try: From 2387f990081ec8fe20e6046a60fd835e1385880c Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 09:18:32 +0000 Subject: [PATCH 06/17] Apply suggestions from code review --- src/anyio/_core/_sockets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/anyio/_core/_sockets.py b/src/anyio/_core/_sockets.py index 438b04ee..801182f1 100644 --- a/src/anyio/_core/_sockets.py +++ b/src/anyio/_core/_sockets.py @@ -237,12 +237,12 @@ async def try_connect(remote_host: str, event: Event) -> None: oserrors[0] if len(oserrors) == 1 else ExceptionGroup( - "multiple connection attempts failed", oserrors.copy() + "multiple connection attempts failed", oserrors ) ) raise OSError("All connection attempts failed") from cause finally: - oserrors.clear() + del oserrors if tls or tls_hostname or ssl_context: try: From 0203d03e78b06bd2b0700825e1abe864939da576 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 09:18:44 +0000 Subject: [PATCH 07/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/anyio/_core/_sockets.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/anyio/_core/_sockets.py b/src/anyio/_core/_sockets.py index 801182f1..cd4a6d32 100644 --- a/src/anyio/_core/_sockets.py +++ b/src/anyio/_core/_sockets.py @@ -236,9 +236,7 @@ async def try_connect(remote_host: str, event: Event) -> None: cause = ( oserrors[0] if len(oserrors) == 1 - else ExceptionGroup( - "multiple connection attempts failed", oserrors - ) + else ExceptionGroup("multiple connection attempts failed", oserrors) ) raise OSError("All connection attempts failed") from cause finally: From 6fba791cfc91d697fc698e591864f3c7cb61f54a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 09:23:36 +0000 Subject: [PATCH 08/17] fixes after merge --- src/anyio/_core/_sockets.py | 2 +- tests/test_sockets.py | 29 +++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/anyio/_core/_sockets.py b/src/anyio/_core/_sockets.py index cd4a6d32..167d785b 100644 --- a/src/anyio/_core/_sockets.py +++ b/src/anyio/_core/_sockets.py @@ -240,7 +240,7 @@ async def try_connect(remote_host: str, event: Event) -> None: ) raise OSError("All connection attempts failed") from cause finally: - del oserrors + oserrors = [] if tls or tls_hostname or ssl_context: try: diff --git a/tests/test_sockets.py b/tests/test_sockets.py index 8e48ba32..f17a1b53 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -44,6 +44,7 @@ create_unix_datagram_socket, create_unix_listener, fail_after, + get_current_task, getaddrinfo, getnameinfo, move_on_after, @@ -59,6 +60,7 @@ SocketAttribute, SocketListener, SocketStream, + TaskStatus, ) from anyio.lowlevel import checkpoint from anyio.streams.stapled import MultiListener @@ -134,14 +136,29 @@ def check_asyncio_bug(anyio_backend_name: str, family: AnyIPAddressFamily) -> No pytest.skip("Does not work due to a known bug (39148)") -if sys.version_info <= (3, 11): +if sys.version_info >= (3, 14): - def no_other_refs() -> list[object]: - return [sys._getframe(1)] -else: + async def no_other_refs() -> list[object]: + frame = sys._getframe(1) + coro = get_current_task().coro + + async def get_coro_for_frame(*, task_status: TaskStatus[object]) -> None: + my_coro = coro + while my_coro.cr_frame is not frame: + my_coro = my_coro.cr_await + task_status.started(my_coro) + + async with create_task_group() as tg: + return [await tg.start(get_coro_for_frame)] - def no_other_refs() -> list[object]: +elif sys.version_info >= (3, 11): + + async def no_other_refs() -> list[object]: return [] +else: + + async def no_other_refs() -> list[object]: + return [sys._getframe(1)] _T = TypeVar("_T") @@ -347,7 +364,7 @@ async def test_happy_eyeballs_refcycles(self) -> None: exc = e.__cause__ assert isinstance(exc, OSError) - assert gc.get_referrers(exc) == no_other_refs() + assert gc.get_referrers(exc) == await no_other_refs() @pytest.mark.parametrize( "target, exception_class", From 6dd20109e2290fb497db9b3193153f3e3c7b2380 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 09:25:09 +0000 Subject: [PATCH 09/17] add ephemeral-port-reserve again --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 45e17608..1aa749cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ test = [ and platform_system != 'Windows' \ and python_version < '3.14'\ """ + "ephemeral-port-reserve >= 1.1.4", ] doc = [ "packaging", From 877855ebc136dc1839dc14e95e47945cff571235 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 09:29:38 +0000 Subject: [PATCH 10/17] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1aa749cb..b5463bc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ test = [ uvloop >= 0.21; platform_python_implementation == 'CPython' \ and platform_system != 'Windows' \ and python_version < '3.14'\ - """ + """, "ephemeral-port-reserve >= 1.1.4", ] doc = [ From 4bb150d8d4a48aa8801e92ea4da480dc4abf9427 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 09:37:14 +0000 Subject: [PATCH 11/17] skip on 3.9 asyncio --- tests/test_sockets.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_sockets.py b/tests/test_sockets.py index f17a1b53..3ac44c51 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -350,10 +350,15 @@ def serve() -> None: "https://github.com/pypy/pypy/issues/5075" ), ) - async def test_happy_eyeballs_refcycles(self) -> None: + async def test_happy_eyeballs_refcycles(self, anyio_backend_name: str) -> None: """ Test derived from https://github.com/python/cpython/pull/124859 """ + if anyio_backend_name == "asyncio" and sys.version_info < (3, 9): + pytest.skip( + "asyncio.BaseEventLoop.create_connection creates refcycles on" + " py 3.9" + ) ip = "127.0.0.1" port = ephemeral_port_reserve.reserve(ip=ip) exc = None From 4eea30f0351924f54783ba0fcc6d6e31a58a66e0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 09:37:31 +0000 Subject: [PATCH 12/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_sockets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_sockets.py b/tests/test_sockets.py index 3ac44c51..26c0b8af 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -356,8 +356,7 @@ async def test_happy_eyeballs_refcycles(self, anyio_backend_name: str) -> None: """ if anyio_backend_name == "asyncio" and sys.version_info < (3, 9): pytest.skip( - "asyncio.BaseEventLoop.create_connection creates refcycles on" - " py 3.9" + "asyncio.BaseEventLoop.create_connection creates refcycles on" " py 3.9" ) ip = "127.0.0.1" port = ephemeral_port_reserve.reserve(ip=ip) From 7ddcd1237a68f4bb04fcb19e55eda2aa1f579ba2 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 09:38:17 +0000 Subject: [PATCH 13/17] Update tests/test_sockets.py --- tests/test_sockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sockets.py b/tests/test_sockets.py index 26c0b8af..9e044428 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -356,7 +356,7 @@ async def test_happy_eyeballs_refcycles(self, anyio_backend_name: str) -> None: """ if anyio_backend_name == "asyncio" and sys.version_info < (3, 9): pytest.skip( - "asyncio.BaseEventLoop.create_connection creates refcycles on" " py 3.9" + "asyncio.BaseEventLoop.create_connection creates refcycles on py 3.9" ) ip = "127.0.0.1" port = ephemeral_port_reserve.reserve(ip=ip) From 0542d94123bb0329dba1dde798954016e46029f2 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 09:41:11 +0000 Subject: [PATCH 14/17] news --- docs/versionhistory.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/versionhistory.rst b/docs/versionhistory.rst index 5c75f286..acc58110 100644 --- a/docs/versionhistory.rst +++ b/docs/versionhistory.rst @@ -3,6 +3,12 @@ Version history This library adheres to `Semantic Versioning 2.0 `_. +**UNRELEASED** + +- Fixed ``connect_tcp()`` producing cyclic references in tracebacks + when raising exceptions (`#809 `_) + (PR by @graingert) + **4.8.0** - Added **experimental** support for running functions in subinterpreters on Python From e53e9ebaf7663b80aa1c01eb0193f8e71ac1f357 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 09:44:03 +0000 Subject: [PATCH 15/17] Update tests/test_sockets.py --- tests/test_sockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sockets.py b/tests/test_sockets.py index 9e044428..8a3761d6 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -354,7 +354,7 @@ async def test_happy_eyeballs_refcycles(self, anyio_backend_name: str) -> None: """ Test derived from https://github.com/python/cpython/pull/124859 """ - if anyio_backend_name == "asyncio" and sys.version_info < (3, 9): + if anyio_backend_name == "asyncio" and sys.version_info < (3, 10): pytest.skip( "asyncio.BaseEventLoop.create_connection creates refcycles on py 3.9" ) From 3d626c932e07d8b77070006a97ff5c63d9d6ef1f Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 13:13:40 +0000 Subject: [PATCH 16/17] remove ephemeral-port-reserve --- pyproject.toml | 3 +-- tests/test_sockets.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b5463bc3..45e17608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,8 +55,7 @@ test = [ uvloop >= 0.21; platform_python_implementation == 'CPython' \ and platform_system != 'Windows' \ and python_version < '3.14'\ - """, - "ephemeral-port-reserve >= 1.1.4", + """ ] doc = [ "packaging", diff --git a/tests/test_sockets.py b/tests/test_sockets.py index 8a3761d6..b17ddfb7 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -19,7 +19,6 @@ from threading import Thread from typing import TYPE_CHECKING, Any, Literal, NoReturn, TypeVar, cast -import ephemeral_port_reserve import psutil import pytest from _pytest.fixtures import SubRequest @@ -359,10 +358,13 @@ async def test_happy_eyeballs_refcycles(self, anyio_backend_name: str) -> None: "asyncio.BaseEventLoop.create_connection creates refcycles on py 3.9" ) ip = "127.0.0.1" - port = ephemeral_port_reserve.reserve(ip=ip) + with socket.socket(AddressFamily.AF_INET6) as dummy_socket: + dummy_socket.bind(("::", 0)) + free_port = dummy_socket.getsockname()[1] + exc = None try: - async with await connect_tcp(ip, port): + async with await connect_tcp(ip, free_port): pass except OSError as e: exc = e.__cause__ @@ -385,10 +387,9 @@ async def test_connection_refused( exception_class: type[ExceptionGroup] | type[ConnectionRefusedError], fake_localhost_dns: None, ) -> None: - dummy_socket = socket.socket(AddressFamily.AF_INET6) - dummy_socket.bind(("::", 0)) - free_port = dummy_socket.getsockname()[1] - dummy_socket.close() + with socket.socket(AddressFamily.AF_INET6) as dummy_socket: + dummy_socket.bind(("::", 0)) + free_port = dummy_socket.getsockname()[1] with pytest.raises(OSError) as exc: await connect_tcp(target, free_port) From 05d9f63315c9444c494614b98915d9fd818dc270 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 7 Jan 2025 13:39:24 +0000 Subject: [PATCH 17/17] Update tests/test_sockets.py --- tests/test_sockets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sockets.py b/tests/test_sockets.py index b17ddfb7..f7917f6a 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -358,8 +358,8 @@ async def test_happy_eyeballs_refcycles(self, anyio_backend_name: str) -> None: "asyncio.BaseEventLoop.create_connection creates refcycles on py 3.9" ) ip = "127.0.0.1" - with socket.socket(AddressFamily.AF_INET6) as dummy_socket: - dummy_socket.bind(("::", 0)) + with socket.socket(AddressFamily.AF_INET) as dummy_socket: + dummy_socket.bind(("0.0.0.0", 0)) free_port = dummy_socket.getsockname()[1] exc = None