Release 3.10 (#159)

3.10.0 (2024-10-21)
------------------

**Added**
- Automatic Advanced Keep-Alive for HTTP/2 and HTTP/3 over QUIC by
sending PING frames.
New Session, and Adapter parameters are now available:
`keepalive_delay`, and `keepalive_idle_window`.
This greatly improves your daily experience working with HTTP/2+ remote
peers.

**Fixed**
- Unshielded picotls assertion error in Python < 3.10 when trying to
fetch the peer intermediate certificate. (#157)

Author: Ousret

.pre-commit-config.yaml

@@ -28,4 +28,4 @@ repos:
   -   id: mypy
       args: [--check-untyped-defs]
       exclude: 'tests/|noxfile.py'
-      additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.10.903', 'wassima>=1.0.1', 'idna', 'kiss_headers']
+      additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.11.900', 'wassima>=1.0.1', 'idna', 'kiss_headers']

HISTORY.md

@@ -1,6 +1,17 @@
 Release History
 ===============
 
+3.10.0 (2024-10-21)
+------------------
+
+**Added**
+- Automatic Advanced Keep-Alive for HTTP/2 and HTTP/3 over QUIC by sending PING frames.
+  New Session, and Adapter parameters are now available: `keepalive_delay`, and `keepalive_idle_window`.
+  This greatly improves your daily experience working with HTTP/2+ remote peers.
+
+**Fixed**
+- Unshielded picotls assertion error in Python < 3.10 when trying to fetch the peer intermediate certificate.
+
 3.9.1 (2024-10-13)
 ------------------
 

README.md

@@ -45,6 +45,7 @@ Niquests, is the “**Safest**, **Fastest[^10]**, **Easiest**, and **Most advanc
 | `Early Responses`                   |       ✅        |     ❌     |       ❌       | ❌             |
 | `WebSocket over HTTP/1`             |       ✅        |  ❌[^14]   |    ❌[^14]     | ✅             |
 | `WebSocket over HTTP/2 and HTTP/3`  |     ✅[^13]     |     ❌     |       ❌       | ❌             |
+| `Automatic Ping for HTTP/2+`        |       ✅        |    N/A    |       ❌       | N/A           |
 | `Automatic Connection Upgrade / Downgrade`  |     ✅     |     N/A     |       ❌       | N/A             |
 </details>
 

docs/user/quickstart.rst

@@ -1249,6 +1249,32 @@ See::
 
 .. note:: The given example are really basic ones. You may adjust at will the settings and algorithm to match your requisites.
 
+Keep-Alive
+----------
+
+.. note:: Available since Niquests v3.10 and before this only HTTP/1.1 were kept alive properly.
+
+Niquests can automatically make sure that your HTTP connection is kept alive
+no matter the used protocol using a discrete scheduled task for each host.
+
+.. code-block:: python
+
+    import niquests
+
+    sess = niquests.Session(keepalive_delay=300, keepalive_idle_window=60)  # already the defaults!, you don't need to specify anything
+
+In that example, we indicate that we wish to keep a connection alive for 5 minutes and
+eventually send ping every 60s after the connection was idle. (Those values are the default ones!)
+
+The pings are only sent when using HTTP/2 or HTTP/3 over QUIC. Any connection activity is considered as used, therefor
+making the ping only 60s after zero activity. If the connection receive unsolicited data, it is also considered used.
+
+.. note:: Setting either keepalive_delay or keepalive_idle_window to None disable this feature.
+
+.. warning:: We do not recommend setting anything lower than 30s for keepalive_idle_window. Anything lower than 1s is considered to be 1s. High frequency ping will lower the performance of your connection pool. And probably end up by getting kicked out by the server.
+
+Once the ``keepalive_delay`` passed, we do not close the connection, we simply cease to ensure it is alive. This is purely for backward compatibility with our predecessor, as some host may retain the connection for hours.
+
 -----------------------
 
 Ready for more? Check out the :ref:`advanced <advanced>` section.

pyproject.toml

@@ -42,7 +42,7 @@ dynamic = ["version"]
 dependencies = [
     "charset_normalizer>=2,<4",
     "idna>=2.5,<4",
-    "urllib3.future>=2.10.904,<3",
+    "urllib3.future>=2.11.900,<3",
     "wassima>=1.0.1,<2",
     "kiss_headers>=2,<4",
 ]

src/niquests/__version__.py

@@ -9,9 +9,9 @@
 __url__: str = "https://niquests.readthedocs.io"
 
 __version__: str
-__version__ = "3.9.1"
+__version__ = "3.10.0"
 
-__build__: int = 0x030901
+__build__: int = 0x031000
 __author__: str = "Kenneth Reitz"
 __author_email__: str = "me@kennethreitz.org"
 __license__: str = "Apache-2.0"

src/niquests/_async.py

@@ -130,6 +130,8 @@ def __init__(
         pool_connections: int = DEFAULT_POOLSIZE,
         pool_maxsize: int = DEFAULT_POOLSIZE,
         happy_eyeballs: bool | int = False,
+        keepalive_delay: float | int | None = 300.0,
+        keepalive_idle_window: float | int | None = 60.0,
     ):
         if [disable_ipv4, disable_ipv6].count(True) == 2:
             raise RuntimeError("Cannot disable both IPv4 and IPv6")
@@ -195,6 +197,9 @@ def __init__(
 
         self._happy_eyeballs = happy_eyeballs
 
+        self._keepalive_delay = keepalive_delay
+        self._keepalive_idle_window = keepalive_idle_window
+
         #: SSL Verification default.
         #: Defaults to `True`, requiring requests to verify the TLS certificate at the
         #: remote end.
@@ -253,6 +258,8 @@ def __init__(
                 pool_connections=pool_connections,
                 pool_maxsize=pool_maxsize,
                 happy_eyeballs=happy_eyeballs,
+                keepalive_delay=keepalive_delay,
+                keepalive_idle_window=keepalive_idle_window,
             ),
         )
         self.mount(
@@ -269,6 +276,8 @@ def __init__(
                 pool_connections=pool_connections,
                 pool_maxsize=pool_maxsize,
                 happy_eyeballs=happy_eyeballs,
+                keepalive_delay=keepalive_delay,
+                keepalive_idle_window=keepalive_idle_window,
             ),
         )
 
@@ -436,6 +445,8 @@ async def on_early_response(early_response: Response) -> None:
                     pool_connections=self._pool_connections,
                     pool_maxsize=self._pool_maxsize,
                     happy_eyeballs=self._happy_eyeballs,
+                    keepalive_delay=self._keepalive_delay,
+                    keepalive_idle_window=self._keepalive_idle_window,
                 ),
             )
             self.mount(
@@ -452,6 +463,8 @@ async def on_early_response(early_response: Response) -> None:
                     pool_connections=self._pool_connections,
                     pool_maxsize=self._pool_maxsize,
                     happy_eyeballs=self._happy_eyeballs,
+                    keepalive_delay=self._keepalive_delay,
+                    keepalive_idle_window=self._keepalive_idle_window,
                 ),
             )
 

src/niquests/adapters.py

@@ -333,6 +333,8 @@ class HTTPAdapter(BaseAdapter):
         "_disable_ipv4",
         "_disable_ipv6",
         "_happy_eyeballs",
+        "_keepalive_delay",
+        "_keepalive_idle_window",
     ]
 
     def __init__(
@@ -341,7 +343,7 @@ def __init__(
         pool_maxsize: int = DEFAULT_POOLSIZE,
         max_retries: RetryType = DEFAULT_RETRIES,
         pool_block: bool = DEFAULT_POOLBLOCK,
-        *,  # todo: revert if any complaint about it... :s
+        *,
         quic_cache_layer: CacheLayerAltSvcType | None = None,
         disable_http1: bool = False,
         disable_http2: bool = False,
@@ -352,6 +354,8 @@ def __init__(
         disable_ipv4: bool = False,
         disable_ipv6: bool = False,
         happy_eyeballs: bool | int = False,
+        keepalive_delay: float | int | None = 300.0,
+        keepalive_idle_window: float | int | None = 60.0,
     ):
         if isinstance(max_retries, bool):
             self.max_retries: RetryType = False
@@ -383,6 +387,8 @@ def __init__(
         self._disable_ipv4 = disable_ipv4
         self._disable_ipv6 = disable_ipv6
         self._happy_eyeballs = happy_eyeballs
+        self._keepalive_delay = keepalive_delay
+        self._keepalive_idle_window = keepalive_idle_window
 
         #: we keep a list of pending (lazy) response
         self._promises: dict[str, Response] = {}
@@ -413,6 +419,8 @@ def __init__(
             source_address=source_address,
             socket_family=resolve_socket_family(disable_ipv4, disable_ipv6),
             happy_eyeballs=happy_eyeballs,
+            keepalive_delay=keepalive_delay,
+            keepalive_idle_window=keepalive_idle_window,
         )
 
     def __getstate__(self) -> dict[str, typing.Any | None]:
@@ -447,6 +455,8 @@ def __setstate__(self, state):
             source_address=self._source_address,
             socket_family=resolve_socket_family(self._disable_ipv4, self._disable_ipv6),
             happy_eyeballs=self._happy_eyeballs,
+            keepalive_delay=self._keepalive_delay,
+            keepalive_idle_window=self._keepalive_idle_window,
         )
 
     def init_poolmanager(
@@ -1335,6 +1345,8 @@ class AsyncHTTPAdapter(AsyncBaseAdapter):
         "_disable_ipv4",
         "_disable_ipv6",
         "_happy_eyeballs",
+        "_keepalive_delay",
+        "_keepalive_idle_window",
     ]
 
     def __init__(
@@ -1354,6 +1366,8 @@ def __init__(
         disable_ipv4: bool = False,
         disable_ipv6: bool = False,
         happy_eyeballs: bool | int = False,
+        keepalive_delay: float | int | None = 300.0,
+        keepalive_idle_window: float | int | None = 60.0,
     ):
         if isinstance(max_retries, bool):
             self.max_retries: RetryType = False
@@ -1386,6 +1400,8 @@ def __init__(
         self._disable_ipv4 = disable_ipv4
         self._disable_ipv6 = disable_ipv6
         self._happy_eyeballs = happy_eyeballs
+        self._keepalive_delay = keepalive_delay
+        self._keepalive_idle_window = keepalive_idle_window
 
         #: we keep a list of pending (lazy) response
         self._promises: dict[str, Response | AsyncResponse] = {}
@@ -1415,6 +1431,8 @@ def __init__(
             source_address=source_address,
             socket_family=resolve_socket_family(disable_ipv4, disable_ipv6),
             happy_eyeballs=happy_eyeballs,
+            keepalive_delay=keepalive_delay,
+            keepalive_idle_window=keepalive_idle_window,
         )
 
     def __getstate__(self) -> dict[str, typing.Any | None]:
@@ -1449,6 +1467,8 @@ def __setstate__(self, state):
             source_address=self._source_address,
             socket_family=resolve_socket_family(self._disable_ipv4, self._disable_ipv6),
             happy_eyeballs=self._happy_eyeballs,
+            keepalive_delay=self._keepalive_delay,
+            keepalive_idle_window=self._keepalive_idle_window,
         )
 
     def init_poolmanager(

src/niquests/extensions/_async_ocsp.py

@@ -51,6 +51,7 @@
     async_recv_tls,
     async_recv_tls_and_decrypt,
     async_send_tls,
+    PicoTLSException,
 )
 from ._ocsp import (
     _str_fingerprint_of,
@@ -73,7 +74,11 @@ async def _ask_nicely_for_issuer(
         sock = AsyncSocket(socket.AF_INET6, socket.SOCK_STREAM)
 
     sock.settimeout(timeout)
-    await sock.connect(dst_address)
+
+    try:
+        await sock.connect(dst_address)
+    except (OSError, socket.timeout, TimeoutError, ConnectionError) as e:
+        raise PicoTLSException from e
 
     SECP256R1_P = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF
     SECP256R1_A = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFC
@@ -408,11 +413,14 @@ async def verify(
                             raise ValueError
 
                         if not proxies:
-                            issuer_certificate = await _ask_nicely_for_issuer(
-                                url_parsed.hostname,
-                                conn_info.destination_address,
-                                timeout,
-                            )
+                            try:
+                                issuer_certificate = await _ask_nicely_for_issuer(
+                                    url_parsed.hostname,
+                                    conn_info.destination_address,
+                                    timeout,
+                                )
+                            except PicoTLSException:
+                                issuer_certificate = None
                         else:
                             issuer_certificate = None
 

src/niquests/extensions/_ocsp.py

@@ -49,6 +49,7 @@
     recv_tls,
     recv_tls_and_decrypt,
     send_tls,
+    PicoTLSException,
 )
 
 
@@ -80,8 +81,11 @@ def _ask_nicely_for_issuer(
     else:
         sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
 
-    sock.connect(dst_address)
     sock.settimeout(timeout)
+    try:
+        sock.connect(dst_address)
+    except (OSError, socket.timeout, TimeoutError, ConnectionError) as e:
+        raise PicoTLSException from e
 
     SECP256R1_P = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF
     SECP256R1_A = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFC
@@ -402,11 +406,14 @@ def verify(
                         raise ValueError
 
                     if not proxies:
-                        issuer_certificate = _ask_nicely_for_issuer(
-                            url_parsed.hostname,
-                            conn_info.destination_address,
-                            timeout,
-                        )
+                        try:
+                            issuer_certificate = _ask_nicely_for_issuer(
+                                url_parsed.hostname,
+                                conn_info.destination_address,
+                                timeout,
+                            )
+                        except PicoTLSException:
+                            issuer_certificate = None
                     else:
                         issuer_certificate = None