Bump version to 0.7.0b5, adding stream_exclusive and more tls options

Makefile

@@ -2,13 +2,13 @@
 SHELL := bash
 
 # this is the upstream libcurl-impersonate version
-VERSION := 0.7.0b4
-CURL_VERSION := curl-8.5.0
+VERSION := 0.7.0b5
+CURL_VERSION := curl-8_7_1
 
 $(CURL_VERSION):
-	curl -L "https://curl.se/download/$(CURL_VERSION).tar.xz" \
-		-o "$(CURL_VERSION).tar.xz"
-	tar -xf $(CURL_VERSION).tar.xz
+	curl -L https://github.com/curl/curl/archive/$(CURL_VERSION).zip -o curl.zip
+	unzip -q -o curl.zip
+	mv curl-$(CURL_VERSION) $(CURL_VERSION)
 
 curl-impersonate-$(VERSION)/chrome/patches: $(CURL_VERSION)
 	curl -L "https://github.com/yifeikong/curl-impersonate/archive/refs/tags/v$(VERSION).tar.gz" \
@@ -25,6 +25,17 @@ curl-impersonate-$(VERSION)/chrome/patches: $(CURL_VERSION)
 	# Sentinel files: https://tech.davis-hansson.com/p/make/
 	touch .preprocessed
 
+local-curl: $(CURL_VERSION)
+	cp /usr/local/lib/libcurl-impersonate-chrome* /Users/runner/work/_temp/install/lib/
+	cd $(CURL_VERSION)
+	for p in ../curl-impersonate/chrome/patches/curl-*.patch; do patch -p1 < ../$$p; done
+	# Re-generate the configure script
+	autoreconf -fi
+	mkdir -p ../include/curl
+	cp -R include/curl/* ../include/curl/
+	# Sentinel files: https://tech.davis-hansson.com/p/make/
+	touch .preprocessed
+
 gen-const: .preprocessed
 	python scripts/generate_consts.py $(CURL_VERSION)
 

curl_cffi/aio.py

@@ -145,6 +145,7 @@ def _setup(self):
         self._self_handle = ffi.new_handle(self)
         self.setopt(CurlMOpt.SOCKETDATA, self._self_handle)
         self.setopt(CurlMOpt.TIMERDATA, self._self_handle)
+        # self.setopt(CurlMOpt.PIPELINING, 0)
 
     async def close(self):
         """Close and cleanup running timers, readers, writers and handles."""
@@ -184,6 +185,7 @@ async def _force_timeout(self):
     def add_handle(self, curl: Curl):
         """Add a curl handle to be managed by curl_multi. This is the equivalent of
         `perform` in the async world."""
+
         # import pdb; pdb.set_trace()
         curl._ensure_cacert()
         lib.curl_multi_add_handle(self._curlm, curl._curl)

curl_cffi/const.py

@@ -278,7 +278,7 @@ class CurlOpt(IntEnum):
     ALTSVC = 10000 + 287
     MAXAGE_CONN = 0 + 288
     SASL_AUTHZID = 10000 + 289
-    MAIL_RCPT_ALLLOWFAILS = 0 + 290
+    MAIL_RCPT_ALLOWFAILS = 0 + 290
     SSLCERT_BLOB = 40000 + 291
     SSLKEY_BLOB = 40000 + 292
     PROXY_SSLCERT_BLOB = 40000 + 293
@@ -311,16 +311,23 @@ class CurlOpt(IntEnum):
     WS_OPTIONS = 0 + 320
     CA_CACHE_TIMEOUT = 0 + 321
     QUICK_EXIT = 0 + 322
-    HTTPBASEHEADER = 10000 + 323
-    SSL_SIG_HASH_ALGS = 10000 + 324
-    SSL_ENABLE_ALPS = 0 + 325
-    SSL_CERT_COMPRESSION = 10000 + 326
-    SSL_ENABLE_TICKET = 0 + 327
-    HTTP2_PSEUDO_HEADERS_ORDER = 10000 + 328
-    HTTP2_SETTINGS = 10000 + 329
-    SSL_PERMUTE_EXTENSIONS = 0 + 330
-    HTTP2_WINDOW_UPDATE = 0 + 331
-    ECH = 10000 + 332
+    HAPROXY_CLIENT_IP = 10000 + 323
+    SERVER_RESPONSE_TIMEOUT_MS = 0 + 324
+    HTTPBASEHEADER = 10000 + 1000
+    SSL_SIG_HASH_ALGS = 10000 + 1001
+    SSL_ENABLE_ALPS = 0 + 1002
+    SSL_CERT_COMPRESSION = 10000 + 1003
+    SSL_ENABLE_TICKET = 0 + 1004
+    HTTP2_PSEUDO_HEADERS_ORDER = 10000 + 1005
+    HTTP2_SETTINGS = 10000 + 1006
+    SSL_PERMUTE_EXTENSIONS = 0 + 1007
+    HTTP2_WINDOW_UPDATE = 0 + 1008
+    ECH = 10000 + 1009
+    HTTP2_STREAMS = 10000 + 1010
+    TLS_GREASE = 0 + 1011
+    TLS_EXTENSION_ORDER = 10000 + 1012
+    STREAM_EXCLUSIVE = 0 + 1013
+    TLS_KEY_USAGE_NO_CHECK = 0 + 1014
 
     if locals().get("WRITEDATA"):
         FILE = locals().get("WRITEDATA")
@@ -395,7 +402,11 @@ class CurlInfo(IntEnum):
     REFERER = 0x100000 + 60
     CAINFO = 0x100000 + 61
     CAPATH = 0x100000 + 62
-    LASTONE = 62
+    XFER_ID = 0x600000 + 63
+    CONN_ID = 0x600000 + 64
+    QUEUE_TIME_T = 0x600000 + 65
+    USED_PROXY = 0x200000 + 66
+    LASTONE = 66
 
     if locals().get("RESPONSE_CODE"):
         HTTP_CODE = locals().get("RESPONSE_CODE")
@@ -527,11 +538,12 @@ class CurlECode(IntEnum):
     PROXY = 97
     SSL_CLIENTCERT = 98
     UNRECOVERABLE_POLL = 99
-    ECH_REQUIRED = 100
+    TOO_LARGE = 100
+    ECH_REQUIRED = 101
 
 
 class CurlHttpVersion(IntEnum):
-    """``CURL_HTTP_VERSION`` constants extracted from libcurl, see comments for details"""
+    """``CURL_HTTP_VERSION`` constants extracted from libcurl, see comments for details."""
 
     NONE = 0
     V1_0 = 1  # please use HTTP 1.0 in the request */
@@ -543,7 +555,7 @@ class CurlHttpVersion(IntEnum):
 
 
 class CurlWsFlag(IntEnum):
-    """``CURL_WS_FLAG`` constancs extracted from libcurl, see comments for details"""
+    """``CURL_WS_FLAG`` constants extracted from libcurl, see comments for details."""
 
     TEXT = 1 << 0
     BINARY = 1 << 1

curl_cffi/requests/session.py

@@ -984,6 +984,8 @@ async def pop_curl(self):
         curl = await self.pool.get()
         if curl is None:
             curl = Curl(debug=self.debug)
+        # curl.setopt(CurlOpt.FRESH_CONNECT, 1)
+        # curl.setopt(CurlOpt.FORBID_REUSE, 1)
         return curl
 
     def push_curl(self, curl):
@@ -1014,6 +1016,7 @@ def release_curl(self, curl):
         if not self._closed:
             self.acurl.remove_handle(curl)
             curl.reset()
+            # curl.setopt(CurlOpt.PIPEWAIT, 1)
             self.push_curl(curl)
         else:
             curl.close()
@@ -1146,9 +1149,9 @@ def cleanup(fut):
         else:
             try:
                 # curl.debug()
+                # print("using curl instance: ", curl)
                 task = self.acurl.add_handle(curl)
                 await task
-                # print(curl.getinfo(CurlInfo.CAINFO))
             except CurlError as e:
                 rsp = self._parse_response(curl, buffer, header_buffer, default_encoding)
                 rsp.request = req

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "curl_cffi"
-version = "0.7.0b4"
+version = "0.7.0b5"
 authors = [{ name = "Yifei Kong", email = "kong@yifei.me" }]
 description = "libcurl ffi bindings for Python, with impersonation support."
 license = { file = "LICENSE" }

scripts/build.py

@@ -10,7 +10,7 @@
 from cffi import FFI
 
 # this is the upstream libcurl-impersonate version
-__version__ = "0.7.0b4"
+__version__ = "0.7.0b5"
 
 tmpdir = None
 

scripts/download_curl.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+CURL_VERSION=curl-8_7_1
+
+curl -L https://github.com/curl/curl/archive/${CURL_VERSION}.zip -o curl.zip
+unzip -q -o curl.zip
+mv curl-${CURL_VERSION} ${CURL_VERSION}
+
+cd ${CURL_VERSION}
+
+patchfile=../../curl-impersonate/chrome/patches/curl-impersonate.patch
+patch -p1 < $patchfile

scripts/generate_consts.py

@@ -13,6 +13,8 @@
     f.write("# This file is automatically generated, do not modify it directly.\n\n")
     f.write("from enum import IntEnum\n\n\n")
     f.write("class CurlOpt(IntEnum):\n")
+    f.write('    """``CULROPT_`` constancs extracted from libcurl,\n')
+    f.write('    see: https://curl.se/libcurl/c/curl_easy_setopt.html"""\n\n')
     cmd = rf"""
         echo '#include "{CURL_VERSION}/include/curl/curl.h"' | gcc -E - | grep -i "CURLOPT_.\+ =" | sed "s/  CURLOPT_/    /g" | sed "s/,//g"
     """  # noqa E501
@@ -30,6 +32,8 @@
     )
 
     f.write("class CurlInfo(IntEnum):\n")
+    f.write('    """``CURLINFO_`` constancs extracted from libcurl,\n')
+    f.write('    see: https://curl.se/libcurl/c/curl_easy_getinfo.html"""\n\n')
     cmd = rf"""
         echo '#include "{CURL_VERSION}/include/curl/curl.h"' | gcc -E - | grep -i "CURLINFO_.\+ =" | sed "s/  CURLINFO_/    /g" | sed "s/,//g"
     """  # noqa E501
@@ -43,6 +47,8 @@
     )
 
     f.write("class CurlMOpt(IntEnum):\n")
+    f.write('    """``CURLMOPT_`` constancs extracted from libcurl,\n')
+    f.write('    see: https://curl.se/libcurl/c/curl_multi_setopt.html"""\n\n')
     cmd = rf"""
         echo '#include "{CURL_VERSION}/include/curl/curl.h"' | gcc -E - | grep -i "CURLMOPT_.\+ =" | sed "s/  CURLMOPT_/    /g" | sed "s/,//g"
     """  # noqa E501
@@ -51,6 +57,8 @@
     f.write("\n\n")
 
     f.write("class CurlECode(IntEnum):\n")
+    f.write('    """``CURLECODE_`` constancs extracted from libcurl,\n')
+    f.write('    see: https://curl.se/libcurl/c/libcurl-errors.html"""\n\n')
     cmd = rf"""
         echo '#include "{CURL_VERSION}/include/curl/curl.h"' | gcc -E - | grep -i CURLE_ | sed "s/[, ][=0]*//g" | sed "s/CURLE_/    /g" | awk '{{print $0 " = " NR-1}}'
     """  # noqa E501
@@ -59,6 +67,7 @@
     f.write("\n\n")
 
     f.write("class CurlHttpVersion(IntEnum):\n")
+    f.write('    """``CURL_HTTP_VERSION`` constants extracted from libcurl, see comments for details."""\n\n')
     f.write("    NONE = 0\n")
     f.write("    V1_0 = 1  # please use HTTP 1.0 in the request */\n")
     f.write("    V1_1 = 2  # please use HTTP 1.1 in the request */\n")
@@ -69,9 +78,10 @@
     f.write("\n\n")
 
     f.write("class CurlWsFlag(IntEnum):\n")
-    f.write("    TEXT       = (1<<0)\n")
-    f.write("    BINARY     = (1<<1)\n")
-    f.write("    CONT       = (1<<2)\n")
-    f.write("    CLOSE      = (1<<3)\n")
-    f.write("    PING       = (1<<4)\n")
-    f.write("    OFFSET     = (1<<5)\n")
+    f.write('    """``CURL_WS_FLAG`` constants extracted from libcurl, see comments for details."""\n\n')
+    f.write("    TEXT = 1 << 0\n")
+    f.write("    BINARY = 1 << 1\n")
+    f.write("    CONT = 1 << 2\n")
+    f.write("    CLOSE = 1 << 3\n")
+    f.write("    PING = 1 << 4\n")
+    f.write("    OFFSET = 1 << 5\n")