Fix #155 - no SSL reuse for libcurl < v7.45

HISTORY.txt

@@ -1,3 +1,6 @@
+v0.8.2 - 2022-06-29
+- Fixed #155 - prevent SSL connection reuse for libcurl < v7.45
+
 v0.8.1 - 2022-06-27
 - Fixed #154 - improved SSL connection handling with libcurl by querying active
   socket to get the sock_fd in select() instead of relying on sockopt_callback()

build.sh

@@ -16,7 +16,7 @@
 #
 # Commands
 #   build - sub-commands: deps nuitka
-#   test
+#   test - sub-commands: alpine ubuntu debian opensuse, test.py flags
 
 if [ -f "/.dockerenv" ]; then
     # Running inside container
@@ -35,6 +35,7 @@ if [ -f "/.dockerenv" ]; then
     fi
 
     # build sub-commands: nuitka or deps
+    # test sub-commands passed as flags to test.py
     SUBCOMMAND=""
     if [ ! -z "$5" ]; then
         SUBCOMMAND="$5"
@@ -47,7 +48,7 @@ if [ -f "/.dockerenv" ]; then
         python3 -m ensurepip
         if [ "$COMMAND" = "build" ]; then
             apk add ccache gcc musl-dev patchelf python3-dev upx
-        elif [ "$COMMAND" = "test" ]; then
+        else
             apk add dbus gnome-keyring
         fi
 
@@ -66,7 +67,7 @@ if [ -f "/.dockerenv" ]; then
         python3 -m ensurepip
         if [ "$COMMAND" = "build" ]; then
             yum install -y ccache libffi-devel patchelf python3-devel upx
-        elif [ "$COMMAND" = "test" ]; then
+        else
             yum install -y gnome-keyring
         fi
 
@@ -107,7 +108,7 @@ if [ -f "/.dockerenv" ]; then
     export PXBIN="/px/px.dist-linux-$ABI-x86_64/px.dist/px"
     export WHEELS="/px/px.dist-wheels-linux-$ABI-x86_64/px.dist-wheels"
 
-    if [ "$COMMAND" = "test" ]; then
+    if [ "$COMMAND" != "build" ]; then
         dbus-run-session -- $SHELL -c 'echo "abc" | gnome-keyring-daemon --unlock'
     fi
 
@@ -147,7 +148,7 @@ if [ -f "/.dockerenv" ]; then
         python3 -m pip install px-proxy --no-index -f $WHEELS
 
         if [ "$COMMAND" = "test" ]; then
-            python3 test.py --binary --pip --proxy=$PROXY --pac=$PAC --username=$USERNAME $AUTH
+            python3 test.py --binary --pip --proxy=$PROXY --pac=$PAC --username=$USERNAME $AUTH $SUBCOMMAND
         else
             $SHELL
         fi
@@ -181,9 +182,24 @@ else
             $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" build $1
         done
     elif [ "$1" = "test" ]; then
-        for image in alpine alpine:3.11 voidlinux/voidlinux-musl ubuntu ubuntu:bionic debian debian:oldstable linuxmintd/mint20.3-amd64 opensuse/tumbleweed opensuse/leap:15.1
+        SUBCOMMAND="$3"
+        if [ "$2" = "alpine" ]; then
+            IMAGES="alpine alpine:3.11"
+        elif [ "$2" = "ubuntu" ]; then
+            IMAGES="ubuntu ubuntu:focal"
+        elif [ "$2" = "debian" ]; then
+            IMAGES="debian debian:oldstable"
+        elif [ "$2" = "mint" ]; then
+            IMAGES="linuxmintd/mint20.3-amd64"
+        elif [ "$2" = "opensuse" ]; then
+            IMAGES="opensuse/tumbleweed opensuse/leap:15.1"
+        else
+            SUBCOMMAND="$2"
+            IMAGES="alpine ubuntu debian opensuse/tumbleweed"
+        fi
+        for image in $IMAGES
         do
-            $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" test
+            $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" test "$SUBCOMMAND"
         done
     else
         if [ "$1" = "musl" ]; then

px/mcurl.py

@@ -253,6 +253,12 @@ def _setup(self, url, method, request_version, connect_timeout):
         if method == "CONNECT":
             libcurl.easy_setopt(self.easy, libcurl.CURLOPT_CONNECT_ONLY, True)
 
+            if curl_version() < 0x072D00:
+                # libcurl < v7.45 does not support CURLINFO_ACTIVESOCKET so it is not possible
+                # to reuse existing connections
+                libcurl.easy_setopt(self.easy, libcurl.CURLOPT_FRESH_CONNECT, True)
+                dprint(self.easyhash + ": Fresh connection requested")
+
             # We want libcurl to make a simple HTTP connection to auth
             # with the upstream proxy and let client establish SSL
             if "://" not in url:
@@ -485,10 +491,11 @@ def set_follow(self, enable = True):
 
     def perform(self):
         "Perform the easy handle"
-        if MCURL.do(self) == False:
+        ret = MCURL.do(self)
+        MCURL.remove(self)
+        if ret is False:
             dprint(self.easyhash + ": Connection failed: " + self.errstr)
-            return False
-        return True
+        return ret
 
 @libcurl.socket_callback
 def _socket_callback(easy, sock_fd, ev_bitmask, userp, socketp):
@@ -666,8 +673,7 @@ def _remove_handle(self, curl: Curl, errstr = ""):
             curl.errstr += errstr + "; "
 
         dprint(curl.easyhash + ": Remove handle: " + curl.errstr)
-        if len(curl.errstr) == 0:
-            libcurl.multi_remove_handle(self._multi, curl.easy)
+        libcurl.multi_remove_handle(self._multi, curl.easy)
 
         self.handles.pop(curl.easyhash)
 
@@ -683,8 +689,8 @@ def stop(self, curl: Curl):
 
     # Executing multi
 
-    def perform(self):
-        "Perform all tasks in the multi instance"
+    def _perform(self):
+        # Perform all tasks in the multi instance
         with self._lock:
             rlen = len(self.rlist)
             wlen = len(self.wlist)
@@ -716,7 +722,7 @@ def do(self, curl: Curl):
         while True:
             if curl.done:
                 break
-            self.perform()
+            self._perform()
             time.sleep(0.01)
 
         if "timed out" in curl.errstr:
@@ -739,26 +745,36 @@ def do(self, curl: Curl):
                 with self._lock:
                     self.failed.append(curl.proxy)
 
+        if curl.is_connect() and curl.sock_fd is None:
+            # Need sock_fd for select()
+            if curl_version() < 0x072D00:
+                # This should never happen since we have set CURLOPT_FRESH_CONNECT = True
+                # for CONNECT
+                out = "Cannot reuse an SSL connection with libcurl < v7.45 - should never happen"
+                dprint(curl.easyhash + ": " + out)
+                curl.errstr += out + "; "
+                curl.resp = 500
+            else:
+                # Get the active socket using getinfo() for select()
+                dprint(curl.easyhash + ": Getting active socket")
+                ret, sock_fd = curl.get_activesocket()
+                if ret == libcurl.CURLE_OK:
+                    curl.sock_fd = sock_fd
+                else:
+                    out = "Failed to get active socket: %d, %d" % (ret, sock_fd)
+                    dprint(curl.easyhash + ": " + out)
+                    curl.errstr += out + "; "
+                    curl.resp = 503
+
         return len(curl.errstr) == 0
 
     def select(self, curl: Curl, client_sock, idle = 30):
         "Run select loop between client and curl"
         # TODO figure out if IPv6 or IPv4
         if curl.sock_fd is None:
-            if curl_version() < 0x072D00:
-                # Reusing an SSL connection but no way to get active socket since
-                # CURLINFO_ACTIVESOCKET was only added in libcurl v7.45
-                dprint(curl.easyhash + ": unable to reuse SSL connection with libcurl < v7.45")
-                return
-
-            # Need to get the active socket using getinfo()
-            dprint(curl.easyhash + ": Getting active socket")
-            ret, sock_fd = curl.get_activesocket()
-            if ret == libcurl.CURLE_OK:
-                curl.sock_fd = sock_fd
-            else:
-                dprint(curl.easyhash + ": Failed to get active socket: %d, %d" % (ret, sock_fd))
-                return
+            dprint(curl.easyhash + ": Cannot select() without active socket")
+            return
+
 
         dprint(curl.easyhash + ": Starting select loop")
         curl_sock = socket.fromfd(curl.sock_fd, socket.AF_INET, socket.SOCK_STREAM)

px/version.py

@@ -1,3 +1,3 @@
 "Px version"
 
-__version__ = "0.8.1"
+__version__ = "0.8.2"

test.py

@@ -41,7 +41,7 @@
 except AttributeError:
     DEVNULL = open(os.devnull, 'wb')
 
-def exec(cmd, port = 0, shell = True):
+def exec(cmd, port = 0, shell = True, delete = False):
     global COUNT
     log = "%d-%d.txt" % (port, COUNT)
     COUNT += 1
@@ -50,6 +50,10 @@ def exec(cmd, port = 0, shell = True):
 
     with open(log, "r") as l:
         data = l.read()
+
+    if delete:
+        os.remove(log)
+
     return p.returncode, data
 
 def curlcli(url, port, method = "GET", data = "", proxy = ""):
@@ -442,8 +446,13 @@ def socketTestSetup():
             TESTS.append((cmd, testproc, ip))
 
 def auto():
-    osname = tools.get_os()
+    prefix = "px.dist"
+    osname, _, dist = tools.get_dirs(prefix)
     if "--norun" not in sys.argv:
+        if sys.platform == "linux":
+            _, distro = exec("cat /etc/os-release | grep ^ID | head -n 1 | cut -d\"=\" -f2 | sed 's/\"//g'", delete = True)
+            _, version = exec("cat /etc/os-release | grep ^VERSION_ID | head -n 1 | cut -d\"=\" -f2 | sed 's/\"//g'", delete = True)
+            osname += "-%s-%s" % (distro.strip(), version.strip())
         testdir = "test-%s-%d-%s" % (osname, PORT, platform.machine().lower())
 
         # Make temp directory
@@ -487,9 +496,7 @@ def auto():
 
     if "--binary" in sys.argv:
         # Nuitka binary test
-        outdir = "px.dist-%s-%s" % (osname, platform.machine().lower())
-
-        cmd = os.path.abspath(os.path.join("..", outdir, "px.dist", "px")) + " "
+        cmd = os.path.abspath(os.path.join(dist, "px")) + " "
         cmds.append(cmd)
         results.extend([pool.apply_async(runTest, args = (TESTS[count], cmd, count + offset, PORT)) for count in range(len(TESTS))])
         offset += len(TESTS)