diff --git a/HISTORY.txt b/HISTORY.txt index d2b64e8..88dab1a 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -1,3 +1,6 @@ +v0.8.2 - 2022-06-29 +- Fixed #155 - prevent SSL connection reuse for libcurl < v7.45 + v0.8.1 - 2022-06-27 - Fixed #154 - improved SSL connection handling with libcurl by querying active socket to get the sock_fd in select() instead of relying on sockopt_callback() diff --git a/build.sh b/build.sh index 2a5a897..691ef84 100644 --- a/build.sh +++ b/build.sh @@ -16,7 +16,7 @@ # # Commands # build - sub-commands: deps nuitka -# test +# test - sub-commands: alpine ubuntu debian opensuse, test.py flags if [ -f "/.dockerenv" ]; then # Running inside container @@ -35,6 +35,7 @@ if [ -f "/.dockerenv" ]; then fi # build sub-commands: nuitka or deps + # test sub-commands passed as flags to test.py SUBCOMMAND="" if [ ! -z "$5" ]; then SUBCOMMAND="$5" @@ -47,7 +48,7 @@ if [ -f "/.dockerenv" ]; then python3 -m ensurepip if [ "$COMMAND" = "build" ]; then apk add ccache gcc musl-dev patchelf python3-dev upx - elif [ "$COMMAND" = "test" ]; then + else apk add dbus gnome-keyring fi @@ -66,7 +67,7 @@ if [ -f "/.dockerenv" ]; then python3 -m ensurepip if [ "$COMMAND" = "build" ]; then yum install -y ccache libffi-devel patchelf python3-devel upx - elif [ "$COMMAND" = "test" ]; then + else yum install -y gnome-keyring fi @@ -107,7 +108,7 @@ if [ -f "/.dockerenv" ]; then export PXBIN="/px/px.dist-linux-$ABI-x86_64/px.dist/px" export WHEELS="/px/px.dist-wheels-linux-$ABI-x86_64/px.dist-wheels" - if [ "$COMMAND" = "test" ]; then + if [ "$COMMAND" != "build" ]; then dbus-run-session -- $SHELL -c 'echo "abc" | gnome-keyring-daemon --unlock' fi @@ -147,7 +148,7 @@ if [ -f "/.dockerenv" ]; then python3 -m pip install px-proxy --no-index -f $WHEELS if [ "$COMMAND" = "test" ]; then - python3 test.py --binary --pip --proxy=$PROXY --pac=$PAC --username=$USERNAME $AUTH + python3 test.py --binary --pip --proxy=$PROXY --pac=$PAC --username=$USERNAME $AUTH $SUBCOMMAND else $SHELL fi @@ -181,9 +182,24 @@ else $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" build $1 done elif [ "$1" = "test" ]; then - for image in alpine alpine:3.11 voidlinux/voidlinux-musl ubuntu ubuntu:bionic debian debian:oldstable linuxmintd/mint20.3-amd64 opensuse/tumbleweed opensuse/leap:15.1 + SUBCOMMAND="$3" + if [ "$2" = "alpine" ]; then + IMAGES="alpine alpine:3.11" + elif [ "$2" = "ubuntu" ]; then + IMAGES="ubuntu ubuntu:focal" + elif [ "$2" = "debian" ]; then + IMAGES="debian debian:oldstable" + elif [ "$2" = "mint" ]; then + IMAGES="linuxmintd/mint20.3-amd64" + elif [ "$2" = "opensuse" ]; then + IMAGES="opensuse/tumbleweed opensuse/leap:15.1" + else + SUBCOMMAND="$2" + IMAGES="alpine ubuntu debian opensuse/tumbleweed" + fi + for image in $IMAGES do - $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" test + $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" test "$SUBCOMMAND" done else if [ "$1" = "musl" ]; then diff --git a/px/mcurl.py b/px/mcurl.py index ca28a81..cf98ab0 100644 --- a/px/mcurl.py +++ b/px/mcurl.py @@ -253,6 +253,12 @@ def _setup(self, url, method, request_version, connect_timeout): if method == "CONNECT": libcurl.easy_setopt(self.easy, libcurl.CURLOPT_CONNECT_ONLY, True) + if curl_version() < 0x072D00: + # libcurl < v7.45 does not support CURLINFO_ACTIVESOCKET so it is not possible + # to reuse existing connections + libcurl.easy_setopt(self.easy, libcurl.CURLOPT_FRESH_CONNECT, True) + dprint(self.easyhash + ": Fresh connection requested") + # We want libcurl to make a simple HTTP connection to auth # with the upstream proxy and let client establish SSL if "://" not in url: @@ -485,10 +491,11 @@ def set_follow(self, enable = True): def perform(self): "Perform the easy handle" - if MCURL.do(self) == False: + ret = MCURL.do(self) + MCURL.remove(self) + if ret is False: dprint(self.easyhash + ": Connection failed: " + self.errstr) - return False - return True + return ret @libcurl.socket_callback def _socket_callback(easy, sock_fd, ev_bitmask, userp, socketp): @@ -666,8 +673,7 @@ def _remove_handle(self, curl: Curl, errstr = ""): curl.errstr += errstr + "; " dprint(curl.easyhash + ": Remove handle: " + curl.errstr) - if len(curl.errstr) == 0: - libcurl.multi_remove_handle(self._multi, curl.easy) + libcurl.multi_remove_handle(self._multi, curl.easy) self.handles.pop(curl.easyhash) @@ -683,8 +689,8 @@ def stop(self, curl: Curl): # Executing multi - def perform(self): - "Perform all tasks in the multi instance" + def _perform(self): + # Perform all tasks in the multi instance with self._lock: rlen = len(self.rlist) wlen = len(self.wlist) @@ -716,7 +722,7 @@ def do(self, curl: Curl): while True: if curl.done: break - self.perform() + self._perform() time.sleep(0.01) if "timed out" in curl.errstr: @@ -739,26 +745,36 @@ def do(self, curl: Curl): with self._lock: self.failed.append(curl.proxy) + if curl.is_connect() and curl.sock_fd is None: + # Need sock_fd for select() + if curl_version() < 0x072D00: + # This should never happen since we have set CURLOPT_FRESH_CONNECT = True + # for CONNECT + out = "Cannot reuse an SSL connection with libcurl < v7.45 - should never happen" + dprint(curl.easyhash + ": " + out) + curl.errstr += out + "; " + curl.resp = 500 + else: + # Get the active socket using getinfo() for select() + dprint(curl.easyhash + ": Getting active socket") + ret, sock_fd = curl.get_activesocket() + if ret == libcurl.CURLE_OK: + curl.sock_fd = sock_fd + else: + out = "Failed to get active socket: %d, %d" % (ret, sock_fd) + dprint(curl.easyhash + ": " + out) + curl.errstr += out + "; " + curl.resp = 503 + return len(curl.errstr) == 0 def select(self, curl: Curl, client_sock, idle = 30): "Run select loop between client and curl" # TODO figure out if IPv6 or IPv4 if curl.sock_fd is None: - if curl_version() < 0x072D00: - # Reusing an SSL connection but no way to get active socket since - # CURLINFO_ACTIVESOCKET was only added in libcurl v7.45 - dprint(curl.easyhash + ": unable to reuse SSL connection with libcurl < v7.45") - return - - # Need to get the active socket using getinfo() - dprint(curl.easyhash + ": Getting active socket") - ret, sock_fd = curl.get_activesocket() - if ret == libcurl.CURLE_OK: - curl.sock_fd = sock_fd - else: - dprint(curl.easyhash + ": Failed to get active socket: %d, %d" % (ret, sock_fd)) - return + dprint(curl.easyhash + ": Cannot select() without active socket") + return + dprint(curl.easyhash + ": Starting select loop") curl_sock = socket.fromfd(curl.sock_fd, socket.AF_INET, socket.SOCK_STREAM) diff --git a/px/version.py b/px/version.py index a634e05..282894d 100644 --- a/px/version.py +++ b/px/version.py @@ -1,3 +1,3 @@ "Px version" -__version__ = "0.8.1" +__version__ = "0.8.2" diff --git a/test.py b/test.py index 2c15b07..d6d12b6 100644 --- a/test.py +++ b/test.py @@ -41,7 +41,7 @@ except AttributeError: DEVNULL = open(os.devnull, 'wb') -def exec(cmd, port = 0, shell = True): +def exec(cmd, port = 0, shell = True, delete = False): global COUNT log = "%d-%d.txt" % (port, COUNT) COUNT += 1 @@ -50,6 +50,10 @@ def exec(cmd, port = 0, shell = True): with open(log, "r") as l: data = l.read() + + if delete: + os.remove(log) + return p.returncode, data def curlcli(url, port, method = "GET", data = "", proxy = ""): @@ -442,8 +446,13 @@ def socketTestSetup(): TESTS.append((cmd, testproc, ip)) def auto(): - osname = tools.get_os() + prefix = "px.dist" + osname, _, dist = tools.get_dirs(prefix) if "--norun" not in sys.argv: + if sys.platform == "linux": + _, distro = exec("cat /etc/os-release | grep ^ID | head -n 1 | cut -d\"=\" -f2 | sed 's/\"//g'", delete = True) + _, version = exec("cat /etc/os-release | grep ^VERSION_ID | head -n 1 | cut -d\"=\" -f2 | sed 's/\"//g'", delete = True) + osname += "-%s-%s" % (distro.strip(), version.strip()) testdir = "test-%s-%d-%s" % (osname, PORT, platform.machine().lower()) # Make temp directory @@ -487,9 +496,7 @@ def auto(): if "--binary" in sys.argv: # Nuitka binary test - outdir = "px.dist-%s-%s" % (osname, platform.machine().lower()) - - cmd = os.path.abspath(os.path.join("..", outdir, "px.dist", "px")) + " " + cmd = os.path.abspath(os.path.join(dist, "px")) + " " cmds.append(cmd) results.extend([pool.apply_async(runTest, args = (TESTS[count], cmd, count + offset, PORT)) for count in range(len(TESTS))]) offset += len(TESTS)