From 39df09c98d9dc768e020f56f140c2b8b85efc828 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 21 Jul 2022 23:34:59 -0500 Subject: [PATCH] Fix #160 - pac issues --- HISTORY.txt | 3 +- build.ps1 | 5 +++- build.sh | 79 ++++++++++++++++++++++++++++++++++++++++++---------- px/pac.py | 17 +++++++++-- px/wproxy.py | 4 +-- test.py | 22 +++++++-------- tools.py | 1 + 7 files changed, 99 insertions(+), 32 deletions(-) diff --git a/HISTORY.txt b/HISTORY.txt index b8740d4..39b0034 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -1,6 +1,7 @@ -v0.8.3 - 2022-07-14 +v0.8.3 - 2022-07-19 - Fixed #157 - libcurl wrapper was missing socket definitions for OSX - Fixed #158 - win32ctypes was not being included in Windows binary +- Fixed #160 - need to convert PAC return values into CURLOPT_PROXY schemes v0.8.2 - 2022-06-29 - Fixed #155 - prevent SSL connection reuse for libcurl < v7.45 diff --git a/build.ps1 b/build.ps1 index 6ae8b1a..8dab40c 100644 --- a/build.ps1 +++ b/build.ps1 @@ -41,4 +41,7 @@ Invoke-Expression "$PY tools.py --wheel" Invoke-Expression "$PY tools.py --depspkg" # Build Nuitka -Invoke-Expression "$PY tools.py --nuitka" \ No newline at end of file +Invoke-Expression "$PY tools.py --nuitka" + +# Uninstall Px +Invoke-Expression "$PY -m pip uninstall px-proxy -y" \ No newline at end of file diff --git a/build.sh b/build.sh index 691ef84..873dd82 100644 --- a/build.sh +++ b/build.sh @@ -8,6 +8,9 @@ # Build nuitka binaries for glibc and musl # ./build.sh nuitka # +# Build both wheels and nuita binaries for glibc and musl +# ./build.sh all +# # Run test across all distros # ./build.sh test # @@ -15,9 +18,11 @@ # ./build.sh IMAGE command subcommand # # Commands -# build - sub-commands: deps nuitka +# build - sub-commands: deps nuitka all # test - sub-commands: alpine ubuntu debian opensuse, test.py flags +OS=`uname -s` + if [ -f "/.dockerenv" ]; then # Running inside container DISTRO=`cat /etc/os-release | grep ^ID | head -n 1 | cut -d"=" -f2 | sed 's/"//g'` @@ -114,18 +119,7 @@ if [ -f "/.dockerenv" ]; then cd /px if [ "$COMMAND" = "build" ]; then - if [ "$SUBCOMMAND" = "nuitka" ]; then - # Install tools - python3 -m pip install --upgrade pip setuptools build wheel - python3 -m pip install --upgrade nuitka - - # Install wheel dependencies - # nuitka depends on deps - run deps first - python3 -m pip install px-proxy --no-index -f $WHEELS - - # Build Nuitka binary - python3 tools.py --nuitka - elif [ "$SUBCOMMAND" = "deps" ]; then + if [ "$SUBCOMMAND" = "deps" ] || [ "$SUBCOMMAND" = "all" ]; then rm -rf $WHEELS # Run for all Python versions @@ -140,7 +134,22 @@ if [ -f "/.dockerenv" ]; then # Package all wheels /opt/python/cp310-cp310/bin/python3 tools.py --depspkg - else + fi + + if [ "$SUBCOMMAND" = "nuitka" ] || [ "$SUBCOMMAND" = "all" ]; then + # Install tools + python3 -m pip install --upgrade pip setuptools build wheel + python3 -m pip install --upgrade nuitka + + # Install wheel dependencies + # nuitka depends on deps - run deps first + python3 -m pip install px-proxy --no-index -f $WHEELS + + # Build Nuitka binary + python3 tools.py --nuitka + fi + + if [ -z "$SUBCOMMAND" ]; then $SHELL fi else @@ -154,6 +163,46 @@ if [ -f "/.dockerenv" ]; then fi fi +elif [ "$OS" = "Darwin" ]; then + + # Install brew + if ! brew -v > /dev/null; then + bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + fi + brew update + + # Delete depspkg directory + rm -rf px.dist-wheels-osx-x86_64 + + for pyver in 3.7 3.8 3.9 3.10 + do + # Install Python + brew install python@$pyver + + PY="/usr/local/opt/python@$pyver/bin/python3" + + # Tools + $PY -m pip install --upgrade pip setuptools build wheel + + # Create wheel dependencies for this Python version + $PY tools.py --deps + done + + # Install build tools + $PY -m pip install --upgrade nuitka twine + + # Install wheel dependencies + $PY -m pip install --upgrade px-proxy --no-index -f px.dist-wheels-osx-x86_64/px.dist-wheels + + # Create package of all dependencies + $PY tools.py --depspkg + + # Build Nuitka + $PY tools.py --nuitka + + # Uninstall Px + $PY -m pip uninstall px-proxy -y + else # Start container if [ -z "$PROXY" ]; then @@ -176,7 +225,7 @@ else GLIBC="quay.io/pypa/manylinux2014_x86_64" DOCKERCMD="docker run -it --rm --network host --privileged -v `pwd`:/px -v /root/.local/share:/root/.local/share" - if [ "$1" = "deps" ] || [ "$1" = "nuitka" ]; then + if [ "$1" = "deps" ] || [ "$1" = "nuitka" ] || [ "$1" = "all" ]; then for image in $MUSL $GLIBC do $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" build $1 diff --git a/px/pac.py b/px/pac.py index fff37cb..c860f65 100644 --- a/px/pac.py +++ b/px/pac.py @@ -54,10 +54,23 @@ def load_url(self, jsurl): def find_proxy_for_url(self, url, host): """ - Return list of proxy servers to use for this url + Return comma-separated list of proxy servers to use for this url DIRECT can be returned as one of the options in the response """ - return self.ctxt.eval("FindProxyForURL")(url, host) + proxies = self.ctxt.eval("FindProxyForURL")(url, host) + + # Fix #160 - convert PAC return values into CURLOPT_PROXY schemes + for ptype in ["PROXY", "HTTP"]: + proxies = proxies.replace(ptype + " ", "") + for ptype in ["HTTPS", "SOCKS4", "SOCKS5"]: + proxies = proxies.replace(ptype + " ", ptype.lower() + "://") + proxies = proxies.replace("SOCKS ", "socks5://") + + # Not sure if SOCKS proxies will be used with Px since they are not + # relevant for NTLM/Kerberos authentication over HTTP but libcurl can + # deal with it for now + + return proxies.replace(" ", ",").replace(";", ",") # Python callables from JS diff --git a/px/wproxy.py b/px/wproxy.py index 0558a9d..6e2959a 100644 --- a/px/wproxy.py +++ b/px/wproxy.py @@ -464,9 +464,9 @@ def find_proxy_for_url(self, url): servers, netloc, path = super().find_proxy_for_url(url) if servers is not None: - # MODE_NONE, MODE_ENV, MODE_CONFIG or url in no_proxy + # MODE_NONE, MODE_ENV, MODE_CONFIG, MODE_CONFIG_PAC or url in no_proxy return servers, netloc, path - elif self.mode in [MODE_AUTO, MODE_PAC, MODE_CONFIG_PAC]: + elif self.mode in [MODE_AUTO, MODE_PAC]: # Use proxies as resolved via WinHttp return parse_proxy(self.winhttp_find_proxy_for_url(url)), netloc, path elif self.mode in [MODE_MANUAL]: diff --git a/test.py b/test.py index be94486..15ce4a8 100644 --- a/test.py +++ b/test.py @@ -223,7 +223,7 @@ def getips(): for addr in ifs[ifname]: if addr.family == socket.AddressFamily.AF_INET: ip = addr.address - if ip not in ip_list: + if ip not in ip_list and not ip.startswith("169"): ip_list.append(ip) return ip_list @@ -474,7 +474,7 @@ def auto(): if "--binary" in sys.argv: # Nuitka binary test - cmd = os.path.abspath(os.path.join(dist, "px")) + " " + cmd = os.path.abspath(os.path.join("..", dist, "px")) + " " cmds.append(cmd) results.extend([pool.apply_async(runTest, args = (TESTS[count], cmd, count + offset, PORT)) for count in range(len(TESTS))]) offset += len(TESTS) @@ -487,11 +487,9 @@ def auto(): exec(cmd) # Install Px - cmd = sys.executable + " -m pip install ../wheel/" - if sys.platform == "win32": - cmd += "px_proxy-%s-py3-none-win_amd64.whl" % __version__ - elif sys.platform == "linux": - cmd += "px_proxy-%s-py2.py3-none-any.whl" % __version__ + prefix = "px.dist-wheels" + _, _, wdist = tools.get_dirs(prefix) + cmd = sys.executable + " -m pip install --upgrade px-proxy --no-index -f ../" + wdist ret, data = exec(cmd) if ret != 0: print("Failed: pip install: %d\n%s" % (ret, data)) @@ -503,9 +501,9 @@ def auto(): offset += len(TESTS) # Run as Python console script - cmd = shutil.which("px").replace(".EXE", "") - if len(cmd) != 0: - cmd += " " + cmd = shutil.which("px") + if cmd is not None: + cmd = cmd.replace(".EXE", "") + " " cmds.append(cmd) results.extend([pool.apply_async(runTest, args = (TESTS[count], cmd, count + offset, PORT)) for count in range(len(TESTS))]) offset += len(TESTS) @@ -630,7 +628,9 @@ def main(): PORT = int(PORT) else: PORT = 3128 - USERNAME = tools.get_argval("username").replace("\\", "\\\\") + USERNAME = tools.get_argval("username") + if sys.platform != "win32": + USERNAME = USERNAME.replace("\\", "\\\\") AUTH = tools.get_argval("auth") BINARY = tools.get_argval("binary") diff --git a/tools.py b/tools.py index b455bbd..4ec1eb1 100644 --- a/tools.py +++ b/tools.py @@ -461,6 +461,7 @@ def create_release(tag, name, body, prerelease): return id def add_asset_to_release(filename, relid): + print("Uploading " + filename) rfile_size = os.stat(filename).st_size with open(filename, "rb") as rfile: headers = get_auth()