Fix #154, linux deps, offline install

HISTORY.txt

@@ -1,3 +1,11 @@
+v0.8.1 - 2022-06-27
+- Fixed #154 - improved SSL connection handling with libcurl by querying active
+  socket to get the sock_fd in select() instead of relying on sockopt_callback()
+  which does not get called when connections get reused
+- Fixed keyring dependencies on Linux
+- Added infrastructure to generate and post binary wheels for Px and all its
+  dependencies for offline installation
+
 v0.8.0 - 2022-06-18
 - Added PAC file support for Linux
 - Local PAC files on Windows are now processed using QuickJS instead of WinHttp

README.md

@@ -17,20 +17,36 @@ supported by [libcurl](https://curl.se/libcurl/c/CURLOPT_HTTPAUTH.html).
 
 ## Installation
 
-If Python is already available, Px can be easily installed using the Python
-package manager `pip`. This will download and install Px as a Python module
-along with all dependencies:
+The whole point of Px is to help tools get through a typical corporate proxy.
+This means using a package manager to install Px might not always be feasible
+which is why Px offers two binary options:
+- If Python is already available, Px and all its dependencies can be easily
+installed by downloading the `wheels` package for the target OS from the
+[releases](https://github.com/genotrance/px/releases) page. After extraction,
+Px and all dependencies can be installed with `pip`:
+
+	python -m pip install px-proxy --no-index -f /path/to/wheels
+
+- If Python is not available, get the latest compiled binary from the
+[releases](https://github.com/genotrance/px/releases) page instead. These
+binaries are compiled with [Nuitka](https://nuitka.net) and contain everything
+needed to run standalone.
+
+If direct internet access is available along with Python, Px can be easily
+installed using the Python package manager `pip`. This will download and install
+Px as a Python module along with all dependencies:
 
 	python -m pip install px-proxy
 
+On Windows, `scoop` can also be used to install Px:
+
+	scoop install px
+
 Once installed, Px can be run as follows:
 - Running `px` directly
 - In the background: `pythonw -m px`
 - In the foreground in a console window: `python -m px`
 
-If Python is not available, get the latest binary from the [releases](https://github.com/genotrance/px/releases)
-page. These binaries are compiled with [Nuitka](https://nuitka.net) and contain
-everything needed to run standalone.
 
 Px requires [libcurl](https://curl.se/libcurl/) and the Windows builds ship with
 a copy. On Linux, it is required to install libcurl using the package manager:
@@ -60,8 +76,12 @@ dependencies :
 
 	python -m pip install .
 
-Note that libcurl will need to be installed on Linux, as described earlier,
-using the package manager. For Windows, [download](https://curl.se/windows/) and
+NOOTE: Source install methods will require internet access since Python will try
+to install Px dependencies from the internet. The binaries mentioned in the
+previous section could be used to bootstrap a source install.
+
+NOTE: libcurl will need to be installed on Linux, as described earlier, using
+the package manager. For Windows, [download](https://curl.se/windows/) and
 extract `libcurl.dll` and `libcurl-x64.dll` to `$PATH`.
 
 ### Without installation
@@ -349,6 +369,10 @@ the following Python packages:
 - [netaddr](https://pypi.org/project/netaddr/)
 - [psutil](https://pypi.org/project/psutil/)
 - [quickjs](https://pypi.org/project/quickjs/)
+- Linux
+  - [jeepney](https://pypi.org/project/jeepney/)
+  - [keyring_jeepney](https://pypi.org/project/keyring_jeepney/)
+  - [keyrings.alt](https://pypi.org/project/keyrings.alt/)
 - [futures](https://pypi.org/project/futures/) on Python 2.x
 
 Px also depends on [libcurl](https://curl.se/libcurl) for all outbound HTTP

build.ps1

@@ -0,0 +1,44 @@
+# Install scoop
+if ((Get-Command "scoop" -ErrorAction SilentlyContinue) -eq $null) {
+    Set-ExecutionPolicy RemoteSigned -Scope CurrentUser
+    irm get.scoop.sh | iex
+}
+
+# Install versions bucket
+scoop bucket add versions -ErrorAction SilentlyContinue
+
+# Python versions
+$PYVERSIONS = @("37", "38", "39", "310")
+$PY = "python310"
+
+# Delete depspkg directory
+Remove-Item -Recurse -Force px.dist-wheels-windows-amd64
+
+# Setup Python and dependencies, build wheels for Px
+foreach ($pyver in $PYVERSIONS) {
+    if ((Get-Command "python$pyver" -ErrorAction SilentlyContinue) -eq $null) {
+        # Install Python
+        scoop install python$pyver
+    }
+
+    # Tools
+    Invoke-Expression "python$pyver -m pip install --upgrade pip setuptools build wheel"
+
+    # Create wheel dependencies for this Python version
+    Invoke-Expression "python$pyver tools.py --deps"
+}
+
+# Install build tools
+Invoke-Expression "$PY -m pip install --upgrade nuitka twine"
+
+# Install wheel dependencies
+Invoke-Expression "$PY -m pip install --upgrade px-proxy --no-index -f px.dist-wheels-windows-amd64\px.dist-wheels"
+
+# Build wheels
+Invoke-Expression "$PY tools.py --wheel"
+
+# Create package of all dependencies
+Invoke-Expression "$PY tools.py --depspkg"
+
+# Build Nuitka
+Invoke-Expression "$PY tools.py --nuitka"

build.sh

@@ -1,5 +1,23 @@
 #! /bin/sh
 
+# Usage
+#
+# Build wheels of all dependencies across glibc and musl
+# ./build.sh deps
+#
+# Build nuitka binaries for glibc and musl
+# ./build.sh nuitka
+#
+# Run test across all distros
+# ./build.sh test
+#
+# Run command on specific container
+# ./build.sh IMAGE command subcommand
+#
+# Commands
+#   build - sub-commands: deps nuitka
+#   test
+
 if [ -f "/.dockerenv" ]; then
     # Running inside container
     DISTRO=`cat /etc/os-release | grep ^ID | head -n 1 | cut -d"=" -f2 | sed 's/"//g'`
@@ -8,52 +26,68 @@ if [ -f "/.dockerenv" ]; then
     export PROXY="$1"
     export PAC="$2"
     export USERNAME="$3"
+    export AUTH=""
 
-    # build or test
-    COMMAND="test"
+    # build or test - else $SHELL
+    COMMAND=""
     if [ ! -z "$4" ]; then
         COMMAND="$4"
     fi
 
+    # build sub-commands: nuitka or deps
+    SUBCOMMAND=""
+    if [ ! -z "$5" ]; then
+        SUBCOMMAND="$5"
+    fi
+
     if [ "$DISTRO" = "alpine" ]; then
 
         apk update && apk upgrade
-        apk add curl dbus gcc gnome-keyring linux-headers musl-dev psmisc python3 python3-dev
+        apk add curl psmisc python3
         python3 -m ensurepip
         if [ "$COMMAND" = "build" ]; then
-            apk add ccache libffi-dev patchelf py3-dbus upx
+            apk add ccache gcc musl-dev patchelf python3-dev upx
+        elif [ "$COMMAND" = "test" ]; then
+            apk add dbus gnome-keyring
         fi
 
         SHELL="sh"
 
     elif [ "$DISTRO" = "centos" ]; then
 
+        # Avoid random mirror
+        cd /etc/yum.repos.d
+        for file in `ls`; do sed -i~ 's/^mirrorlist/#mirrorlist/' $file; done
+        for file in `ls`; do sed -i~~ 's/^#baseurl/baseurl/' $file; done
+        cd
+
         yum update -y
-        yum install -y gnome-keyring psmisc
+        yum install -y psmisc python3
+        python3 -m ensurepip
         if [ "$COMMAND" = "build" ]; then
-            yum install -y ccache dbus-devel libffi-devel patchelf python36-cryptography upx
+            yum install -y ccache libffi-devel patchelf python3-devel upx
+        elif [ "$COMMAND" = "test" ]; then
+            yum install -y gnome-keyring
         fi
 
     elif [ "$DISTRO" = "ubuntu" ] || [ "$DISTRO" = "debian" ] || [ "$DISTRO" = "linuxmint" ]; then
 
         apt update -y && apt upgrade -y
-        apt install -y curl dbus gnome-keyring psmisc python3 python3-dev python3-pip
-        if [ "$COMMAND" = "build" ]; then
-            apt install -y ccache patchelf python3-dbus python3-secretstorage upx zlib1g zlib1g-dev
-        fi
+        apt install -y curl dbus gnome-keyring psmisc python3 python3-pip
+
+        export AUTH="--auth=NONEGOTIATE"
 
     elif [ "$DISTRO" = "opensuse-tumbleweed" ] || [ "$DISTRO" = "opensuse-leap" ]; then
 
         zypper -n update
         zypper -n install curl dbus-1 gnome-keyring psmisc python3 python3-pip
-        if [ "$DISTRO" = "opensuse-leap" ]; then
-            zypper -n install gcc
-        fi
+
+        export AUTH="--auth=NONEGOTIATE"
 
     elif [ "$DISTRO" = "void" ]; then
 
         xbps-install -Suy xbps
-        xbps-install -Sy curl dbus gcc gnome-keyring psmisc python3 python3-devel
+        xbps-install -Sy curl dbus gnome-keyring psmisc python3
         python3 -m ensurepip
 
         SHELL="sh"
@@ -66,27 +100,59 @@ if [ -f "/.dockerenv" ]; then
 
     MUSL=`ldd /bin/ls | grep musl`
     if [ -z "$MUSL" ]; then
-        PXBIN="/px/px.dist-linux-glibc-x86_64/px.dist/px"
+        ABI="glibc"
     else
-        PXBIN="/px/px.dist-linux-musl-x86_64/px.dist/px"
+        ABI="musl"
     fi
+    export PXBIN="/px/px.dist-linux-$ABI-x86_64/px.dist/px"
+    export WHEELS="/px/px.dist-wheels-linux-$ABI-x86_64/px.dist-wheels"
 
-    dbus-run-session -- $SHELL -c 'echo "abc" | gnome-keyring-daemon --unlock'
+    if [ "$COMMAND" = "test" ]; then
+        dbus-run-session -- $SHELL -c 'echo "abc" | gnome-keyring-daemon --unlock'
+    fi
 
     cd /px
     if [ "$COMMAND" = "build" ]; then
-        python3 -m pip install --upgrade pip setuptools jeepney==0.7.1
-
-        python3 tools.py --setup
-
-        python3 px.py --username=$USERNAME --password
+        if [ "$SUBCOMMAND" = "nuitka" ]; then
+            # Install tools
+            python3 -m pip install --upgrade pip setuptools build wheel
+            python3 -m pip install --upgrade nuitka
+
+            # Install wheel dependencies
+            # nuitka depends on deps - run deps first
+            python3 -m pip install px-proxy --no-index -f $WHEELS
+
+            # Build Nuitka binary
+            python3 tools.py --nuitka
+        elif [ "$SUBCOMMAND" = "deps" ]; then
+            rm -rf $WHEELS
+
+            # Run for all Python versions
+            for pyver in `ls /opt/python/cp* -d`
+            do
+                # Install tools
+                $pyver/bin/python3 -m pip install --upgrade pip setuptools build wheel
+
+                # Build dependency wheels
+                $pyver/bin/python3 tools.py --deps
+            done
+
+            # Package all wheels
+            /opt/python/cp310-cp310/bin/python3 tools.py --depspkg
+        else
+            $SHELL
+        fi
     else
-        python3 -m pip install --upgrade pip setuptools netifaces psutil
+        python3 -m pip install --upgrade pip
+        python3 -m pip install px-proxy --no-index -f $WHEELS
 
-        $PXBIN --username=$USERNAME --password
+        if [ "$COMMAND" = "test" ]; then
+            python3 test.py --binary --pip --proxy=$PROXY --pac=$PAC --username=$USERNAME $AUTH
+        else
+            $SHELL
+        fi
     fi
 
-    $SHELL
 else
     # Start container
     if [ -z "$PROXY" ]; then
@@ -104,5 +170,30 @@ else
         exit
     fi
 
-    docker run -it --rm --network host --privileged -v `pwd`:/px $1 /px/build.sh "$PROXY" "$PAC" "$USERNAME" $2
+    # Python wheel containers for musl and glibc
+    MUSL="quay.io/pypa/musllinux_1_1_x86_64"
+    GLIBC="quay.io/pypa/manylinux2014_x86_64"
+
+    DOCKERCMD="docker run -it --rm --network host --privileged -v `pwd`:/px -v /root/.local/share:/root/.local/share"
+    if [ "$1" = "deps" ] || [ "$1" = "nuitka" ]; then
+        for image in $MUSL $GLIBC
+        do
+            $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" build $1
+        done
+    elif [ "$1" = "test" ]; then
+        for image in alpine alpine:3.11 voidlinux/voidlinux-musl ubuntu ubuntu:bionic debian debian:oldstable linuxmintd/mint20.3-amd64 opensuse/tumbleweed opensuse/leap:15.1
+        do
+            $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" test
+        done
+    else
+        if [ "$1" = "musl" ]; then
+            IMAGE="$MUSL"
+        elif [ "$1" = "glibc" ]; then
+            IMAGE="$GLIBC"
+        else
+            IMAGE="$1"
+        fi
+
+        $DOCKERCMD $IMAGE /px/build.sh "$PROXY" "$PAC" "$USERNAME" $2 $3
+    fi
 fi