feat: add urllib3 v2 as network backend (#292)

* requirements: add urllib3 * pyproject: update dependencies - Adds urllib3 as dependency and truststore as an optional dependency * Makefile: add new build target for urllib3 v2 - Automatically includes urllib3 v2 in all targets * __main__: add vendored path to PYTHONPATH * umu_util: remove helper to create HTTPSConnection instances * umu_util: add helper to compute digest and write file * umu_run: refactor to use connection pools from urllib3 v2 * umu_runtime: refactor to use urllib3 v2 * umu_proton: refactor to use urllib3 v2 * umu_test: add tests for _restore_umu * umu_test: add tests for setup_umu * umu_test: add tests for _restore_umu_platformid * umu_test: add tests for write_file_chunks * umu_test: update tests * umu_test_plugins: update tests * workflows: setup venv * workflows: activate venv * workflows: don't use fish shell * workflows: don't append .sh to activate script * workflows: setup venv for e2e * workflows: fix venv in e2e * workflows: setup venv once * workflows: fix unit tests not running in venv * workflows: fix unit tests not running in venv * workflows: remove python 3.10 from matrix - We're targeting latest Debian which is currently Bookworm and it defaults to 3.11 * workflows: fix mypy not running in venv * workflows: fix mypy not running in venv * workflows: set python 3.11 for mypy * umu_runtime: add ignore for mypy * Makefile: fix deb build * Makefile: fix format * deb: add python3-pip as build dependency * README: mention pip as build dependency * pyproject: add comment regarding urllib3 dep * pyproject: update comment for truststore * umu_runtime: update log statement * umu_runtime: remove cached file on digest mismatch * umu_runtime: update error message on digest mismatch * umu_run: handle the digest mismatch when updating runtime - We shouldn't crash here as it would be non-critical for users that already have an existing runtime that's intact. In this case, let the user run their game with the runtime already installed * umu_runtime: include the upstream link in the error message * umu_proton: delete cached file on digest mismatch * umu_runtime: extend archive file with its id to avoid corruption on resume * __main__: prefer list.insert when extending PYTHONPATH
Open-Wine-Components · Nov 29, 2024 · d006222 · d006222
1 parent 2c85618
commit d006222
Show file tree

Hide file tree

Showing 16 changed files with 1,146 additions and 574 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -2,9 +2,9 @@ name: e2e
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
 
 permissions:
   contents: read
@@ -14,37 +14,43 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - name: Install dependencies
-      run: |
-        sudo apt-get install meson scdoc python3-hatchling python3-build python3-installer python3-filelock shellcheck
-        python3 -m pip install --upgrade pip
-        pip install uv
-    - name: Initialize submodules
-      run: |
-        git submodule update --init --recursive
-    - name: Make user install
-      run: |
-        ./configure.sh --user-install
-        make install
-    - name: Run shellcheck
-      run: |
-        shellcheck tests/*.sh
-    - name: Test steamrt install
-      run: |
-        sh tests/test_install.sh
-        rm -rf "$HOME/.local/share/umu" "$HOME/Games/umu" "$HOME/.local/share/Steam/compatibilitytools.d"
-    - name: Test steamrt update
-      run: |
-        sh tests/test_update.sh
-        rm -rf "$HOME/.local/share/umu" "$HOME/Games/umu" "$HOME/.local/share/Steam/compatibilitytools.d"
-    - name: Test winetricks
-      run: |
-        sh tests/test_winetricks.sh
-        rm -rf "$HOME/.local/share/umu" "$HOME/.local/share/Steam/compatibilitytools.d"
-    - name: Test configuration file
-      run: |
-        uv python install 3.11
-        uv run --python 3.11 -- sh tests/test_config.sh
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install dependencies
+        run: |
+          sudo apt-get install meson scdoc python3-hatchling python3-build python3-installer python3-filelock shellcheck
+          python3 -m pip install --upgrade pip
+          pip install uv
+      - name: Initialize submodules
+        run: |
+          git submodule update --init --recursive
+      - name: Make user install
+        run: |
+          ./configure.sh --user-install
+          make install
+      - name: Run shellcheck
+        run: |
+          shellcheck tests/*.sh
+      - name: Setup venv
+        run: |
+          uv venv --python 3.11
+      - name: Test steamrt install
+        run: |
+          source .venv/bin/activate
+          sh tests/test_install.sh
+          rm -rf "$HOME/.local/share/umu" "$HOME/Games/umu" "$HOME/.local/share/Steam/compatibilitytools.d"
+      - name: Test steamrt update
+        run: |
+          source .venv/bin/activate
+          sh tests/test_update.sh
+          rm -rf "$HOME/.local/share/umu" "$HOME/Games/umu" "$HOME/.local/share/Steam/compatibilitytools.d"
+      - name: Test winetricks
+        run: |
+          source .venv/bin/activate
+          sh tests/test_winetricks.sh
+          rm -rf "$HOME/.local/share/umu" "$HOME/.local/share/Steam/compatibilitytools.d"
+      - name: Test configuration file
+        run: |
+          source .venv/bin/activate
+          sh tests/test_config.sh
diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml
@@ -2,9 +2,9 @@ name: mypy
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
 
 permissions:
   contents: read
@@ -13,20 +13,25 @@ jobs:
   build:
     strategy:
       matrix:
-        version: ["3.10"]
+        version: ["3.11"]
 
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.version }}
-    - name: Install dependencies
-      run: |
-        python3 -m pip install --upgrade pip
-    - name: Check types with mypy
-      run: |
-        pip install mypy
-        mypy .
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.version }}
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip uv mypy
+      - name: Setup venv
+        run: |
+          uv venv --python 3.11
+          source .venv/bin/activate
+          uv pip install -r requirements.in
+      - name: Check types with mypy
+        run: |
+          source .venv/bin/activate
+          mypy --python-version 3.11 .
diff --git a/.github/workflows/umu-python.yml b/.github/workflows/umu-python.yml
@@ -2,9 +2,9 @@ name: umu-launcher
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
 
 permissions:
   contents: read
@@ -15,28 +15,40 @@ jobs:
       matrix:
         # tomllib requires Python 3.11
         # Ubuntu latest (Jammy) provides Python 3.10
-        version: ["3.10", "3.11", "3.12", "3.13"]
+        version: ["3.11", "3.12", "3.13"]
 
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.version }}
-    - name: Install dependencies
-      run: |
-        python3 -m pip install --upgrade pip
-        pip install ruff
-        pip install python-xlib
-        pip install filelock
-    - name: Lint umu_*.py files with Ruff
-      run: |
-        pip install ruff
-        ruff check --output-format github ./umu/umu_*.py
-    - name: Test with unittest
-      run: python3 ./umu/umu_test.py
-    - name: Test with unittest for plugins
-      if: ${{ matrix.version == '3.11' || matrix.version == '3.12' || matrix.version == '3.13' }}
-      run: python3 ./umu/umu_test_plugins.py
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.version }}
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          pip install ruff
+          pip install python-xlib
+          pip install filelock
+          pip install uv
+      - name: Lint umu_*.py files with Ruff
+        run: |
+          pip install ruff
+          ruff check --output-format github ./umu/umu_*.py
+      - name: Setup venv
+        run: |
+          uv venv --python $pyver
+          source .venv/bin/activate
+          uv pip install -r requirements.in
+        env:
+          pyver: ${{ matrix.version }}
+      - name: Test with unittest
+        run: |
+          source .venv/bin/activate
+          python3 ./umu/umu_test.py
+      - name: Test with unittest for plugins
+        if: ${{ matrix.version == '3.11' || matrix.version == '3.12' || matrix.version == '3.13' }}
+        run: |
+          source .venv/bin/activate
+          python3 ./umu/umu_test_plugins.py
diff --git a/Makefile.in b/Makefile.in
@@ -21,7 +21,7 @@ FLATPAK ?= xfalse
 
 .PHONY: all
 ifeq ($(FLATPAK), xtrue)
-all: umu-dist umu-launcher
+all: umu-dist umu-launcher umu-vendored
 endif
 
 .PHONY: install
@@ -30,8 +30,8 @@ SOURCE_DATE_EPOCH = $(shell LC_ALL=C date --date='@1580601600')
 all: zipapp
 install: zipapp-install
 else
-all: umu-dist umu-docs umu-launcher
-install: umu-install umu-launcher-install
+all: umu-dist umu-docs umu-launcher umu-vendored
+install: umu-install umu-launcher-install umu-vendored-install
 endif
 
 
@@ -99,20 +99,17 @@ umu-launcher-dist-install:
 umu-launcher-install: umu-launcher-dist-install umu-launcher-bin-install
 
 
-$(OBJDIR)/.build-umu-subprojects: | $(OBJDIR)
-	$(info :: Building subprojects )
-	pip3 install -r requirements.in -t $(OBJDIR)
+$(OBJDIR)/.build-umu-vendored: | $(OBJDIR)
+	$(info :: Building vendored dependencies )
+	python3 -m pip install urllib3 -t $(OBJDIR)
 
-.PHONY: umu-subprojects
-umu-subprojects: $(OBJDIR)/.build-umu-subprojects
+.PHONY: umu-vendored
+umu-vendored: $(OBJDIR)/.build-umu-vendored
 
-umu-subprojects-install:
+umu-vendored-install: umu-vendored
 	$(info :: Installing subprojects )
-	install -d $(DESTDIR)$(PYTHONDIR)
-	cp      -r $(OBJDIR)/*-info     $(DESTDIR)$(PYTHONDIR)
-	cp      -r $(OBJDIR)/Xlib       $(DESTDIR)$(PYTHONDIR)
-	cp      -r $(OBJDIR)/filelock   $(DESTDIR)$(PYTHONDIR)
-	cp         $(OBJDIR)/six.py     $(DESTDIR)$(PYTHONDIR)
+	install -d $(DESTDIR)$(PYTHONDIR)/umu/_vendor
+	cp      -r $(OBJDIR)/urllib3 $(DESTDIR)$(PYTHONDIR)/umu/_vendor
 
 $(OBJDIR):
 	@mkdir -p $(@)
@@ -153,7 +150,7 @@ ZIPAPP_VENV := $(OBJDIR)/zipapp_venv
 $(OBJDIR)/.build-zipapp: | $(OBJDIR)
 	$(info :: Building umu-launcher as zipapp )
 	$(PYTHON_INTERPRETER) -m venv $(ZIPAPP_VENV)
-	. $(ZIPAPP_VENV)/bin/activate && python3 -m pip install -t "$(ZIPAPP_STAGING)" -U --no-compile .
+	. $(ZIPAPP_VENV)/bin/activate && python3 -m pip install -t "$(ZIPAPP_STAGING)" -U --no-compile . truststore
 	cp umu/__main__.py "$(ZIPAPP_STAGING)"
 	find "$(ZIPAPP_STAGING)" -exec touch -h -d "$(SOURCE_DATE_EPOCH)" {} +
 	. $(ZIPAPP_VENV)/bin/activate && python3 -m zipapp $(ZIPAPP_STAGING) -o $(ZIPAPP) -p "$(PYTHON_INTERPRETER)" -c

diff --git a/README.md b/README.md
@@ -79,7 +79,7 @@ Borderlands 3 from EGS store.
 
 ## Building
 
-Building umu-launcher currently requires `bash`, `make`, and `scdoc` for distribution, as well as the following Python build tools: [build](https://github.com/pypa/build), [hatchling](https://github.com/pypa/hatch), and [installer](https://github.com/pypa/installer).
+Building umu-launcher currently requires `bash`, `make`, and `scdoc` for distribution, as well as the following Python build tools: [build](https://github.com/pypa/build), [hatchling](https://github.com/pypa/hatch), [installer](https://github.com/pypa/installer), and [pip](https://github.com/pypa/pip).
 
 To build umu-launcher, after downloading and extracting the source code from this repository, change into the newly extracted directory
 ```shell
@@ -95,7 +95,7 @@ Change the `--prefix` as fit for your distribution, for example `/usr/local`, or
 Then run `make` to build. After a successful build the resulting files should be available in the `./builddir` directory
 
 
-## Installing 
+## Installing
 
 To install umu-launcher run the following command after completing the steps described above
 ```shell

diff --git a/packaging/deb/debian/control b/packaging/deb/debian/control
@@ -13,6 +13,7 @@ Build-Depends:
  python3-hatchling,
  python3-installer,
  python3-build,
+ python3-pip,
 Standards-Version: 4.6.2
 Homepage: https://github.com/Open-Wine-Components/umu-launcher
 Vcs-Browser: https://github.com/Open-Wine-Components/umu-launcher

diff --git a/packaging/deb/ubuntu/control b/packaging/deb/ubuntu/control
@@ -13,6 +13,7 @@ Build-Depends:
  python3-hatchling,
  python3-installer,
  python3-build,
+ python3-pip,
 Standards-Version: 4.6.2
 Homepage: https://github.com/Open-Wine-Components/umu-launcher
 Vcs-Browser: https://github.com/Open-Wine-Components/umu-launcher

diff --git a/pyproject.toml b/pyproject.toml
@@ -33,7 +33,14 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 urls = { repository = "https://github.com/Open-Wine-Components/umu-launcher" }
-dependencies = ["python-xlib>=0.33", "filelock>=3.9.0"]
+# Note: urllib3 is a vendored dependency. When using our Makefile, it will be
+# installed automatically.
+dependencies = ["python-xlib>=0.33", "filelock>=3.9.0", "urllib3>=2.0.0,<3.0.0"]
+
+[project.optional-dependencies]
+# Recommended
+# For network requests, use the system's CA bundle instead of certifi's
+cli = ["truststore"]
 
 [project.scripts]
 umu-run = "umu.__main__:main"

diff --git a/requirements.in b/requirements.in
@@ -1,2 +1,3 @@
 python-xlib>=0.33
-filelock>=3.15.4 
+filelock>=3.15.4
+urllib3>=2.0.0,<3.0.0
diff --git a/umu/__main__.py b/umu/__main__.py
@@ -1,5 +1,13 @@
 import os
 import sys
+from zipfile import is_zipfile
+
+if not is_zipfile(os.path.dirname(__file__)):  # noqa: PTH120
+    sys.path.insert(
+        0,
+        f"{os.path.dirname(os.path.realpath(__file__, strict=True))}/_vendor",  # noqa: PTH120
+    )
+
 from argparse import ArgumentParser, Namespace, RawTextHelpFormatter
 
 from umu import __version__