Merge branch 'develop' into register-email-3pid-race

Author: MatMaul

.ci/scripts/calculate_jobs.py

@@ -36,11 +36,11 @@ def set_output(key: str, value: str):
 # First calculate the various trial jobs.
 #
 # For PRs, we only run each type of test with the oldest Python version supported (which
-# is Python 3.8 right now)
+# is Python 3.9 right now)
 
 trial_sqlite_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.9",
         "database": "sqlite",
         "extras": "all",
     }
@@ -53,12 +53,12 @@ def set_output(key: str, value: str):
             "database": "sqlite",
             "extras": "all",
         }
-        for version in ("3.9", "3.10", "3.11", "3.12")
+        for version in ("3.10", "3.11", "3.12", "3.13")
     )
 
 trial_postgres_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.9",
         "database": "postgres",
         "postgres-version": "11",
         "extras": "all",
@@ -68,16 +68,16 @@ def set_output(key: str, value: str):
 if not IS_PR:
     trial_postgres_tests.append(
         {
-            "python-version": "3.12",
+            "python-version": "3.13",
             "database": "postgres",
-            "postgres-version": "16",
+            "postgres-version": "17",
             "extras": "all",
         }
     )
 
 trial_no_extra_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.9",
         "database": "sqlite",
         "extras": "",
     }
@@ -99,24 +99,24 @@ def set_output(key: str, value: str):
 
 # First calculate the various sytest jobs.
 #
-# For each type of test we only run on focal on PRs
+# For each type of test we only run on bullseye on PRs
 
 
 sytest_tests = [
     {
-        "sytest-tag": "focal",
+        "sytest-tag": "bullseye",
     },
     {
-        "sytest-tag": "focal",
+        "sytest-tag": "bullseye",
         "postgres": "postgres",
     },
     {
-        "sytest-tag": "focal",
+        "sytest-tag": "bullseye",
         "postgres": "multi-postgres",
         "workers": "workers",
     },
     {
-        "sytest-tag": "focal",
+        "sytest-tag": "bullseye",
         "postgres": "multi-postgres",
         "workers": "workers",
         "reactor": "asyncio",
@@ -127,11 +127,11 @@ def set_output(key: str, value: str):
     sytest_tests.extend(
         [
             {
-                "sytest-tag": "focal",
+                "sytest-tag": "bullseye",
                 "reactor": "asyncio",
             },
             {
-                "sytest-tag": "focal",
+                "sytest-tag": "bullseye",
                 "postgres": "postgres",
                 "reactor": "asyncio",
             },

.ci/scripts/prepare_old_deps.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it
+# this script is run by GitHub Actions in a plain `jammy` container; it
 # - installs the minimal system requirements, and poetry;
 # - patches the project definition file to refer to old versions only;
 # - creates a venv with these old versions using poetry; and finally

.github/workflows/docker.yml

@@ -14,7 +14,7 @@ permissions:
   id-token: write # needed for signing the images with GitHub OIDC Token
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Set up QEMU
         id: qemu
@@ -30,7 +30,7 @@ jobs:
         run: docker buildx inspect
 
       - name: Install Cosign
-        uses: sigstore/cosign-installer@v3.5.0
+        uses: sigstore/cosign-installer@v3.7.0
 
       - name: Checkout repository
         uses: actions/checkout@v4

.github/workflows/docs-pr-netlify.yaml

@@ -14,7 +14,7 @@ jobs:
       # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
       # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
       - name: 📥 Download artifact
-        uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6
+        uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
         with:
           workflow: docs-pr.yaml
           run_id: ${{ github.event.workflow_run.id }}

.github/workflows/fix_lint.yaml

@@ -29,17 +29,13 @@ jobs:
         with:
           install-project: "false"
 
-      - name: Import order (isort)
+      - name: Run ruff check
         continue-on-error: true
-        run: poetry run isort .
+        run: poetry run ruff check --fix .
 
-      - name: Code style (black)
+      - name: Run ruff format
         continue-on-error: true
-        run: poetry run black .
-
-      - name: Semantic checks (ruff)
-        continue-on-error: true
-        run: poetry run ruff --fix .
+        run: poetry run ruff format --quiet .
 
       - run: cargo clippy --all-features --fix -- -D warnings
         continue-on-error: true
@@ -49,4 +45,4 @@ jobs:
 
       - uses: stefanzweifel/git-auto-commit-action@v5
         with:
-            commit_message: "Attempt to fix linting"
+          commit_message: "Attempt to fix linting"

.github/workflows/latest_deps.yml

@@ -132,9 +132,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: focal
+          - sytest-tag: bullseye
 
-          - sytest-tag: focal
+          - sytest-tag: bullseye
             postgres: postgres
             workers: workers
             redis: redis

.github/workflows/release-artifacts.yml

@@ -91,18 +91,27 @@ jobs:
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
+      - name: Artifact name
+        id: artifact-name
+        # We can't have colons in the upload name of the artifact, so we convert
+        # e.g. `debian:sid` to `sid`.
+        env:
+          DISTRO: ${{ matrix.distro }}
+        run: |
+          echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT"
+
       - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v3 # Don't upgrade to v4; broken: https://github.com/actions/upload-artifact#breaking-changes
+        uses: actions/upload-artifact@v4
         with:
-          name: debs
+          name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
           path: debs/*
 
   build-wheels:
     name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-12]
+        os: [ubuntu-22.04, macos-13]
         arch: [x86_64, aarch64]
         # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
         # It is not read by the rest of the workflow.
@@ -112,9 +121,9 @@ jobs:
         exclude:
           # Don't build macos wheels on PR CI.
           - is_pr: true
-            os: "macos-12"
+            os: "macos-13"
           # Don't build aarch64 wheels on mac.
-          - os: "macos-12"
+          - os: "macos-13"
             arch: aarch64
           # Don't build aarch64 wheels on PR CI.
           - is_pr: true
@@ -144,7 +153,7 @@ jobs:
 
       - name: Only build a single wheel on PR
         if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp39-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
 
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse
@@ -156,9 +165,9 @@ jobs:
           CARGO_NET_GIT_FETCH_WITH_CLI: true
           CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI
 
-      - uses: actions/upload-artifact@v3 # Don't upgrade to v4; broken: https://github.com/actions/upload-artifact#breaking-changes
+      - uses: actions/upload-artifact@v4
         with:
-          name: Wheel
+          name: Wheel-${{ matrix.os }}-${{ matrix.arch }}
           path: ./wheelhouse/*.whl
 
   build-sdist:
@@ -177,7 +186,7 @@ jobs:
       - name: Build sdist
         run: python -m build --sdist
 
-      - uses: actions/upload-artifact@v3 # Don't upgrade to v4; broken: https://github.com/actions/upload-artifact#breaking-changes
+      - uses: actions/upload-artifact@v4
         with:
           name: Sdist
           path: dist/*.tar.gz
@@ -194,17 +203,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v3 # Don't upgrade to v4, it should match upload-artifact
+        uses: actions/download-artifact@v4
       - name: Build a tarball for the debs
-        run: tar -cvJf debs.tar.xz debs
+        # We need to merge all the debs uploads into one folder, then compress
+        # that.
+        run: |
+          mkdir debs
+          mv debs*/* debs/
+          tar -cvJf debs.tar.xz debs
       - name: Attach to release
-        uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a  # PR#109
+        # Pinned to work around https://github.com/softprops/action-gh-release/issues/445
+        uses: softprops/action-gh-release@v0.1.15
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           files: |
             Sdist/*
-            Wheel/*
+            Wheel*/*
             debs.tar.xz
           # if it's not already published, keep the release as a draft.
           draft: true

.github/workflows/tests.yml

@@ -131,15 +131,11 @@ jobs:
         with:
           install-project: "false"
 
-      - name: Import order (isort)
-        run: poetry run isort --check --diff .
+      - name: Run ruff check
+        run: poetry run ruff check --output-format=github .
 
-      - name: Code style (black)
-        run: poetry run black --check --diff .
-
-      - name: Semantic checks (ruff)
-        # --quiet suppresses the update check.
-        run: poetry run ruff check --quiet .
+      - name: Run ruff format
+        run: poetry run ruff format --check .
 
   lint-mypy:
     runs-on: ubuntu-latest
@@ -401,7 +397,7 @@ jobs:
     needs:
       - linting-done
       - changes
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
 
@@ -413,12 +409,12 @@ jobs:
       # their build dependencies
       - run: |
           sudo apt-get -qq update
-          sudo apt-get -qq install build-essential libffi-dev python-dev \
+          sudo apt-get -qq install build-essential libffi-dev python3-dev \
             libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Prepare old deps
         if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
@@ -462,7 +458,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["pypy-3.8"]
+        python-version: ["pypy-3.9"]
         extras: ["all"]
 
     steps:
@@ -584,11 +580,11 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.8"
+          - python-version: "3.9"
             postgres-version: "11"
 
-          - python-version: "3.11"
-            postgres-version: "15"
+          - python-version: "3.13"
+            postgres-version: "17"
 
     services:
       postgres:

.github/workflows/twisted_trunk.yml

@@ -99,11 +99,11 @@ jobs:
     if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
-      # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
+      # We're using debian:bullseye because it uses Python 3.9 which is our minimum supported Python version.
       # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
       # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
       # version, assuming that any incompatibilities on newer versions would also be present on the oldest.
-      image: matrixdotorg/sytest-synapse:focal
+      image: matrixdotorg/sytest-synapse:bullseye
       volumes:
         - ${{ github.workspace }}:/src