Merge branch 'develop' into register-email-3pid-race

.ci/scripts/calculate_jobs.py

@@ -60,7 +60,7 @@ def set_output(key: str, value: str):
     {
         "python-version": "3.9",
         "database": "postgres",
-        "postgres-version": "11",
+        "postgres-version": "13",
         "extras": "all",
     }
 ]

.github/workflows/tests.yml

@@ -581,7 +581,7 @@ jobs:
       matrix:
         include:
           - python-version: "3.9"
-            postgres-version: "11"
+            postgres-version: "13"
 
           - python-version: "3.13"
             postgres-version: "17"

changelog.d/17994.doc

@@ -0,0 +1 @@
+Fix example in reverse proxy docs to include server port.

changelog.d/18017.misc

@@ -0,0 +1 @@
+Disable DB statement timeout when doing a purge room since it can be quite long.

changelog.d/18020.misc

@@ -0,0 +1 @@
+Remove some remaining uses of `twisted.internet.defer.returnValue`. Contributed by Colin Watson.

changelog.d/18024.bugfix

@@ -0,0 +1 @@
+Properly purge state groups tables when purging a room with the admin API.

changelog.d/18029.bugfix

@@ -0,0 +1 @@
+Fix a bug preventing the admin redaction endpoint from working on messages from remote users.

changelog.d/18034.removal

@@ -0,0 +1 @@
+Remove support for PostgreSQL 11 and 12. Contributed by @clokep.

changelog.d/18043.bugfix

@@ -0,0 +1 @@
+Fix a bug preventing the admin redaction endpoint from working on messages from remote users.

changelog.d/18063.misc

@@ -0,0 +1 @@
+Refactor `get_profile` to no longer include fields with a value of `None`.

contrib/cmdclient/console.py

@@ -245,7 +245,7 @@ def _check_can_login(self):
 
         if "flows" not in json_res:
             print("Failed to find any login flows.")
-            defer.returnValue(False)
+            return False
 
         flow = json_res["flows"][0]  # assume first is the one we want.
         if "type" not in flow or "m.login.password" != flow["type"] or "stages" in flow:
@@ -254,8 +254,8 @@ def _check_can_login(self):
                 "Unable to login via the command line client. Please visit "
                 "%s to login." % fallback_url
             )
-            defer.returnValue(False)
-        defer.returnValue(True)
+            return False
+        return True
 
     def do_emailrequest(self, line):
         """Requests the association of a third party identifier

contrib/cmdclient/http.py

@@ -78,7 +78,7 @@ def put_json(self, url, data):
             url, data, headers_dict={"Content-Type": ["application/json"]}
         )
         body = yield readBody(response)
-        defer.returnValue((response.code, body))
+        return response.code, body
 
     @defer.inlineCallbacks
     def get_json(self, url, args=None):
@@ -88,7 +88,7 @@ def get_json(self, url, args=None):
             url = "%s?%s" % (url, qs)
         response = yield self._create_get_request(url)
         body = yield readBody(response)
-        defer.returnValue(json.loads(body))
+        return json.loads(body)
 
     def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
         """Wrapper of _create_request to issue a PUT request"""
@@ -134,7 +134,7 @@ def do_request(
             response = yield self._create_request(method, url)
 
         body = yield readBody(response)
-        defer.returnValue(json.loads(body))
+        return json.loads(body)
 
     @defer.inlineCallbacks
     def _create_request(
@@ -173,7 +173,7 @@ def _create_request(
         if self.verbose:
             print("Status %s %s" % (response.code, response.phrase))
             print(pformat(list(response.headers.getAllRawHeaders())))
-        defer.returnValue(response)
+        return response
 
     def sleep(self, seconds):
         d = defer.Deferred()

docker/conf-workers/nginx.conf.j2

@@ -38,6 +38,9 @@ server {
 {% if using_unix_sockets %}
         proxy_pass http://unix:/run/main_public.sock;
 {% else %}
+        # note: do not add a path (even a single /) after the port in `proxy_pass`,
+        # otherwise nginx will canonicalise the URI and cause signature verification
+        # errors.
         proxy_pass http://localhost:8080;
 {% endif %}
         proxy_set_header X-Forwarded-For $remote_addr;

docs/reverse_proxy.md

@@ -74,7 +74,7 @@ server {
         proxy_pass http://localhost:8008;
         proxy_set_header X-Forwarded-For $remote_addr;
         proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header Host $host;
+        proxy_set_header Host $host:$server_port;
 
         # Nginx by default only allows file uploads up to 1M in size
         # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml

docs/upgrade.md

@@ -117,6 +117,14 @@ each upgrade are complete before moving on to the next upgrade, to avoid
 stacking them up. You can monitor the currently running background updates with
 [the Admin API](usage/administration/admin_api/background_updates.html#status).
 
+# Upgrading to v1.122.0
+
+## Dropping support for PostgreSQL 11 and 12
+
+In line with our [deprecation policy](deprecation_policy.md), we've dropped
+support for PostgreSQL 11 and 12, as they are no longer supported upstream.
+This release of Synapse requires PostgreSQL 13+.
+
 # Upgrading to v1.120.0
 
 ## Removal of experimental MSC3886 feature

synapse/api/constants.py

@@ -320,3 +320,8 @@ class ApprovalNoticeMedium:
 class Direction(enum.Enum):
     BACKWARDS = "b"
     FORWARDS = "f"
+
+
+class ProfileFields:
+    DISPLAYNAME: Final = "displayname"
+    AVATAR_URL: Final = "avatar_url"

synapse/handlers/admin.py

@@ -473,7 +473,7 @@ async def _redact_all_events(
                     "type": EventTypes.Redaction,
                     "content": {"reason": reason} if reason else {},
                     "room_id": room,
-                    "sender": user_id,
+                    "sender": requester.user.to_string(),
                 }
                 if room_version.updated_redaction_rules:
                     event_dict["content"]["redacts"] = event.event_id

synapse/handlers/profile.py

@@ -22,6 +22,7 @@
 import random
 from typing import TYPE_CHECKING, List, Optional, Union
 
+from synapse.api.constants import ProfileFields
 from synapse.api.errors import (
     AuthError,
     Codes,
@@ -83,7 +84,7 @@ async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDi
 
         Returns:
             A JSON dictionary. For local queries this will include the displayname and avatar_url
-            fields. For remote queries it may contain arbitrary information.
+            fields, if set. For remote queries it may contain arbitrary information.
         """
         target_user = UserID.from_string(user_id)
 
@@ -92,10 +93,13 @@ async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDi
             if profileinfo.display_name is None and profileinfo.avatar_url is None:
                 raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
 
-            return {
-                "displayname": profileinfo.display_name,
-                "avatar_url": profileinfo.avatar_url,
-            }
+            # Do not include display name or avatar if unset.
+            ret = {}
+            if profileinfo.display_name is not None:
+                ret[ProfileFields.DISPLAYNAME] = profileinfo.display_name
+            if profileinfo.avatar_url is not None:
+                ret[ProfileFields.AVATAR_URL] = profileinfo.avatar_url
+            return ret
         else:
             try:
                 result = await self.federation.make_query(

synapse/handlers/sso.py

@@ -43,7 +43,7 @@
 from twisted.web.iweb import IRequest
 from twisted.web.server import Request
 
-from synapse.api.constants import LoginType
+from synapse.api.constants import LoginType, ProfileFields
 from synapse.api.errors import Codes, NotFoundError, RedirectException, SynapseError
 from synapse.config.sso import SsoAttributeRequirement
 from synapse.handlers.device import DeviceHandler
@@ -813,9 +813,10 @@ def is_allowed_mime_type(content_type: str) -> bool:
 
             # bail if user already has the same avatar
             profile = await self._profile_handler.get_profile(user_id)
-            if profile["avatar_url"] is not None:
-                server_name = profile["avatar_url"].split("/")[-2]
-                media_id = profile["avatar_url"].split("/")[-1]
+            if ProfileFields.AVATAR_URL in profile:
+                avatar_url_parts = profile[ProfileFields.AVATAR_URL].split("/")
+                server_name = avatar_url_parts[-2]
+                media_id = avatar_url_parts[-1]
                 if self._is_mine_server_name(server_name):
                     media = await self._media_repo.store.get_local_media(media_id)  # type: ignore[has-type]
                     if media is not None and upload_name == media.upload_name:

synapse/handlers/user_directory.py

@@ -26,7 +26,13 @@
 from twisted.internet.interfaces import IDelayedCall
 
 import synapse.metrics
-from synapse.api.constants import EventTypes, HistoryVisibility, JoinRules, Membership
+from synapse.api.constants import (
+    EventTypes,
+    HistoryVisibility,
+    JoinRules,
+    Membership,
+    ProfileFields,
+)
 from synapse.api.errors import Codes, SynapseError
 from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler
 from synapse.metrics.background_process_metrics import run_as_background_process
@@ -756,6 +762,10 @@ async def _unsafe_refresh_remote_profiles_for_remote_server(
 
                 await self.store.update_profile_in_user_dir(
                     user_id,
-                    display_name=non_null_str_or_none(profile.get("displayname")),
-                    avatar_url=non_null_str_or_none(profile.get("avatar_url")),
+                    display_name=non_null_str_or_none(
+                        profile.get(ProfileFields.DISPLAYNAME)
+                    ),
+                    avatar_url=non_null_str_or_none(
+                        profile.get(ProfileFields.AVATAR_URL)
+                    ),
                 )

synapse/logging/scopecontextmanager.py

@@ -20,13 +20,10 @@
 #
 
 import logging
-from types import TracebackType
-from typing import Optional, Type
+from typing import Optional
 
 from opentracing import Scope, ScopeManager, Span
 
-import twisted
-
 from synapse.logging.context import (
     LoggingContext,
     current_context,
@@ -112,9 +109,6 @@ class _LogContextScope(Scope):
     """
     A custom opentracing scope, associated with a LogContext
 
-      * filters out _DefGen_Return exceptions which arise from calling
-        `defer.returnValue` in Twisted code
-
       * When the scope is closed, the logcontext's active scope is reset to None.
         and - if enter_logcontext was set - the logcontext is finished too.
     """
@@ -146,17 +140,6 @@ def __init__(
         self._finish_on_close = finish_on_close
         self._enter_logcontext = enter_logcontext
 
-    def __exit__(
-        self,
-        exc_type: Optional[Type[BaseException]],
-        value: Optional[BaseException],
-        traceback: Optional[TracebackType],
-    ) -> None:
-        if exc_type == twisted.internet.defer._DefGen_Return:
-            # filter out defer.returnValue() calls
-            exc_type = value = traceback = None
-        super().__exit__(exc_type, value, traceback)
-
     def __str__(self) -> str:
         return f"Scope<{self.span}>"