Merge remote-tracking branch 'upstream/main' into kh2-optimize-client

CommonClient.py

@@ -709,8 +709,16 @@ def handle_connection_loss(self, msg: str) -> None:
         logger.exception(msg, exc_info=exc_info, extra={'compact_gui': True})
         self._messagebox_connection_loss = self.gui_error(msg, exc_info[1])
 
-    def make_gui(self) -> typing.Type["kvui.GameManager"]:
-        """To return the Kivy App class needed for run_gui so it can be overridden before being built"""
+    def make_gui(self) -> "type[kvui.GameManager]":
+        """
+        To return the Kivy `App` class needed for `run_gui` so it can be overridden before being built
+
+        Common changes are changing `base_title` to update the window title of the client and
+        updating `logging_pairs` to automatically make new tabs that can be filled with their respective logger.
+
+        ex. `logging_pairs.append(("Foo", "Bar"))`
+        will add a "Bar" tab which follows the logger returned from `logging.getLogger("Foo")`
+        """
         from kvui import GameManager
 
         class TextManager(GameManager):

MultiServer.py

@@ -28,9 +28,11 @@
 
 if typing.TYPE_CHECKING:
     import ssl
+    from NetUtils import ServerConnection
 
-import websockets
 import colorama
+import websockets
+from websockets.extensions.permessage_deflate import PerMessageDeflate
 try:
     # ponyorm is a requirement for webhost, not default server, so may not be importable
     from pony.orm.dbapiprovider import OperationalError
@@ -119,13 +121,14 @@ def get_saving_second(seed_name: str, interval: int = 60) -> int:
 
 class Client(Endpoint):
     version = Version(0, 0, 0)
-    tags: typing.List[str] = []
+    tags: typing.List[str]
     remote_items: bool
     remote_start_inventory: bool
     no_items: bool
     no_locations: bool
+    no_text: bool
 
-    def __init__(self, socket: websockets.WebSocketServerProtocol, ctx: Context):
+    def __init__(self, socket: "ServerConnection", ctx: Context) -> None:
         super().__init__(socket)
         self.auth = False
         self.team = None
@@ -175,6 +178,7 @@ class Context:
                       "compatibility": int}
     # team -> slot id -> list of clients authenticated to slot.
     clients: typing.Dict[int, typing.Dict[int, typing.List[Client]]]
+    endpoints: list[Client]
     locations: LocationStore  # typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]]
     location_checks: typing.Dict[typing.Tuple[int, int], typing.Set[int]]
     hints_used: typing.Dict[typing.Tuple[int, int], int]
@@ -364,18 +368,28 @@ async def broadcast_send_encoded_msgs(self, endpoints: typing.Iterable[Endpoint]
             return True
 
     def broadcast_all(self, msgs: typing.List[dict]):
-        msgs = self.dumper(msgs)
-        endpoints = (endpoint for endpoint in self.endpoints if endpoint.auth)
-        async_start(self.broadcast_send_encoded_msgs(endpoints, msgs))
+        msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs)
+        data = self.dumper(msgs)
+        endpoints = (
+            endpoint
+            for endpoint in self.endpoints
+            if endpoint.auth and not (msg_is_text and endpoint.no_text)
+        )
+        async_start(self.broadcast_send_encoded_msgs(endpoints, data))
 
     def broadcast_text_all(self, text: str, additional_arguments: dict = {}):
         self.logger.info("Notice (all): %s" % text)
         self.broadcast_all([{**{"cmd": "PrintJSON", "data": [{ "text": text }]}, **additional_arguments}])
 
     def broadcast_team(self, team: int, msgs: typing.List[dict]):
-        msgs = self.dumper(msgs)
-        endpoints = (endpoint for endpoint in itertools.chain.from_iterable(self.clients[team].values()))
-        async_start(self.broadcast_send_encoded_msgs(endpoints, msgs))
+        msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs)
+        data = self.dumper(msgs)
+        endpoints = (
+            endpoint
+            for endpoint in itertools.chain.from_iterable(self.clients[team].values())
+            if not (msg_is_text and endpoint.no_text)
+        )
+        async_start(self.broadcast_send_encoded_msgs(endpoints, data))
 
     def broadcast(self, endpoints: typing.Iterable[Client], msgs: typing.List[dict]):
         msgs = self.dumper(msgs)
@@ -389,13 +403,13 @@ async def disconnect(self, endpoint: Client):
         await on_client_disconnected(self, endpoint)
 
     def notify_client(self, client: Client, text: str, additional_arguments: dict = {}):
-        if not client.auth:
+        if not client.auth or client.no_text:
             return
         self.logger.info("Notice (Player %s in team %d): %s" % (client.name, client.team + 1, text))
         async_start(self.send_msgs(client, [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments}]))
 
     def notify_client_multiple(self, client: Client, texts: typing.List[str], additional_arguments: dict = {}):
-        if not client.auth:
+        if not client.auth or client.no_text:
             return
         async_start(self.send_msgs(client,
                                    [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments}
@@ -760,7 +774,7 @@ def notify_hints(self, team: int, hints: typing.List[Hint], only_new: bool = Fal
             self.on_new_hint(team, slot)
         for slot, hint_data in concerns.items():
             if recipients is None or slot in recipients:
-                clients = self.clients[team].get(slot)
+                clients = filter(lambda c: not c.no_text, self.clients[team].get(slot, []))
                 if not clients:
                     continue
                 client_hints = [datum[1] for datum in sorted(hint_data, key=lambda x: x[0].finding_player != slot)]
@@ -819,7 +833,7 @@ def update_aliases(ctx: Context, team: int):
             async_start(ctx.send_encoded_msgs(client, cmd))
 
 
-async def server(websocket, path: str = "/", ctx: Context = None):
+async def server(websocket: "ServerConnection", path: str = "/", ctx: Context = None) -> None:
     client = Client(websocket, ctx)
     ctx.endpoints.append(client)
 
@@ -910,6 +924,10 @@ async def on_client_joined(ctx: Context, client: Client):
                               "If your client supports it, "
                               "you may have additional local commands you can list with /help.",
                       {"type": "Tutorial"})
+    if not any(isinstance(extension, PerMessageDeflate) for extension in client.socket.extensions):
+        ctx.notify_client(client, "Warning: your client does not support compressed websocket connections! "
+                                  "It may stop working in the future. If you are a player, please report this to the "
+                                  "client's developer.")
     ctx.client_connection_timers[client.team, client.slot] = datetime.datetime.now(datetime.timezone.utc)
 
 
@@ -1060,21 +1078,37 @@ def send_items_to(ctx: Context, team: int, target_slot: int, *items: NetworkItem
 
 def register_location_checks(ctx: Context, team: int, slot: int, locations: typing.Iterable[int],
                              count_activity: bool = True):
+    slot_locations = ctx.locations[slot]
     new_locations = set(locations) - ctx.location_checks[team, slot]
-    new_locations.intersection_update(ctx.locations[slot])  # ignore location IDs unknown to this multidata
+    new_locations.intersection_update(slot_locations)  # ignore location IDs unknown to this multidata
     if new_locations:
         if count_activity:
             ctx.client_activity_timers[team, slot] = datetime.datetime.now(datetime.timezone.utc)
+
+        sortable: list[tuple[int, int, int, int]] = []
         for location in new_locations:
-            item_id, target_player, flags = ctx.locations[slot][location]
+            # extract all fields to avoid runtime overhead in LocationStore
+            item_id, target_player, flags = slot_locations[location]
+            # sort/group by receiver and item
+            sortable.append((target_player, item_id, location, flags))
+
+        info_texts: list[dict[str, typing.Any]] = []
+        for target_player, item_id, location, flags in sorted(sortable):
             new_item = NetworkItem(item_id, location, slot, flags)
             send_items_to(ctx, team, target_player, new_item)
 
             ctx.logger.info('(Team #%d) %s sent %s to %s (%s)' % (
                 team + 1, ctx.player_names[(team, slot)], ctx.item_names[ctx.slot_info[target_player].game][item_id],
                 ctx.player_names[(team, target_player)], ctx.location_names[ctx.slot_info[slot].game][location]))
-            info_text = json_format_send_event(new_item, target_player)
-            ctx.broadcast_team(team, [info_text])
+            if len(info_texts) >= 140:
+                # split into chunks that are close to compression window of 64K but not too big on the wire
+                # (roughly 1300-2600 bytes after compression depending on repetitiveness)
+                ctx.broadcast_team(team, info_texts)
+                info_texts.clear()
+            info_texts.append(json_format_send_event(new_item, target_player))
+        ctx.broadcast_team(team, info_texts)
+        del info_texts
+        del sortable
 
         ctx.location_checks[team, slot] |= new_locations
         send_new_items(ctx)
@@ -1787,7 +1821,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
             ctx.clients[team][slot].append(client)
             client.version = args['version']
             client.tags = args['tags']
-            client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags
+            client.no_locations = "TextOnly" in client.tags or "Tracker" in client.tags
+            # set NoText for old PopTracker clients that predate the tag to save traffic
+            client.no_text = "NoText" in client.tags or ("PopTracker" in client.tags and client.version < (0, 5, 1))
             connected_packet = {
                 "cmd": "Connected",
                 "team": client.team, "slot": client.slot,
@@ -1860,6 +1896,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
                 client.tags = args["tags"]
                 if set(old_tags) != set(client.tags):
                     client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags
+                    client.no_text = "NoText" in client.tags or (
+                        "PopTracker" in client.tags and client.version < (0, 5, 1)
+                    )
                     ctx.broadcast_text_all(
                         f"{ctx.get_aliased_name(client.team, client.slot)} (Team #{client.team + 1}) has changed tags "
                         f"from {old_tags} to {client.tags}.",

NetUtils.py

@@ -5,7 +5,8 @@
 import warnings
 from json import JSONEncoder, JSONDecoder
 
-import websockets
+if typing.TYPE_CHECKING:
+    from websockets import WebSocketServerProtocol as ServerConnection
 
 from Utils import ByValue, Version
 
@@ -151,7 +152,7 @@ def _object_hook(o: typing.Any) -> typing.Any:
 
 
 class Endpoint:
-    socket: websockets.WebSocketServerProtocol
+    socket: "ServerConnection"
 
     def __init__(self, socket):
         self.socket = socket

docs/network protocol.md

@@ -47,6 +47,9 @@ Packets are simple JSON lists in which any number of ordered network commands ca
 
 An object can contain the "class" key, which will tell the content data type, such as "Version" in the following example.
 
+Websocket connections should support per-message compression. Uncompressed connections are deprecated and may stop
+working in the future.
+
 Example:
 ```javascript
 [{"cmd": "RoomInfo", "version": {"major": 0, "minor": 1, "build": 3, "class": "Version"}, "tags": ["WebHost"], ... }]
@@ -745,6 +748,7 @@ Tags are represented as a list of strings, the common client tags follow:
 | HintGame  | Indicates the client is a hint game, made to send hints instead of locations. Special join/leave message,¹ `game` is optional.²      |
 | Tracker   | Indicates the client is a tracker, made to track instead of sending locations. Special join/leave message,¹ `game` is optional.²     |
 | TextOnly  | Indicates the client is a basic client, made to chat instead of sending locations. Special join/leave message,¹ `game` is optional.² |
+| NoText    | Indicates the client does not want to receive text messages, improving performance if not needed.                                    |
 
 ¹: When connecting or disconnecting, the chat message shows e.g. "tracking".\
 ²: Allows `game` to be empty or null in [Connect](#connect). Game and version validation will then be skipped.

test/general/test_memory.py

@@ -1,5 +1,6 @@
 import unittest
 
+from BaseClasses import MultiWorld
 from worlds.AutoWorld import AutoWorldRegister
 from . import setup_solo_multiworld
 
@@ -9,8 +10,12 @@ def test_leak(self) -> None:
         """Tests that worlds don't leak references to MultiWorld or themselves with default options."""
         import gc
         import weakref
+        refs: dict[str, weakref.ReferenceType[MultiWorld]] = {}
         for game_name, world_type in AutoWorldRegister.world_types.items():
-            with self.subTest("Game", game_name=game_name):
+            with self.subTest("Game creation", game_name=game_name):
                 weak = weakref.ref(setup_solo_multiworld(world_type))
-                gc.collect()
+                refs[game_name] = weak
+        gc.collect()
+        for game_name, weak in refs.items():
+            with self.subTest("Game cleanup", game_name=game_name):
                 self.assertFalse(weak(), "World leaked a reference")

worlds/LauncherComponents.py

@@ -87,14 +87,22 @@ def __repr__(self):
 processes = weakref.WeakSet()
 
 
-def launch_subprocess(func: Callable, name: str = None, args: Tuple[str, ...] = ()) -> None:
+def launch_subprocess(func: Callable, name: str | None = None, args: Tuple[str, ...] = ()) -> None:
     global processes
     import multiprocessing
     process = multiprocessing.Process(target=func, name=name, args=args)
     process.start()
     processes.add(process)
 
 
+def launch(func: Callable, name: str | None = None, args: Tuple[str, ...] = ()) -> None:
+    from Utils import is_kivy_running
+    if is_kivy_running():
+        launch_subprocess(func, name, args)
+    else:
+        func(*args)
+
+
 class SuffixIdentifier:
     suffixes: Iterable[str]
 
@@ -111,7 +119,7 @@ def __call__(self, path: str) -> bool:
 
 def launch_textclient(*args):
     import CommonClient
-    launch_subprocess(CommonClient.run_as_textclient, name="TextClient", args=args)
+    launch(CommonClient.run_as_textclient, name="TextClient", args=args)
 
 
 def _install_apworld(apworld_src: str = "") -> Optional[Tuple[pathlib.Path, pathlib.Path]]:

worlds/_bizhawk/README.md

@@ -55,6 +55,7 @@ async def lock(ctx) -> None
 async def unlock(ctx) -> None
 
 async def get_hash(ctx) -> str
+async def get_memory_size(ctx, domain: str) -> int
 async def get_system(ctx) -> str
 async def get_cores(ctx) -> dict[str, str]
 async def ping(ctx) -> None
@@ -168,9 +169,10 @@ select dialog and they will be associated with BizHawkClient. This does not affe
 associate the file extension with Archipelago.
 
 `validate_rom` is called to figure out whether a given ROM belongs to your client. It will only be called when a ROM is
-running on a system you specified in your `system` class variable. In most cases, that will be a single system and you
-can be sure that you're not about to try to read from nonexistent domains or out of bounds. If you decide to claim this
-ROM as yours, this is where you should do setup for things like `items_handling`.
+running on a system you specified in your `system` class variable. Take extra care here, because your code will run
+against ROMs that you have no control over. If you're reading an address deep in ROM, you might want to check the size
+of ROM before you attempt to read it using `get_memory_size`. If you decide to claim this ROM as yours, this is where
+you should do setup for things like `items_handling`.
 
 `game_watcher` is the "main loop" of your client where you should be checking memory and sending new items to the ROM.
 `BizHawkClient` will make sure that your `game_watcher` only runs when your client has validated the ROM, and will do
@@ -268,6 +270,8 @@ server connection before trying to interact with it.
 - By default, the player will be asked to provide their slot name after connecting to the server and validating, and
 that input will be used to authenticate with the `Connect` command. You can override `set_auth` in your own client to
 set it automatically based on data in the ROM or on your client instance.
+- Use `get_memory_size` inside `validate_rom` if you need to read at large addresses, in case some other game has a
+smaller ROM size.
 - You can override `on_package` in your client to watch raw packages, but don't forget you also have access to a
 subclass of `CommonContext` and its API.
 - You can import `BizHawkClientContext` for type hints using `typing.TYPE_CHECKING`. Importing it without conditions at

worlds/factorio/Options.py

@@ -304,6 +304,11 @@ class EvolutionTrapIncrease(Range):
     range_end = 100
 
 
+class InventorySpillTrapCount(TrapCount):
+    """Trap items that when received trigger dropping your main inventory and trash inventory onto the ground."""
+    display_name = "Inventory Spill Traps"
+
+
 class FactorioWorldGen(OptionDict):
     """World Generation settings. Overview of options at https://wiki.factorio.com/Map_generator,
     with in-depth documentation at https://lua-api.factorio.com/latest/Concepts.html#MapGenSettings"""
@@ -484,6 +489,7 @@ class FactorioOptions(PerGameCommonOptions):
     artillery_traps: ArtilleryTrapCount
     atomic_rocket_traps: AtomicRocketTrapCount
     atomic_cliff_remover_traps: AtomicCliffRemoverTrapCount
+    inventory_spill_traps: InventorySpillTrapCount
     attack_traps: AttackTrapCount
     evolution_traps: EvolutionTrapCount
     evolution_trap_increase: EvolutionTrapIncrease
@@ -518,6 +524,7 @@ class FactorioOptions(PerGameCommonOptions):
             ArtilleryTrapCount,
             AtomicRocketTrapCount,
             AtomicCliffRemoverTrapCount,
+            InventorySpillTrapCount,
         ],
         start_collapsed=True
     ),