Optimizations and limits for StreamLit community app

* Removed timeit.

* Minor logging updates.

* Queue on community hosting.

* Map rollback.

* mypy update.

.dockerignore

@@ -17,4 +17,6 @@ run.ps1
 run.sh
 maps4fs.zip
 .DS_Store
-maps/
+maps/
+osmps/
+queue.json

.gitignore

@@ -14,4 +14,5 @@ maps/
 .pytest_cache/
 htmlcov/
 tests/data/
-osmps/
+osmps/
+queue.json

dev/clean_trash.ps1

@@ -1,8 +1,8 @@
 # Directories to be removed
-$dirs = @(".mypy_cache", ".pytest_cache", "htmlcov", "dist", "archives", "cache", "logs", "maps", "temp")
+$dirs = @(".mypy_cache", ".pytest_cache", "htmlcov", "dist", "archives", "cache", "logs", "maps", "temp", "osmps")
 
 # Files to be removed
-$files = @(".coverage")
+$files = @(".coverage", "queue.json")
 
 # Loop through the directories
 foreach ($dir in $dirs) {

dev/clean_trash.sh

@@ -1,10 +1,10 @@
 #!/bin/sh
 
 # Directories to be removed
-dirs=".mypy_cache .pytest_cache htmlcov dist archives cache logs maps temp"
+dirs=".mypy_cache .pytest_cache htmlcov dist archives cache logs maps temp osmps"
 
 # Files to be removed
-files=".coverage"
+files=".coverage queue.json"
 
 # Loop through the directories
 for dir in $dirs

maps4fs/generator/background.py

@@ -17,7 +17,6 @@
 )
 from maps4fs.generator.path_steps import DEFAULT_DISTANCE, PATH_FULL_NAME, get_steps
 from maps4fs.generator.tile import Tile
-from maps4fs.logger import timeit
 
 RESIZE_FACTOR = 1 / 4
 SIMPLIFY_FACTOR = 10
@@ -151,10 +150,9 @@ def generate_obj_files(self) -> None:
             self.logger.debug("Generating obj file for tile %s in path: %s", tile.code, save_path)
 
             dem_data = cv2.imread(tile.dem_path, cv2.IMREAD_UNCHANGED)  # pylint: disable=no-member
-            self.plane_from_np(tile.code, dem_data, save_path)
+            self.plane_from_np(tile.code, dem_data, save_path)  # type: ignore
 
     # pylint: disable=too-many-locals
-    @timeit
     def plane_from_np(self, tile_code: str, dem_data: np.ndarray, save_path: str) -> None:
         """Generates a 3D obj file based on DEM data.
 

maps4fs/generator/tile.py

@@ -34,13 +34,13 @@ def preprocess(self) -> None:
         if not self.code:
             raise ValueError("Tile code was not provided")
 
-        self.logger.debug(f"Generating tile {self.code}")
+        self.logger.debug("Generating tile for code %s", self.code)
 
         tiles_directory = os.path.join(self.map_directory, "objects", "tiles")
         os.makedirs(tiles_directory, exist_ok=True)
 
         self._dem_path = os.path.join(tiles_directory, f"{self.code}.png")
-        self.logger.debug(f"DEM path for tile {self.code} is {self._dem_path}")
+        self.logger.debug("DEM path for tile %s is %s", self.code, self._dem_path)
 
     def get_output_resolution(self) -> tuple[int, int]:
         """Return the resolution of the output image.

maps4fs/logger.py

@@ -4,9 +4,7 @@
 import os
 import sys
 from datetime import datetime
-from logging import getLogger
-from time import perf_counter
-from typing import Any, Callable, Literal
+from typing import Literal
 
 LOGGER_NAME = "maps4fs"
 log_directory = os.path.join(os.getcwd(), "logs")
@@ -46,25 +44,3 @@ def log_file(self) -> str:
         today = datetime.now().strftime("%Y-%m-%d")
         log_file = os.path.join(log_directory, f"{today}.txt")
         return log_file
-
-
-def timeit(func: Callable[..., Any]) -> Callable[..., Any]:
-    """Decorator to log the time taken by a function to execute.
-
-    Args:
-        func (function): The function to be timed.
-
-    Returns:
-        function: The timed function.
-    """
-
-    def timed(*args, **kwargs):
-        logger = getLogger("maps4fs")
-        start = perf_counter()
-        result = func(*args, **kwargs)
-        end = perf_counter()
-        if logger is not None:
-            logger.info("Function %s took %s seconds to execute", func.__name__, end - start)
-        return result
-
-    return timed

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "maps4fs"
-version = "0.9.94"
+version = "0.9.95"
 description = "Generate map templates for Farming Simulator from real places."
 authors = [{name = "iwatkot", email = "iwatkot@gmail.com"}]
 license = {text = "MIT License"}

webui/config.py

@@ -23,10 +23,19 @@
 MD_FILES = {"⛰️ DEM": "dem.md"}
 FAQ_MD = os.path.join(DOCS_DIRECTORY, "FAQ.md")
 
+QUEUE_FILE = os.path.join(WORKING_DIRECTORY, "queue.json")
+QUEUE_TIMEOUT = 600  # 10 minutes
+QUEUE_INTERVAL = 15
+
 REMOVE_DELAY = 300  # 5 minutes
 
 
 def get_mds() -> dict[str, str]:
+    """Get the paths to the Markdown files in the docs directory.
+
+    Returns:
+        dict[str, str]: The paths to the Markdown files in the docs directory.
+    """
     return {
         md_file: os.path.join(DOCS_DIRECTORY, filename) for md_file, filename in MD_FILES.items()
     }

webui/generator.py

@@ -6,6 +6,7 @@
 import streamlit as st
 import streamlit.components.v1 as components
 from PIL import Image
+from queuing import add_to_queue, remove_from_queue, wait_in_queue
 from streamlit_stl import stl_from_file
 from templates import Messages
 
@@ -340,7 +341,6 @@ def generate_map(self) -> None:
 
         session_name = self.get_sesion_name(coordinates)
 
-        # self.status_container.info("Starting...", icon="⏳")
         map_directory = os.path.join(config.MAPS_DIRECTORY, session_name)
         os.makedirs(map_directory, exist_ok=True)
 
@@ -359,36 +359,44 @@ def generate_map(self) -> None:
             light_version=self.community,
         )
 
-        step = int(100 / (len(game.components) + 2))
-        completed = 0
-        progress_bar = st.progress(0)
-        for component_name in mp.generate():
-            # self.status_container.info(f"Generating {component_name}...", icon="⏳")
-            progress_bar.progress(completed, f"⏳ Generating {component_name}...")
-            completed += step
+        if self.community:
+            add_to_queue(session_name)
+            for position in wait_in_queue(session_name):
+                self.status_container.info(
+                    f"Your position in the queue: {position}. Please wait...", icon="⏳"
+                )
 
-        # self.status_container.info("Creating previews...", icon="⏳")
+            self.status_container.info("Started the map generation...", icon="🔄")
 
-        completed += step
-        progress_bar.progress(completed, "🖼️ Creating previews...")
+        try:
+            step = int(100 / (len(game.components) + 2))
+            completed = 0
+            progress_bar = st.progress(0)
+            for component_name in mp.generate():
+                progress_bar.progress(completed, f"⏳ Generating {component_name}...")
+                completed += step
 
-        # Create a preview image.
-        self.show_preview(mp)
-        self.map_preview()
+            completed += step
+            progress_bar.progress(completed, "🖼️ Creating previews...")
 
-        completed += step
-        progress_bar.progress(completed, "🗃️ Packing the map...")
+            # Create a preview image.
+            self.show_preview(mp)
+            self.map_preview()
 
-        # self.status_container.info("Packing the map...", icon="⏳")
+            completed += step
+            progress_bar.progress(completed, "🗃️ Packing the map...")
 
-        # Pack the generated map into a zip archive.
-        archive_path = mp.pack(os.path.join(config.ARCHIVES_DIRECTORY, session_name))
+            # Pack the generated map into a zip archive.
+            archive_path = mp.pack(os.path.join(config.ARCHIVES_DIRECTORY, session_name))
 
-        self.download_path = archive_path
+            self.download_path = archive_path
 
-        st.session_state.generated = True
+            st.session_state.generated = True
 
-        self.status_container.success("Map generation completed!", icon="✅")
+            self.status_container.success("Map generation completed!", icon="✅")
+        finally:
+            if self.community:
+                remove_from_queue(session_name)
 
     def show_preview(self, mp: mfs.Map) -> None:
         """Show the preview of the generated map.

webui/queuing.py

@@ -0,0 +1,115 @@
+import json
+import os
+from time import sleep
+from typing import Generator
+
+from config import QUEUE_FILE, QUEUE_INTERVAL, QUEUE_TIMEOUT
+
+from maps4fs import Logger
+
+logger = Logger(level="DEBUG", to_file=False)
+
+
+def get_queue(force: bool = False) -> list[str]:
+    """Get the queue from the queue file.
+    If the queue file does not exist, create a new one with an empty queue.
+
+    Arguments:
+        force (bool): Whether to force the creation of a new queue file.
+
+    Returns:
+        list[dict[str, str]]: The queue.
+    """
+    if not os.path.isfile(QUEUE_FILE) or force:
+        logger.debug("Queue will be reset.")
+        save_queue([])
+        return []
+    with open(QUEUE_FILE, "r") as f:
+        return json.load(f)
+
+
+def save_queue(queue: list[str]) -> None:
+    """Save the queue to the queue file.
+
+    Arguments:
+        queue (list[str]): The queue to save to the queue file.
+    """
+    with open(QUEUE_FILE, "w") as f:
+        json.dump(queue, f)
+    logger.debug("Queue set to %s.", queue)
+
+
+def add_to_queue(session: str) -> None:
+    """Add a session to the queue.
+
+    Args:
+        session (str): The session to add to the queue.
+    """
+    queue = get_queue()
+    queue.append(session)
+    save_queue(queue)
+    logger.debug("Session %s added to the queue.", session)
+
+
+def get_first_item() -> str | None:
+    """Get the first item from the queue.
+
+    Returns:
+        str: The first item from the queue.
+    """
+    queue = get_queue()
+    if not queue:
+        return None
+    return queue[0]
+
+
+def get_position(session: str) -> int | None:
+    """Get the position of a session in the queue.
+
+    Args:
+        session (str): The session to get the position of.
+
+    Returns:
+        int: The position of the session in the queue.
+    """
+    queue = get_queue()
+    if session not in queue:
+        return None
+    return queue.index(session)
+
+
+def remove_from_queue(session: str) -> None:
+    """Remove a session from the queue.
+
+    Args:
+        session (str): The session to remove from the queue.
+    """
+    queue = get_queue()
+    if session in queue:
+        queue.remove(session)
+        save_queue(queue)
+        logger.debug("Session %s removed from the queue.", session)
+
+
+def wait_in_queue(session: str) -> Generator[int, None, None]:
+    """Wait in the queue until the session is the first item.
+
+    Args:
+        session (str): The session to wait for.
+    """
+    retries = QUEUE_TIMEOUT // QUEUE_INTERVAL
+    logger.debug(
+        "Starting to wait in the queue for session %s with maximum retries %d.", session, retries
+    )
+
+    for _ in range(retries):
+        position = get_position(session)
+        if position == 0 or position is None:
+            logger.debug("Session %s is the first item in the queue.", session)
+            return
+        logger.debug("Session %s is in position %d in the queue.", session, position)
+        yield position
+        sleep(QUEUE_INTERVAL)
+
+
+get_queue(force=True)