Merge branch 'features/lb'

Paweł Kędzia · Paweł Kędzia · commit fa3643355b0f · 2025-11-21T21:29:42.000+01:00
diff --git a/.version b/.version
@@ -1 +1 @@
-0.3.0
+0.3.1
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,4 +13,5 @@
 | 0.2.2   | Update dockerfile and requirements. Fix routing with vLLM.                                                                                                                                                                                                                                  |
 | 0.2.3   | New web configurator: Handling projects, configs for each user separately. First Available strategy is more powerful, a lot of improvements to efficiency.                                                                                                                                  |
 | 0.2.4   | Anonymizer module, integration anonymization with any endpoint (using dynamic payload analysis and full payload anonymisation), dedicated `/api/anonymize_text` endpoint as memory only anonymization. Whole router may be run in `FORCE_ANONYMISATION` mode.                               |
-| 0.3.0   | Anonymization available with three strategies: `fast_masker`, `genai`, `prov_masker`.                                                                                                                                                                                                       |
+| 0.3.0   | Anonymization available with three strategies: `fast_masker`, `genai`, `prov_masker`.                                                                                                                                                                                                       |
+| 0.3.1   | Refactoring `lb.strategies` to be more flexible modular.                                                                                                                                                                                                                                    |
diff --git a/llm_router_api/base/constants.py b/llm_router_api/base/constants.py
@@ -57,6 +57,9 @@
     f"{_DontChangeMe.MAIN_ENV_PREFIX}FORCE_ANONYMISATION"
 )
 
+FORCE_ANONYMISATION_ALGORITHM = None
+FORCE_ANONYMISATION_MODEL = None
+
 # Type of server, default is flask {flask, gunicorn, waitress}
 SERVER_TYPE = (
     os.environ.get(f"{_DontChangeMe.MAIN_ENV_PREFIX}SERVER_TYPE", "flask")
@@ -76,7 +79,7 @@
     ).strip()
 )
 
-# Number of threads (if server supports multithreading), default: 8
+# Number of threads (if the server supports multithreading), default: 8
 SERVER_THREADS_COUNT = int(
     os.environ.get(
         f"{_DontChangeMe.MAIN_ENV_PREFIX}SERVER_THREADS_COUNT", "8"
diff --git a/llm_router_api/base/constants_base.py b/llm_router_api/base/constants_base.py
@@ -26,3 +26,7 @@ class BalanceStrategies:
     BalanceStrategies.FIRST_AVAILABLE,
     BalanceStrategies.FIRST_AVAILABLE_OPTIM,
 ]
+
+#
+# DEFAULT_ANONYMIZE_STRATEGY = "fast_masker"
+# POSSIBLE_ANONYMIZE_STRATEGIES = ["fast_masker", "genai"]
diff --git a/llm_router_api/base/lb/provider_strategy_facase.py b/llm_router_api/base/lb/provider_strategy_facase.py
@@ -25,25 +25,28 @@
 from llm_router_api.base.constants import REST_API_LOG_LEVEL
 from llm_router_api.base.constants_base import BalanceStrategies
 
-from llm_router_api.base.lb.strategy import ChooseProviderStrategyI
-from llm_router_api.base.lb.first_available import FirstAvailableStrategy
-from llm_router_api.base.lb.first_available_optim import (
-    FirstAvailableOptimStrategy,
+from llm_router_api.base.lb.strategy_interface import ChooseProviderStrategyI
+from llm_router_api.base.lb.strategies.first_available import RedisBasedStrategy
+from llm_router_api.base.lb.strategies.first_available_optim import (
+    RedisBasedOptimStrategy,
 )
 
-from llm_router_api.base.lb.balanced import LoadBalancedStrategy
-from llm_router_api.base.lb.weighted import WeightedStrategy, DynamicWeightedStrategy
+from llm_router_api.base.lb.strategies.balanced import LoadBalancedStrategy
+from llm_router_api.base.lb.strategies.weighted import (
+    WeightedStrategy,
+    DynamicWeightedStrategy,
+)
 
 STRATEGIES = {
     BalanceStrategies.BALANCED: LoadBalancedStrategy,
     BalanceStrategies.WEIGHTED: WeightedStrategy,
     BalanceStrategies.DYNAMIC_WEIGHTED: DynamicWeightedStrategy,
-    BalanceStrategies.FIRST_AVAILABLE: FirstAvailableStrategy,
-    BalanceStrategies.FIRST_AVAILABLE_OPTIM: FirstAvailableOptimStrategy,
+    BalanceStrategies.FIRST_AVAILABLE: RedisBasedStrategy,
+    BalanceStrategies.FIRST_AVAILABLE_OPTIM: RedisBasedOptimStrategy,
 }
 
 
-class ProviderChooser:
+class ProviderStrategyFacade:
     """
     Facade for selecting a provider using a configurable load‑balancing strategy.
 
diff --git a/llm_router_api/base/lb/redis_based_interface.py b/llm_router_api/base/lb/redis_based_interface.py
@@ -1,7 +1,8 @@
 import random
-import time
 import logging
+
 from abc import ABC
+from typing import List, Dict, Optional, Any, Tuple
 
 try:
     import redis
@@ -10,14 +11,16 @@
 except ImportError:
     REDIS_IS_AVAILABLE = False
 
-from typing import List, Dict, Optional, Any, Tuple
-
 from llm_router_api.base.constants import REDIS_PORT, REDIS_HOST
-from llm_router_api.base.lb.strategy import ChooseProviderStrategyI
-from llm_router_api.base.lb.provider_monitor import RedisProviderMonitor
+from llm_router_api.base.lb.strategy_interface import ChooseProviderStrategyI
+from llm_router_api.base.monitor.redis_health_interface import (
+    RedisBasedHealthCheckInterface,
+)
 
 
-class FirstAvailableStrategyI(ChooseProviderStrategyI, ABC):
+class RedisBasedStrategyInterface(
+    ChooseProviderStrategyI, RedisBasedHealthCheckInterface, ABC
+):
     """
     Strategy that selects the first free provider for a model using Redis.
 
@@ -39,6 +42,7 @@ def __init__(
         redis_db: int = 0,
         timeout: int = 60,
         check_interval: float = 0.1,
+        monitor_check_interval: float = 30,
         clear_buffers: bool = True,
         logger: Optional[logging.Logger] = None,
         strategy_prefix: Optional[str] = "",
@@ -62,20 +66,29 @@ def __init__(
         check_interval : float, optional
             Time to sleep between checks for available providers (in seconds).
             Default is ``0.1``.
+        monitor_check_interval : float, optional
+            Time to sleep [in monitor module] between checks
+            for available providers (in seconds).
+            Default is ``0.1``.
         clear_buffers:
             Whether to clear all buffers when starting. Default is ``True``.
         """
-        if not REDIS_IS_AVAILABLE:
-            raise RuntimeError("Redis is not available. Please install it first.")
-
-        super().__init__(models_config_path=models_config_path, logger=logger)
+        ChooseProviderStrategyI.__init__(
+            self=self, models_config_path=models_config_path, logger=logger
+        )
 
-        self.redis_client = redis.Redis(
-            host=redis_host, port=redis_port, db=redis_db, decode_responses=True
+        RedisBasedHealthCheckInterface.__init__(
+            self=self,
+            redis_host=redis_host,
+            redis_port=redis_port,
+            redis_db=redis_db,
+            clear_buffers=clear_buffers,
+            logger=logger,
+            check_interval=monitor_check_interval,
         )
+
         self.timeout = timeout
         self.check_interval = check_interval
-
         self.strategy_prefix = strategy_prefix
 
         # Atomic acquire script – treat missing field as “available”
@@ -108,14 +121,6 @@ def __init__(
         if clear_buffers:
             self._clear_buffers()
 
-        # Start providers monitor
-        self._monitor = RedisProviderMonitor(
-            redis_client=self.redis_client,
-            check_interval=30,
-            clear_buffers=clear_buffers,
-            logger=self.logger,
-        )
-
     def init_provider(
         self,
         model_name: str,
diff --git a/llm_router_api/base/lb/strategies/__init__.py b/llm_router_api/base/lb/strategies/__init__.py
diff --git a/llm_router_api/base/lb/strategies/balanced.py b/llm_router_api/base/lb/strategies/balanced.py
@@ -3,7 +3,7 @@
 from collections import defaultdict
 from typing import List, Dict, Optional, Any
 
-from llm_router_api.base.lb.strategy import ChooseProviderStrategyI
+from llm_router_api.base.lb.strategy_interface import ChooseProviderStrategyI
 
 
 class LoadBalancedStrategy(ChooseProviderStrategyI):
diff --git a/llm_router_api/base/lb/strategies/beta/__init__.py b/llm_router_api/base/lb/strategies/beta/__init__.py
diff --git a/llm_router_api/base/lb/strategies/beta/adaptive.py b/llm_router_api/base/lb/strategies/beta/adaptive.py
@@ -1,6 +1,6 @@
 from typing import List, Dict
 
-from llm_router_api.base.lb.weighted import DynamicWeightedStrategy
+from llm_router_api.base.lb.strategies.weighted import DynamicWeightedStrategy
 
 
 class AdaptiveStrategy(DynamicWeightedStrategy):
diff --git a/llm_router_api/base/lb/strategies/first_available.py b/llm_router_api/base/lb/strategies/first_available.py
@@ -24,7 +24,6 @@
 
 import time
 import logging
-import random
 
 try:
     import redis
@@ -36,11 +35,10 @@
 from typing import List, Dict, Optional, Any
 
 from llm_router_api.base.constants import REDIS_PORT, REDIS_HOST
-from llm_router_api.base.lb.provider_monitor import RedisProviderMonitor
-from llm_router_api.base.lb.first_available_i import FirstAvailableStrategyI
+from llm_router_api.base.lb.redis_based_interface import RedisBasedStrategyInterface
 
 
-class FirstAvailableStrategy(FirstAvailableStrategyI):
+class RedisBasedStrategy(RedisBasedStrategyInterface):
     """
     Strategy that selects the first free provider for a model using Redis.
 
diff --git a/llm_router_api/base/lb/strategies/first_available_optim.py b/llm_router_api/base/lb/strategies/first_available_optim.py
@@ -37,10 +37,10 @@
 from typing import List, Dict, Optional, Any
 
 from llm_router_api.base.constants import REDIS_PORT, REDIS_HOST
-from llm_router_api.base.lb.first_available_i import FirstAvailableStrategyI
+from llm_router_api.base.lb.redis_based_interface import RedisBasedStrategyInterface
 
 
-class FirstAvailableOptimStrategy(FirstAvailableStrategyI):
+class RedisBasedOptimStrategy(RedisBasedStrategyInterface):
     """
     Strategy that selects the first free provider for a model using Redis.
 
diff --git a/llm_router_api/base/lb/strategies/weighted.py b/llm_router_api/base/lb/strategies/weighted.py
@@ -29,7 +29,7 @@
 from collections import defaultdict
 from typing import List, Dict, Optional, Any
 
-from llm_router_api.base.lb.strategy import ChooseProviderStrategyI
+from llm_router_api.base.lb.strategy_interface import ChooseProviderStrategyI
 
 
 class WeightedStrategy(ChooseProviderStrategyI):
diff --git a/llm_router_api/base/lb/strategy_interface.py b/llm_router_api/base/lb/strategy_interface.py
@@ -25,6 +25,9 @@ def __init__(
         )
         self.logger = logger
 
+        # Placeholder for any monitor, may be initialized in any provider
+        self._monitor = None
+
     def __str__(self):
         return str(self.__class__.__name__)
 
diff --git a/llm_router_api/base/model_handler.py b/llm_router_api/base/model_handler.py
@@ -10,8 +10,8 @@
 from dataclasses import dataclass
 from typing import Dict, Optional, Any
 
-from llm_router_api.base.lb.chooser import ProviderChooser
 from llm_router_api.base.model_config import ApiModelConfig
+from llm_router_api.base.lb.provider_strategy_facase import ProviderStrategyFacade
 
 
 @dataclass(frozen=True)
@@ -108,7 +108,9 @@ class ModelHandler:
 
     LIST_MODEL_FIELDS_REMOVE = ["model_path", "api_token"]
 
-    def __init__(self, models_config_path: str, provider_chooser: ProviderChooser):
+    def __init__(
+        self, models_config_path: str, provider_chooser: ProviderStrategyFacade
+    ):
         """
         Initialize the handler with the provided configuration path.
 
diff --git a/llm_router_api/base/monitor/__init__.py b/llm_router_api/base/monitor/__init__.py
diff --git a/llm_router_api/base/monitor/provider_monitor.py b/llm_router_api/base/monitor/provider_monitor.py
diff --git a/llm_router_api/base/monitor/redis_health_interface.py b/llm_router_api/base/monitor/redis_health_interface.py
@@ -0,0 +1,40 @@
+import logging
+
+from abc import ABC
+from typing import Optional
+
+try:
+    import redis
+
+    REDIS_IS_AVAILABLE = True
+except ImportError:
+    REDIS_IS_AVAILABLE = False
+
+from llm_router_api.base.constants import REDIS_PORT, REDIS_HOST
+from llm_router_api.base.monitor.provider_monitor import RedisProviderMonitor
+
+
+class RedisBasedHealthCheckInterface(ABC):
+    def __init__(
+        self,
+        redis_host: str = REDIS_HOST,
+        redis_port: int = REDIS_PORT,
+        redis_db: int = 0,
+        check_interval: float = 30,
+        clear_buffers: bool = True,
+        logger: Optional[logging.Logger] = None,
+    ):
+        if not REDIS_IS_AVAILABLE:
+            raise RuntimeError("Redis is not available. Please install it first.")
+
+        self.redis_client = redis.Redis(
+            host=redis_host, port=redis_port, db=redis_db, decode_responses=True
+        )
+
+        # Start providers monitor
+        self._monitor = RedisProviderMonitor(
+            redis_client=self.redis_client,
+            check_interval=check_interval,
+            clear_buffers=clear_buffers,
+            logger=logger,
+        )
diff --git a/llm_router_api/core/engine.py b/llm_router_api/core/engine.py
@@ -23,7 +23,6 @@
 from flask import Flask
 from typing import List, Type, Optional
 
-from llm_router_api.base.lb.chooser import ProviderChooser
 
 from llm_router_api.endpoints.endpoint_i import EndpointI
 from llm_router_api.register.auto_loader import EndpointAutoLoader
@@ -34,6 +33,7 @@
     USE_PROMETHEUS,
     SERVER_BALANCE_STRATEGY,
 )
+from llm_router_api.base.lb.provider_strategy_facase import ProviderStrategyFacade
 
 if USE_PROMETHEUS:
     from llm_router_api.core.metrics import PrometheusMetrics
@@ -104,7 +104,7 @@ def __init__(
         self.logger_level = logger_level
         self.logger_file_name = logger_file_name
 
-        self._provider_chooser = ProviderChooser(
+        self._provider_chooser = ProviderStrategyFacade(
             models_config_path=models_config_path,
             strategy_name=SERVER_BALANCE_STRATEGY,
             logger_level=logger_level,
diff --git a/llm_router_api/endpoints/endpoint_i.py b/llm_router_api/endpoints/endpoint_i.py
diff --git a/llm_router_api/register/auto_loader.py b/llm_router_api/register/auto_loader.py
diff --git a/llm_router_lib/data_models/constants.py b/llm_router_lib/data_models/constants.py

Original file line number	Diff line number	Diff line change
`@@ -26,3 +26,7 @@ class BalanceStrategies:`
`26`	`26`	`BalanceStrategies.FIRST_AVAILABLE,`
`27`	`27`	`BalanceStrategies.FIRST_AVAILABLE_OPTIM,`
`28`	`28`	`]`
	`29`	`+`
	`30`	`+#`
	`31`	`+# DEFAULT_ANONYMIZE_STRATEGY = "fast_masker"`
	`32`	`+# POSSIBLE_ANONYMIZE_STRATEGIES = ["fast_masker", "genai"]`