From 1f930a5f8f10e687e690490da9388d4f4c7b6c56 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Sun, 29 Sep 2024 22:19:05 +0400 Subject: [PATCH 1/6] Update layer.py Following a request to be able to update route thresholds directly without having to fit. --- semantic_router/layer.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index bed1ef28..6c9950ff 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -832,6 +832,22 @@ def _vec_evaluate(self, Xq: Union[List[float], Any], y: List[str]) -> float: def _get_route_names(self) -> List[str]: return [route.name for route in self.routes] + + def update_route_thresholds(self, threshold_dict: Dict[str, float]): + """ + Update thresholds for specified routes. + + :param threshold_dict: A dictionary mapping route names to new threshold values. + :type threshold_dict: Dict[str, float] + """ + for route_name, new_threshold in threshold_dict.items(): + route = self.get(route_name) + if route: + old_threshold = route.score_threshold + route.score_threshold = new_threshold + logger.info(f"Updated threshold for route '{route_name}' from {old_threshold} to {new_threshold}") + else: + logger.warning(f"Route '{route_name}' not found. Skipping threshold update.") def threshold_random_search( From 6cb48dd4d388d36c82e4090e6a203bbce3eeafa4 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Sun, 29 Sep 2024 22:22:00 +0400 Subject: [PATCH 2/6] Update layer.py After a discussion with James we decided to just alter the existing update function. --- semantic_router/layer.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 6c9950ff..4512207f 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -177,7 +177,7 @@ class RouteLayer: index: BaseIndex def __init__( - self, + self encoder: Optional[BaseEncoder] = None, llm: Optional[BaseLLM] = None, routes: Optional[List[Route]] = None, @@ -832,22 +832,6 @@ def _vec_evaluate(self, Xq: Union[List[float], Any], y: List[str]) -> float: def _get_route_names(self) -> List[str]: return [route.name for route in self.routes] - - def update_route_thresholds(self, threshold_dict: Dict[str, float]): - """ - Update thresholds for specified routes. - - :param threshold_dict: A dictionary mapping route names to new threshold values. - :type threshold_dict: Dict[str, float] - """ - for route_name, new_threshold in threshold_dict.items(): - route = self.get(route_name) - if route: - old_threshold = route.score_threshold - route.score_threshold = new_threshold - logger.info(f"Updated threshold for route '{route_name}' from {old_threshold} to {new_threshold}") - else: - logger.warning(f"Route '{route_name}' not found. Skipping threshold update.") def threshold_random_search( From 8b86773a7ff488f77aeaa8c87a76f95940b53132 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Sun, 29 Sep 2024 23:37:00 +0400 Subject: [PATCH 3/6] Update layer.py --- semantic_router/layer.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 4512207f..778af03d 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -177,7 +177,7 @@ class RouteLayer: index: BaseIndex def __init__( - self + self, encoder: Optional[BaseEncoder] = None, llm: Optional[BaseLLM] = None, routes: Optional[List[Route]] = None, @@ -435,8 +435,29 @@ def add(self, route: Route): def list_route_names(self) -> List[str]: return [route.name for route in self.routes] - def update(self, route_name: str, utterances: List[str]): - raise NotImplementedError("This method has not yet been implemented.") + def update(self, name: str, threshold: Optional[float] = None, utterances: Optional[List[str]] = None): + """Updates the route specified in name. Allows the update of + threshold and/or utterances. If no values are provided via the + threshold or utterances parameters, those fields are not updated. + If neither field is provided raises a ValueError. + + The name must exist within the local RouteLayer, if not a + KeyError will be raised. + """ + + if threshold is None and utterances is None: + raise ValueError("At least one of 'threshold' or 'utterances' must be provided.") + if utterances: + raise NotImplementedError("The update method cannot be used for updating utterances yet.") + + route = self.get(name) + if route: + if threshold: + old_threshold = route.score_threshold + route.score_threshold = threshold + logger.info(f"Updated threshold for route '{route.name}' from {old_threshold} to {threshold}") + else: + raise ValueError(f"Route '{name}' not found. Nothing updated.") def delete(self, route_name: str): """Deletes a route given a specific route name. From 0866dccd7997eff0a9d74544a4b2218694d8f904 Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Mon, 30 Sep 2024 15:12:10 +0400 Subject: [PATCH 4/6] Linting. --- semantic_router/layer.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 778af03d..158c12be 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -435,7 +435,12 @@ def add(self, route: Route): def list_route_names(self) -> List[str]: return [route.name for route in self.routes] - def update(self, name: str, threshold: Optional[float] = None, utterances: Optional[List[str]] = None): + def update( + self, + name: str, + threshold: Optional[float] = None, + utterances: Optional[List[str]] = None, + ): """Updates the route specified in name. Allows the update of threshold and/or utterances. If no values are provided via the threshold or utterances parameters, those fields are not updated. @@ -446,16 +451,22 @@ def update(self, name: str, threshold: Optional[float] = None, utterances: Optio """ if threshold is None and utterances is None: - raise ValueError("At least one of 'threshold' or 'utterances' must be provided.") + raise ValueError( + "At least one of 'threshold' or 'utterances' must be provided." + ) if utterances: - raise NotImplementedError("The update method cannot be used for updating utterances yet.") - + raise NotImplementedError( + "The update method cannot be used for updating utterances yet." + ) + route = self.get(name) if route: if threshold: old_threshold = route.score_threshold route.score_threshold = threshold - logger.info(f"Updated threshold for route '{route.name}' from {old_threshold} to {threshold}") + logger.info( + f"Updated threshold for route '{route.name}' from {old_threshold} to {threshold}" + ) else: raise ValueError(f"Route '{name}' not found. Nothing updated.") From 0586c670c244b064197ce5442b97db2cccd8538c Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Mon, 30 Sep 2024 15:35:24 +0400 Subject: [PATCH 5/6] More pytests. --- tests/unit/test_layer.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 12773995..8b985c13 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -811,6 +811,36 @@ def test_refresh_routes_not_implemented(self, openai_encoder, routes, index_cls) ): route_layer._refresh_routes() + def test_update_threshold(self, openai_encoder, routes, index_cls): + index = init_index(index_cls) + route_layer = RouteLayer(encoder=openai_encoder, routes=routes, index=index) + route_name = "Route 1" + new_threshold = 0.8 + route_layer.update(name=route_name, threshold=new_threshold) + updated_route = route_layer.get(route_name) + assert updated_route.score_threshold == new_threshold, f"Expected threshold to be updated to {new_threshold}, but got {updated_route.score_threshold}" + + def test_update_non_existent_route(self, openai_encoder, routes, index_cls): + index = init_index(index_cls) + route_layer = RouteLayer(encoder=openai_encoder, routes=routes, index=index) + non_existent_route = "Non-existent Route" + with pytest.raises(ValueError, match=f"Route '{non_existent_route}' not found. Nothing updated."): + route_layer.update(name=non_existent_route, threshold=0.7) + + def test_update_without_parameters(self, openai_encoder, routes, index_cls): + index = init_index(index_cls) + route_layer = RouteLayer(encoder=openai_encoder, routes=routes, index=index) + with pytest.raises(ValueError, match="At least one of 'threshold' or 'utterances' must be provided."): + route_layer.update(name="Route 1") + + def test_update_utterances_not_implemented(self, openai_encoder, routes, index_cls): + index = init_index(index_cls) + route_layer = RouteLayer(encoder=openai_encoder, routes=routes, index=index) + with pytest.raises(NotImplementedError, match="The update method cannot be used for updating utterances yet."): + route_layer.update(name="Route 1", utterances=["New utterance"]) + + + class TestLayerFit: def test_eval(self, openai_encoder, routes, test_data): @@ -948,3 +978,4 @@ def test_semantic_classify_multiple_routes_with_different_aggregation( elif agg == "max": assert classification == "Route 3" assert score == [0.1, 1.0] + From e6f14149a2ab225fe7fd26296d9fff6cbd1e0f4f Mon Sep 17 00:00:00 2001 From: Siraj R Aizlewood Date: Mon, 30 Sep 2024 15:38:10 +0400 Subject: [PATCH 6/6] Linting. --- tests/unit/test_layer.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py index 8b985c13..3bfcd485 100644 --- a/tests/unit/test_layer.py +++ b/tests/unit/test_layer.py @@ -818,30 +818,39 @@ def test_update_threshold(self, openai_encoder, routes, index_cls): new_threshold = 0.8 route_layer.update(name=route_name, threshold=new_threshold) updated_route = route_layer.get(route_name) - assert updated_route.score_threshold == new_threshold, f"Expected threshold to be updated to {new_threshold}, but got {updated_route.score_threshold}" + assert ( + updated_route.score_threshold == new_threshold + ), f"Expected threshold to be updated to {new_threshold}, but got {updated_route.score_threshold}" def test_update_non_existent_route(self, openai_encoder, routes, index_cls): index = init_index(index_cls) route_layer = RouteLayer(encoder=openai_encoder, routes=routes, index=index) non_existent_route = "Non-existent Route" - with pytest.raises(ValueError, match=f"Route '{non_existent_route}' not found. Nothing updated."): + with pytest.raises( + ValueError, + match=f"Route '{non_existent_route}' not found. Nothing updated.", + ): route_layer.update(name=non_existent_route, threshold=0.7) def test_update_without_parameters(self, openai_encoder, routes, index_cls): index = init_index(index_cls) route_layer = RouteLayer(encoder=openai_encoder, routes=routes, index=index) - with pytest.raises(ValueError, match="At least one of 'threshold' or 'utterances' must be provided."): + with pytest.raises( + ValueError, + match="At least one of 'threshold' or 'utterances' must be provided.", + ): route_layer.update(name="Route 1") def test_update_utterances_not_implemented(self, openai_encoder, routes, index_cls): index = init_index(index_cls) route_layer = RouteLayer(encoder=openai_encoder, routes=routes, index=index) - with pytest.raises(NotImplementedError, match="The update method cannot be used for updating utterances yet."): + with pytest.raises( + NotImplementedError, + match="The update method cannot be used for updating utterances yet.", + ): route_layer.update(name="Route 1", utterances=["New utterance"]) - - class TestLayerFit: def test_eval(self, openai_encoder, routes, test_data): route_layer = RouteLayer(encoder=openai_encoder, routes=routes) @@ -978,4 +987,3 @@ def test_semantic_classify_multiple_routes_with_different_aggregation( elif agg == "max": assert classification == "Route 3" assert score == [0.1, 1.0] -