From 748780c53227ba401616d8beb562cb4d8ba9b689 Mon Sep 17 00:00:00 2001 From: Vittorio Date: Fri, 23 Aug 2024 14:05:06 +0200 Subject: [PATCH 01/13] Implemented aget_routes async method for pinecone index --- semantic_router/index/base.py | 11 +++ semantic_router/index/local.py | 3 + semantic_router/index/pinecone.py | 113 ++++++++++++++++++++++++++++++ semantic_router/index/postgres.py | 4 ++ semantic_router/index/qdrant.py | 3 + 5 files changed, 134 insertions(+) diff --git a/semantic_router/index/base.py b/semantic_router/index/base.py index d0f12ac6..a23f92bf 100644 --- a/semantic_router/index/base.py +++ b/semantic_router/index/base.py @@ -90,6 +90,17 @@ async def aquery( """ raise NotImplementedError("This method should be implemented by subclasses.") + def aget_routes(self): + """ + Asynchronously get a list of route and utterance objects currently stored in the index. + This method should be implemented by subclasses. + + :returns: A list of tuples, each containing a route name and an associated utterance. + :rtype: list[tuple] + :raises NotImplementedError: If the method is not implemented by the subclass. + """ + raise NotImplementedError("This method should be implemented by subclasses.") + def delete_index(self): """ Deletes or resets the index. diff --git a/semantic_router/index/local.py b/semantic_router/index/local.py index 7150b267..f2398618 100644 --- a/semantic_router/index/local.py +++ b/semantic_router/index/local.py @@ -128,6 +128,9 @@ async def aquery( route_names = [self.routes[i] for i in idx] return scores, route_names + def aget_routes(self): + logger.error("Sync remove is not implemented for LocalIndex.") + def delete(self, route_name: str): """ Delete all records of a specific route from the index. diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index b4f6033f..5bb4b4aa 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -528,6 +528,18 @@ async def aquery( route_names = [result["metadata"]["sr_route"] for result in results["matches"]] return np.array(scores), route_names + async def aget_routes(self) -> list[tuple]: + """ + Asynchronously get a list of route and utterance objects currently stored in the index. + + Returns: + List[Tuple]: A list of (route_name, utterance) objects. + """ + if self.async_client is None or self.host is None: + raise ValueError("Async client or host are not initialized.") + + return await self._async_get_routes() + def delete_index(self): self.client.delete_index(self.index_name) @@ -584,5 +596,106 @@ async def _async_describe_index(self, name: str): async with self.async_client.get(f"{self.base_url}/indexes/{name}") as response: return await response.json(content_type=None) + async def _async_get_all( + self, prefix: str | None = None, include_metadata: bool = False + ) -> tuple[list[str], list[dict]]: + """ + Retrieves all vector IDs from the Pinecone index using pagination asynchronously. + """ + if self.index is None: + raise ValueError("Index is None, could not retrieve vector IDs.") + + all_vector_ids = [] + next_page_token = None + + if prefix: + prefix_str = f"?prefix={prefix}" + else: + prefix_str = "" + + list_url = f"https://{self.host}/vectors/list{prefix_str}" + params: dict = {} + if self.namespace: + params["namespace"] = self.namespace + metadata = [] + + while True: + if next_page_token: + params["paginationToken"] = next_page_token + + async with self.async_client.get( + list_url, params=params, headers={"Api-Key": self.api_key} + ) as response: + if response.status != 200: + error_text = await response.text() + print(f"Error listing vectors: {response.status} - {error_text}") + break + + response_data = await response.json(content_type=None) + + vector_ids = [vec["id"] for vec in response_data.get("vectors", [])] + if not vector_ids: + break + all_vector_ids.extend(vector_ids) + + if include_metadata: + metadata_tasks = [self._async_fetch_metadata(id) for id in vector_ids] + metadata_results = await asyncio.gather(*metadata_tasks) + metadata.extend(metadata_results) + + next_page_token = response_data.get("pagination", {}).get("next") + if not next_page_token: + break + + return all_vector_ids, metadata + + async def _async_fetch_metadata(self, vector_id: str) -> dict: + """ + Fetch metadata for a single vector ID asynchronously using the async_client. + """ + url = f"https://{self.host}/vectors/fetch" + + params = { + "ids": [vector_id], + } + + headers = { + "Api-Key": self.api_key, + } + + async with self.async_client.get( + url, params=params, headers=headers + ) as response: + if response.status != 200: + error_text = await response.text() + print( + f"Error fetching metadata for vector {vector_id}: {response.status} - {error_text}" + ) + return {} + + try: + response_data = await response.json(content_type=None) + print(f"RESPONSE: {response_data}") + except Exception as e: + print(f"Failed to decode JSON for vector {vector_id}: {e}") + return {} + + return ( + response_data.get("vectors", {}).get(vector_id, {}).get("metadata", {}) + ) + + async def _async_get_routes(self) -> list[tuple]: + """ + Gets a list of route and utterance objects currently stored in the index. + + Returns: + List[Tuple]: A list of (route_name, utterance) objects. + """ + _, metadata = await self._async_get_all(include_metadata=True) + print("AAAAAAAAAAAAAAAAAAA") + print(metadata) + route_tuples = [(x["sr_route"], x["sr_utterance"]) for x in metadata] + return route_tuples + def __len__(self): return self.index.describe_index_stats()["total_vector_count"] diff --git a/semantic_router/index/postgres.py b/semantic_router/index/postgres.py index 4c971d4d..52afdeec 100644 --- a/semantic_router/index/postgres.py +++ b/semantic_router/index/postgres.py @@ -9,6 +9,7 @@ from semantic_router.index.base import BaseIndex from semantic_router.schema import Metric +from semantic_router.utils.logger import logger class MetricPgVecOperatorMap(Enum): @@ -456,6 +457,9 @@ def delete_index(self) -> None: cur.execute(f"DROP TABLE IF EXISTS {table_name}") self.conn.commit() + def aget_routes(self): + logger.error("Sync remove is not implemented for PostgresIndex.") + def __len__(self): """ Returns the total number of vectors in the index. diff --git a/semantic_router/index/qdrant.py b/semantic_router/index/qdrant.py index c1a5e28b..3077da33 100644 --- a/semantic_router/index/qdrant.py +++ b/semantic_router/index/qdrant.py @@ -317,6 +317,9 @@ async def aquery( route_names = [result.payload[SR_ROUTE_PAYLOAD_KEY] for result in results] return np.array(scores), route_names + def aget_routes(self): + logger.error("Sync remove is not implemented for QdrantIndex.") + def delete_index(self): self.client.delete_collection(self.index_name) From 15560be57605188deb48d0d27a098e562687e4fa Mon Sep 17 00:00:00 2001 From: James Briggs <35938317+jamescalam@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:39:13 +0200 Subject: [PATCH 02/13] Update semantic_router/index/pinecone.py --- semantic_router/index/pinecone.py | 1 - 1 file changed, 1 deletion(-) diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index 5bb4b4aa..8102881d 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -692,7 +692,6 @@ async def _async_get_routes(self) -> list[tuple]: List[Tuple]: A list of (route_name, utterance) objects. """ _, metadata = await self._async_get_all(include_metadata=True) - print("AAAAAAAAAAAAAAAAAAA") print(metadata) route_tuples = [(x["sr_route"], x["sr_utterance"]) for x in metadata] return route_tuples From 4c8020d3b0e0bd24408ac761ff8922ddd8b875f3 Mon Sep 17 00:00:00 2001 From: James Briggs <35938317+jamescalam@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:39:38 +0200 Subject: [PATCH 03/13] Update semantic_router/index/pinecone.py --- semantic_router/index/pinecone.py | 1 - 1 file changed, 1 deletion(-) diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index 8102881d..98dcf4f4 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -628,7 +628,6 @@ async def _async_get_all( ) as response: if response.status != 200: error_text = await response.text() - print(f"Error listing vectors: {response.status} - {error_text}") break response_data = await response.json(content_type=None) From a22b1d17d95208093b409edb3baae6d7dedba7f3 Mon Sep 17 00:00:00 2001 From: James Briggs <35938317+jamescalam@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:40:55 +0200 Subject: [PATCH 04/13] Update semantic_router/index/pinecone.py --- semantic_router/index/pinecone.py | 1 - 1 file changed, 1 deletion(-) diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index 98dcf4f4..38f197ba 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -691,7 +691,6 @@ async def _async_get_routes(self) -> list[tuple]: List[Tuple]: A list of (route_name, utterance) objects. """ _, metadata = await self._async_get_all(include_metadata=True) - print(metadata) route_tuples = [(x["sr_route"], x["sr_utterance"]) for x in metadata] return route_tuples From 9ab0ff97e7f2e0f440e92ce2b1cb260e9dfa6911 Mon Sep 17 00:00:00 2001 From: James Briggs <35938317+jamescalam@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:41:03 +0200 Subject: [PATCH 05/13] Update semantic_router/index/pinecone.py --- semantic_router/index/pinecone.py | 1 - 1 file changed, 1 deletion(-) diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index 38f197ba..12019106 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -676,7 +676,6 @@ async def _async_fetch_metadata(self, vector_id: str) -> dict: response_data = await response.json(content_type=None) print(f"RESPONSE: {response_data}") except Exception as e: - print(f"Failed to decode JSON for vector {vector_id}: {e}") return {} return ( From 682e7f138e06bd97fa903bccac09c6a198d49339 Mon Sep 17 00:00:00 2001 From: James Briggs <35938317+jamescalam@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:41:10 +0200 Subject: [PATCH 06/13] Update semantic_router/index/pinecone.py --- semantic_router/index/pinecone.py | 1 - 1 file changed, 1 deletion(-) diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index 12019106..09f85e24 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -674,7 +674,6 @@ async def _async_fetch_metadata(self, vector_id: str) -> dict: try: response_data = await response.json(content_type=None) - print(f"RESPONSE: {response_data}") except Exception as e: return {} From 32f72c9e17f0f31a80848155b4a65a66969ff7d0 Mon Sep 17 00:00:00 2001 From: James Briggs <35938317+jamescalam@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:41:20 +0200 Subject: [PATCH 07/13] Update semantic_router/index/pinecone.py --- semantic_router/index/pinecone.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index 09f85e24..50469baf 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -667,9 +667,6 @@ async def _async_fetch_metadata(self, vector_id: str) -> dict: ) as response: if response.status != 200: error_text = await response.text() - print( - f"Error fetching metadata for vector {vector_id}: {response.status} - {error_text}" - ) return {} try: From f9c6e0f65084b12fdddea141df9b2704ca6712bc Mon Sep 17 00:00:00 2001 From: James Briggs Date: Fri, 23 Aug 2024 16:29:46 +0200 Subject: [PATCH 08/13] chore: lint --- semantic_router/index/pinecone.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index 50469baf..327d277b 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -628,6 +628,7 @@ async def _async_get_all( ) as response: if response.status != 200: error_text = await response.text() + logger.error(f"Error fetching vectors: {error_text}") break response_data = await response.json(content_type=None) @@ -667,11 +668,15 @@ async def _async_fetch_metadata(self, vector_id: str) -> dict: ) as response: if response.status != 200: error_text = await response.text() + logger.error(f"Error fetching metadata: {error_text}") return {} try: response_data = await response.json(content_type=None) except Exception as e: + logger.warning( + f"No metadata found for vector {vector_id}: {e}" + ) return {} return ( From 615bcdb86ff695ce39e6b0e1ecc2728c5692fbdc Mon Sep 17 00:00:00 2001 From: James Briggs Date: Fri, 23 Aug 2024 16:30:41 +0200 Subject: [PATCH 09/13] chore: black lint --- semantic_router/index/pinecone.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index 327d277b..70c2000f 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -674,9 +674,7 @@ async def _async_fetch_metadata(self, vector_id: str) -> dict: try: response_data = await response.json(content_type=None) except Exception as e: - logger.warning( - f"No metadata found for vector {vector_id}: {e}" - ) + logger.warning(f"No metadata found for vector {vector_id}: {e}") return {} return ( From e2668a85b5468148f9f5d9ef892073967ce416c7 Mon Sep 17 00:00:00 2001 From: James Briggs Date: Fri, 23 Aug 2024 16:46:03 +0200 Subject: [PATCH 10/13] fix: types --- semantic_router/index/pinecone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantic_router/index/pinecone.py b/semantic_router/index/pinecone.py index 70c2000f..28087f75 100644 --- a/semantic_router/index/pinecone.py +++ b/semantic_router/index/pinecone.py @@ -597,7 +597,7 @@ async def _async_describe_index(self, name: str): return await response.json(content_type=None) async def _async_get_all( - self, prefix: str | None = None, include_metadata: bool = False + self, prefix: Optional[str] = None, include_metadata: bool = False ) -> tuple[list[str], list[dict]]: """ Retrieves all vector IDs from the Pinecone index using pagination asynchronously. From 6a4bd38233b6a9fd8b57da4e268eee9aefcf9efb Mon Sep 17 00:00:00 2001 From: James Briggs Date: Fri, 23 Aug 2024 16:48:27 +0200 Subject: [PATCH 11/13] chore: upgrade version --- docs/source/conf.py | 2 +- pyproject.toml | 2 +- semantic_router/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 74b46335..0174df00 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,7 +15,7 @@ project = "Semantic Router" copyright = "2024, Aurelio AI" author = "Aurelio AI" -release = "0.0.60" +release = "0.0.61" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/pyproject.toml b/pyproject.toml index f69ceca0..7b294000 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "semantic-router" -version = "0.0.60" +version = "0.0.61" description = "Super fast semantic router for AI decision making" authors = [ "James Briggs ", diff --git a/semantic_router/__init__.py b/semantic_router/__init__.py index 0664671f..e8ed01d5 100644 --- a/semantic_router/__init__.py +++ b/semantic_router/__init__.py @@ -4,4 +4,4 @@ __all__ = ["RouteLayer", "HybridRouteLayer", "Route", "LayerConfig"] -__version__ = "0.0.60" +__version__ = "0.0.61" From 5cdf1d20e24677e02c2e1a7f540e713a91478b22 Mon Sep 17 00:00:00 2001 From: James Briggs Date: Fri, 23 Aug 2024 17:10:29 +0200 Subject: [PATCH 12/13] fix: vit test condition fix --- tests/unit/encoders/test_vit.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/encoders/test_vit.py b/tests/unit/encoders/test_vit.py index 848093b3..e0cfd8ec 100644 --- a/tests/unit/encoders/test_vit.py +++ b/tests/unit/encoders/test_vit.py @@ -7,7 +7,6 @@ from semantic_router.encoders import VitEncoder test_model_name = "aurelio-ai/sr-test-vit" -vit_encoder = VitEncoder(name=test_model_name) embed_dim = 32 if torch.cuda.is_available(): @@ -53,6 +52,7 @@ def test_vit_encoder__import_errors_torchvision(self, mocker): os.environ.get("RUN_HF_TESTS") is None, reason="Set RUN_HF_TESTS=1 to run" ) def test_vit_encoder_initialization(self): + vit_encoder = VitEncoder(name=test_model_name) assert vit_encoder.name == test_model_name assert vit_encoder.type == "huggingface" assert vit_encoder.score_threshold == 0.5 @@ -62,6 +62,7 @@ def test_vit_encoder_initialization(self): os.environ.get("RUN_HF_TESTS") is None, reason="Set RUN_HF_TESTS=1 to run" ) def test_vit_encoder_call(self, dummy_pil_image): + vit_encoder = VitEncoder(name=test_model_name) encoded_images = vit_encoder([dummy_pil_image] * 3) assert len(encoded_images) == 3 @@ -71,6 +72,7 @@ def test_vit_encoder_call(self, dummy_pil_image): os.environ.get("RUN_HF_TESTS") is None, reason="Set RUN_HF_TESTS=1 to run" ) def test_vit_encoder_call_misshaped(self, dummy_pil_image, misshaped_pil_image): + vit_encoder = VitEncoder(name=test_model_name) encoded_images = vit_encoder([dummy_pil_image, misshaped_pil_image]) assert len(encoded_images) == 2 @@ -80,6 +82,7 @@ def test_vit_encoder_call_misshaped(self, dummy_pil_image, misshaped_pil_image): os.environ.get("RUN_HF_TESTS") is None, reason="Set RUN_HF_TESTS=1 to run" ) def test_vit_encoder_process_images_device(self, dummy_pil_image): + vit_encoder = VitEncoder(name=test_model_name) imgs = vit_encoder._process_images([dummy_pil_image] * 3)["pixel_values"] assert imgs.device.type == device @@ -88,6 +91,7 @@ def test_vit_encoder_process_images_device(self, dummy_pil_image): os.environ.get("RUN_HF_TESTS") is None, reason="Set RUN_HF_TESTS=1 to run" ) def test_vit_encoder_ensure_rgb(self, dummy_black_and_white_img): + vit_encoder = VitEncoder(name=test_model_name) rgb_image = vit_encoder._ensure_rgb(dummy_black_and_white_img) assert rgb_image.mode == "RGB" From 7f2cfaef26712d3f9fd0bd001b3a0c647e25aca7 Mon Sep 17 00:00:00 2001 From: James Briggs Date: Fri, 23 Aug 2024 17:22:58 +0200 Subject: [PATCH 13/13] fix: tests --- tests/unit/encoders/test_vit.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/unit/encoders/test_vit.py b/tests/unit/encoders/test_vit.py index e0cfd8ec..85e66a9f 100644 --- a/tests/unit/encoders/test_vit.py +++ b/tests/unit/encoders/test_vit.py @@ -43,11 +43,6 @@ def test_vit_encoder__import_errors_torch(self, mocker): with pytest.raises(ImportError): VitEncoder() - def test_vit_encoder__import_errors_torchvision(self, mocker): - mocker.patch.dict("sys.modules", {"torchvision": None}) - with pytest.raises(ImportError): - VitEncoder() - @pytest.mark.skipif( os.environ.get("RUN_HF_TESTS") is None, reason="Set RUN_HF_TESTS=1 to run" )