From 8a556ada0ea0363f67d3f34761171c3e27d311c7 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Tue, 9 Jan 2024 12:58:08 +0200 Subject: [PATCH 01/14] feat: compatible with python 3.9 and 3.12 --- README.md | 10 +- coverage.xml | 664 ++++++++++++++++---------- pyproject.toml | 2 +- semantic_router/encoders/bm25.py | 2 +- semantic_router/encoders/fastembed.py | 2 +- 5 files changed, 420 insertions(+), 260 deletions(-) diff --git a/README.md b/README.md index fe5db343..fd4c12a2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ [![Aurelio AI](https://pbs.twimg.com/profile_banners/1671498317455581184/1696285195/1500x500)](https://aurelio.ai) # Semantic Router +

PyPI - Python Version GitHub Contributors @@ -22,7 +23,7 @@ To get started with _semantic-router_ we install it like so: pip install -qU semantic-router ``` -❗️ _If wanting to use local embeddings you can use `FastEmbedEncoder` (`pip install -qU semantic-router[fastembed]`). To use the `HybridRouteLayer` you must `pip install -qU semantic-router[hybrid]`._ +❗️ _If wanting to use local embeddings you can use `FastEmbedEncoder` (`pip install -qU "semantic-router[fastembed]`"). To use the `HybridRouteLayer` you must `pip install -qU "semantic-router[hybrid]"`._ We begin by defining a set of `Route` objects. These are the decision paths that the semantic router can decide to use, let's try two simple routes for now — one for talk on _politics_ and another for _chitchat_: @@ -114,10 +115,3 @@ rl("I'm interested in learning about llama 2").name In this case, no decision could be made as we had no matches — so our route layer returned `None`! ## 📚 [Resources](https://github.com/aurelio-labs/semantic-router/tree/main/docs) - - - - - - - diff --git a/coverage.xml b/coverage.xml index 6726da20..9d154145 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,12 +1,12 @@ - + /Users/jakit/customers/aurelio/semantic-router/semantic_router - + @@ -23,104 +23,100 @@ - - + + + + - - + + - - - - + + - - + + + + - - + - - + - - - + + + - + - + + - - + + - - + - - - + - - - + + + + + + - + + - + - + - + - - + + + - - - - + + - + + - - - - - - - + - + @@ -128,190 +124,196 @@ + + + + - - + + - - - + + + - - - - - - + + + + + - - - - - - - - - + + + + + + + + - + - + - + - + - + - - + + - - + + - - + + - + - - - - + + + + - + - + - + - - + + - + + - - - - - + + - + - - - + + + - - - - - - - + + + + + + + + - - - - - - - - + + + - + + + - + - - + - - - + + + + + - - + + + + + + - - + - - - - - - - + + + - + - - - - + + + + + - + - + + + + + + + + @@ -332,7 +334,7 @@ - + @@ -367,90 +369,105 @@ - + - - - - - - + + + + + - + - - + + + - - + - - + + - - - - - + + + + + + - - + + + + + + + + - + - - + - - + + - - + + + - - - + + + + - + - + + + + + - + + + + + - + @@ -470,52 +487,58 @@ - + - + + - + - + + - + - - - - + + - + + - - + + - - + + + + + + + @@ -528,7 +551,6 @@ - @@ -538,7 +560,7 @@ - + @@ -546,40 +568,42 @@ + - + - - + + - - - - - - - - + + + + + + - - - - + + + + + + + @@ -596,22 +620,21 @@ - - - + + - + - + @@ -623,21 +646,162 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + @@ -663,47 +827,49 @@ - - + + - - - + + + - - - + + - - - - + + + - + + + + + - + - - - - + + + + @@ -746,34 +912,34 @@ - - + - + - + - + - + - - - - - - - + + + + + + + + diff --git a/pyproject.toml b/pyproject.toml index 5b0e1f30..5273afac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ readme = "README.md" packages = [{include = "semantic_router"}] [tool.poetry.dependencies] -python = ">=3.10,<3.12" +python = "^3.9" pydantic = "^1.8.2" openai = "^1.3.9" cohere = "^4.32" diff --git a/semantic_router/encoders/bm25.py b/semantic_router/encoders/bm25.py index 68150cb7..e96e4d7d 100644 --- a/semantic_router/encoders/bm25.py +++ b/semantic_router/encoders/bm25.py @@ -16,7 +16,7 @@ def __init__(self, name: str = "bm25", score_threshold: float = 0.82): except ImportError: raise ImportError( "Please install pinecone-text to use BM25Encoder. " - "You can install it with: `pip install semantic-router[hybrid]`" + "You can install it with: `pip install 'semantic-router[hybrid]'`" ) logger.info("Downloading and initializing BM25 model parameters.") self.model = encoder.default() diff --git a/semantic_router/encoders/fastembed.py b/semantic_router/encoders/fastembed.py index 413e3a6a..98cfc6cc 100644 --- a/semantic_router/encoders/fastembed.py +++ b/semantic_router/encoders/fastembed.py @@ -27,7 +27,7 @@ def _initialize_client(self): raise ImportError( "Please install fastembed to use FastEmbedEncoder. " "You can install it with: " - "`pip install semantic-router[fastembed]`" + "`pip install 'semantic-router[fastembed]'`" ) embedding_args = { From 07b0ecc5afaf81874b759b22c600f239e48fff9e Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Tue, 9 Jan 2024 12:58:44 +0200 Subject: [PATCH 02/14] feat: compatible with python 3.9 and 3.12 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a805290..64e5c9e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.11.3 + python: python3.12 repos: - repo: meta hooks: From c6f4290effff99d159aa4f7b7c631f0412f189bf Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Tue, 9 Jan 2024 13:09:34 +0200 Subject: [PATCH 03/14] feat: added 3.9 to github workflow --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf22ea9b..6717b24c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,10 @@ jobs: strategy: matrix: python-version: + - "3.9" + - "3.10" - "3.11" + - "3.12" steps: - uses: actions/checkout@v4 - name: Cache Poetry From 5cdbe71292bb6506cc2cf2da60fe992439d8f577 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Tue, 9 Jan 2024 13:46:59 +0200 Subject: [PATCH 04/14] fix: Python versions --- poetry.lock | 110 +++++++++++++++++++++++-------------------------- pyproject.toml | 4 +- 2 files changed, 53 insertions(+), 61 deletions(-) diff --git a/poetry.lock b/poetry.lock index d0f80d03..3f7e8103 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -679,42 +679,42 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth [[package]] name = "fastavro" -version = "1.9.2" +version = "1.9.3" description = "Fast read/write of AVRO files" optional = false python-versions = ">=3.8" files = [ - {file = "fastavro-1.9.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:223cecf135fd29b83ca6a30035b15b8db169aeaf8dc4f9a5d34afadc4b31638a"}, - {file = "fastavro-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e08c9be8c6f7eed2cf30f8b64d50094cba38a81b751c7db9f9c4be2656715259"}, - {file = "fastavro-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:394f06cc865c6fbae3bbca323633a28a5d914c55dc2c1cdefb75432456ef8f6f"}, - {file = "fastavro-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7a7caadd47bdd04bda534ff70b4b98d2823800c488fd911918115aec4c4dc09b"}, - {file = "fastavro-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:68478a1b8a583d83ad6550e9dceac6cbb148a99a52c3559a0413bf4c0b9c8786"}, - {file = "fastavro-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:b59a1123f1d534743af33fdbda80dd7b9146685bdd7931eae12bee6203065222"}, - {file = "fastavro-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:887c20dc527a549764c91f9e48ece071f2f26d217af66ebcaeb87bf29578fee5"}, - {file = "fastavro-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46458f78b481c12db62d3d8a81bae09cb0b5b521c0d066c6856fc2746908d00d"}, - {file = "fastavro-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f4a2a4bed0e829f79fa1e4f172d484b2179426e827bcc80c0069cc81328a5af"}, - {file = "fastavro-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6167f9bbe1c5a28fbc2db767f97dbbb4981065e6eeafd4e613f6fe76c576ffd4"}, - {file = "fastavro-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d574bc385f820da0404528157238de4e5fdd775d2cb3d05b3b0f1b475d493837"}, - {file = "fastavro-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:ec600eb15b3ec931904c5bf8da62b3b725cb0f369add83ba47d7b5e9322f92a0"}, - {file = "fastavro-1.9.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:c82b0761503420cd45f7f50bc31975ac1c75b5118e15434c1d724b751abcc249"}, - {file = "fastavro-1.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db62d9b8c944b8d9c481e5f980d5becfd034bdd58c72e27c9333bd504b06bda0"}, - {file = "fastavro-1.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65e61f040bc9494646f42a466e9cd428783b82d7161173f3296710723ba5a453"}, - {file = "fastavro-1.9.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6278b93cdd5bef1778c0232ce1f265137f90bc6be97a5c1dd7e0d99a406c0488"}, - {file = "fastavro-1.9.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cd003ddea5d89720194b6e57011c37221d9fc4ddc750e6f4723516eb659be686"}, - {file = "fastavro-1.9.2-cp312-cp312-win_amd64.whl", hash = "sha256:43f09d100a26e8b59f30dde664d93e423b648e008abfc43132608a18fe8ddcc2"}, - {file = "fastavro-1.9.2-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:3ddffeff5394f285c69f9cd481f47b6cf62379840cdbe6e0dc74683bd589b56e"}, - {file = "fastavro-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e75a2b2ec697d2058a7d96522e921f03f174cf9049ace007c24be7ab58c5370"}, - {file = "fastavro-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd2e8fd0567483eb0fdada1b979ad4d493305dfdd3f351c82a87df301f0ae1f"}, - {file = "fastavro-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c652dbe3f087c943a5b89f9a50a574e64f23790bfbec335ce2b91a2ae354a443"}, - {file = "fastavro-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bba73e9a1822162f1b3a43de0362f29880014c5c4d49d63ad7fcce339ef73ea2"}, - {file = "fastavro-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:beeef2964bbfd09c539424808539b956d7425afbb7055b89e2aa311374748b56"}, - {file = "fastavro-1.9.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:d5fa48266d75e057b27d8586b823d6d7d7c94593fd989d75033eb4c8078009fb"}, - {file = "fastavro-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b69aeb0d063f5955a0e412f9779444fc452568a49db75a90a8d372f9cb4a01c8"}, - {file = "fastavro-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ce336c59fb40fdb8751bda8cc6076cfcdf9767c3c107f6049e049166b26c61f"}, - {file = "fastavro-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:581036e18661f045415a51ad528865e1d7ba5a9690a3dede9e6ea50f94ed6c4c"}, - {file = "fastavro-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:39b6b5c3cda569c0a130fd2d08d4c53a326ede7e05174a24eda08f7698f70eda"}, - {file = "fastavro-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:d33e40f246bf07f106f9d2da68d0234efcc62276b6e35bde00ff920ea7f871fd"}, - {file = "fastavro-1.9.2.tar.gz", hash = "sha256:5c1ffad986200496bd69b5c4748ae90b5d934d3b1456f33147bee3a0bb17f89b"}, + {file = "fastavro-1.9.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:5e9b2e1427fb84c0754bc34923d10cabcf2ed23230201208a1371ab7b6027674"}, + {file = "fastavro-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ef82f86ae276309abc0072598474b6be68105a0b28f8d7cc0398d1d353d7de"}, + {file = "fastavro-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:280ef7ab7232ecb2097038d6842416ec717d0e1c314b80ff245f85201f3396a4"}, + {file = "fastavro-1.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4a36cfc0421ed7576ecb1c22de7bd1dedcce62aebbffcc597379d59171e5d76e"}, + {file = "fastavro-1.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d80f2e20199140eb8c036b4393e9bc9eff325543311b958c72318999499d4279"}, + {file = "fastavro-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:a435f7edd7c5b52cee3f23ca950cd9373ab35cf2aa3d269b3d6aca7e2fc1372c"}, + {file = "fastavro-1.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a7053ed10194ec53754f5337b57b3273a74b48505edcd6edb79fe3c4cd259c0"}, + {file = "fastavro-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853e01f13534d1baa0a3d493a8573e665e93ffa35b4bf1d125e21764d343af8e"}, + {file = "fastavro-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5a279cda25d876e6f120950cadf184a307fd8998f9a22a90bb62e6749f88d1e"}, + {file = "fastavro-1.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:63d6f928840f3fb1f2e1fe20bc8b7d0e1a51ba4bb0e554ecb837a669fba31288"}, + {file = "fastavro-1.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8807046edc78f50b3ea5f55f6a534c87b2a13538e7c56fec3532ef802bcae333"}, + {file = "fastavro-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:e502579da4a51c5630eadbd811a1b3d262d6e783bf19998cfb33d2ea0cf6f516"}, + {file = "fastavro-1.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:6b665efe442061df8d9608c2fb692847df85d52ad825b776c441802f0dfa6571"}, + {file = "fastavro-1.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b8c96d81f0115633489d7f1133a03832922629a61ca81c1d47b482ddcda3b94"}, + {file = "fastavro-1.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338c7ec94dd2474c4679e44d2560a1922cb6fa99acbb7b18957264baf8eadfc7"}, + {file = "fastavro-1.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a509b34c9af71a109c633631ac2f6d2209830e13200d0048f7e9c057fd563f8f"}, + {file = "fastavro-1.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:967edefab470987c024cd5a1fcd04744a50a91e740c7bdf325181043a47f1083"}, + {file = "fastavro-1.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:033c15e8ed02f80f01d58be1cd880b09fd444faf277263d563a727711d47a98a"}, + {file = "fastavro-1.9.3-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:6b38723327603d77080aec56628e13a739415f8596ca0cc41a905615977c6d6b"}, + {file = "fastavro-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:046d75c4400941fd08f0a6855a34ae63bf02ea01f366b5b749942abe10640056"}, + {file = "fastavro-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ab312b8baf0e61ee717878d390022ee1b713d70b244d69efbf3325680f9749"}, + {file = "fastavro-1.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c562fcf8f5091a2446aafd0c2a0da590c24e0b53527a0100d33908e32f20eea8"}, + {file = "fastavro-1.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2aa0111e7ebd076d2a094862bbdf8ea175cebba148fcce6c89ff46b625e334b4"}, + {file = "fastavro-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:652072e0f455ca19a1ee502b527e603389783657c130d81f89df66775979d6f5"}, + {file = "fastavro-1.9.3-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:0a57cdd4edaee36d4216faf801ebc7f53f45e4e1518bdd9832d6f6f1d6e2d88f"}, + {file = "fastavro-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b46a18ebed61573b0823c28eda2716485d283258a83659c7fe6ad3aaeacfed4"}, + {file = "fastavro-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f756f0723f3bd97db20437d0a8e45712839e6ccd7c82f4d82469533be48b4c7"}, + {file = "fastavro-1.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d98d5a08063f5b6d7ac5016a0dfe0698b50d9987cb74686f7dfa8288b7b09e0b"}, + {file = "fastavro-1.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:00698e60db58a2d52cb709df882d451fb7664ebb2f8cb37d9171697e060dc767"}, + {file = "fastavro-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:d021bbc135023194688e88a7431fb0b5e3ce20e27153bf258f2ce08ee1a0106b"}, + {file = "fastavro-1.9.3.tar.gz", hash = "sha256:a30d3d2353f6d3b4f6dcd6a97ae937b3775faddd63f5856fe11ba3b0dbb1756a"}, ] [package.extras] @@ -1069,13 +1069,13 @@ test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio" [[package]] name = "ipython" -version = "8.19.0" +version = "8.18.1" description = "IPython: Productive Interactive Computing" optional = false -python-versions = ">=3.10" +python-versions = ">=3.9" files = [ - {file = "ipython-8.19.0-py3-none-any.whl", hash = "sha256:2f55d59370f59d0d2b2212109fe0e6035cfea436b1c0e6150ad2244746272ec5"}, - {file = "ipython-8.19.0.tar.gz", hash = "sha256:ac4da4ecf0042fb4e0ce57c60430c2db3c719fa8bdf92f8631d6bd8a5785d1f0"}, + {file = "ipython-8.18.1-py3-none-any.whl", hash = "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397"}, + {file = "ipython-8.18.1.tar.gz", hash = "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27"}, ] [package.dependencies] @@ -1089,19 +1089,20 @@ prompt-toolkit = ">=3.0.41,<3.1.0" pygments = ">=2.4.0" stack-data = "*" traitlets = ">=5" +typing-extensions = {version = "*", markers = "python_version < \"3.10\""} [package.extras] -all = ["black", "curio", "docrepr", "exceptiongroup", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.23)", "pandas", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] +all = ["black", "curio", "docrepr", "exceptiongroup", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio (<0.22)", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] black = ["black"] -doc = ["docrepr", "exceptiongroup", "ipykernel", "matplotlib", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] +doc = ["docrepr", "exceptiongroup", "ipykernel", "matplotlib", "pickleshare", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio (<0.22)", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] kernel = ["ipykernel"] nbconvert = ["nbconvert"] nbformat = ["nbformat"] notebook = ["ipywidgets", "notebook"] parallel = ["ipyparallel"] qtconsole = ["qtconsole"] -test = ["pickleshare", "pytest", "pytest-asyncio (<0.22)", "testpath"] -test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.23)", "pandas", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "testpath", "trio"] +test = ["pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath"] +test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath", "trio"] [[package]] name = "jedi" @@ -1145,6 +1146,7 @@ files = [ ] [package.dependencies] +importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" python-dateutil = ">=2.8.2" pyzmq = ">=23.0" @@ -1157,13 +1159,13 @@ test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pyt [[package]] name = "jupyter-core" -version = "5.7.0" +version = "5.7.1" description = "Jupyter core package. A base package on which Jupyter projects rely." optional = false python-versions = ">=3.8" files = [ - {file = "jupyter_core-5.7.0-py3-none-any.whl", hash = "sha256:16eea462f7dad23ba9f86542bdf17f830804e2028eb48d609b6134d91681e983"}, - {file = "jupyter_core-5.7.0.tar.gz", hash = "sha256:cb8d3ed92144d2463a3c5664fdd686a3f0c1442ea45df8babb1c1a9e6333fe03"}, + {file = "jupyter_core-5.7.1-py3-none-any.whl", hash = "sha256:c65c82126453a723a2804aa52409930434598fd9d35091d63dfb919d2b765bb7"}, + {file = "jupyter_core-5.7.1.tar.gz", hash = "sha256:de61a9d7fc71240f688b2fb5ab659fbb56979458dc66a71decd098e03c79e218"}, ] [package.dependencies] @@ -1545,13 +1547,13 @@ sympy = "*" [[package]] name = "openai" -version = "1.6.1" +version = "1.7.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.6.1-py3-none-any.whl", hash = "sha256:bc9f774838d67ac29fb24cdeb2d58faf57de8b311085dcd1348f7aa02a96c7ee"}, - {file = "openai-1.6.1.tar.gz", hash = "sha256:d553ca9dbf9486b08e75b09e8671e4f638462aaadccfced632bf490fc3d75fa2"}, + {file = "openai-1.7.0-py3-none-any.whl", hash = "sha256:2282e8e15acb05df79cccba330c025b8e84284c7ec1f3fa31f167a8479066333"}, + {file = "openai-1.7.0.tar.gz", hash = "sha256:f2a8dcb739e8620c9318a2c6304ea72aebb572ba02fa1d586344405e80d567d3"}, ] [package.dependencies] @@ -1972,7 +1974,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1980,15 +1981,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2005,7 +1999,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2013,7 +2006,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2703,5 +2695,5 @@ hybrid = ["pinecone-text"] [metadata] lock-version = "2.0" -python-versions = ">=3.10,<3.12" -content-hash = "42a58d13a0f9d9a1bca34b4c29cafee6a5c884b80d47848cb7c552ad91e54743" +python-versions = "^3.9" +content-hash = "aaccb399ef67ee627b0bca661f624af24b0fafe9c83dee51297bec5868573ad8" diff --git a/pyproject.toml b/pyproject.toml index 5273afac..127f5534 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,14 +22,14 @@ numpy = "^1.25.2" pinecone-text = {version = "^0.7.0", optional = true} colorlog = "^6.8.0" pyyaml = "^6.0.1" -fastembed = {version = "^0.1.3", optional = true} +fastembed = {version = "^0.1.3", optional = true, python = "<3.12"} [tool.poetry.extras] hybrid = ["pinecone-text"] fastembed = ["fastembed"] [tool.poetry.group.dev.dependencies] -ipykernel = "^6.26.0" +ipykernel = "^6.25.0" ruff = "^0.1.5" black = {extras = ["jupyter"], version = "^23.12.0"} pytest = "^7.4.3" From 8958860c6bb37e31e1bc0173e86a57646b80f816 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Tue, 9 Jan 2024 13:57:56 +0200 Subject: [PATCH 05/14] fix: numpy version downgrade --- poetry.lock | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3f7e8103..3b8775c6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2696,4 +2696,4 @@ hybrid = ["pinecone-text"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "aaccb399ef67ee627b0bca661f624af24b0fafe9c83dee51297bec5868573ad8" +content-hash = "bd435dd14b07738872f14483a7a781ea4a16394c4739917e487acde26235cfae" diff --git a/pyproject.toml b/pyproject.toml index 127f5534..90b97835 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ python = "^3.9" pydantic = "^1.8.2" openai = "^1.3.9" cohere = "^4.32" -numpy = "^1.25.2" +numpy = "^1.24.3" pinecone-text = {version = "^0.7.0", optional = true} colorlog = "^6.8.0" pyyaml = "^6.0.1" From abc8783477266b188719708df9b061e2807745e6 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:54:32 +0200 Subject: [PATCH 06/14] feat: compatible with python 3.9 --- .gitignore | 3 +++ .pre-commit-config.yaml | 14 ++++++-------- Makefile | 4 ++-- poetry.lock | 8 ++++---- pyproject.toml | 4 ++-- replace.py | 23 +++++++++++++++++++++++ semantic_router/encoders/bm25.py | 4 ++-- semantic_router/encoders/cohere.py | 7 ++++--- semantic_router/encoders/openai.py | 7 ++++--- semantic_router/hybrid_layer.py | 3 ++- semantic_router/layer.py | 13 +++++++------ semantic_router/llms/base.py | 4 +++- semantic_router/llms/cohere.py | 7 ++++--- semantic_router/llms/openai.py | 11 ++++++----- semantic_router/llms/openrouter.py | 13 +++++++------ semantic_router/route.py | 5 +++-- semantic_router/schema.py | 9 +++++---- semantic_router/utils/llm.py | 5 +++-- test_output.txt | 0 19 files changed, 90 insertions(+), 54 deletions(-) create mode 100644 replace.py delete mode 100644 test_output.txt diff --git a/.gitignore b/.gitignore index cb4c0022..41aac4aa 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ venv/ .idea **/__pycache__ **/*.py[cod] +node_modules +package-lock.json +package.json # local env files .env*.local diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 64e5c9e1..daf970d5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.12 + python: python3.9 repos: - repo: meta hooks: @@ -17,14 +17,12 @@ repos: - id: blacken-docs additional_dependencies: [black==22.10.0] - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.290 + - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook + rev: v9.11.0 hooks: - - id: ruff - types_or: [ python, pyi, jupyter ] - args: [ --fix ] - - id: ruff-format - types_or: [ python, pyi, jupyter ] + - id: commitlint + stages: [commit-msg] + additional_dependencies: ['@commitlint/config-conventional'] - repo: https://github.com/codespell-project/codespell rev: v2.2.4 diff --git a/Makefile b/Makefile index aeb3d3b1..adf4eb0c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ format: - poetry run black . + poetry run black --target-version py39 . poetry run ruff --select I --fix . PYTHON_FILES=. @@ -7,7 +7,7 @@ lint: PYTHON_FILES=. lint_diff: PYTHON_FILES=$(shell git diff --name-only --diff-filter=d main | grep -E '\.py$$') lint lint_diff: - poetry run black $(PYTHON_FILES) --check + poetry run black --target-version py39 $(PYTHON_FILES) --check poetry run ruff . poetry run mypy $(PYTHON_FILES) diff --git a/poetry.lock b/poetry.lock index 3b8775c6..3131c0d8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -440,13 +440,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cohere" -version = "4.40" +version = "4.41" description = "Python SDK for the Cohere API" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "cohere-4.40-py3-none-any.whl", hash = "sha256:75dac8369d97fadc05901352d9db64a0ca6cd40c08423f3c4691f57eb7b131e7"}, - {file = "cohere-4.40.tar.gz", hash = "sha256:d9e5c1fa7f80a193c03330a634954b927bf188ead7dcfdb51865480f73aebda8"}, + {file = "cohere-4.41-py3-none-any.whl", hash = "sha256:39470cc412fa96a1c612f522d48d7d86b34b3163a04030cff83ec48ebbaff32f"}, + {file = "cohere-4.41.tar.gz", hash = "sha256:8509ca196dc038eca81e474d3cd5896da2ea168a4d3c578b4cb6969994be34ef"}, ] [package.dependencies] @@ -2696,4 +2696,4 @@ hybrid = ["pinecone-text"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "bd435dd14b07738872f14483a7a781ea4a16394c4739917e487acde26235cfae" +content-hash = "3e3b13e2493e7bef6ef1d9487d4618f834f3387a55379edf63d00f76fe4def0a" diff --git a/pyproject.toml b/pyproject.toml index 90b97835..bbe14c56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,10 +18,10 @@ python = "^3.9" pydantic = "^1.8.2" openai = "^1.3.9" cohere = "^4.32" -numpy = "^1.24.3" -pinecone-text = {version = "^0.7.0", optional = true} +numpy = "^1.25.2" colorlog = "^6.8.0" pyyaml = "^6.0.1" +pinecone-text = {version = "^0.7.1", optional = true, python = "<3.12"} fastembed = {version = "^0.1.3", optional = true, python = "<3.12"} [tool.poetry.extras] diff --git a/replace.py b/replace.py new file mode 100644 index 00000000..d1fbe597 --- /dev/null +++ b/replace.py @@ -0,0 +1,23 @@ +import re +import os + + +def replace_type_hints(file_path): + with open(file_path, "rb") as file: + file_data = file.read() + + # Decode the file data with error handling + file_data = file_data.decode("utf-8", errors="ignore") + + # Regular expression pattern to find '| None' and replace with 'Optional' + file_data = re.sub(r"(\w+)\s*\|\s*None", r"Optional[\1]", file_data) + + with open(file_path, "w") as file: + file.write(file_data) + + +# Walk through the repository and update all .py files +for root, dirs, files in os.walk("/Users/jakit/customers/aurelio/semantic-router"): + for file in files: + if file.endswith(".py"): + replace_type_hints(os.path.join(root, file)) diff --git a/semantic_router/encoders/bm25.py b/semantic_router/encoders/bm25.py index e96e4d7d..11a964b6 100644 --- a/semantic_router/encoders/bm25.py +++ b/semantic_router/encoders/bm25.py @@ -1,11 +1,11 @@ -from typing import Any +from typing import Any, Optional from semantic_router.encoders import BaseEncoder from semantic_router.utils.logger import logger class BM25Encoder(BaseEncoder): - model: Any | None = None + model: Optional[Any] = None idx_mapping: dict[int, int] | None = None type: str = "sparse" diff --git a/semantic_router/encoders/cohere.py b/semantic_router/encoders/cohere.py index 2f80aaaf..ec8ee0f8 100644 --- a/semantic_router/encoders/cohere.py +++ b/semantic_router/encoders/cohere.py @@ -1,4 +1,5 @@ import os +from typing import Optional import cohere @@ -6,13 +7,13 @@ class CohereEncoder(BaseEncoder): - client: cohere.Client | None = None + client: Optional[cohere.Client] = None type: str = "cohere" def __init__( self, - name: str | None = None, - cohere_api_key: str | None = None, + name: Optional[str] = None, + cohere_api_key: Optional[str] = None, score_threshold: float = 0.3, ): if name is None: diff --git a/semantic_router/encoders/openai.py b/semantic_router/encoders/openai.py index 4ec87638..4504fefe 100644 --- a/semantic_router/encoders/openai.py +++ b/semantic_router/encoders/openai.py @@ -7,16 +7,17 @@ from semantic_router.encoders import BaseEncoder from semantic_router.utils.logger import logger +from typing import Optional class OpenAIEncoder(BaseEncoder): - client: openai.Client | None + client: Optional[openai.Client] type: str = "openai" def __init__( self, - name: str | None = None, - openai_api_key: str | None = None, + name: Optional[str] = None, + openai_api_key: Optional[str] = None, score_threshold: float = 0.82, ): if name is None: diff --git a/semantic_router/hybrid_layer.py b/semantic_router/hybrid_layer.py index cd9f7ccb..06862a63 100644 --- a/semantic_router/hybrid_layer.py +++ b/semantic_router/hybrid_layer.py @@ -7,6 +7,7 @@ ) from semantic_router.route import Route from semantic_router.utils.logger import logger +from typing import Optional class HybridRouteLayer: @@ -29,7 +30,7 @@ def __init__( # self._add_route(route=route) self._add_routes(routes) - def __call__(self, text: str) -> str | None: + def __call__(self, text: str) -> Optional[str]: results = self._query(text) top_class, top_class_scores = self._semantic_classify(results) passed = self._pass_threshold(top_class_scores, self.score_threshold) diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 08261756..e6a214f9 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -1,5 +1,6 @@ import json import os +from typing import Optional import numpy as np import yaml @@ -52,7 +53,7 @@ def __init__( self, routes: list[Route] = [], encoder_type: str = "openai", - encoder_name: str | None = None, + encoder_name: Optional[str] = None, ): self.encoder_type = encoder_type if encoder_name is None: @@ -131,7 +132,7 @@ def add(self, route: Route): self.routes.append(route) logger.info(f"Added route `{route.name}`") - def get(self, name: str) -> Route | None: + def get(self, name: str) -> Optional[Route]: for route in self.routes: if route.name == name: return route @@ -147,15 +148,15 @@ def remove(self, name: str): class RouteLayer: - index: np.ndarray | None = None - categories: np.ndarray | None = None + index: Optional[np.ndarray] = None + categories: Optional[np.ndarray] = None score_threshold: float encoder: BaseEncoder def __init__( self, - encoder: BaseEncoder | None = None, - llm: BaseLLM | None = None, + encoder: Optional[BaseEncoder] = None, + llm: Optional[BaseLLM] = None, routes: list[Route] | None = None, ): logger.info("Initializing RouteLayer") diff --git a/semantic_router/llms/base.py b/semantic_router/llms/base.py index 51db1fd0..bf5f29b6 100644 --- a/semantic_router/llms/base.py +++ b/semantic_router/llms/base.py @@ -1,3 +1,5 @@ +from typing import Optional + from pydantic import BaseModel from semantic_router.schema import Message @@ -9,5 +11,5 @@ class BaseLLM(BaseModel): class Config: arbitrary_types_allowed = True - def __call__(self, messages: list[Message]) -> str | None: + def __call__(self, messages: list[Message]) -> Optional[str]: raise NotImplementedError("Subclasses must implement this method") diff --git a/semantic_router/llms/cohere.py b/semantic_router/llms/cohere.py index 77581700..0ec21f35 100644 --- a/semantic_router/llms/cohere.py +++ b/semantic_router/llms/cohere.py @@ -1,4 +1,5 @@ import os +from typing import Optional import cohere @@ -7,12 +8,12 @@ class CohereLLM(BaseLLM): - client: cohere.Client | None = None + client: Optional[cohere.Client] = None def __init__( self, - name: str | None = None, - cohere_api_key: str | None = None, + name: Optional[str] = None, + cohere_api_key: Optional[str] = None, ): if name is None: name = os.getenv("COHERE_CHAT_MODEL_NAME", "command") diff --git a/semantic_router/llms/openai.py b/semantic_router/llms/openai.py index 43ddd642..d3b215bf 100644 --- a/semantic_router/llms/openai.py +++ b/semantic_router/llms/openai.py @@ -5,17 +5,18 @@ from semantic_router.llms import BaseLLM from semantic_router.schema import Message from semantic_router.utils.logger import logger +from typing import Optional class OpenAILLM(BaseLLM): - client: openai.OpenAI | None - temperature: float | None - max_tokens: int | None + client: Optional[openai.OpenAI] + temperature: Optional[float] + max_tokens: Optional[int] def __init__( self, - name: str | None = None, - openai_api_key: str | None = None, + name: Optional[str] = None, + openai_api_key: Optional[str] = None, temperature: float = 0.01, max_tokens: int = 200, ): diff --git a/semantic_router/llms/openrouter.py b/semantic_router/llms/openrouter.py index 587eeb12..6130e0a7 100644 --- a/semantic_router/llms/openrouter.py +++ b/semantic_router/llms/openrouter.py @@ -5,18 +5,19 @@ from semantic_router.llms import BaseLLM from semantic_router.schema import Message from semantic_router.utils.logger import logger +from typing import Optional class OpenRouterLLM(BaseLLM): - client: openai.OpenAI | None - base_url: str | None - temperature: float | None - max_tokens: int | None + client: Optional[openai.OpenAI] + base_url: Optional[str] + temperature: Optional[float] + max_tokens: Optional[int] def __init__( self, - name: str | None = None, - openrouter_api_key: str | None = None, + name: Optional[str] = None, + openrouter_api_key: Optional[str] = None, base_url: str = "https://openrouter.ai/api/v1", temperature: float = 0.01, max_tokens: int = 200, diff --git a/semantic_router/route.py b/semantic_router/route.py index 0d8269f0..c2b9b3dc 100644 --- a/semantic_router/route.py +++ b/semantic_router/route.py @@ -8,6 +8,7 @@ from semantic_router.schema import Message, RouteChoice from semantic_router.utils import function_call from semantic_router.utils.logger import logger +from typing import Optional def is_valid(route_config: str) -> bool: @@ -41,9 +42,9 @@ def is_valid(route_config: str) -> bool: class Route(BaseModel): name: str utterances: list[str] - description: str | None = None + description: Optional[str] = None function_schema: dict[str, Any] | None = None - llm: BaseLLM | None = None + llm: Optional[BaseLLM] = None def __call__(self, query: str) -> RouteChoice: if self.function_schema: diff --git a/semantic_router/schema.py b/semantic_router/schema.py index 5e94c23b..9505df24 100644 --- a/semantic_router/schema.py +++ b/semantic_router/schema.py @@ -10,6 +10,7 @@ OpenAIEncoder, ) from semantic_router.utils.splitters import semantic_splitter +from typing import Optional class EncoderType(Enum): @@ -20,17 +21,17 @@ class EncoderType(Enum): class RouteChoice(BaseModel): - name: str | None = None - function_call: dict | None = None + name: Optional[str] = None + function_call: Optional[dict] = None @dataclass class Encoder: type: EncoderType - name: str | None + name: Optional[str] model: BaseEncoder - def __init__(self, type: str, name: str | None): + def __init__(self, type: str, name: Optional[str]): self.type = EncoderType(type) self.name = name if self.type == EncoderType.HUGGINGFACE: diff --git a/semantic_router/utils/llm.py b/semantic_router/utils/llm.py index e92c1bcf..f0db13c8 100644 --- a/semantic_router/utils/llm.py +++ b/semantic_router/utils/llm.py @@ -3,9 +3,10 @@ import openai from semantic_router.utils.logger import logger +from typing import Optional -def llm(prompt: str) -> str | None: +def llm(prompt: str) -> Optional[str]: try: client = openai.OpenAI( base_url="https://openrouter.ai/api/v1", @@ -35,7 +36,7 @@ def llm(prompt: str) -> str | None: # TODO integrate async LLM function -# async def allm(prompt: str) -> str | None: +# async def allm(prompt: str) -> Optional[str]: # try: # client = openai.AsyncOpenAI( # base_url="https://openrouter.ai/api/v1", diff --git a/test_output.txt b/test_output.txt deleted file mode 100644 index e69de29b..00000000 From ffc9788d18919f34503f85b846a9732586ff1615 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:56:31 +0200 Subject: [PATCH 07/14] fix: removed helper replace.py --- .gitignore | 9 ++++++--- replace.py | 23 ----------------------- 2 files changed, 6 insertions(+), 26 deletions(-) delete mode 100644 replace.py diff --git a/.gitignore b/.gitignore index 41aac4aa..c3631d40 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,7 @@ venv/ .idea **/__pycache__ **/*.py[cod] -node_modules -package-lock.json -package.json + # local env files .env*.local @@ -23,3 +21,8 @@ mac.env .pytest_cache test.py output + +node_modules +package-lock.json +package.json +``` diff --git a/replace.py b/replace.py deleted file mode 100644 index d1fbe597..00000000 --- a/replace.py +++ /dev/null @@ -1,23 +0,0 @@ -import re -import os - - -def replace_type_hints(file_path): - with open(file_path, "rb") as file: - file_data = file.read() - - # Decode the file data with error handling - file_data = file_data.decode("utf-8", errors="ignore") - - # Regular expression pattern to find '| None' and replace with 'Optional' - file_data = re.sub(r"(\w+)\s*\|\s*None", r"Optional[\1]", file_data) - - with open(file_path, "w") as file: - file.write(file_data) - - -# Walk through the repository and update all .py files -for root, dirs, files in os.walk("/Users/jakit/customers/aurelio/semantic-router"): - for file in files: - if file.endswith(".py"): - replace_type_hints(os.path.join(root, file)) From 4192ee5d3b536410bfa23311bd204269d5f2a98d Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 10 Jan 2024 11:26:06 +0200 Subject: [PATCH 08/14] fix: gitignore --- .gitignore | 3 +- coverage.xml | 505 ++++++++++++++++++++++++++------------------------- 2 files changed, 260 insertions(+), 248 deletions(-) diff --git a/.gitignore b/.gitignore index c3631d40..cc461499 100644 --- a/.gitignore +++ b/.gitignore @@ -19,10 +19,11 @@ mac.env .coverage .coverage.* .pytest_cache +coverage.xml test.py output - node_modules package-lock.json package.json + ``` diff --git a/coverage.xml b/coverage.xml index 9d154145..dc2d1646 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,12 +1,12 @@ - + /Users/jakit/customers/aurelio/semantic-router/semantic_router - + @@ -25,295 +25,297 @@ - + - - + + - - - + + + - - + + - - + + - + - + - + - - + + - - - + + + - + - + - - - - + + + + - + - - - + + + - + - + - + - + - - + + - + - - + + - + - + - + - + + - + - + - + - + - - + + - + - + - - - - + + + + - - - - + + + + - - - - - - - - - + + + + + + + + + - + - + - - + + - - + + - - + + - + - - - - + + + + - + - + - + - - + + - + - - + + - - + + - - - + + + - + - + - + - - - - - - + + + + + + - + - + - + - - + + - + - + - + - - - - - + + + + + - + - - + + - - + + - + - + - + - - + + - + - + - + - - + + - + - + + @@ -334,7 +336,7 @@ - + @@ -345,76 +347,77 @@ - + - + - + - + - - + + - + - + - - - + + + - + - + - + - + - + - + - + - - + + - - + + - - - - + + + + - + + - + @@ -422,52 +425,53 @@ - + - + - + - + - - + + - - + + - + - + - + - + - - + + + - + @@ -545,13 +549,13 @@ - - - + + + - - + + @@ -561,7 +565,7 @@ - + @@ -569,9 +573,10 @@ + - + @@ -596,17 +601,17 @@ - + - - - - + + + + - + @@ -616,26 +621,26 @@ - + - - + + - - + + - + - + @@ -647,15 +652,16 @@ - - - + + + + - + @@ -672,25 +678,26 @@ - - + + - + - + + - + - - + + - + - - + + @@ -700,20 +707,21 @@ - + - - - - - + + + + + + - + @@ -721,38 +729,39 @@ - + - - + + - - + + - + - - - - - + + + + + + - + @@ -760,42 +769,43 @@ - + - - + + - + - - + + - + - - - - - + + + + + + - + @@ -869,17 +879,18 @@ - + - - - + + + + From c3648c0f5ea0a1b857f0bade01cf542232ddaeff Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 10 Jan 2024 12:22:25 +0200 Subject: [PATCH 09/14] fix: removed 3.12 from github tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6717b24c..017c23c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: - "3.9" - "3.10" - "3.11" - - "3.12" + # - "3.12" steps: - uses: actions/checkout@v4 - name: Cache Poetry From 94766893d13a61933d6edad0a76269c709be08ba Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 10 Jan 2024 12:46:09 +0200 Subject: [PATCH 10/14] fix: typing hint --- docs/examples/function_calling.ipynb | 2 +- semantic_router/encoders/bm25.py | 2 +- semantic_router/hybrid_layer.py | 2 +- semantic_router/layer.py | 1 + semantic_router/schema.py | 2 ++ 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/examples/function_calling.ipynb b/docs/examples/function_calling.ipynb index 401ab04a..5392e54e 100644 --- a/docs/examples/function_calling.ipynb +++ b/docs/examples/function_calling.ipynb @@ -512,7 +512,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.9.18" } }, "nbformat": 4, diff --git a/semantic_router/encoders/bm25.py b/semantic_router/encoders/bm25.py index 451273cd..0fc55173 100644 --- a/semantic_router/encoders/bm25.py +++ b/semantic_router/encoders/bm25.py @@ -6,7 +6,7 @@ class BM25Encoder(BaseEncoder): model: Optional[Any] = None - idx_mapping: Optional[dict[int, int]] = None + idx_mapping: dict[int, int] | None = None type: str = "sparse" def __init__( diff --git a/semantic_router/hybrid_layer.py b/semantic_router/hybrid_layer.py index aa65d7d7..d4c81b13 100644 --- a/semantic_router/hybrid_layer.py +++ b/semantic_router/hybrid_layer.py @@ -20,7 +20,7 @@ class HybridRouteLayer: def __init__( self, encoder: BaseEncoder, - sparse_encoder: Optional[BaseEncoder] = None, + sparse_encoder: Optional[BM25Encoder] = None, routes: list[Route] = [], alpha: float = 0.3, ): diff --git a/semantic_router/layer.py b/semantic_router/layer.py index 082e24b6..cf546bfc 100644 --- a/semantic_router/layer.py +++ b/semantic_router/layer.py @@ -158,6 +158,7 @@ def __init__( encoder: Optional[BaseEncoder] = None, llm: Optional[BaseLLM] = None, routes: Optional[list[Route]] = None, + top_k_routes: int = 3, ): logger.info("Initializing RouteLayer") self.index = None diff --git a/semantic_router/schema.py b/semantic_router/schema.py index 88dd753c..bb1a4c6a 100644 --- a/semantic_router/schema.py +++ b/semantic_router/schema.py @@ -23,6 +23,8 @@ class EncoderType(Enum): class RouteChoice(BaseModel): name: Optional[str] = None function_call: Optional[dict] = None + similarity_score: Optional[float] = None + trigger: Optional[bool] = None @dataclass From 107cdbdd186dd044567a44f56014d39c47f39300 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 10 Jan 2024 12:47:27 +0200 Subject: [PATCH 11/14] fix: lint --- semantic_router/encoders/bm25.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantic_router/encoders/bm25.py b/semantic_router/encoders/bm25.py index 0fc55173..451273cd 100644 --- a/semantic_router/encoders/bm25.py +++ b/semantic_router/encoders/bm25.py @@ -6,7 +6,7 @@ class BM25Encoder(BaseEncoder): model: Optional[Any] = None - idx_mapping: dict[int, int] | None = None + idx_mapping: Optional[dict[int, int]] = None type: str = "sparse" def __init__( From 2292e375616465723c924e01eeb9e0781d5cc5e1 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 10 Jan 2024 13:38:42 +0200 Subject: [PATCH 12/14] feat: updated pre-commit --- .pre-commit-config.yaml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index daf970d5..811aabc8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,13 +34,11 @@ repos: args: [ "--write-changes", "--ignore-words-list", "asend" ] exclude: "poetry.lock" - - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: check-vcs-permalinks - id: end-of-file-fixer - # exclude: "tests/((commands|data)/|test_).+" - id: trailing-whitespace args: [ --markdown-linebreak-ext=md ] - id: debug-statements @@ -60,8 +58,24 @@ repos: - post-commit - push - - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook - rev: v9.11.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.290 + hooks: + - id: ruff + types_or: [python, pyi, jupyter] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.8.0 + hooks: + - id: mypy + args: [--ignore-missing-imports] + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.6 + hooks: + - id: bandit + + - repo: https://github.com/Lucas-C/pre-commit-hooks-safety + rev: v1.3.2 hooks: - - id: commitlint - stages: [commit-msg] + - id: python-safety-dependencies-check From a359453f54c69b28968cfed374b9cb48f1c3a487 Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 10 Jan 2024 13:42:04 +0200 Subject: [PATCH 13/14] feat: updated pre-commit --- .pre-commit-config.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 811aabc8..82eccf67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,16 +64,17 @@ repos: - id: ruff types_or: [python, pyi, jupyter] - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 - hooks: - - id: mypy - args: [--ignore-missing-imports] + # - repo: https://github.com/pre-commit/mirrors-mypy + # rev: v1.8.0 + # hooks: + # - id: mypy + # args: [--ignore-missing-imports] - repo: https://github.com/PyCQA/bandit rev: 1.7.6 hooks: - id: bandit + args: ['-lll'] - repo: https://github.com/Lucas-C/pre-commit-hooks-safety rev: v1.3.2 From c8873e7b08e256a193f2faeebca185418d00fbca Mon Sep 17 00:00:00 2001 From: Simonas <20096648+simjak@users.noreply.github.com> Date: Wed, 10 Jan 2024 13:43:55 +0200 Subject: [PATCH 14/14] fix: removed precommit hooks safety --- .pre-commit-config.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82eccf67..b24594f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -75,8 +75,3 @@ repos: hooks: - id: bandit args: ['-lll'] - - - repo: https://github.com/Lucas-C/pre-commit-hooks-safety - rev: v1.3.2 - hooks: - - id: python-safety-dependencies-check