From 306ee6339269a32bc7e3936252826b1df7b0e810 Mon Sep 17 00:00:00 2001 From: alisalim17 Date: Wed, 27 Mar 2024 10:41:04 +0400 Subject: [PATCH 1/2] feat: add pgvector --- README.md | 2 +- models/vector_database.py | 1 + poetry.lock | 134 +++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + vectordbs/__init__.py | 2 + vectordbs/pgvector.py | 95 +++++++++++++++++++++++++++ 6 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 vectordbs/pgvector.py diff --git a/README.md b/README.md index 4bf1ba4c..d73474d2 100644 --- a/README.md +++ b/README.md @@ -166,5 +166,5 @@ Super-Rag comes with a built in REST API powered by FastApi. - Qdrant - Weaviate - Astra -- Supabase (coming soon) +- PGVector - Chroma (coming soon) diff --git a/models/vector_database.py b/models/vector_database.py index 6fc2cf8d..a74dd01a 100644 --- a/models/vector_database.py +++ b/models/vector_database.py @@ -9,6 +9,7 @@ class DatabaseType(Enum): pinecone = "pinecone" weaviate = "weaviate" astra = "astra" + pgvector = "pgvector" class VectorDatabase(BaseModel): diff --git a/poetry.lock b/poetry.lock index 6889e530..548ea750 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1039,6 +1039,22 @@ mccabe = ">=0.7.0,<0.8.0" pycodestyle = ">=2.11.0,<2.12.0" pyflakes = ">=3.2.0,<3.3.0" +[[package]] +name = "flupy" +version = "1.2.0" +description = "Method chaining built on generators" +optional = false +python-versions = "*" +files = [ + {file = "flupy-1.2.0.tar.gz", hash = "sha256:12487a008e9744cd35d0f6ea3cfa06f4b2b27cb138bf57d0788f5c26e57afe69"}, +] + +[package.dependencies] +typing_extensions = "*" + +[package.extras] +dev = ["black", "mypy", "pre-commit", "pylint", "pytest", "pytest-benchmark", "pytest-cov"] + [[package]] name = "frozenlist" version = "1.4.1" @@ -2420,6 +2436,19 @@ files = [ [package.dependencies] ptyprocess = ">=0.5" +[[package]] +name = "pgvector" +version = "0.1.8" +description = "pgvector support for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pgvector-0.1.8-py2.py3-none-any.whl", hash = "sha256:99dce3a6580ef73863edb9b8441937671f4e1a09383826e6b0838176cd441a96"}, +] + +[package.dependencies] +numpy = "*" + [[package]] name = "pinecone-client" version = "3.1.0" @@ -2536,6 +2565,87 @@ files = [ [package.extras] test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +[[package]] +name = "psycopg2-binary" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -3933,6 +4043,28 @@ testing = ["pytest (>=7.4.0)"] tooling = ["black (>=23.7.0)", "pyright (>=1.1.325)", "ruff (>=0.0.287)"] tooling-extras = ["pyaml (>=23.7.0)", "pypandoc-binary (>=1.11)", "pytest (>=7.4.0)"] +[[package]] +name = "vecs" +version = "0.4.3" +description = "pgvector client" +optional = false +python-versions = "*" +files = [ + {file = "vecs-0.4.3.tar.gz", hash = "sha256:0a60294143aec43bd0344bb9235b6e57f8f919d102538f6b989d7b85095a31ce"}, +] + +[package.dependencies] +deprecated = "==1.2.*" +flupy = "==1.*" +pgvector = "==0.1.*" +psycopg2-binary = "==2.9.*" +sqlalchemy = "==2.*" + +[package.extras] +dev = ["numpy", "parse", "pytest", "pytest-cov"] +docs = ["mike", "mkdocs", "pygments", "pymarkdown", "pymdown-extensions"] +text-embedding = ["sentence-transformers (==2.*)"] + [[package]] name = "vulture" version = "2.11" @@ -4258,4 +4390,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "b9774decb9338d39235bb4599347540fa5a684cecd89d194a95b26257be43364" +content-hash = "a653f64fd5fa81106a858c03a2ef477c23e3a4d93954ce4e1491ced981ca7469" diff --git a/pyproject.toml b/pyproject.toml index 7263d176..eb796451 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ gunicorn = "^21.2.0" unstructured-client = "^0.18.0" unstructured = {extras = ["google-drive"], version = "^0.12.4"} tiktoken = "^0.6.0" +vecs = "^0.4.3" [tool.poetry.group.dev.dependencies] termcolor = "^2.4.0" diff --git a/vectordbs/__init__.py b/vectordbs/__init__.py index a54a58a5..00b4d920 100644 --- a/vectordbs/__init__.py +++ b/vectordbs/__init__.py @@ -10,6 +10,7 @@ from vectordbs.pinecone import PineconeService from vectordbs.qdrant import QdrantService from vectordbs.weaviate import WeaviateService +from vectordbs.pgvector import PGVectorService load_dotenv() @@ -26,6 +27,7 @@ def get_vector_service( "qdrant": QdrantService, "weaviate": WeaviateService, "astra": AstraService, + "pgvector": PGVectorService, # Add other providers here # e.g "weaviate": WeaviateVectorService, } diff --git a/vectordbs/pgvector.py b/vectordbs/pgvector.py new file mode 100644 index 00000000..e6b185ec --- /dev/null +++ b/vectordbs/pgvector.py @@ -0,0 +1,95 @@ +from typing import List + +import vecs +from semantic_router.encoders import BaseEncoder +from tqdm import tqdm + +from qdrant_client.http import models as rest +from models.delete import DeleteResponse +from models.document import BaseDocumentChunk +from vectordbs.base import BaseVectorDatabase + +MAX_QUERY_TOP_K = 5 + + +class PGVectorService(BaseVectorDatabase): + def __init__( + self, index_name: str, dimension: int, credentials: dict, encoder: BaseEncoder + ): + super().__init__( + index_name=index_name, + dimension=dimension, + credentials=credentials, + encoder=encoder, + ) + client = vecs.create_client(connection_string=credentials["database_uri"]) + self.collection = client.get_or_create_collection( + name=self.index_name, + dimension=dimension, + ) + + # TODO: remove this + async def convert_to_rerank_format(self, chunks: List[rest.PointStruct]): + docs = [ + { + "content": chunk.payload.get("content"), + "page_label": chunk.payload.get("page_label"), + "file_url": chunk.payload.get("file_url"), + } + for chunk in chunks + ] + return docs + + async def upsert(self, chunks: List[BaseDocumentChunk]) -> None: + records = [] + for chunk in tqdm(chunks, desc="Upserting to PGVector"): + records.append( + ( + chunk.id, + chunk.dense_embedding, + { + "document_id": chunk.document_id, + "content": chunk.content, + "doc_url": chunk.doc_url, + **(chunk.metadata if chunk.metadata else {}), + }, + ) + ) + self.collection.upsert(records) + self.collection.create_index() + + async def query(self, input: str, top_k: int = MAX_QUERY_TOP_K) -> List: + vectors = await self._generate_vectors(input=input) + + results = self.collection.query( + data=vectors[0], + limit=top_k, + include_metadata=True, + include_value=False, + ) + + chunks = [] + + for result in results: + ( + id, + metadata, + ) = result + + chunks.append( + BaseDocumentChunk( + id=id, + source_type=metadata.get("filetype"), + source=metadata.get("doc_url"), + document_id=metadata.get("document_id"), + content=metadata.get("content"), + doc_url=metadata.get("doc_url"), + page_number=metadata.get("page_number"), + metadata={**metadata}, + ) + ) + return chunks + + async def delete(self, file_url: str) -> None: + deleted = self.collection.delete(filters={"doc_url": {"$eq": file_url}}) + return DeleteResponse(num_of_deleted_chunks=len(deleted)) From 4ba341838fc95b55132b2ffd204210ba1c08ed19 Mon Sep 17 00:00:00 2001 From: alisalim17 Date: Wed, 27 Mar 2024 10:42:00 +0400 Subject: [PATCH 2/2] feat: delete summary index too --- api/delete.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/delete.py b/api/delete.py index 493049da..51644b6b 100644 --- a/api/delete.py +++ b/api/delete.py @@ -3,6 +3,7 @@ from models.delete import RequestPayload, ResponsePayload from vectordbs import get_vector_service from vectordbs.base import BaseVectorDatabase +from utils.summarise import SUMMARY_SUFFIX router = APIRouter() @@ -16,8 +17,15 @@ async def delete(payload: RequestPayload): encoder=encoder, dimensions=payload.encoder.dimensions, ) + summary_vector_service: BaseVectorDatabase = get_vector_service( + index_name=f"{payload.index_name}{SUMMARY_SUFFIX}", + credentials=payload.vector_database, + encoder=encoder, + dimensions=payload.encoder.dimensions, + ) for file in payload.files: data = await vector_service.delete(file_url=file.url) + await summary_vector_service.delete(file_url=file.url) return ResponsePayload(success=True, data=data)