From 98d8e9671f1728461c6075e93406b069252fe705 Mon Sep 17 00:00:00 2001 From: Michele Riva Date: Tue, 18 Feb 2025 12:01:40 +0100 Subject: [PATCH] feat: adds triggers and segments --- Cargo.lock | 210 +++++++++--------- Cargo.toml | 4 +- config.yaml | 30 +-- src/ai_server/src/utils.py | 17 +- src/collection_manager/sides/generic_kv.rs | 25 +++ src/collection_manager/sides/mod.rs | 1 + .../sides/write/collections.rs | 2 + src/collection_manager/sides/write/mod.rs | 3 + 8 files changed, 164 insertions(+), 128 deletions(-) create mode 100644 src/collection_manager/sides/generic_kv.rs diff --git a/Cargo.lock b/Cargo.lock index 6a37d26..9a2c231 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -289,9 +289,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.25.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b2ddd3ada61a305e1d8bb6c005d1eaa7d14d903681edfc400406d523a9b491" +checksum = "54ac4f13dad353b209b34cbec082338202cbc01c8f00336b55c750c13ac91f8f" dependencies = [ "bindgen 0.69.5", "cc", @@ -490,7 +490,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.8.3", + "miniz_oxide 0.8.4", "object", "rustc-demangle", "windows-targets 0.52.6", @@ -672,9 +672,9 @@ dependencies = [ [[package]] name = "built" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" [[package]] name = "bumpalo" @@ -705,9 +705,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" [[package]] name = "capacity_builder" @@ -735,9 +735,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.11" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4730490333d58093109dc02c23174c3f4d490998c3fed3cc8e82d57afedb9cf" +checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" dependencies = [ "jobserver", "libc", @@ -823,18 +823,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.27" +version = "4.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" +checksum = "92b7b18d71fad5313a1e320fa9897994228ce274b60faa4d694fe0ea89cd9e6d" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.27" +version = "4.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +checksum = "a35db2071778a7344791a4fb4f95308b5673d219dee3ae348b86642574ecc90c" dependencies = [ "anstream", "anstyle", @@ -851,9 +851,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] @@ -896,9 +896,9 @@ dependencies = [ [[package]] name = "config" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e26695492a475c4a091cfda61446d5ba01aac2e1dfbcd27a12fdd11aa2e32596" +checksum = "8cf9dc8d4ef88e27a8cb23e85cb116403dedd57f7971964dc4b18ccead548901" dependencies = [ "async-trait", "convert_case 0.6.0", @@ -908,8 +908,8 @@ dependencies = [ "rust-ini", "serde", "serde_json", - "toml 0.8.19", - "winnow 0.7.0", + "toml 0.8.20", + "winnow 0.7.2", "yaml-rust2", ] @@ -1128,9 +1128,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" dependencies = [ "memchr", ] @@ -1172,9 +1172,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.78+curl-8.11.0" +version = "0.4.80+curl-8.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eec768341c5c7789611ae51cf6c459099f22e64a5d5d0ce4892434e33821eaf" +checksum = "55f7df2eac63200c3ab25bde3b2268ef2ee56af3d238e76d61f01c3c49bff734" dependencies = [ "cc", "libc", @@ -1243,9 +1243,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" +checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" [[package]] name = "debugid" @@ -1594,9 +1594,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -1623,7 +1623,7 @@ dependencies = [ "bit_field", "half", "lebe", - "miniz_oxide 0.8.3", + "miniz_oxide 0.8.4", "rayon-core", "smallvec", "zune-inflate", @@ -1689,6 +1689,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.0.35" @@ -1696,7 +1702,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", - "miniz_oxide 0.8.3", + "miniz_oxide 0.8.4", ] [[package]] @@ -2076,7 +2082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c42c851d1dc97de92a1a50574295b1b36f0a9443cb33e05eb379147bd34bf1ff" dependencies = [ "bincode", - "fixedbitset", + "fixedbitset 0.4.2", "log", "num", "rand 0.8.5", @@ -2995,9 +3001,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b" dependencies = [ "adler2", "simd-adler32", @@ -3280,9 +3286,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.2" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "onig" @@ -3314,9 +3320,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" -version = "0.10.70" +version = "0.10.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -3346,9 +3352,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.105" +version = "0.9.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" dependencies = [ "cc", "libc", @@ -3534,7 +3540,7 @@ dependencies = [ "allocator-api2", "bumpalo", "hashbrown 0.15.2", - "rustc-hash 2.1.0", + "rustc-hash 2.1.1", "simdutf8", ] @@ -3620,7 +3626,7 @@ dependencies = [ "oxc_regular_expression", "oxc_span", "oxc_syntax", - "rustc-hash 2.1.0", + "rustc-hash 2.1.1", "seq-macro", ] @@ -3636,7 +3642,7 @@ dependencies = [ "oxc_estree", "oxc_span", "phf", - "rustc-hash 2.1.0", + "rustc-hash 2.1.1", "unicode-id-start", ] @@ -3669,7 +3675,7 @@ dependencies = [ "oxc_index", "oxc_span", "phf", - "rustc-hash 2.1.0", + "rustc-hash 2.1.1", "ryu-js", "unicode-id-start", ] @@ -3762,11 +3768,11 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.5" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ - "fixedbitset", + "fixedbitset 0.5.7", "indexmap 2.7.1", ] @@ -3888,7 +3894,7 @@ dependencies = [ "crc32fast", "fdeflate", "flate2", - "miniz_oxide 0.8.3", + "miniz_oxide 0.8.4", ] [[package]] @@ -4003,9 +4009,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", "prost-derive", @@ -4013,12 +4019,12 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -4033,12 +4039,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -4046,9 +4052,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ "prost", ] @@ -4057,6 +4063,9 @@ dependencies = [ name = "ptrie" version = "0.7.1" source = "git+https://github.com/oramasearch/ptrie.git?branch=feat%2Fexpose-find-postfixes-with-current#44a1c1824702334a2dba9c86219889ecad7579a0" +dependencies = [ + "serde", +] [[package]] name = "pulldown-cmark" @@ -4181,8 +4190,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.0", - "zerocopy 0.8.14", + "rand_core 0.9.1", + "zerocopy 0.8.18", ] [[package]] @@ -4202,7 +4211,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.0", + "rand_core 0.9.1", ] [[package]] @@ -4216,12 +4225,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.14", + "zerocopy 0.8.18", ] [[package]] @@ -4285,9 +4294,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.3.0" +version = "11.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6928fa44c097620b706542d428957635951bade7143269085389d42c8a4927e" +checksum = "529468c1335c1c03919960dfefdb1b3648858c20d7ec2d0663e728e4a717efbc" dependencies = [ "bitflags 2.8.0", ] @@ -4461,15 +4470,14 @@ checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" [[package]] name = "ring" -version = "0.17.8" +version = "0.17.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "e75ec5e92c4d8aede845126adc388046234541629e76029599ed35a003c7ed24" dependencies = [ "cc", "cfg-if", "getrandom 0.2.15", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] @@ -4537,9 +4545,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -4574,9 +4582,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.22" +version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "aws-lc-rs", "log", @@ -4641,9 +4649,9 @@ checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "ryu-js" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad97d4ce1560a5e27cec89519dc8300d1aa6035b099821261c651486a19e44d5" +checksum = "dd29631678d6fb0903b69223673e122c32e9ae559d0960a38d574695ebc0ea15" [[package]] name = "same-file" @@ -4925,9 +4933,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" dependencies = [ "serde", ] @@ -4978,12 +4986,6 @@ dependencies = [ "url", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] name = "spm_precompiled" version = "0.1.4" @@ -5137,7 +5139,7 @@ dependencies = [ "cfg-expr", "heck 0.5.0", "pkg-config", - "toml 0.8.19", + "toml 0.8.20", "version-compare", ] @@ -5166,9 +5168,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.16.0" +version = "3.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" +checksum = "22e5a0acb1f3f55f65cc4a866c361b2fb2a0ff6366785ae6fbb5f85df07ba230" dependencies = [ "cfg-if", "fastrand", @@ -5470,9 +5472,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" dependencies = [ "serde", "serde_spanned", @@ -5491,15 +5493,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.23" +version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", - "winnow 0.7.0", + "winnow 0.7.2", ] [[package]] @@ -5720,9 +5722,9 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38eee4db33814de3d004de9d8d825627ed3320d0989cce0dea30efaf5be4736c" +checksum = "c4013970217383f67b18aef68f6fb2e8d409bc5755227092d32efb0422ba24b8" [[package]] name = "tree-sitter-typescript" @@ -5748,9 +5750,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "ucd-trie" @@ -5896,12 +5898,12 @@ dependencies = [ [[package]] name = "uuid" -version = "1.12.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" +checksum = "8c1f41ffb7cf259f1ecc2876861a17e7142e63ead296f671f81f6ae85903e0d6" dependencies = [ - "getrandom 0.2.15", - "rand 0.8.5", + "getrandom 0.3.1", + "rand 0.9.0", ] [[package]] @@ -6359,9 +6361,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e49d2d35d3fad69b39b94139037ecfb4f359f08958b9c11e7315ce770462419" +checksum = "59690dea168f2198d1a3b0cac23b8063efcd11012f10ae4698f284808c8ef603" dependencies = [ "memchr", ] @@ -6481,11 +6483,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.14" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a367f292d93d4eab890745e75a778da40909cab4d6ff8173693812f79c4a2468" +checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" dependencies = [ - "zerocopy-derive 0.8.14", + "zerocopy-derive 0.8.18", ] [[package]] @@ -6501,9 +6503,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.14" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3931cb58c62c13adec22e38686b559c86a30565e16ad6e8510a337cedc611e1" +checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index d9fbf39..b11cd75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,9 @@ tonic = "0.12.3" dashmap = { version = "6.1.0", features = ["rayon"] } csv = "1.3.1" -ptrie = { git = "https://github.com/oramasearch/ptrie.git", branch = "feat/expose-find-postfixes-with-current" } +ptrie = { git = "https://github.com/oramasearch/ptrie.git", branch = "feat/expose-find-postfixes-with-current", features = [ + "serde", +] } regex = "1.11.1" diff --git a/config.yaml b/config.yaml index 9a4644d..77c5908 100644 --- a/config.yaml +++ b/config.yaml @@ -10,16 +10,16 @@ log: writer_side: output: - # type: in-memory - type: rabbitmq + type: in-memory + # type: rabbitmq # host: localhost # port: 5672 # user: guest # password: guest # v_host: / - stream_name: oramacore-operations - client_provided_name: oramacore-producer - producer_name: write + # stream_name: oramacore-operations + # client_provided_name: oramacore-producer + # producer_name: write # Replace the following value with your own API key master_api_key: my-master-api-key @@ -42,16 +42,16 @@ writer_side: reader_side: input: - # type: in-memory - type: rabbitmq - # host: localhost - # port: 5672 - # user: guest - # password: guest - # v_host: / - stream_name: oramacore-operations - client_provided_name: oramacore-producer - consumer_name: reader + type: in-memory + # type: rabbitmq + # host: localhost + # port: 5672 + # user: guest + # password: guest + # v_host: / + # stream_name: oramacore-operations + # client_provided_name: oramacore-producer + # consumer_name: reader config: data_dir: ./.data/reader diff --git a/src/ai_server/src/utils.py b/src/ai_server/src/utils.py index 469bf77..9a4ad32 100644 --- a/src/ai_server/src/utils.py +++ b/src/ai_server/src/utils.py @@ -4,14 +4,15 @@ from typing import List, Optional from dataclasses import dataclass, field - -DEFAULT_GENERAL_MODEL = "Qwen/Qwen2.5-3B-Instruct" -DEFAULT_VISION_MODEL = "microsoft/Phi-3.5-vision-instruct" -DEFAULT_CONTENT_EXPANSION_MODEL = "Qwen/Qwen2.5-3B-Instruct" -DEFAULT_GOOGLE_QUERY_TRANSLATOR_MODEL = "Qwen/Qwen2.5-3B-Instruct" -DEFAULT_ANSWER_MODEL = "Qwen/Qwen2.5-3B-Instruct" -DEFAULT_ANSWER_PLANNING_MODEL = "Qwen/Qwen2.5-3B-Instruct" -DEFAULT_ACTION_MODEL = "Qwen/Qwen2.5-3B-Instruct" +BASE_MODEL = "Qwen/Qwen2.5-3B-Instruct" + +DEFAULT_GENERAL_MODEL = BASE_MODEL +DEFAULT_VISION_MODEL = BASE_MODEL +DEFAULT_CONTENT_EXPANSION_MODEL = BASE_MODEL +DEFAULT_GOOGLE_QUERY_TRANSLATOR_MODEL = BASE_MODEL +DEFAULT_ANSWER_MODEL = BASE_MODEL +DEFAULT_ANSWER_PLANNING_MODEL = BASE_MODEL +DEFAULT_ACTION_MODEL = BASE_MODEL @dataclass diff --git a/src/collection_manager/sides/generic_kv.rs b/src/collection_manager/sides/generic_kv.rs new file mode 100644 index 0000000..1ae08d1 --- /dev/null +++ b/src/collection_manager/sides/generic_kv.rs @@ -0,0 +1,25 @@ +use ptrie::Trie; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KV { + data: Trie, +} + +impl KV { + pub fn new() -> Self { + KV { data: Trie::new() } + } + + pub fn insert(&mut self, key: String, value: V) { + self.data.insert(key.as_bytes().iter().cloned(), value); + } + + pub fn get(&self, key: &str) -> Option<&V> { + self.data.get(key.as_bytes().iter().cloned()) + } + + pub fn remove(&mut self, key: &str) -> Option { + self.data.remove(key.as_bytes().iter().cloned()) + } +} diff --git a/src/collection_manager/sides/mod.rs b/src/collection_manager/sides/mod.rs index 1dd6ed9..69a0b0f 100644 --- a/src/collection_manager/sides/mod.rs +++ b/src/collection_manager/sides/mod.rs @@ -1,3 +1,4 @@ +pub mod generic_kv; pub mod hooks; mod operation; mod read; diff --git a/src/collection_manager/sides/write/collections.rs b/src/collection_manager/sides/write/collections.rs index 3fd2ffa..5c00168 100644 --- a/src/collection_manager/sides/write/collections.rs +++ b/src/collection_manager/sides/write/collections.rs @@ -7,6 +7,7 @@ use redact::Secret; use tokio::sync::{RwLock, RwLockReadGuard}; use tracing::info; +use crate::collection_manager::sides::generic_kv::KV; use crate::collection_manager::sides::hooks::HooksRuntime; use crate::collection_manager::sides::write::collection::DEFAULT_EMBEDDING_FIELD_NAME; use crate::collection_manager::sides::{OperationSender, OramaModelSerializable, WriteOperation}; @@ -36,6 +37,7 @@ impl CollectionsWriter { embedding_sender: tokio::sync::mpsc::Sender, hooks_runtime: Arc, nlp_service: Arc, + kv: Arc, ) -> Result { let mut collections: HashMap = Default::default(); diff --git a/src/collection_manager/sides/write/mod.rs b/src/collection_manager/sides/write/mod.rs index 0409238..dd6f509 100644 --- a/src/collection_manager/sides/write/mod.rs +++ b/src/collection_manager/sides/write/mod.rs @@ -14,6 +14,7 @@ use std::{ }; use super::{ + generic_kv::KV, hooks::{HookName, HooksRuntime}, Offset, OperationSender, OperationSenderCreator, OutputSideChannelType, }; @@ -82,6 +83,7 @@ impl WriteSide { ai_service: Arc, hook_runtime: Arc, nlp_service: Arc, + kv: Arc, ) -> Result> { let master_api_key = config.master_api_key; let collections_writer_config = config.config; @@ -120,6 +122,7 @@ impl WriteSide { sx, hook_runtime.clone(), nlp_service.clone(), + kv, ) .await .context("Cannot load collections")?;