feat: adds triggers and segments

Cargo.toml

@@ -27,7 +27,9 @@ tonic = "0.12.3"
 dashmap = { version = "6.1.0", features = ["rayon"] }
 csv = "1.3.1"
 
-ptrie = { git = "https://github.com/oramasearch/ptrie.git", branch = "feat/expose-find-postfixes-with-current" }
+ptrie = { git = "https://github.com/oramasearch/ptrie.git", branch = "feat/expose-find-postfixes-with-current", features = [
+    "serde",
+] }
 
 regex = "1.11.1"
 

config.yaml

@@ -10,16 +10,16 @@ log:
 
 writer_side:
     output:
-        # type: in-memory
-        type: rabbitmq
+        type: in-memory
+        # type: rabbitmq
         # host: localhost
         # port: 5672
         # user: guest
         # password: guest
         # v_host: /
-        stream_name: oramacore-operations
-        client_provided_name: oramacore-producer
-        producer_name: write
+        # stream_name: oramacore-operations
+        # client_provided_name: oramacore-producer
+        # producer_name: write
 
     # Replace the following value with your own API key
     master_api_key: my-master-api-key
@@ -42,16 +42,16 @@ writer_side:
 
 reader_side:
     input:
-        # type: in-memory
-        type: rabbitmq
-        # host: localhost
-        # port: 5672
-        # user: guest
-        # password: guest
-        # v_host: /
-        stream_name: oramacore-operations
-        client_provided_name: oramacore-producer
-        consumer_name: reader
+        type: in-memory
+    # type: rabbitmq
+    # host: localhost
+    # port: 5672
+    # user: guest
+    # password: guest
+    # v_host: /
+    # stream_name: oramacore-operations
+    # client_provided_name: oramacore-producer
+    # consumer_name: reader
 
     config:
         data_dir: ./.data/reader

src/ai_server/src/utils.py

@@ -4,14 +4,15 @@
 from typing import List, Optional
 from dataclasses import dataclass, field
 
-
-DEFAULT_GENERAL_MODEL = "Qwen/Qwen2.5-3B-Instruct"
-DEFAULT_VISION_MODEL = "microsoft/Phi-3.5-vision-instruct"
-DEFAULT_CONTENT_EXPANSION_MODEL = "Qwen/Qwen2.5-3B-Instruct"
-DEFAULT_GOOGLE_QUERY_TRANSLATOR_MODEL = "Qwen/Qwen2.5-3B-Instruct"
-DEFAULT_ANSWER_MODEL = "Qwen/Qwen2.5-3B-Instruct"
-DEFAULT_ANSWER_PLANNING_MODEL = "Qwen/Qwen2.5-3B-Instruct"
-DEFAULT_ACTION_MODEL = "Qwen/Qwen2.5-3B-Instruct"
+BASE_MODEL = "Qwen/Qwen2.5-3B-Instruct"
+
+DEFAULT_GENERAL_MODEL = BASE_MODEL
+DEFAULT_VISION_MODEL = BASE_MODEL
+DEFAULT_CONTENT_EXPANSION_MODEL = BASE_MODEL
+DEFAULT_GOOGLE_QUERY_TRANSLATOR_MODEL = BASE_MODEL
+DEFAULT_ANSWER_MODEL = BASE_MODEL
+DEFAULT_ANSWER_PLANNING_MODEL = BASE_MODEL
+DEFAULT_ACTION_MODEL = BASE_MODEL
 
 
 @dataclass

src/collection_manager/sides/generic_kv.rs

@@ -0,0 +1,25 @@
+use ptrie::Trie;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct KV<V = String> {
+    data: Trie<u8, V>,
+}
+
+impl<V: Clone> KV<V> {
+    pub fn new() -> Self {
+        KV { data: Trie::new() }
+    }
+
+    pub fn insert(&mut self, key: String, value: V) {
+        self.data.insert(key.as_bytes().iter().cloned(), value);
+    }
+
+    pub fn get(&self, key: &str) -> Option<&V> {
+        self.data.get(key.as_bytes().iter().cloned())
+    }
+
+    pub fn remove(&mut self, key: &str) -> Option<V> {
+        self.data.remove(key.as_bytes().iter().cloned())
+    }
+}

src/collection_manager/sides/mod.rs

@@ -1,3 +1,4 @@
+pub mod generic_kv;
 pub mod hooks;
 mod operation;
 mod read;

src/collection_manager/sides/write/collections.rs

@@ -7,6 +7,7 @@ use redact::Secret;
 use tokio::sync::{RwLock, RwLockReadGuard};
 use tracing::info;
 
+use crate::collection_manager::sides::generic_kv::KV;
 use crate::collection_manager::sides::hooks::HooksRuntime;
 use crate::collection_manager::sides::write::collection::DEFAULT_EMBEDDING_FIELD_NAME;
 use crate::collection_manager::sides::{OperationSender, OramaModelSerializable, WriteOperation};
@@ -36,6 +37,7 @@ impl CollectionsWriter {
         embedding_sender: tokio::sync::mpsc::Sender<EmbeddingCalculationRequest>,
         hooks_runtime: Arc<HooksRuntime>,
         nlp_service: Arc<NLPService>,
+        kv: Arc<KV>,
     ) -> Result<Self> {
         let mut collections: HashMap<CollectionId, CollectionWriter> = Default::default();
 

src/collection_manager/sides/write/mod.rs

@@ -14,6 +14,7 @@ use std::{
 };
 
 use super::{
+    generic_kv::KV,
     hooks::{HookName, HooksRuntime},
     Offset, OperationSender, OperationSenderCreator, OutputSideChannelType,
 };
@@ -82,6 +83,7 @@ impl WriteSide {
         ai_service: Arc<AIService>,
         hook_runtime: Arc<HooksRuntime>,
         nlp_service: Arc<NLPService>,
+        kv: Arc<KV>,
     ) -> Result<Arc<Self>> {
         let master_api_key = config.master_api_key;
         let collections_writer_config = config.config;
@@ -120,6 +122,7 @@ impl WriteSide {
             sx,
             hook_runtime.clone(),
             nlp_service.clone(),
+            kv,
         )
         .await
         .context("Cannot load collections")?;