From ddec3709a945a36ef1aa65e815bfe3164ab87a97 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 18 Dec 2025 00:33:45 +0000 Subject: [PATCH 1/5] Enhance worker management in start-workers.sh and update docker-compose.yml for OpenMemory MCP - Added conditional start for Deepgram worker based on DEEPGRAM_API_KEY presence. - Updated shutdown and monitoring functions to handle AUDIO_STREAM_WORKER_PID safely. - Changed OpenMemory MCP service to use the official Docker image instead of a local build. - Removed deprecated init-cache.sh script and streamlined setup.sh for clarity. --- backends/advanced/start-workers.sh | 20 ++++++++++----- extras/openmemory-mcp/docker-compose.yml | 11 +++------ extras/openmemory-mcp/init-cache.sh | 31 ------------------------ extras/openmemory-mcp/setup.sh | 25 ++----------------- 4 files changed, 20 insertions(+), 67 deletions(-) delete mode 100755 extras/openmemory-mcp/init-cache.sh diff --git a/backends/advanced/start-workers.sh b/backends/advanced/start-workers.sh index f62b5a42..48f2c5a3 100755 --- a/backends/advanced/start-workers.sh +++ b/backends/advanced/start-workers.sh @@ -51,9 +51,16 @@ start_workers() { uv run python -m advanced_omi_backend.workers.rq_worker_entry audio & AUDIO_PERSISTENCE_WORKER_PID=$! - echo "🎵 Starting audio stream Deepgram worker (1 worker for sequential processing)..." - uv run python -m advanced_omi_backend.workers.audio_stream_deepgram_worker & - AUDIO_STREAM_WORKER_PID=$! + # Only start Deepgram worker if DEEPGRAM_API_KEY is set + if [ -n "$DEEPGRAM_API_KEY" ]; then + echo "🎵 Starting audio stream Deepgram worker (1 worker for sequential processing)..." + uv run python -m advanced_omi_backend.workers.audio_stream_deepgram_worker & + AUDIO_STREAM_WORKER_PID=$! + else + echo "⏭️ Skipping Deepgram worker (DEEPGRAM_API_KEY not set)" + # Set dummy PID to avoid script errors + AUDIO_STREAM_WORKER_PID="" + fi echo "✅ All workers started:" echo " - RQ worker 1: PID $RQ_WORKER_1_PID (transcription, memory, default)" @@ -103,7 +110,8 @@ monitor_worker_health() { echo "🔧 Self-healing: Restarting all workers to restore registration..." # Kill all workers - kill $RQ_WORKER_1_PID $RQ_WORKER_2_PID $RQ_WORKER_3_PID $RQ_WORKER_4_PID $RQ_WORKER_5_PID $RQ_WORKER_6_PID $AUDIO_PERSISTENCE_WORKER_PID $AUDIO_STREAM_WORKER_PID 2>/dev/null || true + kill $RQ_WORKER_1_PID $RQ_WORKER_2_PID $RQ_WORKER_3_PID $RQ_WORKER_4_PID $RQ_WORKER_5_PID $RQ_WORKER_6_PID $AUDIO_PERSISTENCE_WORKER_PID 2>/dev/null || true + [ -n "$AUDIO_STREAM_WORKER_PID" ] && kill $AUDIO_STREAM_WORKER_PID 2>/dev/null || true wait 2>/dev/null || true # Restart workers @@ -128,7 +136,7 @@ shutdown() { kill $RQ_WORKER_5_PID 2>/dev/null || true kill $RQ_WORKER_6_PID 2>/dev/null || true kill $AUDIO_PERSISTENCE_WORKER_PID 2>/dev/null || true - kill $AUDIO_STREAM_WORKER_PID 2>/dev/null || true + [ -n "$AUDIO_STREAM_WORKER_PID" ] && kill $AUDIO_STREAM_WORKER_PID 2>/dev/null || true wait echo "✅ All workers stopped" exit 0 @@ -161,7 +169,7 @@ kill $RQ_WORKER_4_PID 2>/dev/null || true kill $RQ_WORKER_5_PID 2>/dev/null || true kill $RQ_WORKER_6_PID 2>/dev/null || true kill $AUDIO_PERSISTENCE_WORKER_PID 2>/dev/null || true -kill $AUDIO_STREAM_WORKER_PID 2>/dev/null || true +[ -n "$AUDIO_STREAM_WORKER_PID" ] && kill $AUDIO_STREAM_WORKER_PID 2>/dev/null || true wait echo "🔄 All workers stopped" diff --git a/extras/openmemory-mcp/docker-compose.yml b/extras/openmemory-mcp/docker-compose.yml index 4107cf4a..491debb3 100644 --- a/extras/openmemory-mcp/docker-compose.yml +++ b/extras/openmemory-mcp/docker-compose.yml @@ -8,11 +8,9 @@ services: - ./data/mem0_storage:/qdrant/storage restart: unless-stopped - # OpenMemory MCP Server (built from local cache) + # OpenMemory MCP Server (official Docker image) openmemory-mcp: - build: - context: ./cache/mem0/openmemory/api - dockerfile: Dockerfile + image: mem0/openmemory-mcp:latest env_file: - .env environment: @@ -30,7 +28,7 @@ services: retries: 3 start_period: 30s - # OpenMemory UI (optional - can be disabled if not needed) + # OpenMemory UI (starts by default with the MCP server) openmemory-ui: image: mem0/openmemory-ui:latest ports: @@ -40,5 +38,4 @@ services: - NEXT_PUBLIC_USER_ID=openmemory depends_on: - openmemory-mcp - profiles: - - ui # Only starts when --profile ui is used \ No newline at end of file + restart: unless-stopped \ No newline at end of file diff --git a/extras/openmemory-mcp/init-cache.sh b/extras/openmemory-mcp/init-cache.sh deleted file mode 100755 index 18ec6f6f..00000000 --- a/extras/openmemory-mcp/init-cache.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Initialize or update local cached mem0 from Ankush's fork - -CACHE_DIR="./cache/mem0" -FORK_REPO="https://github.com/AnkushMalaker/mem0.git" -BRANCH="fix/get-endpoint" - -echo "🔄 Updating OpenMemory cache from fork..." - -if [ ! -d "$CACHE_DIR/.git" ]; then - echo "📥 Initializing cache from fork..." - rm -rf "$CACHE_DIR" - git clone "$FORK_REPO" "$CACHE_DIR" - cd "$CACHE_DIR" - git checkout "$BRANCH" - echo "✅ Cache initialized from $FORK_REPO ($BRANCH)" -else - echo "🔄 Updating existing cache..." - cd "$CACHE_DIR" - git fetch origin - git checkout "$BRANCH" - git pull origin "$BRANCH" - echo "✅ Cache updated from $FORK_REPO ($BRANCH)" -fi - -echo "" -echo "📂 Cache directory: $(pwd)" -echo "🌿 Current branch: $(git branch --show-current)" -echo "📝 Latest commit: $(git log --oneline -1)" -echo "" -echo "🚀 Ready to build! Run: docker compose build openmemory-mcp --no-cache" \ No newline at end of file diff --git a/extras/openmemory-mcp/setup.sh b/extras/openmemory-mcp/setup.sh index 555720ec..afa8cf57 100755 --- a/extras/openmemory-mcp/setup.sh +++ b/extras/openmemory-mcp/setup.sh @@ -43,24 +43,6 @@ fi # Set restrictive permissions (owner read/write only) chmod 600 .env -# Clone the custom fork of mem0 with OpenMemory fixes -echo "" -echo "📦 Setting up custom mem0 fork with OpenMemory..." -if [ -d "cache/mem0" ]; then - echo " Removing existing mem0 directory..." - rm -rf cache/mem0 -fi - -echo " Cloning mem0 fork from AnkushMalaker/mem0..." -mkdir -p cache -git clone https://github.com/AnkushMalaker/mem0.git cache/mem0 -cd cache/mem0 -echo " Checking out fix/get-endpoint branch..." -git checkout fix/get-endpoint -cd ../.. - -echo "✅ Custom mem0 fork ready with OpenMemory improvements" - # Get OpenAI API Key (prompt only if not provided via command line) if [ -z "$OPENAI_API_KEY" ]; then echo "" @@ -91,9 +73,6 @@ echo "" echo "✅ OpenMemory MCP configured!" echo "📁 Configuration saved to .env" echo "" -echo "🚀 To start: docker compose up --build -d" +echo "🚀 To start: docker compose up -d" echo "🌐 MCP Server: http://localhost:8765" -echo "📱 Web Interface: http://localhost:8765" -echo "🔧 UI (optional): docker compose --profile ui up -d" -echo "" -echo "💡 Note: Using custom mem0 fork from AnkushMalaker/mem0:fix/get-endpoint" \ No newline at end of file +echo "📱 Web UI: http://localhost:3001" \ No newline at end of file From d25c3413e0770f16dd0a363c13491310caa73aa1 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 18 Dec 2025 00:42:04 +0000 Subject: [PATCH 2/5] Enhance audio stream worker management in start-workers.sh - Introduced conditional starting of Parakeet worker based on PARAKEET_ASR_URL or OFFLINE_ASR_TCP_URI presence. - Updated variable names for clarity, changing AUDIO_STREAM_WORKER_PID to AUDIO_STREAM_DEEPGRAM_WORKER_PID and added AUDIO_STREAM_PARAKEET_WORKER_PID. - Improved shutdown and monitoring functions to handle new worker PIDs safely. --- backends/advanced/start-workers.sh | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/backends/advanced/start-workers.sh b/backends/advanced/start-workers.sh index 48f2c5a3..c6052c07 100755 --- a/backends/advanced/start-workers.sh +++ b/backends/advanced/start-workers.sh @@ -51,15 +51,25 @@ start_workers() { uv run python -m advanced_omi_backend.workers.rq_worker_entry audio & AUDIO_PERSISTENCE_WORKER_PID=$! + # Start stream workers based on available configuration # Only start Deepgram worker if DEEPGRAM_API_KEY is set if [ -n "$DEEPGRAM_API_KEY" ]; then echo "🎵 Starting audio stream Deepgram worker (1 worker for sequential processing)..." uv run python -m advanced_omi_backend.workers.audio_stream_deepgram_worker & - AUDIO_STREAM_WORKER_PID=$! + AUDIO_STREAM_DEEPGRAM_WORKER_PID=$! else - echo "⏭️ Skipping Deepgram worker (DEEPGRAM_API_KEY not set)" - # Set dummy PID to avoid script errors - AUDIO_STREAM_WORKER_PID="" + echo "⏭️ Skipping Deepgram stream worker (DEEPGRAM_API_KEY not set)" + AUDIO_STREAM_DEEPGRAM_WORKER_PID="" + fi + + # Only start Parakeet worker if PARAKEET_ASR_URL or OFFLINE_ASR_TCP_URI is set + if [ -n "$PARAKEET_ASR_URL" ] || [ -n "$OFFLINE_ASR_TCP_URI" ]; then + echo "🎵 Starting audio stream Parakeet worker (1 worker for sequential processing)..." + uv run python -m advanced_omi_backend.workers.audio_stream_parakeet_worker & + AUDIO_STREAM_PARAKEET_WORKER_PID=$! + else + echo "⏭️ Skipping Parakeet stream worker (PARAKEET_ASR_URL/OFFLINE_ASR_TCP_URI not set)" + AUDIO_STREAM_PARAKEET_WORKER_PID="" fi echo "✅ All workers started:" @@ -70,7 +80,8 @@ start_workers() { echo " - RQ worker 5: PID $RQ_WORKER_5_PID (transcription, memory, default)" echo " - RQ worker 6: PID $RQ_WORKER_6_PID (transcription, memory, default)" echo " - Audio persistence worker: PID $AUDIO_PERSISTENCE_WORKER_PID (audio queue - file rotation)" - echo " - Audio stream worker: PID $AUDIO_STREAM_WORKER_PID (Redis Streams consumer - sequential processing)" + [ -n "$AUDIO_STREAM_DEEPGRAM_WORKER_PID" ] && echo " - Deepgram stream worker: PID $AUDIO_STREAM_DEEPGRAM_WORKER_PID (real-time transcription)" + [ -n "$AUDIO_STREAM_PARAKEET_WORKER_PID" ] && echo " - Parakeet stream worker: PID $AUDIO_STREAM_PARAKEET_WORKER_PID (real-time transcription)" } # Function to check worker registration health @@ -111,7 +122,8 @@ monitor_worker_health() { # Kill all workers kill $RQ_WORKER_1_PID $RQ_WORKER_2_PID $RQ_WORKER_3_PID $RQ_WORKER_4_PID $RQ_WORKER_5_PID $RQ_WORKER_6_PID $AUDIO_PERSISTENCE_WORKER_PID 2>/dev/null || true - [ -n "$AUDIO_STREAM_WORKER_PID" ] && kill $AUDIO_STREAM_WORKER_PID 2>/dev/null || true + [ -n "$AUDIO_STREAM_DEEPGRAM_WORKER_PID" ] && kill $AUDIO_STREAM_DEEPGRAM_WORKER_PID 2>/dev/null || true + [ -n "$AUDIO_STREAM_PARAKEET_WORKER_PID" ] && kill $AUDIO_STREAM_PARAKEET_WORKER_PID 2>/dev/null || true wait 2>/dev/null || true # Restart workers @@ -136,7 +148,8 @@ shutdown() { kill $RQ_WORKER_5_PID 2>/dev/null || true kill $RQ_WORKER_6_PID 2>/dev/null || true kill $AUDIO_PERSISTENCE_WORKER_PID 2>/dev/null || true - [ -n "$AUDIO_STREAM_WORKER_PID" ] && kill $AUDIO_STREAM_WORKER_PID 2>/dev/null || true + [ -n "$AUDIO_STREAM_DEEPGRAM_WORKER_PID" ] && kill $AUDIO_STREAM_DEEPGRAM_WORKER_PID 2>/dev/null || true + [ -n "$AUDIO_STREAM_PARAKEET_WORKER_PID" ] && kill $AUDIO_STREAM_PARAKEET_WORKER_PID 2>/dev/null || true wait echo "✅ All workers stopped" exit 0 @@ -169,7 +182,8 @@ kill $RQ_WORKER_4_PID 2>/dev/null || true kill $RQ_WORKER_5_PID 2>/dev/null || true kill $RQ_WORKER_6_PID 2>/dev/null || true kill $AUDIO_PERSISTENCE_WORKER_PID 2>/dev/null || true -[ -n "$AUDIO_STREAM_WORKER_PID" ] && kill $AUDIO_STREAM_WORKER_PID 2>/dev/null || true +[ -n "$AUDIO_STREAM_DEEPGRAM_WORKER_PID" ] && kill $AUDIO_STREAM_DEEPGRAM_WORKER_PID 2>/dev/null || true +[ -n "$AUDIO_STREAM_PARAKEET_WORKER_PID" ] && kill $AUDIO_STREAM_PARAKEET_WORKER_PID 2>/dev/null || true wait echo "🔄 All workers stopped" From ef987036ddd81f6ed0c477f5f94bbb5398d61631 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 18 Dec 2025 00:46:50 +0000 Subject: [PATCH 3/5] Update Caddyfile.template to replace friend-backend with chronicle-backend for all proxy configurations - Changed reverse proxy targets for WebSocket, API, Auth, Health, Readiness, Users, and Audio endpoints from friend-backend:8000 to chronicle-backend:8000. - Ensured consistency across all relevant sections of the Caddyfile.template. --- backends/advanced/Caddyfile.template | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/backends/advanced/Caddyfile.template b/backends/advanced/Caddyfile.template index 21caf0ee..ccb2983d 100644 --- a/backends/advanced/Caddyfile.template +++ b/backends/advanced/Caddyfile.template @@ -19,7 +19,7 @@ localhost TAILSCALE_IP { # WebSocket endpoints - proxy to backend with upgrade support handle /ws* { - reverse_proxy friend-backend:8000 { + reverse_proxy chronicle-backend:8000 { # Caddy automatically handles WebSocket upgrades header_up X-Real-IP {remote_host} header_up X-Forwarded-For {remote_host} @@ -29,31 +29,31 @@ localhost TAILSCALE_IP { # API endpoints - proxy to backend handle /api/* { - reverse_proxy friend-backend:8000 + reverse_proxy chronicle-backend:8000 } # Auth endpoints - proxy to backend handle /auth/* { - reverse_proxy friend-backend:8000 + reverse_proxy chronicle-backend:8000 } # Health checks - proxy to backend handle /health { - reverse_proxy friend-backend:8000 + reverse_proxy chronicle-backend:8000 } handle /readiness { - reverse_proxy friend-backend:8000 + reverse_proxy chronicle-backend:8000 } # Users endpoints - proxy to backend handle /users/* { - reverse_proxy friend-backend:8000 + reverse_proxy chronicle-backend:8000 } # Audio files - proxy to backend handle /audio/* { - reverse_proxy friend-backend:8000 + reverse_proxy chronicle-backend:8000 } # Everything else - proxy to webui @@ -68,36 +68,36 @@ localhost TAILSCALE_IP { # # # WebSocket endpoints # handle /ws* { -# reverse_proxy friend-backend:8000 +# reverse_proxy chronicle-backend:8000 # } # # # API endpoints # handle /api/* { -# reverse_proxy friend-backend:8000 +# reverse_proxy chronicle-backend:8000 # } # # # Auth endpoints # handle /auth/* { -# reverse_proxy friend-backend:8000 +# reverse_proxy chronicle-backend:8000 # } # # # Health checks # handle /health { -# reverse_proxy friend-backend:8000 +# reverse_proxy chronicle-backend:8000 # } # # handle /readiness { -# reverse_proxy friend-backend:8000 +# reverse_proxy chronicle-backend:8000 # } # # # Users endpoints # handle /users/* { -# reverse_proxy friend-backend:8000 +# reverse_proxy chronicle-backend:8000 # } # # # Audio files # handle /audio/* { -# reverse_proxy friend-backend:8000 +# reverse_proxy chronicle-backend:8000 # } # # # Everything else - webui From 6bb8ff88f218e9ab1c243210068126166e47a52e Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 18 Dec 2025 00:58:45 +0000 Subject: [PATCH 4/5] Update docker-compose.yml files to use chronicle-network - Changed the default network name in backends/advanced/docker-compose.yml from friend-network to chronicle-network. - Updated extras/openmemory-mcp/docker-compose.yml to include chronicle-network as an external network and added it to the services configuration. --- backends/advanced/docker-compose.yml | 2 +- extras/openmemory-mcp/docker-compose.yml | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/backends/advanced/docker-compose.yml b/backends/advanced/docker-compose.yml index ea2f936b..47cff96e 100644 --- a/backends/advanced/docker-compose.yml +++ b/backends/advanced/docker-compose.yml @@ -226,7 +226,7 @@ services: # Shared network for cross-project communication networks: default: - name: friend-network + name: chronicle-network volumes: ollama_data: diff --git a/extras/openmemory-mcp/docker-compose.yml b/extras/openmemory-mcp/docker-compose.yml index 491debb3..34fbc8e5 100644 --- a/extras/openmemory-mcp/docker-compose.yml +++ b/extras/openmemory-mcp/docker-compose.yml @@ -27,6 +27,9 @@ services: timeout: 10s retries: 3 start_period: 30s + networks: + - default + - chronicle-network # OpenMemory UI (starts by default with the MCP server) openmemory-ui: @@ -38,4 +41,10 @@ services: - NEXT_PUBLIC_USER_ID=openmemory depends_on: - openmemory-mcp - restart: unless-stopped \ No newline at end of file + restart: unless-stopped + +networks: + default: + name: openmemory-mcp_default + chronicle-network: + external: true \ No newline at end of file From 5e2220b5f7765f7e31c33b5ffb744744e2031d3f Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 18 Dec 2025 01:31:53 +0000 Subject: [PATCH 5/5] Update docker-compose.yml to use chronicle-network for speaker recognition service --- extras/speaker-recognition/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extras/speaker-recognition/docker-compose.yml b/extras/speaker-recognition/docker-compose.yml index 0c0f7828..ea41de04 100644 --- a/extras/speaker-recognition/docker-compose.yml +++ b/extras/speaker-recognition/docker-compose.yml @@ -108,5 +108,5 @@ services: # Shared network for cross-project communication networks: default: - name: friend-network + name: chronicle-network external: true \ No newline at end of file