diff --git a/backends/advanced/Docs/plugin-development-guide.md b/backends/advanced/Docs/plugin-development-guide.md index 17c53b4a..a7361469 100644 --- a/backends/advanced/Docs/plugin-development-guide.md +++ b/backends/advanced/Docs/plugin-development-guide.md @@ -24,11 +24,6 @@ Chronicle's plugin system allows you to extend functionality by subscribing to e - **Configurable**: YAML-based configuration with environment variable support - **Isolated**: Each plugin runs independently with proper error handling -### Plugin Types - -- **Core Plugins**: Built-in plugins (`homeassistant`, `test_event`) -- **Community Plugins**: Auto-discovered plugins in `plugins/` directory - ## Quick Start ### 1. Generate Plugin Boilerplate @@ -207,6 +202,84 @@ async def on_memory_processed(self, context: PluginContext): await self.index_memory(memory) ``` +### 4. Button Events (`button.single_press`, `button.double_press`) + +**When**: OMI device button is pressed +**Context Data**: +- `state` (str): Button state (`SINGLE_TAP`, `DOUBLE_TAP`) +- `timestamp` (float): Unix timestamp of the event +- `audio_uuid` (str): Current audio session UUID (may be None) +- `session_id` (str): Streaming session ID (for conversation close) +- `client_id` (str): Client device identifier + +**Data Flow**: +``` +OMI Device (BLE) + → Button press on physical device + → BLE characteristic notifies with 8-byte payload + ↓ +friend-lite-sdk (extras/friend-lite-sdk/) + → parse_button_event() converts payload → ButtonState IntEnum + ↓ +BLE Client (extras/local-omi-bt/ or mobile app) + → Formats as Wyoming protocol: {"type": "button-event", "data": {"state": "SINGLE_TAP"}} + → Sends over WebSocket + ↓ +Backend (websocket_controller.py) + → _handle_button_event() stores marker on client_state + → Maps ButtonState → PluginEvent using enums (plugins/events.py) + → Dispatches granular event to plugin system + ↓ +Plugin System + → Routed to subscribed plugins (e.g., test_button_actions) + → Plugins use PluginServices for system actions and cross-plugin calls +``` + +**Use Cases**: +- Close current conversation (single press) +- Toggle smart home devices (double press) +- Custom actions via cross-plugin communication + +**Example**: +```python +async def on_button_event(self, context: PluginContext): + if context.event == PluginEvent.BUTTON_SINGLE_PRESS: + session_id = context.data.get('session_id') + await context.services.close_conversation(session_id) +``` + +### 5. Plugin Action Events (`plugin_action`) + +**When**: Another plugin calls `context.services.call_plugin()` +**Context Data**: +- `action` (str): Action name (e.g., `toggle_lights`) +- Plus any additional data from the calling plugin + +**Use Cases**: +- Cross-plugin communication (button press → toggle lights) +- Service orchestration between plugins + +**Example**: +```python +async def on_plugin_action(self, context: PluginContext): + action = context.data.get('action') + if action == 'toggle_lights': + # Handle the action + ... +``` + +### PluginServices + +Plugins receive a `services` object on the context for system and cross-plugin interaction: + +```python +# Close the current conversation (triggers post-processing) +await context.services.close_conversation(session_id, reason) + +# Call another plugin's on_plugin_action() handler +result = await context.services.call_plugin("homeassistant", "toggle_lights", data) +``` + ## Creating Your First Plugin ### Step 1: Generate Boilerplate @@ -225,7 +298,7 @@ import logging import re from typing import Any, Dict, List, Optional -from ..base import BasePlugin, PluginContext, PluginResult +from advanced_omi_backend.plugins.base import BasePlugin, PluginContext, PluginResult logger = logging.getLogger(__name__) @@ -671,7 +744,7 @@ async def on_conversation_complete(self, context): **Solution**: - Restart backend after adding dependencies - Verify imports are from correct modules -- Check relative imports use `..base` for base classes +- Use absolute imports for framework classes: `from advanced_omi_backend.plugins.base import BasePlugin` ### Database Connection Issues @@ -749,12 +822,13 @@ class ExternalServicePlugin(BasePlugin): ## Resources -- **Base Plugin Class**: `backends/advanced/src/advanced_omi_backend/plugins/base.py` -- **Example Plugins**: +- **Plugin Framework**: `backends/advanced/src/advanced_omi_backend/plugins/` (base.py, router.py, events.py, services.py) +- **Plugin Implementations**: `plugins/` at repo root - Email Summarizer: `plugins/email_summarizer/` - Home Assistant: `plugins/homeassistant/` - Test Event: `plugins/test_event/` -- **Plugin Generator**: `scripts/create_plugin.py` + - Test Button Actions: `plugins/test_button_actions/` +- **Plugin Generator**: `backends/advanced/scripts/create_plugin.py` - **Configuration**: `config/plugins.yml.template` ## Contributing Plugins diff --git a/backends/advanced/docker-compose-test.yml b/backends/advanced/docker-compose-test.yml index a18b0493..86bb8325 100644 --- a/backends/advanced/docker-compose-test.yml +++ b/backends/advanced/docker-compose-test.yml @@ -21,6 +21,7 @@ services: - ../../config:/app/config # Mount config directory with defaults.yml - ../../tests/configs:/app/test-configs:ro # Mount test-specific configs - ${PLUGINS_CONFIG:-../../tests/config/plugins.test.yml}:/app/config/plugins.yml # Mount test plugins config to correct location + - ../../plugins:/app/plugins # External plugins directory environment: # Override with test-specific settings - MONGODB_URI=mongodb://mongo-test:27017/test_db @@ -223,6 +224,7 @@ services: - ../../config:/app/config # Mount config directory with defaults.yml - ../../tests/configs:/app/test-configs:ro # Mount test-specific configs - ${PLUGINS_CONFIG:-../../tests/config/plugins.test.yml}:/app/config/plugins.yml # Mount test plugins config to correct location + - ../../plugins:/app/plugins # External plugins directory environment: # Same environment as backend - MONGODB_URI=mongodb://mongo-test:27017/test_db diff --git a/backends/advanced/docker-compose.yml b/backends/advanced/docker-compose.yml index 3eb7e108..d8e33c7e 100644 --- a/backends/advanced/docker-compose.yml +++ b/backends/advanced/docker-compose.yml @@ -40,6 +40,7 @@ services: - ./data/debug_dir:/app/debug_dir - ./data:/app/data - ../../config:/app/config # Mount entire config directory (includes config.yml, defaults.yml, plugins.yml) + - ../../plugins:/app/plugins # External plugins directory environment: - DEEPGRAM_API_KEY=${DEEPGRAM_API_KEY} - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} @@ -95,6 +96,7 @@ services: - ./data/audio_chunks:/app/audio_chunks - ./data:/app/data - ../../config:/app/config # Mount entire config directory (includes config.yml, defaults.yml, plugins.yml) + - ../../plugins:/app/plugins # External plugins directory environment: - DEEPGRAM_API_KEY=${DEEPGRAM_API_KEY} - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} @@ -212,8 +214,8 @@ services: - "6033:6033" # gRPC - "6034:6034" # HTTP volumes: - - ./data/qdrant_data:/qdrant/storage - + - ./data/qdrant_data:/qdrant/storage + restart: unless-stopped mongo: image: mongo:8.0.14 @@ -227,6 +229,7 @@ services: timeout: 5s retries: 5 start_period: 10s + restart: unless-stopped redis: image: redis:7-alpine @@ -235,6 +238,7 @@ services: volumes: - ./data/redis_data:/data command: redis-server --appendonly yes + restart: unless-stopped healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s diff --git a/backends/advanced/scripts/create_plugin.py b/backends/advanced/scripts/create_plugin.py index 41b93c83..f24427ad 100755 --- a/backends/advanced/scripts/create_plugin.py +++ b/backends/advanced/scripts/create_plugin.py @@ -37,10 +37,10 @@ def create_plugin(plugin_name: str, force: bool = False): # Convert to class name class_name = snake_to_pascal(plugin_name) + 'Plugin' - # Get plugins directory + # Get plugins directory (repo root plugins/) script_dir = Path(__file__).parent backend_dir = script_dir.parent - plugins_dir = backend_dir / 'src' / 'advanced_omi_backend' / 'plugins' + plugins_dir = backend_dir.parent.parent / 'plugins' plugin_dir = plugins_dir / plugin_name # Check if plugin already exists @@ -83,7 +83,7 @@ def create_plugin(plugin_name: str, force: bool = False): import logging from typing import Any, Dict, List, Optional -from ..base import BasePlugin, PluginContext, PluginResult +from advanced_omi_backend.plugins.base import BasePlugin, PluginContext, PluginResult logger = logging.getLogger(__name__) diff --git a/backends/advanced/src/advanced_omi_backend/client.py b/backends/advanced/src/advanced_omi_backend/client.py index a92fbc10..79ee2957 100644 --- a/backends/advanced/src/advanced_omi_backend/client.py +++ b/backends/advanced/src/advanced_omi_backend/client.py @@ -51,6 +51,9 @@ def __init__( # NOTE: Removed in-memory transcript storage for single source of truth # Transcripts are stored only in MongoDB via TranscriptionManager + # Markers (e.g., button events) collected during the session + self.markers: List[dict] = [] + # Track if conversation has been closed self.conversation_closed: bool = False @@ -102,6 +105,10 @@ def update_transcript_received(self): """Update timestamp when transcript is received (for timeout detection).""" self.last_transcript_time = time.time() + def add_marker(self, marker: dict) -> None: + """Add a marker (e.g., button event) to the current session.""" + self.markers.append(marker) + def should_start_new_conversation(self) -> bool: """Check if we should start a new conversation based on timeout.""" if self.last_transcript_time is None: @@ -114,8 +121,7 @@ def should_start_new_conversation(self) -> bool: return time_since_last_transcript > timeout_seconds async def close_current_conversation(self): - """Close the current conversation and queue necessary processing.""" - # Prevent double closure + """Clean up in-memory speech segments for the current conversation.""" if self.conversation_closed: audio_logger.debug( f"🔒 Conversation already closed for client {self.client_id}, skipping" @@ -125,23 +131,15 @@ async def close_current_conversation(self): self.conversation_closed = True if not self.current_audio_uuid: - audio_logger.info(f"🔒 No active conversation to close for client {self.client_id}") return - # NOTE: ClientState is legacy V1 code. In V2 architecture, conversation closure - # is handled by the websocket controllers using RQ jobs directly. - # This method is kept minimal for backward compatibility. + audio_logger.info(f"🔒 Closing conversation state for client {self.client_id}") - audio_logger.info(f"🔒 Closing conversation for client {self.client_id}, audio_uuid: {self.current_audio_uuid}") - - # Clean up speech segments for this conversation if self.current_audio_uuid in self.speech_segments: del self.speech_segments[self.current_audio_uuid] if self.current_audio_uuid in self.current_speech_start: del self.current_speech_start[self.current_audio_uuid] - audio_logger.info(f"✅ Cleaned up state for {self.current_audio_uuid}") - async def start_new_conversation(self): """Start a new conversation by closing current and resetting state.""" await self.close_current_conversation() @@ -151,11 +149,9 @@ async def start_new_conversation(self): self.conversation_start_time = time.time() self.last_transcript_time = None self.conversation_closed = False + self.markers = [] - audio_logger.info( - f"Client {self.client_id}: Started new conversation due to " - f"{NEW_CONVERSATION_TIMEOUT_MINUTES}min timeout" - ) + audio_logger.info(f"Client {self.client_id}: Started new conversation") async def disconnect(self): """Clean disconnect of client state.""" diff --git a/backends/advanced/src/advanced_omi_backend/controllers/audio_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/audio_controller.py index c9046484..b1646a8e 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/audio_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/audio_controller.py @@ -8,6 +8,7 @@ """ import logging +import os import time import uuid @@ -24,7 +25,10 @@ from advanced_omi_backend.services.transcription import is_transcription_available from advanced_omi_backend.utils.audio_chunk_utils import convert_audio_to_chunks from advanced_omi_backend.utils.audio_utils import ( + SUPPORTED_AUDIO_EXTENSIONS, + VIDEO_EXTENSIONS, AudioValidationError, + convert_any_to_wav, validate_and_prepare_audio, ) from advanced_omi_backend.workers.transcription_jobs import ( @@ -71,22 +75,38 @@ async def upload_and_process_audio_files( for file_index, file in enumerate(files): try: - # Validate file type (only WAV for now) - if not file.filename or not file.filename.lower().endswith(".wav"): + # Validate file type + filename = file.filename or "unknown" + _, ext = os.path.splitext(filename.lower()) + if not ext or ext not in SUPPORTED_AUDIO_EXTENSIONS: + supported = ", ".join(sorted(SUPPORTED_AUDIO_EXTENSIONS)) processed_files.append({ - "filename": file.filename or "unknown", + "filename": filename, "status": "error", - "error": "Only WAV files are currently supported", + "error": f"Unsupported format '{ext}'. Supported: {supported}", }) continue + is_video_source = ext in VIDEO_EXTENSIONS + audio_logger.info( - f"📁 Uploading file {file_index + 1}/{len(files)}: {file.filename}" + f"📁 Uploading file {file_index + 1}/{len(files)}: {filename}" ) # Read file content content = await file.read() + # Convert non-WAV files to WAV via FFmpeg + if ext != ".wav": + try: + content = await convert_any_to_wav(content, ext) + except AudioValidationError as e: + processed_files.append({ + "filename": filename, + "status": "error", + "error": str(e), + }) + continue # Track external source for deduplication (Google Drive, etc.) external_source_id = None @@ -95,7 +115,7 @@ async def upload_and_process_audio_files( external_source_id = getattr(file, "file_id", None) or getattr(file, "audio_uuid", None) external_source_type = "gdrive" if not external_source_id: - audio_logger.warning(f"Missing file_id for gdrive file: {file.filename}") + audio_logger.warning(f"Missing file_id for gdrive file: {filename}") timestamp = int(time.time() * 1000) # Validate and prepare audio (read format from WAV file) @@ -108,21 +128,18 @@ async def upload_and_process_audio_files( ) except AudioValidationError as e: processed_files.append({ - "filename": file.filename, + "filename": filename, "status": "error", "error": str(e), }) continue audio_logger.info( - f"📊 {file.filename}: {duration:.1f}s ({sample_rate}Hz, {channels}ch, {sample_width} bytes/sample)" + f"📊 {filename}: {duration:.1f}s ({sample_rate}Hz, {channels}ch, {sample_width} bytes/sample)" ) - # Create conversation immediately for uploaded files (conversation_id auto-generated) - version_id = str(uuid.uuid4()) - # Generate title from filename - title = file.filename.rsplit('.', 1)[0][:50] if file.filename else "Uploaded Audio" + title = filename.rsplit('.', 1)[0][:50] if filename != "unknown" else "Uploaded Audio" conversation = create_conversation( user_id=user.user_id, @@ -154,7 +171,7 @@ async def upload_and_process_audio_files( # Handle validation errors (e.g., file too long) audio_logger.error(f"Audio validation failed: {val_error}") processed_files.append({ - "filename": file.filename, + "filename": filename, "status": "error", "error": str(val_error), }) @@ -167,7 +184,7 @@ async def upload_and_process_audio_files( exc_info=True ) processed_files.append({ - "filename": file.filename, + "filename": filename, "status": "error", "error": f"Audio conversion failed: {str(chunk_error)}", }) @@ -209,15 +226,18 @@ async def upload_and_process_audio_files( client_id=client_id # Pass client_id for UI tracking ) - processed_files.append({ - "filename": file.filename, + file_result = { + "filename": filename, "status": "started", # RQ standard: job has been enqueued "conversation_id": conversation_id, "transcript_job_id": transcription_job.id if transcription_job else None, "speaker_job_id": job_ids['speaker_recognition'], "memory_job_id": job_ids['memory'], "duration_seconds": round(duration, 2), - }) + } + if is_video_source: + file_result["note"] = "Audio extracted from video file" + processed_files.append(file_result) # Build job chain description job_chain = [] @@ -229,23 +249,23 @@ async def upload_and_process_audio_files( job_chain.append(job_ids['memory']) audio_logger.info( - f"✅ Processed {file.filename} → conversation {conversation_id}, " + f"✅ Processed {filename} → conversation {conversation_id}, " f"jobs: {' → '.join(job_chain) if job_chain else 'none'}" ) except (OSError, IOError) as e: # File I/O errors during audio processing - audio_logger.exception(f"File I/O error processing {file.filename}") + audio_logger.exception(f"File I/O error processing {filename}") processed_files.append({ - "filename": file.filename or "unknown", + "filename": filename, "status": "error", "error": str(e), }) except Exception as e: # Unexpected errors during file processing - audio_logger.exception(f"Unexpected error processing file {file.filename}") + audio_logger.exception(f"Unexpected error processing file {filename}") processed_files.append({ - "filename": file.filename or "unknown", + "filename": filename, "status": "error", "error": str(e), }) diff --git a/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py index f4ffe096..13f2620d 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py @@ -3,16 +3,18 @@ """ import logging +import os import time import uuid from datetime import datetime from pathlib import Path +import redis.asyncio as aioredis from fastapi.responses import JSONResponse from advanced_omi_backend.client_manager import ( - ClientManager, client_belongs_to_user, + get_client_manager, ) from advanced_omi_backend.config_loader import get_service_config from advanced_omi_backend.controllers.queue_controller import ( @@ -21,9 +23,13 @@ memory_queue, transcription_queue, ) +from advanced_omi_backend.controllers.session_controller import ( + request_conversation_close, +) from advanced_omi_backend.models.audio_chunk import AudioChunkDocument from advanced_omi_backend.models.conversation import Conversation from advanced_omi_backend.models.job import JobPriority +from advanced_omi_backend.plugins.events import ConversationCloseReason from advanced_omi_backend.users import User from advanced_omi_backend.workers.conversation_jobs import generate_title_summary_job from advanced_omi_backend.workers.memory_jobs import ( @@ -36,8 +42,12 @@ audio_logger = logging.getLogger("audio_processing") -async def close_current_conversation(client_id: str, user: User, client_manager: ClientManager): - """Close the current conversation for a specific client. Users can only close their own conversations.""" +async def close_current_conversation(client_id: str, user: User): + """Close the current conversation for a specific client. + + Signals the open_conversation_job to close the current conversation + and trigger post-processing. The session stays active for new conversations. + """ # Validate client ownership if not user.is_superuser and not client_belongs_to_user(client_id, user.user_id): logger.warning( @@ -51,50 +61,47 @@ async def close_current_conversation(client_id: str, user: User, client_manager: status_code=403, ) - if not client_manager.has_client(client_id): - return JSONResponse( - content={"error": f"Client '{client_id}' not found or not connected"}, - status_code=404, - ) - + client_manager = get_client_manager() client_state = client_manager.get_client(client_id) - if client_state is None: + if client_state is None or not client_state.connected: return JSONResponse( content={"error": f"Client '{client_id}' not found or not connected"}, status_code=404, ) - if not client_state.connected: + session_id = getattr(client_state, 'stream_session_id', None) + if not session_id: return JSONResponse( - content={"error": f"Client '{client_id}' is not connected"}, status_code=400 + content={"error": "No active session"}, + status_code=400, ) + # Signal the conversation job to close and trigger post-processing + redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0") + r = aioredis.from_url(redis_url) try: - # Close the current conversation - await client_state.close_current_conversation() - - # Reset conversation state but keep client connected - client_state.current_audio_uuid = None - client_state.conversation_start_time = time.time() - client_state.last_transcript_time = None - - logger.info(f"Manually closed conversation for client {client_id} by user {user.id}") - - return JSONResponse( - content={ - "message": f"Successfully closed current conversation for client '{client_id}'", - "client_id": client_id, - "timestamp": int(time.time()), - } + success = await request_conversation_close( + r, session_id, reason=ConversationCloseReason.USER_REQUESTED.value ) + finally: + await r.aclose() - except Exception as e: - logger.error(f"Error closing conversation for client {client_id}: {e}") + if not success: return JSONResponse( - content={"error": f"Failed to close conversation: {str(e)}"}, - status_code=500, + content={"error": "Session not found in Redis"}, + status_code=404, ) + logger.info(f"Conversation close requested for client {client_id} by user {user.user_id}") + + return JSONResponse( + content={ + "message": f"Conversation close requested for client '{client_id}'", + "client_id": client_id, + "timestamp": int(time.time()), + } + ) + async def get_conversation(conversation_id: str, user: User): """Get a single conversation with full transcript details.""" @@ -150,40 +157,85 @@ async def get_conversation(conversation_id: str, user: User): return JSONResponse(status_code=500, content={"error": "Error fetching conversation"}) -async def get_conversations(user: User, include_deleted: bool = False): - """Get conversations with speech only (speech-driven architecture).""" +async def get_conversations( + user: User, + include_deleted: bool = False, + include_unprocessed: bool = False, + limit: int = 200, + offset: int = 0, +): + """Get conversations with speech only (speech-driven architecture). + + Uses a single consolidated query with ``$or`` when ``include_unprocessed`` + is True, eliminating multiple round-trips and Python-side merge/sort. + Results are paginated with ``limit``/``offset``. + """ try: - # Build query based on user permissions using Beanie - if not user.is_superuser: - # Regular users can only see their own conversations - # Filter by deleted status - if not include_deleted: - user_conversations = ( - await Conversation.find( - Conversation.user_id == str(user.user_id), Conversation.deleted == False - ) - .sort(-Conversation.created_at) - .to_list() - ) - else: - user_conversations = ( - await Conversation.find(Conversation.user_id == str(user.user_id)) - .sort(-Conversation.created_at) - .to_list() - ) + user_filter = {} if user.is_superuser else {"user_id": str(user.user_id)} + + # Build query conditions — single $or when orphans are requested + conditions = [] + + # Condition 1: normal (non-deleted or all) conversations + if include_deleted: + conditions.append({}) # no filter on deleted else: - # Admins see all conversations - # Filter by deleted status - if not include_deleted: - user_conversations = ( - await Conversation.find(Conversation.deleted == False) - .sort(-Conversation.created_at) - .to_list() + conditions.append({"deleted": False}) + + if include_unprocessed: + # Orphan type 1: always_persist stuck in pending/failed (not deleted) + conditions.append({ + "always_persist": True, + "processing_status": {"$in": ["pending_transcription", "transcription_failed"]}, + "deleted": False, + }) + # Orphan type 2: soft-deleted due to no speech but have audio data + conditions.append({ + "deleted": True, + "deletion_reason": {"$in": [ + "no_meaningful_speech", + "audio_file_not_ready", + "no_meaningful_speech_batch_transcription", + ]}, + "audio_chunks_count": {"$gt": 0}, + }) + + # Assemble final query + if len(conditions) == 1: + query = {**user_filter, **conditions[0]} + else: + query = {**user_filter, "$or": conditions} + + total = await Conversation.find(query).count() + + user_conversations = ( + await Conversation.find(query) + .sort(-Conversation.created_at) + .skip(offset) + .limit(limit) + .to_list() + ) + + # Mark orphans in results (lightweight in-memory check on the page) + orphan_ids: set = set() + if include_unprocessed: + for conv in user_conversations: + is_orphan_type1 = ( + conv.always_persist + and conv.processing_status in ("pending_transcription", "transcription_failed") + and not conv.deleted ) - else: - user_conversations = ( - await Conversation.find_all().sort(-Conversation.created_at).to_list() + is_orphan_type2 = ( + conv.deleted + and conv.deletion_reason in ( + "no_meaningful_speech", + "audio_file_not_ready", + "no_meaningful_speech_batch_transcription", + ) + and (conv.audio_chunks_count or 0) > 0 ) + if is_orphan_type1 or is_orphan_type2: + orphan_ids.add(conv.conversation_id) # Build response with explicit curated fields - minimal for list view conversations = [] @@ -215,10 +267,16 @@ async def get_conversations(user: User, include_deleted: bool = False): "memory_version_count": conv.memory_version_count, "active_transcript_version_number": conv.active_transcript_version_number, "active_memory_version_number": conv.active_memory_version_number, + "is_orphan": conv.conversation_id in orphan_ids, } ) - return {"conversations": conversations} + return { + "conversations": conversations, + "total": total, + "limit": limit, + "offset": offset, + } except Exception as e: logger.exception(f"Error fetching conversations: {e}") @@ -440,6 +498,134 @@ async def restore_conversation(conversation_id: str, user: User) -> JSONResponse ) +async def reprocess_orphan(conversation_id: str, user: User): + """Reprocess an orphan audio session - restore if deleted and enqueue full processing chain.""" + try: + conversation = await Conversation.find_one(Conversation.conversation_id == conversation_id) + if not conversation: + return JSONResponse(status_code=404, content={"error": "Conversation not found"}) + + # Check ownership + if not user.is_superuser and conversation.user_id != str(user.user_id): + return JSONResponse(status_code=403, content={"error": "Access forbidden"}) + + # Verify audio chunks exist (check both deleted and non-deleted) + total_chunks = await AudioChunkDocument.find( + AudioChunkDocument.conversation_id == conversation_id + ).count() + + if total_chunks == 0: + return JSONResponse( + status_code=400, + content={"error": "No audio data found for this conversation"}, + ) + + # If conversation is soft-deleted, restore it and its chunks + if conversation.deleted: + await AudioChunkDocument.find( + AudioChunkDocument.conversation_id == conversation_id, + AudioChunkDocument.deleted == True, + ).update_many({"$set": {"deleted": False, "deleted_at": None}}) + + conversation.deleted = False + conversation.deletion_reason = None + conversation.deleted_at = None + + # Set processing status and update title + conversation.processing_status = "reprocessing" + conversation.title = "Reprocessing..." + conversation.summary = None + conversation.detailed_summary = None + await conversation.save() + + # Create new transcript version ID + version_id = str(uuid.uuid4()) + + # Enqueue the same 4-job chain as reprocess_transcript + from advanced_omi_backend.workers.transcription_jobs import ( + transcribe_full_audio_job, + ) + + # Job 1: Transcribe audio + transcript_job = transcription_queue.enqueue( + transcribe_full_audio_job, + conversation_id, + version_id, + "reprocess_orphan", + job_timeout=900, + result_ttl=JOB_RESULT_TTL, + job_id=f"orphan_transcribe_{conversation_id[:8]}", + description=f"Transcribe orphan audio for {conversation_id[:8]}", + meta={"conversation_id": conversation_id}, + ) + + # Job 2: Speaker recognition (conditional) + speaker_config = get_service_config("speaker_recognition") + speaker_enabled = speaker_config.get("enabled", True) + speaker_dependency = transcript_job + speaker_job = None + + if speaker_enabled: + speaker_job = transcription_queue.enqueue( + recognise_speakers_job, + conversation_id, + version_id, + depends_on=transcript_job, + job_timeout=600, + result_ttl=JOB_RESULT_TTL, + job_id=f"orphan_speaker_{conversation_id[:8]}", + description=f"Recognize speakers for orphan {conversation_id[:8]}", + meta={"conversation_id": conversation_id}, + ) + speaker_dependency = speaker_job + + # Job 3: Extract memories + memory_job = memory_queue.enqueue( + process_memory_job, + conversation_id, + depends_on=speaker_dependency, + job_timeout=1800, + result_ttl=JOB_RESULT_TTL, + job_id=f"orphan_memory_{conversation_id[:8]}", + description=f"Extract memories for orphan {conversation_id[:8]}", + meta={"conversation_id": conversation_id}, + ) + + # Job 4: Generate title/summary + title_summary_job = default_queue.enqueue( + generate_title_summary_job, + conversation_id, + job_timeout=300, + result_ttl=JOB_RESULT_TTL, + depends_on=memory_job, + job_id=f"orphan_title_{conversation_id[:8]}", + description=f"Generate title/summary for orphan {conversation_id[:8]}", + meta={"conversation_id": conversation_id, "trigger": "reprocess_orphan"}, + ) + + logger.info( + f"Enqueued orphan reprocessing chain for {conversation_id}: " + f"transcribe={transcript_job.id} → speaker={'skipped' if not speaker_job else speaker_job.id} " + f"→ memory={memory_job.id} → title={title_summary_job.id}" + ) + + return JSONResponse( + content={ + "message": f"Orphan reprocessing started for conversation {conversation_id}", + "job_id": transcript_job.id, + "title_summary_job_id": title_summary_job.id, + "version_id": version_id, + "status": "queued", + } + ) + + except Exception as e: + logger.error(f"Error starting orphan reprocessing for {conversation_id}: {e}") + return JSONResponse( + status_code=500, content={"error": "Error starting orphan reprocessing"} + ) + + async def reprocess_transcript(conversation_id: str, user: User): """Reprocess transcript for a conversation. Users can only reprocess their own conversations.""" try: diff --git a/backends/advanced/src/advanced_omi_backend/controllers/session_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/session_controller.py index 7d7d5f2e..9b3a2de9 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/session_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/session_controller.py @@ -65,6 +65,37 @@ async def mark_session_complete( logger.info(f"✅ Session {session_id[:12]} marked finished: {reason} [TIME: {mark_time:.3f}]") +async def request_conversation_close( + redis_client, + session_id: str, + reason: str = "user_requested", +) -> bool: + """ + Request closing the current conversation without killing the session. + + Unlike mark_session_complete() which finalizes the entire session, + this signals open_conversation_job to close just the current conversation + and trigger post-processing. The session stays active for new conversations. + + Sets 'conversation_close_requested' field on the session hash. + The open_conversation_job checks this field every poll iteration. + + Args: + redis_client: Redis async client + session_id: Session UUID + reason: Why the conversation is being closed + + Returns: + True if the close request was set, False if session not found + """ + session_key = f"audio:session:{session_id}" + if not await redis_client.exists(session_key): + return False + await redis_client.hset(session_key, "conversation_close_requested", reason) + logger.info(f"🔒 Conversation close requested for session {session_id[:12]}: {reason}") + return True + + async def get_session_info(redis_client, session_id: str) -> Optional[Dict]: """ Get detailed information about a specific session. diff --git a/backends/advanced/src/advanced_omi_backend/controllers/system_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/system_controller.py index 53e8ff95..c4794a40 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/system_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/system_controller.py @@ -353,7 +353,7 @@ async def save_misc_settings_controller(settings: dict): """Save miscellaneous settings.""" try: # Validate settings - valid_keys = {"always_persist_enabled", "use_provider_segments"} + valid_keys = {"always_persist_enabled", "use_provider_segments", "per_segment_speaker_id"} # Filter to only valid keys filtered_settings = {} @@ -1102,8 +1102,7 @@ async def update_plugin_config_structured(plugin_id: str, config: dict) -> dict: Success message with list of updated files """ try: - import advanced_omi_backend.plugins - from advanced_omi_backend.services.plugin_service import discover_plugins + from advanced_omi_backend.services.plugin_service import _get_plugins_dir, discover_plugins # Validate plugin exists discovered_plugins = discover_plugins() @@ -1151,7 +1150,7 @@ async def update_plugin_config_structured(plugin_id: str, config: dict) -> dict: # 2. Update plugins/{plugin_id}/config.yml (settings with env var references) if 'settings' in config: - plugins_dir = Path(advanced_omi_backend.plugins.__file__).parent + plugins_dir = _get_plugins_dir() plugin_config_path = plugins_dir / plugin_id / "config.yml" # Load current config.yml diff --git a/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py index 98e8f81b..0dc09396 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py @@ -528,6 +528,14 @@ async def _finalize_streaming_session( # Mark session as finalizing with user_stopped reason (audio-stop event) await audio_stream_producer.finalize_session(session_id, completion_reason="user_stopped") + # Store markers in Redis so open_conversation_job can persist them + if client_state.markers: + session_key = f"audio:session:{session_id}" + await audio_stream_producer.redis_client.hset( + session_key, "markers", json.dumps(client_state.markers) + ) + client_state.markers.clear() + # NOTE: Finalize job disabled - open_conversation_job now handles everything # The open_conversation_job will: # 1. Detect the "finalizing" status @@ -945,6 +953,75 @@ async def _handle_audio_session_stop( return False # Switch back to control mode +async def _handle_button_event( + client_state, + button_state: str, + user_id: str, + client_id: str, +) -> None: + """Handle a button event from the device. + + Stores a marker on the client state and dispatches granular events + to the plugin system using typed enums. + + Args: + client_state: Client state object + button_state: Button state string (e.g., "SINGLE_TAP", "DOUBLE_TAP") + user_id: User ID + client_id: Client ID + """ + from advanced_omi_backend.plugins.events import ( + BUTTON_STATE_TO_EVENT, + ButtonState, + ) + from advanced_omi_backend.services.plugin_service import get_plugin_router + + timestamp = time.time() + audio_uuid = client_state.current_audio_uuid + + application_logger.info( + f"🔘 Button event from {client_id}: {button_state} " + f"(audio_uuid={audio_uuid})" + ) + + # Store marker on client state for later persistence to conversation + marker = { + "type": "button_event", + "state": button_state, + "timestamp": timestamp, + "audio_uuid": audio_uuid, + "client_id": client_id, + } + client_state.add_marker(marker) + + # Map device button state to typed plugin event + try: + button_state_enum = ButtonState(button_state) + except ValueError: + application_logger.warning(f"Unknown button state: {button_state}") + return + + event = BUTTON_STATE_TO_EVENT.get(button_state_enum) + if not event: + application_logger.debug(f"No plugin event mapped for {button_state_enum}") + return + + # Dispatch granular event to plugin system + router = get_plugin_router() + if router: + await router.dispatch_event( + event=event.value, + user_id=user_id, + data={ + "state": button_state_enum.value, + "timestamp": timestamp, + "audio_uuid": audio_uuid, + "session_id": getattr(client_state, 'stream_session_id', None), + "client_id": client_id, + }, + ) + + async def _process_rolling_batch( client_state, user_id: str, @@ -1094,6 +1171,10 @@ async def _process_batch_audio_complete( title="Batch Recording", summary="Processing batch audio..." ) + # Attach any markers (e.g., button events) captured during the session + if client_state.markers: + conversation.markers = list(client_state.markers) + client_state.markers.clear() await conversation.insert() conversation_id = conversation.conversation_id # Get the auto-generated ID @@ -1385,7 +1466,15 @@ async def handle_pcm_websocket( # Handle keepalive ping from frontend application_logger.debug(f"🏓 Received ping from {client_id}") continue - + + elif header["type"] == "button-event": + button_data = header.get("data", {}) + button_state = button_data.get("state", "unknown") + await _handle_button_event( + client_state, button_state, user.user_id, client_id + ) + continue + else: # Unknown control message type application_logger.debug( @@ -1466,10 +1555,17 @@ async def handle_pcm_websocket( else: application_logger.warning(f"audio-chunk missing payload_length: {payload_length}") continue + elif control_header.get("type") == "button-event": + button_data = control_header.get("data", {}) + button_state = button_data.get("state", "unknown") + await _handle_button_event( + client_state, button_state, user.user_id, client_id + ) + continue else: application_logger.warning(f"Unknown control message during streaming: {control_header.get('type')}") continue - + except json.JSONDecodeError: application_logger.warning(f"Invalid control message during streaming for {client_id}") continue diff --git a/backends/advanced/src/advanced_omi_backend/models/conversation.py b/backends/advanced/src/advanced_omi_backend/models/conversation.py index 2ec45f33..23fae946 100644 --- a/backends/advanced/src/advanced_omi_backend/models/conversation.py +++ b/backends/advanced/src/advanced_omi_backend/models/conversation.py @@ -39,6 +39,7 @@ class EndReason(str, Enum): INACTIVITY_TIMEOUT = "inactivity_timeout" # No speech detected for threshold period WEBSOCKET_DISCONNECT = "websocket_disconnect" # Connection lost (Bluetooth, network, etc.) MAX_DURATION = "max_duration" # Hit maximum conversation duration + CLOSE_REQUESTED = "close_requested" # External close signal (API, plugin, button) ERROR = "error" # Processing error forced conversation end UNKNOWN = "unknown" # Unknown or legacy reason @@ -122,6 +123,12 @@ class MemoryVersion(BaseModel): description="Compression ratio (compressed_size / original_size), typically ~0.047 for Opus" ) + # Markers (e.g., button events) captured during the session + markers: List[Dict[str, Any]] = Field( + default_factory=list, + description="Markers captured during audio session (button events, bookmarks, etc.)" + ) + # Creation metadata created_at: Indexed(datetime) = Field(default_factory=datetime.utcnow, description="When the conversation was created") @@ -377,7 +384,7 @@ class Settings: "conversation_id", "user_id", "created_at", - [("user_id", 1), ("created_at", -1)], # Compound index for user queries + [("user_id", 1), ("deleted", 1), ("created_at", -1)], # Compound index for paginated list queries IndexModel([("external_source_id", 1)], sparse=True) # Sparse index for deduplication ] diff --git a/backends/advanced/src/advanced_omi_backend/plugins/__init__.py b/backends/advanced/src/advanced_omi_backend/plugins/__init__.py index 3ccea7dc..90c47460 100644 --- a/backends/advanced/src/advanced_omi_backend/plugins/__init__.py +++ b/backends/advanced/src/advanced_omi_backend/plugins/__init__.py @@ -5,6 +5,8 @@ - transcript: When new transcript segment arrives - conversation: When conversation processing completes - memory: After memory extraction finishes +- button: When device button events are received +- plugin_action: Cross-plugin communication Trigger types control when plugins execute: - wake_word: Only when transcript starts with specified wake word @@ -13,6 +15,18 @@ """ from .base import BasePlugin, PluginContext, PluginResult +from .events import ButtonActionType, ButtonState, ConversationCloseReason, PluginEvent from .router import PluginRouter +from .services import PluginServices -__all__ = ['BasePlugin', 'PluginContext', 'PluginResult', 'PluginRouter'] +__all__ = [ + 'BasePlugin', + 'ButtonActionType', + 'ButtonState', + 'ConversationCloseReason', + 'PluginContext', + 'PluginEvent', + 'PluginResult', + 'PluginRouter', + 'PluginServices', +] diff --git a/backends/advanced/src/advanced_omi_backend/plugins/base.py b/backends/advanced/src/advanced_omi_backend/plugins/base.py index bb55128a..2bfe3609 100644 --- a/backends/advanced/src/advanced_omi_backend/plugins/base.py +++ b/backends/advanced/src/advanced_omi_backend/plugins/base.py @@ -18,6 +18,7 @@ class PluginContext: event: str # Event name (e.g., "transcript.streaming", "conversation.complete") data: Dict[str, Any] # Event-specific data metadata: Dict[str, Any] = field(default_factory=dict) + services: Optional[Any] = None # PluginServices instance for system/cross-plugin calls @dataclass @@ -56,24 +57,10 @@ def __init__(self, config: Dict[str, Any]): config: Plugin configuration from config/plugins.yml Contains: enabled, events, condition, and plugin-specific config """ - import logging - logger = logging.getLogger(__name__) - self.config = config self.enabled = config.get('enabled', False) - - # NEW terminology with backward compatibility - self.events = config.get('events') or config.get('subscriptions', []) - self.condition = config.get('condition') or config.get('trigger', {'type': 'always'}) - - # Deprecation warnings - plugin_name = config.get('name', 'unknown') - if 'subscriptions' in config: - logger.warning(f"Plugin '{plugin_name}': 'subscriptions' is deprecated, use 'events' instead") - if 'trigger' in config: - logger.warning(f"Plugin '{plugin_name}': 'condition' is deprecated, use 'condition' instead") - if 'access_level' in config: - logger.warning(f"Plugin '{plugin_name}': 'access_level' is deprecated and ignored") + self.events = config.get('events', []) + self.condition = config.get('condition', {'type': 'always'}) def register_prompts(self, registry) -> None: """Register plugin prompts with the prompt registry. @@ -154,3 +141,30 @@ async def on_memory_processed(self, context: PluginContext) -> Optional[PluginRe PluginResult with success status, optional message, and should_continue flag """ pass + + async def on_button_event(self, context: PluginContext) -> Optional[PluginResult]: + """ + Called when a device button event is received. + + Context data contains: + - state: str - Button state (e.g., "SINGLE_TAP", "DOUBLE_TAP", "LONG_PRESS") + - timestamp: float - Unix timestamp of the event + - audio_uuid: str - Current audio session UUID (may be None) + + Returns: + PluginResult with success status, optional message, and should_continue flag + """ + pass + + async def on_plugin_action(self, context: PluginContext) -> Optional[PluginResult]: + """ + Called when another plugin dispatches an action to this plugin via PluginServices.call_plugin(). + + Context data contains: + - action: str - Action name (e.g., "toggle_lights", "call_service") + - Plus any additional data from the calling plugin + + Returns: + PluginResult with success status, optional message, and should_continue flag + """ + pass diff --git a/backends/advanced/src/advanced_omi_backend/plugins/events.py b/backends/advanced/src/advanced_omi_backend/plugins/events.py new file mode 100644 index 00000000..210c8fd6 --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/plugins/events.py @@ -0,0 +1,57 @@ +""" +Single source of truth for all plugin event types, button states, and action types. + +All event names, button states, and action types live here. No raw strings anywhere else. +Using str, Enum so values work directly as strings in Redis, YAML, JSON — but code +always references the enum member, never a raw string. +""" + +from enum import Enum +from typing import Dict + + +class PluginEvent(str, Enum): + """All events that can trigger plugins.""" + + # Conversation lifecycle + CONVERSATION_COMPLETE = "conversation.complete" + TRANSCRIPT_STREAMING = "transcript.streaming" + TRANSCRIPT_BATCH = "transcript.batch" + MEMORY_PROCESSED = "memory.processed" + + # Button events (from OMI device) + BUTTON_SINGLE_PRESS = "button.single_press" + BUTTON_DOUBLE_PRESS = "button.double_press" + + # Cross-plugin communication (dispatched by PluginServices.call_plugin) + PLUGIN_ACTION = "plugin_action" + + +class ButtonState(str, Enum): + """Raw button states from OMI device firmware.""" + + SINGLE_TAP = "SINGLE_TAP" + DOUBLE_TAP = "DOUBLE_TAP" + LONG_PRESS = "LONG_PRESS" + + +# Maps device button states to plugin events +BUTTON_STATE_TO_EVENT: Dict[ButtonState, PluginEvent] = { + ButtonState.SINGLE_TAP: PluginEvent.BUTTON_SINGLE_PRESS, + ButtonState.DOUBLE_TAP: PluginEvent.BUTTON_DOUBLE_PRESS, +} + + +class ButtonActionType(str, Enum): + """Types of actions a button press can trigger (from test_button_actions plugin config).""" + + CLOSE_CONVERSATION = "close_conversation" + CALL_PLUGIN = "call_plugin" + + +class ConversationCloseReason(str, Enum): + """Reasons for requesting a conversation close.""" + + USER_REQUESTED = "user_requested" + PLUGIN_REQUESTED = "plugin_requested" + BUTTON_CLOSE = "button_close" diff --git a/backends/advanced/src/advanced_omi_backend/plugins/router.py b/backends/advanced/src/advanced_omi_backend/plugins/router.py index 523fe3ed..422a97da 100644 --- a/backends/advanced/src/advanced_omi_backend/plugins/router.py +++ b/backends/advanced/src/advanced_omi_backend/plugins/router.py @@ -4,12 +4,18 @@ Routes pipeline events to appropriate plugins based on access level and triggers. """ +import json import logging +import os import re import string +import time from typing import Dict, List, Optional +import redis + from .base import BasePlugin, PluginContext, PluginResult +from .events import PluginEvent logger = logging.getLogger(__name__) @@ -86,10 +92,26 @@ def extract_command_after_wake_word(transcript: str, wake_word: str) -> str: class PluginRouter: """Routes pipeline events to appropriate plugins based on event subscriptions""" + _EVENT_LOG_KEY = "system:event_log" + _EVENT_LOG_MAX = 1000 + def __init__(self): self.plugins: Dict[str, BasePlugin] = {} # Index plugins by event for fast lookup self._plugins_by_event: Dict[str, List[str]] = {} + self._services = None + + # Sync Redis for event logging (works from both FastAPI and RQ workers) + redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0") + try: + self._event_redis = redis.from_url(redis_url, decode_responses=True) + except Exception: + logger.warning("Could not connect to Redis for event logging") + self._event_redis = None + + def set_services(self, services) -> None: + """Attach PluginServices instance for injection into plugin contexts.""" + self._services = services def register_plugin(self, plugin_id: str, plugin: BasePlugin): """Register a plugin with the router""" @@ -126,16 +148,15 @@ async def dispatch_event( logger.info(f"🔌 ROUTER: Dispatching '{event}' event (user={user_id})") results = [] + executed = [] # Track per-plugin outcomes for event log # Get plugins subscribed to this event plugin_ids = self._plugins_by_event.get(event, []) - # Add subscription check if not plugin_ids: - logger.warning(f"🔌 ROUTER: No plugins subscribed to event '{event}'") - return results - - logger.info(f"🔌 ROUTER: Found {len(plugin_ids)} subscribed plugin(s): {plugin_ids}") + logger.info(f"🔌 ROUTER: No plugins subscribed to event '{event}'") + else: + logger.info(f"🔌 ROUTER: Found {len(plugin_ids)} subscribed plugin(s): {plugin_ids}") for plugin_id in plugin_ids: plugin = self.plugins[plugin_id] @@ -157,7 +178,8 @@ async def dispatch_event( user_id=user_id, event=event, data=data, - metadata=metadata or {} + metadata=metadata or {}, + services=self._services, ) result = await self._execute_plugin(plugin, event, context) @@ -169,6 +191,7 @@ async def dispatch_event( f"success={result.success}, message={result.message}" ) results.append(result) + executed.append({"plugin_id": plugin_id, "success": result.success, "message": result.message}) # If plugin says stop processing, break if not result.should_continue: @@ -181,6 +204,7 @@ async def dispatch_event( f" ✗ Plugin '{plugin_id}' FAILED with exception: {e}", exc_info=True ) + executed.append({"plugin_id": plugin_id, "success": False, "message": str(e)}) # Add at end logger.info( @@ -188,6 +212,14 @@ async def dispatch_event( f"{len(results)} plugin(s) executed successfully" ) + self._log_event( + event=event, + user_id=user_id, + plugins_subscribed=plugin_ids, + plugins_executed=executed, + metadata=metadata, + ) + return results async def _should_execute(self, plugin: BasePlugin, data: Dict) -> bool: @@ -236,16 +268,66 @@ async def _execute_plugin( context: PluginContext ) -> Optional[PluginResult]: """Execute plugin method for specified event""" - # Map events to plugin callback methods - if event.startswith('transcript.'): + # Map events to plugin callback methods using enums + # str(Enum) comparisons work because PluginEvent inherits from str + if event in (PluginEvent.TRANSCRIPT_STREAMING, PluginEvent.TRANSCRIPT_BATCH): return await plugin.on_transcript(context) - elif event.startswith('conversation.'): + elif event in (PluginEvent.CONVERSATION_COMPLETE,): return await plugin.on_conversation_complete(context) - elif event.startswith('memory.'): + elif event in (PluginEvent.MEMORY_PROCESSED,): return await plugin.on_memory_processed(context) + elif event in (PluginEvent.BUTTON_SINGLE_PRESS, PluginEvent.BUTTON_DOUBLE_PRESS): + return await plugin.on_button_event(context) + elif event == PluginEvent.PLUGIN_ACTION: + return await plugin.on_plugin_action(context) + # Fallback for any unrecognized events (forward compatibility) + logger.warning(f"No handler mapping for event '{event}'") return None + def _log_event( + self, + event: str, + user_id: str, + plugins_subscribed: List[str], + plugins_executed: List[Dict], + metadata: Optional[Dict] = None, + ) -> None: + """Append an event record to the Redis event log (capped list).""" + if not self._event_redis: + return + try: + record = json.dumps({ + "timestamp": time.time(), + "event": event, + "user_id": user_id, + "plugins_subscribed": plugins_subscribed, + "plugins_executed": plugins_executed, + "metadata": metadata or {}, + }) + pipe = self._event_redis.pipeline() + pipe.lpush(self._EVENT_LOG_KEY, record) + pipe.ltrim(self._EVENT_LOG_KEY, 0, self._EVENT_LOG_MAX - 1) + pipe.execute() + except Exception: + logger.debug("Failed to log event to Redis", exc_info=True) + + def get_recent_events(self, limit: int = 50, event_type: Optional[str] = None) -> List[Dict]: + """Read recent events from the Redis log.""" + if not self._event_redis: + return [] + try: + # Fetch more than needed when filtering by type + fetch_count = self._EVENT_LOG_MAX if event_type else limit + raw = self._event_redis.lrange(self._EVENT_LOG_KEY, 0, fetch_count - 1) + events = [json.loads(r) for r in raw] + if event_type: + events = [e for e in events if e.get("event") == event_type][:limit] + return events + except Exception: + logger.debug("Failed to read events from Redis", exc_info=True) + return [] + async def cleanup_all(self): """Clean up all registered plugins""" for plugin_id, plugin in self.plugins.items(): diff --git a/backends/advanced/src/advanced_omi_backend/plugins/services.py b/backends/advanced/src/advanced_omi_backend/plugins/services.py new file mode 100644 index 00000000..0322265b --- /dev/null +++ b/backends/advanced/src/advanced_omi_backend/plugins/services.py @@ -0,0 +1,99 @@ +""" +PluginServices — typed interface for plugin-to-system and plugin-to-plugin communication. + +Plugins use this interface (via context.services) to interact with the core system +(e.g., close a conversation) or with other plugins (e.g., call Home Assistant to toggle lights). +""" + +import logging +from typing import TYPE_CHECKING, Optional + +from .base import PluginContext, PluginResult +from .events import ConversationCloseReason, PluginEvent + +if TYPE_CHECKING: + from .router import PluginRouter + +logger = logging.getLogger(__name__) + + +class PluginServices: + """Typed interface for plugin-to-system and plugin-to-plugin communication.""" + + def __init__(self, router: "PluginRouter", redis_url: str): + self._router = router + self._redis_url = redis_url + + async def close_conversation( + self, + session_id: str, + reason: ConversationCloseReason = ConversationCloseReason.PLUGIN_REQUESTED, + ) -> bool: + """Request closing the current conversation for a session. + + Signals the open_conversation_job to close the current conversation + and trigger post-processing. The session stays active for new conversations. + + Args: + session_id: The streaming session ID (typically same as client_id) + reason: Why the conversation is being closed + + Returns: + True if the close request was set successfully + """ + import redis.asyncio as aioredis + + from advanced_omi_backend.controllers.session_controller import ( + request_conversation_close, + ) + + r = aioredis.from_url(self._redis_url) + try: + return await request_conversation_close(r, session_id, reason=reason.value) + finally: + await r.aclose() + + async def call_plugin( + self, + plugin_id: str, + action: str, + data: dict, + user_id: str = "system", + ) -> Optional[PluginResult]: + """Dispatch an action to another plugin's on_plugin_action() handler. + + Args: + plugin_id: Target plugin identifier (e.g., "homeassistant") + action: Action name (e.g., "toggle_lights") + data: Action-specific data + user_id: User context for the action + + Returns: + PluginResult from the target plugin, or error result if plugin not found + """ + plugin = self._router.plugins.get(plugin_id) + if not plugin: + logger.warning(f"Plugin '{plugin_id}' not found for cross-plugin call") + return PluginResult(success=False, message=f"Plugin '{plugin_id}' not found") + if not plugin.enabled: + logger.warning(f"Plugin '{plugin_id}' is disabled, cannot call") + return PluginResult(success=False, message=f"Plugin '{plugin_id}' is disabled") + + context = PluginContext( + user_id=user_id, + event=PluginEvent.PLUGIN_ACTION, + data={**data, "action": action}, + services=self, + ) + + try: + result = await plugin.on_plugin_action(context) + if result: + logger.info( + f"Cross-plugin call {plugin_id}.{action}: " + f"success={result.success}, message={result.message}" + ) + return result + except Exception as e: + logger.error(f"Cross-plugin call to {plugin_id}.{action} failed: {e}", exc_info=True) + return PluginResult(success=False, message=f"Plugin action failed: {e}") diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py index 2de13ae7..c4424904 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/conversation_routes.py @@ -32,10 +32,13 @@ async def close_current_conversation( @router.get("") async def get_conversations( include_deleted: bool = Query(False, description="Include soft-deleted conversations"), + include_unprocessed: bool = Query(False, description="Include orphan audio sessions (always_persist with failed/pending transcription)"), + limit: int = Query(200, ge=1, le=500, description="Max conversations to return"), + offset: int = Query(0, ge=0, description="Number of conversations to skip"), current_user: User = Depends(current_active_user) ): """Get conversations. Admins see all conversations, users see only their own.""" - return await conversation_controller.get_conversations(current_user, include_deleted) + return await conversation_controller.get_conversations(current_user, include_deleted, include_unprocessed, limit, offset) @router.get("/{conversation_id}") @@ -48,6 +51,14 @@ async def get_conversation_detail( # New reprocessing endpoints +@router.post("/{conversation_id}/reprocess-orphan") +async def reprocess_orphan( + conversation_id: str, current_user: User = Depends(current_active_user) +): + """Reprocess an orphan audio session (always_persist conversation with failed/pending transcription).""" + return await conversation_controller.reprocess_orphan(conversation_id, current_user) + + @router.post("/{conversation_id}/reprocess-transcript") async def reprocess_transcript( conversation_id: str, current_user: User = Depends(current_active_user) diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/queue_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/queue_routes.py index 934cf0b1..ff3d460a 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/queue_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/queue_routes.py @@ -321,6 +321,30 @@ def process_job_and_dependents(job, queue_name, base_status): raise HTTPException(status_code=500, detail=f"Failed to get jobs for client: {str(e)}") +@router.get("/events") +async def get_events( + limit: int = Query(50, ge=1, le=200, description="Number of recent events"), + event_type: str = Query(None, description="Filter by event type"), + current_user: User = Depends(current_active_user), +): + """Get recent system events from the event log (admin only).""" + if not current_user.is_superuser: + raise HTTPException(status_code=403, detail="Admin access required") + + try: + from advanced_omi_backend.services.plugin_service import get_plugin_router + + router_instance = get_plugin_router() + if not router_instance: + return {"events": [], "total": 0} + + events = router_instance.get_recent_events(limit=limit, event_type=event_type or None) + return {"events": events, "total": len(events)} + except Exception as e: + logger.error(f"Failed to get events: {e}") + return {"events": [], "total": 0} + + @router.get("/stats") async def get_queue_stats_endpoint( current_user: User = Depends(current_active_user) @@ -1034,6 +1058,21 @@ def get_job_status(job): logger.error(f"Error fetching jobs for client {client_id}: {e}") return {"client_id": client_id, "jobs": []} + async def fetch_events(): + """Fetch recent system events from the event log (admin only).""" + if not current_user.is_superuser: + return [] + try: + from advanced_omi_backend.services.plugin_service import get_plugin_router + + router_instance = get_plugin_router() + if not router_instance: + return [] + return router_instance.get_recent_events(limit=50) + except Exception as e: + logger.error(f"Error fetching events: {e}") + return [] + # Execute all fetches in parallel (using RQ standard status names) queued_jobs_task = fetch_jobs_by_status("queued", limit=100) started_jobs_task = fetch_jobs_by_status("started", limit=100) # RQ standard, not "processing" @@ -1041,6 +1080,7 @@ def get_job_status(job): failed_jobs_task = fetch_jobs_by_status("failed", limit=50) stats_task = fetch_stats() streaming_status_task = fetch_streaming_status() + events_task = fetch_events() client_jobs_tasks = [fetch_client_jobs(cid) for cid in expanded_client_ids] results = await asyncio.gather( @@ -1050,6 +1090,7 @@ def get_job_status(job): failed_jobs_task, stats_task, streaming_status_task, + events_task, *client_jobs_tasks, return_exceptions=True ) @@ -1060,8 +1101,9 @@ def get_job_status(job): failed_jobs = results[3] if not isinstance(results[3], Exception) else [] stats = results[4] if not isinstance(results[4], Exception) else {"total_jobs": 0} streaming_status = results[5] if not isinstance(results[5], Exception) else {"active_sessions": []} + events = results[6] if not isinstance(results[6], Exception) else [] recent_conversations = [] - client_jobs_results = results[6:] if len(results) > 6 else [] + client_jobs_results = results[7:] if len(results) > 7 else [] # Convert client jobs list to dict client_jobs = {} @@ -1092,6 +1134,7 @@ def get_job_status(job): "streaming_status": streaming_status, "recent_conversations": conversations_list, "client_jobs": client_jobs, + "events": events, "timestamp": asyncio.get_event_loop().time() } diff --git a/backends/advanced/src/advanced_omi_backend/services/audio_stream/producer.py b/backends/advanced/src/advanced_omi_backend/services/audio_stream/producer.py index e7fae522..dc5e9b27 100644 --- a/backends/advanced/src/advanced_omi_backend/services/audio_stream/producer.py +++ b/backends/advanced/src/advanced_omi_backend/services/audio_stream/producer.py @@ -4,6 +4,7 @@ import logging import time +import json import redis.asyncio as redis @@ -139,6 +140,19 @@ async def send_session_end_signal(self, session_id: str): stream_name = buffer["stream_name"] # Send special "end" message to signal workers to flush + # Read audio format from Redis session metadata (stored at audio-start time) + sample_rate, channels, sample_width = 16000, 1, 2 + try: + session_key = f"audio:session:{session_id}" + audio_format_raw = await self.redis_client.hget(session_key, "audio_format") + if audio_format_raw: + audio_format = json.loads(audio_format_raw) + sample_rate = int(audio_format.get("rate", 16000)) + channels = int(audio_format.get("channels", 1)) + sample_width = int(audio_format.get("width", 2)) + except Exception: + pass # Fall back to defaults + end_signal = { b"audio_data": b"", # Empty audio data b"session_id": session_id.encode(), @@ -146,9 +160,9 @@ async def send_session_end_signal(self, session_id: str): b"user_id": buffer["user_id"].encode(), b"client_id": buffer["client_id"].encode(), b"timestamp": str(time.time()).encode(), - b"sample_rate": b"16000", - b"channels": b"1", - b"sample_width": b"2", + b"sample_rate": str(sample_rate).encode(), + b"channels": str(channels).encode(), + b"sample_width": str(sample_width).encode(), } await self.redis_client.xadd( diff --git a/backends/advanced/src/advanced_omi_backend/services/plugin_service.py b/backends/advanced/src/advanced_omi_backend/services/plugin_service.py index 2a69e860..e71422f8 100644 --- a/backends/advanced/src/advanced_omi_backend/services/plugin_service.py +++ b/backends/advanced/src/advanced_omi_backend/services/plugin_service.py @@ -9,6 +9,7 @@ import logging import os import re +import sys from pathlib import Path from typing import Any, Dict, List, Optional, Type @@ -16,6 +17,7 @@ from advanced_omi_backend.config_loader import get_plugins_yml_path from advanced_omi_backend.plugins import BasePlugin, PluginRouter +from advanced_omi_backend.plugins.services import PluginServices logger = logging.getLogger(__name__) @@ -23,6 +25,22 @@ _plugin_router: Optional[PluginRouter] = None +def _get_plugins_dir() -> Path: + """Get external plugins directory. + + Priority: PLUGINS_DIR env var > Docker path > local dev path. + """ + env_dir = os.getenv("PLUGINS_DIR") + if env_dir: + return Path(env_dir) + docker_path = Path("/app/plugins") + if docker_path.is_dir(): + return docker_path + # Local dev: plugin_service.py is at /backends/advanced/src/advanced_omi_backend/services/ + repo_root = Path(__file__).resolve().parents[5] + return repo_root / "plugins" + + def expand_env_vars(value: Any) -> Any: """ Recursively expand environment variables in configuration values. @@ -105,9 +123,7 @@ def load_plugin_config(plugin_id: str, orchestration_config: Dict[str, Any]) -> # 1. Load plugin-specific config.yml if it exists try: - import advanced_omi_backend.plugins - - plugins_dir = Path(advanced_omi_backend.plugins.__file__).parent + plugins_dir = _get_plugins_dir() plugin_config_path = plugins_dir / plugin_id / "config.yml" if plugin_config_path.exists(): @@ -284,9 +300,7 @@ def load_schema_yml(plugin_id: str) -> Optional[Dict[str, Any]]: Schema dictionary if schema.yml exists, None otherwise """ try: - import advanced_omi_backend.plugins - - plugins_dir = Path(advanced_omi_backend.plugins.__file__).parent + plugins_dir = _get_plugins_dir() schema_path = plugins_dir / plugin_id / "schema.yml" if schema_path.exists(): @@ -396,9 +410,7 @@ def get_plugin_metadata( """ # Load plugin config.yml try: - import advanced_omi_backend.plugins - - plugins_dir = Path(advanced_omi_backend.plugins.__file__).parent + plugins_dir = _get_plugins_dir() plugin_config_path = plugins_dir / plugin_id / "config.yml" config_dict = {} @@ -469,23 +481,21 @@ def discover_plugins() -> Dict[str, Type[BasePlugin]]: """ discovered_plugins = {} - # Get the plugins directory path - try: - import advanced_omi_backend.plugins - - plugins_dir = Path(advanced_omi_backend.plugins.__file__).parent - except Exception as e: - logger.error(f"Failed to locate plugins directory: {e}") + plugins_dir = _get_plugins_dir() + if not plugins_dir.is_dir(): + logger.warning(f"Plugins directory not found: {plugins_dir}") return discovered_plugins - logger.info(f"🔍 Scanning for plugins in: {plugins_dir}") + # Add plugins dir to sys.path so plugin packages can be imported directly + plugins_dir_str = str(plugins_dir) + if plugins_dir_str not in sys.path: + sys.path.insert(0, plugins_dir_str) - # Skip these known system directories/files - skip_items = {"__pycache__", "__init__.py", "base.py", "router.py"} + logger.info(f"Scanning for plugins in: {plugins_dir}") - # Scan for plugin directories + # Scan for plugin directories (skip hidden/underscore dirs) for item in plugins_dir.iterdir(): - if not item.is_dir() or item.name in skip_items: + if not item.is_dir() or item.name.startswith("_"): continue plugin_id = item.name @@ -500,12 +510,9 @@ def discover_plugins() -> Dict[str, Type[BasePlugin]]: # e.g., email_summarizer -> EmailSummarizerPlugin class_name = "".join(word.capitalize() for word in plugin_id.split("_")) + "Plugin" - # Import the plugin module - module_path = f"advanced_omi_backend.plugins.{plugin_id}" - logger.debug(f"Attempting to import plugin from: {module_path}") - - # Import the plugin package (which should export the class in __init__.py) - plugin_module = importlib.import_module(module_path) + # Import the plugin package directly (it's on sys.path now) + logger.debug(f"Attempting to import plugin: {plugin_id}") + plugin_module = importlib.import_module(plugin_id) # Try to get the plugin class if not hasattr(plugin_module, class_name): @@ -530,14 +537,14 @@ def discover_plugins() -> Dict[str, Type[BasePlugin]]: # Successfully discovered plugin discovered_plugins[plugin_id] = plugin_class - logger.info(f"✅ Discovered plugin: '{plugin_id}' ({class_name})") + logger.info(f"Discovered plugin: '{plugin_id}' ({class_name})") except ImportError as e: logger.warning(f"Failed to import plugin '{plugin_id}': {e}") except Exception as e: logger.error(f"Error discovering plugin '{plugin_id}': {e}", exc_info=True) - logger.info(f"🎉 Plugin discovery complete: {len(discovered_plugins)} plugin(s) found") + logger.info(f"Plugin discovery complete: {len(discovered_plugins)} plugin(s) found") return discovered_plugins @@ -577,9 +584,6 @@ def init_plugin_router() -> Optional[PluginRouter]: # Discover all plugins via auto-discovery discovered_plugins = discover_plugins() - # Core plugin names (for informational logging only) - CORE_PLUGIN_NAMES = {"homeassistant", "test_event"} - # Initialize each plugin listed in config/plugins.yml for plugin_id, orchestration_config in plugins_data.items(): logger.info( @@ -602,7 +606,6 @@ def init_plugin_router() -> Optional[PluginRouter]: # Get plugin class from discovered plugins plugin_class = discovered_plugins[plugin_id] - plugin_type = "core" if plugin_id in CORE_PLUGIN_NAMES else "community" # Instantiate and register the plugin plugin = plugin_class(plugin_config) @@ -616,7 +619,7 @@ def init_plugin_router() -> Optional[PluginRouter]: # Note: async initialization happens in app_factory lifespan _plugin_router.register_plugin(plugin_id, plugin) - logger.info(f"✅ Plugin '{plugin_id}' registered successfully ({plugin_type})") + logger.info(f"Plugin '{plugin_id}' registered successfully") except Exception as e: logger.error(f"Failed to register plugin '{plugin_id}': {e}", exc_info=True) @@ -627,6 +630,11 @@ def init_plugin_router() -> Optional[PluginRouter]: else: logger.info("No plugins.yml found, plugins disabled") + # Attach PluginServices for cross-plugin and system interaction + redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0") + services = PluginServices(router=_plugin_router, redis_url=redis_url) + _plugin_router.set_services(services) + return _plugin_router except Exception as e: diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/__init__.py b/backends/advanced/src/advanced_omi_backend/services/transcription/__init__.py index 9c7f1d21..48637b13 100644 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/__init__.py +++ b/backends/advanced/src/advanced_omi_backend/services/transcription/__init__.py @@ -141,7 +141,13 @@ async def transcribe(self, audio_data: bytes, sample_rate: int, diarize: bool = # Build headers (skip Content-Type for multipart as httpx will set it) headers = {} if not use_multipart: - headers["Content-Type"] = "audio/raw" + # Auto-detect WAV format from RIFF header and use correct Content-Type. + # Sending WAV data as audio/raw can cause Deepgram to silently return + # empty transcripts because it tries to decode the WAV header as raw PCM. + if audio_data[:4] == b"RIFF": + headers["Content-Type"] = "audio/wav" + else: + headers["Content-Type"] = "audio/raw" if self.model.api_key: # Allow templated header, otherwise fallback to Bearer/Token conventions by config diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py b/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py index 052680d2..749558ce 100644 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py +++ b/backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py @@ -19,6 +19,8 @@ import redis.asyncio as redis from redis import exceptions as redis_exceptions +from advanced_omi_backend.plugins.events import PluginEvent + from advanced_omi_backend.client_manager import get_client_owner_async from advanced_omi_backend.plugins.router import PluginRouter from advanced_omi_backend.services.transcription import get_transcription_provider @@ -357,7 +359,7 @@ async def trigger_plugins(self, session_id: str, result: Dict): logger.info(f"🎯 Dispatching transcript.streaming event for user {user_id}, transcript: {plugin_data['transcript'][:50]}...") plugin_results = await self.plugin_router.dispatch_event( - event='transcript.streaming', + event=PluginEvent.TRANSCRIPT_STREAMING, user_id=user_id, data=plugin_data, metadata={'client_id': session_id} @@ -387,8 +389,20 @@ async def process_stream(self, stream_name: str): "started_at": time.time() } + # Read actual sample rate from the session's audio_format stored in Redis + sample_rate = 16000 + session_key = f"audio:session:{session_id}" + try: + audio_format_raw = await self.redis_client.hget(session_key, "audio_format") + if audio_format_raw: + audio_format = json.loads(audio_format_raw) + sample_rate = int(audio_format.get("rate", 16000)) + logger.info(f"📊 Read sample rate {sample_rate}Hz from session {session_id}") + except Exception as e: + logger.warning(f"Failed to read audio_format from Redis for {session_id}: {e}") + # Start WebSocket connection to Deepgram - await self.start_session_stream(session_id) + await self.start_session_stream(session_id, sample_rate=sample_rate) last_id = "0" # Start from beginning stream_ended = False diff --git a/backends/advanced/src/advanced_omi_backend/speaker_recognition_client.py b/backends/advanced/src/advanced_omi_backend/speaker_recognition_client.py index d7ffc6f1..ea8510fe 100644 --- a/backends/advanced/src/advanced_omi_backend/speaker_recognition_client.py +++ b/backends/advanced/src/advanced_omi_backend/speaker_recognition_client.py @@ -181,7 +181,7 @@ async def diarize_identify_match( form_data.add_field("transcript_data", json.dumps(transcript_data)) form_data.add_field("user_id", "1") # TODO: Implement proper user mapping - form_data.add_field("similarity_threshold", str(config.get("similarity_threshold", 0.15))) + form_data.add_field("similarity_threshold", str(config.get("similarity_threshold", 0.45))) form_data.add_field("min_duration", str(config.get("min_duration", 0.5))) # Use /v1/diarize-identify-match endpoint as fallback @@ -194,7 +194,7 @@ async def diarize_identify_match( # Send existing transcript for diarization and speaker matching form_data.add_field("transcript_data", json.dumps(transcript_data)) form_data.add_field("user_id", "1") # TODO: Implement proper user mapping - form_data.add_field("similarity_threshold", str(config.get("similarity_threshold", 0.15))) + form_data.add_field("similarity_threshold", str(config.get("similarity_threshold", 0.45))) # Add pyannote diarization parameters form_data.add_field("min_duration", str(config.get("min_duration", 0.5))) @@ -352,7 +352,7 @@ async def identify_provider_segments( ) config = get_diarization_settings() - similarity_threshold = config.get("similarity_threshold", 0.15) + similarity_threshold = config.get("similarity_threshold", 0.45) MAX_SAMPLES_PER_LABEL = 3 @@ -496,7 +496,7 @@ async def _identify_one(seg: Dict) -> Optional[Dict]: "end": seg["end"], "text": seg.get("text", ""), "speaker": label, - "identified_as": mapped[0] if mapped else label, + "identified_as": mapped[0] if mapped else None, "confidence": mapped[1] if mapped else 0.0, "status": "identified" if mapped else "unknown", }) @@ -610,7 +610,7 @@ async def _identify_one(seg: Dict) -> Optional[Dict]: "end": seg["end"], "text": seg.get("text", ""), "speaker": label, - "identified_as": label, + "identified_as": None, "confidence": 0.0, "status": "too_short", }) @@ -640,7 +640,7 @@ async def _identify_one(seg: Dict) -> Optional[Dict]: "end": seg["end"], "text": seg.get("text", ""), "speaker": label, - "identified_as": label, + "identified_as": None, "confidence": 0.0, "status": "unknown", }) @@ -700,7 +700,7 @@ async def diarize_and_identify( # Add all diarization parameters for the diarize-and-identify endpoint min_duration = diarization_settings.get("min_duration", 0.5) - similarity_threshold = diarization_settings.get("similarity_threshold", 0.15) + similarity_threshold = diarization_settings.get("similarity_threshold", 0.45) collar = diarization_settings.get("collar", 2.0) min_duration_off = diarization_settings.get("min_duration_off", 1.5) @@ -829,7 +829,7 @@ async def identify_speakers(self, audio_path: str, segments: List[Dict]) -> Dict # Add all diarization parameters for the diarize-and-identify endpoint form_data.add_field("min_duration", str(_diarization_settings.get("min_duration", 0.5))) - form_data.add_field("similarity_threshold", str(_diarization_settings.get("similarity_threshold", 0.15))) + form_data.add_field("similarity_threshold", str(_diarization_settings.get("similarity_threshold", 0.45))) form_data.add_field("collar", str(_diarization_settings.get("collar", 2.0))) form_data.add_field("min_duration_off", str(_diarization_settings.get("min_duration_off", 1.5))) if _diarization_settings.get("min_speakers"): diff --git a/backends/advanced/src/advanced_omi_backend/utils/audio_utils.py b/backends/advanced/src/advanced_omi_backend/utils/audio_utils.py index 5b5fa992..4abb1d5d 100644 --- a/backends/advanced/src/advanced_omi_backend/utils/audio_utils.py +++ b/backends/advanced/src/advanced_omi_backend/utils/audio_utils.py @@ -27,6 +27,9 @@ MIN_SPEECH_SEGMENT_DURATION = float(os.getenv("MIN_SPEECH_SEGMENT_DURATION", "1.0")) # seconds CROPPING_CONTEXT_PADDING = float(os.getenv("CROPPING_CONTEXT_PADDING", "0.1")) # seconds +SUPPORTED_AUDIO_EXTENSIONS = {".wav", ".mp3", ".mp4", ".m4a", ".flac", ".ogg", ".webm"} +VIDEO_EXTENSIONS = {".mp4", ".webm"} + class AudioValidationError(Exception): """Exception raised when audio validation fails.""" @@ -107,6 +110,59 @@ async def resample_audio_with_ffmpeg( return stdout +async def convert_any_to_wav(file_data: bytes, file_extension: str) -> bytes: + """ + Convert any supported audio/video file to 16kHz mono WAV using FFmpeg. + + For .wav input, returns the data as-is. + For everything else, runs FFmpeg to extract audio and convert to WAV. + + Args: + file_data: Raw file bytes + file_extension: File extension including dot (e.g. ".mp3", ".mp4") + + Returns: + WAV file bytes (16kHz, mono, 16-bit PCM) + + Raises: + AudioValidationError: If FFmpeg conversion fails + """ + ext = file_extension.lower() + if ext == ".wav": + return file_data + + cmd = [ + "ffmpeg", + "-i", "pipe:0", + "-vn", # Strip video track (no-op for audio-only files) + "-acodec", "pcm_s16le", + "-ar", "16000", + "-ac", "1", + "-f", "wav", + "pipe:1", + ] + + process = await asyncio.create_subprocess_exec( + *cmd, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + stdout, stderr = await process.communicate(input=file_data) + + if process.returncode != 0: + error_msg = stderr.decode() if stderr else "Unknown error" + audio_logger.error(f"FFmpeg conversion failed for {ext}: {error_msg}") + raise AudioValidationError(f"Failed to convert {ext} file to WAV: {error_msg}") + + audio_logger.info( + f"Converted {ext} to WAV: {len(file_data)} → {len(stdout)} bytes" + ) + + return stdout + + async def validate_and_prepare_audio( audio_data: bytes, expected_sample_rate: int = 16000, @@ -207,6 +263,9 @@ async def write_audio_file( timestamp: int, chunk_dir: Optional[Path] = None, validate: bool = True, + pcm_sample_rate: int = 16000, + pcm_channels: int = 1, + pcm_sample_width: int = 2, ) -> tuple[str, str, float]: """ Validate, write audio data to WAV file, and create AudioSession database entry. @@ -223,6 +282,9 @@ async def write_audio_file( timestamp: Timestamp in milliseconds chunk_dir: Optional directory path (defaults to CHUNK_DIR from config) validate: Whether to validate and prepare audio (default: True for uploads, False for WebSocket) + pcm_sample_rate: Sample rate for raw PCM data when validate=False (default: 16000) + pcm_channels: Channel count for raw PCM data when validate=False (default: 1) + pcm_sample_width: Sample width in bytes for raw PCM data when validate=False (default: 2) Returns: Tuple of (relative_audio_path, absolute_file_path, duration) @@ -242,11 +304,11 @@ async def write_audio_file( audio_data, sample_rate, sample_width, channels, duration = \ await validate_and_prepare_audio(raw_audio_data) else: - # For WebSocket path - audio is already processed PCM + # For WebSocket/streaming path - audio is already processed PCM audio_data = raw_audio_data - sample_rate = 16000 # WebSocket always uses 16kHz - sample_width = 2 - channels = 1 + sample_rate = pcm_sample_rate + sample_width = pcm_sample_width + channels = pcm_channels duration = len(audio_data) / (sample_rate * sample_width * channels) # Use provided chunk_dir or default from config diff --git a/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py index 04754431..9b5077e6 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py @@ -5,6 +5,7 @@ """ import asyncio +import json import logging import os import time @@ -20,6 +21,7 @@ ) from advanced_omi_backend.controllers.session_controller import mark_session_complete from advanced_omi_backend.models.job import async_job +from advanced_omi_backend.plugins.events import PluginEvent from advanced_omi_backend.services.plugin_service import ( ensure_plugin_router, get_plugin_router, @@ -303,6 +305,18 @@ async def open_conversation_job( conversation_id = conversation.conversation_id logger.info(f"✅ Created streaming conversation {conversation_id} for session {session_id}") + # Attach markers from Redis session (e.g., button events captured during streaming) + session_key = f"audio:session:{session_id}" + markers_json = await redis_client.hget(session_key, "markers") + if markers_json: + try: + markers_data = markers_json if isinstance(markers_json, str) else markers_json.decode() + conversation.markers = json.loads(markers_data) + await conversation.save() + logger.info(f"📌 Attached {len(conversation.markers)} markers to conversation {conversation_id}") + except Exception as marker_err: + logger.warning(f"⚠️ Failed to parse markers from Redis: {marker_err}") + # Link job metadata to conversation (cascading updates) current_job.meta["conversation_id"] = conversation_id current_job.save_meta() @@ -361,6 +375,7 @@ async def open_conversation_job( 0.0 # Initialize with audio time 0 (will be updated with first speech) ) timeout_triggered = False # Track if closure was due to timeout + close_requested_reason = None # Track if closure was requested via API/plugin/button last_inactivity_log_time = ( time.time() ) # Track when we last logged inactivity (wall-clock for logging) @@ -410,6 +425,17 @@ async def open_conversation_job( ) break # Exit immediately when finalize signal received + # Check for conversation close request (set by API, plugins, button press) + if not finalize_received: + close_reason = await redis_client.hget(session_key, "conversation_close_requested") + if close_reason: + await redis_client.hdel(session_key, "conversation_close_requested") + close_requested_reason = close_reason.decode() if isinstance(close_reason, bytes) else close_reason + logger.info(f"🔒 Conversation close requested: {close_requested_reason}") + timeout_triggered = True # Session stays active (same restart behavior as inactivity timeout) + finalize_received = True + break + # Check max runtime timeout if time.time() - start_time > max_runtime: logger.warning(f"⏱️ Max runtime reached for {conversation_id}") @@ -564,7 +590,7 @@ async def open_conversation_job( ) plugin_results = await plugin_router.dispatch_event( - event="transcript.streaming", + event=PluginEvent.TRANSCRIPT_STREAMING, user_id=user_id, data=plugin_data, metadata={"client_id": client_id}, @@ -602,12 +628,16 @@ async def open_conversation_job( # Determine end_reason with proper precedence: # 1. completion_reason from Redis (set by WebSocket controller: websocket_disconnect, user_stopped) - # 2. inactivity_timeout (no speech for SPEECH_INACTIVITY_THRESHOLD_SECONDS) - # 3. max_duration (conversation exceeded max runtime) - # 4. user_stopped (fallback for any other exit condition) + # 2. close_requested (via API, plugin, or button press) + # 3. inactivity_timeout (no speech for SPEECH_INACTIVITY_THRESHOLD_SECONDS) + # 4. max_duration (conversation exceeded max runtime) + # 5. user_stopped (fallback for any other exit condition) if completion_reason_str: end_reason = completion_reason_str logger.info(f"📊 Using completion_reason from session: {end_reason}") + elif close_requested_reason: + end_reason = "close_requested" + logger.info(f"📊 Conversation closed by request: {close_requested_reason}") elif timeout_triggered: end_reason = "inactivity_timeout" elif time.time() - start_time > max_runtime: @@ -655,30 +685,6 @@ async def open_conversation_job( f"(waited {max_wait_streaming}s), proceeding with available transcript" ) - # Wait for streaming transcription consumer to complete before reading transcript - # This fixes the race condition where conversation job reads transcript before - # streaming consumer stores all final results (seen as 24+ second delay in logs) - completion_key = f"transcription:complete:{session_id}" - max_wait_streaming = 30 # seconds - waited_streaming = 0.0 - while waited_streaming < max_wait_streaming: - completion_status = await redis_client.get(completion_key) - if completion_status: - status_str = completion_status.decode() if isinstance(completion_status, bytes) else completion_status - if status_str == "error": - logger.warning(f"⚠️ Streaming transcription ended with error for {session_id}, proceeding anyway") - else: - logger.info(f"✅ Streaming transcription confirmed complete for {session_id}") - break - await asyncio.sleep(0.5) - waited_streaming += 0.5 - - if waited_streaming >= max_wait_streaming: - logger.warning( - f"⚠️ Timed out waiting for streaming completion signal for {session_id} " - f"(waited {max_wait_streaming}s), proceeding with available transcript" - ) - # Wait for audio_streaming_persistence_job to complete and write MongoDB chunks from advanced_omi_backend.utils.audio_chunk_utils import wait_for_audio_chunks @@ -910,8 +916,8 @@ async def generate_title_summary_job(conversation_id: str, *, redis_client=None) logger.info(f"✅ Generated summary: '{conversation.summary}'") logger.info(f"✅ Generated detailed summary: {len(conversation.detailed_summary)} chars") - # Update processing status for placeholder conversations - if getattr(conversation, "processing_status", None) == "pending_transcription": + # Update processing status for placeholder/reprocessing conversations + if getattr(conversation, "processing_status", None) in ["pending_transcription", "reprocessing"]: conversation.processing_status = "completed" logger.info( f"✅ Updated placeholder conversation {conversation_id} " @@ -921,8 +927,8 @@ async def generate_title_summary_job(conversation_id: str, *, redis_client=None) except Exception as gen_error: logger.error(f"❌ Title/summary generation failed: {gen_error}") - # Mark placeholder conversation as failed - if getattr(conversation, "processing_status", None) == "pending_transcription": + # Mark placeholder/reprocessing conversation as failed + if getattr(conversation, "processing_status", None) in ["pending_transcription", "reprocessing"]: conversation.title = "Audio Recording (Transcription Failed)" conversation.summary = f"Title/summary generation failed: {str(gen_error)}" conversation.processing_status = "transcription_failed" @@ -1080,7 +1086,7 @@ async def dispatch_conversation_complete_event_job( ) plugin_results = await plugin_router.dispatch_event( - event="conversation.complete", + event=PluginEvent.CONVERSATION_COMPLETE, user_id=user_id, data=plugin_data, metadata={"end_reason": actual_end_reason}, diff --git a/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py index f4adb6e3..8bf6d27b 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py @@ -22,6 +22,7 @@ memory_queue, ) from advanced_omi_backend.models.job import JobPriority, async_job +from advanced_omi_backend.plugins.events import PluginEvent from advanced_omi_backend.services.plugin_service import ensure_plugin_router logger = logging.getLogger(__name__) @@ -379,7 +380,7 @@ async def process_memory_job(conversation_id: str, *, redis_client=None) -> Dict ) plugin_results = await plugin_router.dispatch_event( - event="memory.processed", + event=PluginEvent.MEMORY_PROCESSED, user_id=user_id, data=plugin_data, metadata={ diff --git a/backends/advanced/src/advanced_omi_backend/workers/speaker_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/speaker_jobs.py index 5ab6afa6..bfe38c62 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/speaker_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/speaker_jobs.py @@ -452,6 +452,20 @@ async def recognise_speakers_job( speaker_segments = speaker_result["segments"] logger.info(f"🎤 Speaker recognition returned {len(speaker_segments)} segments") + # Build mapping for unknown speakers: diarization_label -> "Unknown Speaker N" + unknown_label_map = {} + unknown_counter = 1 + for seg in speaker_segments: + identified_as = seg.get("identified_as") + if not identified_as: + label = seg.get("speaker", "Unknown") + if label not in unknown_label_map: + unknown_label_map[label] = f"Unknown Speaker {unknown_counter}" + unknown_counter += 1 + + if unknown_label_map: + logger.info(f"🎤 Unknown speaker mapping: {unknown_label_map}") + # Update the transcript version segments with identified speakers # Filter out empty segments (diarization sometimes creates segments with no text) updated_segments = [] @@ -472,7 +486,7 @@ async def recognise_speakers_job( logger.debug(f"Filtered segment with invalid timing: {seg}") continue - speaker_name = seg.get("identified_as") or seg.get("speaker", "Unknown") + speaker_name = seg.get("identified_as") or unknown_label_map.get(seg.get("speaker", "Unknown"), "Unknown Speaker") # Extract words from speaker service response (already matched to this segment) words_data = seg.get("words", []) diff --git a/backends/advanced/src/advanced_omi_backend/workers/transcription_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/transcription_jobs.py index d1cc23e1..f2dda207 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/transcription_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/transcription_jobs.py @@ -5,10 +5,13 @@ """ import asyncio +import io +import json import logging import os import time import uuid +import wave from datetime import datetime from pathlib import Path from typing import Any, Dict @@ -30,6 +33,7 @@ from advanced_omi_backend.models.conversation import Conversation from advanced_omi_backend.models.job import BaseRQJob, JobPriority, async_job from advanced_omi_backend.services.audio_stream import TranscriptionResultsAggregator +from advanced_omi_backend.plugins.events import PluginEvent from advanced_omi_backend.services.plugin_service import ensure_plugin_router from advanced_omi_backend.services.transcription import ( get_transcription_provider, @@ -224,11 +228,18 @@ async def transcribe_full_audio_job( logger.warning(f"Failed to build ASR context: {e}") context_info = None + # Read actual sample rate from WAV header + try: + with wave.open(io.BytesIO(wav_data), "rb") as wf: + actual_sample_rate = wf.getframerate() + except Exception: + actual_sample_rate = 16000 + try: # Transcribe the audio directly from memory (no disk I/O needed) transcribe_kwargs: Dict[str, Any] = { "audio_data": wav_data, - "sample_rate": 16000, + "sample_rate": actual_sample_rate, "diarize": True, } if context_info: @@ -279,7 +290,7 @@ async def transcribe_full_audio_job( ) plugin_results = await plugin_router.dispatch_event( - event="transcript.batch", + event=PluginEvent.TRANSCRIPT_BATCH, user_id=user_id, data=plugin_data, metadata={"client_id": client_id}, @@ -793,9 +804,23 @@ async def transcription_fallback_check_job( sorted_chunks = sorted(audio_chunks.items()) combined_audio = b"".join(data for _, data in sorted_chunks) + # Read audio format from Redis session metadata + sample_rate, channels, sample_width = 16000, 1, 2 + session_key = f"audio:session:{session_id}" + try: + audio_format_raw = await redis_client.hget(session_key, "audio_format") + if audio_format_raw: + audio_format = json.loads(audio_format_raw) + sample_rate = int(audio_format.get("rate", 16000)) + channels = int(audio_format.get("channels", 1)) + sample_width = int(audio_format.get("width", 2)) + except Exception as e: + logger.warning(f"Failed to read audio_format from Redis for {session_id}: {e}") + + bytes_per_second = sample_rate * channels * sample_width logger.info( f"✅ Extracted {len(sorted_chunks)} audio chunks from Redis stream " - f"({len(combined_audio)} bytes, ~{len(combined_audio)/32000:.1f}s)" + f"({len(combined_audio)} bytes, ~{len(combined_audio)/bytes_per_second:.1f}s)" ) # Create conversation placeholder @@ -805,9 +830,9 @@ async def transcription_fallback_check_job( num_chunks = await convert_audio_to_chunks( conversation_id=conversation.conversation_id, audio_data=combined_audio, - sample_rate=16000, - channels=1, - sample_width=2, + sample_rate=sample_rate, + channels=channels, + sample_width=sample_width, ) logger.info( diff --git a/backends/advanced/uv.lock b/backends/advanced/uv.lock index 98dc39d1..9c9a1b1c 100644 --- a/backends/advanced/uv.lock +++ b/backends/advanced/uv.lock @@ -13,6 +13,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, + { name = "croniter" }, { name = "easy-audio-interfaces" }, { name = "en-core-web-sm" }, { name = "fastapi" }, @@ -72,6 +73,7 @@ test = [ [package.metadata] requires-dist = [ { name = "aiohttp", specifier = ">=3.8.0" }, + { name = "croniter", specifier = ">=1.3.0" }, { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = ">=4.0.0" }, { name = "easy-audio-interfaces", specifier = ">=0.7.1" }, { name = "easy-audio-interfaces", extras = ["local-audio"], marker = "extra == 'local-audio'", specifier = ">=0.7.1" }, diff --git a/backends/advanced/webui/src/components/layout/Layout.tsx b/backends/advanced/webui/src/components/layout/Layout.tsx index 814634d9..80d38d37 100644 --- a/backends/advanced/webui/src/components/layout/Layout.tsx +++ b/backends/advanced/webui/src/components/layout/Layout.tsx @@ -18,7 +18,7 @@ export default function Layout() { { path: '/users', label: 'User Management', icon: Users }, ...(isAdmin ? [ { path: '/upload', label: 'Upload Audio', icon: Upload }, - { path: '/queue', label: 'Queue Management', icon: Layers }, + { path: '/queue', label: 'Queue & Events', icon: Layers }, { path: '/plugins', label: 'Plugins', icon: Puzzle }, { path: '/finetuning', label: 'Fine-tuning', icon: Zap }, { path: '/system', label: 'System State', icon: Settings }, diff --git a/backends/advanced/webui/src/components/plugins/OrchestrationSection.tsx b/backends/advanced/webui/src/components/plugins/OrchestrationSection.tsx index f667143c..ceaf51c8 100644 --- a/backends/advanced/webui/src/components/plugins/OrchestrationSection.tsx +++ b/backends/advanced/webui/src/components/plugins/OrchestrationSection.tsx @@ -15,10 +15,14 @@ interface OrchestrationSectionProps { disabled?: boolean } -const AVAILABLE_EVENTS = [ +// Keep in sync with backend PluginEvent enum (plugins/events.py) +const AVAILABLE_EVENTS: { value: string; label: string; note?: string }[] = [ { value: 'conversation.complete', label: 'Conversation Complete' }, { value: 'transcript.streaming', label: 'Transcript Streaming' }, - { value: 'memory.created', label: 'Memory Created' }, + { value: 'memory.processed', label: 'Memory Processed' }, + { value: 'transcript.batch', label: 'Transcript Batch', note: 'file upload' }, + { value: 'button.single_press', label: 'Button Single Press', note: 'from OMI' }, + { value: 'button.double_press', label: 'Button Double Press', note: 'from OMI' }, ] export default function OrchestrationSection({ @@ -132,6 +136,11 @@ export default function OrchestrationSection({ /> {event.label} + {event.note && ( + + ({event.note}) + + )} ))} diff --git a/backends/advanced/webui/src/contexts/RecordingContext.tsx b/backends/advanced/webui/src/contexts/RecordingContext.tsx index ba39badc..7fb2ef19 100644 --- a/backends/advanced/webui/src/contexts/RecordingContext.tsx +++ b/backends/advanced/webui/src/contexts/RecordingContext.tsx @@ -28,6 +28,11 @@ export interface RecordingContextType { stopRecording: () => void setMode: (mode: RecordingMode) => void + // Microphone selection + availableDevices: MediaDeviceInfo[] + selectedDeviceId: string | null + setSelectedDeviceId: (id: string | null) => void + // For components analyser: AnalyserNode | null debugStats: DebugStats @@ -50,6 +55,10 @@ export function RecordingProvider({ children }: { children: ReactNode }) { const [mode, setMode] = useState('streaming') const [analyserState, setAnalyserState] = useState(null) + // Microphone selection + const [availableDevices, setAvailableDevices] = useState([]) + const [selectedDeviceId, setSelectedDeviceId] = useState(null) + // Debug stats const [debugStats, setDebugStats] = useState({ chunksSent: 0, @@ -83,6 +92,24 @@ export function RecordingProvider({ children }: { children: ReactNode }) { const canAccessMicrophone = isLocalhost || isHttps || isDevelopmentHost + // Enumerate audio input devices + const refreshDevices = useCallback(async () => { + try { + const devices = await navigator.mediaDevices.enumerateDevices() + const audioInputs = devices.filter(d => d.kind === 'audioinput') + setAvailableDevices(audioInputs) + } catch (e) { + console.warn('Failed to enumerate audio devices:', e) + } + }, []) + + // Initial device enumeration + listen for device changes + useEffect(() => { + refreshDevices() + navigator.mediaDevices.addEventListener('devicechange', refreshDevices) + return () => navigator.mediaDevices.removeEventListener('devicechange', refreshDevices) + }, [refreshDevices]) + // Format duration helper const formatDuration = useCallback((seconds: number) => { const mins = Math.floor(seconds / 60) @@ -141,18 +168,23 @@ export function RecordingProvider({ children }: { children: ReactNode }) { throw new Error('Microphone access requires HTTPS or localhost') } - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - sampleRate: 16000, - channelCount: 1, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true - } - }) + const audioConstraints: MediaTrackConstraints = { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + } + if (selectedDeviceId) { + audioConstraints.deviceId = { exact: selectedDeviceId } + } + + const stream = await navigator.mediaDevices.getUserMedia({ audio: audioConstraints }) mediaStreamRef.current = stream + // Re-enumerate to get labels after permission grant + refreshDevices() + // Track when mic permission is revoked stream.getTracks().forEach(track => { track.onended = () => { @@ -168,7 +200,7 @@ export function RecordingProvider({ children }: { children: ReactNode }) { console.log('✅ Microphone access granted') return stream - }, [canAccessMicrophone, isRecording, cleanup]) + }, [canAccessMicrophone, selectedDeviceId, isRecording, cleanup, refreshDevices]) // Step 2: Connect WebSocket const connectWebSocket = useCallback(async (): Promise => { @@ -299,10 +331,12 @@ export function RecordingProvider({ children }: { children: ReactNode }) { throw new Error('WebSocket not connected') } + const rate = audioContextRef.current?.sampleRate ?? 16000 + const startMessage = { type: 'audio-start', data: { - rate: 16000, + rate, width: 2, channels: 1, mode: mode // Pass recording mode to backend @@ -311,15 +345,15 @@ export function RecordingProvider({ children }: { children: ReactNode }) { } ws.send(JSON.stringify(startMessage) + '\n') - console.log('✅ Audio-start message sent with mode:', mode) + console.log(`✅ Audio-start message sent with mode: ${mode}, rate: ${rate}`) }, [mode]) // Step 4: Start audio streaming const startAudioStreaming = useCallback(async (stream: MediaStream, ws: WebSocket): Promise => { console.log('🎵 Step 4: Starting audio streaming') - // Set up audio context and analyser for visualization - const audioContext = new AudioContext({ sampleRate: 16000 }) + // Reuse the AudioContext created in startRecording + const audioContext = audioContextRef.current! const analyser = audioContext.createAnalyser() const source = audioContext.createMediaStreamSource(stream) @@ -336,8 +370,6 @@ export function RecordingProvider({ children }: { children: ReactNode }) { await audioContext.resume() console.log('🎧 Audio context resumed, new state:', audioContext.state) } - - audioContextRef.current = audioContext analyserRef.current = analyser setAnalyserState(analyser) @@ -395,7 +427,7 @@ export function RecordingProvider({ children }: { children: ReactNode }) { const chunkHeader = { type: 'audio-chunk', data: { - rate: 16000, + rate: audioContext.sampleRate, width: 2, channels: 1 }, @@ -444,16 +476,22 @@ export function RecordingProvider({ children }: { children: ReactNode }) { // Step 1: Get microphone access const stream = await getMicrophoneAccess() + // Create AudioContext at 16kHz to match the backend pipeline expectation. + // The browser will internally resample from the mic's native rate (e.g. 48kHz). + const audioContext = new AudioContext({ sampleRate: 16000 }) + audioContextRef.current = audioContext + console.log(`🎧 AudioContext created, sample rate: ${audioContext.sampleRate}Hz`) + setCurrentStep('websocket') // Step 2: Connect WebSocket (includes stabilization delay) const ws = await connectWebSocket() setCurrentStep('audio-start') - // Step 3: Send audio-start message + // Step 3: Send audio-start message (uses audioContextRef for sample rate) await sendAudioStartMessage(ws) setCurrentStep('streaming') - // Step 4: Start audio streaming (includes processing delay) + // Step 4: Start audio streaming (reuses existing AudioContext) await startAudioStreaming(stream, ws) // All steps complete - mark as recording @@ -551,6 +589,9 @@ export function RecordingProvider({ children }: { children: ReactNode }) { startRecording, stopRecording, setMode, + availableDevices, + selectedDeviceId, + setSelectedDeviceId, analyser: analyserState, debugStats, formatDuration, diff --git a/backends/advanced/webui/src/pages/Conversations.tsx b/backends/advanced/webui/src/pages/Conversations.tsx index d8861859..cc5d6f98 100644 --- a/backends/advanced/webui/src/pages/Conversations.tsx +++ b/backends/advanced/webui/src/pages/Conversations.tsx @@ -1,5 +1,5 @@ import { useState, useEffect, useRef, useCallback, useMemo } from 'react' -import { MessageSquare, RefreshCw, Calendar, User, Play, Pause, MoreVertical, RotateCcw, Zap, ChevronDown, ChevronUp, Trash2, Save, X, Check } from 'lucide-react' +import { MessageSquare, RefreshCw, Calendar, User, Play, Pause, MoreVertical, RotateCcw, Zap, ChevronDown, ChevronUp, Trash2, Save, X, Check, AlertTriangle } from 'lucide-react' import { conversationsApi, annotationsApi, speakerApi, BACKEND_URL } from '../services/api' import ConversationVersionHeader from '../components/ConversationVersionHeader' import { getStorageKey } from '../utils/storage' @@ -36,6 +36,9 @@ interface Conversation { deleted?: boolean deletion_reason?: string deleted_at?: string + always_persist?: boolean + processing_status?: string + is_orphan?: boolean } // Speaker color palette for consistent colors across conversations @@ -72,6 +75,7 @@ export default function Conversations() { const [reprocessingTranscript, setReprocessingTranscript] = useState>(new Set()) const [reprocessingMemory, setReprocessingMemory] = useState>(new Set()) const [reprocessingSpeakers, setReprocessingSpeakers] = useState>(new Set()) + const [reprocessingOrphan, setReprocessingOrphan] = useState>(new Set()) const [deletingConversation, setDeletingConversation] = useState>(new Set()) // Transcript segment editing state @@ -168,8 +172,8 @@ export default function Conversations() { const loadConversations = async () => { try { setLoading(true) - // Exclude deleted conversations from main view - const response = await conversationsApi.getAll(false) + // Exclude deleted conversations from main view; include orphans in debug mode + const response = await conversationsApi.getAll(false, debugMode ? true : undefined) // API now returns a flat list with client_id as a field const conversationsList = response.data.conversations || [] setConversations(conversationsList) @@ -270,10 +274,14 @@ export default function Conversations() { } useEffect(() => { - loadConversations() loadEnrolledSpeakers() }, []) + // Load conversations on mount and when debug mode toggles (to include/exclude orphans) + useEffect(() => { + loadConversations() + }, [debugMode]) + // Close dropdown when clicking outside useEffect(() => { const handleClickOutside = () => setOpenDropdown(null) @@ -401,6 +409,32 @@ export default function Conversations() { } } + const handleReprocessOrphan = async (conversation: Conversation) => { + try { + if (!conversation.conversation_id) return + + setReprocessingOrphan(prev => new Set(prev).add(conversation.conversation_id!)) + + const response = await conversationsApi.reprocessOrphan(conversation.conversation_id) + + if (response.status === 200) { + await loadConversations() + } else { + setError(`Failed to start orphan reprocessing: ${response.data?.error || 'Unknown error'}`) + } + } catch (err: any) { + setError(`Error starting orphan reprocessing: ${err.message || 'Unknown error'}`) + } finally { + if (conversation.conversation_id) { + setReprocessingOrphan(prev => { + const newSet = new Set(prev) + newSet.delete(conversation.conversation_id!) + return newSet + }) + } + } + } + const handleDeleteConversation = async (conversationId: string) => { try { const confirmed = window.confirm('Are you sure you want to delete this conversation? This action cannot be undone.') @@ -712,8 +746,45 @@ export default function Conversations() { conversations.map((conversation) => (
+ {/* Orphan Audio Session Banner */} + {conversation.is_orphan && ( +
+
+ +
+ + Unprocessed Audio Session + + + {conversation.processing_status === 'transcription_failed' ? 'Transcription failed' : + conversation.processing_status === 'reprocessing' ? 'Reprocessing...' : + conversation.deleted ? `Deleted: ${conversation.deletion_reason}` : + conversation.processing_status || 'Pending'} + {conversation.audio_total_duration ? ` · ${Math.floor(conversation.audio_total_duration / 60)}:${Math.floor(conversation.audio_total_duration % 60).toString().padStart(2, '0')} audio` : ''} + +
+
+ +
+ )} + {/* Version Selector Header */}
+ {/* Microphone Selector */} + {recording.availableDevices.length > 1 && ( +
+ + + +
+ )} + {/* Mode Description */}

diff --git a/backends/advanced/webui/src/pages/MemoryDetail.tsx b/backends/advanced/webui/src/pages/MemoryDetail.tsx index 7f852c56..f49819bb 100644 --- a/backends/advanced/webui/src/pages/MemoryDetail.tsx +++ b/backends/advanced/webui/src/pages/MemoryDetail.tsx @@ -157,18 +157,37 @@ export default function MemoryDetail() { loadMemory() }, [id, user?.id]) - const formatDate = (dateString: string) => { - try { - return new Date(dateString).toLocaleString('en-US', { - year: 'numeric', - month: 'short', - day: 'numeric', - hour: '2-digit', - minute: '2-digit' - }) - } catch { - return dateString + const formatDate = (dateInput: string | number | undefined | null) => { + if (dateInput === undefined || dateInput === null || dateInput === '') { + return 'N/A' + } + + let date: Date + + if (typeof dateInput === 'number') { + date = dateInput > 1e10 ? new Date(dateInput) : new Date(dateInput * 1000) + } else if (typeof dateInput === 'string') { + if (dateInput.match(/^\d+$/)) { + const timestamp = parseInt(dateInput) + date = timestamp > 1e10 ? new Date(timestamp) : new Date(timestamp * 1000) + } else { + date = new Date(dateInput) + } + } else { + date = new Date(dateInput) } + + if (isNaN(date.getTime())) { + return 'N/A' + } + + return date.toLocaleString('en-US', { + year: 'numeric', + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit' + }) } const getMemoryTypeIcon = () => { diff --git a/backends/advanced/webui/src/pages/Queue.tsx b/backends/advanced/webui/src/pages/Queue.tsx index b05f9374..04fca8e6 100644 --- a/backends/advanced/webui/src/pages/Queue.tsx +++ b/backends/advanced/webui/src/pages/Queue.tsx @@ -109,6 +109,27 @@ interface StreamingStatus { timestamp: number; } +interface EventRecord { + timestamp: number; + event: string; + user_id: string; + plugins_subscribed: string[]; + plugins_executed: Array<{ plugin_id: string; success: boolean; message: string }>; + metadata: Record; +} + +// Known event type colors — unknown types fall back to gray via getEventColor() +const EVENT_TYPE_COLORS: Record = { + 'conversation.complete': 'bg-green-100 text-green-700', + 'transcript.batch': 'bg-blue-100 text-blue-700', + 'memory.processed': 'bg-purple-100 text-purple-700', + 'button.single_press': 'bg-orange-100 text-orange-700', + 'button.double_press': 'bg-orange-100 text-orange-700', + 'plugin_action': 'bg-indigo-100 text-indigo-700', +}; +const DEFAULT_EVENT_COLOR = 'bg-gray-100 text-gray-700'; +const getEventColor = (eventType: string) => EVENT_TYPE_COLORS[eventType] || DEFAULT_EVENT_COLOR; + const Queue: React.FC = () => { const [jobs, setJobs] = useState([]); const [stats, setStats] = useState(null); @@ -147,6 +168,26 @@ const Queue: React.FC = () => { return saved !== null ? saved === 'true' : true; }); + // System events + const [events, setEvents] = useState([]); + const [eventFilters, setEventFilters] = useState>({}); + + const cycleEventFilter = (eventType: string) => { + setEventFilters(prev => { + const current = prev[eventType]; + const next = { ...prev }; + if (!current) { + next[eventType] = 'include'; + } else if (current === 'include') { + next[eventType] = 'exclude'; + } else { + delete next[eventType]; + } + return next; + }); + }; + const [eventsExpanded, setEventsExpanded] = useState(true); + // Completed conversations pagination const [completedConvPage, setCompletedConvPage] = useState(1); const [completedConvItemsPerPage] = useState(10); @@ -255,6 +296,7 @@ const Queue: React.FC = () => { setConversationJobs(jobsByConversation); setStats(dashboardData.stats); setStreamingStatus(dashboardData.streaming_status); + setEvents(dashboardData.events || []); setLastUpdate(Date.now()); // Auto-expand active conversations (those with open_conversation_job in progress) @@ -687,7 +729,7 @@ const Queue: React.FC = () => {

-

Queue Management

+

Queue & Events

Last updated: {new Date(lastUpdate).toLocaleTimeString()} • Auto-refresh every 2s

@@ -2021,6 +2063,146 @@ const Queue: React.FC = () => {
)} + {/* Events */} +
+
setEventsExpanded(!eventsExpanded)} + > +
+ +

Events

+ + {(() => { + const includes = Object.entries(eventFilters).filter(([, v]) => v === 'include').map(([k]) => k); + const excludes = Object.entries(eventFilters).filter(([, v]) => v === 'exclude').map(([k]) => k); + const hasFilters = includes.length > 0 || excludes.length > 0; + if (!hasFilters) return `(${events.length})`; + const count = includes.length > 0 + ? events.filter(e => includes.includes(e.event) && !excludes.includes(e.event)).length + : events.filter(e => !excludes.includes(e.event)).length; + return `(${count} / ${events.length})`; + })()} + +
+
+ {eventsExpanded ? : } +
+
+ + {eventsExpanded && [...new Set(events.map(e => e.event))].sort().length > 0 && ( +
+ {[...new Set(events.map(e => e.event))].sort().map(eventType => { + const state = eventFilters[eventType]; + return ( + + ); + })} + {Object.keys(eventFilters).length > 0 && ( + + )} +
+ )} + + {eventsExpanded && ( +
+ {(() => { + const includes = Object.entries(eventFilters).filter(([, v]) => v === 'include').map(([k]) => k); + const excludes = Object.entries(eventFilters).filter(([, v]) => v === 'exclude').map(([k]) => k); + let filtered = events; + if (includes.length > 0) { + filtered = filtered.filter(e => includes.includes(e.event)); + } + filtered = filtered.filter(e => !excludes.includes(e.event)); + + if (filtered.length === 0) { + return ( +
+ No events recorded yet. Events are logged when system actions like conversation.complete, memory.processed, or button presses occur. +
+ ); + } + + return ( + + + + + + + + + + + + {filtered.map((evt, idx) => { + const allSuccess = evt.plugins_executed.length > 0 && evt.plugins_executed.every(p => p.success); + const anyFailure = evt.plugins_executed.some(p => !p.success); + + return ( + + + + + + + + ); + })} + +
TimeEventUserPlugins TriggeredStatus
+ {new Date(evt.timestamp * 1000).toLocaleTimeString()} + + + {evt.event} + + + {evt.user_id.length > 12 ? `${evt.user_id.slice(-8)}` : evt.user_id} + + {evt.plugins_executed.length > 0 + ? evt.plugins_executed.map(p => p.plugin_id).join(', ') + : none + } + + {evt.plugins_executed.length === 0 ? ( + no plugins ran + ) : allSuccess ? ( + + + OK + + ) : anyFailure ? ( + !p.success).map(p => `${p.plugin_id}: ${p.message}`).join('; ')}> + + Error + + ) : ( + partial + )} +
+ ); + })()} +
+ )} +
+ {/* Filters */}

Filters

diff --git a/backends/advanced/webui/src/pages/Upload.tsx b/backends/advanced/webui/src/pages/Upload.tsx index 6c22f4e7..4d455af9 100644 --- a/backends/advanced/webui/src/pages/Upload.tsx +++ b/backends/advanced/webui/src/pages/Upload.tsx @@ -3,6 +3,9 @@ import { Upload as UploadIcon, File, X, CheckCircle, AlertCircle, RefreshCw, Fol import { uploadApi, obsidianApi } from '../services/api' import { useAuth } from '../contexts/AuthContext' +const SUPPORTED_EXTENSIONS = ['.wav', '.mp3', '.m4a', '.flac', '.ogg', '.mp4', '.webm'] +const VIDEO_EXTENSIONS = ['.mp4', '.webm'] + interface UploadFile { file: File id: string @@ -16,6 +19,7 @@ export default function Upload() { const [dragActive, setDragActive] = useState(false) const [uploadProgress, setUploadProgress] = useState(0) const [gdriveFolderId, setGdriveFolderId] = useState('') + const [videoWarning, setVideoWarning] = useState(false) const { isAdmin } = useAuth() @@ -70,16 +74,22 @@ export default function Upload() { const handleFileSelect = (selectedFiles: FileList | null) => { if (!selectedFiles) return - const audioFiles = Array.from(selectedFiles).filter( - (file) => + const acceptedFiles = Array.from(selectedFiles).filter((file) => { + const ext = '.' + file.name.split('.').pop()?.toLowerCase() + return ( file.type.startsWith('audio/') || - file.name.toLowerCase().endsWith('.wav') || - file.name.toLowerCase().endsWith('.mp3') || - file.name.toLowerCase().endsWith('.m4a') || - file.name.toLowerCase().endsWith('.flac') - ) + file.type.startsWith('video/') || + SUPPORTED_EXTENSIONS.includes(ext) + ) + }) + + const hasVideo = acceptedFiles.some((file) => { + const ext = '.' + file.name.split('.').pop()?.toLowerCase() + return file.type.startsWith('video/') || VIDEO_EXTENSIONS.includes(ext) + }) + if (hasVideo) setVideoWarning(true) - const newFiles: UploadFile[] = audioFiles.map((file) => ({ + const newFiles: UploadFile[] = acceptedFiles.map((file) => ({ file, id: generateId(), status: 'pending', @@ -211,7 +221,9 @@ export default function Upload() { }, [obsidianPolling, obsidianJobId]) const clearCompleted = () => { - setFiles(files.filter((f) => f.status === 'pending' || f.status === 'uploading')) + const remaining = files.filter((f) => f.status === 'pending' || f.status === 'uploading') + setFiles(remaining) + if (remaining.length === 0) setVideoWarning(false) } const formatFileSize = (bytes: number) => { @@ -313,13 +325,13 @@ export default function Upload() { Drop audio files here or click to browse

- Supported formats: WAV, MP3, M4A, FLAC + Supported formats: WAV, MP3, M4A, FLAC, OGG, MP4, WebM

handleFileSelect(e.target.files)} className="absolute inset-0 w-full h-full opacity-0 cursor-pointer" /> @@ -332,6 +344,16 @@ export default function Upload() {
+ {/* Video Warning */} + {videoWarning && ( +
+ +

+ Video files detected — only the audio track will be extracted. +

+
+ )} + {/* File List */} {files.length > 0 && (
@@ -442,7 +464,7 @@ export default function Upload() {
  • • Processing time varies based on audio length (roughly 3× duration + 60s)
  • • Large files or multiple files may cause timeout errors
  • • Check the Conversations tab for processed results
    • -
  • • Supported formats: WAV, MP3, M4A, FLAC
    • +
  • • Supported formats: WAV, MP3, M4A, FLAC, OGG, MP4, WebM
    • diff --git a/backends/advanced/webui/src/services/api.ts b/backends/advanced/webui/src/services/api.ts index e349dc91..3f3781f2 100644 --- a/backends/advanced/webui/src/services/api.ts +++ b/backends/advanced/webui/src/services/api.ts @@ -107,8 +107,13 @@ export const authApi = { } export const conversationsApi = { - getAll: (includeDeleted?: boolean) => api.get('/api/conversations', { - params: includeDeleted !== undefined ? { include_deleted: includeDeleted } : {} + getAll: (includeDeleted?: boolean, includeUnprocessed?: boolean, limit?: number, offset?: number) => api.get('/api/conversations', { + params: { + ...(includeDeleted !== undefined && { include_deleted: includeDeleted }), + ...(includeUnprocessed !== undefined && { include_unprocessed: includeUnprocessed }), + ...(limit !== undefined && { limit }), + ...(offset !== undefined && { offset }), + } }), getById: (id: string) => api.get(`/api/conversations/${id}`), delete: (id: string) => api.delete(`/api/conversations/${id}`), @@ -118,6 +123,7 @@ export const conversationsApi = { }), // Reprocessing endpoints + reprocessOrphan: (conversationId: string) => api.post(`/api/conversations/${conversationId}/reprocess-orphan`), reprocessTranscript: (conversationId: string) => api.post(`/api/conversations/${conversationId}/reprocess-transcript`), reprocessMemory: (conversationId: string, transcriptVersionId: string = 'active') => api.post(`/api/conversations/${conversationId}/reprocess-memory`, null, { params: { transcript_version_id: transcriptVersionId } @@ -335,6 +341,11 @@ export const queueApi = { return api.post(endpoint, body) }, + // Plugin events + getEvents: (limit: number = 50, eventType?: string) => api.get('/api/queue/events', { + params: { limit, ...(eventType && { event_type: eventType }) } + }), + // Legacy endpoints - kept for backward compatibility but not used in Queue page // getJobs: (params: URLSearchParams) => api.get(`/api/queue/jobs?${params}`), // getJobsBySession: (sessionId: string) => api.get(`/api/queue/jobs/by-session/${sessionId}`), diff --git a/config/plugins.yml.template b/config/plugins.yml.template index cc9134ca..789cd9ed 100644 --- a/config/plugins.yml.template +++ b/config/plugins.yml.template @@ -76,3 +76,21 @@ plugins: summary_max_sentences: 3 # LLM summary length include_conversation_id: true include_duration: true + + # ======================================== + # Test Button Actions Plugin + # ======================================== + # Maps OMI device button presses to system actions. + # Single press: close current conversation (triggers post-processing) + # Double press: cross-plugin call (e.g., toggle lights via Home Assistant) + # + # Actions are configured in: plugins/test_button_actions/config.yml + # See: backends/advanced/Docs/plugin-development-guide.md for button event data flow + + test_button_actions: + enabled: false # Set to true to enable button actions + events: + - button.single_press # OMI device single tap + - button.double_press # OMI device double tap + condition: + type: always # Always execute on button events diff --git a/extras/asr-services/Dockerfile_Moonshine b/extras/asr-services/Dockerfile_Moonshine deleted file mode 100644 index 0fbfa861..00000000 --- a/extras/asr-services/Dockerfile_Moonshine +++ /dev/null @@ -1,32 +0,0 @@ -# syntax=docker/dockerfile:1 - -######################### builder ################################# -FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS builder -WORKDIR /app - -# NeMo and texterrors need libs and C++ compiler -RUN apt-get update && apt-get install -y --no-install-recommends \ - libsndfile1 \ - build-essential git \ - && rm -rf /var/lib/apt/lists/* - -# Dependency manifest first for cache‑friendly installs -COPY pyproject.toml uv.lock ./ -RUN uv sync --no-install-project --compile-bytecode --group moonshine && \ - uv cache clean - -# Add source and install project itself -COPY . . - -######################### runtime ################################# -FROM python:3.12-slim-bookworm AS runtime -ENV PYTHONUNBUFFERED=1 -WORKDIR /app - -RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1 && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /app /app -ENV PATH="/app/.venv/bin:$PATH" - -EXPOSE 8765 -CMD ["python", "moonshine-online.py", "--port", "8765"] diff --git a/extras/asr-services/charts/moonshine/Chart.yaml b/extras/asr-services/charts/moonshine/Chart.yaml deleted file mode 100644 index 24b53132..00000000 --- a/extras/asr-services/charts/moonshine/Chart.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: parakeet -description: A generated Helm Chart for docker-compose from Skippbox Kompose -version: 0.0.1 -apiVersion: v2 -keywords: - - docker-compose -sources: -home: diff --git a/extras/asr-services/charts/moonshine/templates/moonshine-asr-claim0-persistentvolumeclaim.yaml b/extras/asr-services/charts/moonshine/templates/moonshine-asr-claim0-persistentvolumeclaim.yaml deleted file mode 100644 index 090d2f46..00000000 --- a/extras/asr-services/charts/moonshine/templates/moonshine-asr-claim0-persistentvolumeclaim.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - io.kompose.service: moonshine-claim0 - name: moonshine-claim0 -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi diff --git a/extras/asr-services/charts/moonshine/templates/moonshine-asr-deployment.yaml b/extras/asr-services/charts/moonshine/templates/moonshine-asr-deployment.yaml deleted file mode 100644 index 3fb5916a..00000000 --- a/extras/asr-services/charts/moonshine/templates/moonshine-asr-deployment.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - kompose.cmd: kompose convert -c - kompose.version: 1.36.0 (HEAD) - labels: - io.kompose.service: moonshine-asr - name: moonshine-asr -spec: - replicas: 1 - selector: - matchLabels: - io.kompose.service: moonshine-asr - strategy: - type: Recreate - template: - metadata: - annotations: - kompose.cmd: kompose convert -c - kompose.version: 1.36.0 (HEAD) - labels: - io.kompose.service: moonshine-asr - spec: - containers: - - env: - - name: HF_HOME - value: /models - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - name: moonshine-asr - ports: - - containerPort: 8765 - protocol: TCP - volumeMounts: - - mountPath: /models - name: moonshine-claim0 - restartPolicy: Always - volumes: - - name: moonshine-claim0 - persistentVolumeClaim: - claimName: moonshine-claim0 - tolerations: - - key: "gpu" - operator: "Equal" - value: "only" - effect: "NoSchedule" - nodeSelector: - nvidia.com/gpu.present: "true" diff --git a/extras/asr-services/charts/moonshine/templates/moonshine-asr-service.yaml b/extras/asr-services/charts/moonshine/templates/moonshine-asr-service.yaml deleted file mode 100644 index 1fdb0739..00000000 --- a/extras/asr-services/charts/moonshine/templates/moonshine-asr-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - kompose.cmd: kompose convert -c - kompose.version: 1.36.0 (HEAD) - labels: - io.kompose.service: moonshine-asr - name: moonshine-asr -spec: - ports: - - name: "8765" - port: 8765 - targetPort: 8765 - selector: - io.kompose.service: moonshine-asr diff --git a/extras/asr-services/charts/moonshine/values.yaml b/extras/asr-services/charts/moonshine/values.yaml deleted file mode 100644 index a8568020..00000000 --- a/extras/asr-services/charts/moonshine/values.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Default values for moonshine-asr -image: - repository: moonshine-asr - tag: latest - pullPolicy: IfNotPresent - -service: - type: ClusterIP - port: 8765 - -resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: - nvidia.com/gpu.present: "true" - -tolerations: - - key: "gpu" - operator: "Equal" - value: "only" - effect: "NoSchedule" - -affinity: {} - -persistence: - enabled: true - size: 10Gi - storageClass: "" - accessMode: ReadWriteOnce diff --git a/extras/asr-services/pyproject.toml b/extras/asr-services/pyproject.toml index 334a8b43..5c468016 100644 --- a/extras/asr-services/pyproject.toml +++ b/extras/asr-services/pyproject.toml @@ -60,7 +60,6 @@ conflicts = [ ] [tool.uv.sources] -useful-moonshine-onnx = { git = "https://github.com/usefulsensors/moonshine.git", subdirectory = "moonshine-onnx" } torch = [ { index = "pytorch-cu121", extra = "cu121" }, { index = "pytorch-cu126", extra = "cu126" }, @@ -125,10 +124,6 @@ parakeet = [ "numpy>=1.26,<2.0", ] -moonshine = [ - "useful-moonshine-onnx", -] - # Development dependencies dev = [ "black>=25.1.0", diff --git a/extras/asr-services/uv.lock b/extras/asr-services/uv.lock index 93a8edd4..2bc09f7f 100644 --- a/extras/asr-services/uv.lock +++ b/extras/asr-services/uv.lock @@ -491,9 +491,6 @@ faster-whisper = [ { name = "ctranslate2" }, { name = "faster-whisper" }, ] -moonshine = [ - { name = "useful-moonshine-onnx" }, -] nemo = [ { name = "cuda-python" }, { name = "nemo-toolkit", version = "2.6.1", source = { registry = "https://pypi.org/simple" }, extra = ["asr"], marker = "(extra == 'extra-12-asr-services-cu121' and extra == 'extra-12-asr-services-cu126') or (extra == 'extra-12-asr-services-cu121' and extra == 'extra-12-asr-services-cu128') or (extra == 'extra-12-asr-services-cu126' and extra == 'extra-12-asr-services-cu128') or (extra == 'extra-12-asr-services-cu121' and extra != 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-nemo') or (extra != 'extra-12-asr-services-cu128' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-nemo') or (extra != 'extra-12-asr-services-cu128' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-transformers') or (extra != 'extra-12-asr-services-cu128' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-vibevoice') or (extra != 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-transformers') or (extra != 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-vibevoice') or (extra != 'extra-12-asr-services-cu128' and extra != 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-transformers' and extra == 'group-12-asr-services-vibevoice') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-nemo') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-transformers') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-vibevoice') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra != 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-transformers' and extra == 'group-12-asr-services-vibevoice')" }, @@ -525,7 +522,6 @@ vibevoice = [ { name = "accelerate" }, { name = "bitsandbytes" }, { name = "diffusers" }, - { name = "langfuse" }, { name = "librosa" }, { name = "torch", version = "2.5.1+cu121", source = { registry = "https://download.pytorch.org/whl/cu121" }, marker = "(extra == 'extra-12-asr-services-cu121' and extra == 'extra-12-asr-services-cu126') or (extra == 'extra-12-asr-services-cu121' and extra == 'extra-12-asr-services-cu128') or (extra == 'extra-12-asr-services-cu126' and extra == 'extra-12-asr-services-cu128') or (extra != 'extra-12-asr-services-cu128' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-nemo') or (extra != 'extra-12-asr-services-cu128' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-transformers') or (extra != 'extra-12-asr-services-cu128' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-vibevoice') or (extra != 'extra-12-asr-services-cu128' and extra == 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-transformers') or (extra != 'extra-12-asr-services-cu128' and extra == 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-vibevoice') or (extra == 'extra-12-asr-services-cu121' and extra != 'group-12-asr-services-faster-whisper' and extra != 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-vibevoice') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-nemo') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-transformers') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-vibevoice') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra == 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-transformers') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra == 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-vibevoice') or (extra != 'group-12-asr-services-faster-whisper' and extra != 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-transformers' and extra == 'group-12-asr-services-vibevoice')" }, { name = "torch", version = "2.7.1", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-12-asr-services-cu121' and extra == 'extra-12-asr-services-cu126') or (extra == 'extra-12-asr-services-cu121' and extra == 'extra-12-asr-services-cu128') or (extra == 'extra-12-asr-services-cu126' and extra == 'extra-12-asr-services-cu128') or (extra == 'group-12-asr-services-transformers' and extra == 'group-12-asr-services-vibevoice') or (extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-nemo') or (extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-transformers') or (extra == 'group-12-asr-services-faster-whisper' and extra == 'group-12-asr-services-vibevoice') or (extra == 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-transformers') or (extra == 'group-12-asr-services-nemo' and extra == 'group-12-asr-services-vibevoice') or (extra != 'extra-12-asr-services-cu121' and extra != 'extra-12-asr-services-cu126' and extra != 'extra-12-asr-services-cu128' and extra == 'group-12-asr-services-vibevoice')" }, @@ -578,7 +574,6 @@ faster-whisper = [ { name = "ctranslate2", specifier = ">=4.0.0" }, { name = "faster-whisper", specifier = ">=1.0.0" }, ] -moonshine = [{ name = "useful-moonshine-onnx", git = "https://github.com/usefulsensors/moonshine.git?subdirectory=moonshine-onnx" }] nemo = [ { name = "cuda-python", specifier = ">=12.3" }, { name = "nemo-toolkit", extras = ["asr"], specifier = ">=2.2.0" }, @@ -600,7 +595,6 @@ vibevoice = [ { name = "accelerate", specifier = ">=0.30.0" }, { name = "bitsandbytes", specifier = ">=0.43.0" }, { name = "diffusers", specifier = ">=0.30.0" }, - { name = "langfuse", specifier = ">=3.13.0,<4.0" }, { name = "librosa", specifier = ">=0.10.0" }, { name = "torch", specifier = ">=2.3" }, { name = "torchaudio", specifier = ">=2.3" }, @@ -689,15 +683,6 @@ version = "14.4.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/86/f6/0b473dab52dfdea05f28f3578b1c56b6c796ce85e76951bab7c4e38d5a74/av-14.4.0.tar.gz", hash = "sha256:3ecbf803a7fdf67229c0edada0830d6bfaea4d10bfb24f0c3f4e607cd1064b42", size = 3892203, upload-time = "2025-05-16T19:13:35.737Z" } -[[package]] -name = "backoff" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, -] - [[package]] name = "backports-datetime-fromisoformat" version = "2.0.3" @@ -1304,15 +1289,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, ] -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - [[package]] name = "dnspython" version = "2.7.0" @@ -1745,18 +1721,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/3c/2a19a60a473de48717b4efb19398c3f914795b64a96cf3fbe82588044f78/google_crc32c-1.7.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6efb97eb4369d52593ad6f75e7e10d053cf00c48983f7a973105bc70b0ac4d82", size = 28048, upload-time = "2025-03-26T14:41:46.696Z" }, ] -[[package]] -name = "googleapis-common-protos" -version = "1.72.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e5/7b/adfd75544c415c487b33061fe7ae526165241c1ea133f9a9125a56b39fd8/googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5", size = 147433, upload-time = "2025-11-06T18:29:24.087Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/ab/09169d5a4612a5f92490806649ac8d41e3ec9129c636754575b3553f4ea4/googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038", size = 297515, upload-time = "2025-11-06T18:29:13.14Z" }, -] - [[package]] name = "gradio" version = "5.33.0" @@ -2382,103 +2346,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] -[[package]] -name = "jiter" -version = "0.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0d/5e/4ec91646aee381d01cdb9974e30882c9cd3b8c5d1079d6b5ff4af522439a/jiter-0.13.0.tar.gz", hash = "sha256:f2839f9c2c7e2dffc1bc5929a510e14ce0a946be9365fd1219e7ef342dae14f4", size = 164847, upload-time = "2026-02-02T12:37:56.441Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/5a/41da76c5ea07bec1b0472b6b2fdb1b651074d504b19374d7e130e0cdfb25/jiter-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2ffc63785fd6c7977defe49b9824ae6ce2b2e2b77ce539bdaf006c26da06342e", size = 311164, upload-time = "2026-02-02T12:35:17.688Z" }, - { url = "https://files.pythonhosted.org/packages/40/cb/4a1bf994a3e869f0d39d10e11efb471b76d0ad70ecbfb591427a46c880c2/jiter-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4a638816427006c1e3f0013eb66d391d7a3acda99a7b0cf091eff4497ccea33a", size = 320296, upload-time = "2026-02-02T12:35:19.828Z" }, - { url = "https://files.pythonhosted.org/packages/09/82/acd71ca9b50ecebadc3979c541cd717cce2fe2bc86236f4fa597565d8f1a/jiter-0.13.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19928b5d1ce0ff8c1ee1b9bdef3b5bfc19e8304f1b904e436caf30bc15dc6cf5", size = 352742, upload-time = "2026-02-02T12:35:21.258Z" }, - { url = "https://files.pythonhosted.org/packages/71/03/d1fc996f3aecfd42eb70922edecfb6dd26421c874503e241153ad41df94f/jiter-0.13.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:309549b778b949d731a2f0e1594a3f805716be704a73bf3ad9a807eed5eb5721", size = 363145, upload-time = "2026-02-02T12:35:24.653Z" }, - { url = "https://files.pythonhosted.org/packages/f1/61/a30492366378cc7a93088858f8991acd7d959759fe6138c12a4644e58e81/jiter-0.13.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bcdabaea26cb04e25df3103ce47f97466627999260290349a88c8136ecae0060", size = 487683, upload-time = "2026-02-02T12:35:26.162Z" }, - { url = "https://files.pythonhosted.org/packages/20/4e/4223cffa9dbbbc96ed821c5aeb6bca510848c72c02086d1ed3f1da3d58a7/jiter-0.13.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a3a377af27b236abbf665a69b2bdd680e3b5a0bd2af825cd3b81245279a7606c", size = 373579, upload-time = "2026-02-02T12:35:27.582Z" }, - { url = "https://files.pythonhosted.org/packages/fe/c9/b0489a01329ab07a83812d9ebcffe7820a38163c6d9e7da644f926ff877c/jiter-0.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe49d3ff6db74321f144dff9addd4a5874d3105ac5ba7c5b77fac099cfae31ae", size = 362904, upload-time = "2026-02-02T12:35:28.925Z" }, - { url = "https://files.pythonhosted.org/packages/05/af/53e561352a44afcba9a9bc67ee1d320b05a370aed8df54eafe714c4e454d/jiter-0.13.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2113c17c9a67071b0f820733c0893ed1d467b5fcf4414068169e5c2cabddb1e2", size = 392380, upload-time = "2026-02-02T12:35:30.385Z" }, - { url = "https://files.pythonhosted.org/packages/76/2a/dd805c3afb8ed5b326c5ae49e725d1b1255b9754b1b77dbecdc621b20773/jiter-0.13.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ab1185ca5c8b9491b55ebf6c1e8866b8f68258612899693e24a92c5fdb9455d5", size = 517939, upload-time = "2026-02-02T12:35:31.865Z" }, - { url = "https://files.pythonhosted.org/packages/20/2a/7b67d76f55b8fe14c937e7640389612f05f9a4145fc28ae128aaa5e62257/jiter-0.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9621ca242547edc16400981ca3231e0c91c0c4c1ab8573a596cd9bb3575d5c2b", size = 551696, upload-time = "2026-02-02T12:35:33.306Z" }, - { url = "https://files.pythonhosted.org/packages/85/9c/57cdd64dac8f4c6ab8f994fe0eb04dc9fd1db102856a4458fcf8a99dfa62/jiter-0.13.0-cp310-cp310-win32.whl", hash = "sha256:a7637d92b1c9d7a771e8c56f445c7f84396d48f2e756e5978840ecba2fac0894", size = 204592, upload-time = "2026-02-02T12:35:34.58Z" }, - { url = "https://files.pythonhosted.org/packages/a7/38/f4f3ea5788b8a5bae7510a678cdc747eda0c45ffe534f9878ff37e7cf3b3/jiter-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c1b609e5cbd2f52bb74fb721515745b407df26d7b800458bd97cb3b972c29e7d", size = 206016, upload-time = "2026-02-02T12:35:36.435Z" }, - { url = "https://files.pythonhosted.org/packages/71/29/499f8c9eaa8a16751b1c0e45e6f5f1761d180da873d417996cc7bddc8eef/jiter-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ea026e70a9a28ebbdddcbcf0f1323128a8db66898a06eaad3a4e62d2f554d096", size = 311157, upload-time = "2026-02-02T12:35:37.758Z" }, - { url = "https://files.pythonhosted.org/packages/50/f6/566364c777d2ab450b92100bea11333c64c38d32caf8dc378b48e5b20c46/jiter-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66aa3e663840152d18cc8ff1e4faad3dd181373491b9cfdc6004b92198d67911", size = 319729, upload-time = "2026-02-02T12:35:39.246Z" }, - { url = "https://files.pythonhosted.org/packages/73/dd/560f13ec5e4f116d8ad2658781646cca91b617ae3b8758d4a5076b278f70/jiter-0.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3524798e70655ff19aec58c7d05adb1f074fecff62da857ea9be2b908b6d701", size = 354766, upload-time = "2026-02-02T12:35:40.662Z" }, - { url = "https://files.pythonhosted.org/packages/7c/0d/061faffcfe94608cbc28a0d42a77a74222bdf5055ccdbe5fd2292b94f510/jiter-0.13.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ec7e287d7fbd02cb6e22f9a00dd9c9cd504c40a61f2c61e7e1f9690a82726b4c", size = 362587, upload-time = "2026-02-02T12:35:42.025Z" }, - { url = "https://files.pythonhosted.org/packages/92/c9/c66a7864982fd38a9773ec6e932e0398d1262677b8c60faecd02ffb67bf3/jiter-0.13.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:47455245307e4debf2ce6c6e65a717550a0244231240dcf3b8f7d64e4c2f22f4", size = 487537, upload-time = "2026-02-02T12:35:43.459Z" }, - { url = "https://files.pythonhosted.org/packages/6c/86/84eb4352cd3668f16d1a88929b5888a3fe0418ea8c1dfc2ad4e7bf6e069a/jiter-0.13.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ee9da221dca6e0429c2704c1b3655fe7b025204a71d4d9b73390c759d776d165", size = 373717, upload-time = "2026-02-02T12:35:44.928Z" }, - { url = "https://files.pythonhosted.org/packages/6e/09/9fe4c159358176f82d4390407a03f506a8659ed13ca3ac93a843402acecf/jiter-0.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24ab43126d5e05f3d53a36a8e11eb2f23304c6c1117844aaaf9a0aa5e40b5018", size = 362683, upload-time = "2026-02-02T12:35:46.636Z" }, - { url = "https://files.pythonhosted.org/packages/c9/5e/85f3ab9caca0c1d0897937d378b4a515cae9e119730563572361ea0c48ae/jiter-0.13.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9da38b4fedde4fb528c740c2564628fbab737166a0e73d6d46cb4bb5463ff411", size = 392345, upload-time = "2026-02-02T12:35:48.088Z" }, - { url = "https://files.pythonhosted.org/packages/12/4c/05b8629ad546191939e6f0c2f17e29f542a398f4a52fb987bc70b6d1eb8b/jiter-0.13.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0b34c519e17658ed88d5047999a93547f8889f3c1824120c26ad6be5f27b6cf5", size = 517775, upload-time = "2026-02-02T12:35:49.482Z" }, - { url = "https://files.pythonhosted.org/packages/4d/88/367ea2eb6bc582c7052e4baf5ddf57ebe5ab924a88e0e09830dfb585c02d/jiter-0.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2a6394e6af690d462310a86b53c47ad75ac8c21dc79f120714ea449979cb1d3", size = 551325, upload-time = "2026-02-02T12:35:51.104Z" }, - { url = "https://files.pythonhosted.org/packages/f3/12/fa377ffb94a2f28c41afaed093e0d70cfe512035d5ecb0cad0ae4792d35e/jiter-0.13.0-cp311-cp311-win32.whl", hash = "sha256:0f0c065695f616a27c920a56ad0d4fc46415ef8b806bf8fc1cacf25002bd24e1", size = 204709, upload-time = "2026-02-02T12:35:52.467Z" }, - { url = "https://files.pythonhosted.org/packages/cb/16/8e8203ce92f844dfcd3d9d6a5a7322c77077248dbb12da52d23193a839cd/jiter-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:0733312953b909688ae3c2d58d043aa040f9f1a6a75693defed7bc2cc4bf2654", size = 204560, upload-time = "2026-02-02T12:35:53.925Z" }, - { url = "https://files.pythonhosted.org/packages/44/26/97cc40663deb17b9e13c3a5cf29251788c271b18ee4d262c8f94798b8336/jiter-0.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:5d9b34ad56761b3bf0fbe8f7e55468704107608512350962d3317ffd7a4382d5", size = 189608, upload-time = "2026-02-02T12:35:55.304Z" }, - { url = "https://files.pythonhosted.org/packages/2e/30/7687e4f87086829955013ca12a9233523349767f69653ebc27036313def9/jiter-0.13.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0a2bd69fc1d902e89925fc34d1da51b2128019423d7b339a45d9e99c894e0663", size = 307958, upload-time = "2026-02-02T12:35:57.165Z" }, - { url = "https://files.pythonhosted.org/packages/c3/27/e57f9a783246ed95481e6749cc5002a8a767a73177a83c63ea71f0528b90/jiter-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f917a04240ef31898182f76a332f508f2cc4b57d2b4d7ad2dbfebbfe167eb505", size = 318597, upload-time = "2026-02-02T12:35:58.591Z" }, - { url = "https://files.pythonhosted.org/packages/cf/52/e5719a60ac5d4d7c5995461a94ad5ef962a37c8bf5b088390e6fad59b2ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1e2b199f446d3e82246b4fd9236d7cb502dc2222b18698ba0d986d2fecc6152", size = 348821, upload-time = "2026-02-02T12:36:00.093Z" }, - { url = "https://files.pythonhosted.org/packages/61/db/c1efc32b8ba4c740ab3fc2d037d8753f67685f475e26b9d6536a4322bcdd/jiter-0.13.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04670992b576fa65bd056dbac0c39fe8bd67681c380cb2b48efa885711d9d726", size = 364163, upload-time = "2026-02-02T12:36:01.937Z" }, - { url = "https://files.pythonhosted.org/packages/55/8a/fb75556236047c8806995671a18e4a0ad646ed255276f51a20f32dceaeec/jiter-0.13.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a1aff1fbdb803a376d4d22a8f63f8e7ccbce0b4890c26cc7af9e501ab339ef0", size = 483709, upload-time = "2026-02-02T12:36:03.41Z" }, - { url = "https://files.pythonhosted.org/packages/7e/16/43512e6ee863875693a8e6f6d532e19d650779d6ba9a81593ae40a9088ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b3fb8c2053acaef8580809ac1d1f7481a0a0bdc012fd7f5d8b18fb696a5a089", size = 370480, upload-time = "2026-02-02T12:36:04.791Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4c/09b93e30e984a187bc8aaa3510e1ec8dcbdcd71ca05d2f56aac0492453aa/jiter-0.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdaba7d87e66f26a2c45d8cbadcbfc4bf7884182317907baf39cfe9775bb4d93", size = 360735, upload-time = "2026-02-02T12:36:06.994Z" }, - { url = "https://files.pythonhosted.org/packages/1a/1b/46c5e349019874ec5dfa508c14c37e29864ea108d376ae26d90bee238cd7/jiter-0.13.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7b88d649135aca526da172e48083da915ec086b54e8e73a425ba50999468cc08", size = 391814, upload-time = "2026-02-02T12:36:08.368Z" }, - { url = "https://files.pythonhosted.org/packages/15/9e/26184760e85baee7162ad37b7912797d2077718476bf91517641c92b3639/jiter-0.13.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e404ea551d35438013c64b4f357b0474c7abf9f781c06d44fcaf7a14c69ff9e2", size = 513990, upload-time = "2026-02-02T12:36:09.993Z" }, - { url = "https://files.pythonhosted.org/packages/e9/34/2c9355247d6debad57a0a15e76ab1566ab799388042743656e566b3b7de1/jiter-0.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1f4748aad1b4a93c8bdd70f604d0f748cdc0e8744c5547798acfa52f10e79228", size = 548021, upload-time = "2026-02-02T12:36:11.376Z" }, - { url = "https://files.pythonhosted.org/packages/ac/4a/9f2c23255d04a834398b9c2e0e665382116911dc4d06b795710503cdad25/jiter-0.13.0-cp312-cp312-win32.whl", hash = "sha256:0bf670e3b1445fc4d31612199f1744f67f889ee1bbae703c4b54dc097e5dd394", size = 203024, upload-time = "2026-02-02T12:36:12.682Z" }, - { url = "https://files.pythonhosted.org/packages/09/ee/f0ae675a957ae5a8f160be3e87acea6b11dc7b89f6b7ab057e77b2d2b13a/jiter-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:15db60e121e11fe186c0b15236bd5d18381b9ddacdcf4e659feb96fc6c969c92", size = 205424, upload-time = "2026-02-02T12:36:13.93Z" }, - { url = "https://files.pythonhosted.org/packages/1b/02/ae611edf913d3cbf02c97cdb90374af2082c48d7190d74c1111dde08bcdd/jiter-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:41f92313d17989102f3cb5dd533a02787cdb99454d494344b0361355da52fcb9", size = 186818, upload-time = "2026-02-02T12:36:15.308Z" }, - { url = "https://files.pythonhosted.org/packages/91/9c/7ee5a6ff4b9991e1a45263bfc46731634c4a2bde27dfda6c8251df2d958c/jiter-0.13.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1f8a55b848cbabf97d861495cd65f1e5c590246fabca8b48e1747c4dfc8f85bf", size = 306897, upload-time = "2026-02-02T12:36:16.748Z" }, - { url = "https://files.pythonhosted.org/packages/7c/02/be5b870d1d2be5dd6a91bdfb90f248fbb7dcbd21338f092c6b89817c3dbf/jiter-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f556aa591c00f2c45eb1b89f68f52441a016034d18b65da60e2d2875bbbf344a", size = 317507, upload-time = "2026-02-02T12:36:18.351Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/b25d2ec333615f5f284f3a4024f7ce68cfa0604c322c6808b2344c7f5d2b/jiter-0.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7e1d61da332ec412350463891923f960c3073cf1aae93b538f0bb4c8cd46efb", size = 350560, upload-time = "2026-02-02T12:36:19.746Z" }, - { url = "https://files.pythonhosted.org/packages/be/ec/74dcb99fef0aca9fbe56b303bf79f6bd839010cb18ad41000bf6cc71eec0/jiter-0.13.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3097d665a27bc96fd9bbf7f86178037db139f319f785e4757ce7ccbf390db6c2", size = 363232, upload-time = "2026-02-02T12:36:21.243Z" }, - { url = "https://files.pythonhosted.org/packages/1b/37/f17375e0bb2f6a812d4dd92d7616e41917f740f3e71343627da9db2824ce/jiter-0.13.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d01ecc3a8cbdb6f25a37bd500510550b64ddf9f7d64a107d92f3ccb25035d0f", size = 483727, upload-time = "2026-02-02T12:36:22.688Z" }, - { url = "https://files.pythonhosted.org/packages/77/d2/a71160a5ae1a1e66c1395b37ef77da67513b0adba73b993a27fbe47eb048/jiter-0.13.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed9bbc30f5d60a3bdf63ae76beb3f9db280d7f195dfcfa61af792d6ce912d159", size = 370799, upload-time = "2026-02-02T12:36:24.106Z" }, - { url = "https://files.pythonhosted.org/packages/01/99/ed5e478ff0eb4e8aa5fd998f9d69603c9fd3f32de3bd16c2b1194f68361c/jiter-0.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98fbafb6e88256f4454de33c1f40203d09fc33ed19162a68b3b257b29ca7f663", size = 359120, upload-time = "2026-02-02T12:36:25.519Z" }, - { url = "https://files.pythonhosted.org/packages/16/be/7ffd08203277a813f732ba897352797fa9493faf8dc7995b31f3d9cb9488/jiter-0.13.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5467696f6b827f1116556cb0db620440380434591e93ecee7fd14d1a491b6daa", size = 390664, upload-time = "2026-02-02T12:36:26.866Z" }, - { url = "https://files.pythonhosted.org/packages/d1/84/e0787856196d6d346264d6dcccb01f741e5f0bd014c1d9a2ebe149caf4f3/jiter-0.13.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:2d08c9475d48b92892583df9da592a0e2ac49bcd41fae1fec4f39ba6cf107820", size = 513543, upload-time = "2026-02-02T12:36:28.217Z" }, - { url = "https://files.pythonhosted.org/packages/65/50/ecbd258181c4313cf79bca6c88fb63207d04d5bf5e4f65174114d072aa55/jiter-0.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:aed40e099404721d7fcaf5b89bd3b4568a4666358bcac7b6b15c09fb6252ab68", size = 547262, upload-time = "2026-02-02T12:36:29.678Z" }, - { url = "https://files.pythonhosted.org/packages/27/da/68f38d12e7111d2016cd198161b36e1f042bd115c169255bcb7ec823a3bf/jiter-0.13.0-cp313-cp313-win32.whl", hash = "sha256:36ebfbcffafb146d0e6ffb3e74d51e03d9c35ce7c625c8066cdbfc7b953bdc72", size = 200630, upload-time = "2026-02-02T12:36:31.808Z" }, - { url = "https://files.pythonhosted.org/packages/25/65/3bd1a972c9a08ecd22eb3b08a95d1941ebe6938aea620c246cf426ae09c2/jiter-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:8d76029f077379374cf0dbc78dbe45b38dec4a2eb78b08b5194ce836b2517afc", size = 202602, upload-time = "2026-02-02T12:36:33.679Z" }, - { url = "https://files.pythonhosted.org/packages/15/fe/13bd3678a311aa67686bb303654792c48206a112068f8b0b21426eb6851e/jiter-0.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:bb7613e1a427cfcb6ea4544f9ac566b93d5bf67e0d48c787eca673ff9c9dff2b", size = 185939, upload-time = "2026-02-02T12:36:35.065Z" }, - { url = "https://files.pythonhosted.org/packages/49/19/a929ec002ad3228bc97ca01dbb14f7632fffdc84a95ec92ceaf4145688ae/jiter-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fa476ab5dd49f3bf3a168e05f89358c75a17608dbabb080ef65f96b27c19ab10", size = 316616, upload-time = "2026-02-02T12:36:36.579Z" }, - { url = "https://files.pythonhosted.org/packages/52/56/d19a9a194afa37c1728831e5fb81b7722c3de18a3109e8f282bfc23e587a/jiter-0.13.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ade8cb6ff5632a62b7dbd4757d8c5573f7a2e9ae285d6b5b841707d8363205ef", size = 346850, upload-time = "2026-02-02T12:36:38.058Z" }, - { url = "https://files.pythonhosted.org/packages/36/4a/94e831c6bf287754a8a019cb966ed39ff8be6ab78cadecf08df3bb02d505/jiter-0.13.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9950290340acc1adaded363edd94baebcee7dabdfa8bee4790794cd5cfad2af6", size = 358551, upload-time = "2026-02-02T12:36:39.417Z" }, - { url = "https://files.pythonhosted.org/packages/a2/ec/a4c72c822695fa80e55d2b4142b73f0012035d9fcf90eccc56bc060db37c/jiter-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2b4972c6df33731aac0742b64fd0d18e0a69bc7d6e03108ce7d40c85fd9e3e6d", size = 201950, upload-time = "2026-02-02T12:36:40.791Z" }, - { url = "https://files.pythonhosted.org/packages/b6/00/393553ec27b824fbc29047e9c7cd4a3951d7fbe4a76743f17e44034fa4e4/jiter-0.13.0-cp313-cp313t-win_arm64.whl", hash = "sha256:701a1e77d1e593c1b435315ff625fd071f0998c5f02792038a5ca98899261b7d", size = 185852, upload-time = "2026-02-02T12:36:42.077Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f5/f1997e987211f6f9bd71b8083047b316208b4aca0b529bb5f8c96c89ef3e/jiter-0.13.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cc5223ab19fe25e2f0bf2643204ad7318896fe3729bf12fde41b77bfc4fafff0", size = 308804, upload-time = "2026-02-02T12:36:43.496Z" }, - { url = "https://files.pythonhosted.org/packages/cd/8f/5482a7677731fd44881f0204981ce2d7175db271f82cba2085dd2212e095/jiter-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9776ebe51713acf438fd9b4405fcd86893ae5d03487546dae7f34993217f8a91", size = 318787, upload-time = "2026-02-02T12:36:45.071Z" }, - { url = "https://files.pythonhosted.org/packages/f3/b9/7257ac59778f1cd025b26a23c5520a36a424f7f1b068f2442a5b499b7464/jiter-0.13.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:879e768938e7b49b5e90b7e3fecc0dbec01b8cb89595861fb39a8967c5220d09", size = 353880, upload-time = "2026-02-02T12:36:47.365Z" }, - { url = "https://files.pythonhosted.org/packages/c3/87/719eec4a3f0841dad99e3d3604ee4cba36af4419a76f3cb0b8e2e691ad67/jiter-0.13.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:682161a67adea11e3aae9038c06c8b4a9a71023228767477d683f69903ebc607", size = 366702, upload-time = "2026-02-02T12:36:48.871Z" }, - { url = "https://files.pythonhosted.org/packages/d2/65/415f0a75cf6921e43365a1bc227c565cb949caca8b7532776e430cbaa530/jiter-0.13.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a13b68cd1cd8cc9de8f244ebae18ccb3e4067ad205220ef324c39181e23bbf66", size = 486319, upload-time = "2026-02-02T12:36:53.006Z" }, - { url = "https://files.pythonhosted.org/packages/54/a2/9e12b48e82c6bbc6081fd81abf915e1443add1b13d8fc586e1d90bb02bb8/jiter-0.13.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87ce0f14c6c08892b610686ae8be350bf368467b6acd5085a5b65441e2bf36d2", size = 372289, upload-time = "2026-02-02T12:36:54.593Z" }, - { url = "https://files.pythonhosted.org/packages/4e/c1/e4693f107a1789a239c759a432e9afc592366f04e901470c2af89cfd28e1/jiter-0.13.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c365005b05505a90d1c47856420980d0237adf82f70c4aff7aebd3c1cc143ad", size = 360165, upload-time = "2026-02-02T12:36:56.112Z" }, - { url = "https://files.pythonhosted.org/packages/17/08/91b9ea976c1c758240614bd88442681a87672eebc3d9a6dde476874e706b/jiter-0.13.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1317fdffd16f5873e46ce27d0e0f7f4f90f0cdf1d86bf6abeaea9f63ca2c401d", size = 389634, upload-time = "2026-02-02T12:36:57.495Z" }, - { url = "https://files.pythonhosted.org/packages/18/23/58325ef99390d6d40427ed6005bf1ad54f2577866594bcf13ce55675f87d/jiter-0.13.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c05b450d37ba0c9e21c77fef1f205f56bcee2330bddca68d344baebfc55ae0df", size = 514933, upload-time = "2026-02-02T12:36:58.909Z" }, - { url = "https://files.pythonhosted.org/packages/5b/25/69f1120c7c395fd276c3996bb8adefa9c6b84c12bb7111e5c6ccdcd8526d/jiter-0.13.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:775e10de3849d0631a97c603f996f518159272db00fdda0a780f81752255ee9d", size = 548842, upload-time = "2026-02-02T12:37:00.433Z" }, - { url = "https://files.pythonhosted.org/packages/18/05/981c9669d86850c5fbb0d9e62bba144787f9fba84546ba43d624ee27ef29/jiter-0.13.0-cp314-cp314-win32.whl", hash = "sha256:632bf7c1d28421c00dd8bbb8a3bac5663e1f57d5cd5ed962bce3c73bf62608e6", size = 202108, upload-time = "2026-02-02T12:37:01.718Z" }, - { url = "https://files.pythonhosted.org/packages/8d/96/cdcf54dd0b0341db7d25413229888a346c7130bd20820530905fdb65727b/jiter-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:f22ef501c3f87ede88f23f9b11e608581c14f04db59b6a801f354397ae13739f", size = 204027, upload-time = "2026-02-02T12:37:03.075Z" }, - { url = "https://files.pythonhosted.org/packages/fb/f9/724bcaaab7a3cd727031fe4f6995cb86c4bd344909177c186699c8dec51a/jiter-0.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:07b75fe09a4ee8e0c606200622e571e44943f47254f95e2436c8bdcaceb36d7d", size = 187199, upload-time = "2026-02-02T12:37:04.414Z" }, - { url = "https://files.pythonhosted.org/packages/62/92/1661d8b9fd6a3d7a2d89831db26fe3c1509a287d83ad7838831c7b7a5c7e/jiter-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:964538479359059a35fb400e769295d4b315ae61e4105396d355a12f7fef09f0", size = 318423, upload-time = "2026-02-02T12:37:05.806Z" }, - { url = "https://files.pythonhosted.org/packages/4f/3b/f77d342a54d4ebcd128e520fc58ec2f5b30a423b0fd26acdfc0c6fef8e26/jiter-0.13.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e104da1db1c0991b3eaed391ccd650ae8d947eab1480c733e5a3fb28d4313e40", size = 351438, upload-time = "2026-02-02T12:37:07.189Z" }, - { url = "https://files.pythonhosted.org/packages/76/b3/ba9a69f0e4209bd3331470c723c2f5509e6f0482e416b612431a5061ed71/jiter-0.13.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e3a5f0cde8ff433b8e88e41aa40131455420fb3649a3c7abdda6145f8cb7202", size = 364774, upload-time = "2026-02-02T12:37:08.579Z" }, - { url = "https://files.pythonhosted.org/packages/b3/16/6cdb31fa342932602458dbb631bfbd47f601e03d2e4950740e0b2100b570/jiter-0.13.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:57aab48f40be1db920a582b30b116fe2435d184f77f0e4226f546794cedd9cf0", size = 487238, upload-time = "2026-02-02T12:37:10.066Z" }, - { url = "https://files.pythonhosted.org/packages/ed/b1/956cc7abaca8d95c13aa8d6c9b3f3797241c246cd6e792934cc4c8b250d2/jiter-0.13.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7772115877c53f62beeb8fd853cab692dbc04374ef623b30f997959a4c0e7e95", size = 372892, upload-time = "2026-02-02T12:37:11.656Z" }, - { url = "https://files.pythonhosted.org/packages/26/c4/97ecde8b1e74f67b8598c57c6fccf6df86ea7861ed29da84629cdbba76c4/jiter-0.13.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1211427574b17b633cfceba5040de8081e5abf114f7a7602f73d2e16f9fdaa59", size = 360309, upload-time = "2026-02-02T12:37:13.244Z" }, - { url = "https://files.pythonhosted.org/packages/4b/d7/eabe3cf46715854ccc80be2cd78dd4c36aedeb30751dbf85a1d08c14373c/jiter-0.13.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7beae3a3d3b5212d3a55d2961db3c292e02e302feb43fce6a3f7a31b90ea6dfe", size = 389607, upload-time = "2026-02-02T12:37:14.881Z" }, - { url = "https://files.pythonhosted.org/packages/df/2d/03963fc0804e6109b82decfb9974eb92df3797fe7222428cae12f8ccaa0c/jiter-0.13.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:e5562a0f0e90a6223b704163ea28e831bd3a9faa3512a711f031611e6b06c939", size = 514986, upload-time = "2026-02-02T12:37:16.326Z" }, - { url = "https://files.pythonhosted.org/packages/f6/6c/8c83b45eb3eb1c1e18d841fe30b4b5bc5619d781267ca9bc03e005d8fd0a/jiter-0.13.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:6c26a424569a59140fb51160a56df13f438a2b0967365e987889186d5fc2f6f9", size = 548756, upload-time = "2026-02-02T12:37:17.736Z" }, - { url = "https://files.pythonhosted.org/packages/47/66/eea81dfff765ed66c68fd2ed8c96245109e13c896c2a5015c7839c92367e/jiter-0.13.0-cp314-cp314t-win32.whl", hash = "sha256:24dc96eca9f84da4131cdf87a95e6ce36765c3b156fc9ae33280873b1c32d5f6", size = 201196, upload-time = "2026-02-02T12:37:19.101Z" }, - { url = "https://files.pythonhosted.org/packages/ff/32/4ac9c7a76402f8f00d00842a7f6b83b284d0cf7c1e9d4227bc95aa6d17fa/jiter-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0a8d76c7524087272c8ae913f5d9d608bd839154b62c4322ef65723d2e5bb0b8", size = 204215, upload-time = "2026-02-02T12:37:20.495Z" }, - { url = "https://files.pythonhosted.org/packages/f9/8e/7def204fea9f9be8b3c21a6f2dd6c020cf56c7d5ff753e0e23ed7f9ea57e/jiter-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2c26cf47e2cad140fa23b6d58d435a7c0161f5c514284802f25e87fddfe11024", size = 187152, upload-time = "2026-02-02T12:37:22.124Z" }, - { url = "https://files.pythonhosted.org/packages/79/b3/3c29819a27178d0e461a8571fb63c6ae38be6dc36b78b3ec2876bbd6a910/jiter-0.13.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b1cbfa133241d0e6bdab48dcdc2604e8ba81512f6bbd68ec3e8e1357dd3c316c", size = 307016, upload-time = "2026-02-02T12:37:42.755Z" }, - { url = "https://files.pythonhosted.org/packages/eb/ae/60993e4b07b1ac5ebe46da7aa99fdbb802eb986c38d26e3883ac0125c4e0/jiter-0.13.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:db367d8be9fad6e8ebbac4a7578b7af562e506211036cba2c06c3b998603c3d2", size = 305024, upload-time = "2026-02-02T12:37:44.774Z" }, - { url = "https://files.pythonhosted.org/packages/77/fa/2227e590e9cf98803db2811f172b2d6460a21539ab73006f251c66f44b14/jiter-0.13.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45f6f8efb2f3b0603092401dc2df79fa89ccbc027aaba4174d2d4133ed661434", size = 339337, upload-time = "2026-02-02T12:37:46.668Z" }, - { url = "https://files.pythonhosted.org/packages/2d/92/015173281f7eb96c0ef580c997da8ef50870d4f7f4c9e03c845a1d62ae04/jiter-0.13.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:597245258e6ad085d064780abfb23a284d418d3e61c57362d9449c6c7317ee2d", size = 346395, upload-time = "2026-02-02T12:37:48.09Z" }, - { url = "https://files.pythonhosted.org/packages/80/60/e50fa45dd7e2eae049f0ce964663849e897300433921198aef94b6ffa23a/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:3d744a6061afba08dd7ae375dcde870cffb14429b7477e10f67e9e6d68772a0a", size = 305169, upload-time = "2026-02-02T12:37:50.376Z" }, - { url = "https://files.pythonhosted.org/packages/d2/73/a009f41c5eed71c49bec53036c4b33555afcdee70682a18c6f66e396c039/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:ff732bd0a0e778f43d5009840f20b935e79087b4dc65bd36f1cd0f9b04b8ff7f", size = 303808, upload-time = "2026-02-02T12:37:52.092Z" }, - { url = "https://files.pythonhosted.org/packages/c4/10/528b439290763bff3d939268085d03382471b442f212dca4ff5f12802d43/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab44b178f7981fcaea7e0a5df20e773c663d06ffda0198f1a524e91b2fde7e59", size = 337384, upload-time = "2026-02-02T12:37:53.582Z" }, - { url = "https://files.pythonhosted.org/packages/67/8a/a342b2f0251f3dac4ca17618265d93bf244a2a4d089126e81e4c1056ac50/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19", size = 343768, upload-time = "2026-02-02T12:37:55.055Z" }, -] - [[package]] name = "jiwer" version = "3.1.0" @@ -2621,27 +2488,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/1d/50ad811d1c5dae091e4cf046beba925bcae0a610e79ae4c538f996f63ed5/kiwisolver-1.4.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:65ea09a5a3faadd59c2ce96dc7bf0f364986a315949dc6374f04396b0d60e09b", size = 71762, upload-time = "2024-12-24T18:30:48.903Z" }, ] -[[package]] -name = "langfuse" -version = "3.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "backoff" }, - { name = "httpx" }, - { name = "openai" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-sdk" }, - { name = "packaging" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/24/d0/744e5613c728427330ac2049da0f54fc313e8bf84622f71b025bfba65496/langfuse-3.13.0.tar.gz", hash = "sha256:dacea8111ca4442e97dbfec4f8d676cf9709b35357a26e468f8887b95de0012f", size = 233420, upload-time = "2026-02-06T19:54:14.415Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/63/148382e8e79948f7e5c9c137288e504bb88117574eb7e7c886b4fb470b4b/langfuse-3.13.0-py3-none-any.whl", hash = "sha256:71912ddac1cc831a65df895eae538a556f564c094ae51473e747426e9ded1a9d", size = 417626, upload-time = "2026-02-06T19:54:12.547Z" }, -] - [[package]] name = "lazy-loader" version = "0.4" @@ -5453,107 +5299,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/16/873b955beda7bada5b0d798d3a601b2ff210e44ad5169f6d405b93892103/onnxruntime-1.22.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64845709f9e8a2809e8e009bc4c8f73b788cee9c6619b7d9930344eae4c9cd36", size = 16427482, upload-time = "2025-05-09T20:26:20.376Z" }, ] -[[package]] -name = "openai" -version = "2.17.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9c/a2/677f22c4b487effb8a09439fb6134034b5f0a39ca27df8b95fac23a93720/openai-2.17.0.tar.gz", hash = "sha256:47224b74bd20f30c6b0a6a329505243cb2f26d5cf84d9f8d0825ff8b35e9c999", size = 631445, upload-time = "2026-02-05T16:27:40.953Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/97/284535aa75e6e84ab388248b5a323fc296b1f70530130dee37f7f4fbe856/openai-2.17.0-py3-none-any.whl", hash = "sha256:4f393fd886ca35e113aac7ff239bcd578b81d8f104f5aedc7d3693eb2af1d338", size = 1069524, upload-time = "2026-02-05T16:27:38.941Z" }, -] - -[[package]] -name = "opentelemetry-api" -version = "1.39.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-metadata" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/97/b9/3161be15bb8e3ad01be8be5a968a9237c3027c5be504362ff800fca3e442/opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c", size = 65767, upload-time = "2025-12-11T13:32:39.182Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/df/d3f1ddf4bb4cb50ed9b1139cc7b1c54c34a1e7ce8fd1b9a37c0d1551a6bd/opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950", size = 66356, upload-time = "2025-12-11T13:32:17.304Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-common" -version = "1.39.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-proto" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/9d/22d241b66f7bbde88a3bfa6847a351d2c46b84de23e71222c6aae25c7050/opentelemetry_exporter_otlp_proto_common-1.39.1.tar.gz", hash = "sha256:763370d4737a59741c89a67b50f9e39271639ee4afc999dadfe768541c027464", size = 20409, upload-time = "2025-12-11T13:32:40.885Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/02/ffc3e143d89a27ac21fd557365b98bd0653b98de8a101151d5805b5d4c33/opentelemetry_exporter_otlp_proto_common-1.39.1-py3-none-any.whl", hash = "sha256:08f8a5862d64cc3435105686d0216c1365dc5701f86844a8cd56597d0c764fde", size = 18366, upload-time = "2025-12-11T13:32:20.2Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-http" -version = "1.39.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/80/04/2a08fa9c0214ae38880df01e8bfae12b067ec0793446578575e5080d6545/opentelemetry_exporter_otlp_proto_http-1.39.1.tar.gz", hash = "sha256:31bdab9745c709ce90a49a0624c2bd445d31a28ba34275951a6a362d16a0b9cb", size = 17288, upload-time = "2025-12-11T13:32:42.029Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/f1/b27d3e2e003cd9a3592c43d099d2ed8d0a947c15281bf8463a256db0b46c/opentelemetry_exporter_otlp_proto_http-1.39.1-py3-none-any.whl", hash = "sha256:d9f5207183dd752a412c4cd564ca8875ececba13be6e9c6c370ffb752fd59985", size = 19641, upload-time = "2025-12-11T13:32:22.248Z" }, -] - -[[package]] -name = "opentelemetry-proto" -version = "1.39.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/1d/f25d76d8260c156c40c97c9ed4511ec0f9ce353f8108ca6e7561f82a06b2/opentelemetry_proto-1.39.1.tar.gz", hash = "sha256:6c8e05144fc0d3ed4d22c2289c6b126e03bcd0e6a7da0f16cedd2e1c2772e2c8", size = 46152, upload-time = "2025-12-11T13:32:48.681Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/95/b40c96a7b5203005a0b03d8ce8cd212ff23f1793d5ba289c87a097571b18/opentelemetry_proto-1.39.1-py3-none-any.whl", hash = "sha256:22cdc78efd3b3765d09e68bfbd010d4fc254c9818afd0b6b423387d9dee46007", size = 72535, upload-time = "2025-12-11T13:32:33.866Z" }, -] - -[[package]] -name = "opentelemetry-sdk" -version = "1.39.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/fb/c76080c9ba07e1e8235d24cdcc4d125ef7aa3edf23eb4e497c2e50889adc/opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6", size = 171460, upload-time = "2025-12-11T13:32:49.369Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/98/e91cf858f203d86f4eccdf763dcf01cf03f1dae80c3750f7e635bfa206b6/opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c", size = 132565, upload-time = "2025-12-11T13:32:35.069Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.60b1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/df/553f93ed38bf22f4b999d9be9c185adb558982214f33eae539d3b5cd0858/opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953", size = 137935, upload-time = "2025-12-11T13:32:50.487Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/5e/5958555e09635d09b75de3c4f8b9cae7335ca545d77392ffe7331534c402/opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb", size = 219982, upload-time = "2025-12-11T13:32:36.955Z" }, -] - [[package]] name = "optuna" version = "4.3.0" @@ -8280,17 +8025,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6b/11/cc635220681e93a0183390e26485430ca2c7b5f9d33b15c74c2861cb8091/urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813", size = 128680, upload-time = "2025-04-10T15:23:37.377Z" }, ] -[[package]] -name = "useful-moonshine-onnx" -version = "20241016" -source = { git = "https://github.com/usefulsensors/moonshine.git?subdirectory=moonshine-onnx#89981f1ba33fc471ecb2e7a64531ee44f05e7de6" } -dependencies = [ - { name = "huggingface-hub" }, - { name = "librosa" }, - { name = "onnxruntime" }, - { name = "tokenizers" }, -] - [[package]] name = "uvicorn" version = "0.34.3" diff --git a/extras/friend-lite-sdk/LICENSE b/extras/friend-lite-sdk/LICENSE new file mode 100644 index 00000000..4130f88b --- /dev/null +++ b/extras/friend-lite-sdk/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Chronicle AI Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/extras/friend-lite-sdk/NOTICE b/extras/friend-lite-sdk/NOTICE new file mode 100644 index 00000000..786c9e49 --- /dev/null +++ b/extras/friend-lite-sdk/NOTICE @@ -0,0 +1,7 @@ +This package is derived from the OMI Python SDK: + https://github.com/BasedHardware/omi/tree/main/sdks/python + +Original work: Copyright (c) 2024 Based Hardware Contributors +Licensed under the MIT License. + +Fork: https://github.com/AnkushMalaker/Friend diff --git a/extras/friend-lite-sdk/README.md b/extras/friend-lite-sdk/README.md new file mode 100644 index 00000000..89735516 --- /dev/null +++ b/extras/friend-lite-sdk/README.md @@ -0,0 +1,31 @@ +# friend-lite-sdk + +Python SDK for OMI / Friend Lite BLE devices — audio streaming, button events, and transcription. + +Derived from the [OMI Python SDK](https://github.com/BasedHardware/omi/tree/main/sdks/python) (MIT license, Based Hardware Contributors). See `NOTICE` for attribution. + +## Installation + +```bash +pip install -e extras/friend-lite-sdk +``` + +With optional transcription support: + +```bash +pip install -e "extras/friend-lite-sdk[deepgram,wyoming]" +``` + +## Usage + +```python +import asyncio +from friend_lite import OmiConnection, ButtonState, parse_button_event + +async def main(): + async with OmiConnection("AA:BB:CC:DD:EE:FF") as conn: + await conn.subscribe_audio(lambda _handle, data: print(len(data), "bytes")) + await conn.wait_until_disconnected() + +asyncio.run(main()) +``` diff --git a/extras/friend-lite-sdk/friend_lite/__init__.py b/extras/friend-lite-sdk/friend_lite/__init__.py new file mode 100644 index 00000000..6292b3eb --- /dev/null +++ b/extras/friend-lite-sdk/friend_lite/__init__.py @@ -0,0 +1,18 @@ +from .bluetooth import OmiConnection, listen_to_omi, print_devices +from .button import ButtonState, parse_button_event +from .uuids import ( + OMI_AUDIO_CHAR_UUID, + OMI_BUTTON_CHAR_UUID, + OMI_BUTTON_SERVICE_UUID, +) + +__all__ = [ + "ButtonState", + "OMI_AUDIO_CHAR_UUID", + "OMI_BUTTON_CHAR_UUID", + "OMI_BUTTON_SERVICE_UUID", + "OmiConnection", + "listen_to_omi", + "parse_button_event", + "print_devices", +] diff --git a/extras/friend-lite-sdk/friend_lite/bluetooth.py b/extras/friend-lite-sdk/friend_lite/bluetooth.py new file mode 100644 index 00000000..ce7ea505 --- /dev/null +++ b/extras/friend-lite-sdk/friend_lite/bluetooth.py @@ -0,0 +1,70 @@ +import asyncio +from typing import Callable, Optional + +from bleak import BleakClient, BleakScanner + +from .uuids import OMI_AUDIO_CHAR_UUID, OMI_BUTTON_CHAR_UUID + + +def print_devices() -> None: + devices = asyncio.run(BleakScanner.discover()) + for i, d in enumerate(devices): + print(f"{i}. {d.name} [{d.address}]") + + +class OmiConnection: + def __init__(self, mac_address: str) -> None: + self._mac_address = mac_address + self._client: Optional[BleakClient] = None + self._disconnected = asyncio.Event() + + async def __aenter__(self) -> "OmiConnection": + await self.connect() + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + await self.disconnect() + + async def connect(self) -> None: + if self._client is not None: + return + + def _on_disconnect(_client: BleakClient) -> None: + self._disconnected.set() + + self._client = BleakClient( + self._mac_address, + disconnected_callback=_on_disconnect, + ) + await self._client.connect() + + async def disconnect(self) -> None: + if self._client is None: + return + await self._client.disconnect() + self._client = None + self._disconnected.set() + + async def subscribe_audio(self, callback: Callable[[int, bytearray], None]) -> None: + await self.subscribe(OMI_AUDIO_CHAR_UUID, callback) + + async def subscribe_button(self, callback: Callable[[int, bytearray], None]) -> None: + await self.subscribe(OMI_BUTTON_CHAR_UUID, callback) + + async def subscribe(self, uuid: str, callback: Callable[[int, bytearray], None]) -> None: + if self._client is None: + raise RuntimeError("Not connected to OMI device") + await self._client.start_notify(uuid, callback) + + async def wait_until_disconnected(self, timeout: float | None = None) -> None: + if timeout is None: + await self._disconnected.wait() + else: + await asyncio.wait_for(self._disconnected.wait(), timeout=timeout) + + +async def listen_to_omi(mac_address: str, char_uuid: str, data_handler) -> None: + """Backward-compatible wrapper for older consumers.""" + async with OmiConnection(mac_address) as conn: + await conn.subscribe(char_uuid, data_handler) + await conn.wait_until_disconnected() diff --git a/extras/friend-lite-sdk/friend_lite/button.py b/extras/friend-lite-sdk/friend_lite/button.py new file mode 100644 index 00000000..421a87b1 --- /dev/null +++ b/extras/friend-lite-sdk/friend_lite/button.py @@ -0,0 +1,24 @@ +"""Button event parsing for Omi BLE button characteristic.""" + +import struct +from enum import IntEnum + + +class ButtonState(IntEnum): + IDLE = 0 + SINGLE_TAP = 1 + DOUBLE_TAP = 2 + LONG_PRESS = 3 + PRESS = 4 + RELEASE = 5 + + +def parse_button_event(data: bytes) -> ButtonState: + """Parse the button event payload into a ButtonState. + + Payload is two little-endian uint32 values: [state, 0]. + """ + if len(data) < 8: + raise ValueError(f"Expected 8 bytes for button event, got {len(data)}") + state, _unused = struct.unpack("= WINDOW_SECONDS: + logger.debug(f"Window time ({WINDOW_SECONDS}s) elapsed.") + break + + try: + # Calculate remaining time in window for timeout + timeout = max(0.1, WINDOW_SECONDS - time_elapsed) # Use small minimum timeout + # Get chunk with timeout to prevent blocking indefinitely if queue is empty + chunk = await asyncio.wait_for(audio_queue.get(), timeout=timeout) + + if chunk is None: # Handle queue termination signal + logger.info("Audio queue finished during chunk sending. Exiting.") + # No need to send AudioStop if queue ended before window completion + return # Exit the entire function if queue is done + + # Send audio chunk + logger.debug(f"Wyoming: Sending AudioChunk ({len(chunk)} bytes)...") + await client.write_event(AudioChunk(audio=chunk, rate=SAMPLE_RATE, width=SAMPLE_WIDTH, channels=CHANNELS).event()) + segment_has_audio = True # Mark that we sent audio in this segment + logger.debug("Wyoming: AudioChunk sent.") + + except asyncio.TimeoutError: + # Expected if no audio comes within the window's remaining time + logger.debug("Timeout waiting for audio chunk, window likely finished.") + break # Exit chunk sending loop + except asyncio.CancelledError: + logger.info("Chunk sending task cancelled.") + raise # Re-raise cancellation + except Exception as e: + logger.error(f"Error getting/sending audio chunk: {e}", exc_info=True) + raise # Re-raise other exceptions to break segment processing + + # 5. Stop Audio Segment (only if audio was sent in this window) + if segment_has_audio: + logger.debug(f"Wyoming: Sending AudioStop...") + await client.write_event(AudioStop().event()) + logger.debug("Wyoming: AudioStop sent.") + + # 6. Read Transcript for the Segment + logger.debug(f"Wyoming: Reading events for transcript (timeout={READ_TIMEOUT_SECONDS}s)...") + try: + while True: # Loop to read events until transcript or timeout/error + event = await asyncio.wait_for(client.read_event(), timeout=READ_TIMEOUT_SECONDS) + + if event is None: + logger.warning("Wyoming connection closed by server unexpectedly while waiting for transcript.") + # Server might close connection after sending transcript/error, treat as end of segment read + break + + logger.debug(f"Wyoming: Received event raw: {event}") # DEBUG + logger.debug(f"Wyoming: Received event type: {type(event)}") # DEBUG + if hasattr(event, 'data'): + logger.debug(f"Wyoming: Received event data: {event.data}") # DEBUG + + # Check for transcription event + if isinstance(event, Event) and event.type == 'transcript' and 'text' in event.data: + transcript = event.data['text'] + if transcript and transcript.strip(): + logger.info(f"Transcript: {transcript.strip()}") + # Assume one transcript per segment, break after receiving it + logger.debug("Breaking read loop after receiving transcript.") + break + elif isinstance(event, Event) and event.type == 'error': + logger.error(f"Wyoming server error event: {event.data}") + # Break on error, segment finished (with error) + break + else: + logger.debug(f"Received non-transcript/non-error event: type={type(event)}") + # Continue reading other events until transcript/error or timeout + + except asyncio.TimeoutError: + logger.warning(f"Timeout waiting for transcript after {READ_TIMEOUT_SECONDS}s.") + # Continue to the next segment even if no transcript was received + except (ConnectionClosedOK, ConnectionClosedError, ConnectionResetError) as close_err: + logger.info(f"Wyoming connection closed gracefully/expectedly after sending audio: {close_err}") + # This is expected if the server closes after sending the transcript/error + except asyncio.CancelledError: + logger.info("Transcript reading task cancelled.") + raise # Re-raise cancellation + except Exception as e: + logger.error(f"Error reading transcript event: {e}", exc_info=True) + # Depending on the error, might want to raise or just log and continue + # For now, log and continue to the finally block/next segment + else: + logger.info("Skipping AudioStop and transcript read as no audio was sent in this window.") + + except (ConnectionRefusedError, ConnectionResetError, ConnectionError, WebSocketException) as conn_err: + logger.error(f"Wyoming connection/websocket error during segment: {conn_err}") + # Connection error for a segment, wait before retrying the *next* segment + await asyncio.sleep(5) + except asyncio.CancelledError: + logger.info("Main transcription task cancelled during segment processing.") + break # Exit outer loop if cancelled + except Exception as e: + logger.error(f"Unexpected error during segment processing: {e}", exc_info=True) + # Log unexpected error and wait before next segment attempt + await asyncio.sleep(5) + finally: + if client: # Check if client was successfully created + logger.info("Attempting to disconnect from Wyoming server for this segment...") + with suppress(Exception): # Suppress errors during cleanup disconnect + await client.disconnect() + logger.info("Disconnected from Wyoming server for this segment.") + client = None # Ensure client is reset for the next loop iteration + + # Check if the task was cancelled before potentially sleeping/looping + task = asyncio.current_task() + if task and task.cancelled(): + logger.info("Task cancelled, exiting transcribe_wyoming loop.") + break + + logger.info("Exiting transcribe_wyoming function.") + +# Ensure logging is configured if this module is run directly or imported early +# logging.basicConfig(level=logging.DEBUG) diff --git a/extras/friend-lite-sdk/friend_lite/uuids.py b/extras/friend-lite-sdk/friend_lite/uuids.py new file mode 100644 index 00000000..452a2416 --- /dev/null +++ b/extras/friend-lite-sdk/friend_lite/uuids.py @@ -0,0 +1,8 @@ +"""UUID constants for OMI BLE services and characteristics.""" + +# Standard Omi audio characteristic UUID +OMI_AUDIO_CHAR_UUID = "19B10001-E8F2-537E-4F6C-D104768A1214" + +# Omi button service + characteristic UUIDs +OMI_BUTTON_SERVICE_UUID = "23BA7924-0000-1000-7450-346EAC492E92" +OMI_BUTTON_CHAR_UUID = "23BA7925-0000-1000-7450-346EAC492E92" diff --git a/extras/friend-lite-sdk/pyproject.toml b/extras/friend-lite-sdk/pyproject.toml new file mode 100644 index 00000000..7e11d3af --- /dev/null +++ b/extras/friend-lite-sdk/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "friend-lite-sdk" +version = "0.2.0" +description = "Python SDK for OMI/Friend Lite BLE devices — audio streaming, button events, and transcription" +requires-python = ">= 3.10" +license = "MIT" +dependencies = [ + "bleak>=0.22.3", + "numpy>=1.26", + "opuslib>=3.0.1", + "websockets>=14.0.0", +] + +[project.urls] +Homepage = "https://github.com/AnkushMalaker/chronicle" +"Original Project" = "https://github.com/BasedHardware/omi" + +[project.optional-dependencies] +deepgram = ["deepgram-sdk>=3.11.0"] +wyoming = ["wyoming"] +dev = ["mypy>=1.15.0"] + +[build-system] +requires = ["setuptools >= 80.0.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["friend_lite*"] diff --git a/extras/local-omi-bt/README.md b/extras/local-omi-bt/README.md index e9b34820..3a61352e 100644 --- a/extras/local-omi-bt/README.md +++ b/extras/local-omi-bt/README.md @@ -1,3 +1,39 @@ -# Usage -Run using `uv run --with-requirements requirements.txt python connect-omi.py` -from this directory. \ No newline at end of file +# Local OMI BT + +Connect to an OMI device over Bluetooth and stream audio to the Chronicle backend. + +## Prerequisites + +- **Python 3.12+** (managed via `uv`) +- **Opus codec library** (required by `opuslib`) + +### Installing Opus + +**macOS (Homebrew):** +```bash +brew install opus +``` + +**Linux (Debian/Ubuntu):** +```bash +sudo apt install libopus-dev +``` + +## Usage + +```bash +./start.sh +``` + +Or run directly: +```bash +uv run --with-requirements requirements.txt python connect-omi.py +``` + +### macOS: Opus library not found + +If you see `Could not find Opus library`, you need to tell the dynamic linker where to find it. The `start.sh` script handles this automatically, but if running manually: + +```bash +DYLD_LIBRARY_PATH="$(brew --prefix opus)/lib" uv run --with-requirements requirements.txt python connect-omi.py +``` diff --git a/extras/local-omi-bt/connect-omi.py b/extras/local-omi-bt/connect-omi.py index 302a17d7..474b74bf 100644 --- a/extras/local-omi-bt/connect-omi.py +++ b/extras/local-omi-bt/connect-omi.py @@ -7,11 +7,10 @@ import asyncstdlib as asyncstd from bleak import BleakClient, BleakScanner -from bleak.backends.device import BLEDevice from dotenv import load_dotenv, set_key from easy_audio_interfaces.filesystem import RollingFileSink -from chronicle.bluetooth import listen_to_omi, print_devices -from chronicle.decoder import OmiOpusDecoder +from friend_lite import ButtonState, OmiConnection, parse_button_event +from friend_lite.decoder import OmiOpusDecoder from wyoming.audio import AudioChunk # Setup logging @@ -25,15 +24,12 @@ load_dotenv(env_path) sys.path.append(os.path.dirname(__file__)) -from send_to_adv import stream_to_backend +from send_to_adv import send_button_event, stream_to_backend OMI_MAC = os.getenv("OMI_MAC") if not OMI_MAC: logger.info("OMI_MAC not found in .env. Will try to find and set.") -# Standard Omi audio characteristic UUID -OMI_CHAR_UUID = "19B10001-E8F2-537E-4F6C-D104768A1214" - async def source_bytes(audio_queue: Queue[bytes]) -> AsyncGenerator[bytes, None]: """Single source iterator from the queue.""" while True: @@ -49,16 +45,6 @@ async def as_audio_chunks(it) -> AsyncGenerator[AudioChunk, None]: async for data in it: yield AudioChunk(audio=data, rate=16000, width=2, channels=1) -# Add this to chronicle sdk -async def list_devices(prefix: str = "OMI") -> list[BLEDevice]: - devices = await BleakScanner.discover() - filtered_devices = [] - for d in devices: - if d.name: - if prefix.casefold() in d.name.casefold(): - filtered_devices.append(d) - return filtered_devices - def main() -> None: # api_key: str | None = os.getenv("DEEPGRAM_API_KEY") @@ -76,15 +62,78 @@ def handle_ble_data(sender: Any, data: bytes) -> None: audio_queue.put_nowait(decoded_pcm) except Exception as e: logger.error("Queue Error: %s", e) + + def handle_button_event(sender: Any, data: bytes) -> None: + try: + state = parse_button_event(data) + except Exception as e: + logger.error("Button event parse error: %s", e) + return + if state != ButtonState.IDLE: + logger.info("Button event: %s", state.name) + try: + loop = asyncio.get_running_loop() + loop.create_task(send_button_event(state.name)) + except RuntimeError: + logger.debug("No running event loop, cannot send button event") + def prompt_user_to_pick_device(all_devices) -> str | None: + """Interactively prompt the user to select an OMI/Neo device from scan results. + + Returns the selected MAC address string, or None if no selection was made. + Saves the selected MAC to .env via set_key(). + """ + omi_devices = [ + d for d in all_devices + if d.name and ("omi" in d.name.casefold() or "neo" in d.name.casefold()) + ] + + if not omi_devices: + logger.info("No OMI/Neo devices found. All discovered BLE devices:") + if all_devices: + for i, d in enumerate(all_devices): + logger.info(" %d. %s [%s]", i + 1, d.name or "(unnamed)", d.address) + else: + logger.info(" (no BLE devices found at all)") + return None + + if len(omi_devices) == 1: + device = omi_devices[0] + answer = input(f"Found {device.name} [{device.address}]. Use this device? [Y/n] ").strip().lower() + if answer in ("", "y", "yes"): + set_key(env_path, "OMI_MAC", device.address) + logger.info("OMI_MAC set to %s and saved to .env", device.address) + return device.address + return None + + # Multiple OMI/Neo devices found + logger.info("Multiple OMI/Neo devices found:") + for i, d in enumerate(omi_devices): + logger.info(" %d. %s [%s]", i + 1, d.name, d.address) + choice = input("Enter number to select (or q to quit): ").strip().lower() + if choice == "q": + return None + try: + idx = int(choice) - 1 + if 0 <= idx < len(omi_devices): + selected = omi_devices[idx] + set_key(env_path, "OMI_MAC", selected.address) + logger.info("OMI_MAC set to %s and saved to .env", selected.address) + return selected.address + else: + logger.error("Invalid selection: %s", choice) + return None + except ValueError: + logger.error("Invalid input: %s", choice) + return None + async def find_and_set_omi_mac() -> str: - devices = await list_devices() - assert len(devices) == 1, "Expected 1 Omi device, got %d" % len(devices) - discovered_mac = devices[0].address - set_key(env_path, "OMI_MAC", discovered_mac) - logger.info("OMI_MAC set to %s and saved to .env" % discovered_mac) - return discovered_mac + all_devices = await BleakScanner.discover() + selected = prompt_user_to_pick_device(all_devices) + if not selected: + raise SystemExit(1) + return selected async def run() -> None: logger.info("Starting OMI Bluetooth connection and audio streaming") @@ -101,8 +150,20 @@ async def run() -> None: logger.info(f"Successfully connected to device {mac_address}") except Exception as e: logger.error(f"Failed to connect to device {mac_address}: {e}") - logger.error("Exiting without creating audio sink or backend connection") - return + logger.info("Scanning for nearby BLE devices...") + all_devices = await BleakScanner.discover() + selected = prompt_user_to_pick_device(all_devices) + if not selected: + return + mac_address = selected + # Verify the newly selected device is reachable + logger.info("Connecting to newly selected device %s...", mac_address) + try: + async with BleakClient(mac_address) as test_client: + logger.info(f"Successfully connected to device {mac_address}") + except Exception as e2: + logger.error(f"Failed to connect to newly selected device {mac_address}: {e2}") + return # Device is available, now setup audio sink and backend connection logger.info("Device found and connected, setting up audio pipeline...") @@ -148,11 +209,14 @@ async def queue_to_stream(): async with file_sink: try: - await asyncio.gather( - listen_to_omi(mac_address, OMI_CHAR_UUID, handle_ble_data), - process_audio(), - backend_stream_wrapper(), - ) + async with OmiConnection(mac_address) as conn: + await conn.subscribe_audio(handle_ble_data) + await conn.subscribe_button(handle_button_event) + await asyncio.gather( + conn.wait_until_disconnected(), + process_audio(), + backend_stream_wrapper(), + ) except Exception as e: logger.error(f"Error in audio processing: {e}", exc_info=True) finally: diff --git a/extras/local-omi-bt/requirements.txt b/extras/local-omi-bt/requirements.txt index 5d0f167f..3438c2d5 100644 --- a/extras/local-omi-bt/requirements.txt +++ b/extras/local-omi-bt/requirements.txt @@ -2,7 +2,7 @@ bleak==0.22.3 numpy>=1.26.4 scipy>=1.12.0 opuslib>=3.0.1 -friend-lite-sdk +friend-lite-sdk @ file:../friend-lite-sdk easy_audio_interfaces python-dotenv asyncstdlib diff --git a/extras/local-omi-bt/scan_devices.py b/extras/local-omi-bt/scan_devices.py new file mode 100644 index 00000000..f00cd278 --- /dev/null +++ b/extras/local-omi-bt/scan_devices.py @@ -0,0 +1,60 @@ +"""Quick BLE scanner to find neo1/neosapien devices.""" +import asyncio +from bleak import BleakScanner, BleakClient + +async def scan_all_devices(): + """Scan for all BLE devices.""" + print("Scanning for BLE devices (10 seconds)...") + devices = await BleakScanner.discover(timeout=10.0) + + print(f"\nFound {len(devices)} devices:\n") + + neo_devices = [] + for d in sorted(devices, key=lambda x: x.name or ""): + name = d.name or "(no name)" + print(f" {name:<30} | {d.address}") + + # Look for neo/neosapien devices + if d.name and any(x in d.name.lower() for x in ["neo", "sapien"]): + neo_devices.append(d) + + return neo_devices + +async def explore_device(address: str): + """Connect to a device and list its services/characteristics.""" + print(f"\nConnecting to {address}...") + try: + async with BleakClient(address, timeout=20.0) as client: + print(f"Connected: {client.is_connected}") + print("\nServices and Characteristics:") + + for service in client.services: + print(f"\n Service: {service.uuid}") + print(f" Description: {service.description}") + + for char in service.characteristics: + props = ", ".join(char.properties) + print(f" Char: {char.uuid}") + print(f" Properties: {props}") + print(f" Handle: {char.handle}") + except Exception as e: + print(f"Error connecting: {e}") + +async def main(): + neo_devices = await scan_all_devices() + + if neo_devices: + print(f"\n{'='*60}") + print(f"Found {len(neo_devices)} neo/neosapien device(s)!") + for d in neo_devices: + print(f" - {d.name}: {d.address}") + + # Explore the first neo device + print(f"\n{'='*60}") + await explore_device(neo_devices[0].address) + else: + print("\nNo neo/neosapien devices found.") + print("Make sure the device is powered on and in pairing mode.") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/extras/local-omi-bt/send_to_adv.py b/extras/local-omi-bt/send_to_adv.py index e878a404..39763e9b 100644 --- a/extras/local-omi-bt/send_to_adv.py +++ b/extras/local-omi-bt/send_to_adv.py @@ -32,6 +32,28 @@ logger = logging.getLogger(__name__) +# Module-level websocket reference for sending control messages (e.g., button events) +_active_websocket = None + + +async def send_button_event(button_state: str) -> None: + """Send a button event to the backend via the active WebSocket connection. + + Args: + button_state: Button state string (e.g., "SINGLE_TAP", "DOUBLE_TAP") + """ + if _active_websocket is None: + logger.debug("No active websocket, dropping button event: %s", button_state) + return + + event = { + "type": "button-event", + "data": {"state": button_state}, + "payload_length": None, + } + await _active_websocket.send(json.dumps(event) + "\n") + logger.info("Sent button event to backend: %s", button_state) + async def get_jwt_token(username: str, password: str) -> Optional[str]: """ @@ -152,6 +174,8 @@ async def stream_to_backend(stream: AsyncGenerator[AudioChunk, None]): ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE + global _active_websocket + logger.info(f"Connecting to WebSocket: {websocket_uri}") async with websockets.connect( uri_with_token, @@ -160,6 +184,8 @@ async def stream_to_backend(stream: AsyncGenerator[AudioChunk, None]): ping_timeout=120, # Wait up to 120 seconds for pong (increased from default 20s) close_timeout=10, # Graceful close timeout ) as websocket: + _active_websocket = websocket + # Wait for ready message from backend ready_msg = await websocket.recv() logger.info(f"Backend ready: {ready_msg}") @@ -217,6 +243,7 @@ async def stream_to_backend(stream: AsyncGenerator[AudioChunk, None]): logger.info(f"Sent audio-stop event. Total chunks: {chunk_count}") finally: + _active_websocket = None # Clean up receive task receive_task.cancel() try: diff --git a/extras/local-omi-bt/start.sh b/extras/local-omi-bt/start.sh index 6fd8947e..381a2c78 100755 --- a/extras/local-omi-bt/start.sh +++ b/extras/local-omi-bt/start.sh @@ -1,2 +1,12 @@ #!/bin/bash + +# macOS: opuslib needs the Opus shared library on the dynamic linker path. +# Install with: brew install opus +if [ "$(uname)" = "Darwin" ] && command -v brew &>/dev/null; then + OPUS_PREFIX="$(brew --prefix opus 2>/dev/null)" + if [ -d "$OPUS_PREFIX/lib" ]; then + export DYLD_LIBRARY_PATH="${OPUS_PREFIX}/lib${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" + fi +fi + uv run --with-requirements requirements.txt python connect-omi.py diff --git a/extras/speaker-recognition/src/simple_speaker_recognition/api/routers/identification.py b/extras/speaker-recognition/src/simple_speaker_recognition/api/routers/identification.py index 15a6ef7f..1f215242 100644 --- a/extras/speaker-recognition/src/simple_speaker_recognition/api/routers/identification.py +++ b/extras/speaker-recognition/src/simple_speaker_recognition/api/routers/identification.py @@ -311,7 +311,7 @@ async def diarize_identify_match( conversation_id: Optional[str] = Form(default=None, description="Conversation ID to fetch audio from backend"), backend_token: Optional[str] = Form(default=None, description="JWT token for backend API authentication"), min_duration: float = Form(default=0.5, description="Minimum segment duration in seconds"), - similarity_threshold: float = Form(default=0.15, description="Speaker similarity threshold"), + similarity_threshold: float = Form(default=0.45, description="Speaker similarity threshold"), min_speakers: Optional[int] = Form(default=None, description="Minimum number of speakers to detect"), max_speakers: Optional[int] = Form(default=None, description="Maximum number of speakers to detect"), collar: float = Form(default=2.0, description="Collar duration (seconds) around speaker boundaries to merge segments"), diff --git a/extras/speaker-recognition/src/simple_speaker_recognition/api/service.py b/extras/speaker-recognition/src/simple_speaker_recognition/api/service.py index d849f9d5..6d76c7f4 100644 --- a/extras/speaker-recognition/src/simple_speaker_recognition/api/service.py +++ b/extras/speaker-recognition/src/simple_speaker_recognition/api/service.py @@ -63,7 +63,7 @@ def load_speaker_config_from_root() -> dict: class Settings(BaseSettings): """Service configuration settings.""" - similarity_threshold: float = Field(default=0.15, description="Cosine similarity threshold for speaker identification (0.1-0.3 typical for ECAPA-TDNN)") + similarity_threshold: float = Field(default=0.45, description="Cosine similarity threshold for speaker identification") data_dir: Path = Field(default_factory=get_data_directory, description="Directory for storing speaker data") enrollment_audio_dir: Path = Field(default_factory=lambda: get_data_directory() / "enrollment_audio", description="Directory for storing enrollment audio files") max_file_seconds: int = Field(default=180, description="Maximum file duration in seconds") diff --git a/backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/README.md b/plugins/email_summarizer/README.md similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/README.md rename to plugins/email_summarizer/README.md diff --git a/backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/__init__.py b/plugins/email_summarizer/__init__.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/__init__.py rename to plugins/email_summarizer/__init__.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/config.yml b/plugins/email_summarizer/config.yml similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/config.yml rename to plugins/email_summarizer/config.yml diff --git a/backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/email_service.py b/plugins/email_summarizer/email_service.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/email_service.py rename to plugins/email_summarizer/email_service.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/plugin.py b/plugins/email_summarizer/plugin.py similarity index 99% rename from backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/plugin.py rename to plugins/email_summarizer/plugin.py index 36958cca..2f9459df 100644 --- a/backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/plugin.py +++ b/plugins/email_summarizer/plugin.py @@ -11,7 +11,7 @@ from advanced_omi_backend.llm_client import async_generate from advanced_omi_backend.utils.logging_utils import mask_dict -from ..base import BasePlugin, PluginContext, PluginResult +from advanced_omi_backend.plugins.base import BasePlugin, PluginContext, PluginResult from .email_service import SMTPEmailService from .templates import format_html_email, format_text_email diff --git a/backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/setup.py b/plugins/email_summarizer/setup.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/setup.py rename to plugins/email_summarizer/setup.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/templates.py b/plugins/email_summarizer/templates.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/email_summarizer/templates.py rename to plugins/email_summarizer/templates.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/homeassistant/__init__.py b/plugins/homeassistant/__init__.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/homeassistant/__init__.py rename to plugins/homeassistant/__init__.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/homeassistant/command_parser.py b/plugins/homeassistant/command_parser.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/homeassistant/command_parser.py rename to plugins/homeassistant/command_parser.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/homeassistant/config.yml b/plugins/homeassistant/config.yml similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/homeassistant/config.yml rename to plugins/homeassistant/config.yml diff --git a/backends/advanced/src/advanced_omi_backend/plugins/homeassistant/entity_cache.py b/plugins/homeassistant/entity_cache.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/homeassistant/entity_cache.py rename to plugins/homeassistant/entity_cache.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/homeassistant/mcp_client.py b/plugins/homeassistant/mcp_client.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/homeassistant/mcp_client.py rename to plugins/homeassistant/mcp_client.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/homeassistant/plugin.py b/plugins/homeassistant/plugin.py similarity index 88% rename from backends/advanced/src/advanced_omi_backend/plugins/homeassistant/plugin.py rename to plugins/homeassistant/plugin.py index 0fa7f04d..00a89c95 100644 --- a/backends/advanced/src/advanced_omi_backend/plugins/homeassistant/plugin.py +++ b/plugins/homeassistant/plugin.py @@ -9,7 +9,7 @@ import logging from typing import Any, Dict, List, Optional -from ..base import BasePlugin, PluginContext, PluginResult +from advanced_omi_backend.plugins.base import BasePlugin, PluginContext, PluginResult from .entity_cache import EntityCache from .mcp_client import HAMCPClient, MCPError @@ -250,6 +250,87 @@ async def on_transcript(self, context: PluginContext) -> Optional[PluginResult]: should_continue=True, ) + async def on_plugin_action(self, context: PluginContext) -> Optional[PluginResult]: + """Handle cross-plugin action calls (e.g., toggle lights from button press). + + Supported actions: + - toggle_lights / call_service: Call a Home Assistant service on resolved entities + Data keys: target_type, target, entity_type, service (all optional with defaults) + """ + action = context.data.get("action", "") + + if action not in ("toggle_lights", "call_service"): + return PluginResult( + success=False, + message=f"Unknown action: {action}", + ) + + if not self.mcp_client: + logger.error("MCP client not initialized for plugin action") + return PluginResult( + success=False, + message="Home Assistant is not connected", + ) + + try: + from .command_parser import ParsedCommand + + # Build a ParsedCommand from the action data + target_type = context.data.get("target_type", "area") + target = context.data.get("target", "") + entity_type = context.data.get("entity_type", "light") + service = context.data.get("service", "toggle") + + if not target: + return PluginResult(success=False, message="No target specified") + + parsed = ParsedCommand( + action=service, + target_type=target_type, + target=target, + entity_type=entity_type, + parameters={}, + ) + + # Resolve entities using existing cache-based resolution + entity_ids = await self._resolve_entities(parsed) + domain = entity_ids[0].split(".")[0] if entity_ids else entity_type + + # Call the service + logger.info( + f"Plugin action: {domain}.{service} for {len(entity_ids)} entities: {entity_ids}" + ) + result = await self.mcp_client.call_service( + domain=domain, service=service, entity_ids=entity_ids + ) + + message = ( + f"Called {domain}.{service} on {len(entity_ids)} " + f"{entity_type}{'s' if len(entity_ids) != 1 else ''} in {target}" + ) + logger.info(f"Plugin action executed: {message}") + + return PluginResult( + success=True, + data={ + "action": service, + "entity_ids": entity_ids, + "ha_result": result, + }, + message=message, + should_continue=True, + ) + + except ValueError as e: + logger.warning(f"Entity resolution failed for plugin action: {e}") + return PluginResult(success=False, message=str(e)) + except MCPError as e: + logger.error(f"HA API error during plugin action: {e}") + return PluginResult(success=False, message=f"Home Assistant error: {e}") + except Exception as e: + logger.error(f"Plugin action failed: {e}", exc_info=True) + return PluginResult(success=False, message=f"Action failed: {e}") + async def cleanup(self): """Clean up resources""" if self.mcp_client: diff --git a/plugins/test_button_actions/__init__.py b/plugins/test_button_actions/__init__.py new file mode 100644 index 00000000..947efccb --- /dev/null +++ b/plugins/test_button_actions/__init__.py @@ -0,0 +1,10 @@ +""" +Test Button Actions plugin for Chronicle. + +Maps device button events to configurable actions like closing conversations +or triggering cross-plugin calls. +""" + +from .plugin import TestButtonActionsPlugin + +__all__ = ['TestButtonActionsPlugin'] diff --git a/plugins/test_button_actions/config.yml b/plugins/test_button_actions/config.yml new file mode 100644 index 00000000..059a5881 --- /dev/null +++ b/plugins/test_button_actions/config.yml @@ -0,0 +1,3 @@ +actions: + single_press: + type: close_conversation diff --git a/plugins/test_button_actions/plugin.py b/plugins/test_button_actions/plugin.py new file mode 100644 index 00000000..29e086e5 --- /dev/null +++ b/plugins/test_button_actions/plugin.py @@ -0,0 +1,131 @@ +""" +Test Button Actions plugin — maps device button events to configurable actions. + +Single press: close conversation (triggers post-processing pipeline) +Double press: cross-plugin call (e.g., toggle study lights via Home Assistant) + +Actions are configured in config.yml with typed enums for safety. +""" + +import logging +from typing import Any, Dict, List, Optional + +from advanced_omi_backend.plugins.base import BasePlugin, PluginContext, PluginResult +from advanced_omi_backend.plugins.events import ButtonActionType, ConversationCloseReason, PluginEvent + +logger = logging.getLogger(__name__) + + +class TestButtonActionsPlugin(BasePlugin): + """Maps button press events to configurable system actions.""" + + SUPPORTED_ACCESS_LEVELS: List[str] = ["button"] + + name = "Test Button Actions" + description = "Map OMI device button presses to actions (close conversation, toggle lights, etc.)" + + def __init__(self, config: Dict[str, Any]): + super().__init__(config) + self.actions = config.get("actions", {}) + + async def initialize(self): + if not self.enabled: + logger.info("Test Button Actions plugin is disabled, skipping initialization") + return + logger.info( + f"Test Button Actions plugin initialized with actions: " + f"{list(self.actions.keys())}" + ) + + async def on_button_event(self, context: PluginContext) -> Optional[PluginResult]: + """Handle button events by dispatching configured actions.""" + event = context.event + + # Map plugin event to action config key + if event == PluginEvent.BUTTON_SINGLE_PRESS: + action_key = "single_press" + elif event == PluginEvent.BUTTON_DOUBLE_PRESS: + action_key = "double_press" + else: + logger.debug(f"No action mapping for event: {event}") + return None + + action_config = self.actions.get(action_key) + if not action_config: + logger.debug(f"No action configured for {action_key}") + return None + + try: + action_type = ButtonActionType(action_config.get("type", "")) + except ValueError: + logger.warning(f"Unknown action type: {action_config.get('type')}") + return PluginResult( + success=False, + message=f"Unknown action type: {action_config.get('type')}", + ) + + if action_type == ButtonActionType.CLOSE_CONVERSATION: + return await self._handle_close_conversation(context, action_config) + elif action_type == ButtonActionType.CALL_PLUGIN: + return await self._handle_call_plugin(context, action_config) + + return None + + async def _handle_close_conversation( + self, context: PluginContext, action_config: dict + ) -> PluginResult: + """Close the current conversation via PluginServices.""" + if not context.services: + logger.error("PluginServices not available in context") + return PluginResult(success=False, message="Services not available") + + session_id = context.data.get("session_id") + if not session_id: + logger.warning("No session_id in button event data, cannot close conversation") + return PluginResult(success=False, message="No active session") + + success = await context.services.close_conversation( + session_id=session_id, + reason=ConversationCloseReason.BUTTON_CLOSE, + ) + + if success: + logger.info(f"Button press closed conversation for session {session_id[:12]}") + return PluginResult( + success=True, + message="Conversation closed by button press", + should_continue=False, + ) + else: + logger.warning(f"Failed to close conversation for session {session_id[:12]}") + return PluginResult(success=False, message="Failed to close conversation") + + async def _handle_call_plugin( + self, context: PluginContext, action_config: dict + ) -> PluginResult: + """Dispatch action to another plugin via PluginServices.""" + if not context.services: + logger.error("PluginServices not available in context") + return PluginResult(success=False, message="Services not available") + + plugin_id = action_config.get("plugin_id") + action = action_config.get("action") + data = action_config.get("data", {}) + + if not plugin_id or not action: + logger.warning(f"call_plugin action missing plugin_id or action: {action_config}") + return PluginResult( + success=False, message="Invalid call_plugin configuration" + ) + + result = await context.services.call_plugin( + plugin_id=plugin_id, + action=action, + data=data, + user_id=context.user_id, + ) + + if result: + return result + + return PluginResult(success=False, message=f"No response from plugin '{plugin_id}'") diff --git a/backends/advanced/src/advanced_omi_backend/plugins/test_event/__init__.py b/plugins/test_event/__init__.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/test_event/__init__.py rename to plugins/test_event/__init__.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/test_event/config.yml b/plugins/test_event/config.yml similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/test_event/config.yml rename to plugins/test_event/config.yml diff --git a/backends/advanced/src/advanced_omi_backend/plugins/test_event/event_storage.py b/plugins/test_event/event_storage.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/test_event/event_storage.py rename to plugins/test_event/event_storage.py diff --git a/backends/advanced/src/advanced_omi_backend/plugins/test_event/plugin.py b/plugins/test_event/plugin.py similarity index 100% rename from backends/advanced/src/advanced_omi_backend/plugins/test_event/plugin.py rename to plugins/test_event/plugin.py