Refactor connect-omi.py for improved device selection and user intera…

backends/advanced/src/advanced_omi_backend/client.py

@@ -51,6 +51,9 @@ def __init__(
         # NOTE: Removed in-memory transcript storage for single source of truth
         # Transcripts are stored only in MongoDB via TranscriptionManager
 
+        # Markers (e.g., button events) collected during the session
+        self.markers: List[dict] = []
+
         # Track if conversation has been closed
         self.conversation_closed: bool = False
 
@@ -102,6 +105,10 @@ def update_transcript_received(self):
         """Update timestamp when transcript is received (for timeout detection)."""
         self.last_transcript_time = time.time()
 
+    def add_marker(self, marker: dict) -> None:
+        """Add a marker (e.g., button event) to the current session."""
+        self.markers.append(marker)
+
     def should_start_new_conversation(self) -> bool:
         """Check if we should start a new conversation based on timeout."""
         if self.last_transcript_time is None:

backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py

@@ -528,6 +528,14 @@ async def _finalize_streaming_session(
         # Mark session as finalizing with user_stopped reason (audio-stop event)
         await audio_stream_producer.finalize_session(session_id, completion_reason="user_stopped")
 
+        # Store markers in Redis so open_conversation_job can persist them
+        if client_state.markers:
+            session_key = f"audio:session:{session_id}"
+            await audio_stream_producer.redis_client.hset(
+                session_key, "markers", json.dumps(client_state.markers)
+            )
+            client_state.markers.clear()
+
         # NOTE: Finalize job disabled - open_conversation_job now handles everything
         # The open_conversation_job will:
         # 1. Detect the "finalizing" status
@@ -945,6 +953,56 @@ async def _handle_audio_session_stop(
     return False  # Switch back to control mode
 
 
+async def _handle_button_event(
+    client_state,
+    button_state: str,
+    user_id: str,
+    client_id: str,
+) -> None:
+    """Handle a button event from the device.
+
+    Stores a marker on the client state and dispatches to the plugin system.
+
+    Args:
+        client_state: Client state object
+        button_state: Button state string (e.g., "SINGLE_TAP", "DOUBLE_TAP")
+        user_id: User ID
+        client_id: Client ID
+    """
+    from advanced_omi_backend.services.plugin_service import get_plugin_router
+
+    timestamp = time.time()
+    audio_uuid = client_state.current_audio_uuid
+
+    application_logger.info(
+        f"🔘 Button event from {client_id}: {button_state} "
+        f"(audio_uuid={audio_uuid})"
+    )
+
+    # Store marker on client state for later persistence to conversation
+    marker = {
+        "type": "button_event",
+        "state": button_state,
+        "timestamp": timestamp,
+        "audio_uuid": audio_uuid,
+        "client_id": client_id,
+    }
+    client_state.add_marker(marker)
+
+    # Dispatch to plugin system
+    router = get_plugin_router()
+    if router:
+        await router.dispatch_event(
+            event="button.event",
+            user_id=user_id,
+            data={
+                "state": button_state,
+                "timestamp": timestamp,
+                "audio_uuid": audio_uuid,
+            },
+        )
+
+
 async def _process_rolling_batch(
     client_state,
     user_id: str,
@@ -1094,6 +1152,10 @@ async def _process_batch_audio_complete(
             title="Batch Recording",
             summary="Processing batch audio..."
         )
+        # Attach any markers (e.g., button events) captured during the session
+        if client_state.markers:
+            conversation.markers = list(client_state.markers)
+            client_state.markers.clear()
         await conversation.insert()
         conversation_id = conversation.conversation_id  # Get the auto-generated ID
 
@@ -1385,7 +1447,15 @@ async def handle_pcm_websocket(
                         # Handle keepalive ping from frontend
                         application_logger.debug(f"🏓 Received ping from {client_id}")
                         continue
-
+
+                    elif header["type"] == "button-event":
+                        button_data = header.get("data", {})
+                        button_state = button_data.get("state", "unknown")
+                        await _handle_button_event(
+                            client_state, button_state, user.user_id, client_id
+                        )
+                        continue
+
                     else:
                         # Unknown control message type
                         application_logger.debug(
@@ -1466,10 +1536,17 @@ async def handle_pcm_websocket(
                                     else:
                                         application_logger.warning(f"audio-chunk missing payload_length: {payload_length}")
                                     continue
+                                elif control_header.get("type") == "button-event":
+                                    button_data = control_header.get("data", {})
+                                    button_state = button_data.get("state", "unknown")
+                                    await _handle_button_event(
+                                        client_state, button_state, user.user_id, client_id
+                                    )
+                                    continue
                                 else:
                                     application_logger.warning(f"Unknown control message during streaming: {control_header.get('type')}")
                                     continue
-                                            
+
                             except json.JSONDecodeError:
                                 application_logger.warning(f"Invalid control message during streaming for {client_id}")
                                 continue

backends/advanced/src/advanced_omi_backend/models/conversation.py

@@ -122,6 +122,12 @@ class MemoryVersion(BaseModel):
         description="Compression ratio (compressed_size / original_size), typically ~0.047 for Opus"
     )
 
+    # Markers (e.g., button events) captured during the session
+    markers: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        description="Markers captured during audio session (button events, bookmarks, etc.)"
+    )
+
     # Creation metadata
     created_at: Indexed(datetime) = Field(default_factory=datetime.utcnow, description="When the conversation was created")
 

backends/advanced/src/advanced_omi_backend/plugins/base.py

@@ -143,3 +143,17 @@ async def on_memory_processed(self, context: PluginContext) -> Optional[PluginRe
             PluginResult with success status, optional message, and should_continue flag
         """
         pass
+
+    async def on_button_event(self, context: PluginContext) -> Optional[PluginResult]:
+        """
+        Called when a device button event is received.
+
+        Context data contains:
+            - state: str - Button state (e.g., "SINGLE_TAP", "DOUBLE_TAP", "LONG_PRESS")
+            - timestamp: float - Unix timestamp of the event
+            - audio_uuid: str - Current audio session UUID (may be None)
+
+        Returns:
+            PluginResult with success status, optional message, and should_continue flag
+        """
+        pass

backends/advanced/src/advanced_omi_backend/plugins/router.py

@@ -243,6 +243,8 @@ async def _execute_plugin(
             return await plugin.on_conversation_complete(context)
         elif event.startswith('memory.'):
             return await plugin.on_memory_processed(context)
+        elif event.startswith('button.'):
+            return await plugin.on_button_event(context)
 
         return None
 

backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py

@@ -5,6 +5,7 @@
 """
 
 import asyncio
+import json
 import logging
 import os
 import time
@@ -303,6 +304,17 @@ async def open_conversation_job(
         conversation_id = conversation.conversation_id
         logger.info(f"✅ Created streaming conversation {conversation_id} for session {session_id}")
 
+    # Attach markers from Redis session (e.g., button events captured during streaming)
+    markers_json = await redis_client.hget(session_key, "markers")
+    if markers_json:
+        try:
+            markers_data = markers_json if isinstance(markers_json, str) else markers_json.decode()
+            conversation.markers = json.loads(markers_data)
+            await conversation.save()
+            logger.info(f"📌 Attached {len(conversation.markers)} markers to conversation {conversation_id}")
+        except Exception as marker_err:
+            logger.warning(f"⚠️ Failed to parse markers from Redis: {marker_err}")
+
     # Link job metadata to conversation (cascading updates)
     current_job.meta["conversation_id"] = conversation_id
     current_job.save_meta()

extras/friend-lite-sdk/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Chronicle AI Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

extras/friend-lite-sdk/NOTICE

@@ -0,0 +1,7 @@
+This package is derived from the OMI Python SDK:
+  https://github.com/BasedHardware/omi/tree/main/sdks/python
+
+Original work: Copyright (c) 2024 Based Hardware Contributors
+Licensed under the MIT License.
+
+Fork: https://github.com/AnkushMalaker/Friend

extras/friend-lite-sdk/README.md

@@ -0,0 +1,31 @@
+# friend-lite-sdk
+
+Python SDK for OMI / Friend Lite BLE devices — audio streaming, button events, and transcription.
+
+Derived from the [OMI Python SDK](https://github.com/BasedHardware/omi/tree/main/sdks/python) (MIT license, Based Hardware Contributors). See `NOTICE` for attribution.
+
+## Installation
+
+```bash
+pip install -e extras/friend-lite-sdk
+```
+
+With optional transcription support:
+
+```bash
+pip install -e "extras/friend-lite-sdk[deepgram,wyoming]"
+```
+
+## Usage
+
+```python
+import asyncio
+from friend_lite import OmiConnection, ButtonState, parse_button_event
+
+async def main():
+    async with OmiConnection("AA:BB:CC:DD:EE:FF") as conn:
+        await conn.subscribe_audio(lambda _handle, data: print(len(data), "bytes"))
+        await conn.wait_until_disconnected()
+
+asyncio.run(main())
+```

extras/friend-lite-sdk/friend_lite/__init__.py

@@ -0,0 +1,18 @@
+from .bluetooth import OmiConnection, listen_to_omi, print_devices
+from .button import ButtonState, parse_button_event
+from .uuids import (
+    OMI_AUDIO_CHAR_UUID,
+    OMI_BUTTON_CHAR_UUID,
+    OMI_BUTTON_SERVICE_UUID,
+)
+
+__all__ = [
+    "ButtonState",
+    "OMI_AUDIO_CHAR_UUID",
+    "OMI_BUTTON_CHAR_UUID",
+    "OMI_BUTTON_SERVICE_UUID",
+    "OmiConnection",
+    "listen_to_omi",
+    "parse_button_event",
+    "print_devices",
+]

extras/friend-lite-sdk/friend_lite/bluetooth.py

@@ -0,0 +1,70 @@
+import asyncio
+from typing import Callable, Optional
+
+from bleak import BleakClient, BleakScanner
+
+from .uuids import OMI_AUDIO_CHAR_UUID, OMI_BUTTON_CHAR_UUID
+
+
+def print_devices() -> None:
+    devices = asyncio.run(BleakScanner.discover())
+    for i, d in enumerate(devices):
+        print(f"{i}. {d.name} [{d.address}]")
+
+
+class OmiConnection:
+    def __init__(self, mac_address: str) -> None:
+        self._mac_address = mac_address
+        self._client: Optional[BleakClient] = None
+        self._disconnected = asyncio.Event()
+
+    async def __aenter__(self) -> "OmiConnection":
+        await self.connect()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.disconnect()
+
+    async def connect(self) -> None:
+        if self._client is not None:
+            return
+
+        def _on_disconnect(_client: BleakClient) -> None:
+            self._disconnected.set()
+
+        self._client = BleakClient(
+            self._mac_address,
+            disconnected_callback=_on_disconnect,
+        )
+        await self._client.connect()
+
+    async def disconnect(self) -> None:
+        if self._client is None:
+            return
+        await self._client.disconnect()
+        self._client = None
+        self._disconnected.set()
+
+    async def subscribe_audio(self, callback: Callable[[int, bytearray], None]) -> None:
+        await self.subscribe(OMI_AUDIO_CHAR_UUID, callback)
+
+    async def subscribe_button(self, callback: Callable[[int, bytearray], None]) -> None:
+        await self.subscribe(OMI_BUTTON_CHAR_UUID, callback)
+
+    async def subscribe(self, uuid: str, callback: Callable[[int, bytearray], None]) -> None:
+        if self._client is None:
+            raise RuntimeError("Not connected to OMI device")
+        await self._client.start_notify(uuid, callback)
+
+    async def wait_until_disconnected(self, timeout: float | None = None) -> None:
+        if timeout is None:
+            await self._disconnected.wait()
+        else:
+            await asyncio.wait_for(self._disconnected.wait(), timeout=timeout)
+
+
+async def listen_to_omi(mac_address: str, char_uuid: str, data_handler) -> None:
+    """Backward-compatible wrapper for older consumers."""
+    async with OmiConnection(mac_address) as conn:
+        await conn.subscribe(char_uuid, data_handler)
+        await conn.wait_until_disconnected()

extras/friend-lite-sdk/friend_lite/button.py

@@ -0,0 +1,24 @@
+"""Button event parsing for Omi BLE button characteristic."""
+
+import struct
+from enum import IntEnum
+
+
+class ButtonState(IntEnum):
+    IDLE = 0
+    SINGLE_TAP = 1
+    DOUBLE_TAP = 2
+    LONG_PRESS = 3
+    PRESS = 4
+    RELEASE = 5
+
+
+def parse_button_event(data: bytes) -> ButtonState:
+    """Parse the button event payload into a ButtonState.
+
+    Payload is two little-endian uint32 values: [state, 0].
+    """
+    if len(data) < 8:
+        raise ValueError(f"Expected 8 bytes for button event, got {len(data)}")
+    state, _unused = struct.unpack("<II", data[:8])
+    return ButtonState(state)