From 44b3b7fbc6d6e3b0e8e9a112ed43e3cb091f7bf2 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sat, 7 Feb 2026 13:35:34 +0530 Subject: [PATCH 1/3] Refactor connect-omi.py for improved device selection and user interaction - Replaced references to the chronicle Bluetooth library with friend_lite for device management. - Removed the list_devices function and implemented a new prompt_user_to_pick_device function to enhance user interaction when selecting OMI/Neo devices. - Updated the find_and_set_omi_mac function to utilize the new device selection method, improving the overall flow of device connection. - Added a new scan_devices.py script for quick scanning of neo/neosapien devices, enhancing usability. - Updated README.md to reflect new usage instructions and prerequisites for connecting to OMI devices over Bluetooth. - Enhanced start.sh to ensure proper environment variable setup for macOS users. --- extras/local-omi-bt/README.md | 42 ++++++++++++- extras/local-omi-bt/connect-omi.py | 92 ++++++++++++++++++++++------- extras/local-omi-bt/scan_devices.py | 60 +++++++++++++++++++ extras/local-omi-bt/start.sh | 10 ++++ 4 files changed, 180 insertions(+), 24 deletions(-) create mode 100644 extras/local-omi-bt/scan_devices.py diff --git a/extras/local-omi-bt/README.md b/extras/local-omi-bt/README.md index e9b34820..3a61352e 100644 --- a/extras/local-omi-bt/README.md +++ b/extras/local-omi-bt/README.md @@ -1,3 +1,39 @@ -# Usage -Run using `uv run --with-requirements requirements.txt python connect-omi.py` -from this directory. \ No newline at end of file +# Local OMI BT + +Connect to an OMI device over Bluetooth and stream audio to the Chronicle backend. + +## Prerequisites + +- **Python 3.12+** (managed via `uv`) +- **Opus codec library** (required by `opuslib`) + +### Installing Opus + +**macOS (Homebrew):** +```bash +brew install opus +``` + +**Linux (Debian/Ubuntu):** +```bash +sudo apt install libopus-dev +``` + +## Usage + +```bash +./start.sh +``` + +Or run directly: +```bash +uv run --with-requirements requirements.txt python connect-omi.py +``` + +### macOS: Opus library not found + +If you see `Could not find Opus library`, you need to tell the dynamic linker where to find it. The `start.sh` script handles this automatically, but if running manually: + +```bash +DYLD_LIBRARY_PATH="$(brew --prefix opus)/lib" uv run --with-requirements requirements.txt python connect-omi.py +``` diff --git a/extras/local-omi-bt/connect-omi.py b/extras/local-omi-bt/connect-omi.py index 302a17d7..9b1bcaeb 100644 --- a/extras/local-omi-bt/connect-omi.py +++ b/extras/local-omi-bt/connect-omi.py @@ -7,11 +7,10 @@ import asyncstdlib as asyncstd from bleak import BleakClient, BleakScanner -from bleak.backends.device import BLEDevice from dotenv import load_dotenv, set_key from easy_audio_interfaces.filesystem import RollingFileSink -from chronicle.bluetooth import listen_to_omi, print_devices -from chronicle.decoder import OmiOpusDecoder +from friend_lite.bluetooth import listen_to_omi +from friend_lite.decoder import OmiOpusDecoder from wyoming.audio import AudioChunk # Setup logging @@ -49,16 +48,6 @@ async def as_audio_chunks(it) -> AsyncGenerator[AudioChunk, None]: async for data in it: yield AudioChunk(audio=data, rate=16000, width=2, channels=1) -# Add this to chronicle sdk -async def list_devices(prefix: str = "OMI") -> list[BLEDevice]: - devices = await BleakScanner.discover() - filtered_devices = [] - for d in devices: - if d.name: - if prefix.casefold() in d.name.casefold(): - filtered_devices.append(d) - return filtered_devices - def main() -> None: # api_key: str | None = os.getenv("DEEPGRAM_API_KEY") @@ -78,13 +67,62 @@ def handle_ble_data(sender: Any, data: bytes) -> None: logger.error("Queue Error: %s", e) + def prompt_user_to_pick_device(all_devices) -> str | None: + """Interactively prompt the user to select an OMI/Neo device from scan results. + + Returns the selected MAC address string, or None if no selection was made. + Saves the selected MAC to .env via set_key(). + """ + omi_devices = [ + d for d in all_devices + if d.name and ("omi" in d.name.casefold() or "neo" in d.name.casefold()) + ] + + if not omi_devices: + logger.info("No OMI/Neo devices found. All discovered BLE devices:") + if all_devices: + for i, d in enumerate(all_devices): + logger.info(" %d. %s [%s]", i + 1, d.name or "(unnamed)", d.address) + else: + logger.info(" (no BLE devices found at all)") + return None + + if len(omi_devices) == 1: + device = omi_devices[0] + answer = input(f"Found {device.name} [{device.address}]. Use this device? [Y/n] ").strip().lower() + if answer in ("", "y", "yes"): + set_key(env_path, "OMI_MAC", device.address) + logger.info("OMI_MAC set to %s and saved to .env", device.address) + return device.address + return None + + # Multiple OMI/Neo devices found + logger.info("Multiple OMI/Neo devices found:") + for i, d in enumerate(omi_devices): + logger.info(" %d. %s [%s]", i + 1, d.name, d.address) + choice = input("Enter number to select (or q to quit): ").strip().lower() + if choice == "q": + return None + try: + idx = int(choice) - 1 + if 0 <= idx < len(omi_devices): + selected = omi_devices[idx] + set_key(env_path, "OMI_MAC", selected.address) + logger.info("OMI_MAC set to %s and saved to .env", selected.address) + return selected.address + else: + logger.error("Invalid selection: %s", choice) + return None + except ValueError: + logger.error("Invalid input: %s", choice) + return None + async def find_and_set_omi_mac() -> str: - devices = await list_devices() - assert len(devices) == 1, "Expected 1 Omi device, got %d" % len(devices) - discovered_mac = devices[0].address - set_key(env_path, "OMI_MAC", discovered_mac) - logger.info("OMI_MAC set to %s and saved to .env" % discovered_mac) - return discovered_mac + all_devices = await BleakScanner.discover() + selected = prompt_user_to_pick_device(all_devices) + if not selected: + raise SystemExit(1) + return selected async def run() -> None: logger.info("Starting OMI Bluetooth connection and audio streaming") @@ -101,8 +139,20 @@ async def run() -> None: logger.info(f"Successfully connected to device {mac_address}") except Exception as e: logger.error(f"Failed to connect to device {mac_address}: {e}") - logger.error("Exiting without creating audio sink or backend connection") - return + logger.info("Scanning for nearby BLE devices...") + all_devices = await BleakScanner.discover() + selected = prompt_user_to_pick_device(all_devices) + if not selected: + return + mac_address = selected + # Verify the newly selected device is reachable + logger.info("Connecting to newly selected device %s...", mac_address) + try: + async with BleakClient(mac_address) as test_client: + logger.info(f"Successfully connected to device {mac_address}") + except Exception as e2: + logger.error(f"Failed to connect to newly selected device {mac_address}: {e2}") + return # Device is available, now setup audio sink and backend connection logger.info("Device found and connected, setting up audio pipeline...") diff --git a/extras/local-omi-bt/scan_devices.py b/extras/local-omi-bt/scan_devices.py new file mode 100644 index 00000000..f00cd278 --- /dev/null +++ b/extras/local-omi-bt/scan_devices.py @@ -0,0 +1,60 @@ +"""Quick BLE scanner to find neo1/neosapien devices.""" +import asyncio +from bleak import BleakScanner, BleakClient + +async def scan_all_devices(): + """Scan for all BLE devices.""" + print("Scanning for BLE devices (10 seconds)...") + devices = await BleakScanner.discover(timeout=10.0) + + print(f"\nFound {len(devices)} devices:\n") + + neo_devices = [] + for d in sorted(devices, key=lambda x: x.name or ""): + name = d.name or "(no name)" + print(f" {name:<30} | {d.address}") + + # Look for neo/neosapien devices + if d.name and any(x in d.name.lower() for x in ["neo", "sapien"]): + neo_devices.append(d) + + return neo_devices + +async def explore_device(address: str): + """Connect to a device and list its services/characteristics.""" + print(f"\nConnecting to {address}...") + try: + async with BleakClient(address, timeout=20.0) as client: + print(f"Connected: {client.is_connected}") + print("\nServices and Characteristics:") + + for service in client.services: + print(f"\n Service: {service.uuid}") + print(f" Description: {service.description}") + + for char in service.characteristics: + props = ", ".join(char.properties) + print(f" Char: {char.uuid}") + print(f" Properties: {props}") + print(f" Handle: {char.handle}") + except Exception as e: + print(f"Error connecting: {e}") + +async def main(): + neo_devices = await scan_all_devices() + + if neo_devices: + print(f"\n{'='*60}") + print(f"Found {len(neo_devices)} neo/neosapien device(s)!") + for d in neo_devices: + print(f" - {d.name}: {d.address}") + + # Explore the first neo device + print(f"\n{'='*60}") + await explore_device(neo_devices[0].address) + else: + print("\nNo neo/neosapien devices found.") + print("Make sure the device is powered on and in pairing mode.") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/extras/local-omi-bt/start.sh b/extras/local-omi-bt/start.sh index 6fd8947e..381a2c78 100755 --- a/extras/local-omi-bt/start.sh +++ b/extras/local-omi-bt/start.sh @@ -1,2 +1,12 @@ #!/bin/bash + +# macOS: opuslib needs the Opus shared library on the dynamic linker path. +# Install with: brew install opus +if [ "$(uname)" = "Darwin" ] && command -v brew &>/dev/null; then + OPUS_PREFIX="$(brew --prefix opus 2>/dev/null)" + if [ -d "$OPUS_PREFIX/lib" ]; then + export DYLD_LIBRARY_PATH="${OPUS_PREFIX}/lib${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" + fi +fi + uv run --with-requirements requirements.txt python connect-omi.py From 3a5b242a6b66c2a8fe596817bf20b676d2420f42 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sun, 8 Feb 2026 21:11:30 +0530 Subject: [PATCH 2/3] Add friend-lite-sdk: Initial implementation of Python SDK for OMI/Friend Lite BLE devices - Introduced the friend-lite-sdk, a Python SDK for OMI/Friend Lite BLE devices, enabling audio streaming, button events, and transcription functionalities. - Added LICENSE and NOTICE files to clarify licensing and attribution. - Created pyproject.toml for package management, specifying dependencies and project metadata. - Developed core modules including bluetooth connection handling, button event parsing, audio decoding, and transcription capabilities. - Implemented example usage in README.md to guide users on installation and basic functionality. - Enhanced connect-omi.py to utilize the new SDK for improved device management and event handling. - Updated requirements.txt to reference the new SDK for local development. This commit lays the foundation for further enhancements and integrations with OMI devices. --- .../controllers/websocket_controller.py | 81 +++++- .../models/conversation.py | 6 + extras/friend-lite-sdk/LICENSE | 21 ++ extras/friend-lite-sdk/NOTICE | 7 + extras/friend-lite-sdk/README.md | 31 +++ .../friend-lite-sdk/friend_lite/__init__.py | 18 ++ .../friend-lite-sdk/friend_lite/bluetooth.py | 70 ++++++ extras/friend-lite-sdk/friend_lite/button.py | 24 ++ extras/friend-lite-sdk/friend_lite/decoder.py | 24 ++ .../friend_lite/discover_characteristics.py | 19 ++ extras/friend-lite-sdk/friend_lite/py.typed | 0 .../friend-lite-sdk/friend_lite/transcribe.py | 235 ++++++++++++++++++ extras/friend-lite-sdk/friend_lite/uuids.py | 8 + extras/friend-lite-sdk/pyproject.toml | 28 +++ extras/local-omi-bt/connect-omi.py | 34 ++- extras/local-omi-bt/requirements.txt | 2 +- extras/local-omi-bt/send_to_adv.py | 27 ++ 17 files changed, 622 insertions(+), 13 deletions(-) create mode 100644 extras/friend-lite-sdk/LICENSE create mode 100644 extras/friend-lite-sdk/NOTICE create mode 100644 extras/friend-lite-sdk/README.md create mode 100644 extras/friend-lite-sdk/friend_lite/__init__.py create mode 100644 extras/friend-lite-sdk/friend_lite/bluetooth.py create mode 100644 extras/friend-lite-sdk/friend_lite/button.py create mode 100644 extras/friend-lite-sdk/friend_lite/decoder.py create mode 100644 extras/friend-lite-sdk/friend_lite/discover_characteristics.py create mode 100644 extras/friend-lite-sdk/friend_lite/py.typed create mode 100644 extras/friend-lite-sdk/friend_lite/transcribe.py create mode 100644 extras/friend-lite-sdk/friend_lite/uuids.py create mode 100644 extras/friend-lite-sdk/pyproject.toml diff --git a/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py index fcf80de4..f301bed7 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py @@ -528,6 +528,14 @@ async def _finalize_streaming_session( # Mark session as finalizing with user_stopped reason (audio-stop event) await audio_stream_producer.finalize_session(session_id, completion_reason="user_stopped") + # Store markers in Redis so open_conversation_job can persist them + if client_state.markers: + session_key = f"audio:session:{session_id}" + await audio_stream_producer.redis_client.hset( + session_key, "markers", json.dumps(client_state.markers) + ) + client_state.markers.clear() + # NOTE: Finalize job disabled - open_conversation_job now handles everything # The open_conversation_job will: # 1. Detect the "finalizing" status @@ -945,6 +953,56 @@ async def _handle_audio_session_stop( return False # Switch back to control mode +async def _handle_button_event( + client_state, + button_state: str, + user_id: str, + client_id: str, +) -> None: + """Handle a button event from the device. + + Stores a marker on the client state and dispatches to the plugin system. + + Args: + client_state: Client state object + button_state: Button state string (e.g., "SINGLE_TAP", "DOUBLE_TAP") + user_id: User ID + client_id: Client ID + """ + from advanced_omi_backend.services.plugin_service import get_plugin_router + + timestamp = time.time() + audio_uuid = client_state.current_audio_uuid + + application_logger.info( + f"🔘 Button event from {client_id}: {button_state} " + f"(audio_uuid={audio_uuid})" + ) + + # Store marker on client state for later persistence to conversation + marker = { + "type": "button_event", + "state": button_state, + "timestamp": timestamp, + "audio_uuid": audio_uuid, + "client_id": client_id, + } + client_state.add_marker(marker) + + # Dispatch to plugin system + router = get_plugin_router() + if router: + await router.dispatch_event( + event="button.event", + user_id=user_id, + data={ + "state": button_state, + "timestamp": timestamp, + "audio_uuid": audio_uuid, + }, + ) + + async def _process_rolling_batch( client_state, user_id: str, @@ -1094,6 +1152,10 @@ async def _process_batch_audio_complete( title="Batch Recording", summary="Processing batch audio..." ) + # Attach any markers (e.g., button events) captured during the session + if client_state.markers: + conversation.markers = list(client_state.markers) + client_state.markers.clear() await conversation.insert() conversation_id = conversation.conversation_id # Get the auto-generated ID @@ -1385,7 +1447,15 @@ async def handle_pcm_websocket( # Handle keepalive ping from frontend application_logger.debug(f"🏓 Received ping from {client_id}") continue - + + elif header["type"] == "button-event": + button_data = header.get("data", {}) + button_state = button_data.get("state", "unknown") + await _handle_button_event( + client_state, button_state, user.user_id, client_id + ) + continue + else: # Unknown control message type application_logger.debug( @@ -1466,10 +1536,17 @@ async def handle_pcm_websocket( else: application_logger.warning(f"audio-chunk missing payload_length: {payload_length}") continue + elif control_header.get("type") == "button-event": + button_data = control_header.get("data", {}) + button_state = button_data.get("state", "unknown") + await _handle_button_event( + client_state, button_state, user.user_id, client_id + ) + continue else: application_logger.warning(f"Unknown control message during streaming: {control_header.get('type')}") continue - + except json.JSONDecodeError: application_logger.warning(f"Invalid control message during streaming for {client_id}") continue diff --git a/backends/advanced/src/advanced_omi_backend/models/conversation.py b/backends/advanced/src/advanced_omi_backend/models/conversation.py index 2ec45f33..ffbb8260 100644 --- a/backends/advanced/src/advanced_omi_backend/models/conversation.py +++ b/backends/advanced/src/advanced_omi_backend/models/conversation.py @@ -122,6 +122,12 @@ class MemoryVersion(BaseModel): description="Compression ratio (compressed_size / original_size), typically ~0.047 for Opus" ) + # Markers (e.g., button events) captured during the session + markers: List[Dict[str, Any]] = Field( + default_factory=list, + description="Markers captured during audio session (button events, bookmarks, etc.)" + ) + # Creation metadata created_at: Indexed(datetime) = Field(default_factory=datetime.utcnow, description="When the conversation was created") diff --git a/extras/friend-lite-sdk/LICENSE b/extras/friend-lite-sdk/LICENSE new file mode 100644 index 00000000..4130f88b --- /dev/null +++ b/extras/friend-lite-sdk/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Chronicle AI Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/extras/friend-lite-sdk/NOTICE b/extras/friend-lite-sdk/NOTICE new file mode 100644 index 00000000..786c9e49 --- /dev/null +++ b/extras/friend-lite-sdk/NOTICE @@ -0,0 +1,7 @@ +This package is derived from the OMI Python SDK: + https://github.com/BasedHardware/omi/tree/main/sdks/python + +Original work: Copyright (c) 2024 Based Hardware Contributors +Licensed under the MIT License. + +Fork: https://github.com/AnkushMalaker/Friend diff --git a/extras/friend-lite-sdk/README.md b/extras/friend-lite-sdk/README.md new file mode 100644 index 00000000..89735516 --- /dev/null +++ b/extras/friend-lite-sdk/README.md @@ -0,0 +1,31 @@ +# friend-lite-sdk + +Python SDK for OMI / Friend Lite BLE devices — audio streaming, button events, and transcription. + +Derived from the [OMI Python SDK](https://github.com/BasedHardware/omi/tree/main/sdks/python) (MIT license, Based Hardware Contributors). See `NOTICE` for attribution. + +## Installation + +```bash +pip install -e extras/friend-lite-sdk +``` + +With optional transcription support: + +```bash +pip install -e "extras/friend-lite-sdk[deepgram,wyoming]" +``` + +## Usage + +```python +import asyncio +from friend_lite import OmiConnection, ButtonState, parse_button_event + +async def main(): + async with OmiConnection("AA:BB:CC:DD:EE:FF") as conn: + await conn.subscribe_audio(lambda _handle, data: print(len(data), "bytes")) + await conn.wait_until_disconnected() + +asyncio.run(main()) +``` diff --git a/extras/friend-lite-sdk/friend_lite/__init__.py b/extras/friend-lite-sdk/friend_lite/__init__.py new file mode 100644 index 00000000..6292b3eb --- /dev/null +++ b/extras/friend-lite-sdk/friend_lite/__init__.py @@ -0,0 +1,18 @@ +from .bluetooth import OmiConnection, listen_to_omi, print_devices +from .button import ButtonState, parse_button_event +from .uuids import ( + OMI_AUDIO_CHAR_UUID, + OMI_BUTTON_CHAR_UUID, + OMI_BUTTON_SERVICE_UUID, +) + +__all__ = [ + "ButtonState", + "OMI_AUDIO_CHAR_UUID", + "OMI_BUTTON_CHAR_UUID", + "OMI_BUTTON_SERVICE_UUID", + "OmiConnection", + "listen_to_omi", + "parse_button_event", + "print_devices", +] diff --git a/extras/friend-lite-sdk/friend_lite/bluetooth.py b/extras/friend-lite-sdk/friend_lite/bluetooth.py new file mode 100644 index 00000000..ce7ea505 --- /dev/null +++ b/extras/friend-lite-sdk/friend_lite/bluetooth.py @@ -0,0 +1,70 @@ +import asyncio +from typing import Callable, Optional + +from bleak import BleakClient, BleakScanner + +from .uuids import OMI_AUDIO_CHAR_UUID, OMI_BUTTON_CHAR_UUID + + +def print_devices() -> None: + devices = asyncio.run(BleakScanner.discover()) + for i, d in enumerate(devices): + print(f"{i}. {d.name} [{d.address}]") + + +class OmiConnection: + def __init__(self, mac_address: str) -> None: + self._mac_address = mac_address + self._client: Optional[BleakClient] = None + self._disconnected = asyncio.Event() + + async def __aenter__(self) -> "OmiConnection": + await self.connect() + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + await self.disconnect() + + async def connect(self) -> None: + if self._client is not None: + return + + def _on_disconnect(_client: BleakClient) -> None: + self._disconnected.set() + + self._client = BleakClient( + self._mac_address, + disconnected_callback=_on_disconnect, + ) + await self._client.connect() + + async def disconnect(self) -> None: + if self._client is None: + return + await self._client.disconnect() + self._client = None + self._disconnected.set() + + async def subscribe_audio(self, callback: Callable[[int, bytearray], None]) -> None: + await self.subscribe(OMI_AUDIO_CHAR_UUID, callback) + + async def subscribe_button(self, callback: Callable[[int, bytearray], None]) -> None: + await self.subscribe(OMI_BUTTON_CHAR_UUID, callback) + + async def subscribe(self, uuid: str, callback: Callable[[int, bytearray], None]) -> None: + if self._client is None: + raise RuntimeError("Not connected to OMI device") + await self._client.start_notify(uuid, callback) + + async def wait_until_disconnected(self, timeout: float | None = None) -> None: + if timeout is None: + await self._disconnected.wait() + else: + await asyncio.wait_for(self._disconnected.wait(), timeout=timeout) + + +async def listen_to_omi(mac_address: str, char_uuid: str, data_handler) -> None: + """Backward-compatible wrapper for older consumers.""" + async with OmiConnection(mac_address) as conn: + await conn.subscribe(char_uuid, data_handler) + await conn.wait_until_disconnected() diff --git a/extras/friend-lite-sdk/friend_lite/button.py b/extras/friend-lite-sdk/friend_lite/button.py new file mode 100644 index 00000000..421a87b1 --- /dev/null +++ b/extras/friend-lite-sdk/friend_lite/button.py @@ -0,0 +1,24 @@ +"""Button event parsing for Omi BLE button characteristic.""" + +import struct +from enum import IntEnum + + +class ButtonState(IntEnum): + IDLE = 0 + SINGLE_TAP = 1 + DOUBLE_TAP = 2 + LONG_PRESS = 3 + PRESS = 4 + RELEASE = 5 + + +def parse_button_event(data: bytes) -> ButtonState: + """Parse the button event payload into a ButtonState. + + Payload is two little-endian uint32 values: [state, 0]. + """ + if len(data) < 8: + raise ValueError(f"Expected 8 bytes for button event, got {len(data)}") + state, _unused = struct.unpack("= WINDOW_SECONDS: + logger.debug(f"Window time ({WINDOW_SECONDS}s) elapsed.") + break + + try: + # Calculate remaining time in window for timeout + timeout = max(0.1, WINDOW_SECONDS - time_elapsed) # Use small minimum timeout + # Get chunk with timeout to prevent blocking indefinitely if queue is empty + chunk = await asyncio.wait_for(audio_queue.get(), timeout=timeout) + + if chunk is None: # Handle queue termination signal + logger.info("Audio queue finished during chunk sending. Exiting.") + # No need to send AudioStop if queue ended before window completion + return # Exit the entire function if queue is done + + # Send audio chunk + logger.debug(f"Wyoming: Sending AudioChunk ({len(chunk)} bytes)...") + await client.write_event(AudioChunk(audio=chunk, rate=SAMPLE_RATE, width=SAMPLE_WIDTH, channels=CHANNELS).event()) + segment_has_audio = True # Mark that we sent audio in this segment + logger.debug("Wyoming: AudioChunk sent.") + + except asyncio.TimeoutError: + # Expected if no audio comes within the window's remaining time + logger.debug("Timeout waiting for audio chunk, window likely finished.") + break # Exit chunk sending loop + except asyncio.CancelledError: + logger.info("Chunk sending task cancelled.") + raise # Re-raise cancellation + except Exception as e: + logger.error(f"Error getting/sending audio chunk: {e}", exc_info=True) + raise # Re-raise other exceptions to break segment processing + + # 5. Stop Audio Segment (only if audio was sent in this window) + if segment_has_audio: + logger.debug(f"Wyoming: Sending AudioStop...") + await client.write_event(AudioStop().event()) + logger.debug("Wyoming: AudioStop sent.") + + # 6. Read Transcript for the Segment + logger.debug(f"Wyoming: Reading events for transcript (timeout={READ_TIMEOUT_SECONDS}s)...") + try: + while True: # Loop to read events until transcript or timeout/error + event = await asyncio.wait_for(client.read_event(), timeout=READ_TIMEOUT_SECONDS) + + if event is None: + logger.warning("Wyoming connection closed by server unexpectedly while waiting for transcript.") + # Server might close connection after sending transcript/error, treat as end of segment read + break + + logger.debug(f"Wyoming: Received event raw: {event}") # DEBUG + logger.debug(f"Wyoming: Received event type: {type(event)}") # DEBUG + if hasattr(event, 'data'): + logger.debug(f"Wyoming: Received event data: {event.data}") # DEBUG + + # Check for transcription event + if isinstance(event, Event) and event.type == 'transcript' and 'text' in event.data: + transcript = event.data['text'] + if transcript and transcript.strip(): + logger.info(f"Transcript: {transcript.strip()}") + # Assume one transcript per segment, break after receiving it + logger.debug("Breaking read loop after receiving transcript.") + break + elif isinstance(event, Event) and event.type == 'error': + logger.error(f"Wyoming server error event: {event.data}") + # Break on error, segment finished (with error) + break + else: + logger.debug(f"Received non-transcript/non-error event: type={type(event)}") + # Continue reading other events until transcript/error or timeout + + except asyncio.TimeoutError: + logger.warning(f"Timeout waiting for transcript after {READ_TIMEOUT_SECONDS}s.") + # Continue to the next segment even if no transcript was received + except (ConnectionClosedOK, ConnectionClosedError, ConnectionResetError) as close_err: + logger.info(f"Wyoming connection closed gracefully/expectedly after sending audio: {close_err}") + # This is expected if the server closes after sending the transcript/error + except asyncio.CancelledError: + logger.info("Transcript reading task cancelled.") + raise # Re-raise cancellation + except Exception as e: + logger.error(f"Error reading transcript event: {e}", exc_info=True) + # Depending on the error, might want to raise or just log and continue + # For now, log and continue to the finally block/next segment + else: + logger.info("Skipping AudioStop and transcript read as no audio was sent in this window.") + + except (ConnectionRefusedError, ConnectionResetError, ConnectionError, WebSocketException) as conn_err: + logger.error(f"Wyoming connection/websocket error during segment: {conn_err}") + # Connection error for a segment, wait before retrying the *next* segment + await asyncio.sleep(5) + except asyncio.CancelledError: + logger.info("Main transcription task cancelled during segment processing.") + break # Exit outer loop if cancelled + except Exception as e: + logger.error(f"Unexpected error during segment processing: {e}", exc_info=True) + # Log unexpected error and wait before next segment attempt + await asyncio.sleep(5) + finally: + if client: # Check if client was successfully created + logger.info("Attempting to disconnect from Wyoming server for this segment...") + with suppress(Exception): # Suppress errors during cleanup disconnect + await client.disconnect() + logger.info("Disconnected from Wyoming server for this segment.") + client = None # Ensure client is reset for the next loop iteration + + # Check if the task was cancelled before potentially sleeping/looping + task = asyncio.current_task() + if task and task.cancelled(): + logger.info("Task cancelled, exiting transcribe_wyoming loop.") + break + + logger.info("Exiting transcribe_wyoming function.") + +# Ensure logging is configured if this module is run directly or imported early +# logging.basicConfig(level=logging.DEBUG) diff --git a/extras/friend-lite-sdk/friend_lite/uuids.py b/extras/friend-lite-sdk/friend_lite/uuids.py new file mode 100644 index 00000000..452a2416 --- /dev/null +++ b/extras/friend-lite-sdk/friend_lite/uuids.py @@ -0,0 +1,8 @@ +"""UUID constants for OMI BLE services and characteristics.""" + +# Standard Omi audio characteristic UUID +OMI_AUDIO_CHAR_UUID = "19B10001-E8F2-537E-4F6C-D104768A1214" + +# Omi button service + characteristic UUIDs +OMI_BUTTON_SERVICE_UUID = "23BA7924-0000-1000-7450-346EAC492E92" +OMI_BUTTON_CHAR_UUID = "23BA7925-0000-1000-7450-346EAC492E92" diff --git a/extras/friend-lite-sdk/pyproject.toml b/extras/friend-lite-sdk/pyproject.toml new file mode 100644 index 00000000..7e11d3af --- /dev/null +++ b/extras/friend-lite-sdk/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "friend-lite-sdk" +version = "0.2.0" +description = "Python SDK for OMI/Friend Lite BLE devices — audio streaming, button events, and transcription" +requires-python = ">= 3.10" +license = "MIT" +dependencies = [ + "bleak>=0.22.3", + "numpy>=1.26", + "opuslib>=3.0.1", + "websockets>=14.0.0", +] + +[project.urls] +Homepage = "https://github.com/AnkushMalaker/chronicle" +"Original Project" = "https://github.com/BasedHardware/omi" + +[project.optional-dependencies] +deepgram = ["deepgram-sdk>=3.11.0"] +wyoming = ["wyoming"] +dev = ["mypy>=1.15.0"] + +[build-system] +requires = ["setuptools >= 80.0.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["friend_lite*"] diff --git a/extras/local-omi-bt/connect-omi.py b/extras/local-omi-bt/connect-omi.py index 9b1bcaeb..474b74bf 100644 --- a/extras/local-omi-bt/connect-omi.py +++ b/extras/local-omi-bt/connect-omi.py @@ -9,7 +9,7 @@ from bleak import BleakClient, BleakScanner from dotenv import load_dotenv, set_key from easy_audio_interfaces.filesystem import RollingFileSink -from friend_lite.bluetooth import listen_to_omi +from friend_lite import ButtonState, OmiConnection, parse_button_event from friend_lite.decoder import OmiOpusDecoder from wyoming.audio import AudioChunk @@ -24,15 +24,12 @@ load_dotenv(env_path) sys.path.append(os.path.dirname(__file__)) -from send_to_adv import stream_to_backend +from send_to_adv import send_button_event, stream_to_backend OMI_MAC = os.getenv("OMI_MAC") if not OMI_MAC: logger.info("OMI_MAC not found in .env. Will try to find and set.") -# Standard Omi audio characteristic UUID -OMI_CHAR_UUID = "19B10001-E8F2-537E-4F6C-D104768A1214" - async def source_bytes(audio_queue: Queue[bytes]) -> AsyncGenerator[bytes, None]: """Single source iterator from the queue.""" while True: @@ -65,6 +62,20 @@ def handle_ble_data(sender: Any, data: bytes) -> None: audio_queue.put_nowait(decoded_pcm) except Exception as e: logger.error("Queue Error: %s", e) + + def handle_button_event(sender: Any, data: bytes) -> None: + try: + state = parse_button_event(data) + except Exception as e: + logger.error("Button event parse error: %s", e) + return + if state != ButtonState.IDLE: + logger.info("Button event: %s", state.name) + try: + loop = asyncio.get_running_loop() + loop.create_task(send_button_event(state.name)) + except RuntimeError: + logger.debug("No running event loop, cannot send button event") def prompt_user_to_pick_device(all_devices) -> str | None: @@ -198,11 +209,14 @@ async def queue_to_stream(): async with file_sink: try: - await asyncio.gather( - listen_to_omi(mac_address, OMI_CHAR_UUID, handle_ble_data), - process_audio(), - backend_stream_wrapper(), - ) + async with OmiConnection(mac_address) as conn: + await conn.subscribe_audio(handle_ble_data) + await conn.subscribe_button(handle_button_event) + await asyncio.gather( + conn.wait_until_disconnected(), + process_audio(), + backend_stream_wrapper(), + ) except Exception as e: logger.error(f"Error in audio processing: {e}", exc_info=True) finally: diff --git a/extras/local-omi-bt/requirements.txt b/extras/local-omi-bt/requirements.txt index 5d0f167f..3438c2d5 100644 --- a/extras/local-omi-bt/requirements.txt +++ b/extras/local-omi-bt/requirements.txt @@ -2,7 +2,7 @@ bleak==0.22.3 numpy>=1.26.4 scipy>=1.12.0 opuslib>=3.0.1 -friend-lite-sdk +friend-lite-sdk @ file:../friend-lite-sdk easy_audio_interfaces python-dotenv asyncstdlib diff --git a/extras/local-omi-bt/send_to_adv.py b/extras/local-omi-bt/send_to_adv.py index e878a404..39763e9b 100644 --- a/extras/local-omi-bt/send_to_adv.py +++ b/extras/local-omi-bt/send_to_adv.py @@ -32,6 +32,28 @@ logger = logging.getLogger(__name__) +# Module-level websocket reference for sending control messages (e.g., button events) +_active_websocket = None + + +async def send_button_event(button_state: str) -> None: + """Send a button event to the backend via the active WebSocket connection. + + Args: + button_state: Button state string (e.g., "SINGLE_TAP", "DOUBLE_TAP") + """ + if _active_websocket is None: + logger.debug("No active websocket, dropping button event: %s", button_state) + return + + event = { + "type": "button-event", + "data": {"state": button_state}, + "payload_length": None, + } + await _active_websocket.send(json.dumps(event) + "\n") + logger.info("Sent button event to backend: %s", button_state) + async def get_jwt_token(username: str, password: str) -> Optional[str]: """ @@ -152,6 +174,8 @@ async def stream_to_backend(stream: AsyncGenerator[AudioChunk, None]): ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE + global _active_websocket + logger.info(f"Connecting to WebSocket: {websocket_uri}") async with websockets.connect( uri_with_token, @@ -160,6 +184,8 @@ async def stream_to_backend(stream: AsyncGenerator[AudioChunk, None]): ping_timeout=120, # Wait up to 120 seconds for pong (increased from default 20s) close_timeout=10, # Graceful close timeout ) as websocket: + _active_websocket = websocket + # Wait for ready message from backend ready_msg = await websocket.recv() logger.info(f"Backend ready: {ready_msg}") @@ -217,6 +243,7 @@ async def stream_to_backend(stream: AsyncGenerator[AudioChunk, None]): logger.info(f"Sent audio-stop event. Total chunks: {chunk_count}") finally: + _active_websocket = None # Clean up receive task receive_task.cancel() try: From 55da001c84f6e601775af92200cb3784ab003c48 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sun, 8 Feb 2026 21:12:15 +0530 Subject: [PATCH 3/3] Enhance client state and plugin architecture for button event handling - Introduced a new `markers` list in `ClientState` to collect button event data during sessions. - Added `add_marker` method to facilitate the addition of markers to the current session. - Implemented `on_button_event` method in the `BasePlugin` class to handle device button events, providing context data for button state and timestamps. - Updated `PluginRouter` to route button events to the appropriate plugin handler. - Enhanced conversation job handling to attach markers from Redis sessions, improving the tracking of button events during conversations. --- .../advanced/src/advanced_omi_backend/client.py | 7 +++++++ .../src/advanced_omi_backend/plugins/base.py | 14 ++++++++++++++ .../src/advanced_omi_backend/plugins/router.py | 2 ++ .../workers/conversation_jobs.py | 12 ++++++++++++ 4 files changed, 35 insertions(+) diff --git a/backends/advanced/src/advanced_omi_backend/client.py b/backends/advanced/src/advanced_omi_backend/client.py index a92fbc10..40adcf41 100644 --- a/backends/advanced/src/advanced_omi_backend/client.py +++ b/backends/advanced/src/advanced_omi_backend/client.py @@ -51,6 +51,9 @@ def __init__( # NOTE: Removed in-memory transcript storage for single source of truth # Transcripts are stored only in MongoDB via TranscriptionManager + # Markers (e.g., button events) collected during the session + self.markers: List[dict] = [] + # Track if conversation has been closed self.conversation_closed: bool = False @@ -102,6 +105,10 @@ def update_transcript_received(self): """Update timestamp when transcript is received (for timeout detection).""" self.last_transcript_time = time.time() + def add_marker(self, marker: dict) -> None: + """Add a marker (e.g., button event) to the current session.""" + self.markers.append(marker) + def should_start_new_conversation(self) -> bool: """Check if we should start a new conversation based on timeout.""" if self.last_transcript_time is None: diff --git a/backends/advanced/src/advanced_omi_backend/plugins/base.py b/backends/advanced/src/advanced_omi_backend/plugins/base.py index fefcc6a0..4dd83b8f 100644 --- a/backends/advanced/src/advanced_omi_backend/plugins/base.py +++ b/backends/advanced/src/advanced_omi_backend/plugins/base.py @@ -143,3 +143,17 @@ async def on_memory_processed(self, context: PluginContext) -> Optional[PluginRe PluginResult with success status, optional message, and should_continue flag """ pass + + async def on_button_event(self, context: PluginContext) -> Optional[PluginResult]: + """ + Called when a device button event is received. + + Context data contains: + - state: str - Button state (e.g., "SINGLE_TAP", "DOUBLE_TAP", "LONG_PRESS") + - timestamp: float - Unix timestamp of the event + - audio_uuid: str - Current audio session UUID (may be None) + + Returns: + PluginResult with success status, optional message, and should_continue flag + """ + pass diff --git a/backends/advanced/src/advanced_omi_backend/plugins/router.py b/backends/advanced/src/advanced_omi_backend/plugins/router.py index 523fe3ed..927624a9 100644 --- a/backends/advanced/src/advanced_omi_backend/plugins/router.py +++ b/backends/advanced/src/advanced_omi_backend/plugins/router.py @@ -243,6 +243,8 @@ async def _execute_plugin( return await plugin.on_conversation_complete(context) elif event.startswith('memory.'): return await plugin.on_memory_processed(context) + elif event.startswith('button.'): + return await plugin.on_button_event(context) return None diff --git a/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py index 18420ddf..8dff66cf 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py @@ -5,6 +5,7 @@ """ import asyncio +import json import logging import os import time @@ -303,6 +304,17 @@ async def open_conversation_job( conversation_id = conversation.conversation_id logger.info(f"✅ Created streaming conversation {conversation_id} for session {session_id}") + # Attach markers from Redis session (e.g., button events captured during streaming) + markers_json = await redis_client.hget(session_key, "markers") + if markers_json: + try: + markers_data = markers_json if isinstance(markers_json, str) else markers_json.decode() + conversation.markers = json.loads(markers_data) + await conversation.save() + logger.info(f"📌 Attached {len(conversation.markers)} markers to conversation {conversation_id}") + except Exception as marker_err: + logger.warning(f"⚠️ Failed to parse markers from Redis: {marker_err}") + # Link job metadata to conversation (cascading updates) current_job.meta["conversation_id"] = conversation_id current_job.save_meta()