Desktop: remove GEMINI_API_KEY, route proactive AI through /v4/listen (#5396)

backend/models/message_event.py

@@ -181,3 +181,102 @@ def to_json(self):
         j["type"] = self.event_type
         del j["event_type"]
         return j
+
+
+# Desktop proactive AI events (Phase 2 — #5396)
+
+
+class FocusResultEvent(MessageEvent):
+    event_type: str = "focus_result"
+    frame_id: str
+    status: str
+    app_or_site: str
+    description: str
+    message: Optional[str] = None
+
+    def to_json(self):
+        j = self.model_dump(mode="json")
+        j["type"] = self.event_type
+        del j["event_type"]
+        return j
+
+
+class TasksExtractedEvent(MessageEvent):
+    event_type: str = "tasks_extracted"
+    frame_id: str
+    tasks: List = []
+
+    def to_json(self):
+        j = self.model_dump(mode="json")
+        j["type"] = self.event_type
+        del j["event_type"]
+        return j
+
+
+class MemoriesExtractedEvent(MessageEvent):
+    event_type: str = "memories_extracted"
+    frame_id: str
+    memories: List = []
+
+    def to_json(self):
+        j = self.model_dump(mode="json")
+        j["type"] = self.event_type
+        del j["event_type"]
+        return j
+
+
+class AdviceExtractedEvent(MessageEvent):
+    event_type: str = "advice_extracted"
+    frame_id: str
+    advice: Optional[Any] = None
+
+    def to_json(self):
+        j = self.model_dump(mode="json")
+        j["type"] = self.event_type
+        del j["event_type"]
+        return j
+
+
+class LiveNoteEvent(MessageEvent):
+    event_type: str = "live_note"
+    text: str
+
+    def to_json(self):
+        j = self.model_dump(mode="json")
+        j["type"] = self.event_type
+        del j["event_type"]
+        return j
+
+
+class ProfileUpdatedEvent(MessageEvent):
+    event_type: str = "profile_updated"
+    profile_text: str
+
+    def to_json(self):
+        j = self.model_dump(mode="json")
+        j["type"] = self.event_type
+        del j["event_type"]
+        return j
+
+
+class RerankCompleteEvent(MessageEvent):
+    event_type: str = "rerank_complete"
+    updated_tasks: List = []
+
+    def to_json(self):
+        j = self.model_dump(mode="json")
+        j["type"] = self.event_type
+        del j["event_type"]
+        return j
+
+
+class DedupCompleteEvent(MessageEvent):
+    event_type: str = "dedup_complete"
+    deleted_ids: List = []
+    reason: str = ""
+
+    def to_json(self):
+        j = self.model_dump(mode="json")
+        j["type"] = self.event_type
+        del j["event_type"]
+        return j

backend/routers/transcribe.py

@@ -51,16 +51,24 @@
     TranscriptSegment,
 )
 from models.message_event import (
+    AdviceExtractedEvent,
     ConversationEvent,
+    DedupCompleteEvent,
+    FocusResultEvent,
     FREEMIUM_ACTION_SETUP_ON_DEVICE_STT,
     FreemiumThresholdReachedEvent,
     LastConversationEvent,
+    LiveNoteEvent,
+    MemoriesExtractedEvent,
     MessageEvent,
     MessageServiceStatusEvent,
     PhotoDescribedEvent,
     PhotoProcessingEvent,
+    ProfileUpdatedEvent,
+    RerankCompleteEvent,
     SegmentsDeletedEvent,
     SpeakerLabelSuggestionEvent,
+    TasksExtractedEvent,
     TranslationEvent,
 )
 from models.transcript_segment import Translation
@@ -101,6 +109,13 @@
     SPEAKER_MATCH_THRESHOLD,
 )
 from utils.speaker_sample_migration import maybe_migrate_person_samples
+from utils.desktop.advice import generate_advice
+from utils.desktop.focus import analyze_focus
+from utils.desktop.live_notes import generate_live_note
+from utils.desktop.memories import extract_memories
+from utils.desktop.profile import generate_profile
+from utils.desktop.task_ops import dedup_tasks, rerank_tasks
+from utils.desktop.tasks import extract_tasks
 from utils.log_sanitizer import sanitize, sanitize_pii
 
 logger = logging.getLogger(__name__)
@@ -2431,6 +2446,98 @@ async def close_soniox_profile():
                                 logger.info(
                                     f"Speaker assignment ignored: missing speaker_id/person_id/person_name. {uid} {session_id}"
                                 )
+                        # Desktop proactive AI — screen_frame analysis (#5396)
+                        elif json_data.get('type') == 'screen_frame':
+                            frame_id = json_data.get('frame_id', '')
+                            image_b64 = json_data.get('image_b64', '')
+                            analyze_types = json_data.get('analyze', [])
+                            sf_app = json_data.get('app_name', '')
+                            sf_wtitle = json_data.get('window_title', '')
+                            if not image_b64:
+                                logger.warning(f"screen_frame missing image_b64 {uid} {session_id}")
+                            else:
+                                # Fan out to parallel handlers per analyze type
+                                if 'focus' in analyze_types:
+                                    async def _handle_focus(fid, img, app, wtitle):
+                                        try:
+                                            result = await analyze_focus(uid=uid, image_b64=img, app_name=app, window_title=wtitle)
+                                            _send_message_event(FocusResultEvent(
+                                                frame_id=fid, status=result['status'], app_or_site=result['app_or_site'],
+                                                description=result['description'], message=result.get('message'),
+                                            ))
+                                        except Exception as e:
+                                            logger.error(f"Focus analysis failed: {e} {uid} {session_id}")
+                                    spawn(_handle_focus(frame_id, image_b64, sf_app, sf_wtitle))
+
+                                if 'tasks' in analyze_types:
+                                    async def _handle_tasks(fid, img, app, wtitle):
+                                        try:
+                                            result = await extract_tasks(uid=uid, image_b64=img, app_name=app, window_title=wtitle)
+                                            _send_message_event(TasksExtractedEvent(frame_id=fid, tasks=result.get('tasks', [])))
+                                        except Exception as e:
+                                            logger.error(f"Task extraction failed: {e} {uid} {session_id}")
+                                    spawn(_handle_tasks(frame_id, image_b64, sf_app, sf_wtitle))
+
+                                if 'memories' in analyze_types:
+                                    async def _handle_memories(fid, img, app, wtitle):
+                                        try:
+                                            result = await extract_memories(uid=uid, image_b64=img, app_name=app, window_title=wtitle)
+                                            _send_message_event(MemoriesExtractedEvent(frame_id=fid, memories=result.get('memories', [])))
+                                        except Exception as e:
+                                            logger.error(f"Memory extraction failed: {e} {uid} {session_id}")
+                                    spawn(_handle_memories(frame_id, image_b64, sf_app, sf_wtitle))
+
+                                if 'advice' in analyze_types:
+                                    async def _handle_advice(fid, img, app, wtitle):
+                                        try:
+                                            result = await generate_advice(uid=uid, image_b64=img, app_name=app, window_title=wtitle)
+                                            _send_message_event(AdviceExtractedEvent(
+                                                frame_id=fid, advice=result.get('advice'),
+                                            ))
+                                        except Exception as e:
+                                            logger.error(f"Advice generation failed: {e} {uid} {session_id}")
+                                    spawn(_handle_advice(frame_id, image_b64, sf_app, sf_wtitle))
+
+                        # Desktop proactive AI — text-only message types (#5396)
+                        elif json_data.get('type') == 'live_notes_text':
+                            async def _handle_live_notes(text, ctx):
+                                try:
+                                    result = await generate_live_note(text=text, session_context=ctx)
+                                    if result.get('text'):
+                                        _send_message_event(LiveNoteEvent(text=result['text']))
+                                except Exception as e:
+                                    logger.error(f"Live note generation failed: {e} {uid} {session_id}")
+                            spawn(_handle_live_notes(json_data.get('text', ''), json_data.get('session_context', '')))
+
+                        elif json_data.get('type') == 'profile_request':
+                            async def _handle_profile():
+                                try:
+                                    result = await generate_profile(uid=uid)
+                                    _send_message_event(ProfileUpdatedEvent(profile_text=result['profile_text']))
+                                except Exception as e:
+                                    logger.error(f"Profile generation failed: {e} {uid} {session_id}")
+                            spawn(_handle_profile())
+
+                        elif json_data.get('type') == 'task_rerank':
+                            async def _handle_rerank():
+                                try:
+                                    result = await rerank_tasks(uid=uid)
+                                    _send_message_event(RerankCompleteEvent(updated_tasks=result['updated_tasks']))
+                                except Exception as e:
+                                    logger.error(f"Task reranking failed: {e} {uid} {session_id}")
+                            spawn(_handle_rerank())
+
+                        elif json_data.get('type') == 'task_dedup':
+                            async def _handle_dedup():
+                                try:
+                                    result = await dedup_tasks(uid=uid)
+                                    _send_message_event(DedupCompleteEvent(
+                                        deleted_ids=result['deleted_ids'], reason=result['reason'],
+                                    ))
+                                except Exception as e:
+                                    logger.error(f"Task dedup failed: {e} {uid} {session_id}")
+                            spawn(_handle_dedup())
+
                     except json.JSONDecodeError:
                         logger.info(
                             f"Received non-json text message: {sanitize(message.get('text'))} {uid} {session_id}"

backend/test.sh

@@ -43,3 +43,10 @@ pytest tests/unit/test_storage_upload_audio_chunk_data_protection.py -v
 pytest tests/unit/test_people_conversations_500s.py -v
 pytest tests/unit/test_firestore_read_ops_cache.py -v
 pytest tests/unit/test_ws_auth_handshake.py -v
+pytest tests/unit/test_desktop_focus.py -v
+pytest tests/unit/test_desktop_tasks.py -v
+pytest tests/unit/test_desktop_memories.py -v
+pytest tests/unit/test_desktop_advice.py -v
+pytest tests/unit/test_desktop_live_notes.py -v
+pytest tests/unit/test_desktop_profile.py -v
+pytest tests/unit/test_desktop_task_ops.py -v