feat(realtime): unify initial state with modelName, image, and prompt (#29)

* feat(realtime): replace AvatarOptions with ModelState initial state API

Remove AvatarOptions and InitialPromptOptions in favor of unified
ModelState with optional image (bytes, str, or Path) and Prompt fields.
This aligns the Python SDK with the TS SDK's PR #84 API changes.

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

* feat(realtime): wire model_name and initial state through WebRTC stack

- Replace is_avatar_live with model_name throughout the connection pipeline
- Unify initial image/prompt handling into Phase 2 pre-handshake
- Add receive-only video transceiver for avatar-live with local track
- Add audio transceiver for subscribe (receive-only) mode
- Accept Path, raw base64, data URI, URL, and file paths in _image_to_base64

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

* feat(realtime): update avatar-live example and tests for new API

- Update avatar_live.py to use ModelState(image=...) instead of AvatarOptions
- Update existing tests for new initial_state wiring
- Add unit tests for _image_to_base64: raw base64, Path, bytes, data URI, file path

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

* chore: use lucy 2

---------

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

Author: AdirAmsalem

decart/__init__.py

@@ -36,7 +36,6 @@
         decode_subscribe_token,
         RealtimeConnectOptions,
         ConnectionState,
-        AvatarOptions,
     )
 
     REALTIME_AVAILABLE = True
@@ -50,7 +49,6 @@
     decode_subscribe_token = None  # type: ignore
     RealtimeConnectOptions = None  # type: ignore
     ConnectionState = None  # type: ignore
-    AvatarOptions = None  # type: ignore
 
 __version__ = "0.0.1"
 
@@ -93,6 +91,5 @@
             "decode_subscribe_token",
             "RealtimeConnectOptions",
             "ConnectionState",
-            "AvatarOptions",
         ]
     )

decart/realtime/__init__.py

@@ -6,7 +6,7 @@
     decode_subscribe_token,
 )
 from .messages import GenerationTickMessage
-from .types import RealtimeConnectOptions, ConnectionState, AvatarOptions
+from .types import RealtimeConnectOptions, ConnectionState
 
 __all__ = [
     "RealtimeClient",
@@ -18,5 +18,4 @@
     "GenerationTickMessage",
     "RealtimeConnectOptions",
     "ConnectionState",
-    "AvatarOptions",
 ]

decart/realtime/client.py

@@ -18,6 +18,7 @@
 )
 from .types import ConnectionState, RealtimeConnectOptions
 from ..types import FileInput
+from ..models import RealTimeModels
 from ..errors import DecartSDKError, InvalidInputError, WebRTCError
 from ..process.request import file_input_to_bytes
 
@@ -34,9 +35,13 @@ class SetInput(BaseModel):
 
 
 async def _image_to_base64(
-    image: Union[bytes, str],
+    image: Union[bytes, str, Path],
     http_session: aiohttp.ClientSession,
 ) -> str:
+    if isinstance(image, Path):
+        image_bytes, _ = await file_input_to_bytes(image, http_session)
+        return base64.b64encode(image_bytes).decode("utf-8")
+
     if isinstance(image, bytes):
         return base64.b64encode(image).decode("utf-8")
 
@@ -56,21 +61,20 @@ async def _image_to_base64(
             image_bytes, _ = await file_input_to_bytes(image, http_session)
             return base64.b64encode(image_bytes).decode("utf-8")
 
-        raise InvalidInputError(
-            "Invalid image input: string is not a data URI, URL, or valid file path"
-        )
+        # Non-URL, non-file string — treat as raw base64 (matches TS SDK behavior)
+        return image
 
 
 class RealtimeClient:
     def __init__(
         self,
         manager: WebRTCManager,
         http_session: Optional[aiohttp.ClientSession] = None,
-        is_avatar_live: bool = False,
+        model_name: Optional[str] = None,
     ):
         self._manager = manager
         self._http_session = http_session
-        self._is_avatar_live = is_avatar_live
+        self._model_name = model_name
         self._connection_callbacks: list[Callable[[ConnectionState], None]] = []
         self._error_callbacks: list[Callable[[DecartSDKError], None]] = []
         self._generation_tick_callbacks: list[Callable[[GenerationTickMessage], None]] = []
@@ -105,7 +109,7 @@ async def connect(
         ws_url = f"{base_url}{options.model.url_path}"
         ws_url += f"?api_key={quote(api_key)}&model={quote(options.model.name)}"
 
-        is_avatar_live = options.model.name == "avatar-live"
+        model_name: RealTimeModels = options.model.name  # type: ignore[assignment]
 
         config = WebRTCConfiguration(
             webrtc_url=ws_url,
@@ -119,7 +123,7 @@ async def connect(
             initial_state=options.initial_state,
             customize_offer=options.customize_offer,
             integration=integration,
-            is_avatar_live=is_avatar_live,
+            model_name=model_name,
         )
 
         # Create HTTP session for file conversions
@@ -129,7 +133,7 @@ async def connect(
         client = cls(
             manager=manager,
             http_session=http_session,
-            is_avatar_live=is_avatar_live,
+            model_name=model_name,
         )
 
         config.on_connection_state_change = client._emit_connection_change
@@ -138,35 +142,22 @@ async def connect(
         config.on_generation_tick = client._emit_generation_tick
 
         try:
-            # For avatar-live, convert and send avatar image before WebRTC connection
-            avatar_image_base64: Optional[str] = None
-            if is_avatar_live and options.avatar:
-                image_bytes, _ = await file_input_to_bytes(
-                    options.avatar.avatar_image, http_session
-                )
-                avatar_image_base64 = base64.b64encode(image_bytes).decode("utf-8")
-
-            # Prepare initial prompt if provided
+            initial_image: Optional[str] = None
+            if options.initial_state and options.initial_state.image:
+                initial_image = await _image_to_base64(options.initial_state.image, http_session)
+
             initial_prompt: Optional[dict] = None
-            if options.initial_prompt:
+            if options.initial_state and options.initial_state.prompt:
                 initial_prompt = {
-                    "text": options.initial_prompt.text,
-                    "enhance": options.initial_prompt.enhance,
+                    "text": options.initial_state.prompt.text,
+                    "enhance": options.initial_state.prompt.enhance,
                 }
 
             await manager.connect(
                 local_track,
-                avatar_image_base64=avatar_image_base64,
+                initial_image=initial_image,
                 initial_prompt=initial_prompt,
             )
-
-            # Handle initial_state.prompt for backward compatibility (after WebRTC connection)
-            if options.initial_state:
-                if options.initial_state.prompt:
-                    await client.set_prompt(
-                        options.initial_state.prompt.text,
-                        enhance=options.initial_state.prompt.enhance,
-                    )
         except Exception as e:
             await manager.cleanup()
             await http_session.close()

decart/realtime/types.py

@@ -1,7 +1,7 @@
 from typing import Literal, Callable, Optional
 from dataclasses import dataclass
 from ..models import ModelDefinition
-from ..types import ModelState, FileInput
+from ..types import ModelState
 
 try:
     from aiortc import MediaStreamTrack
@@ -12,31 +12,9 @@
 ConnectionState = Literal["connecting", "connected", "generating", "disconnected", "reconnecting"]
 
 
-@dataclass
-class AvatarOptions:
-    """Options for avatar-live model."""
-
-    avatar_image: FileInput
-    """The avatar image to use. Can be bytes, Path, URL string, or file-like object."""
-
-
-@dataclass
-class InitialPromptOptions:
-    """Options for initial prompt sent before WebRTC handshake."""
-
-    text: str
-    """The prompt text to send."""
-
-    enhance: bool = True
-    """Whether to enhance the prompt. Defaults to True."""
-
-
 @dataclass
 class RealtimeConnectOptions:
     model: ModelDefinition
     on_remote_stream: Callable[[MediaStreamTrack], None]
     initial_state: Optional[ModelState] = None
     customize_offer: Optional[Callable] = None
-    avatar: Optional[AvatarOptions] = None
-    initial_prompt: Optional[InitialPromptOptions] = None
-    """Initial prompt to send before WebRTC handshake (optional)."""

decart/realtime/webrtc_connection.py

@@ -61,21 +61,21 @@ def __init__(
         self._pending_prompts: dict[str, tuple[asyncio.Event, dict]] = {}
         self._pending_image_set: Optional[tuple[asyncio.Event, dict]] = None
         self._local_track: Optional[MediaStreamTrack] = None
-        self._is_avatar_live: bool = False
+        self._model_name: Optional[str] = None
 
     async def connect(
         self,
         url: str,
         local_track: Optional[MediaStreamTrack],
         timeout: float,
         integration: Optional[str] = None,
-        is_avatar_live: bool = False,
-        avatar_image_base64: Optional[str] = None,
+        model_name: Optional[str] = None,
+        initial_image: Optional[str] = None,
         initial_prompt: Optional[dict] = None,
     ) -> None:
         try:
             self._local_track = local_track
-            self._is_avatar_live = is_avatar_live
+            self._model_name = model_name
 
             await self._set_state("connecting")
 
@@ -90,13 +90,16 @@ async def connect(
 
             self._ws_task = asyncio.create_task(self._receive_messages())
 
-            if is_avatar_live and avatar_image_base64:
-                await self._send_avatar_image_and_wait(avatar_image_base64)
-
-            if initial_prompt:
+            if initial_image:
+                await self._send_initial_image_and_wait(
+                    initial_image,
+                    prompt=initial_prompt.get("text") if initial_prompt else None,
+                    enhance=initial_prompt.get("enhance") if initial_prompt else None,
+                )
+            elif initial_prompt:
                 await self._send_initial_prompt_and_wait(initial_prompt)
 
-            await self._setup_peer_connection(local_track, is_avatar_live=is_avatar_live)
+            await self._setup_peer_connection(local_track, model_name=model_name)
 
             await self._create_and_send_offer()
 
@@ -115,23 +118,32 @@ async def connect(
                 self._on_error(e)
             raise WebRTCError(str(e), cause=e)
 
-    async def _send_avatar_image_and_wait(self, image_base64: str, timeout: float = 30.0) -> None:
-        """Send avatar image and wait for acknowledgment."""
+    async def _send_initial_image_and_wait(
+        self,
+        image_base64: str,
+        prompt: Optional[str] = None,
+        enhance: Optional[bool] = None,
+        timeout: float = 30.0,
+    ) -> None:
         event, result = self.register_image_set_wait()
 
         try:
-            await self._send_message(
-                SetAvatarImageMessage(type="set_image", image_data=image_base64)
-            )
+            message = SetAvatarImageMessage(type="set_image", image_data=image_base64)
+            if prompt is not None:
+                message.prompt = prompt
+            if enhance is not None:
+                message.enhance_prompt = enhance
+
+            await self._send_message(message)
 
             try:
                 await asyncio.wait_for(event.wait(), timeout=timeout)
             except asyncio.TimeoutError:
-                raise WebRTCError("Avatar image acknowledgment timed out")
+                raise WebRTCError("Initial image acknowledgment timed out")
 
             if not result["success"]:
                 raise WebRTCError(
-                    f"Failed to set avatar image: {result.get('error', 'unknown error')}"
+                    f"Failed to set initial image: {result.get('error', 'unknown error')}"
                 )
         finally:
             self.unregister_image_set_wait()
@@ -163,7 +175,7 @@ async def _send_initial_prompt_and_wait(self, prompt: dict, timeout: float = 15.
     async def _setup_peer_connection(
         self,
         local_track: Optional[MediaStreamTrack],
-        is_avatar_live: bool = False,
+        model_name: Optional[str] = None,
     ) -> None:
         config = RTCConfiguration(iceServers=[RTCIceServer(urls=["stun:stun.l.google.com:19302"])])
 
@@ -205,8 +217,12 @@ async def on_ice_connection_state_change():
 
         if local_track is None:
             self._pc.addTransceiver("video", direction="recvonly")
-            logger.debug("Added video transceiver (recvonly) for receive-only mode")
+            self._pc.addTransceiver("audio", direction="recvonly")
+            logger.debug("Added video+audio transceivers (recvonly) for subscribe mode")
         else:
+            if model_name == "avatar-live":
+                self._pc.addTransceiver("video", direction="recvonly")
+                logger.debug("Added video transceiver (recvonly) for avatar-live mode")
             self._pc.addTrack(local_track)
             logger.debug("Added local track to peer connection")
 

decart/realtime/webrtc_manager.py

@@ -48,7 +48,7 @@ class WebRTCConfiguration:
     initial_state: Optional[ModelState] = None
     customize_offer: Optional[Callable] = None
     integration: Optional[str] = None
-    is_avatar_live: bool = False
+    model_name: Optional[str] = None
 
 
 def _is_permanent_error(exception: BaseException) -> bool:
@@ -155,7 +155,7 @@ async def _attempt():
                 local_track=self._local_track,
                 timeout=CONNECTION_TIMEOUT,
                 integration=self._config.integration,
-                is_avatar_live=self._config.is_avatar_live,
+                model_name=self._config.model_name,
             )
 
             if self._intentional_disconnect or reconnect_generation != self._reconnect_generation:
@@ -174,7 +174,7 @@ async def _attempt():
     async def connect(
         self,
         local_track: Optional[MediaStreamTrack],
-        avatar_image_base64: Optional[str] = None,
+        initial_image: Optional[str] = None,
         initial_prompt: Optional[dict] = None,
     ) -> bool:
         self._local_track = local_track
@@ -192,8 +192,8 @@ async def connect(
                 local_track=local_track,
                 timeout=CONNECTION_TIMEOUT,
                 integration=self._config.integration,
-                is_avatar_live=self._config.is_avatar_live,
-                avatar_image_base64=avatar_image_base64,
+                model_name=self._config.model_name,
+                initial_image=initial_image,
                 initial_prompt=initial_prompt,
             )
             return True

decart/types.py

@@ -20,6 +20,7 @@ class Prompt(BaseModel):
 
 class ModelState(BaseModel):
     prompt: Optional[Prompt] = None
+    image: Optional[Union[bytes, str, Path]] = None
 
 
 class MotionTrajectoryInput(BaseModel):

examples/avatar_live.py

@@ -76,7 +76,8 @@ async def main():
 
     try:
         from decart.realtime.client import RealtimeClient
-        from decart.realtime.types import RealtimeConnectOptions, AvatarOptions
+        from decart.realtime.types import RealtimeConnectOptions
+        from decart.types import ModelState
     except ImportError:
         print("Error: Realtime API not available")
         print("Install with: pip install decart[realtime]")
@@ -120,7 +121,7 @@ def on_error(error):
                 options=RealtimeConnectOptions(
                     model=model,
                     on_remote_stream=on_remote_stream,
-                    avatar=AvatarOptions(avatar_image=Path(avatar_image)),
+                    initial_state=ModelState(image=avatar_image),
                 ),
             )
 

examples/files/image.png

@@ -73,7 +73,7 @@ async def main():
         print("Creating synthetic video track...")
         video_track = SyntheticVideoTrack()
 
-        model = models.realtime("mirage_v2")
+        model = models.realtime("lucy_2_rt")
         print(f"Using model: {model.name}")
         print(f"Model config - FPS: {model.fps}, Size: {model.width}x{model.height}")
 
@@ -111,7 +111,13 @@ def on_error(error):
                 options=RealtimeConnectOptions(
                     model=model,
                     on_remote_stream=on_remote_stream,
-                    initial_state=ModelState(prompt=Prompt(text="Anime style", enhance=True)),
+                    initial_state=ModelState(
+                        prompt=Prompt(
+                            text="use the image as a reference",
+                            enhance=True,
+                        ),
+                        image=Path("examples/files/image.png"),
+                    ),
                 ),
             )