feat: support custom model definitions in realtime.connect()

examples/sdk-core/README.md

@@ -57,6 +57,7 @@ See `examples/nextjs-realtime` or `examples/react-vite` for runnable demos.
 - `realtime/live-avatar.ts` - Live avatar (audio-driven avatar with playAudio or mic input)
 - `realtime/connection-events.ts` - Handling connection state and errors
 - `realtime/prompt-update.ts` - Updating prompt dynamically
+- `realtime/custom-model.ts` - Using a custom model definition (e.g., preview/experimental models)
 
 ## API Reference
 

examples/sdk-core/realtime/custom-model.ts

@@ -0,0 +1,62 @@
+/**
+ * Custom Model Definition Example
+ *
+ * Demonstrates how to define and use a custom model that isn't
+ * built into the SDK. This is useful for preview/experimental models
+ * or private deployments.
+ *
+ * Browser-only example - requires WebRTC APIs
+ * See examples/nextjs-realtime or examples/react-vite for runnable demos
+ */
+
+import { createDecartClient } from "@decartai/sdk";
+import type { CustomModelDefinition } from "@decartai/sdk";
+
+async function main() {
+  // Define a custom model that isn't in the SDK's built-in registry.
+  // This works for any model that conforms to the CustomModelDefinition shape.
+  const lucy2RtPreview: CustomModelDefinition = {
+    name: "lucy_2_rt_preview",
+    urlPath: "/v1/stream",
+    fps: 20,
+    width: 1280,
+    height: 720,
+  };
+
+  // Get webcam stream using the custom model's settings
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: true,
+    video: {
+      frameRate: lucy2RtPreview.fps,
+      width: lucy2RtPreview.width,
+      height: lucy2RtPreview.height,
+    },
+  });
+
+  const client = createDecartClient({
+    apiKey: process.env.DECART_API_KEY!,
+  });
+
+  // Pass the custom model directly to realtime.connect()
+  const realtimeClient = await client.realtime.connect(stream, {
+    model: lucy2RtPreview,
+    onRemoteStream: (transformedStream) => {
+      const video = document.getElementById("output") as HTMLVideoElement;
+      video.srcObject = transformedStream;
+    },
+    initialState: {
+      prompt: {
+        text: "cinematic lighting, film grain",
+        enhance: true,
+      },
+    },
+  });
+
+  console.log("Session ID:", realtimeClient.sessionId);
+  console.log("Connected:", realtimeClient.isConnected());
+
+  // Update prompt dynamically, same as built-in models
+  realtimeClient.setPrompt("watercolor painting style");
+}
+
+main();

packages/sdk/src/index.ts

@@ -47,6 +47,7 @@ export type {
 export type { ConnectionState } from "./realtime/types";
 export type { WebRTCStats } from "./realtime/webrtc-stats";
 export {
+  type CustomModelDefinition,
   type ImageModelDefinition,
   type ImageModels,
   isImageModel,

packages/sdk/src/realtime/client.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { modelDefinitionSchema, type RealTimeModels } from "../shared/model";
+import { type CustomModelDefinition, type ModelDefinition, modelDefinitionSchema } from "../shared/model";
 import { modelStateSchema } from "../shared/types";
 import { classifyWebrtcError, type DecartSDKError } from "../utils/errors";
 import type { Logger } from "../utils/logger";
@@ -94,7 +94,9 @@ const realTimeClientConnectOptionsSchema = z.object({
   initialState: realTimeClientInitialStateSchema.optional(),
   customizeOffer: createAsyncFunctionSchema(z.function()).optional(),
 });
-export type RealTimeClientConnectOptions = z.infer<typeof realTimeClientConnectOptionsSchema>;
+export type RealTimeClientConnectOptions = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model"> & {
+  model: ModelDefinition | CustomModelDefinition;
+};
 
 export type Events = {
   connectionChange: ConnectionState;
@@ -189,7 +191,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
         customizeOffer: options.customizeOffer as ((offer: RTCSessionDescriptionInit) => Promise<void>) | undefined,
         vp8MinBitrate: 300,
         vp8StartBitrate: 600,
-        modelName: options.model.name as RealTimeModels,
+        modelName: options.model.name,
         initialImage,
         initialPrompt,
       });

packages/sdk/src/realtime/webrtc-connection.ts

@@ -1,5 +1,5 @@
 import mitt from "mitt";
-import type { RealTimeModels } from "../shared/model";
+
 import type { Logger } from "../utils/logger";
 import { buildUserAgent } from "../utils/user-agent";
 import type { DiagnosticEmitter, IceCandidateEvent } from "./diagnostics";
@@ -24,7 +24,7 @@ interface ConnectionCallbacks {
   customizeOffer?: (offer: RTCSessionDescriptionInit) => Promise<void>;
   vp8MinBitrate?: number;
   vp8StartBitrate?: number;
-  modelName?: RealTimeModels;
+  modelName?: string;
   initialImage?: string;
   initialPrompt?: { text: string; enhance?: boolean };
   logger?: Logger;

packages/sdk/src/realtime/webrtc-manager.ts

@@ -1,5 +1,5 @@
 import pRetry, { AbortError } from "p-retry";
-import type { RealTimeModels } from "../shared/model";
+
 import type { Logger } from "../utils/logger";
 import type { DiagnosticEmitter } from "./diagnostics";
 import type { ConnectionState, OutgoingMessage } from "./types";
@@ -16,7 +16,7 @@ export interface WebRTCConfig {
   customizeOffer?: (offer: RTCSessionDescriptionInit) => Promise<void>;
   vp8MinBitrate?: number;
   vp8StartBitrate?: number;
-  modelName?: RealTimeModels;
+  modelName?: string;
   initialImage?: string;
   initialPrompt?: { text: string; enhance?: boolean };
 }

packages/sdk/src/shared/model.ts

@@ -208,6 +208,15 @@ export type ModelDefinition<T extends Model = Model> = {
   inputSchema: T extends keyof ModelInputSchemas ? ModelInputSchemas[T] : z.ZodTypeAny;
 };
 
+/**
+ * A model definition with an arbitrary (non-registry) model name.
+ * Use this when providing your own model configuration.
+ */
+export type CustomModelDefinition = Omit<ModelDefinition, "name" | "inputSchema"> & {
+  name: string;
+  inputSchema?: z.ZodTypeAny;
+};
+
 /**
  * Type alias for model definitions that support synchronous processing.
  * Only image models support the sync/process API.
@@ -221,7 +230,7 @@ export type ImageModelDefinition = ModelDefinition<ImageModels>;
 export type VideoModelDefinition = ModelDefinition<VideoModels>;
 
 export const modelDefinitionSchema = z.object({
-  name: modelSchema,
+  name: z.string(),
   urlPath: z.string(),
   queueUrlPath: z.string().optional(),
   fps: z.number().min(1),

packages/sdk/tests/unit.test.ts

@@ -3117,3 +3117,33 @@ describe("VideoStall Diagnostic", () => {
     expect(event.data.durationMs).toBe(1500);
   });
 });
+
+describe("CustomModelDefinition", () => {
+  it("allows arbitrary model names in modelDefinitionSchema", async () => {
+    const { modelDefinitionSchema } = await import("../src/shared/model.js");
+
+    const customModel = {
+      name: "lucy_2_rt_preview",
+      urlPath: "/v1/stream",
+      fps: 20,
+      width: 1280,
+      height: 720,
+    };
+
+    const result = modelDefinitionSchema.safeParse(customModel);
+    expect(result.success).toBe(true);
+  });
+
+  it("rejects invalid custom model definitions", async () => {
+    const { modelDefinitionSchema } = await import("../src/shared/model.js");
+
+    const invalidModel = {
+      name: "my_custom_model",
+      urlPath: "/v1/stream",
+      // missing fps, width, height
+    };
+
+    const result = modelDefinitionSchema.safeParse(invalidModel);
+    expect(result.success).toBe(false);
+  });
+});