From 079f2df4df0b02decb55e063ff9d72110d999f68 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 26 Nov 2025 11:02:21 +0200 Subject: [PATCH 01/23] new architecture for realtime manager --- Example.xcodeproj/project.pbxproj | 6 + .../xcshareddata/xcschemes/Example.xcscheme | 2 +- Example/Example/DecartSDK/DecartConfig.swift | 2 +- .../DecartSDK/DecartRealtimeManager.swift | 45 +++-- Example/Example/Views/RealtimeView.swift | 2 +- .../Capture/RealtimeCameraCapture.swift | 19 +- Sources/DecartSDK/DecartSDK.swift | 4 +- .../DecartSDK/Realtime/RealtimeClient.swift | 94 ---------- .../Realtime/RealtimeConfiguration.swift | 2 +- ...edia.swift => RealtimeManager+Media.swift} | 15 +- .../DecartSDK/Realtime/RealtimeManager.swift | 167 ++++++++++++++++++ .../Realtime/WebRTC/SignalingClient.swift | 50 ++++++ .../Realtime/WebRTC/SignalingManager.swift | 127 ------------- .../Realtime/WebRTC/SignalingModel.swift | 39 +++- .../Realtime/WebRTC/WebRTCClient.swift | 154 ++++++++++++++++ .../WebRTC/WebRTCDelegateHandler.swift | 57 ++++++ .../Realtime/WebRTC/WebRTCManager.swift | 138 --------------- .../Realtime/Websocket/WebSocketClient.swift | 49 ++--- Sources/DecartSDK/Shared/DecartError.swift | 5 + 19 files changed, 527 insertions(+), 450 deletions(-) delete mode 100644 Sources/DecartSDK/Realtime/RealtimeClient.swift rename Sources/DecartSDK/Realtime/{RealtimeClient+Media.swift => RealtimeManager+Media.swift} (69%) create mode 100644 Sources/DecartSDK/Realtime/RealtimeManager.swift create mode 100644 Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift delete mode 100644 Sources/DecartSDK/Realtime/WebRTC/SignalingManager.swift create mode 100644 Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift create mode 100644 Sources/DecartSDK/Realtime/WebRTC/WebRTCDelegateHandler.swift delete mode 100644 Sources/DecartSDK/Realtime/WebRTC/WebRTCManager.swift diff --git a/Example.xcodeproj/project.pbxproj b/Example.xcodeproj/project.pbxproj index 34ca959..b2b7148 100644 --- a/Example.xcodeproj/project.pbxproj +++ b/Example.xcodeproj/project.pbxproj @@ -217,7 +217,9 @@ ONLY_ACTIVE_ARCH = YES; SDKROOT = iphoneos; SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; + SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_STRICT_CONCURRENCY = complete; }; name = Debug; }; @@ -274,6 +276,8 @@ MTL_FAST_MATH = YES; SDKROOT = iphoneos; SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; + SWIFT_STRICT_CONCURRENCY = complete; VALIDATE_PRODUCT = YES; }; name = Release; @@ -306,6 +310,7 @@ SWIFT_APPROACHABLE_CONCURRENCY = YES; SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_STRICT_CONCURRENCY = complete; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; @@ -340,6 +345,7 @@ SWIFT_APPROACHABLE_CONCURRENCY = YES; SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_STRICT_CONCURRENCY = complete; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; diff --git a/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme b/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme index 0f704ee..1c38a71 100644 --- a/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme +++ b/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme @@ -56,7 +56,7 @@ diff --git a/Example/Example/DecartSDK/DecartConfig.swift b/Example/Example/DecartSDK/DecartConfig.swift index 6102b74..dc290ed 100644 --- a/Example/Example/DecartSDK/DecartConfig.swift +++ b/Example/Example/DecartSDK/DecartConfig.swift @@ -9,7 +9,7 @@ import DecartSDK import Factory import WebRTC -protocol RealtimeManager { +protocol RealtimeManagerProtocol { var currentPrompt: Prompt { get set } var shouldMirror: Bool { get set } diff --git a/Example/Example/DecartSDK/DecartRealtimeManager.swift b/Example/Example/DecartSDK/DecartRealtimeManager.swift index 13dce01..250f346 100644 --- a/Example/Example/DecartSDK/DecartRealtimeManager.swift +++ b/Example/Example/DecartSDK/DecartRealtimeManager.swift @@ -12,21 +12,15 @@ import WebRTC @MainActor @Observable -final class DecartRealtimeManager: RealtimeManager { +final class DecartRealtimeManager: RealtimeManagerProtocol { @ObservationIgnored private let decartClient = Container.shared.decartClient() var currentPrompt: Prompt { didSet { Task { [weak self] in - guard let self, let client = self.realtimeClient else { return } - do { - try await client.setPrompt(currentPrompt) - } catch { - DecartLogger.log( - "failed to update prompt: \(error.localizedDescription)", level: .error - ) - } + guard let self, let manager = self.realtimeManager else { return } + manager.setPrompt(currentPrompt) } } } @@ -35,13 +29,12 @@ final class DecartRealtimeManager: RealtimeManager { private(set) var connectionState: DecartRealtimeConnectionState = .idle - @ObservationIgnored private(set) var localMediaStream: RealtimeMediaStream? - @ObservationIgnored + private(set) var remoteMediaStreams: RealtimeMediaStream? @ObservationIgnored - private var realtimeClient: RealtimeClient? + private var realtimeManager: RealtimeManager? @ObservationIgnored private var videoCapturer: RTCCameraVideoCapturer? @ObservationIgnored @@ -56,56 +49,60 @@ final class DecartRealtimeManager: RealtimeManager { } func switchCamera() async { + #if !targetEnvironment(simulator) print("switching camera to \(shouldMirror ? "back" : "front") camera") - guard let videoCapturer, let realtimeClient else { + guard let videoCapturer, let realtimeManager else { preconditionFailure("🚨 videoCapturer is nil when switching camera") } do { try await RealtimeCameraCapture.switchCamera( capturer: videoCapturer, - realtimeClient: realtimeClient, + realtimeManager: realtimeManager, newPosition: shouldMirror ? .back : .front ) shouldMirror.toggle() } catch { DecartLogger.log("error while switching camera!", level: .error) } + #endif } func connect(model: RealtimeModel) async { - if connectionState.isInSession || realtimeClient != nil { + if connectionState.isInSession || realtimeManager != nil { await cleanup() } connectionState = .connecting do { - realtimeClient = + realtimeManager = try decartClient - .createRealtimeClient( + .createRealtimeManager( options: RealtimeConfiguration( model: Models.realtime(model), initialState: ModelState( prompt: currentPrompt ) )) - guard let realtimeClient else { - preconditionFailure("🚨 realtimeClient is nil after creating it") + guard let realtimeManager else { + preconditionFailure("🚨 realtimeManager is nil after creating it") } monitorEvents() + #if !targetEnvironment(simulator) (localMediaStream, videoCapturer) = try await RealtimeCameraCapture .captureLocalCameraStream( - realtimeClient: realtimeClient, + realtimeManager: realtimeManager, cameraFacing: .front ) DecartLogger.log("Connecting to WebRTC...", level: .info) remoteMediaStreams = - try await realtimeClient + try await realtimeManager .connect(localStream: localMediaStream!) + #endif } catch { DecartLogger.log( "Connection failed with error: \(error.localizedDescription)", level: .error @@ -119,7 +116,7 @@ final class DecartRealtimeManager: RealtimeManager { eventTask?.cancel() eventTask = Task { [weak self] in - guard let self, let stream = self.realtimeClient?.events else { return } + guard let self, let stream = self.realtimeManager?.events else { return } for await state in stream { if Task.isCancelled { return } @@ -148,8 +145,8 @@ final class DecartRealtimeManager: RealtimeManager { } } videoCapturer = nil - await realtimeClient?.disconnect() - realtimeClient = nil + await realtimeManager?.disconnect() + realtimeManager = nil remoteMediaStreams = nil localMediaStream = nil connectionState = .idle diff --git a/Example/Example/Views/RealtimeView.swift b/Example/Example/Views/RealtimeView.swift index 1cfa6ff..21396cd 100644 --- a/Example/Example/Views/RealtimeView.swift +++ b/Example/Example/Views/RealtimeView.swift @@ -14,7 +14,7 @@ struct RealtimeView: View { private let realtimeAiModel: RealtimeModel @State private var prompt: String = DecartConfig.defaultPrompt - @State private var realtimeManager: RealtimeManager + @State private var realtimeManager: RealtimeManagerProtocol init(realtimeModel: RealtimeModel) { self.realtimeAiModel = realtimeModel diff --git a/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift b/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift index 038282e..81ebae1 100644 --- a/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift +++ b/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift @@ -9,13 +9,13 @@ import WebRTC #if !targetEnvironment(simulator) public enum RealtimeCameraCapture { - public static func captureLocalCameraStream(realtimeClient: RealtimeClient, cameraFacing: AVCaptureDevice.Position) async throws -> ( + public static func captureLocalCameraStream(realtimeManager: RealtimeManager, cameraFacing: AVCaptureDevice.Position) async throws -> ( RealtimeMediaStream, RTCCameraVideoCapturer ) { - let currentRealtimeModel = realtimeClient.options.model - // 1) Source & capturer - let videoSource = realtimeClient.createVideoSource() + let currentRealtimeModel = realtimeManager.options.model + + let videoSource = realtimeManager.createVideoSource() let capturer = RTCCameraVideoCapturer(delegate: videoSource) let device = try AVCaptureDevice.pickCamera(position: cameraFacing) @@ -25,13 +25,13 @@ public enum RealtimeCameraCapture { ) let targetFPS = try device.pickFPS(for: format, preferred: currentRealtimeModel.fps) - // 3) Start capture try await startCameraCapture(capturer: capturer, device: device, format: format, fps: targetFPS) - let localVideoTrack = realtimeClient.createVideoTrack( + + let localVideoTrack = realtimeManager.createVideoTrack( source: videoSource, trackId: "video0" ) - // 4) Create track & stream + return ( RealtimeMediaStream(videoTrack: localVideoTrack, id: .localStream), capturer @@ -55,11 +55,10 @@ public enum RealtimeCameraCapture { @discardableResult public static func switchCamera( capturer: RTCCameraVideoCapturer, - realtimeClient: RealtimeClient, + realtimeManager: RealtimeManager, newPosition: AVCaptureDevice.Position ) async throws -> AVCaptureDevice.Position { - let currentRealtimeModel = realtimeClient.options.model - let newPosition: AVCaptureDevice.Position = newPosition + let currentRealtimeModel = realtimeManager.options.model let newDevice = try AVCaptureDevice.pickCamera(position: newPosition) let format = try newDevice.pickFormat( diff --git a/Sources/DecartSDK/DecartSDK.swift b/Sources/DecartSDK/DecartSDK.swift index b4ead5a..2cb2e6e 100644 --- a/Sources/DecartSDK/DecartSDK.swift +++ b/Sources/DecartSDK/DecartSDK.swift @@ -37,7 +37,7 @@ public struct DecartClient { self.decartConfiguration = decartConfiguration } - public func createRealtimeClient(options: RealtimeConfiguration) throws -> RealtimeClient { + public func createRealtimeManager(options: RealtimeConfiguration) throws -> RealtimeManager { let urlString = "\(decartConfiguration.signalingServerUrl)\(options.model.urlPath)?api_key=\(decartConfiguration.apiKey)&model=\(options.model.name)" @@ -46,7 +46,7 @@ public struct DecartClient { throw DecartError.invalidBaseURL(urlString) } - return try RealtimeClient( + return RealtimeManager( signalingServerURL: signalingServerURL, options: options ) diff --git a/Sources/DecartSDK/Realtime/RealtimeClient.swift b/Sources/DecartSDK/Realtime/RealtimeClient.swift deleted file mode 100644 index 7f421c4..0000000 --- a/Sources/DecartSDK/Realtime/RealtimeClient.swift +++ /dev/null @@ -1,94 +0,0 @@ -import Foundation -import WebRTC - -public final class RealtimeClient { - let webRTCManager: WebRTCManager - private let signalingServerURL: URL - public let options: RealtimeConfiguration - - public let events: AsyncStream - - public init(signalingServerURL: URL, options: RealtimeConfiguration) throws { - self.options = options - self.signalingServerURL = signalingServerURL - - self.webRTCManager = WebRTCManager( - realtimeConfig: options - ) - self.events = webRTCManager.signalingManager.events - } - - public func connect(localStream: RealtimeMediaStream) async throws -> RealtimeMediaStream { - webRTCManager.onWebrtcConnectedCallback = { [weak self] in - guard let self = self else { return } - self.setPrompt(self.options.initialState.prompt) - } - return try await connectWithRetry( - localStream: localStream, - maxRetries: 3, - permanentErrors: ["permission denied", "not allowed", "invalid session"] - ) - } - - public func disconnect() async { - await webRTCManager.disconnect() - } - - public func setPrompt(_ prompt: Prompt) { - webRTCManager.sendWebsocketMessage(.prompt(PromptMessage(prompt: prompt.text))) - } - - // MARK: - Private Helpers - - private func connectWithRetry( - localStream: RealtimeMediaStream, - maxRetries: Int, - permanentErrors: [String] - ) async throws -> RealtimeMediaStream { - var retries = 0 - var delay: TimeInterval = 1.0 - - while retries < maxRetries { - do { - try await webRTCManager.connect(url: signalingServerURL, localStream: localStream) - - guard - let remoteVideoTrack = getTransceivers().first(where: { $0.mediaType == .video } - )?.receiver.track as? RTCVideoTrack - else { - throw DecartError.webRTCError("Remote video track not found after connection.") - } - - let remoteAudioTrack = - getTransceivers().first(where: { $0.mediaType == .audio })?.receiver.track - as? RTCAudioTrack - - return RealtimeMediaStream( - videoTrack: remoteVideoTrack, - audioTrack: remoteAudioTrack, - id: .remoteStream - ) - } catch { - let errorMessage = error.localizedDescription.lowercased() - if permanentErrors.contains(where: { errorMessage.contains($0) }) { - DecartLogger.log( - "[RealtimeClient] Permanent error detected, aborting retries.", - level: .error - ) - throw error - } - - retries += 1 - if retries >= maxRetries { - DecartLogger.log("[RealtimeClient] Max retries reached.", level: .error) - throw error - } - - try await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) - delay = min(delay * 2, 10.0) - } - } - - throw DecartError.webRTCError("Connection failed after max retries.") - } -} diff --git a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift index 31f21cf..e974ff9 100644 --- a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift +++ b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift @@ -35,7 +35,7 @@ public struct RealtimeConfiguration: Sendable { public init( iceServers: [String] = ["stun:stun.l.google.com:19302"], - connectionTimeout: Int32 = 7000, + connectionTimeout: Int32 = 15000, pingInterval: Int32 = 2000 ) { self.iceServers = iceServers diff --git a/Sources/DecartSDK/Realtime/RealtimeClient+Media.swift b/Sources/DecartSDK/Realtime/RealtimeManager+Media.swift similarity index 69% rename from Sources/DecartSDK/Realtime/RealtimeClient+Media.swift rename to Sources/DecartSDK/Realtime/RealtimeManager+Media.swift index 8d4c5a9..ab26555 100644 --- a/Sources/DecartSDK/Realtime/RealtimeClient+Media.swift +++ b/Sources/DecartSDK/Realtime/RealtimeManager+Media.swift @@ -1,27 +1,25 @@ import Foundation import WebRTC -extension RealtimeClient { - // MARK: - Media Factory Methods - +extension RealtimeManager { public func getTransceivers() -> [RTCRtpTransceiver] { - webRTCManager.peerConnection.transceivers + webRTCClient.transceivers } public func createAudioSource(constraints: RTCMediaConstraints? = nil) -> RTCAudioSource { - webRTCManager.factory.audioSource(with: constraints) + webRTCClient.createAudioSource(constraints: constraints) } public func createAudioTrack(source: RTCAudioSource, trackId: String) -> RTCAudioTrack { - webRTCManager.factory.audioTrack(with: source, trackId: trackId) + webRTCClient.createAudioTrack(source: source, trackId: trackId) } public func createVideoSource() -> RTCVideoSource { - webRTCManager.factory.videoSource() + webRTCClient.createVideoSource() } public func createVideoTrack(source: RTCVideoSource, trackId: String) -> RTCVideoTrack { - webRTCManager.factory.videoTrack(with: source, trackId: trackId) + webRTCClient.createVideoTrack(source: source, trackId: trackId) } public func createLocalVideoTrack() -> (RTCVideoTrack, RTCCameraVideoCapturer) { @@ -31,4 +29,3 @@ extension RealtimeClient { return (videoTrack, videoCapturer) } } - diff --git a/Sources/DecartSDK/Realtime/RealtimeManager.swift b/Sources/DecartSDK/Realtime/RealtimeManager.swift new file mode 100644 index 0000000..d3e489e --- /dev/null +++ b/Sources/DecartSDK/Realtime/RealtimeManager.swift @@ -0,0 +1,167 @@ +import Foundation +@preconcurrency import WebRTC + +public final class RealtimeManager: @unchecked Sendable { + public let options: RealtimeConfiguration + public let events: AsyncStream + + let webRTCClient: WebRTCClient + private var webSocketClient: WebSocketClient? + + private let signalingServerURL: URL + private let stateContinuation: AsyncStream.Continuation + private var webSocketListenerTask: Task? + private var connectionStateListenerTask: Task? + + private var connectionState: DecartRealtimeConnectionState = .idle { + didSet { + guard oldValue != connectionState else { return } + stateContinuation.yield(connectionState) + } + } + + public init(signalingServerURL: URL, options: RealtimeConfiguration) { + self.signalingServerURL = signalingServerURL + self.options = options + + let (stream, continuation) = AsyncStream.makeStream( + of: DecartRealtimeConnectionState.self, + bufferingPolicy: .bufferingNewest(1) + ) + self.events = stream + self.stateContinuation = continuation + self.webRTCClient = WebRTCClient() + } + + // MARK: - Public API + + public func connect(localStream: RealtimeMediaStream) async throws -> RealtimeMediaStream { + connectionState = .connecting + + let wsClient = WebSocketClient() + webSocketClient = wsClient + await wsClient.connect(url: signalingServerURL) + setupWebSocketListener(wsClient) + + webRTCClient.createPeerConnection( + config: options.connection.makeRTCConfiguration(), + constraints: options.media.connectionConstraints, + sendMessage: { [weak self] in self?.sendMessage($0) } + ) + setupConnectionStateListener() + + webRTCClient.addTrack(localStream.videoTrack, streamIds: [localStream.id]) + if let audioTrack = localStream.audioTrack { + webRTCClient.addTrack(audioTrack, streamIds: [localStream.id]) + } + + webRTCClient.configureVideoTransceiver(videoConfig: options.media.video) + + let offer = try await webRTCClient.createOffer(constraints: options.media.offerConstraints) + try await webRTCClient.setLocalDescription(offer) + sendMessage(.offer(OfferMessage(sdp: offer.sdp))) + + try await waitForConnection(timeout: TimeInterval(options.connection.connectionTimeout) / 1000) + + return try extractRemoteStream() + } + + public func disconnect() async { + await cleanup() + } + + public func setPrompt(_ prompt: Prompt) { + sendMessage(.prompt(PromptMessage(prompt: prompt.text))) + } + + public func switchCamera(rotateY: Int) { + sendMessage(.switchCamera(SwitchCameraMessage(rotateY: rotateY))) + } + + // MARK: - Private + + private func waitForConnection(timeout: TimeInterval) async throws { + let startTime = Date() + while connectionState != .connected { + if connectionState == .error || connectionState == .disconnected { + throw DecartError.webRTCError("Connection failed") + } + if Date().timeIntervalSince(startTime) > timeout { + throw DecartError.webRTCError("Connection timeout") + } + try await Task.sleep(nanoseconds: 100_000_000) + } + sendMessage(.prompt(PromptMessage(prompt: options.initialState.prompt.text))) + } + + private func extractRemoteStream() throws -> RealtimeMediaStream { + guard let videoTransceiver = webRTCClient.transceivers.first(where: { $0.mediaType == .video }) else { + throw DecartError.webRTCError("Video transceiver not found") + } + guard let remoteVideoTrack = videoTransceiver.receiver.track as? RTCVideoTrack else { + throw DecartError.webRTCError("Remote video track not found") + } + let remoteAudioTrack = webRTCClient.transceivers + .first(where: { $0.mediaType == .audio })? + .receiver.track as? RTCAudioTrack + + return RealtimeMediaStream( + videoTrack: remoteVideoTrack, + audioTrack: remoteAudioTrack, + id: .remoteStream + ) + } + + private func setupWebSocketListener(_ wsClient: WebSocketClient) { + webSocketListenerTask?.cancel() + webSocketListenerTask = Task { [weak self] in + do { + for try await message in wsClient.websocketEventStream { + guard !Task.isCancelled, let self else { return } + try await self.webRTCClient.handleSignalingMessage(message) + } + } catch { + self?.connectionState = .error + } + } + } + + private func setupConnectionStateListener() { + connectionStateListenerTask?.cancel() + guard let stateStream = webRTCClient.connectionStateStream else { return } + connectionStateListenerTask = Task { [weak self] in + for await rtcState in stateStream { + guard !Task.isCancelled, let self else { return } + switch rtcState { + case .connected: self.connectionState = .connected + case .failed, .closed, .disconnected: self.connectionState = .disconnected + case .connecting where self.connectionState == .idle: self.connectionState = .connecting + default: break + } + } + } + } + + private func sendMessage(_ message: OutgoingWebSocketMessage) { + guard let webSocketClient else { return } + Task { try? await webSocketClient.send(message) } + } + + private func cleanup() async { + connectionState = .disconnected + webSocketListenerTask?.cancel() + webSocketListenerTask = nil + connectionStateListenerTask?.cancel() + connectionStateListenerTask = nil + webRTCClient.closePeerConnection() + await webSocketClient?.disconnect() + webSocketClient = nil + } + + deinit { + webSocketListenerTask?.cancel() + connectionStateListenerTask?.cancel() + webRTCClient.closePeerConnection() + stateContinuation.finish() + } +} diff --git a/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift b/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift new file mode 100644 index 0000000..894761d --- /dev/null +++ b/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift @@ -0,0 +1,50 @@ +import Foundation +@preconcurrency import WebRTC + +struct SignalingClient { + private let peerConnection: RTCPeerConnection + private let factory: RTCPeerConnectionFactory + private let sendMessage: (OutgoingWebSocketMessage) -> Void + + init( + peerConnection: RTCPeerConnection, + factory: RTCPeerConnectionFactory, + sendMessage: @escaping (OutgoingWebSocketMessage) -> Void + ) { + self.peerConnection = peerConnection + self.factory = factory + self.sendMessage = sendMessage + } + + func handleMessage(_ message: IncomingWebSocketMessage) async throws { + switch message { + case .offer(let msg): + let sdp = RTCSessionDescription(type: .offer, sdp: msg.sdp) + try await peerConnection.setRemoteDescription(sdp) + let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil) + guard let answer = try await peerConnection.answer(for: constraints) else { + throw DecartError.webRTCError("Failed to create answer") + } + try await peerConnection.setLocalDescription(answer) + sendMessage(.answer(AnswerMessage(type: "answer", sdp: answer.sdp))) + + case .answer(let msg): + let sdp = RTCSessionDescription(type: .answer, sdp: msg.sdp) + try await peerConnection.setRemoteDescription(sdp) + + case .iceCandidate(let msg): + let candidate = RTCIceCandidate( + sdp: msg.candidate.candidate, + sdpMLineIndex: msg.candidate.sdpMLineIndex, + sdpMid: msg.candidate.sdpMid + ) + try await peerConnection.add(candidate) + + case .error(let msg): + throw DecartError.serverError(msg.message ?? msg.error ?? "Unknown server error") + + case .sessionId, .promptAck: + break + } + } +} diff --git a/Sources/DecartSDK/Realtime/WebRTC/SignalingManager.swift b/Sources/DecartSDK/Realtime/WebRTC/SignalingManager.swift deleted file mode 100644 index f5beb58..0000000 --- a/Sources/DecartSDK/Realtime/WebRTC/SignalingManager.swift +++ /dev/null @@ -1,127 +0,0 @@ -import Foundation -@preconcurrency import WebRTC - -/// Manages WebSocket signaling connection with AsyncStream-based message delivery -actor SignalingManager { - private let webSocket: WebSocketClient - private let peerConnection: RTCPeerConnection - private var wsListenerTask: Task? - - private var state: DecartRealtimeConnectionState = .idle { - didSet { - guard oldValue != state else { return } - stateContinuation.yield(state) - } - } - - private let stateContinuation: AsyncStream.Continuation - nonisolated let events: AsyncStream - - init(pc: RTCPeerConnection) { - peerConnection = pc - webSocket = WebSocketClient() - let (stream, continuation) = AsyncStream.makeStream( - of: DecartRealtimeConnectionState.self, - bufferingPolicy: .bufferingNewest(1) - ) - events = stream - stateContinuation = continuation - } - - func connect(url: URL, timeout: TimeInterval = 30) async { - state = .connecting - await webSocket.connect(url: url) - let task = Task { - let eventStream = self.webSocket.websocketEventStream - do { - for try await event in eventStream { - if Task.isCancelled { return } - await self.handle(event) - } - } catch { - DecartLogger.log("error in signaling loop: \(error)", level: .error) - self.state = .error - } - } - if wsListenerTask != nil { - wsListenerTask?.cancel() - wsListenerTask = nil - } - - wsListenerTask = task - } - - func updatePeerConnectionState(_ newState: RTCPeerConnectionState) { - switch newState { - case .connected: - state = .connected - case .failed, .closed: - state = .disconnected - case .connecting: - // Keep as connecting if we are already there, or set it if we were idle - if state != .connecting, state != .connected { - state = .connecting - } - case .disconnected: - state = .disconnected - case .new: - break // Initial state, usually - @unknown default: - break - } - } - - func handle(_ message: IncomingWebSocketMessage) async { - do { - switch message { - case .offer(let msg): - let sdp = RTCSessionDescription(type: .offer, sdp: msg.sdp) - try await peerConnection.setRemoteDescription(sdp) - - let constraints = RTCMediaConstraints( - mandatoryConstraints: nil, - optionalConstraints: nil - ) - - guard let answer = try? await peerConnection.answer(for: constraints) else { - DecartLogger.log("[WebRTCConnection] Failed to create answer", level: .error) - throw DecartError.webRTCError("failed to create answer, check logs") - } - - try await peerConnection.setLocalDescription(answer) - await send(.answer(AnswerMessage(type: "answer", sdp: answer.sdp))) - - case .answer(let msg): - let sdp = RTCSessionDescription(type: .answer, sdp: msg.sdp) - try await peerConnection.setRemoteDescription(sdp) - - case .iceCandidate(let msg): - let candidate = RTCIceCandidate( - sdp: msg.candidate.candidate, - sdpMLineIndex: msg.candidate.sdpMLineIndex, - sdpMid: msg.candidate.sdpMid - ) - try await peerConnection.add(candidate) - } - } catch { - DecartLogger.log("error while handling websocket message: \(error)", level: .error) - } - } - - nonisolated func send(_ message: OutgoingWebSocketMessage) { - Task { - do { - try await webSocket.send(message) - } catch { - DecartLogger.log("error while sending websocket message: \(error)", level: .error) - } - } - } - - func disconnect() async { - state = .disconnected - await webSocket.disconnect() - wsListenerTask?.cancel() - wsListenerTask = nil - } -} diff --git a/Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift b/Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift index 60b159f..bcf3132 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift @@ -43,10 +43,7 @@ struct IceCandidateMessage: Codable, Sendable { init(candidate: RTCIceCandidate) { guard let sdpMid = candidate.sdpMid else { - DecartLogger.log("found invalid candidate without sdpMid", level: .warning) - fatalError( - "found invalid candidate without sdpMid. This should never happen." - ) + fatalError("found invalid candidate without sdpMid") } self.type = "ice-candidate" @@ -78,15 +75,35 @@ struct SwitchCameraMessage: Codable, Sendable { } } +struct ServerErrorMessage: Codable, Sendable { + let type: String + let message: String? + let error: String? +} + +struct SessionIdMessage: Codable, Sendable { + let type: String + let sessionId: String? + let session_id: String? + + var id: String? { sessionId ?? session_id } +} + +struct PromptAckMessage: Codable, Sendable { + let type: String +} + enum IncomingWebSocketMessage: Codable, Sendable { case offer(OfferMessage) case answer(AnswerMessage) case iceCandidate(IceCandidateMessage) + case error(ServerErrorMessage) + case sessionId(SessionIdMessage) + case promptAck(PromptAckMessage) init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) let type = try container.decode(String.self, forKey: .type) - DecartLogger.log("got incoming message \(type)", level: .info) switch type { case "offer": @@ -95,6 +112,12 @@ enum IncomingWebSocketMessage: Codable, Sendable { self = try .answer(AnswerMessage(from: decoder)) case "ice-candidate": self = try .iceCandidate(IceCandidateMessage(from: decoder)) + case "error": + self = try .error(ServerErrorMessage(from: decoder)) + case "session_id": + self = try .sessionId(SessionIdMessage(from: decoder)) + case "prompt_ack": + self = try .promptAck(PromptAckMessage(from: decoder)) default: throw DecodingError.dataCorruptedError( forKey: .type, @@ -112,6 +135,12 @@ enum IncomingWebSocketMessage: Codable, Sendable { try msg.encode(to: encoder) case .iceCandidate(let msg): try msg.encode(to: encoder) + case .error(let msg): + try msg.encode(to: encoder) + case .sessionId(let msg): + try msg.encode(to: encoder) + case .promptAck(let msg): + try msg.encode(to: encoder) } } diff --git a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift new file mode 100644 index 0000000..ebf3b68 --- /dev/null +++ b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift @@ -0,0 +1,154 @@ +import Foundation +@preconcurrency import WebRTC + +final class WebRTCClient { + let factory: RTCPeerConnectionFactory + private(set) var peerConnection: RTCPeerConnection? + private(set) var connectionStateStream: AsyncStream? + + private var delegateHandler: WebRTCDelegateHandler? + private var signalingClient: SignalingClient? + private var connectionStateContinuation: AsyncStream.Continuation? + + init() { + #if IS_DEVELOPMENT + RTCSetMinDebugLogLevel(.verbose) + #endif + RTCInitializeSSL() + + let videoEncoderFactory = RTCDefaultVideoEncoderFactory() + let videoDecoderFactory = RTCDefaultVideoDecoderFactory() + self.factory = RTCPeerConnectionFactory( + encoderFactory: videoEncoderFactory, + decoderFactory: videoDecoderFactory + ) + } + + func createPeerConnection( + config: RTCConfiguration, + constraints: RTCMediaConstraints, + sendMessage: @escaping (OutgoingWebSocketMessage) -> Void + ) { + let (stream, continuation) = AsyncStream.makeStream(of: RTCPeerConnectionState.self) + self.connectionStateStream = stream + self.connectionStateContinuation = continuation + + self.delegateHandler = WebRTCDelegateHandler( + sendMessage: sendMessage, + connectionStateContinuation: continuation + ) + + self.peerConnection = factory.peerConnection( + with: config, + constraints: constraints, + delegate: delegateHandler + )! + + self.signalingClient = SignalingClient( + peerConnection: peerConnection!, + factory: factory, + sendMessage: sendMessage + ) + } + + func handleSignalingMessage(_ message: IncomingWebSocketMessage) async throws { + try await signalingClient?.handleMessage(message) + } + + // MARK: - Track Operations + + func addTrack(_ track: RTCMediaStreamTrack, streamIds: [String]) { + peerConnection?.add(track, streamIds: streamIds) + } + + func configureVideoTransceiver(videoConfig: RealtimeConfiguration.VideoConfig) { + if let transceiver = peerConnection?.transceivers.first(where: { $0.mediaType == .video }) { + videoConfig.configure(transceiver: transceiver, factory: factory) + } + } + + var transceivers: [RTCRtpTransceiver] { + peerConnection?.transceivers ?? [] + } + + // MARK: - SDP Operations + + func createOffer(constraints: RTCMediaConstraints) async throws -> RTCSessionDescription { + guard let peerConnection else { + throw DecartError.webRTCError("peer connection not initialized") + } + guard let offer = try await peerConnection.offer(for: constraints) else { + throw DecartError.webRTCError("failed to create offer") + } + return offer + } + + func createAnswer(constraints: RTCMediaConstraints) async throws -> RTCSessionDescription { + guard let peerConnection else { + throw DecartError.webRTCError("peer connection not initialized") + } + guard let answer = try await peerConnection.answer(for: constraints) else { + throw DecartError.webRTCError("failed to create answer") + } + return answer + } + + func setLocalDescription(_ sdp: RTCSessionDescription) async throws { + guard let peerConnection else { + throw DecartError.webRTCError("peer connection not initialized") + } + try await peerConnection.setLocalDescription(sdp) + } + + func setRemoteDescription(_ sdp: RTCSessionDescription) async throws { + guard let peerConnection else { + throw DecartError.webRTCError("peer connection not initialized") + } + try await peerConnection.setRemoteDescription(sdp) + } + + // MARK: - ICE Operations + + func addIceCandidate(_ candidate: RTCIceCandidate) async throws { + guard let peerConnection else { + throw DecartError.webRTCError("peer connection not initialized") + } + try await peerConnection.add(candidate) + } + + // MARK: - Media Factory + + func createVideoSource() -> RTCVideoSource { + factory.videoSource() + } + + func createVideoTrack(source: RTCVideoSource, trackId: String) -> RTCVideoTrack { + factory.videoTrack(with: source, trackId: trackId) + } + + func createAudioSource(constraints: RTCMediaConstraints?) -> RTCAudioSource { + factory.audioSource(with: constraints) + } + + func createAudioTrack(source: RTCAudioSource, trackId: String) -> RTCAudioTrack { + factory.audioTrack(with: source, trackId: trackId) + } + + // MARK: - Cleanup + + func closePeerConnection() { + delegateHandler?.cleanup() + connectionStateContinuation?.finish() + peerConnection?.close() + peerConnection?.delegate = nil + peerConnection = nil + signalingClient = nil + delegateHandler = nil + connectionStateStream = nil + connectionStateContinuation = nil + } + + deinit { + closePeerConnection() + } +} diff --git a/Sources/DecartSDK/Realtime/WebRTC/WebRTCDelegateHandler.swift b/Sources/DecartSDK/Realtime/WebRTC/WebRTCDelegateHandler.swift new file mode 100644 index 0000000..5eacef0 --- /dev/null +++ b/Sources/DecartSDK/Realtime/WebRTC/WebRTCDelegateHandler.swift @@ -0,0 +1,57 @@ +import Foundation +@preconcurrency import WebRTC + +final class WebRTCDelegateHandler: NSObject { + private let sendMessage: (OutgoingWebSocketMessage) -> Void + private let connectionStateContinuation: AsyncStream.Continuation + + init( + sendMessage: @escaping (OutgoingWebSocketMessage) -> Void, + connectionStateContinuation: AsyncStream.Continuation + ) { + self.sendMessage = sendMessage + self.connectionStateContinuation = connectionStateContinuation + } + + func cleanup() { + connectionStateContinuation.finish() + } +} + +extension WebRTCDelegateHandler: RTCPeerConnectionDelegate { + func peerConnection( + _ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState + ) {} + + func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) {} + + func peerConnection(_ peerConnection: RTCPeerConnection, didRemove stream: RTCMediaStream) {} + + func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) {} + + func peerConnection( + _ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState + ) {} + + func peerConnection( + _ peerConnection: RTCPeerConnection, didChange newState: RTCIceGatheringState + ) {} + + func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) { + sendMessage(.iceCandidate(IceCandidateMessage(candidate: candidate))) + } + + func peerConnection( + _ peerConnection: RTCPeerConnection, didRemove candidates: [RTCIceCandidate] + ) {} + + func peerConnection(_ peerConnection: RTCPeerConnection, didOpen dataChannel: RTCDataChannel) {} + + func peerConnection( + _ peerConnection: RTCPeerConnection, didChange newState: RTCPeerConnectionState + ) { + connectionStateContinuation.yield(newState) + } +} + +extension WebRTCDelegateHandler: @unchecked Sendable {} diff --git a/Sources/DecartSDK/Realtime/WebRTC/WebRTCManager.swift b/Sources/DecartSDK/Realtime/WebRTC/WebRTCManager.swift deleted file mode 100644 index 4a31b10..0000000 --- a/Sources/DecartSDK/Realtime/WebRTC/WebRTCManager.swift +++ /dev/null @@ -1,138 +0,0 @@ -import Foundation -@preconcurrency import WebRTC - -final class WebRTCManager: NSObject { - let factory: RTCPeerConnectionFactory - - @objc let peerConnection: RTCPeerConnection - - let signalingManager: SignalingManager - private let realtimeConfig: RealtimeConfiguration - var onWebrtcConnectedCallback: (() -> Void)? - - init( - realtimeConfig: RealtimeConfiguration - ) { - #if IS_DEVELOPMENT - RTCSetMinDebugLogLevel(.verbose) - #endif - RTCInitializeSSL() - let videoEncoderFactory = RTCDefaultVideoEncoderFactory() - let videoDecoderFactory = RTCDefaultVideoDecoderFactory() - self.factory = RTCPeerConnectionFactory( - encoderFactory: videoEncoderFactory, decoderFactory: videoDecoderFactory) - - let config = realtimeConfig.connection.makeRTCConfiguration() - let constraints = realtimeConfig.media.connectionConstraints - - self.peerConnection = factory.peerConnection( - with: config, - constraints: constraints, - delegate: nil)! - self.signalingManager = SignalingManager(pc: peerConnection) - self.realtimeConfig = realtimeConfig - super.init() - peerConnection.delegate = self - } - - func connect(url: URL, localStream: RealtimeMediaStream, timeout: TimeInterval = 30) - async throws - { - do { - peerConnection.add(localStream.videoTrack, streamIds: [localStream.id]) - if let audioTrack = localStream.audioTrack { - peerConnection.add(audioTrack, streamIds: [localStream.id]) - } - - if let transceiver = peerConnection.transceivers.first(where: { $0.mediaType == .video } - ) { - await realtimeConfig.media.video.configure( - transceiver: transceiver, factory: factory) - } - - await signalingManager.connect(url: url) - try await sendOffer() - } catch { - DecartLogger.log("failed to create webrtc connection", level: .error) - await cleanup() - throw error - } - } - - func disconnect() async { - await cleanup() - } - - func sendWebsocketMessage(_ message: OutgoingWebSocketMessage) { - signalingManager.send(message) - } - - private func cleanup() async { - peerConnection.close() - peerConnection.delegate = nil - await signalingManager.disconnect() - } - - private func handleConnectionStateChange(_ rtcState: RTCPeerConnectionState) { - DecartLogger.log("got new state: \(rtcState)", level: .info) - Task { - await signalingManager.updatePeerConnectionState(rtcState) - } - } - - private func sendOffer() async throws { - let constraints = realtimeConfig.media.offerConstraints - guard let offer = try? await peerConnection.offer(for: constraints) else { - throw DecartError.webRTCError("failed to create offer, aborting") - } - - try await peerConnection.setLocalDescription(offer) - signalingManager.send(.offer(OfferMessage(sdp: offer.sdp))) - } - - deinit { - DecartLogger.log("WebRTCManager deinit", level: .info) - } -} - -extension WebRTCManager: RTCPeerConnectionDelegate, @unchecked Sendable { - func peerConnection( - _ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState - ) {} - - func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) {} - - func peerConnection(_ peerConnection: RTCPeerConnection, didRemove stream: RTCMediaStream) {} - - func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) {} - - func peerConnection( - _ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState - ) {} - - func peerConnection( - _ peerConnection: RTCPeerConnection, didChange newState: RTCIceGatheringState - ) {} - - func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) { - signalingManager.send( - OutgoingWebSocketMessage.iceCandidate( - .init(candidate: candidate))) - } - - func peerConnection( - _ peerConnection: RTCPeerConnection, didRemove candidates: [RTCIceCandidate] - ) {} - - func peerConnection(_ peerConnection: RTCPeerConnection, didOpen dataChannel: RTCDataChannel) {} - - func peerConnection( - _ peerConnection: RTCPeerConnection, didChange newState: RTCPeerConnectionState - ) { - if newState == .connected { - onWebrtcConnectedCallback?() - } - - handleConnectionStateChange(newState) - } -} diff --git a/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift b/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift index 32417b9..d289c0a 100644 --- a/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift +++ b/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift @@ -1,24 +1,15 @@ -// -// WebSocketClient.swift -// DecartSDK -// -// Created by Alon Bar-el on 03/11/2025. -// - import Foundation -import Observation actor WebSocketClient { var isConnected: Bool = false - var socketError: DecartError? private var stream: SocketStream? private var listeningTask: Task? private let decoder = JSONDecoder() private let encoder = JSONEncoder() - private var eventStreamContinuation: AsyncStream.Continuation - let websocketEventStream: AsyncStream + private nonisolated(unsafe) let eventStreamContinuation: AsyncStream.Continuation + nonisolated let websocketEventStream: AsyncStream init() { let (websocketEventStream, eventStreamContinuation) = AsyncStream.makeStream(of: IncomingWebSocketMessage.self) @@ -27,13 +18,12 @@ actor WebSocketClient { } func connect(url: URL) { - if stream != nil { return } + guard stream == nil else { return } let socketConnection = URLSession.shared.webSocketTask(with: url) stream = SocketStream(task: socketConnection) + isConnected = true listeningTask = Task { [weak self] in - guard let self = self, let stream = await self.stream else { - return - } + guard let self, let stream = await self.stream else { return } do { for try await msg in stream { switch msg { @@ -47,8 +37,6 @@ actor WebSocketClient { } } } catch { - DecartLogger - .log("error in ws listening loop: \(error)", level: .error) await self.eventStreamContinuation.finish() } } @@ -56,35 +44,20 @@ actor WebSocketClient { private func handleIncomingMessage(_ text: String) async { guard let data = text.data(using: .utf8) else { return } - - do { - let message = try decoder.decode(IncomingWebSocketMessage.self, from: data) - eventStreamContinuation.yield(message) - } catch { - DecartLogger - .log("error while handling incoming message: \(error)", level: .error) - eventStreamContinuation.finish() - } + guard let message = try? decoder.decode(IncomingWebSocketMessage.self, from: data) else { return } + eventStreamContinuation.yield(message) } func send(_ message: T) throws { - guard let stream = stream else { - DecartLogger.log("tried to send ws message when its closed", level: .warning) - return - } - + guard let stream else { return } let data = try encoder.encode(message) guard let jsonString = String(data: data, encoding: .utf8) else { - DecartLogger.log("unable to encode message", level: .warning) throw DecartError.websocketError("unable to encode message") } - Task { [stream] in - try await stream.sendMessage(.string(jsonString)) - } + Task { [stream] in try await stream.sendMessage(.string(jsonString)) } } func disconnect() async { - DecartLogger.log("disconnecting from websocket", level: .info) eventStreamContinuation.finish() listeningTask?.cancel() listeningTask = nil @@ -93,5 +66,7 @@ actor WebSocketClient { isConnected = false } - deinit { DecartLogger.log("Websocket Client deinit", level: .info) } + deinit { + eventStreamContinuation.finish() + } } diff --git a/Sources/DecartSDK/Shared/DecartError.swift b/Sources/DecartSDK/Shared/DecartError.swift index a8ec5f3..176fd96 100644 --- a/Sources/DecartSDK/Shared/DecartError.swift +++ b/Sources/DecartSDK/Shared/DecartError.swift @@ -11,6 +11,7 @@ public enum DecartError: Error { case connectionTimeout case websocketError(String) case networkError(Error) + case serverError(String) public var errorDescription: String? { switch self { @@ -37,6 +38,8 @@ public enum DecartError: Error { return "WebSocket error: \(message)" case .networkError(let error): return "Network error: \(error.localizedDescription)" + case .serverError(let message): + return "Server error: \(message)" } } @@ -62,6 +65,8 @@ public enum DecartError: Error { return "WEBSOCKET_ERROR" case .networkError: return "NETWORK_ERROR" + case .serverError: + return "SERVER_ERROR" } } } From 22e33c89add7ab7993f43d6eba1b76bdf1d8d6c0 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 26 Nov 2025 12:00:10 +0200 Subject: [PATCH 02/23] now uses swift 6 --- Example.xcodeproj/project.pbxproj | 6 ++++-- Example/Example/DecartSDK/DecartRealtimeManager.swift | 4 ++-- Sources/DecartSDK/Capture/RealtimeCameraCapture.swift | 2 +- Sources/DecartSDK/Process/ProcessClient.swift | 4 ++-- .../Realtime/WebRTC/RTCPeerConnection+Ext.swift | 1 + Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift | 11 ++++++----- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Example.xcodeproj/project.pbxproj b/Example.xcodeproj/project.pbxproj index b2b7148..42dd79a 100644 --- a/Example.xcodeproj/project.pbxproj +++ b/Example.xcodeproj/project.pbxproj @@ -220,6 +220,7 @@ SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_STRICT_CONCURRENCY = complete; + SWIFT_VERSION = 6.0; }; name = Debug; }; @@ -278,6 +279,7 @@ SWIFT_COMPILATION_MODE = wholemodule; SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; SWIFT_STRICT_CONCURRENCY = complete; + SWIFT_VERSION = 6.0; VALIDATE_PRODUCT = YES; }; name = Release; @@ -312,7 +314,7 @@ SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_STRICT_CONCURRENCY = complete; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; - SWIFT_VERSION = 5.0; + SWIFT_VERSION = 6.0; TARGETED_DEVICE_FAMILY = "1,2"; }; name = Debug; @@ -347,7 +349,7 @@ SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_STRICT_CONCURRENCY = complete; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; - SWIFT_VERSION = 5.0; + SWIFT_VERSION = 6.0; TARGETED_DEVICE_FAMILY = "1,2"; }; name = Release; diff --git a/Example/Example/DecartSDK/DecartRealtimeManager.swift b/Example/Example/DecartSDK/DecartRealtimeManager.swift index 250f346..37c5b40 100644 --- a/Example/Example/DecartSDK/DecartRealtimeManager.swift +++ b/Example/Example/DecartSDK/DecartRealtimeManager.swift @@ -131,11 +131,11 @@ final class DecartRealtimeManager: RealtimeManagerProtocol { // For now, just updating state is enough as UI reacts to it. } } + DecartLogger.log("Event monitoring task completed.", level: .info) } } func cleanup() async { - DecartLogger.log("Starting cleanup...", level: .info) eventTask?.cancel() eventTask = nil @@ -151,6 +151,6 @@ final class DecartRealtimeManager: RealtimeManagerProtocol { localMediaStream = nil connectionState = .idle - DecartLogger.log("Cleanup complete.", level: .success) + DecartLogger.log("Cleanup of realtime modelview completed.", level: .success) } } diff --git a/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift b/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift index 81ebae1..5b86d0b 100644 --- a/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift +++ b/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift @@ -5,7 +5,7 @@ // Created by Alon Bar-el on 05/11/2025. // import AVFoundation -import WebRTC +@preconcurrency import WebRTC #if !targetEnvironment(simulator) public enum RealtimeCameraCapture { diff --git a/Sources/DecartSDK/Process/ProcessClient.swift b/Sources/DecartSDK/Process/ProcessClient.swift index ba60fd5..e69953d 100644 --- a/Sources/DecartSDK/Process/ProcessClient.swift +++ b/Sources/DecartSDK/Process/ProcessClient.swift @@ -1,6 +1,6 @@ import Foundation -public struct ProcessClient { +public struct ProcessClient: Sendable { private let session: URLSession private let request: URLRequest @@ -188,7 +188,7 @@ public struct ProcessClient { // MARK: - Process - public func process() async throws -> Data { + public nonisolated func process() async throws -> Data { let (data, response) = try await session.data(for: request) guard let httpResponse = response as? HTTPURLResponse else { diff --git a/Sources/DecartSDK/Realtime/WebRTC/RTCPeerConnection+Ext.swift b/Sources/DecartSDK/Realtime/WebRTC/RTCPeerConnection+Ext.swift index 86ee2a7..b42d003 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/RTCPeerConnection+Ext.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/RTCPeerConnection+Ext.swift @@ -7,6 +7,7 @@ @preconcurrency import WebRTC extension RTCSessionDescription: @unchecked @retroactive Sendable {} +extension RTCCameraVideoCapturer: @unchecked @retroactive Sendable {} extension RTCPeerConnection { func offer(for constraints: RTCMediaConstraints) async throws -> RTCSessionDescription? { diff --git a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift index ebf3b68..a6c2f6f 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift @@ -30,21 +30,21 @@ final class WebRTCClient { sendMessage: @escaping (OutgoingWebSocketMessage) -> Void ) { let (stream, continuation) = AsyncStream.makeStream(of: RTCPeerConnectionState.self) - self.connectionStateStream = stream - self.connectionStateContinuation = continuation + connectionStateStream = stream + connectionStateContinuation = continuation - self.delegateHandler = WebRTCDelegateHandler( + delegateHandler = WebRTCDelegateHandler( sendMessage: sendMessage, connectionStateContinuation: continuation ) - self.peerConnection = factory.peerConnection( + peerConnection = factory.peerConnection( with: config, constraints: constraints, delegate: delegateHandler )! - self.signalingClient = SignalingClient( + signalingClient = SignalingClient( peerConnection: peerConnection!, factory: factory, sendMessage: sendMessage @@ -149,6 +149,7 @@ final class WebRTCClient { } deinit { + DecartLogger.log("Webrtc client deinitialized", level: .info) closePeerConnection() } } From 45ecc798bf6cc55b6cbd31aab4c8251d98bf7682 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 26 Nov 2025 15:57:40 +0200 Subject: [PATCH 03/23] fixes cpu usage + webRTC configuration --- Example/Example/ContentView.swift | 20 +-- .../DecartSDK/DecartRealtimeManager.swift | 73 ++++---- .../Example/Views/DraggableRTCVideoView.swift | 2 +- Example/Example/Views/RealtimeView.swift | 122 +++++++------ .../DecartSDK/Capture/CaptureExtensions.swift | 6 +- .../Capture/RealtimeCameraCapture.swift | 80 --------- .../DecartSDK/Capture/RealtimeCapture.swift | 164 ++++++++++++++++++ .../Realtime/RealtimeConfiguration.swift | 5 +- .../DecartSDK/Realtime/RealtimeManager.swift | 13 +- .../Realtime/WebRTC/SignalingModel.swift | 13 -- .../Realtime/WebRTC/WebRTCClient.swift | 32 +++- .../WebRTC/WebRTCDelegateHandler.swift | 5 + 12 files changed, 309 insertions(+), 226 deletions(-) delete mode 100644 Sources/DecartSDK/Capture/RealtimeCameraCapture.swift create mode 100644 Sources/DecartSDK/Capture/RealtimeCapture.swift diff --git a/Example/Example/ContentView.swift b/Example/Example/ContentView.swift index 538be83..c5a0848 100644 --- a/Example/Example/ContentView.swift +++ b/Example/Example/ContentView.swift @@ -10,7 +10,7 @@ import SwiftUI struct ContentView: View { var body: some View { - NavigationView { + NavigationStack { List { Section(header: Text("Realtime")) { ForEach(RealtimeModel.allCases, id: \.self) { model in @@ -22,29 +22,21 @@ struct ContentView: View { Section(header: Text("Image Generation")) { ForEach(ImageModel.allCases, id: \.self) { model in - NavigationLink( - destination: GenerateImageView( - model: model - ) - ) { - Text("Image - \(model.rawValue)") + NavigationLink("Image - \(model.rawValue)") { + GenerateImageView(model: model) } } } Section(header: Text("Video Generation")) { ForEach(VideoModel.allCases, id: \.self) { model in - NavigationLink( - destination: GenerateVideoView( - model: model - ) - ) { - Text("Video - \(model.rawValue)") + NavigationLink("Video - \(model.rawValue)") { + GenerateVideoView(model: model) } } } } - .navigationBarTitle("Example") + .navigationTitle("Example") } } } diff --git a/Example/Example/DecartSDK/DecartRealtimeManager.swift b/Example/Example/DecartSDK/DecartRealtimeManager.swift index 37c5b40..f43a416 100644 --- a/Example/Example/DecartSDK/DecartRealtimeManager.swift +++ b/Example/Example/DecartSDK/DecartRealtimeManager.swift @@ -8,7 +8,7 @@ import Combine import DecartSDK import Factory import SwiftUI -import WebRTC +@preconcurrency import WebRTC @MainActor @Observable @@ -35,14 +35,16 @@ final class DecartRealtimeManager: RealtimeManagerProtocol { @ObservationIgnored private var realtimeManager: RealtimeManager? + #if !targetEnvironment(simulator) @ObservationIgnored - private var videoCapturer: RTCCameraVideoCapturer? + private var capture: RealtimeCapture? + #endif @ObservationIgnored private var eventTask: Task? init( currentPrompt: Prompt, - isMirroringEnabled: Bool = true // since the initial camera is the front facing one + isMirroringEnabled: Bool = true ) { self.currentPrompt = currentPrompt self.shouldMirror = isMirroringEnabled @@ -50,17 +52,10 @@ final class DecartRealtimeManager: RealtimeManagerProtocol { func switchCamera() async { #if !targetEnvironment(simulator) - print("switching camera to \(shouldMirror ? "back" : "front") camera") - guard let videoCapturer, let realtimeManager else { - preconditionFailure("🚨 videoCapturer is nil when switching camera") - } + guard let capture else { return } do { - try await RealtimeCameraCapture.switchCamera( - capturer: videoCapturer, - realtimeManager: realtimeManager, - newPosition: shouldMirror ? .back : .front - ) - shouldMirror.toggle() + try await capture.switchCamera() + shouldMirror = capture.position == .front } catch { DecartLogger.log("error while switching camera!", level: .error) } @@ -75,33 +70,31 @@ final class DecartRealtimeManager: RealtimeManagerProtocol { connectionState = .connecting do { + let modelConfig = Models.realtime(model) realtimeManager = try decartClient .createRealtimeManager( options: RealtimeConfiguration( - model: Models.realtime(model), + model: modelConfig, initialState: ModelState( prompt: currentPrompt ) )) guard let realtimeManager else { - preconditionFailure("🚨 realtimeManager is nil after creating it") + preconditionFailure("realtimeManager is nil after creating it") } monitorEvents() #if !targetEnvironment(simulator) - (localMediaStream, videoCapturer) = - try await RealtimeCameraCapture - .captureLocalCameraStream( - realtimeManager: realtimeManager, - cameraFacing: .front - ) - - DecartLogger.log("Connecting to WebRTC...", level: .info) - remoteMediaStreams = - try await realtimeManager - .connect(localStream: localMediaStream!) + let videoSource = realtimeManager.createVideoSource() + capture = RealtimeCapture(model: modelConfig, videoSource: videoSource) + try await capture?.startCapture() + + let localVideoTrack = realtimeManager.createVideoTrack(source: videoSource, trackId: "video0") + localMediaStream = RealtimeMediaStream(videoTrack: localVideoTrack, id: .localStream) + + remoteMediaStreams = try await realtimeManager.connect(localStream: localMediaStream!) #endif } catch { DecartLogger.log( @@ -126,9 +119,6 @@ final class DecartRealtimeManager: RealtimeManagerProtocol { if state == .error { DecartLogger.log("Error state received", level: .error) - // Should we disconnect on error? The connection might already be broken. - // Cleanup handles it if needed, or we can just stay in error state. - // For now, just updating state is enough as UI reacts to it. } } DecartLogger.log("Event monitoring task completed.", level: .info) @@ -136,21 +126,26 @@ final class DecartRealtimeManager: RealtimeManagerProtocol { } func cleanup() async { + connectionState = .idle + try? await Task.sleep(nanoseconds: 100_000_000) + eventTask?.cancel() eventTask = nil - if let capturer = videoCapturer { - await withCheckedContinuation { (k: CheckedContinuation) in - capturer.stopCapture { k.resume() } - } - } - videoCapturer = nil + localMediaStream?.videoTrack.isEnabled = false + localMediaStream?.audioTrack?.isEnabled = false + remoteMediaStreams?.videoTrack.isEnabled = false + remoteMediaStreams?.audioTrack?.isEnabled = false + + #if !targetEnvironment(simulator) + await capture?.stopCapture() + capture = nil + #endif + await realtimeManager?.disconnect() realtimeManager = nil - remoteMediaStreams = nil - localMediaStream = nil - connectionState = .idle - DecartLogger.log("Cleanup of realtime modelview completed.", level: .success) + localMediaStream = nil + remoteMediaStreams = nil } } diff --git a/Example/Example/Views/DraggableRTCVideoView.swift b/Example/Example/Views/DraggableRTCVideoView.swift index 3e42a3b..b509fcb 100644 --- a/Example/Example/Views/DraggableRTCVideoView.swift +++ b/Example/Example/Views/DraggableRTCVideoView.swift @@ -10,7 +10,7 @@ import SwiftUI import WebRTC struct DraggableRTCVideoView: View { - let track: RTCVideoTrack + let track: RTCVideoTrack? let mirror: Bool @State private var offset: CGSize = .zero diff --git a/Example/Example/Views/RealtimeView.swift b/Example/Example/Views/RealtimeView.swift index 21396cd..16569e3 100644 --- a/Example/Example/Views/RealtimeView.swift +++ b/Example/Example/Views/RealtimeView.swift @@ -1,10 +1,3 @@ -// -// RealtimeView.swift -// Example -// -// Created by Alon Bar-el on 19/11/2025. -// - import DecartSDK import Factory import SwiftUI @@ -13,25 +6,48 @@ import WebRTC struct RealtimeView: View { private let realtimeAiModel: RealtimeModel @State private var prompt: String = DecartConfig.defaultPrompt - - @State private var realtimeManager: RealtimeManagerProtocol + @State private var realtimeManager: DecartRealtimeManager? init(realtimeModel: RealtimeModel) { self.realtimeAiModel = realtimeModel - _realtimeManager = State( - initialValue: DecartRealtimeManager( - currentPrompt: Prompt( - text: DecartConfig.defaultPrompt, - enrich: false + } + + var body: some View { + ZStack { + if let manager = realtimeManager { + RealtimeContentView( + realtimeManager: manager, + realtimeAiModel: realtimeAiModel, + prompt: $prompt ) - ) - ) + } else { + ProgressView("Loading...") + } + } + .onAppear { + if realtimeManager == nil { + realtimeManager = DecartRealtimeManager( + currentPrompt: Prompt(text: DecartConfig.defaultPrompt, enrich: false) + ) + } + } + .onDisappear { + Task { [realtimeManager] in + await realtimeManager?.cleanup() + } + realtimeManager = nil + } } +} + +private struct RealtimeContentView: View { + @Bindable var realtimeManager: DecartRealtimeManager + let realtimeAiModel: RealtimeModel + @Binding var prompt: String var body: some View { ZStack { if realtimeManager.remoteMediaStreams != nil { - // we listen to shouldMirror here since the demo reflects the user camera. RTCMLVideoViewWrapper( track: realtimeManager.remoteMediaStreams?.videoTrack, mirror: realtimeManager.shouldMirror @@ -39,9 +55,8 @@ struct RealtimeView: View { .background(Color.black) .edgesIgnoringSafeArea(.all) } - // UI overlay + VStack(spacing: 5) { - // Top bar HStack { VStack(alignment: .center, spacing: 1) { Text(realtimeManager.connectionState.rawValue) @@ -57,38 +72,31 @@ struct RealtimeView: View { Spacer() - // Local video preview if realtimeManager.connectionState.isInSession, - realtimeManager.localMediaStream != nil + let localStream = realtimeManager.localMediaStream { DraggableRTCVideoView( - track: realtimeManager.localMediaStream!.videoTrack, + track: localStream.videoTrack, mirror: realtimeManager.shouldMirror ) } - // Controls VStack(spacing: 12) { if realtimeManager.connectionState == .error { - Text( - "Error while connecting to decart realtime servers, please try again later." - ) - .foregroundColor(.red) - .font(.caption) - .padding(8) - .background(Color.black.opacity(0.8)) - .cornerRadius(8) + Text("Error while connecting to decart realtime servers, please try again later.") + .foregroundColor(.red) + .font(.caption) + .padding(8) + .background(Color.black.opacity(0.8)) + .cornerRadius(8) } HStack(spacing: 12) { TextField("Prompt", text: $prompt) .textFieldStyle(RoundedBorderTextFieldStyle()) - // .disabled(!viewModel.isConnected) Button(action: { - Task { - realtimeManager.currentPrompt = Prompt(text: prompt, enrich: false) - } + realtimeManager.currentPrompt = Prompt(text: prompt, enrich: false) }) { Image(systemName: "paperplane.fill") .foregroundColor(.white) @@ -99,46 +107,37 @@ struct RealtimeView: View { ) .cornerRadius(8) } - // .disabled(!viewModel.isConnected) } HStack(spacing: 12) { Toggle("Mirror", isOn: $realtimeManager.shouldMirror) .toggleStyle(SwitchToggleStyle(tint: .blue)) - // .disabled(!viewModel.isConnected) + Button(action: { - Task { - await realtimeManager.switchCamera() - } + Task { await realtimeManager.switchCamera() } }) { Image(systemName: "arrow.trianglehead.2.counterclockwise.rotate.90") } + Spacer() Button(action: { if realtimeManager.connectionState.isInSession { - Task { - await realtimeManager.cleanup() - } + Task { await realtimeManager.cleanup() } } else { - let model = self.realtimeAiModel // Capture value - Task { - await realtimeManager.connect(model: model) - } + Task { await realtimeManager.connect(model: realtimeAiModel) } } }) { - Text( - realtimeManager.connectionState.rawValue - ) - .fontWeight(.semibold) - .foregroundColor(.white) - .frame(maxWidth: .infinity) - .padding() - .background( - realtimeManager.connectionState.isConnected - ? Color.red : Color.green - ) - .cornerRadius(12) + Text(realtimeManager.connectionState.rawValue) + .fontWeight(.semibold) + .foregroundColor(.white) + .frame(maxWidth: .infinity) + .padding() + .background( + realtimeManager.connectionState.isConnected + ? Color.red : Color.green + ) + .cornerRadius(12) } } } @@ -146,11 +145,6 @@ struct RealtimeView: View { .background(Color.black.opacity(0.8)) .cornerRadius(16) .padding(.all, 5) - .onDisappear { - Task { [realtimeManager] in - await realtimeManager.cleanup() - } - } } } } diff --git a/Sources/DecartSDK/Capture/CaptureExtensions.swift b/Sources/DecartSDK/Capture/CaptureExtensions.swift index caf3a6f..238247a 100644 --- a/Sources/DecartSDK/Capture/CaptureExtensions.swift +++ b/Sources/DecartSDK/Capture/CaptureExtensions.swift @@ -8,13 +8,15 @@ import AVFoundation import WebRTC public extension AVCaptureDevice { - /// Pick a format that meets (or exceeds) the requested dimensions; falls back to the first available. + /// Pick a format that meets (or exceeds) the requested dimensions in either orientation. func pickFormat(minWidth: Int, minHeight: Int) throws -> AVCaptureDevice.Format { let formats = RTCCameraVideoCapturer.supportedFormats(for: self) if let match = formats.first(where: { let d = CMVideoFormatDescriptionGetDimensions($0.formatDescription) - return d.width >= minWidth && d.height >= minHeight + let landscape = d.width >= minWidth && d.height >= minHeight + let portrait = d.height >= minWidth && d.width >= minHeight + return landscape || portrait }) { return match } diff --git a/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift b/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift deleted file mode 100644 index 5b86d0b..0000000 --- a/Sources/DecartSDK/Capture/RealtimeCameraCapture.swift +++ /dev/null @@ -1,80 +0,0 @@ -// -// CaptureUtils.swift -// DecartSDK -// -// Created by Alon Bar-el on 05/11/2025. -// -import AVFoundation -@preconcurrency import WebRTC - -#if !targetEnvironment(simulator) -public enum RealtimeCameraCapture { - public static func captureLocalCameraStream(realtimeManager: RealtimeManager, cameraFacing: AVCaptureDevice.Position) async throws -> ( - RealtimeMediaStream, - RTCCameraVideoCapturer - ) { - let currentRealtimeModel = realtimeManager.options.model - - let videoSource = realtimeManager.createVideoSource() - let capturer = RTCCameraVideoCapturer(delegate: videoSource) - - let device = try AVCaptureDevice.pickCamera(position: cameraFacing) - let format = try device.pickFormat( - minWidth: currentRealtimeModel.width, - minHeight: currentRealtimeModel.height - ) - let targetFPS = try device.pickFPS(for: format, preferred: currentRealtimeModel.fps) - - try await startCameraCapture(capturer: capturer, device: device, format: format, fps: targetFPS) - - let localVideoTrack = realtimeManager.createVideoTrack( - source: videoSource, - trackId: "video0" - ) - - return ( - RealtimeMediaStream(videoTrack: localVideoTrack, id: .localStream), - capturer - ) - } - - private static func startCameraCapture( - capturer: RTCCameraVideoCapturer, - device: AVCaptureDevice, - format: AVCaptureDevice.Format, - fps: Int - ) async throws { - try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in - capturer.startCapture(with: device, format: format, fps: fps) { error in - if let error { cont.resume(throwing: error) } - else { cont.resume() } - } - } - } - - @discardableResult - public static func switchCamera( - capturer: RTCCameraVideoCapturer, - realtimeManager: RealtimeManager, - newPosition: AVCaptureDevice.Position - ) async throws -> AVCaptureDevice.Position { - let currentRealtimeModel = realtimeManager.options.model - - let newDevice = try AVCaptureDevice.pickCamera(position: newPosition) - let format = try newDevice.pickFormat( - minWidth: currentRealtimeModel.width, - minHeight: currentRealtimeModel.height - ) - let targetFPS = try newDevice.pickFPS(for: format, preferred: currentRealtimeModel.fps) - - try await startCameraCapture( - capturer: capturer, - device: newDevice, - format: format, - fps: targetFPS - ) - - return newPosition - } -} -#endif diff --git a/Sources/DecartSDK/Capture/RealtimeCapture.swift b/Sources/DecartSDK/Capture/RealtimeCapture.swift new file mode 100644 index 0000000..955c85b --- /dev/null +++ b/Sources/DecartSDK/Capture/RealtimeCapture.swift @@ -0,0 +1,164 @@ +import AVFoundation +@preconcurrency import WebRTC + +public enum CaptureOrientation: Sendable { + case portrait + case landscape +} + +#if !targetEnvironment(simulator) +public final class RealtimeCapture: @unchecked Sendable { + public private(set) var position: AVCaptureDevice.Position + public let orientation: CaptureOrientation + public let targetWidth: Int + public let targetHeight: Int + + public var captureSession: AVCaptureSession { capturer.captureSession } + + private let model: ModelDefinition + private let videoSource: RTCVideoSource + private let capturer: RTCCameraVideoCapturer + + public init( + model: ModelDefinition, + videoSource: RTCVideoSource, + orientation: CaptureOrientation = .portrait, + initialPosition: AVCaptureDevice.Position = .front + ) { + self.model = model + self.videoSource = videoSource + self.orientation = orientation + self.position = initialPosition + + switch orientation { + case .landscape: + self.targetWidth = model.width + self.targetHeight = model.height + case .portrait: + self.targetWidth = model.height + self.targetHeight = model.width + } + + self.capturer = RTCCameraVideoCapturer(delegate: videoSource) + } + + public func startCapture() async throws { + try await startCapture(position: position) + } + + public func switchCamera() async throws { + let newPosition: AVCaptureDevice.Position = position == .front ? .back : .front + try await startCapture(position: newPosition) + position = newPosition + } + + public func stopCapture() async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + capturer.stopCapture { continuation.resume() } + } + + let session = capturer.captureSession + session.beginConfiguration() + session.outputs.forEach { session.removeOutput($0) } + session.inputs.forEach { session.removeInput($0) } + session.commitConfiguration() + } + + private func startCapture(position: AVCaptureDevice.Position) async throws { + let device = try AVCaptureDevice.pickCamera(position: position) + let format = try device.pickFormat(minWidth: targetWidth, minHeight: targetHeight) + let targetFPS = try device.pickFPS(for: format, preferred: model.fps) + + videoSource.adaptOutputFormat( + toWidth: Int32(targetWidth), + height: Int32(targetHeight), + fps: Int32(targetFPS) + ) + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + capturer.startCapture(with: device, format: format, fps: targetFPS) { error in + if let error { continuation.resume(throwing: error) } + else { continuation.resume() } + } + } + } +} + +private final class VideoFrameAdapter: NSObject, RTCVideoCapturerDelegate { + private weak var targetSource: RTCVideoSource? + let targetWidth: Int32 + let targetHeight: Int32 + + init(source: RTCVideoSource, targetWidth: Int32, targetHeight: Int32) { + self.targetSource = source + self.targetWidth = targetWidth + self.targetHeight = targetHeight + super.init() + } + + func capturer(_ capturer: RTCVideoCapturer, didCapture frame: RTCVideoFrame) { + guard let source = targetSource else { return } + + guard frame.width != targetWidth || frame.height != targetHeight else { + source.capturer(capturer, didCapture: frame) + return + } + + guard let adaptedFrame = cropAndScaleFromCenter(frame: frame) else { + source.capturer(capturer, didCapture: frame) + return + } + + source.capturer(capturer, didCapture: adaptedFrame) + } + + private func cropAndScaleFromCenter(frame: RTCVideoFrame) -> RTCVideoFrame? { + let sourceWidth = frame.width + let sourceHeight = frame.height + + let scaleWidth: Int32 + let scaleHeight: Int32 + + if targetWidth > sourceWidth || targetHeight > sourceHeight { + let widthScale = Double(targetWidth) / Double(sourceWidth) + let heightScale = Double(targetHeight) / Double(sourceHeight) + let scale = max(widthScale, heightScale) + scaleWidth = Int32(Double(targetWidth) / scale) + scaleHeight = Int32(Double(targetHeight) / scale) + } else { + scaleWidth = targetWidth + scaleHeight = targetHeight + } + + let sourceRatio = Double(sourceWidth) / Double(sourceHeight) + let targetRatio = Double(scaleWidth) / Double(scaleHeight) + + let cropWidth: Int32 + let cropHeight: Int32 + + if sourceRatio > targetRatio { + cropHeight = sourceHeight + cropWidth = Int32(Double(sourceHeight) * targetRatio) + } else { + cropWidth = sourceWidth + cropHeight = Int32(Double(sourceWidth) / targetRatio) + } + + let offsetX = (sourceWidth - cropWidth) / 2 + let offsetY = (sourceHeight - cropHeight) / 2 + + guard let newBuffer = frame.buffer.cropAndScale?( + with: offsetX, + offsetY: offsetY, + cropWidth: cropWidth, + cropHeight: cropHeight, + scaleWidth: scaleWidth, + scaleHeight: scaleHeight + ) else { + return nil + } + + return RTCVideoFrame(buffer: newBuffer, rotation: frame.rotation, timeStampNs: frame.timeStampNs) + } +} +#endif diff --git a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift index e974ff9..d6bf902 100644 --- a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift +++ b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift @@ -83,8 +83,8 @@ public struct RealtimeConfiguration: Sendable { public let preferredCodec: String public init( - maxBitrate: Int = 3_800_000, - minBitrate: Int = 100_000, + maxBitrate: Int = 2_000_000, + minBitrate: Int = 500_000, maxFramerate: Int = 26, preferredCodec: String = "VP8" ) { @@ -126,6 +126,7 @@ public struct RealtimeConfiguration: Sendable { encodingParam.maxBitrateBps = NSNumber(value: maxBitrate) encodingParam.minBitrateBps = NSNumber(value: minBitrate) encodingParam.maxFramerate = NSNumber(value: maxFramerate) + encodingParam.scaleResolutionDownBy = NSNumber(value: 1.0) parameters.encodings[0] = encodingParam sender.parameters = parameters diff --git a/Sources/DecartSDK/Realtime/RealtimeManager.swift b/Sources/DecartSDK/Realtime/RealtimeManager.swift index d3e489e..964db4b 100644 --- a/Sources/DecartSDK/Realtime/RealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/RealtimeManager.swift @@ -74,8 +74,8 @@ public final class RealtimeManager: @unchecked Sendable { sendMessage(.prompt(PromptMessage(prompt: prompt.text))) } - public func switchCamera(rotateY: Int) { - sendMessage(.switchCamera(SwitchCameraMessage(rotateY: rotateY))) + public func getStats() async -> RTCStatisticsReport? { + await webRTCClient.peerConnection?.statistics() } // MARK: - Private @@ -154,6 +154,14 @@ public final class RealtimeManager: @unchecked Sendable { connectionStateListenerTask?.cancel() connectionStateListenerTask = nil webRTCClient.closePeerConnection() + + let audioSession = RTCAudioSession.sharedInstance() + if audioSession.isActive { + audioSession.lockForConfiguration() + try? audioSession.setActive(false) + audioSession.unlockForConfiguration() + } + await webSocketClient?.disconnect() webSocketClient = nil } @@ -163,5 +171,6 @@ public final class RealtimeManager: @unchecked Sendable { connectionStateListenerTask?.cancel() webRTCClient.closePeerConnection() stateContinuation.finish() + DecartLogger.log("RealtimeManager (SDK) deinitialized", level: .info) } } diff --git a/Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift b/Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift index bcf3132..ccdc271 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift @@ -65,16 +65,6 @@ struct PromptMessage: Codable, Sendable { } } -struct SwitchCameraMessage: Codable, Sendable { - let type: String - let rotateY: Int - - init(rotateY: Int) { - self.type = "switch_camera" - self.rotateY = rotateY - } -} - struct ServerErrorMessage: Codable, Sendable { let type: String let message: String? @@ -154,7 +144,6 @@ enum OutgoingWebSocketMessage: Codable, Sendable { case answer(AnswerMessage) case iceCandidate(IceCandidateMessage) case prompt(PromptMessage) - case switchCamera(SwitchCameraMessage) func encode(to encoder: Encoder) throws { switch self { @@ -166,8 +155,6 @@ enum OutgoingWebSocketMessage: Codable, Sendable { try msg.encode(to: encoder) case .prompt(let msg): try msg.encode(to: encoder) - case .switchCamera(let msg): - try msg.encode(to: encoder) } } } diff --git a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift index a6c2f6f..2263e00 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift @@ -2,6 +2,9 @@ import Foundation @preconcurrency import WebRTC final class WebRTCClient { + private nonisolated(unsafe) static var sharedFactory: RTCPeerConnectionFactory? + private static let factoryLock = NSLock() + let factory: RTCPeerConnectionFactory private(set) var peerConnection: RTCPeerConnection? private(set) var connectionStateStream: AsyncStream? @@ -10,18 +13,28 @@ final class WebRTCClient { private var signalingClient: SignalingClient? private var connectionStateContinuation: AsyncStream.Continuation? - init() { - #if IS_DEVELOPMENT - RTCSetMinDebugLogLevel(.verbose) - #endif + private static func getOrCreateFactory() -> RTCPeerConnectionFactory { + factoryLock.lock() + defer { factoryLock.unlock() } + + if let factory = sharedFactory { + return factory + } + RTCInitializeSSL() + RTCSetMinDebugLogLevel(.warning) - let videoEncoderFactory = RTCDefaultVideoEncoderFactory() - let videoDecoderFactory = RTCDefaultVideoDecoderFactory() - self.factory = RTCPeerConnectionFactory( - encoderFactory: videoEncoderFactory, - decoderFactory: videoDecoderFactory + let factory = RTCPeerConnectionFactory( + encoderFactory: RTCDefaultVideoEncoderFactory(), + decoderFactory: RTCDefaultVideoDecoderFactory() ) + sharedFactory = factory + return factory + } + + init() { + self.factory = Self.getOrCreateFactory() + RTCSetMinDebugLogLevel(.verbose) } func createPeerConnection( @@ -151,5 +164,6 @@ final class WebRTCClient { deinit { DecartLogger.log("Webrtc client deinitialized", level: .info) closePeerConnection() + // Note: Don't call RTCCleanupSSL() - factory is singleton, SSL stays initialized } } diff --git a/Sources/DecartSDK/Realtime/WebRTC/WebRTCDelegateHandler.swift b/Sources/DecartSDK/Realtime/WebRTC/WebRTCDelegateHandler.swift index 5eacef0..fb8d531 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/WebRTCDelegateHandler.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/WebRTCDelegateHandler.swift @@ -16,6 +16,11 @@ final class WebRTCDelegateHandler: NSObject { func cleanup() { connectionStateContinuation.finish() } + + deinit { + DecartLogger.log("WebRTCDelegateHandler deinitialized", level: .info) + cleanup() + } } extension WebRTCDelegateHandler: RTCPeerConnectionDelegate { From 77512be0a67be10d9fd77356406e44b857d3834d Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 26 Nov 2025 16:58:06 +0200 Subject: [PATCH 04/23] fixed image and video fetchers + lucy-fast-v2v --- Example/Example/DecartSDK/ImageFetcher.swift | 19 +-- Example/Example/DecartSDK/VideoFetcher.swift | 38 ++--- Example/Example/Views/GenerateImageView.swift | 20 ++- .../DecartSDK/Capture/RealtimeCapture.swift | 78 ---------- Sources/DecartSDK/Models/Models.swift | 8 +- .../DecartSDK/Models/ModelsInputFactory.swift | 144 ++++++++++++------ Sources/DecartSDK/Process/ProcessClient.swift | 2 +- Sources/DecartSDK/Shared/DecartError.swift | 2 +- 8 files changed, 132 insertions(+), 179 deletions(-) diff --git a/Example/Example/DecartSDK/ImageFetcher.swift b/Example/Example/DecartSDK/ImageFetcher.swift index f70feba..137f4d4 100644 --- a/Example/Example/DecartSDK/ImageFetcher.swift +++ b/Example/Example/DecartSDK/ImageFetcher.swift @@ -30,12 +30,9 @@ final class ImageFetcher { isProcessing = false } - func fetchImage(model: ImageModel, inputType: ModelInputType, selectedItem: PhotosPickerItem) + func fetchImage(model: ImageModel, inputType: ModelInputType, selectedItem: PhotosPickerItem?) async { - let trimmedPrompt = prompt.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmedPrompt.isEmpty else { return } - isProcessing = true errorMessage = nil generatedImage = nil @@ -49,23 +46,21 @@ final class ImageFetcher { switch inputType { case .textToImage: - let input = TextToImageInput(prompt: trimmedPrompt) + let input = try TextToImageInput(prompt: prompt) processClient = try decartClient.createProcessClient( model: model, input: input ) case .imageToImage: - guard let referenceData = try await selectedItem.loadTransferable(type: Data.self) + guard let selectedItem, + let referenceData = try await selectedItem.loadTransferable(type: Data.self) else { - throw DecartError.invalidInput("Failed to load the selected image") + throw DecartError.invalidInput("No image selected") } - let fileInput = FileInput.image( - data: referenceData, - filename: "reference.jpg" - ) - let input = ImageToImageInput(prompt: trimmedPrompt, data: fileInput) + let fileInput = try FileInput.image(data: referenceData) + let input = try ImageToImageInput(prompt: prompt, data: fileInput) processClient = try decartClient.createProcessClient( model: model, input: input diff --git a/Example/Example/DecartSDK/VideoFetcher.swift b/Example/Example/DecartSDK/VideoFetcher.swift index 6c99d0d..dbf204d 100644 --- a/Example/Example/DecartSDK/VideoFetcher.swift +++ b/Example/Example/DecartSDK/VideoFetcher.swift @@ -120,17 +120,17 @@ final class VideoFetcher { ) async throws -> ProcessClient { switch inputType { case .textToVideo: - let input = TextToVideoInput(prompt: prompt) + let input = try TextToVideoInput(prompt: prompt) return try decartClient.createProcessClient(model: model, input: input) case .imageToVideo: - let fileInput = try await fileInput(from: selectedItem, requiresVideo: false) - let input = ImageToVideoInput(prompt: prompt, data: fileInput) + let fileInput = try await loadFileInput(from: selectedItem) + let input = try ImageToVideoInput(prompt: prompt, data: fileInput) return try decartClient.createProcessClient(model: model, input: input) case .videoToVideo: - let fileInput = try await fileInput(from: selectedItem, requiresVideo: true) - let input = VideoToVideoInput(prompt: prompt, data: fileInput) + let fileInput = try await loadFileInput(from: selectedItem) + let input = try VideoToVideoInput(prompt: prompt, data: fileInput) return try decartClient.createProcessClient(model: model, input: input) default: @@ -138,37 +138,19 @@ final class VideoFetcher { } } - private func fileInput(from item: PhotosPickerItem?, requiresVideo: Bool) async throws - -> FileInput - { + private func loadFileInput(from item: PhotosPickerItem?) async throws -> FileInput { guard let item else { - throw DecartError.invalidInput( - requiresVideo ? "Please attach a video first" : "Please attach an image first" - ) + throw DecartError.invalidInput("No media selected") } guard let data = try await item.loadTransferable(type: Data.self) else { throw DecartError.invalidInput("Failed to load selected media") } - guard let mediaType = resolveMediaType(for: item) else { - throw DecartError.invalidInput("Unsupported media type") - } - - if requiresVideo && mediaType.conforms(to: .video) == false { - throw DecartError.invalidInput("Please attach a video file") - } - - if !requiresVideo && mediaType.conforms(to: .image) == false { - throw DecartError.invalidInput("Please attach an image file") - } + let mediaType = item.supportedContentTypes.first(where: { + $0.conforms(to: .movie) || $0.conforms(to: .video) || $0.conforms(to: .image) + }) return try FileInput.from(data: data, uniformType: mediaType) } - - private func resolveMediaType(for item: PhotosPickerItem) -> UTType? { - item.supportedContentTypes.first(where: { - $0.conforms(to: .video) || $0.conforms(to: .image) - }) ?? item.supportedContentTypes.first - } } diff --git a/Example/Example/Views/GenerateImageView.swift b/Example/Example/Views/GenerateImageView.swift index 2a10996..32c1324 100644 --- a/Example/Example/Views/GenerateImageView.swift +++ b/Example/Example/Views/GenerateImageView.swift @@ -155,14 +155,20 @@ struct GenerateImageView: View { private func generate() { dismissKeyboard() Task { - guard let selectedItem else { - return + if requiresReference { + guard let selectedItem else { return } + await imageFetcher.fetchImage( + model: model, + inputType: inputType, + selectedItem: selectedItem + ) + } else { + await imageFetcher.fetchImage( + model: model, + inputType: inputType, + selectedItem: nil + ) } - await imageFetcher.fetchImage( - model: model, - inputType: inputType, - selectedItem: selectedItem - ) } } diff --git a/Sources/DecartSDK/Capture/RealtimeCapture.swift b/Sources/DecartSDK/Capture/RealtimeCapture.swift index 955c85b..0c983a8 100644 --- a/Sources/DecartSDK/Capture/RealtimeCapture.swift +++ b/Sources/DecartSDK/Capture/RealtimeCapture.swift @@ -83,82 +83,4 @@ public final class RealtimeCapture: @unchecked Sendable { } } } - -private final class VideoFrameAdapter: NSObject, RTCVideoCapturerDelegate { - private weak var targetSource: RTCVideoSource? - let targetWidth: Int32 - let targetHeight: Int32 - - init(source: RTCVideoSource, targetWidth: Int32, targetHeight: Int32) { - self.targetSource = source - self.targetWidth = targetWidth - self.targetHeight = targetHeight - super.init() - } - - func capturer(_ capturer: RTCVideoCapturer, didCapture frame: RTCVideoFrame) { - guard let source = targetSource else { return } - - guard frame.width != targetWidth || frame.height != targetHeight else { - source.capturer(capturer, didCapture: frame) - return - } - - guard let adaptedFrame = cropAndScaleFromCenter(frame: frame) else { - source.capturer(capturer, didCapture: frame) - return - } - - source.capturer(capturer, didCapture: adaptedFrame) - } - - private func cropAndScaleFromCenter(frame: RTCVideoFrame) -> RTCVideoFrame? { - let sourceWidth = frame.width - let sourceHeight = frame.height - - let scaleWidth: Int32 - let scaleHeight: Int32 - - if targetWidth > sourceWidth || targetHeight > sourceHeight { - let widthScale = Double(targetWidth) / Double(sourceWidth) - let heightScale = Double(targetHeight) / Double(sourceHeight) - let scale = max(widthScale, heightScale) - scaleWidth = Int32(Double(targetWidth) / scale) - scaleHeight = Int32(Double(targetHeight) / scale) - } else { - scaleWidth = targetWidth - scaleHeight = targetHeight - } - - let sourceRatio = Double(sourceWidth) / Double(sourceHeight) - let targetRatio = Double(scaleWidth) / Double(scaleHeight) - - let cropWidth: Int32 - let cropHeight: Int32 - - if sourceRatio > targetRatio { - cropHeight = sourceHeight - cropWidth = Int32(Double(sourceHeight) * targetRatio) - } else { - cropWidth = sourceWidth - cropHeight = Int32(Double(sourceWidth) / targetRatio) - } - - let offsetX = (sourceWidth - cropWidth) / 2 - let offsetY = (sourceHeight - cropHeight) / 2 - - guard let newBuffer = frame.buffer.cropAndScale?( - with: offsetX, - offsetY: offsetY, - cropWidth: cropWidth, - cropHeight: cropHeight, - scaleWidth: scaleWidth, - scaleHeight: scaleHeight - ) else { - return nil - } - - return RTCVideoFrame(buffer: newBuffer, rotation: frame.rotation, timeStampNs: frame.timeStampNs) - } -} #endif diff --git a/Sources/DecartSDK/Models/Models.swift b/Sources/DecartSDK/Models/Models.swift index af74347..1a443fd 100644 --- a/Sources/DecartSDK/Models/Models.swift +++ b/Sources/DecartSDK/Models/Models.swift @@ -18,7 +18,7 @@ public enum ImageModel: String, CaseIterable { public enum VideoModel: String, CaseIterable { case lucy_dev_i2v = "lucy-dev-i2v" - case lucy_dev_v2v = "lucy-dev-v2v" + case lucy_fast_v2v = "lucy-fast-v2v" case lucy_pro_t2v = "lucy-pro-t2v" case lucy_pro_i2v = "lucy-pro-i2v" case lucy_pro_v2v = "lucy-pro-v2v" @@ -85,10 +85,10 @@ public enum Models { width: 1280, height: 704 ) - case .lucy_dev_v2v: + case .lucy_fast_v2v: return ModelDefinition( - name: "lucy-dev-v2v", - urlPath: "/v1/generate/lucy-dev-v2v", + name: "lucy-fast-v2v", + urlPath: "/v1/generate/lucy-fast-v2v", fps: 25, width: 1280, height: 704 diff --git a/Sources/DecartSDK/Models/ModelsInputFactory.swift b/Sources/DecartSDK/Models/ModelsInputFactory.swift index b8fd645..14eaf85 100644 --- a/Sources/DecartSDK/Models/ModelsInputFactory.swift +++ b/Sources/DecartSDK/Models/ModelsInputFactory.swift @@ -10,60 +10,92 @@ public enum DevResolution: String, Codable, Sendable { case res720p = "720p" } -public enum FileInputError: Error, LocalizedError { - case missingType - case unsupportedType +public enum InputValidationError: LocalizedError { + case emptyPrompt + case emptyFileData + case expectedImage + case expectedVideo + case unsupportedMediaType public var errorDescription: String? { switch self { - case .missingType: - return "Unable to determine the media type. Only image and video files are supported." - case .unsupportedType: - return "Unsupported media type. Only image and video files are supported." + case .emptyPrompt: + return "Prompt cannot be empty" + case .emptyFileData: + return "File data cannot be empty" + case .expectedImage: + return "Expected an image file" + case .expectedVideo: + return "Expected a video file" + case .unsupportedMediaType: + return "Unsupported media type. Only image and video files are supported" } } } +public enum MediaType: Sendable { + case image + case video +} + public struct FileInput: Codable, Sendable { public let data: Data public let filename: String + public let mediaType: MediaType - public init(data: Data, filename: String) { + private enum CodingKeys: String, CodingKey { + case data, filename + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.data = try container.decode(Data.self, forKey: .data) + self.filename = try container.decode(String.self, forKey: .filename) + self.mediaType = FileInput.inferMediaType(from: filename) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(data, forKey: .data) + try container.encode(filename, forKey: .filename) + } + + private init(data: Data, filename: String, mediaType: MediaType) { self.data = data - self.filename = FileInput.ensureExtension( - for: filename, - defaultExtension: FileInput.defaultExtension(forFilename: filename) - ) + self.filename = filename + self.mediaType = mediaType } - public static func image(data: Data, filename: String = "image.jpg") -> FileInput { - FileInput( + public static func image(data: Data, filename: String = "image.jpg") throws -> FileInput { + guard !data.isEmpty else { throw InputValidationError.emptyFileData } + return FileInput( data: data, - filename: ensureExtension(for: filename, defaultExtension: "jpg") + filename: ensureExtension(for: filename, defaultExtension: "jpg"), + mediaType: .image ) } - public static func video(data: Data, filename: String = "video.mp4") -> FileInput { - FileInput( + public static func video(data: Data, filename: String = "video.mp4") throws -> FileInput { + guard !data.isEmpty else { throw InputValidationError.emptyFileData } + return FileInput( data: data, - filename: ensureExtension(for: filename, defaultExtension: "mp4") + filename: ensureExtension(for: filename, defaultExtension: "mp4"), + mediaType: .video ) } public static func from(data: Data, uniformType: UTType?) throws -> FileInput { - guard let uniformType else { - throw FileInputError.missingType - } + guard !data.isEmpty else { throw InputValidationError.emptyFileData } - if uniformType.conforms(to: .image) { - return image(data: data) + if let type = uniformType, type.conforms(to: .image) { + return try image(data: data) } - if uniformType.conforms(to: .video) { - return video(data: data) + if let type = uniformType, type.conforms(to: .video) || type.conforms(to: .movie) { + return try video(data: data) } - throw FileInputError.unsupportedType + throw InputValidationError.unsupportedMediaType } private static func ensureExtension(for filename: String, defaultExtension: String) -> String { @@ -79,15 +111,13 @@ public struct FileInput: Codable, Sendable { return trimmed } - private static func defaultExtension(forFilename filename: String) -> String { - let pathExtension = (filename as NSString).pathExtension.lowercased() - switch pathExtension { - case "jpg", "jpeg", "png", "heic": - return "jpg" - case "mp4", "mov", "m4v": - return "mp4" + private static func inferMediaType(from filename: String) -> MediaType { + let ext = (filename as NSString).pathExtension.lowercased() + switch ext { + case "jpg", "jpeg", "png", "heic", "webp": + return .image default: - return "bin" + return .video } } } @@ -103,8 +133,11 @@ public struct TextToVideoInput: Codable, Sendable { seed: Int? = nil, resolution: ProResolution? = .res720p, orientation: String? = nil - ) { - self.prompt = prompt + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + + self.prompt = trimmed self.seed = seed self.resolution = resolution self.orientation = orientation @@ -122,8 +155,11 @@ public struct TextToImageInput: Codable, Sendable { seed: Int? = nil, resolution: ProResolution? = .res720p, orientation: String? = nil - ) { - self.prompt = prompt + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + + self.prompt = trimmed self.seed = seed self.resolution = resolution self.orientation = orientation @@ -132,17 +168,21 @@ public struct TextToImageInput: Codable, Sendable { public struct ImageToVideoInput: Codable, Sendable { public let prompt: String - public let data: FileInput // We need to handle how this is serialized (e.g. multipart or base64) + public let data: FileInput public let seed: Int? - public let resolution: ProResolution? // Or separate structs for dev/pro if needed, but factory can handle types + public let resolution: ProResolution? public init( prompt: String, data: FileInput, seed: Int? = nil, resolution: ProResolution? = .res720p - ) { - self.prompt = prompt + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + guard data.mediaType == .image else { throw InputValidationError.expectedImage } + + self.prompt = trimmed self.data = data self.seed = seed self.resolution = resolution @@ -162,8 +202,12 @@ public struct ImageToImageInput: Codable, Sendable { seed: Int? = nil, resolution: ProResolution? = .res720p, enhancePrompt: Bool? = nil - ) { - self.prompt = prompt + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + guard data.mediaType == .image else { throw InputValidationError.expectedImage } + + self.prompt = trimmed self.data = data self.seed = seed self.resolution = resolution @@ -175,7 +219,7 @@ public struct VideoToVideoInput: Codable, Sendable { public let prompt: String public let data: FileInput public let seed: Int? - public let resolution: ProResolution? // pro supports 480p/720p, dev supports 720p. + public let resolution: ProResolution? public let enhancePrompt: Bool? public let numInferenceSteps: Int? @@ -186,8 +230,12 @@ public struct VideoToVideoInput: Codable, Sendable { resolution: ProResolution? = .res720p, enhancePrompt: Bool? = nil, numInferenceSteps: Int? = nil - ) { - self.prompt = prompt + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + guard data.mediaType == .video else { throw InputValidationError.expectedVideo } + + self.prompt = trimmed self.data = data self.seed = seed self.resolution = resolution @@ -211,7 +259,7 @@ public enum ModelsInputFactory: Sendable { return .textToVideo case .lucy_dev_i2v, .lucy_pro_i2v: return .imageToVideo - case .lucy_dev_v2v, .lucy_pro_v2v: + case .lucy_fast_v2v, .lucy_pro_v2v: return .videoToVideo } } diff --git a/Sources/DecartSDK/Process/ProcessClient.swift b/Sources/DecartSDK/Process/ProcessClient.swift index e69953d..3c44479 100644 --- a/Sources/DecartSDK/Process/ProcessClient.swift +++ b/Sources/DecartSDK/Process/ProcessClient.swift @@ -197,7 +197,7 @@ public struct ProcessClient: Sendable { guard (200 ... 299).contains(httpResponse.statusCode) else { let errorText = String(data: data, encoding: .utf8) ?? "Unknown error" - DecartLogger.log("error processing request: \(errorText), for route: \(request.url?.absoluteString ?? "unknown"), and body: \(String(decoding: request.httpBody ?? Data(), as: UTF8.self))", level: .error) + DecartLogger.log("error processing request: \(errorText), for route: \(request.url?.absoluteString ?? "unknown"), and body:", level: .error) throw DecartError.processingError( "Processing failed: \(httpResponse.statusCode) - \(errorText)") } diff --git a/Sources/DecartSDK/Shared/DecartError.swift b/Sources/DecartSDK/Shared/DecartError.swift index 176fd96..b95bc3f 100644 --- a/Sources/DecartSDK/Shared/DecartError.swift +++ b/Sources/DecartSDK/Shared/DecartError.swift @@ -1,6 +1,6 @@ import Foundation -public enum DecartError: Error { +public enum DecartError: LocalizedError { case invalidAPIKey case invalidBaseURL(String?) case webRTCError(String) From babba91a883ec0426d92f25f8151c043835516e2 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 26 Nov 2025 18:17:53 +0200 Subject: [PATCH 05/23] Fix memory leak in image/video fetchers - Add custom URLSession with caching disabled to prevent request body retention - Implement proper task cancellation with [weak self] pattern in ImageFetcher - Add tracked preview load tasks with cancellation support in views - Fix image orientation using UIGraphicsBeginImageContext (replaces UIGraphicsImageRenderer which caused memory retention) - Add proper cleanup on view disappear for all async tasks --- Example.xcodeproj/project.pbxproj | 4 + Example/Example/DecartSDK/ImageFetcher.swift | 73 ++++- .../DecartSDK/UIImage+Normalized.swift | 22 ++ Example/Example/DecartSDK/VideoFetcher.swift | 293 +++++++++--------- Example/Example/Views/GenerateImageView.swift | 44 ++- Example/Example/Views/GenerateVideoView.swift | 13 +- .../Realtime/WebRTC/WebRTCClient.swift | 3 +- 7 files changed, 284 insertions(+), 168 deletions(-) create mode 100644 Example/Example/DecartSDK/UIImage+Normalized.swift diff --git a/Example.xcodeproj/project.pbxproj b/Example.xcodeproj/project.pbxproj index 42dd79a..9cd3c33 100644 --- a/Example.xcodeproj/project.pbxproj +++ b/Example.xcodeproj/project.pbxproj @@ -289,6 +289,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = BQ3Q4NZWWC; @@ -308,6 +309,7 @@ MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = ai.decart.Example; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; STRING_CATALOG_GENERATE_SYMBOLS = YES; SWIFT_APPROACHABLE_CONCURRENCY = YES; SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; @@ -324,6 +326,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = BQ3Q4NZWWC; @@ -343,6 +346,7 @@ MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = ai.decart.Examplea; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; STRING_CATALOG_GENERATE_SYMBOLS = YES; SWIFT_APPROACHABLE_CONCURRENCY = YES; SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; diff --git a/Example/Example/DecartSDK/ImageFetcher.swift b/Example/Example/DecartSDK/ImageFetcher.swift index 137f4d4..dcecf3c 100644 --- a/Example/Example/DecartSDK/ImageFetcher.swift +++ b/Example/Example/DecartSDK/ImageFetcher.swift @@ -18,25 +18,59 @@ final class ImageFetcher { @ObservationIgnored private let decartClient = Container.shared.decartClient() + @ObservationIgnored + private static let urlSession: URLSession = { + let config = URLSessionConfiguration.default + config.urlCache = nil + config.requestCachePolicy = .reloadIgnoringLocalCacheData + return URLSession(configuration: config) + }() + + private var generateImageTask: Task? + var prompt: String = "" var generatedImage: UIImage? var isProcessing: Bool = false var errorMessage: String? + func cancelGeneration() { + generateImageTask?.cancel() + generateImageTask = nil + } + func reset() { + cancelGeneration() prompt = "" generatedImage = nil errorMessage = nil isProcessing = false } - func fetchImage(model: ImageModel, inputType: ModelInputType, selectedItem: PhotosPickerItem?) - async - { + func fetchImage(model: ImageModel, inputType: ModelInputType, selectedItem: PhotosPickerItem?) { + let currentPrompt = prompt + guard !currentPrompt.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } + + generateImageTask?.cancel() isProcessing = true errorMessage = nil generatedImage = nil + generateImageTask = Task { [weak self] in + await self?.performFetchImage( + model: model, + inputType: inputType, + selectedItem: selectedItem, + prompt: currentPrompt + ) + } + } + + private func performFetchImage( + model: ImageModel, + inputType: ModelInputType, + selectedItem: PhotosPickerItem?, + prompt: String + ) async { defer { isProcessing = false } @@ -49,35 +83,54 @@ final class ImageFetcher { let input = try TextToImageInput(prompt: prompt) processClient = try decartClient.createProcessClient( model: model, - input: input + input: input, + session: Self.urlSession ) case .imageToImage: - guard let selectedItem, - let referenceData = try await selectedItem.loadTransferable(type: Data.self) - else { + guard let selectedItem else { throw DecartError.invalidInput("No image selected") } - let fileInput = try FileInput.image(data: referenceData) + guard !Task.isCancelled else { return } + + guard let rawData = try await selectedItem.loadTransferable(type: Data.self), + let image = UIImage(data: rawData), + let fixedImage = image.fixOrientation(), + let imageData = fixedImage.jpegData(compressionQuality: 0.9) + else { + throw DecartError.invalidInput("Failed to load image data") + } + + guard !Task.isCancelled else { return } + + let fileInput = try FileInput.image(data: imageData) let input = try ImageToImageInput(prompt: prompt, data: fileInput) processClient = try decartClient.createProcessClient( model: model, - input: input + input: input, + session: Self.urlSession ) default: throw DecartError.invalidInput("Unsupported input type") } + guard !Task.isCancelled else { return } + let data = try await processClient.process() + + guard !Task.isCancelled else { return } + guard let image = UIImage(data: data) else { errorMessage = "Failed to decode image data" return } generatedImage = image } catch { - errorMessage = error.localizedDescription + if !Task.isCancelled { + errorMessage = error.localizedDescription + } } } } diff --git a/Example/Example/DecartSDK/UIImage+Normalized.swift b/Example/Example/DecartSDK/UIImage+Normalized.swift new file mode 100644 index 0000000..efb3272 --- /dev/null +++ b/Example/Example/DecartSDK/UIImage+Normalized.swift @@ -0,0 +1,22 @@ +// +// UIImage+Normalized.swift +// Example +// +// Created by Alon Bar-el on 23/11/2025. +// + +import UIKit + +extension UIImage { + func fixOrientation() -> UIImage? { + if imageOrientation == .up { + return self + } + + UIGraphicsBeginImageContext(size) + draw(in: CGRect(origin: .zero, size: size)) + let normalizedImage = UIGraphicsGetImageFromCurrentImageContext() + UIGraphicsEndImageContext() + return normalizedImage + } +} diff --git a/Example/Example/DecartSDK/VideoFetcher.swift b/Example/Example/DecartSDK/VideoFetcher.swift index dbf204d..4559196 100644 --- a/Example/Example/DecartSDK/VideoFetcher.swift +++ b/Example/Example/DecartSDK/VideoFetcher.swift @@ -15,142 +15,159 @@ import SwiftUI @MainActor @Observable final class VideoFetcher { - @ObservationIgnored - private let decartClient = Container.shared.decartClient() - - private var generateVideoTask: Task? - - var prompt: String = "" - var generatedVideoURL: URL? - var videoPlayer: AVPlayer? - var isProcessing: Bool = false - var errorMessage: String? - - func reset() { - prompt = "" - - // Delete temporary video file if it exists - if let videoURL = generatedVideoURL { - try? FileManager.default.removeItem(at: videoURL) - } - - generatedVideoURL = nil - errorMessage = nil - isProcessing = false - videoPlayer?.pause() - videoPlayer = nil - generateVideoTask?.cancel() - generateVideoTask = nil - } - - func cancelGeneration() { - generateVideoTask?.cancel() - generateVideoTask = nil - videoPlayer?.pause() - } - - func fetchVideo(model: VideoModel, inputType: ModelInputType, selectedItem: PhotosPickerItem?) { - let trimmedPrompt = prompt.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmedPrompt.isEmpty else { return } - - generateVideoTask?.cancel() - isProcessing = true - errorMessage = nil - generatedVideoURL = nil - - generateVideoTask = Task { [weak self, selectedItem] in - if let videoURL = self?.generatedVideoURL { - try? FileManager.default.removeItem(at: videoURL) - } - - await self?.generateVideo( - trimmedPrompt: trimmedPrompt, - model: model, - inputType: inputType, - selectedItem: selectedItem - ) - } - } - - private func generateVideo( - trimmedPrompt: String, - model: VideoModel, - inputType: ModelInputType, - selectedItem: PhotosPickerItem? - ) async { - defer { - isProcessing = false - } - - do { - let processClient = try await buildProcessClient( - prompt: trimmedPrompt, - model: model, - inputType: inputType, - selectedItem: selectedItem - ) - guard !Task.isCancelled else { return } - - let data = try await processClient.process() - let tempURL = FileManager.default.temporaryDirectory - .appendingPathComponent(UUID().uuidString) - .appendingPathExtension("mp4") - try data.write(to: tempURL, options: .atomic) - - if Task.isCancelled { - return - } - - videoPlayer?.pause() - generatedVideoURL = tempURL - videoPlayer = AVPlayer(url: tempURL) - } catch { - if Task.isCancelled { - return - } - errorMessage = error.localizedDescription - } - } - - private func buildProcessClient( - prompt: String, - model: VideoModel, - inputType: ModelInputType, - selectedItem: PhotosPickerItem? - ) async throws -> ProcessClient { - switch inputType { - case .textToVideo: - let input = try TextToVideoInput(prompt: prompt) - return try decartClient.createProcessClient(model: model, input: input) - - case .imageToVideo: - let fileInput = try await loadFileInput(from: selectedItem) - let input = try ImageToVideoInput(prompt: prompt, data: fileInput) - return try decartClient.createProcessClient(model: model, input: input) - - case .videoToVideo: - let fileInput = try await loadFileInput(from: selectedItem) - let input = try VideoToVideoInput(prompt: prompt, data: fileInput) - return try decartClient.createProcessClient(model: model, input: input) - - default: - throw DecartError.invalidInput("Unsupported input type") - } - } - - private func loadFileInput(from item: PhotosPickerItem?) async throws -> FileInput { - guard let item else { - throw DecartError.invalidInput("No media selected") - } - - guard let data = try await item.loadTransferable(type: Data.self) else { - throw DecartError.invalidInput("Failed to load selected media") - } - - let mediaType = item.supportedContentTypes.first(where: { - $0.conforms(to: .movie) || $0.conforms(to: .video) || $0.conforms(to: .image) - }) - - return try FileInput.from(data: data, uniformType: mediaType) - } + @ObservationIgnored + private let decartClient = Container.shared.decartClient() + + @ObservationIgnored + private static let urlSession: URLSession = { + let config = URLSessionConfiguration.default + config.urlCache = nil + config.requestCachePolicy = .reloadIgnoringLocalCacheData + return URLSession(configuration: config) + }() + + private var generateVideoTask: Task? + + var prompt: String = "" + var generatedVideoURL: URL? + var videoPlayer: AVPlayer? + var isProcessing: Bool = false + var errorMessage: String? + + func reset() { + prompt = "" + + if let videoURL = generatedVideoURL { + try? FileManager.default.removeItem(at: videoURL) + } + + generatedVideoURL = nil + errorMessage = nil + isProcessing = false + videoPlayer?.pause() + videoPlayer = nil + generateVideoTask?.cancel() + generateVideoTask = nil + } + + func cancelGeneration() { + generateVideoTask?.cancel() + generateVideoTask = nil + videoPlayer?.pause() + } + + func fetchVideo(model: VideoModel, inputType: ModelInputType, selectedItem: PhotosPickerItem?) { + let trimmedPrompt = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedPrompt.isEmpty else { return } + + generateVideoTask?.cancel() + isProcessing = true + errorMessage = nil + generatedVideoURL = nil + + generateVideoTask = Task { [weak self, selectedItem] in + if let videoURL = self?.generatedVideoURL { + try? FileManager.default.removeItem(at: videoURL) + } + + await self?.generateVideo( + trimmedPrompt: trimmedPrompt, + model: model, + inputType: inputType, + selectedItem: selectedItem + ) + } + } + + private func generateVideo( + trimmedPrompt: String, + model: VideoModel, + inputType: ModelInputType, + selectedItem: PhotosPickerItem? + ) async { + defer { + isProcessing = false + } + + do { + let processClient = try await buildProcessClient( + prompt: trimmedPrompt, + model: model, + inputType: inputType, + selectedItem: selectedItem + ) + guard !Task.isCancelled else { return } + + let data = try await processClient.process() + let tempURL = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathExtension("mp4") + try data.write(to: tempURL, options: .atomic) + + if Task.isCancelled { + return + } + + videoPlayer?.pause() + generatedVideoURL = tempURL + videoPlayer = AVPlayer(url: tempURL) + } catch { + if Task.isCancelled { + return + } + errorMessage = error.localizedDescription + } + } + + private func buildProcessClient( + prompt: String, + model: VideoModel, + inputType: ModelInputType, + selectedItem: PhotosPickerItem? + ) async throws -> ProcessClient { + switch inputType { + case .textToVideo: + let input = try TextToVideoInput(prompt: prompt) + return try decartClient.createProcessClient(model: model, input: input, session: Self.urlSession) + + case .imageToVideo: + let fileInput = try await loadFileInput(from: selectedItem) + let input = try ImageToVideoInput(prompt: prompt, data: fileInput) + return try decartClient.createProcessClient(model: model, input: input, session: Self.urlSession) + + case .videoToVideo: + let fileInput = try await loadFileInput(from: selectedItem) + let input = try VideoToVideoInput(prompt: prompt, data: fileInput) + return try decartClient.createProcessClient(model: model, input: input, session: Self.urlSession) + + default: + throw DecartError.invalidInput("Unsupported input type") + } + } + + private func loadFileInput(from item: PhotosPickerItem?) async throws -> FileInput { + guard let item else { + throw DecartError.invalidInput("No media selected") + } + + guard var data = try await item.loadTransferable(type: Data.self) else { + throw DecartError.invalidInput("Failed to load selected media") + } + + let mediaType = item.supportedContentTypes.first(where: { + $0.conforms(to: .movie) || $0.conforms(to: .video) || $0.conforms(to: .image) + }) + + if let type = mediaType, type.conforms(to: .image) { + guard let image = UIImage(data: data), + let fixedImage = image.fixOrientation(), + let jpegData = fixedImage.jpegData(compressionQuality: 0.9) + else { + throw DecartError.invalidInput("Failed to process image") + } + data = jpegData + } + + return try FileInput.from(data: data, uniformType: mediaType) + } } diff --git a/Example/Example/Views/GenerateImageView.swift b/Example/Example/Views/GenerateImageView.swift index 32c1324..1f20110 100644 --- a/Example/Example/Views/GenerateImageView.swift +++ b/Example/Example/Views/GenerateImageView.swift @@ -16,6 +16,7 @@ struct GenerateImageView: View { @State private var imageFetcher = ImageFetcher() @State private var selectedItem: PhotosPickerItem? @State private var selectedImagePreview: UIImage? + @State private var previewLoadTask: Task? @FocusState private var promptFocused: Bool private var inputType: ModelInputType { @@ -50,6 +51,14 @@ struct GenerateImageView: View { .padding(.horizontal) .padding(.bottom) } + .onDisappear { + previewLoadTask?.cancel() + previewLoadTask = nil + imageFetcher.cancelGeneration() + imageFetcher.reset() + selectedItem = nil + selectedImagePreview = nil + } .navigationTitle(model.rawValue) .navigationBarTitleDisplayMode(.inline) } @@ -113,7 +122,7 @@ struct GenerateImageView: View { } TextField("Enter prompt…", text: $imageFetcher.prompt, axis: .vertical) - .lineLimit(1...3) + .lineLimit(1 ... 3) .textFieldStyle(.roundedBorder) .disabled(imageFetcher.isProcessing) .focused($promptFocused) @@ -137,14 +146,17 @@ struct GenerateImageView: View { .disabled(!canSend) } }.onChange(of: selectedItem) { + previewLoadTask?.cancel() guard let selectedItem else { selectedImagePreview = nil return } - Task { + previewLoadTask = Task { + guard !Task.isCancelled else { return } let imagePreview = try? await selectedItem.loadTransferable( type: Data.self ) + guard !Task.isCancelled else { return } if let uiImage = UIImage(data: imagePreview ?? Data()) { selectedImagePreview = uiImage } @@ -154,21 +166,19 @@ struct GenerateImageView: View { private func generate() { dismissKeyboard() - Task { - if requiresReference { - guard let selectedItem else { return } - await imageFetcher.fetchImage( - model: model, - inputType: inputType, - selectedItem: selectedItem - ) - } else { - await imageFetcher.fetchImage( - model: model, - inputType: inputType, - selectedItem: nil - ) - } + if requiresReference { + guard let selectedItem else { return } + imageFetcher.fetchImage( + model: model, + inputType: inputType, + selectedItem: selectedItem + ) + } else { + imageFetcher.fetchImage( + model: model, + inputType: inputType, + selectedItem: nil + ) } } diff --git a/Example/Example/Views/GenerateVideoView.swift b/Example/Example/Views/GenerateVideoView.swift index 4a390ff..f812054 100644 --- a/Example/Example/Views/GenerateVideoView.swift +++ b/Example/Example/Views/GenerateVideoView.swift @@ -16,6 +16,7 @@ struct GenerateVideoView: View { @State private var videoFetcher = VideoFetcher() @State private var selectedItem: PhotosPickerItem? @State private var selectedMediaPreview: UIImage? + @State private var previewLoadTask: Task? @FocusState private var promptFocused: Bool private var trimmedPrompt: String { @@ -65,8 +66,12 @@ struct GenerateVideoView: View { .padding(.bottom) } .onDisappear { + previewLoadTask?.cancel() + previewLoadTask = nil videoFetcher.cancelGeneration() videoFetcher.reset() + selectedItem = nil + selectedMediaPreview = nil } .navigationTitle(model.rawValue) .navigationBarTitleDisplayMode(.inline) @@ -176,12 +181,16 @@ struct GenerateVideoView: View { } private func handleSelectionChange(_ item: PhotosPickerItem?) { + previewLoadTask?.cancel() + guard let item else { selectedMediaPreview = nil return } - Task { + previewLoadTask = Task { + guard !Task.isCancelled else { return } + let resolvedType = item.supportedContentTypes.first(where: { $0.conforms(to: .video) || $0.conforms(to: .image) @@ -195,6 +204,8 @@ struct GenerateVideoView: View { previewImage = image } + guard !Task.isCancelled else { return } + await MainActor.run { selectedMediaPreview = previewImage } diff --git a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift index 2263e00..810ba41 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift @@ -22,7 +22,7 @@ final class WebRTCClient { } RTCInitializeSSL() - RTCSetMinDebugLogLevel(.warning) + // RTCSetMinDebugLogLevel(.warning) let factory = RTCPeerConnectionFactory( encoderFactory: RTCDefaultVideoEncoderFactory(), @@ -34,7 +34,6 @@ final class WebRTCClient { init() { self.factory = Self.getOrCreateFactory() - RTCSetMinDebugLogLevel(.verbose) } func createPeerConnection( From a71c3274d53335abaa23a0579453b9022feee116 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 26 Nov 2025 18:28:26 +0200 Subject: [PATCH 06/23] Update README with current API documentation --- .../xcshareddata/xcschemes/Example.xcscheme | 5 + README.md | 241 ++++++++++-------- Sources/DecartSDK/Shared/Logger.swift | 7 +- .../SwiftUI/RTCMLVideoViewWrapper.swift | 1 - 4 files changed, 146 insertions(+), 108 deletions(-) diff --git a/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme b/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme index 1c38a71..b32be68 100644 --- a/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme +++ b/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme @@ -59,6 +59,11 @@ value = "funcall_eIJgFmMEbcRfsbTNRGdAWFumYkXOJpWyzoXPPcLClZYATEaiiKuHYFzvDEiOGxtO" isEnabled = "YES"> + + RealtimeManager -Real-time video streaming with WebRTC. +// Create process clients +func createProcessClient(model: ImageModel, input: TextToImageInput) throws -> ProcessClient +func createProcessClient(model: ImageModel, input: ImageToImageInput) throws -> ProcessClient +func createProcessClient(model: VideoModel, input: TextToVideoInput) throws -> ProcessClient +func createProcessClient(model: VideoModel, input: ImageToVideoInput) throws -> ProcessClient +func createProcessClient(model: VideoModel, input: VideoToVideoInput) throws -> ProcessClient +``` -#### Methods +### RealtimeManager ```swift -func createRealtimeClient(options: RealtimeConfiguration) throws -> RealtimeClient func connect(localStream: RealtimeMediaStream) async throws -> RealtimeMediaStream func disconnect() async func setPrompt(_ prompt: Prompt) -``` - -#### Events +func getStats() async -> RTCStatisticsReport? -```swift let events: AsyncStream - // States: .idle, .connecting, .connected, .disconnected, .error ``` -#### Available Models - -```swift -Models.realtime(.mirage) -Models.realtime(.mirage_v2) -Models.realtime(.lucy_v2v_720p_rt) -``` - ### ProcessClient -Batch image and video generation. - -#### Methods - ```swift -func createProcessClient(model: ImageModel, input: TextToImageInput) throws -> ProcessClient -func createProcessClient(model: ImageModel, input: ImageToImageInput) throws -> ProcessClient -func createProcessClient(model: VideoModel, input: TextToVideoInput) throws -> ProcessClient -func createProcessClient(model: VideoModel, input: ImageToVideoInput) throws -> ProcessClient -func createProcessClient(model: VideoModel, input: VideoToVideoInput) throws -> ProcessClient - func process() async throws -> Data ``` -#### Available Models +### Available Models + +**Realtime Models:** +- `RealtimeModel.mirage` +- `RealtimeModel.mirage_v2` +- `RealtimeModel.lucy_v2v_720p_rt` **Image Models:** -- `.lucy_pro_t2i` - Text to image -- `.lucy_pro_i2i` - Image to image +- `ImageModel.lucy_pro_t2i` - Text to image +- `ImageModel.lucy_pro_i2i` - Image to image **Video Models:** -- `.lucy_pro_t2v` - Text to video -- `.lucy_pro_i2v` - Image to video -- `.lucy_pro_v2v` - Video to video -- `.lucy_dev_i2v` - Image to video (dev) -- `.lucy_dev_v2v` - Video to video (dev) +- `VideoModel.lucy_pro_t2v` - Text to video +- `VideoModel.lucy_pro_i2v` - Image to video +- `VideoModel.lucy_pro_v2v` - Video to video +- `VideoModel.lucy_dev_i2v` - Image to video (dev) +- `VideoModel.lucy_fast_v2v` - Fast video to video ### Input Types ```swift // Text-based inputs -TextToImageInput(prompt: String, seed: Int? = nil, resolution: ProResolution? = .res720p) -TextToVideoInput(prompt: String, seed: Int? = nil, resolution: ProResolution? = .res720p) +TextToImageInput(prompt: String, seed: Int?, resolution: ProResolution?) +TextToVideoInput(prompt: String, seed: Int?, resolution: ProResolution?) // File-based inputs -ImageToImageInput(prompt: String, data: FileInput, seed: Int? = nil) -ImageToVideoInput(prompt: String, data: FileInput, seed: Int? = nil) -VideoToVideoInput(prompt: String, data: FileInput, seed: Int? = nil) +ImageToImageInput(prompt: String, data: FileInput, seed: Int?) +ImageToVideoInput(prompt: String, data: FileInput, seed: Int?) +VideoToVideoInput(prompt: String, data: FileInput, seed: Int?) // File input helpers -FileInput.image(data: Data, filename: String = "image.jpg") -FileInput.video(data: Data, filename: String = "video.mp4") +FileInput.image(data: Data, filename: String) +FileInput.video(data: Data, filename: String) FileInput.from(data: Data, uniformType: UTType?) ``` +### RealtimeConfiguration + +```swift +RealtimeConfiguration( + model: ModelDefinition, + initialState: ModelState, + connection: ConnectionConfig, // Optional + media: MediaConfig // Optional +) + +// Connection config +ConnectionConfig( + iceServers: [String], + connectionTimeout: Int32, + pingInterval: Int32 +) + +// Media config +MediaConfig( + video: VideoConfig +) + +// Video config +VideoConfig( + maxBitrate: Int, + minBitrate: Int, + maxFramerate: Int, + preferredCodec: String // "VP8" or "H264" +) +``` + ## Requirements - iOS 15.0+ / macOS 12.0+ - Swift 5.9+ - Xcode 15.0+ -## Architecture - -The SDK follows Swift best practices: - -- **Value types** (structs) for configuration and data models -- **Reference types** (classes) for connection management -- **AsyncStream** for reactive event streams -- **async/await** for asynchronous operations -- **Structured concurrency** with Task-based cancellation -- **Type-safe protocols** for proper Swift error handling - ## Dependencies -- [WebRTC](https://github.com/stasel/WebRTC) - WebRTC framework for iOS/macOS +- [WebRTC](https://github.com/nickkjordan/WebRTC) - WebRTC framework for iOS/macOS ## License diff --git a/Sources/DecartSDK/Shared/Logger.swift b/Sources/DecartSDK/Shared/Logger.swift index d580e1e..7bcc2a4 100644 --- a/Sources/DecartSDK/Shared/Logger.swift +++ b/Sources/DecartSDK/Shared/Logger.swift @@ -7,8 +7,7 @@ import Foundation public enum DecartLogger: Sendable { - public static let printImportantOnly: Bool = ProcessInfo.processInfo.environment["printImportantOnly"] == "YES" - + public static let pringDebugLogs: Bool = ProcessInfo.processInfo.environment["ENABLE_DECART_SDK_DUBUG_LOGS"] == "YES" public enum Level: String, Sendable { case info = "ℹ️" case warning = "⚠️" @@ -30,8 +29,8 @@ public enum DecartLogger: Sendable { public static func log(_ string: String, level: Level, logBreadcrumbEnabled: Bool = true) { let logString = "[DecartSDK -\(dateFormatter.string(from: Date.now)) \(level.rawValue)] - \(string)" - if DecartLogger.printImportantOnly { - if level == .important { + if !DecartLogger.pringDebugLogs { + if level == .important || level == .error { print(logString) } } else { diff --git a/Sources/DecartSDK/SwiftUI/RTCMLVideoViewWrapper.swift b/Sources/DecartSDK/SwiftUI/RTCMLVideoViewWrapper.swift index adcba51..b46ae75 100644 --- a/Sources/DecartSDK/SwiftUI/RTCMLVideoViewWrapper.swift +++ b/Sources/DecartSDK/SwiftUI/RTCMLVideoViewWrapper.swift @@ -63,7 +63,6 @@ public struct RTCMLVideoViewWrapper: UIViewRepresentable { } public static func dismantleUIView(_ uiView: RTCMTLVideoView, coordinator: Coordinator) { - print("dismissing video view and nilling track!") coordinator.lastTrack?.remove(uiView) coordinator.view = nil coordinator.lastTrack = nil From 013018a40fca1207c3dc62599013aa57af5ffd4e Mon Sep 17 00:00:00 2001 From: Verion1 Date: Sun, 30 Nov 2025 12:29:31 +0200 Subject: [PATCH 07/23] rename decart realtime manager --- .../DecartSDK/DecartRealtimeManager.swift | 159 ++++++++++-------- Example/Example/Views/RealtimeView.swift | 6 +- README.md | 10 ++ Sources/DecartSDK/DecartSDK.swift | 4 +- ...ager.swift => DecartRealtimeManager.swift} | 2 +- .../Realtime/RealtimeConfiguration.swift | 2 +- .../Realtime/RealtimeManager+Media.swift | 2 +- .../SignalingModel.swift | 0 8 files changed, 107 insertions(+), 78 deletions(-) rename Sources/DecartSDK/Realtime/{RealtimeManager.swift => DecartRealtimeManager.swift} (98%) rename Sources/DecartSDK/Realtime/{WebRTC => Websocket}/SignalingModel.swift (100%) diff --git a/Example/Example/DecartSDK/DecartRealtimeManager.swift b/Example/Example/DecartSDK/DecartRealtimeManager.swift index f43a416..00673f5 100644 --- a/Example/Example/DecartSDK/DecartRealtimeManager.swift +++ b/Example/Example/DecartSDK/DecartRealtimeManager.swift @@ -1,5 +1,5 @@ // -// RealtimeManager.swift +// DecartRealtimeManager.swift // Example // // Created by Alon Bar-el on 04/11/2025. @@ -12,55 +12,46 @@ import SwiftUI @MainActor @Observable -final class DecartRealtimeManager: RealtimeManagerProtocol { - @ObservationIgnored - private let decartClient = Container.shared.decartClient() +final class RealtimeManager: RealtimeManagerProtocol { + // MARK: - Public State var currentPrompt: Prompt { didSet { - Task { [weak self] in - guard let self, let manager = self.realtimeManager else { return } - manager.setPrompt(currentPrompt) - } + // Send updated prompt to the server for real-time style changes + realtimeManager?.setPrompt(currentPrompt) } } var shouldMirror: Bool private(set) var connectionState: DecartRealtimeConnectionState = .idle - private(set) var localMediaStream: RealtimeMediaStream? - private(set) var remoteMediaStreams: RealtimeMediaStream? + // MARK: - Private + + @ObservationIgnored + private let decartClient = Container.shared.decartClient() + + @ObservationIgnored + private var realtimeManager: DecartRealtimeManager? + @ObservationIgnored - private var realtimeManager: RealtimeManager? + private var eventTask: Task? + #if !targetEnvironment(simulator) @ObservationIgnored private var capture: RealtimeCapture? #endif - @ObservationIgnored - private var eventTask: Task? - init( - currentPrompt: Prompt, - isMirroringEnabled: Bool = true - ) { + // MARK: - Init + + init(currentPrompt: Prompt, isMirroringEnabled: Bool = true) { self.currentPrompt = currentPrompt self.shouldMirror = isMirroringEnabled } - func switchCamera() async { - #if !targetEnvironment(simulator) - guard let capture else { return } - do { - try await capture.switchCamera() - shouldMirror = capture.position == .front - } catch { - DecartLogger.log("error while switching camera!", level: .error) - } - #endif - } + // MARK: - Public API func connect(model: RealtimeModel) async { if connectionState.isInSession || realtimeManager != nil { @@ -71,81 +62,109 @@ final class DecartRealtimeManager: RealtimeManagerProtocol { do { let modelConfig = Models.realtime(model) - realtimeManager = - try decartClient - .createRealtimeManager( - options: RealtimeConfiguration( - model: modelConfig, - initialState: ModelState( - prompt: currentPrompt - ) - )) + + // Initialize the WebRTC manager with model config and initial prompt + realtimeManager = try decartClient.createRealtimeManager( + options: RealtimeConfiguration( + model: modelConfig, + initialState: ModelState(prompt: currentPrompt) + ) + ) + guard let realtimeManager else { - preconditionFailure("realtimeManager is nil after creating it") + preconditionFailure("realtimeManager is nil after creation") } - monitorEvents() + // Listen for connection state changes (connecting, connected, error, etc.) + startEventMonitoring() #if !targetEnvironment(simulator) - let videoSource = realtimeManager.createVideoSource() - capture = RealtimeCapture(model: modelConfig, videoSource: videoSource) - try await capture?.startCapture() - - let localVideoTrack = realtimeManager.createVideoTrack(source: videoSource, trackId: "video0") - localMediaStream = RealtimeMediaStream(videoTrack: localVideoTrack, id: .localStream) + try await startCapture(model: modelConfig) + // Establish WebRTC connection - sends local video, receives AI-processed video remoteMediaStreams = try await realtimeManager.connect(localStream: localMediaStream!) #endif } catch { - DecartLogger.log( - "Connection failed with error: \(error.localizedDescription)", level: .error - ) - DecartLogger.log("Error details: \(error)", level: .error) + DecartLogger.log("Connection failed: \(error.localizedDescription)", level: .error) await cleanup() } } - private func monitorEvents() { - eventTask?.cancel() - - eventTask = Task { [weak self] in - guard let self, let stream = self.realtimeManager?.events else { return } - - for await state in stream { - if Task.isCancelled { return } - - DecartLogger.log("Connection state changed: \(state)", level: .info) - self.connectionState = state - - if state == .error { - DecartLogger.log("Error state received", level: .error) - } - } - DecartLogger.log("Event monitoring task completed.", level: .info) + func switchCamera() async { + #if !targetEnvironment(simulator) + guard let capture else { return } + do { + // Toggle between front and back camera + try await capture.switchCamera() + shouldMirror = capture.position == .front + } catch { + DecartLogger.log("Failed to switch camera", level: .error) } + #endif } func cleanup() async { connectionState = .idle + + // Brief delay to allow UI to update before teardown try? await Task.sleep(nanoseconds: 100_000_000) eventTask?.cancel() eventTask = nil - localMediaStream?.videoTrack.isEnabled = false - localMediaStream?.audioTrack?.isEnabled = false - remoteMediaStreams?.videoTrack.isEnabled = false - remoteMediaStreams?.audioTrack?.isEnabled = false + disableMediaTracks() #if !targetEnvironment(simulator) + // Release camera resources await capture?.stopCapture() capture = nil #endif + // Close WebRTC connection and release server resources await realtimeManager?.disconnect() realtimeManager = nil localMediaStream = nil remoteMediaStreams = nil } + + // MARK: - Private Helpers + + #if !targetEnvironment(simulator) + private func startCapture(model: ModelDefinition) async throws { + guard let realtimeManager else { return } + + // Create a video source that camera frames will be written to + let videoSource = realtimeManager.createVideoSource() + + // Initialize camera capture with model-specific settings (resolution, fps) + capture = RealtimeCapture(model: model, videoSource: videoSource) + try await capture?.startCapture() + + // Wrap the video source in a track for WebRTC transmission + let videoTrack = realtimeManager.createVideoTrack(source: videoSource, trackId: "video0") + localMediaStream = RealtimeMediaStream(videoTrack: videoTrack, id: .localStream) + } + #endif + + private func startEventMonitoring() { + eventTask?.cancel() + + eventTask = Task { [weak self] in + // Subscribe to connection state updates from the SDK + guard let self, let stream = self.realtimeManager?.events else { return } + + for await state in stream { + if Task.isCancelled { return } + self.connectionState = state + } + } + } + + private func disableMediaTracks() { + localMediaStream?.videoTrack.isEnabled = false + localMediaStream?.audioTrack?.isEnabled = false + remoteMediaStreams?.videoTrack.isEnabled = false + remoteMediaStreams?.audioTrack?.isEnabled = false + } } diff --git a/Example/Example/Views/RealtimeView.swift b/Example/Example/Views/RealtimeView.swift index 16569e3..b3202f1 100644 --- a/Example/Example/Views/RealtimeView.swift +++ b/Example/Example/Views/RealtimeView.swift @@ -6,7 +6,7 @@ import WebRTC struct RealtimeView: View { private let realtimeAiModel: RealtimeModel @State private var prompt: String = DecartConfig.defaultPrompt - @State private var realtimeManager: DecartRealtimeManager? + @State private var realtimeManager: RealtimeManager? init(realtimeModel: RealtimeModel) { self.realtimeAiModel = realtimeModel @@ -26,7 +26,7 @@ struct RealtimeView: View { } .onAppear { if realtimeManager == nil { - realtimeManager = DecartRealtimeManager( + realtimeManager = RealtimeManager( currentPrompt: Prompt(text: DecartConfig.defaultPrompt, enrich: false) ) } @@ -41,7 +41,7 @@ struct RealtimeView: View { } private struct RealtimeContentView: View { - @Bindable var realtimeManager: DecartRealtimeManager + @Bindable var realtimeManager: RealtimeManager let realtimeAiModel: RealtimeModel @Binding var prompt: String diff --git a/README.md b/README.md index 1694425..cffd857 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,16 @@ VideoConfig( - Swift 5.9+ - Xcode 15.0+ +## Environment Variables + +Configure these in your Xcode scheme (Edit Scheme → Run → Environment Variables): + +| Variable | Required | Description | +|----------|----------|-------------| +| `DECART_API_KEY` | Yes | Your Decart API key from [platform.decart.ai](https://platform.decart.ai) | +| `DECART_DEFAULT_PROMPT` | No | Default prompt for realtime sessions (defaults to "Simpsons") | +| `ENABLE_DECART_SDK_DUBUG_LOGS` | No | Set to `YES` to enable verbose SDK logging | + ## Dependencies - [WebRTC](https://github.com/nickkjordan/WebRTC) - WebRTC framework for iOS/macOS diff --git a/Sources/DecartSDK/DecartSDK.swift b/Sources/DecartSDK/DecartSDK.swift index 2cb2e6e..79790bc 100644 --- a/Sources/DecartSDK/DecartSDK.swift +++ b/Sources/DecartSDK/DecartSDK.swift @@ -37,7 +37,7 @@ public struct DecartClient { self.decartConfiguration = decartConfiguration } - public func createRealtimeManager(options: RealtimeConfiguration) throws -> RealtimeManager { + public func createRealtimeManager(options: RealtimeConfiguration) throws -> DecartRealtimeManager { let urlString = "\(decartConfiguration.signalingServerUrl)\(options.model.urlPath)?api_key=\(decartConfiguration.apiKey)&model=\(options.model.name)" @@ -46,7 +46,7 @@ public struct DecartClient { throw DecartError.invalidBaseURL(urlString) } - return RealtimeManager( + return DecartRealtimeManager( signalingServerURL: signalingServerURL, options: options ) diff --git a/Sources/DecartSDK/Realtime/RealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift similarity index 98% rename from Sources/DecartSDK/Realtime/RealtimeManager.swift rename to Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index 964db4b..e42474d 100644 --- a/Sources/DecartSDK/Realtime/RealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -1,7 +1,7 @@ import Foundation @preconcurrency import WebRTC -public final class RealtimeManager: @unchecked Sendable { +public final class DecartRealtimeManager: @unchecked Sendable { public let options: RealtimeConfiguration public let events: AsyncStream diff --git a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift index d6bf902..1906603 100644 --- a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift +++ b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift @@ -83,7 +83,7 @@ public struct RealtimeConfiguration: Sendable { public let preferredCodec: String public init( - maxBitrate: Int = 2_000_000, + maxBitrate: Int = 3_000_000, minBitrate: Int = 500_000, maxFramerate: Int = 26, preferredCodec: String = "VP8" diff --git a/Sources/DecartSDK/Realtime/RealtimeManager+Media.swift b/Sources/DecartSDK/Realtime/RealtimeManager+Media.swift index ab26555..eb3877d 100644 --- a/Sources/DecartSDK/Realtime/RealtimeManager+Media.swift +++ b/Sources/DecartSDK/Realtime/RealtimeManager+Media.swift @@ -1,7 +1,7 @@ import Foundation import WebRTC -extension RealtimeManager { +extension DecartRealtimeManager { public func getTransceivers() -> [RTCRtpTransceiver] { webRTCClient.transceivers } diff --git a/Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift b/Sources/DecartSDK/Realtime/Websocket/SignalingModel.swift similarity index 100% rename from Sources/DecartSDK/Realtime/WebRTC/SignalingModel.swift rename to Sources/DecartSDK/Realtime/Websocket/SignalingModel.swift From f6bd965df60c2152abfa67de93b2d22f3be0dec2 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Sun, 30 Nov 2025 14:21:35 +0200 Subject: [PATCH 08/23] refactor connection logic --- ...imeManager.swift => RealtimeManager.swift} | 0 .../Realtime/DecartRealtimeManager.swift | 163 +++++++------- .../Realtime/RealtimeConfiguration.swift | 41 ++-- .../Realtime/RealtimeManager+Media.swift | 31 --- .../Realtime/WebRTC/WebRTCClient.swift | 199 ++++++++++-------- .../Realtime/Websocket/WebSocketClient.swift | 34 ++- 6 files changed, 228 insertions(+), 240 deletions(-) rename Example/Example/DecartSDK/{DecartRealtimeManager.swift => RealtimeManager.swift} (100%) delete mode 100644 Sources/DecartSDK/Realtime/RealtimeManager+Media.swift diff --git a/Example/Example/DecartSDK/DecartRealtimeManager.swift b/Example/Example/DecartSDK/RealtimeManager.swift similarity index 100% rename from Example/Example/DecartSDK/DecartRealtimeManager.swift rename to Example/Example/DecartSDK/RealtimeManager.swift diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index e42474d..a804bba 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -5,7 +5,7 @@ public final class DecartRealtimeManager: @unchecked Sendable { public let options: RealtimeConfiguration public let events: AsyncStream - let webRTCClient: WebRTCClient + private var webRTCClient: WebRTCClient? private var webSocketClient: WebSocketClient? private let signalingServerURL: URL @@ -30,57 +30,78 @@ public final class DecartRealtimeManager: @unchecked Sendable { ) self.events = stream self.stateContinuation = continuation - self.webRTCClient = WebRTCClient() } - // MARK: - Public API + deinit { + webSocketListenerTask?.cancel() + connectionStateListenerTask?.cancel() + webRTCClient?.close() + stateContinuation.finish() + DecartLogger.log("RealtimeManager (SDK) deinitialized", level: .info) + } +} - public func connect(localStream: RealtimeMediaStream) async throws -> RealtimeMediaStream { +// MARK: - Public API + +public extension DecartRealtimeManager { + func connect(localStream: RealtimeMediaStream) async throws -> RealtimeMediaStream { connectionState = .connecting - let wsClient = WebSocketClient() + let wsClient = await WebSocketClient(url: signalingServerURL) webSocketClient = wsClient - await wsClient.connect(url: signalingServerURL) setupWebSocketListener(wsClient) - webRTCClient.createPeerConnection( + let rtcClient = WebRTCClient( config: options.connection.makeRTCConfiguration(), constraints: options.media.connectionConstraints, - sendMessage: { [weak self] in self?.sendMessage($0) } + videoConfig: options.media.video, + sendMessage: { [weak self] in self?.sendMessage($0) }, + withAudio: localStream.audioTrack != nil ) - setupConnectionStateListener() - - webRTCClient.addTrack(localStream.videoTrack, streamIds: [localStream.id]) - if let audioTrack = localStream.audioTrack { - webRTCClient.addTrack(audioTrack, streamIds: [localStream.id]) - } + webRTCClient = rtcClient + setupConnectionStateListener(rtcClient) - webRTCClient.configureVideoTransceiver(videoConfig: options.media.video) + rtcClient.startLocalStreaming( + videoTrack: localStream.videoTrack, + audioTrack: localStream.audioTrack + ) - let offer = try await webRTCClient.createOffer(constraints: options.media.offerConstraints) - try await webRTCClient.setLocalDescription(offer) + let offer = try await rtcClient.createOffer(constraints: options.media.offerConstraints) + try await rtcClient.setLocalDescription(offer) sendMessage(.offer(OfferMessage(sdp: offer.sdp))) try await waitForConnection(timeout: TimeInterval(options.connection.connectionTimeout) / 1000) - return try extractRemoteStream() - } + guard let remoteStream = rtcClient.getRemoteRealtimeStream() else { + throw DecartError.webRTCError("couldn't get remote stream, check video transceiver") + } - public func disconnect() async { - await cleanup() + return remoteStream } - public func setPrompt(_ prompt: Prompt) { - sendMessage(.prompt(PromptMessage(prompt: prompt.text))) - } + func disconnect() async { + connectionState = .disconnected + webSocketListenerTask?.cancel() + webSocketListenerTask = nil + connectionStateListenerTask?.cancel() + connectionStateListenerTask = nil + webRTCClient?.close() + webRTCClient = nil - public func getStats() async -> RTCStatisticsReport? { - await webRTCClient.peerConnection?.statistics() + let audioSession = RTCAudioSession.sharedInstance() + if audioSession.isActive { + audioSession.lockForConfiguration() + try? audioSession.setActive(false) + audioSession.unlockForConfiguration() + } + webSocketClient = nil } - // MARK: - Private + func setPrompt(_ prompt: Prompt) { + sendMessage(.prompt(PromptMessage(prompt: prompt.text))) + } - private func waitForConnection(timeout: TimeInterval) async throws { + func waitForConnection(timeout: TimeInterval) async throws { let startTime = Date() while connectionState != .connected { if connectionState == .error || connectionState == .disconnected { @@ -93,32 +114,42 @@ public final class DecartRealtimeManager: @unchecked Sendable { } sendMessage(.prompt(PromptMessage(prompt: options.initialState.prompt.text))) } +} - private func extractRemoteStream() throws -> RealtimeMediaStream { - guard let videoTransceiver = webRTCClient.transceivers.first(where: { $0.mediaType == .video }) else { - throw DecartError.webRTCError("Video transceiver not found") - } - guard let remoteVideoTrack = videoTransceiver.receiver.track as? RTCVideoTrack else { - throw DecartError.webRTCError("Remote video track not found") - } - let remoteAudioTrack = webRTCClient.transceivers - .first(where: { $0.mediaType == .audio })? - .receiver.track as? RTCAudioTrack - - return RealtimeMediaStream( - videoTrack: remoteVideoTrack, - audioTrack: remoteAudioTrack, - id: .remoteStream - ) +// MARK: - Connection + +public extension DecartRealtimeManager { + func createVideoSource() -> RTCVideoSource { + WebRTCClient.createVideoSource() } - private func setupWebSocketListener(_ wsClient: WebSocketClient) { + func replaceVideoTrack(with newTrack: RTCVideoTrack) { + webRTCClient?.replaceVideoTrack(with: newTrack) + } + + func createVideoTrack(source: RTCVideoSource, trackId: String) -> RTCVideoTrack { + WebRTCClient.createVideoTrack(source: source, trackId: trackId) + } + + func createAudioSource(constraints: RTCMediaConstraints? = nil) -> RTCAudioSource { + WebRTCClient.createAudioSource(constraints: constraints) + } + + func createAudioTrack(source: RTCAudioSource, trackId: String) -> RTCAudioTrack { + WebRTCClient.createAudioTrack(source: source, trackId: trackId) + } +} + +// MARK: - Listeners + +private extension DecartRealtimeManager { + func setupWebSocketListener(_ wsClient: WebSocketClient) { webSocketListenerTask?.cancel() webSocketListenerTask = Task { [weak self] in do { for try await message in wsClient.websocketEventStream { - guard !Task.isCancelled, let self else { return } - try await self.webRTCClient.handleSignalingMessage(message) + guard !Task.isCancelled, let self, let webRTCClient = self.webRTCClient else { return } + try await webRTCClient.handleSignalingMessage(message) } } catch { self?.connectionState = .error @@ -126,11 +157,10 @@ public final class DecartRealtimeManager: @unchecked Sendable { } } - private func setupConnectionStateListener() { + func setupConnectionStateListener(_ rtcClient: WebRTCClient) { connectionStateListenerTask?.cancel() - guard let stateStream = webRTCClient.connectionStateStream else { return } connectionStateListenerTask = Task { [weak self] in - for await rtcState in stateStream { + for await rtcState in rtcClient.connectionStateStream { guard !Task.isCancelled, let self else { return } switch rtcState { case .connected: self.connectionState = .connected @@ -141,36 +171,13 @@ public final class DecartRealtimeManager: @unchecked Sendable { } } } +} + +// MARK: - Messaging - private func sendMessage(_ message: OutgoingWebSocketMessage) { +private extension DecartRealtimeManager { + func sendMessage(_ message: OutgoingWebSocketMessage) { guard let webSocketClient else { return } Task { try? await webSocketClient.send(message) } } - - private func cleanup() async { - connectionState = .disconnected - webSocketListenerTask?.cancel() - webSocketListenerTask = nil - connectionStateListenerTask?.cancel() - connectionStateListenerTask = nil - webRTCClient.closePeerConnection() - - let audioSession = RTCAudioSession.sharedInstance() - if audioSession.isActive { - audioSession.lockForConfiguration() - try? audioSession.setActive(false) - audioSession.unlockForConfiguration() - } - - await webSocketClient?.disconnect() - webSocketClient = nil - } - - deinit { - webSocketListenerTask?.cancel() - connectionStateListenerTask?.cancel() - webRTCClient.closePeerConnection() - stateContinuation.finish() - DecartLogger.log("RealtimeManager (SDK) deinitialized", level: .info) - } } diff --git a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift index 1906603..5193883 100644 --- a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift +++ b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift @@ -83,8 +83,8 @@ public struct RealtimeConfiguration: Sendable { public let preferredCodec: String public init( - maxBitrate: Int = 3_000_000, - minBitrate: Int = 500_000, + maxBitrate: Int = 3_500_000, + minBitrate: Int = 400_000, maxFramerate: Int = 26, preferredCodec: String = "VP8" ) { @@ -94,22 +94,32 @@ public struct RealtimeConfiguration: Sendable { self.preferredCodec = preferredCodec } - public func configure(transceiver: RTCRtpTransceiver, factory: RTCPeerConnectionFactory) { + func makeTransceiverInit() -> RTCRtpTransceiverInit { + let transceiverInit = RTCRtpTransceiverInit() + transceiverInit.direction = .sendRecv + + let encoding = RTCRtpEncodingParameters() + encoding.maxBitrateBps = NSNumber(value: maxBitrate) + encoding.minBitrateBps = NSNumber(value: minBitrate) + encoding.maxFramerate = NSNumber(value: maxFramerate) + transceiverInit.sendEncodings = [encoding] + + return transceiverInit + } + + func configureTransceiver(_ transceiver: RTCRtpTransceiver, factory: RTCPeerConnectionFactory) { let supportedCodecs = factory.rtpSenderCapabilities(forKind: "video").codecs + let preferredCodecName = preferredCodec.uppercased() var preferredCodecs: [RTCRtpCodecCapability] = [] var otherCodecs: [RTCRtpCodecCapability] = [] var utilityCodecs: [RTCRtpCodecCapability] = [] - let preferredCodecName = preferredCodec.uppercased() - for codec in supportedCodecs { let codecNameUpper = codec.name.uppercased() if codecNameUpper == preferredCodecName { preferredCodecs.append(codec) - } else if codecNameUpper == "RTX" || codecNameUpper == "RED" - || codecNameUpper == "ULPFEC" - { + } else if codecNameUpper == "RTX" || codecNameUpper == "RED" || codecNameUpper == "ULPFEC" { utilityCodecs.append(codec) } else { otherCodecs.append(codec) @@ -117,20 +127,7 @@ public struct RealtimeConfiguration: Sendable { } let sortedCodecs = preferredCodecs + otherCodecs + utilityCodecs - try? transceiver.setCodecPreferences(sortedCodecs, error: ()) - - let sender = transceiver.sender - let parameters = sender.parameters - if parameters.encodings.indices.contains(0) { - let encodingParam = parameters.encodings[0] - encodingParam.maxBitrateBps = NSNumber(value: maxBitrate) - encodingParam.minBitrateBps = NSNumber(value: minBitrate) - encodingParam.maxFramerate = NSNumber(value: maxFramerate) - encodingParam.scaleResolutionDownBy = NSNumber(value: 1.0) - - parameters.encodings[0] = encodingParam - sender.parameters = parameters - } + transceiver.setCodecPreferences(sortedCodecs) } } } diff --git a/Sources/DecartSDK/Realtime/RealtimeManager+Media.swift b/Sources/DecartSDK/Realtime/RealtimeManager+Media.swift deleted file mode 100644 index eb3877d..0000000 --- a/Sources/DecartSDK/Realtime/RealtimeManager+Media.swift +++ /dev/null @@ -1,31 +0,0 @@ -import Foundation -import WebRTC - -extension DecartRealtimeManager { - public func getTransceivers() -> [RTCRtpTransceiver] { - webRTCClient.transceivers - } - - public func createAudioSource(constraints: RTCMediaConstraints? = nil) -> RTCAudioSource { - webRTCClient.createAudioSource(constraints: constraints) - } - - public func createAudioTrack(source: RTCAudioSource, trackId: String) -> RTCAudioTrack { - webRTCClient.createAudioTrack(source: source, trackId: trackId) - } - - public func createVideoSource() -> RTCVideoSource { - webRTCClient.createVideoSource() - } - - public func createVideoTrack(source: RTCVideoSource, trackId: String) -> RTCVideoTrack { - webRTCClient.createVideoTrack(source: source, trackId: trackId) - } - - public func createLocalVideoTrack() -> (RTCVideoTrack, RTCCameraVideoCapturer) { - let videoSource = createVideoSource() - let videoTrack = createVideoTrack(source: videoSource, trackId: UUID().uuidString) - let videoCapturer = RTCCameraVideoCapturer(delegate: videoSource) - return (videoTrack, videoCapturer) - } -} diff --git a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift index 810ba41..f2babf3 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift @@ -1,17 +1,20 @@ import Foundation @preconcurrency import WebRTC -final class WebRTCClient { +final class WebRTCClient: @unchecked Sendable { private nonisolated(unsafe) static var sharedFactory: RTCPeerConnectionFactory? private static let factoryLock = NSLock() - let factory: RTCPeerConnectionFactory - private(set) var peerConnection: RTCPeerConnection? - private(set) var connectionStateStream: AsyncStream? + nonisolated(unsafe) let factory: RTCPeerConnectionFactory + let peerConnection: RTCPeerConnection + let connectionStateStream: AsyncStream - private var delegateHandler: WebRTCDelegateHandler? - private var signalingClient: SignalingClient? - private var connectionStateContinuation: AsyncStream.Continuation? + private let delegateHandler: WebRTCDelegateHandler + private let signalingClient: SignalingClient + private let connectionStateContinuation: AsyncStream.Continuation + + nonisolated(unsafe) var videoTransceiver: RTCRtpTransceiver? + nonisolated(unsafe) var audioTransceiver: RTCRtpTransceiver? private static func getOrCreateFactory() -> RTCPeerConnectionFactory { factoryLock.lock() @@ -22,7 +25,6 @@ final class WebRTCClient { } RTCInitializeSSL() - // RTCSetMinDebugLogLevel(.warning) let factory = RTCPeerConnectionFactory( encoderFactory: RTCDefaultVideoEncoderFactory(), @@ -32,137 +34,158 @@ final class WebRTCClient { return factory } - init() { - self.factory = Self.getOrCreateFactory() - } - - func createPeerConnection( + init( config: RTCConfiguration, constraints: RTCMediaConstraints, - sendMessage: @escaping (OutgoingWebSocketMessage) -> Void + videoConfig: RealtimeConfiguration.VideoConfig, + sendMessage: @escaping (OutgoingWebSocketMessage) -> Void, + withAudio: Bool ) { + self.factory = Self.getOrCreateFactory() + let (stream, continuation) = AsyncStream.makeStream(of: RTCPeerConnectionState.self) - connectionStateStream = stream - connectionStateContinuation = continuation + self.connectionStateStream = stream + self.connectionStateContinuation = continuation - delegateHandler = WebRTCDelegateHandler( + self.delegateHandler = WebRTCDelegateHandler( sendMessage: sendMessage, connectionStateContinuation: continuation ) - peerConnection = factory.peerConnection( + self.peerConnection = factory.peerConnection( with: config, constraints: constraints, delegate: delegateHandler )! - signalingClient = SignalingClient( - peerConnection: peerConnection!, + self.signalingClient = SignalingClient( + peerConnection: peerConnection, factory: factory, sendMessage: sendMessage ) + + prepareTransceivers(videoConfig: videoConfig, withAudio: withAudio) } func handleSignalingMessage(_ message: IncomingWebSocketMessage) async throws { - try await signalingClient?.handleMessage(message) + try await signalingClient.handleMessage(message) } - // MARK: - Track Operations - - func addTrack(_ track: RTCMediaStreamTrack, streamIds: [String]) { - peerConnection?.add(track, streamIds: streamIds) + deinit { + DecartLogger.log("Webrtc client deinitialized", level: .info) + close() } +} - func configureVideoTransceiver(videoConfig: RealtimeConfiguration.VideoConfig) { - if let transceiver = peerConnection?.transceivers.first(where: { $0.mediaType == .video }) { - videoConfig.configure(transceiver: transceiver, factory: factory) +// MARK: - Track Operations + +extension WebRTCClient { + func prepareTransceivers(videoConfig: RealtimeConfiguration.VideoConfig, withAudio: Bool) { + if withAudio { + let audioInit = RTCRtpTransceiverInit() + audioInit.direction = .sendRecv + audioTransceiver = peerConnection.addTransceiver(of: .audio, init: audioInit) } - } - var transceivers: [RTCRtpTransceiver] { - peerConnection?.transceivers ?? [] + videoTransceiver = peerConnection.addTransceiver(of: .video, init: videoConfig.makeTransceiverInit()) + if let videoTransceiver { + videoConfig.configureTransceiver(videoTransceiver, factory: factory) + } } - // MARK: - SDP Operations - - func createOffer(constraints: RTCMediaConstraints) async throws -> RTCSessionDescription { - guard let peerConnection else { - throw DecartError.webRTCError("peer connection not initialized") + nonisolated func replaceVideoTrack(with newTrack: RTCVideoTrack) { + guard let videoTransceiver else { + fatalError("Video track does not exist") } - guard let offer = try await peerConnection.offer(for: constraints) else { - throw DecartError.webRTCError("failed to create offer") - } - return offer + videoTransceiver.sender.track = newTrack } - func createAnswer(constraints: RTCMediaConstraints) async throws -> RTCSessionDescription { - guard let peerConnection else { - throw DecartError.webRTCError("peer connection not initialized") - } - guard let answer = try await peerConnection.answer(for: constraints) else { - throw DecartError.webRTCError("failed to create answer") - } - return answer + nonisolated static func createVideoSource() -> RTCVideoSource { + WebRTCClient.getOrCreateFactory().videoSource() } - func setLocalDescription(_ sdp: RTCSessionDescription) async throws { - guard let peerConnection else { - throw DecartError.webRTCError("peer connection not initialized") - } - try await peerConnection.setLocalDescription(sdp) + nonisolated static func createVideoTrack(source: RTCVideoSource, trackId: String) -> RTCVideoTrack { + WebRTCClient.getOrCreateFactory().videoTrack(with: source, trackId: trackId) } - func setRemoteDescription(_ sdp: RTCSessionDescription) async throws { - guard let peerConnection else { - throw DecartError.webRTCError("peer connection not initialized") - } - try await peerConnection.setRemoteDescription(sdp) + nonisolated static func createAudioSource(constraints: RTCMediaConstraints? = nil) -> RTCAudioSource { + WebRTCClient.getOrCreateFactory().audioSource(with: constraints) } - // MARK: - ICE Operations + nonisolated static func createAudioTrack(source: RTCAudioSource, trackId: String) -> RTCAudioTrack { + WebRTCClient.getOrCreateFactory().audioTrack(with: source, trackId: trackId) + } +} - func addIceCandidate(_ candidate: RTCIceCandidate) async throws { - guard let peerConnection else { - throw DecartError.webRTCError("peer connection not initialized") +// MARK: - Streaming + +extension WebRTCClient { + nonisolated func getRemoteRealtimeStream() -> RealtimeMediaStream? { + guard let remoteVideoTrack = videoTransceiver?.receiver.track as? RTCVideoTrack else { + return nil } - try await peerConnection.add(candidate) - } - // MARK: - Media Factory + let remoteAudioTrack = audioTransceiver?.receiver.track as? RTCAudioTrack - func createVideoSource() -> RTCVideoSource { - factory.videoSource() + return RealtimeMediaStream( + videoTrack: remoteVideoTrack, + audioTrack: remoteAudioTrack, + id: .remoteStream + ) } - func createVideoTrack(source: RTCVideoSource, trackId: String) -> RTCVideoTrack { - factory.videoTrack(with: source, trackId: trackId) + @discardableResult + nonisolated func startLocalStreaming(videoTrack: RTCVideoTrack, audioTrack: RTCAudioTrack? = nil) -> RealtimeMediaStream { + if let videoSender = videoTransceiver?.sender { + videoSender.track = videoTrack + } + + if let audioSender = audioTransceiver?.sender { + audioSender.track = audioTrack + } + + return RealtimeMediaStream( + videoTrack: videoTrack, + audioTrack: audioTrack, + id: .localStream + ) } +} + +// MARK: - SDP Operations - func createAudioSource(constraints: RTCMediaConstraints?) -> RTCAudioSource { - factory.audioSource(with: constraints) +extension WebRTCClient { + func createOffer(constraints: RTCMediaConstraints) async throws -> RTCSessionDescription { + guard let offer = try await peerConnection.offer(for: constraints) else { + throw DecartError.webRTCError("failed to create offer") + } + return offer } - func createAudioTrack(source: RTCAudioSource, trackId: String) -> RTCAudioTrack { - factory.audioTrack(with: source, trackId: trackId) + func setLocalDescription(_ sdp: RTCSessionDescription) async throws { + try await peerConnection.setLocalDescription(sdp) } +} - // MARK: - Cleanup +// MARK: - ICE Operations - func closePeerConnection() { - delegateHandler?.cleanup() - connectionStateContinuation?.finish() - peerConnection?.close() - peerConnection?.delegate = nil - peerConnection = nil - signalingClient = nil - delegateHandler = nil - connectionStateStream = nil - connectionStateContinuation = nil +extension WebRTCClient { + func addIceCandidate(_ candidate: RTCIceCandidate) async throws { + try await peerConnection.add(candidate) } +} - deinit { - DecartLogger.log("Webrtc client deinitialized", level: .info) - closePeerConnection() - // Note: Don't call RTCCleanupSSL() - factory is singleton, SSL stays initialized +// MARK: - Cleanup + +extension WebRTCClient { + func close() { + videoTransceiver?.sender.track = nil + audioTransceiver?.sender.track = nil + delegateHandler.cleanup() + connectionStateContinuation.finish() + peerConnection.close() + peerConnection.delegate = nil + videoTransceiver = nil + audioTransceiver = nil } } diff --git a/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift b/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift index d289c0a..e3dc87b 100644 --- a/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift +++ b/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift @@ -3,29 +3,29 @@ import Foundation actor WebSocketClient { var isConnected: Bool = false - private var stream: SocketStream? + private let stream: SocketStream private var listeningTask: Task? private let decoder = JSONDecoder() private let encoder = JSONEncoder() - private nonisolated(unsafe) let eventStreamContinuation: AsyncStream.Continuation + private let eventStreamContinuation: AsyncStream.Continuation nonisolated let websocketEventStream: AsyncStream - init() { + init(url: URL) async { let (websocketEventStream, eventStreamContinuation) = AsyncStream.makeStream(of: IncomingWebSocketMessage.self) self.eventStreamContinuation = eventStreamContinuation self.websocketEventStream = websocketEventStream - } - - func connect(url: URL) { - guard stream == nil else { return } let socketConnection = URLSession.shared.webSocketTask(with: url) stream = SocketStream(task: socketConnection) - isConnected = true + mountListener() + } + + private func mountListener() { listeningTask = Task { [weak self] in - guard let self, let stream = await self.stream else { return } + guard let self else { return } do { - for try await msg in stream { + for try await msg in self.stream { + if Task.isCancelled { return } switch msg { case .string(let text): await self.handleIncomingMessage(text) @@ -37,7 +37,7 @@ actor WebSocketClient { } } } catch { - await self.eventStreamContinuation.finish() + self.eventStreamContinuation.finish() } } } @@ -49,7 +49,6 @@ actor WebSocketClient { } func send(_ message: T) throws { - guard let stream else { return } let data = try encoder.encode(message) guard let jsonString = String(data: data, encoding: .utf8) else { throw DecartError.websocketError("unable to encode message") @@ -57,16 +56,9 @@ actor WebSocketClient { Task { [stream] in try await stream.sendMessage(.string(jsonString)) } } - func disconnect() async { - eventStreamContinuation.finish() - listeningTask?.cancel() - listeningTask = nil - stream?.cancel() - stream = nil - isConnected = false - } - deinit { + listeningTask?.cancel() eventStreamContinuation.finish() + stream.cancel() } } From c0a3bac30e8c30cca54c9ea24a32ddcf9ca6f29e Mon Sep 17 00:00:00 2001 From: Verion1 Date: Sun, 30 Nov 2025 14:53:21 +0200 Subject: [PATCH 09/23] changed config interface --- Example.xcodeproj/project.pbxproj | 32 ++++++++++++------- .../xcshareddata/swiftpm/Package.resolved | 2 +- .../xcshareddata/xcschemes/Example.xcscheme | 6 ++-- .../Realtime/DecartRealtimeManager.swift | 2 +- .../Realtime/RealtimeConfiguration.swift | 14 +++----- .../Realtime/WebRTC/WebRTCClient.swift | 5 ++- 6 files changed, 35 insertions(+), 26 deletions(-) diff --git a/Example.xcodeproj/project.pbxproj b/Example.xcodeproj/project.pbxproj index 9cd3c33..703b3c2 100644 --- a/Example.xcodeproj/project.pbxproj +++ b/Example.xcodeproj/project.pbxproj @@ -9,7 +9,7 @@ /* Begin PBXBuildFile section */ 165B2B9B2ECE09E9004D848C /* Factory in Frameworks */ = {isa = PBXBuildFile; productRef = 165B2B9A2ECE09E9004D848C /* Factory */; }; 165B2B9D2ECE09E9004D848C /* FactoryKit in Frameworks */ = {isa = PBXBuildFile; productRef = 165B2B9C2ECE09E9004D848C /* FactoryKit */; }; - 16E67C3C2EB8CC9F00AF5515 /* DecartSDK in Frameworks */ = {isa = PBXBuildFile; productRef = 16E67C3B2EB8CC9F00AF5515 /* DecartSDK */; }; + 165C49642EDC6F2F00FC00B8 /* DecartSDK in Frameworks */ = {isa = PBXBuildFile; productRef = 165C49632EDC6F2F00FC00B8 /* DecartSDK */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -30,8 +30,8 @@ buildActionMask = 2147483647; files = ( 165B2B9D2ECE09E9004D848C /* FactoryKit in Frameworks */, - 16E67C3C2EB8CC9F00AF5515 /* DecartSDK in Frameworks */, 165B2B9B2ECE09E9004D848C /* Factory in Frameworks */, + 165C49642EDC6F2F00FC00B8 /* DecartSDK in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -76,7 +76,7 @@ buildRules = ( ); dependencies = ( - 16E67C3E2EB8D01600AF5515 /* PBXTargetDependency */, + 165C49662EDC6F3A00FC00B8 /* PBXTargetDependency */, ); fileSystemSynchronizedGroups = ( 16D70B982EB8CA0C00455077 /* Example */, @@ -86,6 +86,8 @@ 16E67C3B2EB8CC9F00AF5515 /* DecartSDK */, 165B2B9A2ECE09E9004D848C /* Factory */, 165B2B9C2ECE09E9004D848C /* FactoryKit */, + 1648B2072ED88F6D00EF4368 /* DecartSDK */, + 165C49632EDC6F2F00FC00B8 /* DecartSDK */, ); productName = Example; productReference = 16D70B962EB8CA0C00455077 /* Example.app */; @@ -116,8 +118,8 @@ mainGroup = 16D70B8D2EB8CA0C00455077; minimizedProjectReferenceProxies = 1; packageReferences = ( - 16E67C3A2EB8CC9F00AF5515 /* XCLocalSwiftPackageReference "../decart-ios" */, 165B2B992ECE09E9004D848C /* XCRemoteSwiftPackageReference "Factory" */, + 165C49622EDC6F2F00FC00B8 /* XCLocalSwiftPackageReference "../decart-ios" */, ); preferredProjectObjectVersion = 77; productRefGroup = 16D70B972EB8CA0C00455077 /* Products */; @@ -150,9 +152,9 @@ /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ - 16E67C3E2EB8D01600AF5515 /* PBXTargetDependency */ = { + 165C49662EDC6F3A00FC00B8 /* PBXTargetDependency */ = { isa = PBXTargetDependency; - productRef = 16E67C3D2EB8D01600AF5515 /* DecartSDK */; + productRef = 165C49652EDC6F3A00FC00B8 /* DecartSDK */; }; /* End PBXTargetDependency section */ @@ -344,7 +346,7 @@ "@executable_path/Frameworks", ); MARKETING_VERSION = 1.0; - PRODUCT_BUNDLE_IDENTIFIER = ai.decart.Examplea; + PRODUCT_BUNDLE_IDENTIFIER = ai.decart.Example; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; STRING_CATALOG_GENERATE_SYMBOLS = YES; @@ -382,7 +384,7 @@ /* End XCConfigurationList section */ /* Begin XCLocalSwiftPackageReference section */ - 16E67C3A2EB8CC9F00AF5515 /* XCLocalSwiftPackageReference "../decart-ios" */ = { + 165C49622EDC6F2F00FC00B8 /* XCLocalSwiftPackageReference "../decart-ios" */ = { isa = XCLocalSwiftPackageReference; relativePath = "../decart-ios"; }; @@ -400,6 +402,10 @@ /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ + 1648B2072ED88F6D00EF4368 /* DecartSDK */ = { + isa = XCSwiftPackageProductDependency; + productName = DecartSDK; + }; 165B2B9A2ECE09E9004D848C /* Factory */ = { isa = XCSwiftPackageProductDependency; package = 165B2B992ECE09E9004D848C /* XCRemoteSwiftPackageReference "Factory" */; @@ -410,13 +416,17 @@ package = 165B2B992ECE09E9004D848C /* XCRemoteSwiftPackageReference "Factory" */; productName = FactoryKit; }; - 16E67C3B2EB8CC9F00AF5515 /* DecartSDK */ = { + 165C49632EDC6F2F00FC00B8 /* DecartSDK */ = { isa = XCSwiftPackageProductDependency; productName = DecartSDK; }; - 16E67C3D2EB8D01600AF5515 /* DecartSDK */ = { + 165C49652EDC6F3A00FC00B8 /* DecartSDK */ = { + isa = XCSwiftPackageProductDependency; + package = 165C49622EDC6F2F00FC00B8 /* XCLocalSwiftPackageReference "../decart-ios" */; + productName = DecartSDK; + }; + 16E67C3B2EB8CC9F00AF5515 /* DecartSDK */ = { isa = XCSwiftPackageProductDependency; - package = 16E67C3A2EB8CC9F00AF5515 /* XCLocalSwiftPackageReference "../decart-ios" */; productName = DecartSDK; }; /* End XCSwiftPackageProductDependency section */ diff --git a/Example.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Example.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index d63e0f8..a11c13a 100644 --- a/Example.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Example.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "99ec935609087934ec5bde71ae62e18771f6ea5ae003c7c000ef8a1cd073784c", + "originHash" : "98d4c41180155d7ad94e6d792307b58cb8c04adedbfdc05c3629362f72d2d6da", "pins" : [ { "identity" : "factory", diff --git a/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme b/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme index b32be68..0f41522 100644 --- a/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme +++ b/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme @@ -31,9 +31,9 @@ shouldAutocreateTestPlan = "YES"> RTCConfiguration { @@ -48,8 +45,7 @@ public struct RealtimeConfiguration: Sendable { config.iceServers = [RTCIceServer(urlStrings: iceServers)] config.sdpSemantics = .unifiedPlan config.continualGatheringPolicy = .gatherContinually - config.iceConnectionReceivingTimeout = connectionTimeout -// config.iceBackupCandidatePairPingInterval = pingInterval + config.iceCandidatePoolSize = 10 return config } } @@ -83,8 +79,8 @@ public struct RealtimeConfiguration: Sendable { public let preferredCodec: String public init( - maxBitrate: Int = 3_500_000, - minBitrate: Int = 400_000, + maxBitrate: Int = 2_500_000, + minBitrate: Int = 300_000, maxFramerate: Int = 26, preferredCodec: String = "VP8" ) { diff --git a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift index f2babf3..1565ae9 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/WebRTCClient.swift @@ -43,7 +43,10 @@ final class WebRTCClient: @unchecked Sendable { ) { self.factory = Self.getOrCreateFactory() - let (stream, continuation) = AsyncStream.makeStream(of: RTCPeerConnectionState.self) + let (stream, continuation) = AsyncStream.makeStream( + of: RTCPeerConnectionState.self, + bufferingPolicy: .bufferingNewest(1) + ) self.connectionStateStream = stream self.connectionStateContinuation = continuation From 164e9fd40898901014e135e612c25b590a117839 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 3 Dec 2025 19:57:51 +0200 Subject: [PATCH 10/23] switched to new arch --- Example.xcodeproj/project.pbxproj | 35 +-- .../xcshareddata/swiftpm/Package.resolved | 36 +++ .../xcshareddata/xcschemes/Example.xcscheme | 6 +- Example/Example/Config.swift | 191 ++++++++++++- ...tConfig.swift => DecartClientShared.swift} | 2 +- .../Example/DecartSDK/RealtimeManager.swift | 8 +- .../Example/Views/RealtimeControlsView.swift | 250 ++++++++++++++++++ Example/Example/Views/RealtimeView.swift | 91 ++----- Package.resolved | 39 ++- Package.swift | 11 +- README.md | 26 +- .../Realtime/DecartRealtimeManager.swift | 4 +- .../Realtime/RealtimeConfiguration.swift | 33 ++- .../Realtime/Websocket/SocketStream.swift | 30 ++- .../Realtime/Websocket/WebSocketClient.swift | 54 ++-- 15 files changed, 652 insertions(+), 164 deletions(-) rename Example/Example/DecartSDK/{DecartConfig.swift => DecartClientShared.swift} (94%) create mode 100644 Example/Example/Views/RealtimeControlsView.swift diff --git a/Example.xcodeproj/project.pbxproj b/Example.xcodeproj/project.pbxproj index 703b3c2..a9e3407 100644 --- a/Example.xcodeproj/project.pbxproj +++ b/Example.xcodeproj/project.pbxproj @@ -9,7 +9,7 @@ /* Begin PBXBuildFile section */ 165B2B9B2ECE09E9004D848C /* Factory in Frameworks */ = {isa = PBXBuildFile; productRef = 165B2B9A2ECE09E9004D848C /* Factory */; }; 165B2B9D2ECE09E9004D848C /* FactoryKit in Frameworks */ = {isa = PBXBuildFile; productRef = 165B2B9C2ECE09E9004D848C /* FactoryKit */; }; - 165C49642EDC6F2F00FC00B8 /* DecartSDK in Frameworks */ = {isa = PBXBuildFile; productRef = 165C49632EDC6F2F00FC00B8 /* DecartSDK */; }; + 16B8AC262EDE0490005EEA8A /* DecartSDK in Frameworks */ = {isa = PBXBuildFile; productRef = 16B8AC252EDE0490005EEA8A /* DecartSDK */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -31,7 +31,7 @@ files = ( 165B2B9D2ECE09E9004D848C /* FactoryKit in Frameworks */, 165B2B9B2ECE09E9004D848C /* Factory in Frameworks */, - 165C49642EDC6F2F00FC00B8 /* DecartSDK in Frameworks */, + 16B8AC262EDE0490005EEA8A /* DecartSDK in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -76,18 +76,15 @@ buildRules = ( ); dependencies = ( - 165C49662EDC6F3A00FC00B8 /* PBXTargetDependency */, ); fileSystemSynchronizedGroups = ( 16D70B982EB8CA0C00455077 /* Example */, ); name = Example; packageProductDependencies = ( - 16E67C3B2EB8CC9F00AF5515 /* DecartSDK */, 165B2B9A2ECE09E9004D848C /* Factory */, 165B2B9C2ECE09E9004D848C /* FactoryKit */, - 1648B2072ED88F6D00EF4368 /* DecartSDK */, - 165C49632EDC6F2F00FC00B8 /* DecartSDK */, + 16B8AC252EDE0490005EEA8A /* DecartSDK */, ); productName = Example; productReference = 16D70B962EB8CA0C00455077 /* Example.app */; @@ -119,7 +116,7 @@ minimizedProjectReferenceProxies = 1; packageReferences = ( 165B2B992ECE09E9004D848C /* XCRemoteSwiftPackageReference "Factory" */, - 165C49622EDC6F2F00FC00B8 /* XCLocalSwiftPackageReference "../decart-ios" */, + 16B8AC242EDE0490005EEA8A /* XCLocalSwiftPackageReference "../decart-ios" */, ); preferredProjectObjectVersion = 77; productRefGroup = 16D70B972EB8CA0C00455077 /* Products */; @@ -151,13 +148,6 @@ }; /* End PBXSourcesBuildPhase section */ -/* Begin PBXTargetDependency section */ - 165C49662EDC6F3A00FC00B8 /* PBXTargetDependency */ = { - isa = PBXTargetDependency; - productRef = 165C49652EDC6F3A00FC00B8 /* DecartSDK */; - }; -/* End PBXTargetDependency section */ - /* Begin XCBuildConfiguration section */ 16D70B9F2EB8CA0D00455077 /* Debug */ = { isa = XCBuildConfiguration; @@ -384,7 +374,7 @@ /* End XCConfigurationList section */ /* Begin XCLocalSwiftPackageReference section */ - 165C49622EDC6F2F00FC00B8 /* XCLocalSwiftPackageReference "../decart-ios" */ = { + 16B8AC242EDE0490005EEA8A /* XCLocalSwiftPackageReference "../decart-ios" */ = { isa = XCLocalSwiftPackageReference; relativePath = "../decart-ios"; }; @@ -402,10 +392,6 @@ /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ - 1648B2072ED88F6D00EF4368 /* DecartSDK */ = { - isa = XCSwiftPackageProductDependency; - productName = DecartSDK; - }; 165B2B9A2ECE09E9004D848C /* Factory */ = { isa = XCSwiftPackageProductDependency; package = 165B2B992ECE09E9004D848C /* XCRemoteSwiftPackageReference "Factory" */; @@ -416,16 +402,7 @@ package = 165B2B992ECE09E9004D848C /* XCRemoteSwiftPackageReference "Factory" */; productName = FactoryKit; }; - 165C49632EDC6F2F00FC00B8 /* DecartSDK */ = { - isa = XCSwiftPackageProductDependency; - productName = DecartSDK; - }; - 165C49652EDC6F3A00FC00B8 /* DecartSDK */ = { - isa = XCSwiftPackageProductDependency; - package = 165C49622EDC6F2F00FC00B8 /* XCLocalSwiftPackageReference "../decart-ios" */; - productName = DecartSDK; - }; - 16E67C3B2EB8CC9F00AF5515 /* DecartSDK */ = { + 16B8AC252EDE0490005EEA8A /* DecartSDK */ = { isa = XCSwiftPackageProductDependency; productName = DecartSDK; }; diff --git a/Example.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Example.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index a11c13a..fb456d7 100644 --- a/Example.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Example.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,6 +1,24 @@ { "originHash" : "98d4c41180155d7ad94e6d792307b58cb8c04adedbfdc05c3629362f72d2d6da", "pins" : [ + { + "identity" : "async-extensions", + "kind" : "remoteSourceControl", + "location" : "https://github.com/shareup/async-extensions.git", + "state" : { + "revision" : "3088474141debc75b78257a0db28adf734bcea0f", + "version" : "4.4.0" + } + }, + { + "identity" : "dispatch-timer", + "kind" : "remoteSourceControl", + "location" : "https://github.com/shareup/dispatch-timer.git", + "state" : { + "revision" : "2d8c304aa6f382a7a362cd5a814884f3930c5662", + "version" : "3.0.1" + } + }, { "identity" : "factory", "kind" : "remoteSourceControl", @@ -10,6 +28,15 @@ "version" : "2.5.3" } }, + { + "identity" : "synchronized", + "kind" : "remoteSourceControl", + "location" : "https://github.com/shareup/synchronized.git", + "state" : { + "revision" : "85653e23270ec88ae19f8d494157769487e34aed", + "version" : "4.0.1" + } + }, { "identity" : "webrtc", "kind" : "remoteSourceControl", @@ -18,6 +45,15 @@ "revision" : "86dbb5cb57e4da009b859a6245b1c10d610f215a", "version" : "140.0.0" } + }, + { + "identity" : "websocket-apple", + "kind" : "remoteSourceControl", + "location" : "https://github.com/shareup/websocket-apple.git", + "state" : { + "revision" : "a176fc4f7b9f7ad4f0a1fa245a2bbb024658a30e", + "version" : "4.1.0" + } } ], "version" : 3 diff --git a/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme b/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme index 0f41522..b32be68 100644 --- a/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme +++ b/Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme @@ -31,9 +31,9 @@ shouldAutocreateTestPlan = "YES"> [PromptPreset] { + switch model { + case .mirage, .mirage_v2: + return miragePresets + case .lucy_v2v_720p_rt: + return lucyEditPresets + } + } + + private static let miragePresets: [PromptPreset] = [ + PromptPreset( + label: "Pirates", + prompt: "Transform the image into Pirates of the Caribbean swashbuckling fantasy style while maintaining the same composition. Use weathered nautical browns and blues, supernatural green ghost effects and tropical Caribbean colors with golden treasure accents. Add water-damaged wooden ship textures, weathered pirate clothing with character-specific details and supernatural decay with ghostly material properties. Apply dramatic lantern and moonlight with supernatural transformation effects, reimagining the elements with historical pirate meets cursed treasure qualities while keeping the same overall arrangement." + ), + PromptPreset( + label: "Bee Hive", + prompt: "Transform the image into a bee hive style with whimsical synthetic-2D animation while maintaining the original composition. Apply a color palette of warm yellows and rich browns reflecting organic honey and wax materials. Create soft, rounded shapes and playful designs with vibrant, saturated colors evoking a lively, bustling atmosphere. Use bright, cheerful lighting simulating sunlight filtering through the hive for a sense of warmth and activity while keeping all elements in their current positions." + ), + PromptPreset( + label: "Van Gogh", + prompt: "Transform the image into Van Gogh painting style while maintaining the same composition. Use vibrant, emotionally expressive yellows, blues and greens with complementary color contrasts and pure unmixed pigments. Create swirling, directional brushwork with thick impasto application that suggests physical dimension. Make lighting appear to radiate from within objects with halos and auras around light sources, reimagining the elements with passionate Post-Impressionist qualities while keeping the same overall arrangement." + ), + PromptPreset( + label: "Yellow Cartoon", + prompt: "Transform the image into The Simpsons animation style while maintaining the same composition. Use the iconic yellow skin tone with bright primary colors and suburban American color schemes in bold saturation. Apply flat cartoon rendering with bold outlines and simple shading techniques. Create bright television lighting with minimal shadows, keeping all elements in their original positions." + ), + PromptPreset( + label: "Cyborgs", + prompt: "Transform the image into Terminator tech-noir style while maintaining the same composition. Use cold blue lighting for future scenes, warm human tones contrasted with metallic silver machine elements and electrical energy effects with blue-white intensity. Add hyperdetailed robotic components with exposed mechanical workings, battle-damaged cyborg elements revealing metal under flesh and post-apocalyptic environmental features. Apply harsh mechanical lighting emphasizing robotic elements with glowing red accents, reimagining the elements with technological horror qualities while keeping the same overall arrangement." + ), + PromptPreset( + label: "Manga", + prompt: "Transform the image into black and white manga illustration style while maintaining the same composition. Use pure blacks, clean whites and varied gray tones achieved through screen tones and hatching techniques. Apply crisp ink lines with varying weights, detailed texture work through stippling and cross-hatching. Create high contrast lighting with dramatic shadows and bright highlights, keeping all elements in their original positions." + ), + PromptPreset( + label: "Animation", + prompt: "Transform the image into Pixar 3D animation style while maintaining the same composition. Use carefully crafted color theory with warm family-friendly tones and cinematic color grading. Apply realistic 3D textures, advanced material shaders and subtle subsurface scattering effects. Create cinematic 3D lighting with realistic light behavior and atmospheric effects, keeping all elements in their original positions." + ), + PromptPreset( + label: "Golden Hour", + prompt: "Transform the image into golden hour style while maintaining the original composition. Apply warm tones with soft lighting that creates long shadows. Enhance the image with warm golden and orange hues throughout. Create a serene, tranquil atmosphere typical of sunset lighting while keeping all elements in their current positions." + ), + PromptPreset( + label: "Ghibli Inspired", + prompt: "Transform the image into Studio Ghibli style with hand-painted watercolor-like visuals while maintaining the same composition. Apply fine lines with soft shading and warm pastel colors in greens, blues and oranges with low saturation. Add delicate watercolor textures, gentle brushstrokes and subtle paper grain. Use soft diffused natural lighting reminiscent of light filtering through shoji screens, keeping all elements in their original positions." + ), + PromptPreset( + label: "War Zone", + prompt: "Transform the image into Mad Max post-apocalyptic style while maintaining the same composition. Use harsh desert oranges and yellows, desaturated dusty neutrals and night scenes illuminated by fire with high contrast. Add rust-covered vehicular modifications with practical mechanical detail, makeshift clothing and armor from salvaged materials and barren wasteland environmental elements. Apply harsh desert lighting with silhouettes against vast wasteland horizons, reimagining the elements with brutal survival qualities while keeping the same overall arrangement." + ), + PromptPreset( + label: "Zombies", + prompt: "Transform the image into a synthetic-3D horror video game style featuring stylized zombies while maintaining the original composition. Apply exaggerated features with high detail and texture that highlight decayed, undead elements. Use a color palette of muted earth tones with pops of eerie greens and purples. Add dramatic lighting with high contrast and deep shadows for a suspenseful, mysterious atmosphere while keeping all elements in their current positions." + ), + PromptPreset( + label: "K-pop", + prompt: "Apply K-pop style with vibrant hyper-polished aesthetic, perfectly styled performers, coordinated fashion-forward outfits, candy-colored pastels alongside bold neons, immaculate styling with glossy skin, and flawless soft-focus lighting" + ), + PromptPreset( + label: "Mythic", + prompt: "Transform the image into God of War game style while maintaining the same composition. Use cold Nordic blues and whites, rich wooden browns and divine gold accents with blood-red highlights. Add weathered leather and metal with intricate Norse knotwork, godly artifacts with magical properties and massive environmental scale elements. Apply dramatic cinematic lighting emphasizing character moments and epic scale, reimagining the elements with a Norse mythological action aesthetic while keeping the same overall arrangement." + ), + PromptPreset( + label: "Wild West", + prompt: "Transform the image into Red Dead Redemption Western game style while maintaining the same composition. Use warm dusty earth tones with golden hour lighting, weathered browns and tans for frontier elements and natural greens for wilderness areas. Add weathered leather and denim with appropriate aging, wooden structures with detailed grain and wear patterns and natural environments with ecological detail. Apply dramatic lighting with stunning sunsets and atmospheric weather effects, reimagining the elements with an American frontier aesthetic while keeping the same overall arrangement." + ), + PromptPreset( + label: "Sci-fi Anime", + prompt: "Transform the image into Rick and Morty animation style while maintaining the same composition. Combine sci-fi colors with earth tones featuring portal greens, space blues and alien color schemes in vivid saturation. Add sci-fi technology textures, interdimensional effect work and adult animation detail levels. Apply dynamic sci-fi lighting with portal effects and alien illumination, keeping all elements in their original positions." + ), + PromptPreset( + label: "Classic Anime", + prompt: "Transform the image into Naruto anime style while maintaining the same composition. Use earth tones with ninja blues, forest greens and warm orange accents in moderate saturation. Combine traditional anime techniques with subtle texture work representing fabric and natural materials. Apply natural outdoor lighting mixed with mystical chakra effects, keeping all elements in their original positions." + ), + PromptPreset( + label: "Blocky", + prompt: "Transform the image into a Minecraft-inspired blocky 3D style with pixelated visual design while maintaining the original composition. Apply distinct cubic shapes throughout. Use a clean, consistent color palette dominated by stone grays, browns, and whites. Create smooth, pixel-consistent textures with low-resolution detail. Add bright, even lighting for an open, constructive atmosphere while keeping all elements in their current positions." + ), + PromptPreset( + label: "Football", + prompt: "Transform the image into American football style with powerful, armored athletic elements. Apply team-specific uniform colors with bold primary hues against green field background. Add football leather grain textures, helmet sheen details, grass-stained jersey effects, and player exertion elements. Use dramatic stadium lighting creating strong contrasts with harsh shadows. Maintain the subject's position while incorporating football action characteristics. Create an intense, gritty atmosphere filled with tension and explosive athleticism." + ), + PromptPreset( + label: "Picasso", + prompt: "Transform the image into Picasso Cubist style while maintaining the same composition. Use flat, bold hues of blues, reds and earth tones applied in geometric patches with strong black outlines. Create flattened surfaces with angular planes that break traditional perspective rules, revealing multiple viewpoints simultaneously. Apply non-naturalistic, symbolic lighting rather than representational, reimagining the elements with fragmented, multi-perspective Cubist qualities while keeping the same overall arrangement." + ), + PromptPreset( + label: "Super Hero", + prompt: "Transform the image into Marvel Cinematic Universe superhero style while maintaining the same composition. Use character-specific signature colors, location-specific color grading and vibrant energy effect colors with high saturation. Add practical superhero costumes with functional detailing, urban environments with appropriate destruction physics and magical/technological effects with distinctive visual signatures. Apply dynamic action lighting emphasizing heroic moments with dramatic highlights, reimagining the elements with colorful optimistic superhero qualities while keeping the same overall arrangement." + ), + PromptPreset( + label: "Neon Nostalgia", + prompt: "Transform the image into a cyberpunk anime style with neon colors and vibrant contrasts while maintaining the original composition. Apply a color palette of electric blues, pinks, and purples against a dark, rainy backdrop. Make surfaces appear reflective and wet with shimmering reflections. Create an intense, moody atmosphere with dynamic lighting to evoke an urban, futuristic aesthetic while keeping all elements in their current positions." + ), + ] + + private static let lucyEditPresets: [PromptPreset] = [ + PromptPreset( + label: "Anime Character", + prompt: "Transform the person into a 2D anime character with smooth cel-shaded lines, soft pastel highlights, large expressive eyes, clean contours, even lighting, simplified textures, and a bright studio-style background for a polished anime look." + ), + PromptPreset( + label: "Knight Armor", + prompt: "Change the uniform to a full medieval knight's armor with polished steel plates, engraved trim, articulated joints, matte underpadding, subtle battle scuffs, and cool directional lighting reflecting off the metal surfaces." + ), + PromptPreset( + label: "Spooky Skeleton", + prompt: "Replace the person with a Halloween-style skeleton featuring clean ivory bones, deep sockets, subtle surface cracks, articulated joints, and soft overhead lighting creating dramatic shadows across the ribcage." + ), + PromptPreset( + label: "Leather Jacket", + prompt: "Change the jacket to a black leather biker jacket with weathered grain texture, silver zippers, reinforced seams, slightly creased sleeves, and cool diffuse lighting suggesting an overcast outdoor feel." + ), + PromptPreset( + label: "Origami", + prompt: "Replace the person with a full-body origami figure built from crisp white folded paper, sharp geometric edges, layered segments, subtle crease shadows, and clean studio lighting enhancing the sculptural form." + ), + PromptPreset( + label: "Business Casual", + prompt: "Change the outfit to a light blue buttoned shirt paired with a tailored charcoal jacket, smooth cotton texture, clean stitching, structured shoulders, and balanced indoor lighting for a polished business-casual look." + ), + PromptPreset( + label: "Summer Dress", + prompt: "Change the outfit to a light floral summer dress with thin spaghetti straps, soft flowing fabric, pastel bloom patterns, gentle folds, and warm natural lighting suggesting an outdoor summer setting." + ), + PromptPreset( + label: "Lizard Person", + prompt: "Transform the person into a humanoid lizard figure with green scaled skin, subtle iridescence, angular cheek structure, elongated pupils, fine texture detail, and directional lighting emphasizing the reptilian contours." + ), + PromptPreset( + label: "Pink Shirt", + prompt: "Change the top color to bright pink with smooth fabric texture, preserved seams, soft shading along natural folds, and consistent lighting for an even saturated look." + ), + PromptPreset( + label: "Plastic Doll", + prompt: "Transform the person into a realistic fashion-doll version with smooth porcelain-like skin, glossy lips, defined lashes, polished facial symmetry, bright studio lighting, perfectly styled hair, and a fitted pink outfit with clean plastic-like highlights." + ), + PromptPreset( + label: "Sunglasses", + prompt: "Add a pair of dark tinted sunglasses resting naturally on the person's face, smooth acetate frames, subtle reflections on the lenses, accurate nose placement, and soft shadows across the cheeks." + ), + PromptPreset( + label: "Super Hero", + prompt: "Transform the person into a superhero wearing a fitted suit with bold color panels, textured fabric, sculpted contours, a flowing cape, subtle rim lighting, and dramatic cinematic shading." + ), + PromptPreset( + label: "Polar Bear", + prompt: "Replace the person with a small polar bear featuring dense white fur, rounded ears, soft muzzle, gentle expression, and cool ambient lighting highlighting the fluffy texture." + ), + PromptPreset( + label: "Alien", + prompt: "Transform the person into a realistic alien form with pale luminescent skin, smooth reflective surface tones, large glassy eyes, subtle facial ridges, elongated contours, and cinematic lighting emphasizing the otherworldly texture." + ), + PromptPreset( + label: "Parrot on Shoulder", + prompt: "Add a bright green parrot perched on the person's shoulder with layered feathers, a curved beak, slight head tilt, natural talon grip, and a soft contact shadow on the clothing." + ), + PromptPreset( + label: "Icy Hair", + prompt: "Change the hair color to icy platinum blonde with a cool metallic sheen, fine reflective highlights, smooth strands, and bright soft lighting emphasizing the frosted tone." + ), + PromptPreset( + label: "Tux", + prompt: "Change the shirt to a formal tuxedo ensemble featuring a crisp white dress shirt, black satin lapels, structured fit, smooth fabric textures, and balanced indoor lighting for an elegant look." + ), + PromptPreset( + label: "Kitty", + prompt: "Add a small cat sitting gently on the person's head, soft striped fur, relaxed posture, curved tail, clear whiskers, natural grip on the hair, and a soft contact shadow for realism." + ), + PromptPreset( + label: "Super Spider", + prompt: "Transform the person into a Spider-Man–style hero wearing a red and blue textured suit, raised web patterns, fitted contours, reflective eye lenses, and dramatic city-style lighting." + ), + PromptPreset( + label: "Car Racer", + prompt: "Transform the person into a professional car racer wearing a padded racing suit with bold sponsor patches, high-contrast stitching, protective collar, and bright track-side lighting." + ), + PromptPreset( + label: "Happy Birthday", + prompt: "Add a mix of colorful helium balloons floating around the person, glossy surfaces, thin strings, soft reflections, varied sizes, and warm ambient party lighting." + ), + ] } diff --git a/Example/Example/DecartSDK/DecartConfig.swift b/Example/Example/DecartSDK/DecartClientShared.swift similarity index 94% rename from Example/Example/DecartSDK/DecartConfig.swift rename to Example/Example/DecartSDK/DecartClientShared.swift index dc290ed..439dbc1 100644 --- a/Example/Example/DecartSDK/DecartConfig.swift +++ b/Example/Example/DecartSDK/DecartClientShared.swift @@ -18,7 +18,7 @@ protocol RealtimeManagerProtocol { var localMediaStream: RealtimeMediaStream? { get } var remoteMediaStreams: RealtimeMediaStream? { get } - func connect(model: RealtimeModel) async + func connect() async func switchCamera() async func cleanup() async } diff --git a/Example/Example/DecartSDK/RealtimeManager.swift b/Example/Example/DecartSDK/RealtimeManager.swift index 00673f5..43d3030 100644 --- a/Example/Example/DecartSDK/RealtimeManager.swift +++ b/Example/Example/DecartSDK/RealtimeManager.swift @@ -33,6 +33,9 @@ final class RealtimeManager: RealtimeManagerProtocol { @ObservationIgnored private let decartClient = Container.shared.decartClient() + @ObservationIgnored + private let model: RealtimeModel + @ObservationIgnored private var realtimeManager: DecartRealtimeManager? @@ -46,14 +49,15 @@ final class RealtimeManager: RealtimeManagerProtocol { // MARK: - Init - init(currentPrompt: Prompt, isMirroringEnabled: Bool = true) { + init(model: RealtimeModel, currentPrompt: Prompt, isMirroringEnabled: Bool = true) { + self.model = model self.currentPrompt = currentPrompt self.shouldMirror = isMirroringEnabled } // MARK: - Public API - func connect(model: RealtimeModel) async { + func connect() async { if connectionState.isInSession || realtimeManager != nil { await cleanup() } diff --git a/Example/Example/Views/RealtimeControlsView.swift b/Example/Example/Views/RealtimeControlsView.swift new file mode 100644 index 0000000..8d9353f --- /dev/null +++ b/Example/Example/Views/RealtimeControlsView.swift @@ -0,0 +1,250 @@ +import DecartSDK +import SwiftUI + +struct RealtimeControlsView: View { + let presets: [PromptPreset] + let connectionState: DecartRealtimeConnectionState + let onPresetSelected: (PromptPreset) -> Void + let onSwitchCamera: () -> Void + let onConnectToggle: () -> Void + + @State private var selectedPresetId: UUID? + + var body: some View { + VStack(spacing: 16) { + if connectionState == .error { + ErrorBanner() + } + + PresetChipsScrollView( + presets: presets, + selectedPresetId: $selectedPresetId, + onPresetSelected: { preset in + selectedPresetId = preset.id + onPresetSelected(preset) + } + ) + + ControlButtonsRow( + connectionState: connectionState, + onSwitchCamera: onSwitchCamera, + onConnectToggle: onConnectToggle + ) + } + .padding(16) + .background( + RoundedRectangle(cornerRadius: 20) + .fill(.ultraThinMaterial) + .overlay( + RoundedRectangle(cornerRadius: 20) + .stroke( + LinearGradient( + colors: [ + Color.white.opacity(0.3), + Color.white.opacity(0.1), + Color.clear, + ], + startPoint: .topLeading, + endPoint: .bottomTrailing + ), + lineWidth: 1 + ) + ) + ) + .padding(.horizontal, 8) + .padding(.bottom, 8) + .onAppear { + if selectedPresetId == nil, let firstPreset = presets.first { + selectedPresetId = firstPreset.id + } + } + } +} + +private struct ErrorBanner: View { + var body: some View { + Text("Connection error. Please try again.") + .font(.caption) + .fontWeight(.medium) + .foregroundStyle(.white) + .padding(.horizontal, 16) + .padding(.vertical, 10) + .background( + Capsule() + .fill(Color.red.opacity(0.8)) + .overlay( + Capsule() + .stroke(Color.red.opacity(0.5), lineWidth: 1) + ) + ) + } +} + +private struct PresetChipsScrollView: View { + let presets: [PromptPreset] + @Binding var selectedPresetId: UUID? + let onPresetSelected: (PromptPreset) -> Void + + var body: some View { + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 10) { + ForEach(presets) { preset in + PresetChip( + preset: preset, + isSelected: selectedPresetId == preset.id, + onTap: { onPresetSelected(preset) } + ) + } + } + .padding(.horizontal, 4) + } + .frame(height: 44) + } +} + +private struct PresetChip: View { + let preset: PromptPreset + let isSelected: Bool + let onTap: () -> Void + + var body: some View { + Button(action: onTap) { + Text(preset.label) + .font(.subheadline) + .fontWeight(isSelected ? .semibold : .medium) + .foregroundStyle(isSelected ? .white : .white.opacity(0.8)) + .padding(.horizontal, 16) + .padding(.vertical, 10) + .background( + Capsule() + .fill( + isSelected + ? LinearGradient( + colors: [ + Color(red: 0.4, green: 0.3, blue: 1.0), + Color(red: 0.6, green: 0.2, blue: 0.9), + ], + startPoint: .topLeading, + endPoint: .bottomTrailing + ) + : LinearGradient( + colors: [ + Color.white.opacity(0.15), + Color.white.opacity(0.08), + ], + startPoint: .topLeading, + endPoint: .bottomTrailing + ) + ) + .overlay( + Capsule() + .stroke( + isSelected + ? Color.white.opacity(0.4) + : Color.white.opacity(0.2), + lineWidth: 1 + ) + ) + ) + .shadow( + color: isSelected ? Color(red: 0.5, green: 0.3, blue: 1.0).opacity(0.5) : .clear, + radius: 8, + y: 2 + ) + } + .buttonStyle(.plain) + .animation(.easeInOut(duration: 0.2), value: isSelected) + } +} + +private struct ControlButtonsRow: View { + let connectionState: DecartRealtimeConnectionState + let onSwitchCamera: () -> Void + let onConnectToggle: () -> Void + + var body: some View { + HStack(spacing: 12) { + CameraSwitchButton(onTap: onSwitchCamera) + + Spacer() + + ConnectButton(connectionState: connectionState, onTap: onConnectToggle) + } + } +} + +private struct CameraSwitchButton: View { + let onTap: () -> Void + + var body: some View { + Button(action: onTap) { + Image(systemName: "arrow.trianglehead.2.counterclockwise.rotate.90") + .font(.system(size: 16, weight: .medium)) + .foregroundStyle(.white.opacity(0.8)) + .frame(width: 40, height: 40) + .background( + Circle() + .fill(Color.white.opacity(0.1)) + .overlay( + Circle() + .stroke(Color.white.opacity(0.2), lineWidth: 1) + ) + ) + } + .buttonStyle(.plain) + } +} + +private struct ConnectButton: View { + let connectionState: DecartRealtimeConnectionState + let onTap: () -> Void + + private var buttonGradient: LinearGradient { + if connectionState.isInSession { + return LinearGradient( + colors: [ + Color(red: 0.9, green: 0.2, blue: 0.3), + Color(red: 0.8, green: 0.1, blue: 0.2), + ], + startPoint: .topLeading, + endPoint: .bottomTrailing + ) + } else { + return LinearGradient( + colors: [ + Color(red: 0.2, green: 0.8, blue: 0.4), + Color(red: 0.1, green: 0.7, blue: 0.3), + ], + startPoint: .topLeading, + endPoint: .bottomTrailing + ) + } + } + + private var shadowColor: Color { + connectionState.isInSession + ? Color.red.opacity(0.4) + : Color.green.opacity(0.4) + } + + var body: some View { + Button(action: onTap) { + Text(connectionState.rawValue) + .font(.subheadline) + .fontWeight(.semibold) + .foregroundStyle(.white) + .padding(.horizontal, 24) + .padding(.vertical, 12) + .background( + Capsule() + .fill(buttonGradient) + .overlay( + Capsule() + .stroke(Color.white.opacity(0.3), lineWidth: 1) + ) + ) + .shadow(color: shadowColor, radius: 8, y: 2) + } + .buttonStyle(.plain) + } +} diff --git a/Example/Example/Views/RealtimeView.swift b/Example/Example/Views/RealtimeView.swift index b3202f1..58d00f6 100644 --- a/Example/Example/Views/RealtimeView.swift +++ b/Example/Example/Views/RealtimeView.swift @@ -5,11 +5,12 @@ import WebRTC struct RealtimeView: View { private let realtimeAiModel: RealtimeModel - @State private var prompt: String = DecartConfig.defaultPrompt + private let presets: [PromptPreset] @State private var realtimeManager: RealtimeManager? init(realtimeModel: RealtimeModel) { self.realtimeAiModel = realtimeModel + self.presets = DecartConfig.presets(for: realtimeModel) } var body: some View { @@ -17,8 +18,7 @@ struct RealtimeView: View { if let manager = realtimeManager { RealtimeContentView( realtimeManager: manager, - realtimeAiModel: realtimeAiModel, - prompt: $prompt + presets: presets ) } else { ProgressView("Loading...") @@ -26,9 +26,14 @@ struct RealtimeView: View { } .onAppear { if realtimeManager == nil { + let defaultPrompt = presets.first?.prompt ?? "" realtimeManager = RealtimeManager( - currentPrompt: Prompt(text: DecartConfig.defaultPrompt, enrich: false) + model: realtimeAiModel, + currentPrompt: Prompt(text: defaultPrompt, enrich: false) ) + Task { + await realtimeManager?.connect() + } } } .onDisappear { @@ -42,8 +47,7 @@ struct RealtimeView: View { private struct RealtimeContentView: View { @Bindable var realtimeManager: RealtimeManager - let realtimeAiModel: RealtimeModel - @Binding var prompt: String + let presets: [PromptPreset] var body: some View { ZStack { @@ -81,70 +85,21 @@ private struct RealtimeContentView: View { ) } - VStack(spacing: 12) { - if realtimeManager.connectionState == .error { - Text("Error while connecting to decart realtime servers, please try again later.") - .foregroundColor(.red) - .font(.caption) - .padding(8) - .background(Color.black.opacity(0.8)) - .cornerRadius(8) - } - - HStack(spacing: 12) { - TextField("Prompt", text: $prompt) - .textFieldStyle(RoundedBorderTextFieldStyle()) - - Button(action: { - realtimeManager.currentPrompt = Prompt(text: prompt, enrich: false) - }) { - Image(systemName: "paperplane.fill") - .foregroundColor(.white) - .padding(12) - .background( - realtimeManager.connectionState.isConnected - ? Color.blue : Color.gray - ) - .cornerRadius(8) - } - } - - HStack(spacing: 12) { - Toggle("Mirror", isOn: $realtimeManager.shouldMirror) - .toggleStyle(SwitchToggleStyle(tint: .blue)) - - Button(action: { - Task { await realtimeManager.switchCamera() } - }) { - Image(systemName: "arrow.trianglehead.2.counterclockwise.rotate.90") - } - - Spacer() - - Button(action: { - if realtimeManager.connectionState.isInSession { - Task { await realtimeManager.cleanup() } - } else { - Task { await realtimeManager.connect(model: realtimeAiModel) } - } - }) { - Text(realtimeManager.connectionState.rawValue) - .fontWeight(.semibold) - .foregroundColor(.white) - .frame(maxWidth: .infinity) - .padding() - .background( - realtimeManager.connectionState.isConnected - ? Color.red : Color.green - ) - .cornerRadius(12) + RealtimeControlsView( + presets: presets, + connectionState: realtimeManager.connectionState, + onPresetSelected: { preset in + realtimeManager.currentPrompt = Prompt(text: preset.prompt, enrich: false) + }, + onSwitchCamera: { Task { await realtimeManager.switchCamera() } }, + onConnectToggle: { + if realtimeManager.connectionState.isInSession { + Task { await realtimeManager.cleanup() } + } else { + Task { await realtimeManager.connect() } } } - } - .padding() - .background(Color.black.opacity(0.8)) - .cornerRadius(16) - .padding(.all, 5) + ) } } } diff --git a/Package.resolved b/Package.resolved index e5b2e01..f1d0374 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,33 @@ { + "originHash" : "785f4c91530100bd3dbb4a3bc1b4980520c27ac047f0c5fbfa119f340d376698", "pins" : [ + { + "identity" : "async-extensions", + "kind" : "remoteSourceControl", + "location" : "https://github.com/shareup/async-extensions.git", + "state" : { + "revision" : "3088474141debc75b78257a0db28adf734bcea0f", + "version" : "4.4.0" + } + }, + { + "identity" : "dispatch-timer", + "kind" : "remoteSourceControl", + "location" : "https://github.com/shareup/dispatch-timer.git", + "state" : { + "revision" : "2d8c304aa6f382a7a362cd5a814884f3930c5662", + "version" : "3.0.1" + } + }, + { + "identity" : "synchronized", + "kind" : "remoteSourceControl", + "location" : "https://github.com/shareup/synchronized.git", + "state" : { + "revision" : "85653e23270ec88ae19f8d494157769487e34aed", + "version" : "4.0.1" + } + }, { "identity" : "webrtc", "kind" : "remoteSourceControl", @@ -8,7 +36,16 @@ "revision" : "86dbb5cb57e4da009b859a6245b1c10d610f215a", "version" : "140.0.0" } + }, + { + "identity" : "websocket-apple", + "kind" : "remoteSourceControl", + "location" : "https://github.com/shareup/websocket-apple.git", + "state" : { + "revision" : "a176fc4f7b9f7ad4f0a1fa245a2bbb024658a30e", + "version" : "4.1.0" + } } ], - "version" : 2 + "version" : 3 } diff --git a/Package.swift b/Package.swift index 83c615b..553ea35 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.0.3 +// swift-tools-version: 6.2.0 import PackageDescription let package = Package( @@ -14,13 +14,18 @@ let package = Package( ) ], dependencies: [ - .package(url: "https://github.com/stasel/WebRTC.git", from: "140.0.0") + .package(url: "https://github.com/stasel/WebRTC.git", from: "140.0.0"), + .package( + url: "https://github.com/shareup/websocket-apple.git", + from: "4.1.0" + ) ], targets: [ .target( name: "DecartSDK", dependencies: [ - .product(name: "WebRTC", package: "WebRTC") + .product(name: "WebRTC", package: "WebRTC"), + .product(name: "WebSocket", package: "websocket-apple") ], path: "Sources/DecartSDK" ) diff --git a/README.md b/README.md index cffd857..2c87831 100644 --- a/README.md +++ b/README.md @@ -56,24 +56,16 @@ import DecartSDK let config = DecartConfiguration(apiKey: "your-api-key") let client = DecartClient(decartConfiguration: config) -let model = Models.realtime(.mirage) +let model: RealtimeModel = .mirage +let modelConfig = Models.realtime(model) + let realtimeManager = try client.createRealtimeManager( options: RealtimeConfiguration( - model: model, + model: modelConfig, initialState: ModelState(prompt: Prompt(text: "Lego World")) ) ) -// Create video source and camera capture -let videoSource = realtimeManager.createVideoSource() -let capture = RealtimeCapture(model: model, videoSource: videoSource) -try await capture.startCapture() - -// Create local stream and connect -let videoTrack = realtimeManager.createVideoTrack(source: videoSource, trackId: "video0") -let localStream = RealtimeMediaStream(videoTrack: videoTrack, id: .localStream) -let remoteStream = try await realtimeManager.connect(localStream: localStream) - // Listen to connection events Task { for await state in realtimeManager.events { @@ -90,6 +82,16 @@ Task { } } +// Create video source and camera capture +let videoSource = realtimeManager.createVideoSource() +let capture = RealtimeCapture(model: modelConfig, videoSource: videoSource) +try await capture.startCapture() + +// Create local stream and connect +let videoTrack = realtimeManager.createVideoTrack(source: videoSource, trackId: "video0") +let localStream = RealtimeMediaStream(videoTrack: videoTrack, id: .localStream) +let remoteStream = try await realtimeManager.connect(localStream: localStream) + // Update prompt in real-time realtimeManager.setPrompt(Prompt(text: "Anime World")) diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index e3dd4e4..1d7e249 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -52,7 +52,7 @@ public extension DecartRealtimeManager { setupWebSocketListener(wsClient) let rtcClient = WebRTCClient( - config: options.connection.makeRTCConfiguration(), + config: options.connection.rtcConfiguration, constraints: options.media.connectionConstraints, videoConfig: options.media.video, sendMessage: { [weak self] in self?.sendMessage($0) }, @@ -178,6 +178,6 @@ private extension DecartRealtimeManager { private extension DecartRealtimeManager { func sendMessage(_ message: OutgoingWebSocketMessage) { guard let webSocketClient else { return } - Task { try? await webSocketClient.send(message) } + Task { [webSocketClient] in try? await webSocketClient.send(message) } } } diff --git a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift index c6fcb8c..400ac8b 100644 --- a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift +++ b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift @@ -31,22 +31,25 @@ public struct RealtimeConfiguration: Sendable { public struct ConnectionConfig: Sendable { public let iceServers: [String] public let connectionTimeout: TimeInterval + public let rtcConfiguration: RTCConfiguration public init( iceServers: [String] = ["stun:stun.l.google.com:19302"], - connectionTimeout: TimeInterval = 15 + connectionTimeout: TimeInterval = 15, + rtcConfiguration: RTCConfiguration? = nil ) { self.iceServers = iceServers self.connectionTimeout = connectionTimeout - } - - public func makeRTCConfiguration() -> RTCConfiguration { - let config = RTCConfiguration() - config.iceServers = [RTCIceServer(urlStrings: iceServers)] - config.sdpSemantics = .unifiedPlan - config.continualGatheringPolicy = .gatherContinually - config.iceCandidatePoolSize = 10 - return config + if let rtcConfiguration { + self.rtcConfiguration = rtcConfiguration + } else { + let config = RTCConfiguration() + config.iceServers = [RTCIceServer(urlStrings: iceServers)] + config.sdpSemantics = .unifiedPlan + config.continualGatheringPolicy = .gatherContinually + config.iceCandidatePoolSize = 10 + self.rtcConfiguration = config + } } } @@ -123,7 +126,15 @@ public struct RealtimeConfiguration: Sendable { } let sortedCodecs = preferredCodecs + otherCodecs + utilityCodecs - transceiver.setCodecPreferences(sortedCodecs) + do { + try transceiver.setCodecPreferences(sortedCodecs, error: ()) + } catch { + DecartLogger + .log( + "error while setting codec preferences: \(error)", + level: .error + ) + } } } } diff --git a/Sources/DecartSDK/Realtime/Websocket/SocketStream.swift b/Sources/DecartSDK/Realtime/Websocket/SocketStream.swift index 40cddb4..7af7e65 100644 --- a/Sources/DecartSDK/Realtime/Websocket/SocketStream.swift +++ b/Sources/DecartSDK/Realtime/Websocket/SocketStream.swift @@ -49,18 +49,24 @@ final class SocketStream: AsyncSequence, @unchecked Sendable { continuation?.finish() return } - task.receive(completionHandler: { [weak self] result in - guard let continuation = self?.continuation else { - return - } - do { - let message = try result.get() - continuation.yield(message) - self?.waitForNextValue() - } catch { - continuation.finish(throwing: error) - } - }) + task.receive( + completionHandler: { [weak self] result in + guard let continuation = self?.continuation else { + return + } + do { + let message = try result.get() + continuation.yield(message) + self?.waitForNextValue() + } catch { + DecartLogger + .log( + "Error in decart realtime websocket: \(error)", + level: .error + ) + continuation.finish(throwing: error) + } + }) } deinit { diff --git a/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift b/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift index e3dc87b..d7f4a33 100644 --- a/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift +++ b/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift @@ -1,10 +1,11 @@ import Foundation +import WebSocket -actor WebSocketClient { - var isConnected: Bool = false +final class WebSocketClient: Sendable { +// var isConnected: Bool = false - private let stream: SocketStream - private var listeningTask: Task? + private let socket: WebSocket? + private nonisolated(unsafe) var listeningTask: Task? private let decoder = JSONDecoder() private let encoder = JSONEncoder() @@ -15,8 +16,9 @@ actor WebSocketClient { let (websocketEventStream, eventStreamContinuation) = AsyncStream.makeStream(of: IncomingWebSocketMessage.self) self.eventStreamContinuation = eventStreamContinuation self.websocketEventStream = websocketEventStream - let socketConnection = URLSession.shared.webSocketTask(with: url) - stream = SocketStream(task: socketConnection) + let newSocket = try? await WebSocket.system(url: url) + socket = newSocket + try? await newSocket?.open() mountListener() } @@ -24,14 +26,15 @@ actor WebSocketClient { listeningTask = Task { [weak self] in guard let self else { return } do { - for try await msg in self.stream { + guard let socket = self.socket else { return } + for try await msg in socket.messages { if Task.isCancelled { return } switch msg { - case .string(let text): - await self.handleIncomingMessage(text) + case .text(let text): + self.handleIncomingMessage(text) case .data(let d): if let text = String(data: d, encoding: .utf8) { - await self.handleIncomingMessage(text) + self.handleIncomingMessage(text) } @unknown default: break } @@ -42,23 +45,38 @@ actor WebSocketClient { } } - private func handleIncomingMessage(_ text: String) async { + private func handleIncomingMessage(_ text: String) { guard let data = text.data(using: .utf8) else { return } - guard let message = try? decoder.decode(IncomingWebSocketMessage.self, from: data) else { return } - eventStreamContinuation.yield(message) + + do { + let message = try decoder.decode(IncomingWebSocketMessage.self, from: data) + eventStreamContinuation.yield(message) + } catch { + DecartLogger.log( + "unable to decode websocket message: \(error)", + level: .warning + ) + } } - func send(_ message: T) throws { + func send(_ message: T) async throws { let data = try encoder.encode(message) - guard let jsonString = String(data: data, encoding: .utf8) else { - throw DecartError.websocketError("unable to encode message") + guard let jsonString = String(data: data, encoding: .utf8) else { return } + guard let socket else { return } + try await socket.send(.text(jsonString)) + } + + func disconnect() { + Task { [socket] in + if let socket { + try? await socket.close() + } } - Task { [stream] in try await stream.sendMessage(.string(jsonString)) } } deinit { listeningTask?.cancel() eventStreamContinuation.finish() - stream.cancel() + disconnect() } } From 2beebe91f7fb239072f00eab3c20056470917e27 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 3 Dec 2025 20:26:29 +0200 Subject: [PATCH 11/23] fix swift version --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 553ea35..45877c3 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.2.0 +// swift-tools-version: 6.0.3 import PackageDescription let package = Package( From aa1ffadfa98ecddc6903afc87e7210e7ffc8ab72 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Tue, 6 Jan 2026 18:59:26 +0200 Subject: [PATCH 12/23] try to fix ci --- .github/workflows/ci.yml | 44 ++++++++++--------- Package.swift | 2 +- .../Realtime/DecartRealtimeManager.swift | 2 + 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c503de3..c7d45a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,29 +8,31 @@ on: jobs: build-and-test: - name: Build and Test - runs-on: macos-15 + name: Build and Test (Swift 6.2.1) + runs-on: macos-latest + env: + IOS_DEVICE_NAME: "iPhone 16 Pro" + IOS_OS_VERSION: "18.5" steps: - name: Checkout uses: actions/checkout@v4 - - name: Select Xcode - run: sudo xcode-select -s /Applications/Xcode_16.2.app - - - name: Swift Version - run: swift --version - - - name: Build SDK - run: swift build -c release - - # - name: Run Tests - # run: swift test - - # - name: Build Example App - # run: | - # cd Examples/RealtimeExample - # xcodebuild -scheme RealtimeExample \ - # -destination 'platform=iOS Simulator,name=iPhone 15' \ - # clean build \ - # CODE_SIGNING_ALLOWED=NO + - name: Build iOS App + uses: brightdigit/swift-build@v1.4.2 + with: + build-only: true + # 1. Provide the scheme name + scheme: DecartSDK + + # 2. Specify 'ios' to trigger xcodebuild (instead of 'swift build') + type: ios + deviceName: ${{ env.IOS_DEVICE_NAME }} + osVersion: ${{ env.IOS_OS_VERSION }} + download-platform: true + + # 3. FIX: Path issue. This moves the context into the subfolder + xcode: /Applications/Xcode_26.1.app + # 6. Prettify the logs (optional but recommended) + use-xcbeautify: true + xcbeautify-renderer: github-actions diff --git a/Package.swift b/Package.swift index 45877c3..087b855 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.0.3 +// swift-tools-version: 6.2.1 import PackageDescription let package = Package( diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index 1d7e249..2162bff 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -88,6 +88,7 @@ public extension DecartRealtimeManager { webRTCClient?.close() webRTCClient = nil + #if canImport(WebRTC) && (os(iOS)) let audioSession = RTCAudioSession.sharedInstance() if audioSession.isActive { audioSession.lockForConfiguration() @@ -95,6 +96,7 @@ public extension DecartRealtimeManager { audioSession.unlockForConfiguration() } webSocketClient = nil + #endif } func setPrompt(_ prompt: Prompt) { From a63dc7242aef40b4ec5c04174a2b25a157ee8586 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Wed, 7 Jan 2026 14:04:03 +0200 Subject: [PATCH 13/23] fixed websocket to use the apple-ws sdk, and the release yaml --- .github/workflows/release.yml | 20 ++++- Example.xcodeproj/project.pbxproj | 18 ++-- .../Realtime/DecartRealtimeManager.swift | 3 +- .../Realtime/Websocket/SocketStream.swift | 89 ------------------- .../Realtime/Websocket/WebSocketClient.swift | 77 ++++++++-------- 5 files changed, 66 insertions(+), 141 deletions(-) delete mode 100644 Sources/DecartSDK/Realtime/Websocket/SocketStream.swift diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c8ee2d1..e853e9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,10 @@ on: jobs: release: name: Create Release - runs-on: macos-15 + runs-on: macos-latest + env: + IOS_DEVICE_NAME: "iPhone 16 Pro" + IOS_OS_VERSION: "18.5" permissions: contents: write @@ -18,10 +21,21 @@ jobs: uses: actions/checkout@v4 - name: Select Xcode - run: sudo xcode-select -s /Applications/Xcode_16.2.app + run: sudo xcode-select -s /Applications/Xcode_26.1.app - name: Build Release - run: swift build -c release + uses: brightdigit/swift-build@v1.4.2 + with: + build-only: true + scheme: DecartSDK + type: ios + configuration: release + deviceName: ${{ env.IOS_DEVICE_NAME }} + osVersion: ${{ env.IOS_OS_VERSION }} + download-platform: true + xcode: /Applications/Xcode_26.1.app + use-xcbeautify: true + xcbeautify-renderer: github-actions - name: Extract Version id: version diff --git a/Example.xcodeproj/project.pbxproj b/Example.xcodeproj/project.pbxproj index a9e3407..ee4960c 100644 --- a/Example.xcodeproj/project.pbxproj +++ b/Example.xcodeproj/project.pbxproj @@ -7,9 +7,9 @@ objects = { /* Begin PBXBuildFile section */ + 161F07602F0E80DC001B6765 /* DecartSDK in Frameworks */ = {isa = PBXBuildFile; productRef = 161F075F2F0E80DC001B6765 /* DecartSDK */; }; 165B2B9B2ECE09E9004D848C /* Factory in Frameworks */ = {isa = PBXBuildFile; productRef = 165B2B9A2ECE09E9004D848C /* Factory */; }; 165B2B9D2ECE09E9004D848C /* FactoryKit in Frameworks */ = {isa = PBXBuildFile; productRef = 165B2B9C2ECE09E9004D848C /* FactoryKit */; }; - 16B8AC262EDE0490005EEA8A /* DecartSDK in Frameworks */ = {isa = PBXBuildFile; productRef = 16B8AC252EDE0490005EEA8A /* DecartSDK */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -31,7 +31,7 @@ files = ( 165B2B9D2ECE09E9004D848C /* FactoryKit in Frameworks */, 165B2B9B2ECE09E9004D848C /* Factory in Frameworks */, - 16B8AC262EDE0490005EEA8A /* DecartSDK in Frameworks */, + 161F07602F0E80DC001B6765 /* DecartSDK in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -84,7 +84,7 @@ packageProductDependencies = ( 165B2B9A2ECE09E9004D848C /* Factory */, 165B2B9C2ECE09E9004D848C /* FactoryKit */, - 16B8AC252EDE0490005EEA8A /* DecartSDK */, + 161F075F2F0E80DC001B6765 /* DecartSDK */, ); productName = Example; productReference = 16D70B962EB8CA0C00455077 /* Example.app */; @@ -116,7 +116,7 @@ minimizedProjectReferenceProxies = 1; packageReferences = ( 165B2B992ECE09E9004D848C /* XCRemoteSwiftPackageReference "Factory" */, - 16B8AC242EDE0490005EEA8A /* XCLocalSwiftPackageReference "../decart-ios" */, + 161F075C2F0E80B8001B6765 /* XCLocalSwiftPackageReference "../decart-ios" */, ); preferredProjectObjectVersion = 77; productRefGroup = 16D70B972EB8CA0C00455077 /* Products */; @@ -374,7 +374,7 @@ /* End XCConfigurationList section */ /* Begin XCLocalSwiftPackageReference section */ - 16B8AC242EDE0490005EEA8A /* XCLocalSwiftPackageReference "../decart-ios" */ = { + 161F075C2F0E80B8001B6765 /* XCLocalSwiftPackageReference "../decart-ios" */ = { isa = XCLocalSwiftPackageReference; relativePath = "../decart-ios"; }; @@ -392,6 +392,10 @@ /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ + 161F075F2F0E80DC001B6765 /* DecartSDK */ = { + isa = XCSwiftPackageProductDependency; + productName = DecartSDK; + }; 165B2B9A2ECE09E9004D848C /* Factory */ = { isa = XCSwiftPackageProductDependency; package = 165B2B992ECE09E9004D848C /* XCRemoteSwiftPackageReference "Factory" */; @@ -402,10 +406,6 @@ package = 165B2B992ECE09E9004D848C /* XCRemoteSwiftPackageReference "Factory" */; productName = FactoryKit; }; - 16B8AC252EDE0490005EEA8A /* DecartSDK */ = { - isa = XCSwiftPackageProductDependency; - productName = DecartSDK; - }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 16D70B8E2EB8CA0C00455077 /* Project object */; diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index 2162bff..daea7ee 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -87,6 +87,7 @@ public extension DecartRealtimeManager { connectionStateListenerTask = nil webRTCClient?.close() webRTCClient = nil + await webSocketClient?.disconnect() #if canImport(WebRTC) && (os(iOS)) let audioSession = RTCAudioSession.sharedInstance() @@ -95,8 +96,8 @@ public extension DecartRealtimeManager { try? audioSession.setActive(false) audioSession.unlockForConfiguration() } - webSocketClient = nil #endif + webSocketClient = nil } func setPrompt(_ prompt: Prompt) { diff --git a/Sources/DecartSDK/Realtime/Websocket/SocketStream.swift b/Sources/DecartSDK/Realtime/Websocket/SocketStream.swift deleted file mode 100644 index 7af7e65..0000000 --- a/Sources/DecartSDK/Realtime/Websocket/SocketStream.swift +++ /dev/null @@ -1,89 +0,0 @@ -// -// SocketStream.swift -// DecartSDK -// -// Created by Alon Bar-el on 03/11/2025. -// -import Foundation - -typealias WebSocketStream = AsyncThrowingStream - -extension URLSessionWebSocketTask { - var stream: WebSocketStream { - return WebSocketStream { continuation in - Task { - var isAlive = true - while isAlive && closeCode == .invalid { - do { - let value = try await receive() - continuation.yield(value) - } catch { - continuation.finish(throwing: error) - isAlive = false - } - } - } - } - } -} - -final class SocketStream: AsyncSequence, @unchecked Sendable { - typealias AsyncIterator = WebSocketStream.Iterator - typealias Element = URLSessionWebSocketTask.Message - - private var continuation: WebSocketStream.Continuation? - private let task: URLSessionWebSocketTask - - private lazy var stream: WebSocketStream = WebSocketStream { continuation in - self.continuation = continuation - waitForNextValue() - } - - init(task: URLSessionWebSocketTask) { - self.task = task - task.resume() - } - - private func waitForNextValue() { - guard task.closeCode == .invalid else { - continuation?.finish() - return - } - task.receive( - completionHandler: { [weak self] result in - guard let continuation = self?.continuation else { - return - } - do { - let message = try result.get() - continuation.yield(message) - self?.waitForNextValue() - } catch { - DecartLogger - .log( - "Error in decart realtime websocket: \(error)", - level: .error - ) - continuation.finish(throwing: error) - } - }) - } - - deinit { - cancel() - continuation?.finish() - } - - func sendMessage(_ message: URLSessionWebSocketTask.Message) async throws { - try await task.send(message) - } - - func makeAsyncIterator() -> AsyncIterator { - return stream.makeAsyncIterator() - } - - func cancel() { - task.cancel(with: .goingAway, reason: nil) - continuation?.finish() - } -} diff --git a/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift b/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift index d7f4a33..59859a4 100644 --- a/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift +++ b/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift @@ -1,11 +1,9 @@ import Foundation import WebSocket -final class WebSocketClient: Sendable { -// var isConnected: Bool = false - - private let socket: WebSocket? - private nonisolated(unsafe) var listeningTask: Task? +actor WebSocketClient { + private var socket: WebSocket? + private var listeningTask: Task? private let decoder = JSONDecoder() private let encoder = JSONEncoder() @@ -13,40 +11,43 @@ final class WebSocketClient: Sendable { nonisolated let websocketEventStream: AsyncStream init(url: URL) async { - let (websocketEventStream, eventStreamContinuation) = AsyncStream.makeStream(of: IncomingWebSocketMessage.self) + let (websocketEventStream, eventStreamContinuation) = + AsyncStream.makeStream(of: IncomingWebSocketMessage.self) self.eventStreamContinuation = eventStreamContinuation self.websocketEventStream = websocketEventStream - let newSocket = try? await WebSocket.system(url: url) - socket = newSocket - try? await newSocket?.open() - mountListener() + + do { + let newSocket = try await WebSocket.system(url: url) + socket = newSocket + try await newSocket.open() + mountListener(socket: newSocket) + } catch { + socket = nil + eventStreamContinuation.finish() + DecartLogger.log( + "unable to open websocket: \(error)", + level: .error + ) + } } - private func mountListener() { + private func mountListener(socket: WebSocket) { + listeningTask?.cancel() listeningTask = Task { [weak self] in guard let self else { return } - do { - guard let socket = self.socket else { return } - for try await msg in socket.messages { - if Task.isCancelled { return } - switch msg { - case .text(let text): - self.handleIncomingMessage(text) - case .data(let d): - if let text = String(data: d, encoding: .utf8) { - self.handleIncomingMessage(text) - } - @unknown default: break - } - } - } catch { - self.eventStreamContinuation.finish() + for await msg in socket.messages { + if Task.isCancelled { return } + await self.handleIncomingMessage(msg) } + await self.finishStream() } } - private func handleIncomingMessage(_ text: String) { - guard let data = text.data(using: .utf8) else { return } + private func handleIncomingMessage(_ message: WebSocketMessage) { + guard + let text = message.stringValue, + let data = text.data(using: .utf8) + else { return } do { let message = try decoder.decode(IncomingWebSocketMessage.self, from: data) @@ -59,6 +60,10 @@ final class WebSocketClient: Sendable { } } + private func finishStream() { + eventStreamContinuation.finish() + } + func send(_ message: T) async throws { let data = try encoder.encode(message) guard let jsonString = String(data: data, encoding: .utf8) else { return } @@ -66,17 +71,11 @@ final class WebSocketClient: Sendable { try await socket.send(.text(jsonString)) } - func disconnect() { - Task { [socket] in - if let socket { - try? await socket.close() - } - } - } - - deinit { + func disconnect() async { listeningTask?.cancel() + listeningTask = nil eventStreamContinuation.finish() - disconnect() + guard let socket else { return } + try? await socket.close() } } From 1262246024a08fbf0d81350d72e61992c9f5cdb7 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Mon, 26 Jan 2026 22:03:25 +0200 Subject: [PATCH 14/23] added lucy 14 RT --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 1 - README.md | 8 ++ Sources/DecartSDK/Models/Models.swift | 9 ++ .../Realtime/DecartRealtimeManager.swift | 95 ++++++++++++++++++- .../Realtime/WebRTC/SignalingClient.swift | 2 +- .../Realtime/Websocket/SignalingModel.swift | 36 ++++++- 7 files changed, 148 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7d45a7..c1b89ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: jobs: build-and-test: name: Build and Test (Swift 6.2.1) - runs-on: macos-latest + runs-on: macos-latest ? env: IOS_DEVICE_NAME: "iPhone 16 Pro" IOS_OS_VERSION: "18.5" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e853e9b..cd20c27 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,6 @@ jobs: build-only: true scheme: DecartSDK type: ios - configuration: release deviceName: ${{ env.IOS_DEVICE_NAME }} osVersion: ${{ env.IOS_OS_VERSION }} download-platform: true diff --git a/README.md b/README.md index 2c87831..cb56d40 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,13 @@ let remoteStream = try await realtimeManager.connect(localStream: localStream) // Update prompt in real-time realtimeManager.setPrompt(Prompt(text: "Anime World")) +// Send reference image (base64) to the realtime session +try await realtimeManager.setImageBase64( + imageBase64String, + prompt: "Use this as reference", + enhance: true +) + // Cleanup await capture.stopCapture() await realtimeManager.disconnect() @@ -226,6 +233,7 @@ func createProcessClient(model: VideoModel, input: VideoToVideoInput) throws -> func connect(localStream: RealtimeMediaStream) async throws -> RealtimeMediaStream func disconnect() async func setPrompt(_ prompt: Prompt) +func setImageBase64(_ imageBase64: String?, prompt: String?, enhance: Bool?, timeout: TimeInterval?) async throws func getStats() async -> RTCStatisticsReport? let events: AsyncStream diff --git a/Sources/DecartSDK/Models/Models.swift b/Sources/DecartSDK/Models/Models.swift index 1a443fd..a1bfd0e 100644 --- a/Sources/DecartSDK/Models/Models.swift +++ b/Sources/DecartSDK/Models/Models.swift @@ -9,6 +9,7 @@ public enum RealtimeModel: String, CaseIterable { case mirage case mirage_v2 case lucy_v2v_720p_rt + case lucy_v2v_14b_rt } public enum ImageModel: String, CaseIterable { @@ -51,6 +52,14 @@ public enum Models { width: 1280, height: 704 ) + case .lucy_v2v_14b_rt: + return ModelDefinition( + name: "lucy_v2v_14b_rt", + urlPath: "/v1/stream", + fps: 15, + width: 1280, + height: 704 + ) } } diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index daea7ee..6925845 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -12,6 +12,10 @@ public final class DecartRealtimeManager: @unchecked Sendable { private let stateContinuation: AsyncStream.Continuation private var webSocketListenerTask: Task? private var connectionStateListenerTask: Task? + private var setImageAckTimeoutTask: Task? + private var setImageAckContinuation: CheckedContinuation? + private let setImageAckLock = NSLock() + private let defaultImageSendTimeout: TimeInterval = 15 private var connectionState: DecartRealtimeConnectionState = .idle { didSet { @@ -36,6 +40,7 @@ public final class DecartRealtimeManager: @unchecked Sendable { webSocketListenerTask?.cancel() connectionStateListenerTask?.cancel() webRTCClient?.close() + cancelSetImageAckIfNeeded(error: DecartError.websocketError("Realtime manager deinitialized")) stateContinuation.finish() DecartLogger.log("RealtimeManager (SDK) deinitialized", level: .info) } @@ -85,6 +90,7 @@ public extension DecartRealtimeManager { webSocketListenerTask = nil connectionStateListenerTask?.cancel() connectionStateListenerTask = nil + cancelSetImageAckIfNeeded(error: DecartError.websocketError("Realtime disconnected")) webRTCClient?.close() webRTCClient = nil await webSocketClient?.disconnect() @@ -104,6 +110,53 @@ public extension DecartRealtimeManager { sendMessage(.prompt(PromptMessage(prompt: prompt.text))) } + func setImageBase64( + _ imageBase64: String?, + prompt: String? = nil, + enhance: Bool? = nil, + timeout: TimeInterval? = nil + ) async throws { + try await withCheckedThrowingContinuation { continuation in + setImageAckLock.lock() + if setImageAckContinuation != nil { + setImageAckLock.unlock() + continuation.resume( + throwing: DecartError.invalidOptions("setImageBase64 already in progress") + ) + return + } + setImageAckContinuation = continuation + setImageAckLock.unlock() + + sendMessage( + .setImage( + SetImageMessage( + imageData: imageBase64, + prompt: prompt, + enhancePrompt: enhance + ) + ) + ) + + let timeoutSeconds = timeout ?? defaultImageSendTimeout + setImageAckTimeoutTask?.cancel() + setImageAckTimeoutTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds * 1_000_000_000)) + guard let self else { return } + self.setImageAckLock.lock() + guard let continuation = self.setImageAckContinuation else { + self.setImageAckLock.unlock() + return + } + self.setImageAckContinuation = nil + self.setImageAckLock.unlock() + continuation.resume( + throwing: DecartError.websocketError("Image send timed out") + ) + } + } + } + func waitForConnection(timeout: TimeInterval) async throws { let startTime = Date() while connectionState != .connected { @@ -152,7 +205,14 @@ private extension DecartRealtimeManager { do { for try await message in wsClient.websocketEventStream { guard !Task.isCancelled, let self, let webRTCClient = self.webRTCClient else { return } - try await webRTCClient.handleSignalingMessage(message) + switch message { + case .setImageAck(let ack): + self.handleSetImageAck(ack) + case .promptAck, .sessionId: + break + default: + try await webRTCClient.handleSignalingMessage(message) + } } } catch { self?.connectionState = .error @@ -183,4 +243,37 @@ private extension DecartRealtimeManager { guard let webSocketClient else { return } Task { [webSocketClient] in try? await webSocketClient.send(message) } } + + func handleSetImageAck(_ message: SetImageAckMessage) { + setImageAckLock.lock() + guard let continuation = setImageAckContinuation else { + setImageAckLock.unlock() + return + } + setImageAckContinuation = nil + setImageAckTimeoutTask?.cancel() + setImageAckTimeoutTask = nil + setImageAckLock.unlock() + + if message.success { + continuation.resume() + } else { + continuation.resume( + throwing: DecartError.serverError(message.error ?? "Failed to send image") + ) + } + } + + func cancelSetImageAckIfNeeded(error: Error) { + setImageAckLock.lock() + guard let continuation = setImageAckContinuation else { + setImageAckLock.unlock() + return + } + setImageAckContinuation = nil + setImageAckTimeoutTask?.cancel() + setImageAckTimeoutTask = nil + setImageAckLock.unlock() + continuation.resume(throwing: error) + } } diff --git a/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift b/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift index 894761d..8fe4b5f 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift @@ -43,7 +43,7 @@ struct SignalingClient { case .error(let msg): throw DecartError.serverError(msg.message ?? msg.error ?? "Unknown server error") - case .sessionId, .promptAck: + case .sessionId, .promptAck, .setImageAck: break } } diff --git a/Sources/DecartSDK/Realtime/Websocket/SignalingModel.swift b/Sources/DecartSDK/Realtime/Websocket/SignalingModel.swift index ccdc271..7e2f392 100644 --- a/Sources/DecartSDK/Realtime/Websocket/SignalingModel.swift +++ b/Sources/DecartSDK/Realtime/Websocket/SignalingModel.swift @@ -65,6 +65,27 @@ struct PromptMessage: Codable, Sendable { } } +struct SetImageMessage: Codable, Sendable { + let type: String + let imageData: String? + let prompt: String? + let enhancePrompt: Bool? + + init(imageData: String?, prompt: String? = nil, enhancePrompt: Bool? = nil) { + self.type = "set_image" + self.imageData = imageData + self.prompt = prompt + self.enhancePrompt = enhancePrompt + } + + private enum CodingKeys: String, CodingKey { + case type + case imageData = "image_data" + case prompt + case enhancePrompt = "enhance_prompt" + } +} + struct ServerErrorMessage: Codable, Sendable { let type: String let message: String? @@ -83,6 +104,12 @@ struct PromptAckMessage: Codable, Sendable { let type: String } +struct SetImageAckMessage: Codable, Sendable { + let type: String + let success: Bool + let error: String? +} + enum IncomingWebSocketMessage: Codable, Sendable { case offer(OfferMessage) case answer(AnswerMessage) @@ -90,6 +117,7 @@ enum IncomingWebSocketMessage: Codable, Sendable { case error(ServerErrorMessage) case sessionId(SessionIdMessage) case promptAck(PromptAckMessage) + case setImageAck(SetImageAckMessage) init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) @@ -108,6 +136,8 @@ enum IncomingWebSocketMessage: Codable, Sendable { self = try .sessionId(SessionIdMessage(from: decoder)) case "prompt_ack": self = try .promptAck(PromptAckMessage(from: decoder)) + case "set_image_ack": + self = try .setImageAck(SetImageAckMessage(from: decoder)) default: throw DecodingError.dataCorruptedError( forKey: .type, @@ -131,6 +161,8 @@ enum IncomingWebSocketMessage: Codable, Sendable { try msg.encode(to: encoder) case .promptAck(let msg): try msg.encode(to: encoder) + case .setImageAck(let msg): + try msg.encode(to: encoder) } } @@ -144,6 +176,7 @@ enum OutgoingWebSocketMessage: Codable, Sendable { case answer(AnswerMessage) case iceCandidate(IceCandidateMessage) case prompt(PromptMessage) + case setImage(SetImageMessage) func encode(to encoder: Encoder) throws { switch self { @@ -155,7 +188,8 @@ enum OutgoingWebSocketMessage: Codable, Sendable { try msg.encode(to: encoder) case .prompt(let msg): try msg.encode(to: encoder) + case .setImage(let msg): + try msg.encode(to: encoder) } } } - From 6803ee22e446634ee03a7250c2e68f1883f3c956 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Mon, 26 Jan 2026 22:58:06 +0200 Subject: [PATCH 15/23] fix build error --- .../Realtime/DecartRealtimeManager.swift | 133 +++++++++--------- 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index 6925845..1ac2278 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -12,9 +12,7 @@ public final class DecartRealtimeManager: @unchecked Sendable { private let stateContinuation: AsyncStream.Continuation private var webSocketListenerTask: Task? private var connectionStateListenerTask: Task? - private var setImageAckTimeoutTask: Task? - private var setImageAckContinuation: CheckedContinuation? - private let setImageAckLock = NSLock() + private let setImageAckState = SetImageAckState() private let defaultImageSendTimeout: TimeInterval = 15 private var connectionState: DecartRealtimeConnectionState = .idle { @@ -40,7 +38,9 @@ public final class DecartRealtimeManager: @unchecked Sendable { webSocketListenerTask?.cancel() connectionStateListenerTask?.cancel() webRTCClient?.close() - cancelSetImageAckIfNeeded(error: DecartError.websocketError("Realtime manager deinitialized")) + Task { [setImageAckState] in + await setImageAckState.cancel(error: DecartError.websocketError("Realtime manager deinitialized")) + } stateContinuation.finish() DecartLogger.log("RealtimeManager (SDK) deinitialized", level: .info) } @@ -90,12 +90,12 @@ public extension DecartRealtimeManager { webSocketListenerTask = nil connectionStateListenerTask?.cancel() connectionStateListenerTask = nil - cancelSetImageAckIfNeeded(error: DecartError.websocketError("Realtime disconnected")) + await setImageAckState.cancel(error: DecartError.websocketError("Realtime disconnected")) webRTCClient?.close() webRTCClient = nil await webSocketClient?.disconnect() - #if canImport(WebRTC) && (os(iOS)) + #if canImport(WebRTC) && os(iOS) let audioSession = RTCAudioSession.sharedInstance() if audioSession.isActive { audioSession.lockForConfiguration() @@ -116,45 +116,19 @@ public extension DecartRealtimeManager { enhance: Bool? = nil, timeout: TimeInterval? = nil ) async throws { - try await withCheckedThrowingContinuation { continuation in - setImageAckLock.lock() - if setImageAckContinuation != nil { - setImageAckLock.unlock() - continuation.resume( - throwing: DecartError.invalidOptions("setImageBase64 already in progress") - ) - return - } - setImageAckContinuation = continuation - setImageAckLock.unlock() - - sendMessage( - .setImage( - SetImageMessage( - imageData: imageBase64, - prompt: prompt, - enhancePrompt: enhance - ) - ) - ) - - let timeoutSeconds = timeout ?? defaultImageSendTimeout - setImageAckTimeoutTask?.cancel() - setImageAckTimeoutTask = Task { [weak self] in - try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds * 1_000_000_000)) - guard let self else { return } - self.setImageAckLock.lock() - guard let continuation = self.setImageAckContinuation else { - self.setImageAckLock.unlock() - return - } - self.setImageAckContinuation = nil - self.setImageAckLock.unlock() - continuation.resume( - throwing: DecartError.websocketError("Image send timed out") - ) + let timeoutSeconds = timeout ?? defaultImageSendTimeout + let message = SetImageMessage( + imageData: imageBase64, + prompt: prompt, + enhancePrompt: enhance + ) + try await setImageAckState.send( + message: message, + timeout: timeoutSeconds, + sendMessage: { [weak self] outgoing in + self?.sendMessage(outgoing) } - } + ) } func waitForConnection(timeout: TimeInterval) async throws { @@ -245,16 +219,54 @@ private extension DecartRealtimeManager { } func handleSetImageAck(_ message: SetImageAckMessage) { - setImageAckLock.lock() - guard let continuation = setImageAckContinuation else { - setImageAckLock.unlock() - return + Task { [setImageAckState] in + await setImageAckState.handleAck(message: message) } - setImageAckContinuation = nil - setImageAckTimeoutTask?.cancel() - setImageAckTimeoutTask = nil - setImageAckLock.unlock() + } +} + +// MARK: - SetImageAckState Actor +private actor SetImageAckState { + private var continuation: CheckedContinuation? + private var timeoutTask: Task? + + func send( + message: SetImageMessage, + timeout: TimeInterval, + sendMessage: (OutgoingWebSocketMessage) -> Void + ) async throws { + guard continuation == nil else { + throw DecartError.invalidOptions("setImageBase64 already in progress") + } + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.continuation = continuation + sendMessage(.setImage(message)) + + timeoutTask?.cancel() + timeoutTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: UInt64(timeout * 1_000_000_000)) + await self?.timeout() + } + } + } + + private func timeout() { + guard let continuation else { return } + self.continuation = nil + self.timeoutTask = nil + continuation.resume( + throwing: DecartError.websocketError("Image send timed out") + ) + } + + func handleAck(message: SetImageAckMessage) { + guard let continuation = self.continuation else { return } + self.continuation = nil + timeoutTask?.cancel() + timeoutTask = nil + if message.success { continuation.resume() } else { @@ -263,17 +275,12 @@ private extension DecartRealtimeManager { ) } } - - func cancelSetImageAckIfNeeded(error: Error) { - setImageAckLock.lock() - guard let continuation = setImageAckContinuation else { - setImageAckLock.unlock() - return - } - setImageAckContinuation = nil - setImageAckTimeoutTask?.cancel() - setImageAckTimeoutTask = nil - setImageAckLock.unlock() + + func cancel(error: Error) { + guard let continuation = self.continuation else { return } + self.continuation = nil + timeoutTask?.cancel() + timeoutTask = nil continuation.resume(throwing: error) } } From 2f5f41be898a2b00ed9d8ce1e74674ed8c8f4e2d Mon Sep 17 00:00:00 2001 From: Verion1 Date: Tue, 27 Jan 2026 17:49:48 +0200 Subject: [PATCH 16/23] remove timeout on realtime connection for lucy 14 B queue --- .../Realtime/DecartRealtimeManager.swift | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index 1ac2278..3842d22 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -140,7 +140,7 @@ public extension DecartRealtimeManager { if Date().timeIntervalSince(startTime) > timeout { throw DecartError.webRTCError("Connection timeout") } - try await Task.sleep(nanoseconds: 100_000_000) + try await Task.sleep(nanoseconds: 100_000_000 * 100) // 1000 seconds } sendMessage(.prompt(PromptMessage(prompt: options.initialState.prompt.text))) } @@ -230,7 +230,7 @@ private extension DecartRealtimeManager { private actor SetImageAckState { private var continuation: CheckedContinuation? private var timeoutTask: Task? - + func send( message: SetImageMessage, timeout: TimeInterval, @@ -255,18 +255,18 @@ private actor SetImageAckState { private func timeout() { guard let continuation else { return } self.continuation = nil - self.timeoutTask = nil + timeoutTask = nil continuation.resume( throwing: DecartError.websocketError("Image send timed out") ) } - + func handleAck(message: SetImageAckMessage) { - guard let continuation = self.continuation else { return } + guard let continuation = continuation else { return } self.continuation = nil timeoutTask?.cancel() timeoutTask = nil - + if message.success { continuation.resume() } else { @@ -275,9 +275,9 @@ private actor SetImageAckState { ) } } - + func cancel(error: Error) { - guard let continuation = self.continuation else { return } + guard let continuation = continuation else { return } self.continuation = nil timeoutTask?.cancel() timeoutTask = nil From e2d2359d834675d481fe244caf8bd48046f59f31 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Fri, 30 Jan 2026 14:43:30 +0200 Subject: [PATCH 17/23] refactor DecartPrompt with ref image --- .github/workflows/ci.yml | 2 +- Sources/DecartSDK/Models/ModelState.swift | 15 +++++---------- .../Realtime/DecartRealtimeManager.swift | 9 ++++++--- .../Realtime/RealtimeConfiguration.swift | 6 +++--- .../DecartSDK/Realtime/RealtimeDataTypes.swift | 2 +- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1b89ab..c7d45a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: jobs: build-and-test: name: Build and Test (Swift 6.2.1) - runs-on: macos-latest ? + runs-on: macos-latest env: IOS_DEVICE_NAME: "iPhone 16 Pro" IOS_OS_VERSION: "18.5" diff --git a/Sources/DecartSDK/Models/ModelState.swift b/Sources/DecartSDK/Models/ModelState.swift index 6dd3f91..29f30ac 100644 --- a/Sources/DecartSDK/Models/ModelState.swift +++ b/Sources/DecartSDK/Models/ModelState.swift @@ -1,19 +1,14 @@ import Foundation -public struct Prompt: Sendable { +public struct DecartPrompt: Sendable { public let text: String public let enrich: Bool + // for lucy 14b we must send a ref image with text prompt + public let referenceImageBase64: String? - public init(text: String, enrich: Bool = true) { + public init(text: String, referenceImageBase64: String? = nil, enrich: Bool = false) { self.text = text + self.referenceImageBase64 = referenceImageBase64 self.enrich = enrich } } - -public struct ModelState: Sendable { - public let prompt: Prompt - - public init(prompt: Prompt) { - self.prompt = prompt - } -} diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index 3842d22..4315f44 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -77,6 +77,10 @@ public extension DecartRealtimeManager { try await waitForConnection(timeout: options.connection.connectionTimeout) + sendMessage( + .prompt(PromptMessage(prompt: options.initialPrompt.text)) + ) + guard let remoteStream = rtcClient.getRemoteRealtimeStream() else { throw DecartError.webRTCError("couldn't get remote stream, check video transceiver") } @@ -106,7 +110,7 @@ public extension DecartRealtimeManager { webSocketClient = nil } - func setPrompt(_ prompt: Prompt) { + func setPrompt(_ prompt: DecartPrompt) { sendMessage(.prompt(PromptMessage(prompt: prompt.text))) } @@ -140,9 +144,8 @@ public extension DecartRealtimeManager { if Date().timeIntervalSince(startTime) > timeout { throw DecartError.webRTCError("Connection timeout") } - try await Task.sleep(nanoseconds: 100_000_000 * 100) // 1000 seconds + try await Task.sleep(nanoseconds: 100_000_000 * 100) // 100 seconds } - sendMessage(.prompt(PromptMessage(prompt: options.initialState.prompt.text))) } } diff --git a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift index 400ac8b..e114585 100644 --- a/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift +++ b/Sources/DecartSDK/Realtime/RealtimeConfiguration.swift @@ -10,18 +10,18 @@ import Foundation public struct RealtimeConfiguration: Sendable { public let model: ModelDefinition - public let initialState: ModelState + public let initialPrompt: DecartPrompt public let connection: ConnectionConfig public let media: MediaConfig public init( model: ModelDefinition, - initialState: ModelState, + initialPrompt: DecartPrompt, connection: ConnectionConfig = .init(), media: MediaConfig = .init() ) { self.model = model - self.initialState = initialState + self.initialPrompt = initialPrompt self.connection = connection self.media = media } diff --git a/Sources/DecartSDK/Realtime/RealtimeDataTypes.swift b/Sources/DecartSDK/Realtime/RealtimeDataTypes.swift index f427358..53c784f 100644 --- a/Sources/DecartSDK/Realtime/RealtimeDataTypes.swift +++ b/Sources/DecartSDK/Realtime/RealtimeDataTypes.swift @@ -15,7 +15,7 @@ public struct RealtimeMediaStream: Sendable { case .localStream: return "stream-local" case .remoteStream: - return "stream-remote" // It's good practice to handle all cases + return "stream-remote" } } } From 3b14478845bfb21f2f52cb2ddfea59e8bb008d14 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Fri, 30 Jan 2026 15:56:51 +0200 Subject: [PATCH 18/23] support queue and ref data image --- .../DecartClient.swift} | 30 -- .../DecartSDK/API/DecartConfiguration.swift | 31 ++ ...ptureDataTypes.swift => CameraError.swift} | 0 .../Models/Inputs/InputSupport.swift | 123 ++++++++ .../DecartSDK/Models/Inputs/ModelInputs.swift | 123 ++++++++ .../Models/Inputs/ModelsInputFactory.swift | 29 ++ Sources/DecartSDK/Models/ModelDataTypes.swift | 4 +- Sources/DecartSDK/Models/Models.swift | 3 +- .../DecartSDK/Models/ModelsInputFactory.swift | 275 ------------------ .../Realtime/DecartRealtimeManager.swift | 152 ++++------ .../Models/DecartPrompt.swift} | 6 +- .../Models/RealtimeConnectionState.swift | 15 + .../RealtimeMediaStream.swift} | 22 -- .../Models/RealtimeServiceStatus.swift | 16 + .../WebSocket}/SignalingModel.swift | 27 ++ .../WebSocket}/WebSocketClient.swift | 0 .../Realtime/WebRTC/SignalingClient.swift | 2 +- .../RTCMLVideoViewWrapper.swift | 0 18 files changed, 427 insertions(+), 431 deletions(-) rename Sources/DecartSDK/{DecartSDK.swift => API/DecartClient.swift} (68%) create mode 100644 Sources/DecartSDK/API/DecartConfiguration.swift rename Sources/DecartSDK/Capture/{CaptureDataTypes.swift => CameraError.swift} (100%) create mode 100644 Sources/DecartSDK/Models/Inputs/InputSupport.swift create mode 100644 Sources/DecartSDK/Models/Inputs/ModelInputs.swift create mode 100644 Sources/DecartSDK/Models/Inputs/ModelsInputFactory.swift delete mode 100644 Sources/DecartSDK/Models/ModelsInputFactory.swift rename Sources/DecartSDK/{Models/ModelState.swift => Realtime/Models/DecartPrompt.swift} (54%) create mode 100644 Sources/DecartSDK/Realtime/Models/RealtimeConnectionState.swift rename Sources/DecartSDK/Realtime/{RealtimeDataTypes.swift => Models/RealtimeMediaStream.swift} (57%) create mode 100644 Sources/DecartSDK/Realtime/Models/RealtimeServiceStatus.swift rename Sources/DecartSDK/Realtime/{Websocket => Transport/WebSocket}/SignalingModel.swift (86%) rename Sources/DecartSDK/Realtime/{Websocket => Transport/WebSocket}/WebSocketClient.swift (100%) rename Sources/DecartSDK/SwiftUI/{ => Realtime}/RTCMLVideoViewWrapper.swift (100%) diff --git a/Sources/DecartSDK/DecartSDK.swift b/Sources/DecartSDK/API/DecartClient.swift similarity index 68% rename from Sources/DecartSDK/DecartSDK.swift rename to Sources/DecartSDK/API/DecartClient.swift index 79790bc..c202c6c 100644 --- a/Sources/DecartSDK/DecartSDK.swift +++ b/Sources/DecartSDK/API/DecartClient.swift @@ -1,35 +1,5 @@ import Foundation -public struct DecartConfiguration { - public let baseURL: URL - public let apiKey: String - - var headers: [String: String] { ["Authorization": "Bearer \(apiKey)"] } - - var signalingServerUrl: String { - var baseURLString = baseURL.absoluteString - if baseURLString.hasPrefix("https://") { - baseURLString = baseURLString.replacingOccurrences(of: "https://", with: "wss://") - } else if baseURLString.hasPrefix("http://") { - baseURLString = baseURLString.replacingOccurrences(of: "http://", with: "ws://") - } - return baseURLString - } - - public init(baseURL: String = "https://api3.decart.ai", apiKey: String) { - guard let url = URL(string: baseURL) else { - DecartLogger.log("Unable to create URL from: \(baseURL)", level: .error) - fatalError("Unable to create URL from: \(baseURL)") - } - guard !apiKey.isEmpty else { - DecartLogger.log("API key is empty", level: .error) - fatalError("Api key is empty") - } - self.baseURL = url - self.apiKey = apiKey - } -} - public struct DecartClient { let decartConfiguration: DecartConfiguration diff --git a/Sources/DecartSDK/API/DecartConfiguration.swift b/Sources/DecartSDK/API/DecartConfiguration.swift new file mode 100644 index 0000000..72847cc --- /dev/null +++ b/Sources/DecartSDK/API/DecartConfiguration.swift @@ -0,0 +1,31 @@ +import Foundation + +public struct DecartConfiguration { + public let baseURL: URL + public let apiKey: String + + var headers: [String: String] { ["Authorization": "Bearer \(apiKey)"] } + + var signalingServerUrl: String { + var baseURLString = baseURL.absoluteString + if baseURLString.hasPrefix("https://") { + baseURLString = baseURLString.replacingOccurrences(of: "https://", with: "wss://") + } else if baseURLString.hasPrefix("http://") { + baseURLString = baseURLString.replacingOccurrences(of: "http://", with: "ws://") + } + return baseURLString + } + + public init(baseURL: String = "https://api.decart.ai", apiKey: String) { + guard let url = URL(string: baseURL) else { + DecartLogger.log("Unable to create URL from: \(baseURL)", level: .error) + fatalError("Unable to create URL from: \(baseURL)") + } + guard !apiKey.isEmpty else { + DecartLogger.log("API key is empty", level: .error) + fatalError("Api key is empty") + } + self.baseURL = url + self.apiKey = apiKey + } +} diff --git a/Sources/DecartSDK/Capture/CaptureDataTypes.swift b/Sources/DecartSDK/Capture/CameraError.swift similarity index 100% rename from Sources/DecartSDK/Capture/CaptureDataTypes.swift rename to Sources/DecartSDK/Capture/CameraError.swift diff --git a/Sources/DecartSDK/Models/Inputs/InputSupport.swift b/Sources/DecartSDK/Models/Inputs/InputSupport.swift new file mode 100644 index 0000000..b2eea6f --- /dev/null +++ b/Sources/DecartSDK/Models/Inputs/InputSupport.swift @@ -0,0 +1,123 @@ +import Foundation +import UniformTypeIdentifiers + +public enum ProResolution: String, Codable, Sendable { + case res720p = "720p" + case res480p = "480p" +} + +public enum DevResolution: String, Codable, Sendable { + case res720p = "720p" +} + +public enum InputValidationError: LocalizedError { + case emptyPrompt + case emptyFileData + case expectedImage + case expectedVideo + case unsupportedMediaType + + public var errorDescription: String? { + switch self { + case .emptyPrompt: + return "Prompt cannot be empty" + case .emptyFileData: + return "File data cannot be empty" + case .expectedImage: + return "Expected an image file" + case .expectedVideo: + return "Expected a video file" + case .unsupportedMediaType: + return "Unsupported media type. Only image and video files are supported" + } + } +} + +public enum MediaType: Sendable { + case image + case video +} + +public struct FileInput: Codable, Sendable { + public let data: Data + public let filename: String + public let mediaType: MediaType + + private enum CodingKeys: String, CodingKey { + case data, filename + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.data = try container.decode(Data.self, forKey: .data) + self.filename = try container.decode(String.self, forKey: .filename) + self.mediaType = FileInput.inferMediaType(from: filename) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(data, forKey: .data) + try container.encode(filename, forKey: .filename) + } + + private init(data: Data, filename: String, mediaType: MediaType) { + self.data = data + self.filename = filename + self.mediaType = mediaType + } + + public static func image(data: Data, filename: String = "image.jpg") throws -> FileInput { + guard !data.isEmpty else { throw InputValidationError.emptyFileData } + return FileInput( + data: data, + filename: ensureExtension(for: filename, defaultExtension: "jpg"), + mediaType: .image + ) + } + + public static func video(data: Data, filename: String = "video.mp4") throws -> FileInput { + guard !data.isEmpty else { throw InputValidationError.emptyFileData } + return FileInput( + data: data, + filename: ensureExtension(for: filename, defaultExtension: "mp4"), + mediaType: .video + ) + } + + public static func from(data: Data, uniformType: UTType?) throws -> FileInput { + guard !data.isEmpty else { throw InputValidationError.emptyFileData } + + if let type = uniformType, type.conforms(to: .image) { + return try image(data: data) + } + + if let type = uniformType, type.conforms(to: .video) || type.conforms(to: .movie) { + return try video(data: data) + } + + throw InputValidationError.unsupportedMediaType + } + + private static func ensureExtension(for filename: String, defaultExtension: String) -> String { + var trimmed = (filename as NSString).lastPathComponent + if trimmed.isEmpty { + trimmed = "attachment.\(defaultExtension)" + } + + if (trimmed as NSString).pathExtension.isEmpty { + trimmed.append(".\(defaultExtension)") + } + + return trimmed + } + + private static func inferMediaType(from filename: String) -> MediaType { + let ext = (filename as NSString).pathExtension.lowercased() + switch ext { + case "jpg", "jpeg", "png", "heic", "webp": + return .image + default: + return .video + } + } +} diff --git a/Sources/DecartSDK/Models/Inputs/ModelInputs.swift b/Sources/DecartSDK/Models/Inputs/ModelInputs.swift new file mode 100644 index 0000000..a293674 --- /dev/null +++ b/Sources/DecartSDK/Models/Inputs/ModelInputs.swift @@ -0,0 +1,123 @@ +import Foundation + +public struct TextToVideoInput: Codable, Sendable { + public let prompt: String + public let seed: Int? + public let resolution: ProResolution? + public let orientation: String? + + public init( + prompt: String, + seed: Int? = nil, + resolution: ProResolution? = .res720p, + orientation: String? = nil + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + + self.prompt = trimmed + self.seed = seed + self.resolution = resolution + self.orientation = orientation + } +} + +public struct TextToImageInput: Codable, Sendable { + public let prompt: String + public let seed: Int? + public let resolution: ProResolution? + public let orientation: String? + + public init( + prompt: String, + seed: Int? = nil, + resolution: ProResolution? = .res720p, + orientation: String? = nil + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + + self.prompt = trimmed + self.seed = seed + self.resolution = resolution + self.orientation = orientation + } +} + +public struct ImageToVideoInput: Codable, Sendable { + public let prompt: String + public let data: FileInput + public let seed: Int? + public let resolution: ProResolution? + + public init( + prompt: String, + data: FileInput, + seed: Int? = nil, + resolution: ProResolution? = .res720p + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + guard data.mediaType == .image else { throw InputValidationError.expectedImage } + + self.prompt = trimmed + self.data = data + self.seed = seed + self.resolution = resolution + } +} + +public struct ImageToImageInput: Codable, Sendable { + public let prompt: String + public let data: FileInput + public let seed: Int? + public let resolution: ProResolution? + public let enhancePrompt: Bool? + + public init( + prompt: String, + data: FileInput, + seed: Int? = nil, + resolution: ProResolution? = .res720p, + enhancePrompt: Bool? = nil + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + guard data.mediaType == .image else { throw InputValidationError.expectedImage } + + self.prompt = trimmed + self.data = data + self.seed = seed + self.resolution = resolution + self.enhancePrompt = enhancePrompt + } +} + +public struct VideoToVideoInput: Codable, Sendable { + public let prompt: String + public let data: FileInput + public let seed: Int? + public let resolution: ProResolution? + public let enhancePrompt: Bool? + public let numInferenceSteps: Int? + + public init( + prompt: String, + data: FileInput, + seed: Int? = nil, + resolution: ProResolution? = .res720p, + enhancePrompt: Bool? = nil, + numInferenceSteps: Int? = nil + ) throws { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } + guard data.mediaType == .video else { throw InputValidationError.expectedVideo } + + self.prompt = trimmed + self.data = data + self.seed = seed + self.resolution = resolution + self.enhancePrompt = enhancePrompt + self.numInferenceSteps = numInferenceSteps + } +} diff --git a/Sources/DecartSDK/Models/Inputs/ModelsInputFactory.swift b/Sources/DecartSDK/Models/Inputs/ModelsInputFactory.swift new file mode 100644 index 0000000..3702379 --- /dev/null +++ b/Sources/DecartSDK/Models/Inputs/ModelsInputFactory.swift @@ -0,0 +1,29 @@ +public enum ModelInputType: Sendable { + case textToVideo + case textToImage + case imageToVideo + case imageToImage + case videoToVideo +} + +public enum ModelsInputFactory: Sendable { + public static func videoInputType(for model: VideoModel) -> ModelInputType { + switch model { + case .lucy_pro_t2v: + return .textToVideo + case .lucy_dev_i2v, .lucy_pro_i2v: + return .imageToVideo + case .lucy_fast_v2v, .lucy_pro_v2v: + return .videoToVideo + } + } + + public static func imageInputType(for model: ImageModel) -> ModelInputType { + switch model { + case .lucy_pro_t2i: + return .textToImage + case .lucy_pro_i2i: + return .imageToImage + } + } +} diff --git a/Sources/DecartSDK/Models/ModelDataTypes.swift b/Sources/DecartSDK/Models/ModelDataTypes.swift index 25daecb..02c3ee9 100644 --- a/Sources/DecartSDK/Models/ModelDataTypes.swift +++ b/Sources/DecartSDK/Models/ModelDataTypes.swift @@ -11,12 +11,14 @@ public struct ModelDefinition: Sendable { public let fps: Int public let width: Int public let height: Int + public let hasReferenceImage: Bool - public init(name: String, urlPath: String, fps: Int, width: Int, height: Int) { + public init(name: String, urlPath: String, fps: Int, width: Int, height: Int, hasReferenceImage: Bool = false) { self.name = name self.urlPath = urlPath self.fps = fps self.width = width self.height = height + self.hasReferenceImage = hasReferenceImage } } diff --git a/Sources/DecartSDK/Models/Models.swift b/Sources/DecartSDK/Models/Models.swift index a1bfd0e..22ad866 100644 --- a/Sources/DecartSDK/Models/Models.swift +++ b/Sources/DecartSDK/Models/Models.swift @@ -58,7 +58,8 @@ public enum Models { urlPath: "/v1/stream", fps: 15, width: 1280, - height: 704 + height: 704, + hasReferenceImage: true ) } } diff --git a/Sources/DecartSDK/Models/ModelsInputFactory.swift b/Sources/DecartSDK/Models/ModelsInputFactory.swift deleted file mode 100644 index 14eaf85..0000000 --- a/Sources/DecartSDK/Models/ModelsInputFactory.swift +++ /dev/null @@ -1,275 +0,0 @@ -import Foundation -import UniformTypeIdentifiers - -public enum ProResolution: String, Codable, Sendable { - case res720p = "720p" - case res480p = "480p" -} - -public enum DevResolution: String, Codable, Sendable { - case res720p = "720p" -} - -public enum InputValidationError: LocalizedError { - case emptyPrompt - case emptyFileData - case expectedImage - case expectedVideo - case unsupportedMediaType - - public var errorDescription: String? { - switch self { - case .emptyPrompt: - return "Prompt cannot be empty" - case .emptyFileData: - return "File data cannot be empty" - case .expectedImage: - return "Expected an image file" - case .expectedVideo: - return "Expected a video file" - case .unsupportedMediaType: - return "Unsupported media type. Only image and video files are supported" - } - } -} - -public enum MediaType: Sendable { - case image - case video -} - -public struct FileInput: Codable, Sendable { - public let data: Data - public let filename: String - public let mediaType: MediaType - - private enum CodingKeys: String, CodingKey { - case data, filename - } - - public init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - self.data = try container.decode(Data.self, forKey: .data) - self.filename = try container.decode(String.self, forKey: .filename) - self.mediaType = FileInput.inferMediaType(from: filename) - } - - public func encode(to encoder: Encoder) throws { - var container = encoder.container(keyedBy: CodingKeys.self) - try container.encode(data, forKey: .data) - try container.encode(filename, forKey: .filename) - } - - private init(data: Data, filename: String, mediaType: MediaType) { - self.data = data - self.filename = filename - self.mediaType = mediaType - } - - public static func image(data: Data, filename: String = "image.jpg") throws -> FileInput { - guard !data.isEmpty else { throw InputValidationError.emptyFileData } - return FileInput( - data: data, - filename: ensureExtension(for: filename, defaultExtension: "jpg"), - mediaType: .image - ) - } - - public static func video(data: Data, filename: String = "video.mp4") throws -> FileInput { - guard !data.isEmpty else { throw InputValidationError.emptyFileData } - return FileInput( - data: data, - filename: ensureExtension(for: filename, defaultExtension: "mp4"), - mediaType: .video - ) - } - - public static func from(data: Data, uniformType: UTType?) throws -> FileInput { - guard !data.isEmpty else { throw InputValidationError.emptyFileData } - - if let type = uniformType, type.conforms(to: .image) { - return try image(data: data) - } - - if let type = uniformType, type.conforms(to: .video) || type.conforms(to: .movie) { - return try video(data: data) - } - - throw InputValidationError.unsupportedMediaType - } - - private static func ensureExtension(for filename: String, defaultExtension: String) -> String { - var trimmed = (filename as NSString).lastPathComponent - if trimmed.isEmpty { - trimmed = "attachment.\(defaultExtension)" - } - - if (trimmed as NSString).pathExtension.isEmpty { - trimmed.append(".\(defaultExtension)") - } - - return trimmed - } - - private static func inferMediaType(from filename: String) -> MediaType { - let ext = (filename as NSString).pathExtension.lowercased() - switch ext { - case "jpg", "jpeg", "png", "heic", "webp": - return .image - default: - return .video - } - } -} - -public struct TextToVideoInput: Codable, Sendable { - public let prompt: String - public let seed: Int? - public let resolution: ProResolution? - public let orientation: String? - - public init( - prompt: String, - seed: Int? = nil, - resolution: ProResolution? = .res720p, - orientation: String? = nil - ) throws { - let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } - - self.prompt = trimmed - self.seed = seed - self.resolution = resolution - self.orientation = orientation - } -} - -public struct TextToImageInput: Codable, Sendable { - public let prompt: String - public let seed: Int? - public let resolution: ProResolution? - public let orientation: String? - - public init( - prompt: String, - seed: Int? = nil, - resolution: ProResolution? = .res720p, - orientation: String? = nil - ) throws { - let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } - - self.prompt = trimmed - self.seed = seed - self.resolution = resolution - self.orientation = orientation - } -} - -public struct ImageToVideoInput: Codable, Sendable { - public let prompt: String - public let data: FileInput - public let seed: Int? - public let resolution: ProResolution? - - public init( - prompt: String, - data: FileInput, - seed: Int? = nil, - resolution: ProResolution? = .res720p - ) throws { - let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } - guard data.mediaType == .image else { throw InputValidationError.expectedImage } - - self.prompt = trimmed - self.data = data - self.seed = seed - self.resolution = resolution - } -} - -public struct ImageToImageInput: Codable, Sendable { - public let prompt: String - public let data: FileInput - public let seed: Int? - public let resolution: ProResolution? - public let enhancePrompt: Bool? - - public init( - prompt: String, - data: FileInput, - seed: Int? = nil, - resolution: ProResolution? = .res720p, - enhancePrompt: Bool? = nil - ) throws { - let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } - guard data.mediaType == .image else { throw InputValidationError.expectedImage } - - self.prompt = trimmed - self.data = data - self.seed = seed - self.resolution = resolution - self.enhancePrompt = enhancePrompt - } -} - -public struct VideoToVideoInput: Codable, Sendable { - public let prompt: String - public let data: FileInput - public let seed: Int? - public let resolution: ProResolution? - public let enhancePrompt: Bool? - public let numInferenceSteps: Int? - - public init( - prompt: String, - data: FileInput, - seed: Int? = nil, - resolution: ProResolution? = .res720p, - enhancePrompt: Bool? = nil, - numInferenceSteps: Int? = nil - ) throws { - let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { throw InputValidationError.emptyPrompt } - guard data.mediaType == .video else { throw InputValidationError.expectedVideo } - - self.prompt = trimmed - self.data = data - self.seed = seed - self.resolution = resolution - self.enhancePrompt = enhancePrompt - self.numInferenceSteps = numInferenceSteps - } -} - -public enum ModelInputType: Sendable { - case textToVideo - case textToImage - case imageToVideo - case imageToImage - case videoToVideo -} - -public enum ModelsInputFactory: Sendable { - public static func videoInputType(for model: VideoModel) -> ModelInputType { - switch model { - case .lucy_pro_t2v: - return .textToVideo - case .lucy_dev_i2v, .lucy_pro_i2v: - return .imageToVideo - case .lucy_fast_v2v, .lucy_pro_v2v: - return .videoToVideo - } - } - - public static func imageInputType(for model: ImageModel) -> ModelInputType { - switch model { - case .lucy_pro_t2i: - return .textToImage - case .lucy_pro_i2i: - return .imageToImage - } - } -} diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index 4315f44..ec657d9 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -4,6 +4,9 @@ import Foundation public final class DecartRealtimeManager: @unchecked Sendable { public let options: RealtimeConfiguration public let events: AsyncStream + public private(set) var serviceStatus: RealtimeServiceStatus = .unknown + public private(set) var queuePosition: Int? + public private(set) var queueSize: Int? private var webRTCClient: WebRTCClient? private var webSocketClient: WebSocketClient? @@ -12,8 +15,6 @@ public final class DecartRealtimeManager: @unchecked Sendable { private let stateContinuation: AsyncStream.Continuation private var webSocketListenerTask: Task? private var connectionStateListenerTask: Task? - private let setImageAckState = SetImageAckState() - private let defaultImageSendTimeout: TimeInterval = 15 private var connectionState: DecartRealtimeConnectionState = .idle { didSet { @@ -38,9 +39,6 @@ public final class DecartRealtimeManager: @unchecked Sendable { webSocketListenerTask?.cancel() connectionStateListenerTask?.cancel() webRTCClient?.close() - Task { [setImageAckState] in - await setImageAckState.cancel(error: DecartError.websocketError("Realtime manager deinitialized")) - } stateContinuation.finish() DecartLogger.log("RealtimeManager (SDK) deinitialized", level: .info) } @@ -56,6 +54,10 @@ public extension DecartRealtimeManager { webSocketClient = wsClient setupWebSocketListener(wsClient) + if serviceStatus == .enteringQueue { + try await waitForServiceReady() + } + let rtcClient = WebRTCClient( config: options.connection.rtcConfiguration, constraints: options.media.connectionConstraints, @@ -76,10 +78,7 @@ public extension DecartRealtimeManager { sendMessage(.offer(OfferMessage(sdp: offer.sdp))) try await waitForConnection(timeout: options.connection.connectionTimeout) - - sendMessage( - .prompt(PromptMessage(prompt: options.initialPrompt.text)) - ) + setPrompt(options.initialPrompt) guard let remoteStream = rtcClient.getRemoteRealtimeStream() else { throw DecartError.webRTCError("couldn't get remote stream, check video transceiver") @@ -94,7 +93,6 @@ public extension DecartRealtimeManager { webSocketListenerTask = nil connectionStateListenerTask?.cancel() connectionStateListenerTask = nil - await setImageAckState.cancel(error: DecartError.websocketError("Realtime disconnected")) webRTCClient?.close() webRTCClient = nil await webSocketClient?.disconnect() @@ -111,28 +109,25 @@ public extension DecartRealtimeManager { } func setPrompt(_ prompt: DecartPrompt) { - sendMessage(.prompt(PromptMessage(prompt: prompt.text))) - } + guard + let referenceImageData = prompt.referenceImageData, + options.model.hasReferenceImage + else { + // if !options.model.hasReferenceImage { + sendMessage(.prompt(PromptMessage(prompt: prompt.text))) + // } + return + } - func setImageBase64( - _ imageBase64: String?, - prompt: String? = nil, - enhance: Bool? = nil, - timeout: TimeInterval? = nil - ) async throws { - let timeoutSeconds = timeout ?? defaultImageSendTimeout - let message = SetImageMessage( - imageData: imageBase64, - prompt: prompt, - enhancePrompt: enhance - ) - try await setImageAckState.send( - message: message, - timeout: timeoutSeconds, - sendMessage: { [weak self] outgoing in - self?.sendMessage(outgoing) - } - ) + let base64Image = referenceImageData.base64EncodedString() + Task { [weak self] in + guard let self else { return } + await self.sendImageWithPrompt( + base64Image, + prompt: prompt.text, + enhance: prompt.enrich + ) + } } func waitForConnection(timeout: TimeInterval) async throws { @@ -144,7 +139,7 @@ public extension DecartRealtimeManager { if Date().timeIntervalSince(startTime) > timeout { throw DecartError.webRTCError("Connection timeout") } - try await Task.sleep(nanoseconds: 100_000_000 * 100) // 100 seconds + try await Task.sleep(nanoseconds: 3_000_000_000) // 10 seconds } } } @@ -181,13 +176,17 @@ private extension DecartRealtimeManager { webSocketListenerTask = Task { [weak self] in do { for try await message in wsClient.websocketEventStream { - guard !Task.isCancelled, let self, let webRTCClient = self.webRTCClient else { return } + guard !Task.isCancelled, let self else { return } switch message { - case .setImageAck(let ack): - self.handleSetImageAck(ack) + case .status(let status): + self.serviceStatus = RealtimeServiceStatus.fromStatusString(status.status) + case .queuePosition(let queue): + self.queuePosition = queue.queuePosition + self.queueSize = queue.queueSize case .promptAck, .sessionId: break default: + guard let webRTCClient = self.webRTCClient else { break } try await webRTCClient.handleSignalingMessage(message) } } @@ -205,7 +204,7 @@ private extension DecartRealtimeManager { switch rtcState { case .connected: self.connectionState = .connected case .failed, .closed, .disconnected: self.connectionState = .disconnected - case .connecting where self.connectionState == .idle: self.connectionState = .connecting + case .connecting: self.connectionState = .connecting default: break } } @@ -213,77 +212,34 @@ private extension DecartRealtimeManager { } } -// MARK: - Messaging +// MARK: - Service Status private extension DecartRealtimeManager { - func sendMessage(_ message: OutgoingWebSocketMessage) { - guard let webSocketClient else { return } - Task { [webSocketClient] in try? await webSocketClient.send(message) } - } - - func handleSetImageAck(_ message: SetImageAckMessage) { - Task { [setImageAckState] in - await setImageAckState.handleAck(message: message) + func waitForServiceReady() async throws { + while serviceStatus == .enteringQueue { + try await Task.sleep(nanoseconds: 3000_000_000) // 3 seconds } } } -// MARK: - SetImageAckState Actor - -private actor SetImageAckState { - private var continuation: CheckedContinuation? - private var timeoutTask: Task? - - func send( - message: SetImageMessage, - timeout: TimeInterval, - sendMessage: (OutgoingWebSocketMessage) -> Void - ) async throws { - guard continuation == nil else { - throw DecartError.invalidOptions("setImageBase64 already in progress") - } - - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.continuation = continuation - sendMessage(.setImage(message)) +// MARK: - Messaging - timeoutTask?.cancel() - timeoutTask = Task { [weak self] in - try? await Task.sleep(nanoseconds: UInt64(timeout * 1_000_000_000)) - await self?.timeout() - } - } +private extension DecartRealtimeManager { + private func sendMessage(_ message: OutgoingWebSocketMessage) { + guard let webSocketClient else { return } + Task { [webSocketClient] in try? await webSocketClient.send(message) } } - private func timeout() { - guard let continuation else { return } - self.continuation = nil - timeoutTask = nil - continuation.resume( - throwing: DecartError.websocketError("Image send timed out") + func sendImageWithPrompt( + _ imageBase64: String?, + prompt: String, + enhance: Bool + ) async { + let message = SetImageMessage( + imageData: imageBase64, + prompt: prompt, + enhancePrompt: enhance ) - } - - func handleAck(message: SetImageAckMessage) { - guard let continuation = continuation else { return } - self.continuation = nil - timeoutTask?.cancel() - timeoutTask = nil - - if message.success { - continuation.resume() - } else { - continuation.resume( - throwing: DecartError.serverError(message.error ?? "Failed to send image") - ) - } - } - - func cancel(error: Error) { - guard let continuation = continuation else { return } - self.continuation = nil - timeoutTask?.cancel() - timeoutTask = nil - continuation.resume(throwing: error) + sendMessage(.setImage(message)) } } diff --git a/Sources/DecartSDK/Models/ModelState.swift b/Sources/DecartSDK/Realtime/Models/DecartPrompt.swift similarity index 54% rename from Sources/DecartSDK/Models/ModelState.swift rename to Sources/DecartSDK/Realtime/Models/DecartPrompt.swift index 29f30ac..0d31766 100644 --- a/Sources/DecartSDK/Models/ModelState.swift +++ b/Sources/DecartSDK/Realtime/Models/DecartPrompt.swift @@ -4,11 +4,11 @@ public struct DecartPrompt: Sendable { public let text: String public let enrich: Bool // for lucy 14b we must send a ref image with text prompt - public let referenceImageBase64: String? + public let referenceImageData: Data? - public init(text: String, referenceImageBase64: String? = nil, enrich: Bool = false) { + public init(text: String, referenceImageData: Data? = nil, enrich: Bool = false) { self.text = text - self.referenceImageBase64 = referenceImageBase64 + self.referenceImageData = referenceImageData self.enrich = enrich } } diff --git a/Sources/DecartSDK/Realtime/Models/RealtimeConnectionState.swift b/Sources/DecartSDK/Realtime/Models/RealtimeConnectionState.swift new file mode 100644 index 0000000..771a920 --- /dev/null +++ b/Sources/DecartSDK/Realtime/Models/RealtimeConnectionState.swift @@ -0,0 +1,15 @@ +public enum DecartRealtimeConnectionState: String, Sendable { + case connecting = "Connecting" + case connected = "Connected" + case disconnected = "Disconnected" + case idle = "Idle" + case error = "Error" + + public var isConnected: Bool { + self == .connected + } + + public var isInSession: Bool { + self == .connected || self == .connecting + } +} diff --git a/Sources/DecartSDK/Realtime/RealtimeDataTypes.swift b/Sources/DecartSDK/Realtime/Models/RealtimeMediaStream.swift similarity index 57% rename from Sources/DecartSDK/Realtime/RealtimeDataTypes.swift rename to Sources/DecartSDK/Realtime/Models/RealtimeMediaStream.swift index 53c784f..ccb1004 100644 --- a/Sources/DecartSDK/Realtime/RealtimeDataTypes.swift +++ b/Sources/DecartSDK/Realtime/Models/RealtimeMediaStream.swift @@ -1,9 +1,3 @@ -// -// Realtime.swift -// DecartSDK -// -// Created by Alon Bar-el on 03/11/2025. -// @preconcurrency import WebRTC public struct RealtimeMediaStream: Sendable { @@ -34,19 +28,3 @@ public struct RealtimeMediaStream: Sendable { self.id = id.id } } - -public enum DecartRealtimeConnectionState: String, Sendable { - case connecting = "Connecting" - case connected = "Connected" - case disconnected = "Disconnected" - case idle = "Idle" - case error = "Error" - - public var isConnected: Bool { - self == .connected - } - - public var isInSession: Bool { - self == .connected || self == .connecting - } -} diff --git a/Sources/DecartSDK/Realtime/Models/RealtimeServiceStatus.swift b/Sources/DecartSDK/Realtime/Models/RealtimeServiceStatus.swift new file mode 100644 index 0000000..b49e0c6 --- /dev/null +++ b/Sources/DecartSDK/Realtime/Models/RealtimeServiceStatus.swift @@ -0,0 +1,16 @@ +public enum RealtimeServiceStatus: String, Sendable { + case unknown + case enteringQueue = "Entering queue" + case ready = "Ready" + + static func fromStatusString(_ status: String) -> RealtimeServiceStatus { + let normalized = status.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if normalized.contains("ready") { + return .ready + } + if normalized.contains("entering queue") { + return .enteringQueue + } + return .unknown + } +} diff --git a/Sources/DecartSDK/Realtime/Websocket/SignalingModel.swift b/Sources/DecartSDK/Realtime/Transport/WebSocket/SignalingModel.swift similarity index 86% rename from Sources/DecartSDK/Realtime/Websocket/SignalingModel.swift rename to Sources/DecartSDK/Realtime/Transport/WebSocket/SignalingModel.swift index 7e2f392..4bca201 100644 --- a/Sources/DecartSDK/Realtime/Websocket/SignalingModel.swift +++ b/Sources/DecartSDK/Realtime/Transport/WebSocket/SignalingModel.swift @@ -110,6 +110,23 @@ struct SetImageAckMessage: Codable, Sendable { let error: String? } +struct StatusMessage: Codable, Sendable { + let type: String + let status: String +} + +struct QueuePositionMessage: Codable, Sendable { + let type: String + let queuePosition: Int? + let queueSize: Int? + + private enum CodingKeys: String, CodingKey { + case type + case queuePosition = "queue_position" + case queueSize = "queue_size" + } +} + enum IncomingWebSocketMessage: Codable, Sendable { case offer(OfferMessage) case answer(AnswerMessage) @@ -118,6 +135,8 @@ enum IncomingWebSocketMessage: Codable, Sendable { case sessionId(SessionIdMessage) case promptAck(PromptAckMessage) case setImageAck(SetImageAckMessage) + case status(StatusMessage) + case queuePosition(QueuePositionMessage) init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) @@ -138,6 +157,10 @@ enum IncomingWebSocketMessage: Codable, Sendable { self = try .promptAck(PromptAckMessage(from: decoder)) case "set_image_ack": self = try .setImageAck(SetImageAckMessage(from: decoder)) + case "status": + self = try .status(StatusMessage(from: decoder)) + case "queue_position": + self = try .queuePosition(QueuePositionMessage(from: decoder)) default: throw DecodingError.dataCorruptedError( forKey: .type, @@ -163,6 +186,10 @@ enum IncomingWebSocketMessage: Codable, Sendable { try msg.encode(to: encoder) case .setImageAck(let msg): try msg.encode(to: encoder) + case .status(let msg): + try msg.encode(to: encoder) + case .queuePosition(let msg): + try msg.encode(to: encoder) } } diff --git a/Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift b/Sources/DecartSDK/Realtime/Transport/WebSocket/WebSocketClient.swift similarity index 100% rename from Sources/DecartSDK/Realtime/Websocket/WebSocketClient.swift rename to Sources/DecartSDK/Realtime/Transport/WebSocket/WebSocketClient.swift diff --git a/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift b/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift index 8fe4b5f..d0a067b 100644 --- a/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift +++ b/Sources/DecartSDK/Realtime/WebRTC/SignalingClient.swift @@ -43,7 +43,7 @@ struct SignalingClient { case .error(let msg): throw DecartError.serverError(msg.message ?? msg.error ?? "Unknown server error") - case .sessionId, .promptAck, .setImageAck: + case .sessionId, .promptAck, .setImageAck, .status, .queuePosition: break } } diff --git a/Sources/DecartSDK/SwiftUI/RTCMLVideoViewWrapper.swift b/Sources/DecartSDK/SwiftUI/Realtime/RTCMLVideoViewWrapper.swift similarity index 100% rename from Sources/DecartSDK/SwiftUI/RTCMLVideoViewWrapper.swift rename to Sources/DecartSDK/SwiftUI/Realtime/RTCMLVideoViewWrapper.swift From 01f2f776ca916a26c1095399aab1907c5c9ba1e8 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Fri, 30 Jan 2026 18:35:35 +0200 Subject: [PATCH 19/23] change connection state to disconnected on ws disconnecnt --- Example/Example/DecartSDK/RealtimeManager.swift | 7 ++++++- Sources/DecartSDK/Realtime/DecartRealtimeManager.swift | 1 + .../Realtime/Transport/WebSocket/SignalingModel.swift | 6 +++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Example/Example/DecartSDK/RealtimeManager.swift b/Example/Example/DecartSDK/RealtimeManager.swift index 43d3030..f602fd2 100644 --- a/Example/Example/DecartSDK/RealtimeManager.swift +++ b/Example/Example/DecartSDK/RealtimeManager.swift @@ -160,7 +160,12 @@ final class RealtimeManager: RealtimeManagerProtocol { for await state in stream { if Task.isCancelled { return } - self.connectionState = state + if state == .error { + // Treat signaling (WS) disconnects as disconnected in the example UI. + self.connectionState = .error + } else { + self.connectionState = state + } } } } diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index ec657d9..6289970 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -190,6 +190,7 @@ private extension DecartRealtimeManager { try await webRTCClient.handleSignalingMessage(message) } } + self?.connectionState = .disconnected } catch { self?.connectionState = .error } diff --git a/Sources/DecartSDK/Realtime/Transport/WebSocket/SignalingModel.swift b/Sources/DecartSDK/Realtime/Transport/WebSocket/SignalingModel.swift index 4bca201..9b802ba 100644 --- a/Sources/DecartSDK/Realtime/Transport/WebSocket/SignalingModel.swift +++ b/Sources/DecartSDK/Realtime/Transport/WebSocket/SignalingModel.swift @@ -67,21 +67,21 @@ struct PromptMessage: Codable, Sendable { struct SetImageMessage: Codable, Sendable { let type: String - let imageData: String? let prompt: String? + let imageData: String? let enhancePrompt: Bool? init(imageData: String?, prompt: String? = nil, enhancePrompt: Bool? = nil) { self.type = "set_image" - self.imageData = imageData self.prompt = prompt + self.imageData = imageData self.enhancePrompt = enhancePrompt } private enum CodingKeys: String, CodingKey { case type - case imageData = "image_data" case prompt + case imageData = "image_data" case enhancePrompt = "enhance_prompt" } } From 4b7cf2a27c71e42175e98db4dc7791b7809ea2db Mon Sep 17 00:00:00 2001 From: Verion1 Date: Thu, 5 Feb 2026 10:50:43 +0200 Subject: [PATCH 20/23] exposed queue event --- .../Realtime/DecartRealtimeManager.swift | 53 +++++++++++++++---- .../Realtime/Models/DecartRealtimeState.swift | 18 +++++++ 2 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 Sources/DecartSDK/Realtime/Models/DecartRealtimeState.swift diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index 6289970..d72ee44 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -3,36 +3,70 @@ import Foundation public final class DecartRealtimeManager: @unchecked Sendable { public let options: RealtimeConfiguration - public let events: AsyncStream - public private(set) var serviceStatus: RealtimeServiceStatus = .unknown - public private(set) var queuePosition: Int? - public private(set) var queueSize: Int? + public let events: AsyncStream + public private(set) var serviceStatus: RealtimeServiceStatus = .unknown { + didSet { + guard oldValue != serviceStatus else { return } + emitStateIfChanged() + } + } + public private(set) var queuePosition: Int? { + didSet { + guard oldValue != queuePosition else { return } + emitStateIfChanged() + } + } + public private(set) var queueSize: Int? { + didSet { + guard oldValue != queueSize else { return } + emitStateIfChanged() + } + } private var webRTCClient: WebRTCClient? private var webSocketClient: WebSocketClient? private let signalingServerURL: URL - private let stateContinuation: AsyncStream.Continuation + private let stateContinuation: AsyncStream.Continuation private var webSocketListenerTask: Task? private var connectionStateListenerTask: Task? private var connectionState: DecartRealtimeConnectionState = .idle { didSet { guard oldValue != connectionState else { return } - stateContinuation.yield(connectionState) + emitStateIfChanged() } } + private var lastEmittedState: DecartRealtimeState? + private var currentState: DecartRealtimeState { + DecartRealtimeState( + connectionState: connectionState, + serviceStatus: serviceStatus, + queuePosition: queuePosition, + queueSize: queueSize + ) + } + public init(signalingServerURL: URL, options: RealtimeConfiguration) { self.signalingServerURL = signalingServerURL self.options = options let (stream, continuation) = AsyncStream.makeStream( - of: DecartRealtimeConnectionState.self, + of: DecartRealtimeState.self, bufferingPolicy: .bufferingNewest(1) ) self.events = stream self.stateContinuation = continuation + emitStateIfChanged() + } + + private func emitStateIfChanged() { + let state = currentState + if lastEmittedState != state { + lastEmittedState = state + stateContinuation.yield(state) + } } deinit { @@ -139,7 +173,7 @@ public extension DecartRealtimeManager { if Date().timeIntervalSince(startTime) > timeout { throw DecartError.webRTCError("Connection timeout") } - try await Task.sleep(nanoseconds: 3_000_000_000) // 10 seconds + try await Task.sleep(nanoseconds: 1_000_000_000) // 1 second } } } @@ -213,12 +247,13 @@ private extension DecartRealtimeManager { } } + // MARK: - Service Status private extension DecartRealtimeManager { func waitForServiceReady() async throws { while serviceStatus == .enteringQueue { - try await Task.sleep(nanoseconds: 3000_000_000) // 3 seconds + try await Task.sleep(nanoseconds: 3_000_000_000) } } } diff --git a/Sources/DecartSDK/Realtime/Models/DecartRealtimeState.swift b/Sources/DecartSDK/Realtime/Models/DecartRealtimeState.swift new file mode 100644 index 0000000..172ea10 --- /dev/null +++ b/Sources/DecartSDK/Realtime/Models/DecartRealtimeState.swift @@ -0,0 +1,18 @@ +public struct DecartRealtimeState: Sendable, Equatable { + public let connectionState: DecartRealtimeConnectionState + public let serviceStatus: RealtimeServiceStatus + public let queuePosition: Int? + public let queueSize: Int? + + public init( + connectionState: DecartRealtimeConnectionState, + serviceStatus: RealtimeServiceStatus, + queuePosition: Int?, + queueSize: Int? + ) { + self.connectionState = connectionState + self.serviceStatus = serviceStatus + self.queuePosition = queuePosition + self.queueSize = queueSize + } +} From f1de3b393fb015e025d2bf7c15188680bbc0a75a Mon Sep 17 00:00:00 2001 From: Verion1 Date: Fri, 6 Feb 2026 18:29:31 +0200 Subject: [PATCH 21/23] fix set_image prompt for lucy 14b --- Sources/DecartSDK/Realtime/DecartRealtimeManager.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift index d72ee44..248ca69 100644 --- a/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift +++ b/Sources/DecartSDK/Realtime/DecartRealtimeManager.swift @@ -144,7 +144,6 @@ public extension DecartRealtimeManager { func setPrompt(_ prompt: DecartPrompt) { guard - let referenceImageData = prompt.referenceImageData, options.model.hasReferenceImage else { // if !options.model.hasReferenceImage { @@ -153,7 +152,7 @@ public extension DecartRealtimeManager { return } - let base64Image = referenceImageData.base64EncodedString() + let base64Image = prompt.referenceImageData?.base64EncodedString() Task { [weak self] in guard let self else { return } await self.sendImageWithPrompt( From 294dbce53368c9ce9aec67dda7a97752738754e7 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Mon, 23 Feb 2026 12:10:37 +0200 Subject: [PATCH 22/23] Add macOS WebRTC capture and view support --- .github/workflows/ci.yml | 5 ++ .github/workflows/release.yml | 5 ++ Sources/DecartSDK/Capture/CameraError.swift | 2 + .../DecartSDK/Capture/CaptureExtensions.swift | 52 ++++++++++++-- .../DecartSDK/Capture/RealtimeCapture.swift | 54 ++++++++++++--- .../Realtime/RTCMLVideoViewWrapper.swift | 69 +++++++++++++++++++ 6 files changed, 175 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7d45a7..cc17484 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Build Hygiene (Reset Package State) + run: | + swift package reset + swift build -c debug + - name: Build iOS App uses: brightdigit/swift-build@v1.4.2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd20c27..d355549 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,11 @@ jobs: - name: Select Xcode run: sudo xcode-select -s /Applications/Xcode_26.1.app + - name: Build Hygiene (Reset Package State) + run: | + swift package reset + swift build -c debug + - name: Build Release uses: brightdigit/swift-build@v1.4.2 with: diff --git a/Sources/DecartSDK/Capture/CameraError.swift b/Sources/DecartSDK/Capture/CameraError.swift index 9264a73..de96537 100644 --- a/Sources/DecartSDK/Capture/CameraError.swift +++ b/Sources/DecartSDK/Capture/CameraError.swift @@ -7,6 +7,7 @@ enum CameraError: Error { case simulatorUnsupported + case noCameraDeviceAvailable case noFrontCameraDetected case noBackCameraDetected case noSupportedFormatFound @@ -16,6 +17,7 @@ enum CameraError: Error { var errorDescription: String? { switch self { case .simulatorUnsupported: return "Camera is not available on the simulator." + case .noCameraDeviceAvailable: return "No camera device is available." case .noFrontCameraDetected: return "No front camera detected." case .noSupportedFormatFound: return "No supported camera format found for the requested resolution." case .noSuitableFPSRange: return "No suitable FPS range available for the requested FPS." diff --git a/Sources/DecartSDK/Capture/CaptureExtensions.swift b/Sources/DecartSDK/Capture/CaptureExtensions.swift index 238247a..96ae3e1 100644 --- a/Sources/DecartSDK/Capture/CaptureExtensions.swift +++ b/Sources/DecartSDK/Capture/CaptureExtensions.swift @@ -8,6 +8,16 @@ import AVFoundation import WebRTC public extension AVCaptureDevice { + static func availableCameras() -> [AVCaptureDevice] { + RTCCameraVideoCapturer.captureDevices().sorted { + let nameCompare = $0.localizedName.localizedStandardCompare($1.localizedName) + if nameCompare != .orderedSame { + return nameCompare == .orderedAscending + } + return $0.uniqueID < $1.uniqueID + } + } + /// Pick a format that meets (or exceeds) the requested dimensions in either orientation. func pickFormat(minWidth: Int, minHeight: Int) throws -> AVCaptureDevice.Format { let formats = RTCCameraVideoCapturer.supportedFormats(for: self) @@ -42,11 +52,45 @@ public extension AVCaptureDevice { throw CameraError.noSuitableFPSRange } - static func pickCamera(position: AVCaptureDevice.Position) throws -> AVCaptureDevice { - let devices = RTCCameraVideoCapturer.captureDevices() - guard let front = devices.first(where: { $0.position == position }) else { + static func pickCamera( + position: AVCaptureDevice.Position, + fallbackToAny: Bool = false + ) throws -> AVCaptureDevice { + let devices = availableCameras() + guard !devices.isEmpty else { + throw CameraError.noCameraDeviceAvailable + } + + if let matchingDevice = devices.first(where: { $0.position == position }) { + return matchingDevice + } + + if fallbackToAny, let firstDevice = devices.first { + return firstDevice + } + + switch position { + case .front: throw CameraError.noFrontCameraDetected + case .back: + throw CameraError.noBackCameraDetected + default: + throw CameraError.noCameraDeviceAvailable } - return front + } + + static func nextCamera(after currentDeviceID: String?) -> AVCaptureDevice? { + let devices = availableCameras() + guard !devices.isEmpty else { return nil } + + guard + let currentDeviceID, + let currentIndex = devices.firstIndex(where: { $0.uniqueID == currentDeviceID }) + else { + return devices.first + } + + let nextIndex = (currentIndex + 1) % devices.count + return devices[nextIndex] } } diff --git a/Sources/DecartSDK/Capture/RealtimeCapture.swift b/Sources/DecartSDK/Capture/RealtimeCapture.swift index 0c983a8..5e477aa 100644 --- a/Sources/DecartSDK/Capture/RealtimeCapture.swift +++ b/Sources/DecartSDK/Capture/RealtimeCapture.swift @@ -18,6 +18,7 @@ public final class RealtimeCapture: @unchecked Sendable { private let model: ModelDefinition private let videoSource: RTCVideoSource private let capturer: RTCCameraVideoCapturer + private var activeDeviceID: String? public init( model: ModelDefinition, @@ -43,13 +44,39 @@ public final class RealtimeCapture: @unchecked Sendable { } public func startCapture() async throws { - try await startCapture(position: position) + #if os(macOS) + try await startCapture(position: position, fallbackToAny: true) + #else + try await startCapture(position: position, fallbackToAny: false) + #endif } public func switchCamera() async throws { + #if os(macOS) + let devices = AVCaptureDevice.availableCameras() + guard devices.count > 1 else { return } + + let currentDeviceID: String + if let activeDeviceID { + currentDeviceID = activeDeviceID + } else { + currentDeviceID = try AVCaptureDevice.pickCamera( + position: position, + fallbackToAny: true + ).uniqueID + } + guard let nextDevice = AVCaptureDevice.nextCamera(after: currentDeviceID) else { + throw CameraError.noCameraDeviceAvailable + } + + guard nextDevice.uniqueID != currentDeviceID else { return } + try await startCapture(with: nextDevice) + position = nextDevice.position + #else let newPosition: AVCaptureDevice.Position = position == .front ? .back : .front - try await startCapture(position: newPosition) + try await startCapture(position: newPosition, fallbackToAny: false) position = newPosition + #endif } public func stopCapture() async { @@ -62,10 +89,19 @@ public final class RealtimeCapture: @unchecked Sendable { session.outputs.forEach { session.removeOutput($0) } session.inputs.forEach { session.removeInput($0) } session.commitConfiguration() + activeDeviceID = nil } - private func startCapture(position: AVCaptureDevice.Position) async throws { - let device = try AVCaptureDevice.pickCamera(position: position) + private func startCapture( + position: AVCaptureDevice.Position, + fallbackToAny: Bool + ) async throws { + let device = try AVCaptureDevice.pickCamera(position: position, fallbackToAny: fallbackToAny) + try await startCapture(with: device) + self.position = device.position + } + + private func startCapture(with device: AVCaptureDevice) async throws { let format = try device.pickFormat(minWidth: targetWidth, minHeight: targetHeight) let targetFPS = try device.pickFPS(for: format, preferred: model.fps) @@ -76,11 +112,13 @@ public final class RealtimeCapture: @unchecked Sendable { ) try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - capturer.startCapture(with: device, format: format, fps: targetFPS) { error in - if let error { continuation.resume(throwing: error) } - else { continuation.resume() } + capturer.startCapture(with: device, format: format, fps: targetFPS) { error in + if let error { continuation.resume(throwing: error) } + else { continuation.resume() } + } } - } + + activeDeviceID = device.uniqueID } } #endif diff --git a/Sources/DecartSDK/SwiftUI/Realtime/RTCMLVideoViewWrapper.swift b/Sources/DecartSDK/SwiftUI/Realtime/RTCMLVideoViewWrapper.swift index b46ae75..33e93b7 100644 --- a/Sources/DecartSDK/SwiftUI/Realtime/RTCMLVideoViewWrapper.swift +++ b/Sources/DecartSDK/SwiftUI/Realtime/RTCMLVideoViewWrapper.swift @@ -6,6 +6,10 @@ // import SwiftUI import WebRTC +#if os(macOS) +import AppKit +import QuartzCore +#endif #if os(iOS) /// A SwiftUI View that renders a WebRTC video track. @@ -68,4 +72,69 @@ public struct RTCMLVideoViewWrapper: UIViewRepresentable { coordinator.lastTrack = nil } } +#elseif os(macOS) +/// A SwiftUI View that renders a WebRTC video track. +public struct RTCMLVideoViewWrapper: NSViewRepresentable { + public weak var track: RTCVideoTrack? + public var mirror: Bool + + /// Creates a new video view for the given track. + public init(track: RTCVideoTrack?, mirror: Bool = false) { + self.track = track + self.mirror = mirror + } + + public final class Coordinator { + weak var view: RTCMTLNSVideoView? + weak var lastTrack: RTCVideoTrack? + var lastMirror: Bool = false + + public init() {} + } + + public func makeCoordinator() -> Coordinator { + Coordinator() + } + + public func makeNSView(context: Context) -> RTCMTLNSVideoView { + let view = RTCMTLNSVideoView(frame: .zero) + applyMirrorIfPossible(view, mirror: mirror) + context.coordinator.view = view + context.coordinator.lastMirror = mirror + + if let track { + track.add(view) + context.coordinator.lastTrack = track + } + return view + } + + public func updateNSView(_ nsView: RTCMTLNSVideoView, context: Context) { + // If the track changed, rewire attachment + if context.coordinator.lastTrack !== track { + context.coordinator.lastTrack?.remove(nsView) + if let track { + track.add(nsView) + } + context.coordinator.lastTrack = track + } + + if context.coordinator.lastMirror != mirror { + applyMirrorIfPossible(nsView, mirror: mirror) + context.coordinator.lastMirror = mirror + } + } + + public static func dismantleNSView(_ nsView: RTCMTLNSVideoView, coordinator: Coordinator) { + coordinator.lastTrack?.remove(nsView) + coordinator.view = nil + coordinator.lastTrack = nil + } + + private func applyMirrorIfPossible(_ view: RTCMTLNSVideoView, mirror: Bool) { + view.wantsLayer = true + guard let layer = view.layer else { return } + layer.transform = mirror ? CATransform3DMakeScale(-1, 1, 1) : CATransform3DIdentity + } +} #endif From e121b029a839fb2bd5a7406cf549cea35eaf7b94 Mon Sep 17 00:00:00 2001 From: Verion1 Date: Mon, 23 Feb 2026 14:24:10 +0200 Subject: [PATCH 23/23] Fix CI toolchain selection before hygiene build --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc17484..9b150fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Select Xcode + run: sudo xcode-select -s /Applications/Xcode_26.1.app + - name: Build Hygiene (Reset Package State) run: | swift package reset