diff --git a/Sources/LiveKit/Agent/SessionOptions.swift b/Sources/LiveKit/Agent/SessionOptions.swift index 96bddf01c..14e1a4b4f 100644 --- a/Sources/LiveKit/Agent/SessionOptions.swift +++ b/Sources/LiveKit/Agent/SessionOptions.swift @@ -32,10 +32,19 @@ public struct SessionOptions: Sendable { public init( room: Room = .init(), preConnectAudio: Bool = true, - agentConnectTimeout: TimeInterval = 20 + agentConnectTimeout: TimeInterval = 20, + audioProcessor: AudioFrameProcessor? = nil, + videoProcessor: VideoFrameProcessor? = nil ) { self.room = room self.preConnectAudio = preConnectAudio self.agentConnectTimeout = agentConnectTimeout + + if let audioProcessor { + room.add(audioFrameProcessor: audioProcessor) + } + if let videoProcessor { + room.add(videoFrameProcessor: videoProcessor) + } } } diff --git a/Sources/LiveKit/Core/Room+FrameProcessor.swift b/Sources/LiveKit/Core/Room+FrameProcessor.swift new file mode 100644 index 000000000..6460de968 --- /dev/null +++ b/Sources/LiveKit/Core/Room+FrameProcessor.swift @@ -0,0 +1,30 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +@objc +public extension Room { + func add(audioFrameProcessor: AudioFrameProcessor) { + add(delegate: audioFrameProcessor) + AudioManager.shared.capturePostProcessingDelegate = audioFrameProcessor + } + + func add(videoFrameProcessor: VideoFrameProcessor) { + add(delegate: videoFrameProcessor) + (localParticipant.firstCameraVideoTrack as? LocalVideoTrack)?.capturer.processor = videoFrameProcessor + } +} diff --git a/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift index 14284dfb1..89fb208b8 100644 --- a/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift +++ b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift @@ -21,9 +21,14 @@ internal import LiveKitWebRTC public let kLiveKitKrispAudioProcessorName = "livekit_krisp_noise_cancellation" +public typealias AudioFrameProcessor = AudioCustomProcessingDelegate & RoomDelegate + /// Used to modify audio buffers before they are sent to the network or played to the user @objc public protocol AudioCustomProcessingDelegate: Sendable { + @objc + var isEnabled: Bool { get set } + /// An optional identifier for the audio processor implementation. /// This can be used to identify different types of audio processing (e.g. noise cancellation). /// Generally you can leave this as the default value. @@ -112,7 +117,10 @@ class AudioCustomProcessingDelegateAdapter: MulticastDelegate, @u func audioProcessingProcess(audioBuffer: LKRTCAudioBuffer) { let lkAudioBuffer = LKAudioBuffer(audioBuffer: audioBuffer) - target?.audioProcessingProcess(audioBuffer: lkAudioBuffer) + + if let target, target.isEnabled { + target.audioProcessingProcess(audioBuffer: lkAudioBuffer) + } // Convert to pcmBuffer and notify only if an audioRenderer is added. if isDelegatesNotEmpty, let pcmBuffer = lkAudioBuffer.toAVAudioPCMBuffer() { diff --git a/Sources/LiveKit/Protocols/VideoProcessor.swift b/Sources/LiveKit/Protocols/VideoProcessor.swift index 8acc68b95..1487267ba 100644 --- a/Sources/LiveKit/Protocols/VideoProcessor.swift +++ b/Sources/LiveKit/Protocols/VideoProcessor.swift @@ -16,7 +16,11 @@ import Foundation +public typealias VideoFrameProcessor = RoomDelegate & VideoProcessor + @objc public protocol VideoProcessor { + var isEnabled: Bool { get set } + func process(frame: VideoFrame) -> VideoFrame? } diff --git a/Sources/LiveKit/Track/Capturers/VideoCapturer.swift b/Sources/LiveKit/Track/Capturers/VideoCapturer.swift index fcd664c91..ba819c67a 100644 --- a/Sources/LiveKit/Track/Capturers/VideoCapturer.swift +++ b/Sources/LiveKit/Track/Capturers/VideoCapturer.swift @@ -313,7 +313,7 @@ extension VideoCapturer { } // Apply processing if we have a processor attached. - if let processor = _state.processor { + if let processor = _state.processor, processor.isEnabled { guard let processedFrame = processor.process(frame: lkFrame) else { log("VideoProcessor didn't return a frame, skipping frame.", .warning) return diff --git a/Sources/LiveKit/VideoProcessors/BackgroundBlurVideoProcessor.swift b/Sources/LiveKit/VideoProcessors/BackgroundBlurVideoProcessor.swift index 6907dd1f5..3834e2d97 100644 --- a/Sources/LiveKit/VideoProcessors/BackgroundBlurVideoProcessor.swift +++ b/Sources/LiveKit/VideoProcessors/BackgroundBlurVideoProcessor.swift @@ -30,7 +30,7 @@ import os.signpost /// - Important: This class is not thread safe and will be called on a dedicated serial `processingQueue`. @available(iOS 15.0, macOS 12.0, tvOS 15.0, visionOS 1.0, *) @objc -public final class BackgroundBlurVideoProcessor: NSObject, @unchecked Sendable, VideoProcessor, Loggable { +public final class BackgroundBlurVideoProcessor: NSObject, @unchecked Sendable, VideoFrameProcessor, Loggable { #if LK_SIGNPOSTS private let signpostLog = OSLog(subsystem: Bundle.main.bundleIdentifier ?? "", category: "BackgroundBlur") #endif @@ -69,6 +69,8 @@ public final class BackgroundBlurVideoProcessor: NSObject, @unchecked Sendable, private var cachedPixelBuffer: CVPixelBuffer? private var cachedPixelBufferSize: CGSize? + public var isEnabled: Bool = true + // MARK: Init /// Initialize the background blur video processor.