From 655d4fa92e72b6a52362aed0573867e12a24d05a Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 14 Jan 2026 11:02:54 +0100 Subject: [PATCH 1/7] Add FrameProcessor noiseCancellation support --- agents/package.json | 4 +- agents/src/voice/room_io/_input.ts | 6 +-- agents/src/voice/room_io/room_io.ts | 4 +- pnpm-lock.yaml | 70 +++++++++++++++++++++++++++-- 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/agents/package.json b/agents/package.json index 30efa4fc0..4c3d952d8 100644 --- a/agents/package.json +++ b/agents/package.json @@ -35,7 +35,7 @@ "api:update": "api-extractor run --local --typescript-compiler-folder ../node_modules/typescript --verbose" }, "devDependencies": { - "@livekit/rtc-node": "^0.13.22", + "@livekit/rtc-node": "^0.13.23", "@microsoft/api-extractor": "^7.35.0", "@types/fluent-ffmpeg": "^2.1.28", "@types/json-schema": "^7.0.15", @@ -79,7 +79,7 @@ "zod-to-json-schema": "^3.24.6" }, "peerDependencies": { - "@livekit/rtc-node": "^0.13.22", + "@livekit/rtc-node": "^0.13.23", "zod": "^3.25.76 || ^4.1.8" } } diff --git a/agents/src/voice/room_io/_input.ts b/agents/src/voice/room_io/_input.ts index ef8c54a0d..e499f6b0c 100644 --- a/agents/src/voice/room_io/_input.ts +++ b/agents/src/voice/room_io/_input.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { AudioFrame } from '@livekit/rtc-node'; +import type { AudioFrame, FrameProcessor } from '@livekit/rtc-node'; import { AudioStream, type NoiseCancellationOptions, @@ -21,7 +21,7 @@ export class ParticipantAudioInputStream extends AudioInput { private room: Room; private sampleRate: number; private numChannels: number; - private noiseCancellation?: NoiseCancellationOptions; + private noiseCancellation?: NoiseCancellationOptions | FrameProcessor; private publication: RemoteTrackPublication | null = null; private participantIdentity: string | null = null; private logger = log(); @@ -34,7 +34,7 @@ export class ParticipantAudioInputStream extends AudioInput { room: Room; sampleRate: number; numChannels: number; - noiseCancellation?: NoiseCancellationOptions; + noiseCancellation?: NoiseCancellationOptions | FrameProcessor; }) { super(); this.room = room; diff --git a/agents/src/voice/room_io/room_io.ts b/agents/src/voice/room_io/room_io.ts index 43a69bb31..5812821c6 100644 --- a/agents/src/voice/room_io/room_io.ts +++ b/agents/src/voice/room_io/room_io.ts @@ -2,8 +2,10 @@ // // SPDX-License-Identifier: Apache-2.0 import { + type AudioFrame, ConnectionState, DisconnectReason, + type FrameProcessor, type NoiseCancellationOptions, type Participant, ParticipantKind, @@ -74,7 +76,7 @@ export interface RoomInputOptions { Can be overridden by the `participant` argument of RoomIO constructor or `set_participant`. */ participantIdentity?: string; - noiseCancellation?: NoiseCancellationOptions; + noiseCancellation?: NoiseCancellationOptions | FrameProcessor; textInputCallback?: TextInputCallback; /** Participant kinds accepted for auto subscription. If not provided, accept `DEFAULT_PARTICIPANT_KINDS` diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1dce72646..5d6446f2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -189,8 +189,8 @@ importers: version: 3.24.6(zod@3.25.76) devDependencies: '@livekit/rtc-node': - specifier: ^0.13.22 - version: 0.13.22 + specifier: ^0.13.23 + version: 0.13.23 '@microsoft/api-extractor': specifier: ^7.35.0 version: 7.43.7(@types/node@22.15.30) @@ -1887,34 +1887,68 @@ packages: cpu: [arm64] os: [darwin] + '@livekit/rtc-node-darwin-arm64@0.13.23': + resolution: {integrity: sha512-lwe2ZZSh/nG71tyD4AhKgSNiRPmMZXReG83S6G30EssSVAbDGWe8HT1iYnuAp86W5/sH0yZlAZBuZ7tZzZxh6g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + '@livekit/rtc-node-darwin-x64@0.13.22': resolution: {integrity: sha512-ZeywKqFqmfe+BWH2KKea+s5WD/3yq93zsomYzBzzfsHGehiJMvsTbSUcznzs+eY5vRLu7zYMYHsYYa2XAJp8pw==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] + '@livekit/rtc-node-darwin-x64@0.13.23': + resolution: {integrity: sha512-YmtxDM9NOr+ArGdl8MuLOBqyfi+hwN2fksZPMA9UcrCzNKDTLccwKCFmXmjVAyRMAQu4CxP1/qj1yenwCBSQdQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + '@livekit/rtc-node-linux-arm64-gnu@0.13.22': resolution: {integrity: sha512-WVCWkImhoQC2eC+zofhbFq2HyGjkhqgTrClosrWcCvU6a1d56wMaCnVvG34xZ+30v+p/pYdE+QAP7ngdACfJjg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + '@livekit/rtc-node-linux-arm64-gnu@0.13.23': + resolution: {integrity: sha512-PpUCJgixsKe/wm+KOPoYUQn8cGB8+vrH4m8B1o9/dGp3mtK/XPMuGF0QxFa5hAQrC4S1TyqICLRxykZCLYQgBw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + '@livekit/rtc-node-linux-x64-gnu@0.13.22': resolution: {integrity: sha512-cykrWbvthtTzqi+4+3G8vN1vvs+9F/r7VZQbrXY/wExTWCnogKqyEPQtgfT3P2joOLZ9rAVT7kmUMSxkN8y+Xg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + '@livekit/rtc-node-linux-x64-gnu@0.13.23': + resolution: {integrity: sha512-fL7uvDS1gKtOI+sXD6Bas2TCUPK1XTUrZOmp/Ba0x6sl/n1TlGeyd5RAIxGaDptnsFaX42Ym0dZVxM9lSLQ3cg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + '@livekit/rtc-node-win32-x64-msvc@0.13.22': resolution: {integrity: sha512-StbHpNjRBKtti4u7f8p6r+59gaHEM2M5dekUvMM/jOd5uiEygg8UNV1UeJZlpWYlDVBK3GtY99AzoEG6SJZmbw==} engines: {node: '>= 10'} cpu: [x64] os: [win32] + '@livekit/rtc-node-win32-x64-msvc@0.13.23': + resolution: {integrity: sha512-eKDl3t96fGqG5xfxJ2Cig2pQJk1KWbj3o6azF9odlqVi8AAoHBpNTcaAFUVsejUjgDBxbqvk7VQc0Pd/HZTFgQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + '@livekit/rtc-node@0.13.22': resolution: {integrity: sha512-VkDVu+KA+pJDSQ7d4pf5SGLtZc+qDHi2zXo/zYyisfrQVO6Sjb0/0LN8IYC283YfAlOayj9BBlU6p7pP6F7o1A==} engines: {node: '>= 18'} + '@livekit/rtc-node@0.13.23': + resolution: {integrity: sha512-5/YTrV1M6fQ7iLr/SFddWRsHYsG9n0dFwlGfzaXsrZRLW6ftzg7Zrp7N4RUiknLSijyr1XQ+aKOvZGq7bbax8w==} + engines: {node: '>= 18'} + '@livekit/typed-emitter@3.0.0': resolution: {integrity: sha512-9bl0k4MgBPZu3Qu3R3xy12rmbW17e3bE9yf4YY85gJIQ3ezLEj/uzpKHWBsLaDoL5Mozz8QCgggwIBudYQWeQg==} @@ -6306,7 +6340,7 @@ snapshots: '@livekit/noise-cancellation-node@0.1.9': dependencies: - '@livekit/rtc-node': 0.13.22 + '@livekit/rtc-node': 0.13.23 tsx: 4.20.4 optionalDependencies: '@livekit/noise-cancellation-darwin-arm64': 0.1.9 @@ -6325,18 +6359,33 @@ snapshots: '@livekit/rtc-node-darwin-arm64@0.13.22': optional: true + '@livekit/rtc-node-darwin-arm64@0.13.23': + optional: true + '@livekit/rtc-node-darwin-x64@0.13.22': optional: true + '@livekit/rtc-node-darwin-x64@0.13.23': + optional: true + '@livekit/rtc-node-linux-arm64-gnu@0.13.22': optional: true + '@livekit/rtc-node-linux-arm64-gnu@0.13.23': + optional: true + '@livekit/rtc-node-linux-x64-gnu@0.13.22': optional: true + '@livekit/rtc-node-linux-x64-gnu@0.13.23': + optional: true + '@livekit/rtc-node-win32-x64-msvc@0.13.22': optional: true + '@livekit/rtc-node-win32-x64-msvc@0.13.23': + optional: true + '@livekit/rtc-node@0.13.22': dependencies: '@bufbuild/protobuf': 1.10.1 @@ -6352,6 +6401,21 @@ snapshots: '@livekit/rtc-node-linux-x64-gnu': 0.13.22 '@livekit/rtc-node-win32-x64-msvc': 0.13.22 + '@livekit/rtc-node@0.13.23': + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@datastructures-js/deque': 1.0.8 + '@livekit/mutex': 1.1.1 + '@livekit/typed-emitter': 3.0.0 + pino: 9.6.0 + pino-pretty: 13.0.0 + optionalDependencies: + '@livekit/rtc-node-darwin-arm64': 0.13.23 + '@livekit/rtc-node-darwin-x64': 0.13.23 + '@livekit/rtc-node-linux-arm64-gnu': 0.13.23 + '@livekit/rtc-node-linux-x64-gnu': 0.13.23 + '@livekit/rtc-node-win32-x64-msvc': 0.13.23 + '@livekit/typed-emitter@3.0.0': {} '@manypkg/find-root@1.1.0': From 30650d7e935bf089bdc5255bdd6140ef3f69c800 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 14 Jan 2026 11:36:30 +0100 Subject: [PATCH 2/7] fix up integration --- agents/src/voice/room_io/_input.ts | 36 ++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/agents/src/voice/room_io/_input.ts b/agents/src/voice/room_io/_input.ts index e499f6b0c..6df90c4e6 100644 --- a/agents/src/voice/room_io/_input.ts +++ b/agents/src/voice/room_io/_input.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { AudioFrame, FrameProcessor } from '@livekit/rtc-node'; +import { type AudioFrame, FrameProcessor } from '@livekit/rtc-node'; import { AudioStream, type NoiseCancellationOptions, @@ -21,7 +21,8 @@ export class ParticipantAudioInputStream extends AudioInput { private room: Room; private sampleRate: number; private numChannels: number; - private noiseCancellation?: NoiseCancellationOptions | FrameProcessor; + private noiseCancellation?: NoiseCancellationOptions; + private frameProcessor?: FrameProcessor; private publication: RemoteTrackPublication | null = null; private participantIdentity: string | null = null; private logger = log(); @@ -40,10 +41,15 @@ export class ParticipantAudioInputStream extends AudioInput { this.room = room; this.sampleRate = sampleRate; this.numChannels = numChannels; - this.noiseCancellation = noiseCancellation; + if (noiseCancellation instanceof FrameProcessor) { + this.frameProcessor = noiseCancellation; + } else { + this.noiseCancellation = noiseCancellation; + } this.room.on(RoomEvent.TrackSubscribed, this.onTrackSubscribed); this.room.on(RoomEvent.TrackUnpublished, this.onTrackUnpublished); + this.room.on(RoomEvent.TokenRefreshed, this.onTokenRefreshed); } setParticipant(participant: RemoteParticipant | string | null) { @@ -116,6 +122,9 @@ export class ParticipantAudioInputStream extends AudioInput { if (this.deferredStream.isSourceSet) { this.deferredStream.detachSource(); } + + this.frameProcessor?.close(); + this.publication = null; } @@ -140,14 +149,32 @@ export class ParticipantAudioInputStream extends AudioInput { outputRate: this.sampleRate, }), ); + this.frameProcessor?.onStreamInfoUpdated({ + participantIdentity: participant.identity, + roomName: this.room.name!, + publicationSid: publication.sid!, + }); + this.frameProcessor?.onCredentialsUpdated({ + token: this.room.token!, + url: this.room.serverUrl!, + }); return true; }; + private onTokenRefreshed = () => { + if (this.room.token && this.room.serverUrl) { + this.frameProcessor?.onCredentialsUpdated({ + token: this.room.token, + url: this.room.serverUrl, + }); + } + }; + private createStream(track: RemoteTrack): ReadableStream { return new AudioStream(track, { sampleRate: this.sampleRate, numChannels: this.numChannels, - noiseCancellation: this.noiseCancellation, + noiseCancellation: this.frameProcessor || this.noiseCancellation, // TODO(AJS-269): resolve compatibility issue with node-sdk to remove the forced type casting }) as unknown as ReadableStream; } @@ -155,6 +182,7 @@ export class ParticipantAudioInputStream extends AudioInput { async close() { this.room.off(RoomEvent.TrackSubscribed, this.onTrackSubscribed); this.room.off(RoomEvent.TrackUnpublished, this.onTrackUnpublished); + this.room.off(RoomEvent.TokenRefreshed, this.onTokenRefreshed); this.closeStream(); // Ignore errors - stream may be locked by RecorderIO or already cancelled await this.deferredStream.stream.cancel().catch(() => {}); From f9825a39151b6f994c72d7b849da5f62fc6a9bc9 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 14 Jan 2026 11:37:59 +0100 Subject: [PATCH 3/7] Add noiseCancellation support to frameProcessors --- .changeset/large-cars-pull.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/large-cars-pull.md diff --git a/.changeset/large-cars-pull.md b/.changeset/large-cars-pull.md new file mode 100644 index 000000000..1570b8afc --- /dev/null +++ b/.changeset/large-cars-pull.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents": patch +--- + +Add support for noiseCancellation frameProcessors From 01561fd299acc2354aa8ee28e3fdb3ba64f18548 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 14 Jan 2026 18:05:03 +0100 Subject: [PATCH 4/7] add aic to example --- examples/package.json | 3 ++- examples/src/basic_agent.ts | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/package.json b/examples/package.json index 531ef544e..9f473e219 100644 --- a/examples/package.json +++ b/examples/package.json @@ -39,7 +39,8 @@ "@livekit/agents-plugin-silero": "workspace:*", "@livekit/agents-plugin-xai": "workspace:*", "@livekit/noise-cancellation-node": "^0.1.9", - "@livekit/rtc-node": "^0.13.22", + "@livekit/plugins-ai-coustics": "0.1.4", + "@livekit/rtc-node": "^0.13.23", "@opentelemetry/api": "^1.9.0", "@opentelemetry/api-logs": "^0.54.0", "@opentelemetry/exporter-logs-otlp-http": "^0.54.0", diff --git a/examples/src/basic_agent.ts b/examples/src/basic_agent.ts index 6228c056b..7c6cb75d4 100644 --- a/examples/src/basic_agent.ts +++ b/examples/src/basic_agent.ts @@ -13,7 +13,7 @@ import { } from '@livekit/agents'; import * as livekit from '@livekit/agents-plugin-livekit'; import * as silero from '@livekit/agents-plugin-silero'; -import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node'; +import * as aic from '@livekit/plugins-ai-coustics'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; @@ -82,7 +82,7 @@ export default defineAgent({ agent, room: ctx.room, inputOptions: { - noiseCancellation: BackgroundVoiceCancellation(), + noiseCancellation: aic.audioEnhancement({ model: 'quailS' }), }, }); From 509ab45006a570db5e2e060a3c86a708171f51a2 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Tue, 20 Jan 2026 16:53:26 +0100 Subject: [PATCH 5/7] lint --- examples/src/inworld_tts.ts | 1 + turbo.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/src/inworld_tts.ts b/examples/src/inworld_tts.ts index bb365c4c4..fec5c552d 100644 --- a/examples/src/inworld_tts.ts +++ b/examples/src/inworld_tts.ts @@ -80,6 +80,7 @@ export default defineAgent({ }); // timestamp handling (if enabled) + // eslint-disable-next-line @typescript-eslint/no-explicit-any session.tts!.on('alignment' as any, (data: any) => { if (data.wordAlignment) { const { words, starts, ends } = data.wordAlignment; diff --git a/turbo.json b/turbo.json index 992f47edc..2ee601f20 100644 --- a/turbo.json +++ b/turbo.json @@ -52,7 +52,8 @@ "LK_OPENAI_DEBUG", "LIVEKIT_EVALS_VERBOSE", "OVHCLOUD_API_KEY", - "INWORLD_API_KEY" + "INWORLD_API_KEY", + "VITEST" ], "pipeline": { "build": { From def53515e9bed4c5eac35acfed24a6949d1be8a5 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 21 Jan 2026 13:54:43 +0100 Subject: [PATCH 6/7] format --- examples/src/basic_agent.ts | 3 ++- pnpm-lock.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/src/basic_agent.ts b/examples/src/basic_agent.ts index 84c91bb21..3efcc9ad1 100644 --- a/examples/src/basic_agent.ts +++ b/examples/src/basic_agent.ts @@ -13,6 +13,7 @@ import { } from '@livekit/agents'; import * as livekit from '@livekit/agents-plugin-livekit'; import * as silero from '@livekit/agents-plugin-silero'; +import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node'; import * as aic from '@livekit/plugins-ai-coustics'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; @@ -82,7 +83,7 @@ export default defineAgent({ agent, room: ctx.room, inputOptions: { - noiseCancellation: aic.audioEnhancement(), + noiseCancellation: BackgroundVoiceCancellation(), }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8d80b9844..935381ff8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -586,7 +586,7 @@ importers: version: 8.21.0 tsup: specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.5.6)(tsx@4.20.4)(typescript@5.9.3) + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.5.6)(tsx@4.21.0)(typescript@5.9.3) typescript: specifier: ^5.0.0 version: 5.9.3 From 4fc984ea69bfdc4787740007062f79ec3a3c5c6a Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 21 Jan 2026 13:57:34 +0100 Subject: [PATCH 7/7] update example --- examples/src/basic_agent.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/src/basic_agent.ts b/examples/src/basic_agent.ts index 3efcc9ad1..91e549bff 100644 --- a/examples/src/basic_agent.ts +++ b/examples/src/basic_agent.ts @@ -13,7 +13,7 @@ import { } from '@livekit/agents'; import * as livekit from '@livekit/agents-plugin-livekit'; import * as silero from '@livekit/agents-plugin-silero'; -import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node'; +// import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node'; import * as aic from '@livekit/plugins-ai-coustics'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; @@ -83,7 +83,9 @@ export default defineAgent({ agent, room: ctx.room, inputOptions: { - noiseCancellation: BackgroundVoiceCancellation(), + noiseCancellation: aic.audioEnhancement(), + // or for krisp use + // noiseCancellation: BackgroundVoiceCancellation(), }, });