From 57f518d99bf81ee250ebccd061ecce0e1ff022a3 Mon Sep 17 00:00:00 2001 From: ggolda Date: Tue, 10 Jun 2025 16:07:47 -0600 Subject: [PATCH 01/41] wip realtime highlighter --- .../realtime-highlighter-service.ts | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 app/services/highlighter/realtime-highlighter-service.ts diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts new file mode 100644 index 000000000000..d9b80b4d4fdb --- /dev/null +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -0,0 +1,221 @@ +import { InitAfter, Inject, Service } from 'services/core'; +import { EventEmitter } from 'events'; +import { EStreamingState, StreamingService } from 'services/streaming'; +import { Subscription } from 'rxjs'; +import { IAiClip } from './models/highlighter.models'; + +/** + * Just a mock class to represent a vision service events + * that would be available when it is ready by another team. + */ +class VisionService extends EventEmitter { + private timeoutId: NodeJS.Timeout | null = null; + private isRunning = false; + + constructor() { + super(); + } + + subscribe(event: string | symbol, listener: (...args: any[]) => void) { + this.on(event, listener); + } + + unsubscribe(event: string | symbol, listener: (...args: any[]) => void) { + this.removeListener(event, listener); + } + + start() { + if (this.isRunning) { + console.warn('VisionService is already running'); + return; + } + + this.isRunning = true; + console.log('Starting VisionService'); + this.scheduleNext(); + } + + stop() { + if (!this.isRunning) { + console.warn('VisionService is not running'); + return; + } + + this.isRunning = false; + console.log('Stopping VisionService'); + if (this.timeoutId) { + clearTimeout(this.timeoutId); + this.timeoutId = null; + } + } + + private scheduleNext(): void { + const maxDelay = 30 * 1000; + const minDelay = 5 * 1000; + + const delay = Math.floor(Math.random() * (maxDelay - minDelay + 1)) + minDelay; + console.log(`Next trigger in ${delay / 1000} seconds`); + + this.timeoutId = setTimeout(() => { + this.emitRandomEvent(); + this.scheduleNext(); + }, delay); + } + + private emitRandomEvent(): void { + // const events = ['elimination', 'deploy', 'game_start', 'game_end']; + const events = { + elimination: { + highlight: { start_adjust: 9, end_adjust: 4, score: 3 }, + }, + knockout: { + highlight: { start_adjust: 9, end_adjust: 4, score: 3 }, + }, + victory: { + highlight: { start_adjust: 7, end_adjust: 5, score: 5 }, + }, + defeat: { + highlight: { start_adjust: 7, end_adjust: 5, score: 3 }, + }, + death: { + highlight: { start_adjust: 7, end_adjust: 5, score: 3 }, + }, + deploy: { + highlight: { start_adjust: 5, end_adjust: 5, score: 2 }, + }, + } as const; + + type EventKey = keyof typeof events; + const eventKeys = Object.keys(events) as EventKey[]; + const randomEvent = eventKeys[Math.floor(Math.random() * eventKeys.length)]; + + console.log(`Emitting event: ${randomEvent}`); + const settings = events[randomEvent]; + + this.emit('event', { + name: randomEvent, + timestamp: new Date().toISOString(), + highlight: settings.highlight, + }); + } +} + +@InitAfter('StreamingService') +export class RealtimeHighlighterService extends Service { + @Inject() streamingService: StreamingService; + visionService = new VisionService(); + + private isRunning = false; + private highlights: IAiClip[] = []; + + private replayBufferFileReadySubscription: Subscription | null = null; + + // timestamp of when the replay should be saved after the event was received + private saveReplayAt: number | null = null; + // events that are currently being observer in the replay buffer window + // (in case there are multiple events in a row that should land in the same replay) + private currentReplayEvents: any[] = []; + + async start() { + if (this.isRunning) { + console.warn('RealtimeHighlighterService is already running'); + return; + } + this.isRunning = true; + // start replay buffer if its not already running + this.streamingService.startReplayBuffer(); + this.replayBufferFileReadySubscription = this.streamingService.replayBufferFileWrite.subscribe( + this.onReplayReady.bind(this), + ); + + this.saveReplayAt = null; + this.currentReplayEvents = []; + + this.visionService.subscribe('event', this.onEvent.bind(this)); + this.visionService.start(); + + // start the periodic tick to process replay queue + this.tick(); + } + + async stop() { + // don't stop replay buffer here, probably better places for it exist + this.visionService.unsubscribe('event', this.onEvent.bind(this)); + this.visionService.stop(); + + this.replayBufferFileReadySubscription?.unsubscribe(); + + this.isRunning = false; + } + + /** + * This method is called periodically to save replay events to file at correct time + * when the highlight ends. + */ + private async tick() { + if (!this.saveReplayAt) { + return; + } + + const now = Date.now(); + if (now >= this.saveReplayAt) { + // save the replay events to file + if (this.currentReplayEvents.length > 0) { + console.log('Saving replay buffer'); + this.streamingService.saveReplay(); + } + + // reset the save time + this.saveReplayAt = null; + } + + if (!this.isRunning) { + return; + } + + // call this method again in 1 second. + // setTimeout instead of setInterval to avoid overlapping calls + setTimeout(() => this.tick(), 1000); + } + + private onEvent(event: any) { + // ignore events that have no highlight data + if (!event.highlight) { + return; + } + + const endAdjust = event.highlight.end_adjust || 0; + + this.saveReplayAt = Date.now() + endAdjust * 1000; + this.currentReplayEvents.push(event); + } + + private onReplayReady(path: string) { + const events = this.currentReplayEvents; + this.currentReplayEvents = []; + + const clip: IAiClip = { + path, + aiInfo: { + inputs: events.map(event => ({ + type: event.name, + })), + score: events.reduce((acc, event) => acc + (event.highlight.score || 0), 0), + metadata: { + round: 0, + webcam_coordinates: undefined, + }, + }, + enabled: true, + loaded: true, + deleted: false, + source: 'AiClip', + startTrim: 0, + endTrim: 0, + globalOrderPosition: 0, + streamInfo: {}, + }; + this.highlights.push(clip); + console.log(`New highlight added: ${clip.path}`); + } +} From 8a25c66aee5804d1cebae691ed7e3f60a47059f3 Mon Sep 17 00:00:00 2001 From: ggolda Date: Tue, 10 Jun 2025 16:23:49 -0600 Subject: [PATCH 02/41] added realtime service into highlighter service --- app/services/highlighter/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index d0d9a2b15342..fb0fab558447 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -78,6 +78,7 @@ import { extractDateTimeFromPath, fileExists } from './file-utils'; import { addVerticalFilterToExportOptions } from './vertical-export'; import { isGameSupported } from './models/game-config.models'; import Utils from 'services/utils'; +import { RealtimeHighlighterService } from './realtime-highlighter-service'; @InitAfter('StreamingService') export class HighlighterService extends PersistentStatefulService { @@ -90,6 +91,7 @@ export class HighlighterService extends PersistentStatefulService Date: Wed, 11 Jun 2025 15:53:11 -0600 Subject: [PATCH 03/41] wip --- app/app-services.ts | 3 + app/services/highlighter/index.ts | 39 +++-- .../realtime-highlighter-service.ts | 161 ++++++++++++++---- package.json | 2 +- 4 files changed, 162 insertions(+), 43 deletions(-) diff --git a/app/app-services.ts b/app/app-services.ts index 368f2c7ff359..651cdd621ad3 100644 --- a/app/app-services.ts +++ b/app/app-services.ts @@ -62,6 +62,7 @@ export { VideoSettingsService } from 'services/settings-v2/video'; export { SettingsManagerService } from 'services/settings-manager'; export { MarkersService } from 'services/markers'; export { RealmService } from 'services/realm'; +export { RealtimeHighlighterService } from 'services/highlighter/realtime-highlighter-service'; import { VirtualWebcamService } from 'services/virtual-webcam'; // ONLINE SERVICES @@ -164,6 +165,7 @@ import { PatchNotesService } from './services/patch-notes'; import { VideoService } from './services/video'; import { ChatService } from './services/chat'; import { HighlighterService } from './services/highlighter'; +import { RealtimeHighlighterService } from './services/highlighter/realtime-highlighter-service'; import { GrowService } from './services/grow/grow'; import { TransitionsService } from './services/transitions'; import { TcpServerService } from './services/api/tcp-server'; @@ -293,4 +295,5 @@ export const AppServices = { RemoteControlService, UrlService, VirtualWebcamService, + RealtimeHighlighterService, }; diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index fb0fab558447..d654d7a77753 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -403,21 +403,33 @@ export class HighlighterService extends PersistentStatefulService { - const streamId = streamInfo?.id || undefined; - let endTime: number | undefined; + // add check if ai realtime highlighter is enabled? + // if yes add hook to realtime highlighter service? + console.log('feature enabled: ', this.aiHighlighterFeatureEnabled); + if (true) { + console.log('AI Highlighter feature is enabled, using realtime highlighter service'); + this.realtimeHighlighterService.highlightsReady.subscribe(async highlights => { + console.log('Realtime highlights received:', highlights); + this.addAiClips(highlights, { id: streamInfo.id || '', game: streamInfo.game }); + }); + } else { + this.streamingService.replayBufferFileWrite.subscribe(async clipPath => { + console.log('replay buffer clip received'); + const streamId = streamInfo?.id || undefined; + let endTime: number | undefined; - if (streamId) { - endTime = moment().diff(aiRecordingStartTime, 'seconds'); - } else { - endTime = undefined; - } + if (streamId) { + endTime = moment().diff(aiRecordingStartTime, 'seconds'); + } else { + endTime = undefined; + } - const REPLAY_BUFFER_DURATION = 20; // TODO M: Replace with settingsservice - const startTime = Math.max(0, endTime ? endTime - REPLAY_BUFFER_DURATION : 0); + const REPLAY_BUFFER_DURATION = 20; // TODO M: Replace with settingsservice + const startTime = Math.max(0, endTime ? endTime - REPLAY_BUFFER_DURATION : 0); - this.addClips([{ path: clipPath, startTime, endTime }], streamId, 'ReplayBuffer'); - }); + this.addClips([{ path: clipPath, startTime, endTime }], streamId, 'ReplayBuffer'); + }); + } this.streamingService.streamingStatusChange.subscribe(async status => { if (status === EStreamingState.Live) { @@ -744,6 +756,9 @@ export class HighlighterService extends PersistentStatefulService { - this.emitRandomEvent(); this.scheduleNext(); }, delay); } @@ -102,27 +106,34 @@ class VisionService extends EventEmitter { @InitAfter('StreamingService') export class RealtimeHighlighterService extends Service { - @Inject() streamingService: StreamingService; - visionService = new VisionService(); + highlightsReady = new Subject(); + + @Inject() private streamingService: StreamingService; + @Inject() private settingsService: SettingsService; + private visionService = new VisionService(); private isRunning = false; - private highlights: IAiClip[] = []; + private highlights: INewClipData[] = []; private replayBufferFileReadySubscription: Subscription | null = null; // timestamp of when the replay should be saved after the event was received private saveReplayAt: number | null = null; + private replaySavedAt: number | null = null; // events that are currently being observer in the replay buffer window // (in case there are multiple events in a row that should land in the same replay) private currentReplayEvents: any[] = []; async start() { + console.log('Starting RealtimeHighlighterService'); if (this.isRunning) { console.warn('RealtimeHighlighterService is already running'); return; } + this.isRunning = true; // start replay buffer if its not already running + this.setReplayBufferDurationSeconds(30); this.streamingService.startReplayBuffer(); this.replayBufferFileReadySubscription = this.streamingService.replayBufferFileWrite.subscribe( this.onReplayReady.bind(this), @@ -134,11 +145,16 @@ export class RealtimeHighlighterService extends Service { this.visionService.subscribe('event', this.onEvent.bind(this)); this.visionService.start(); - // start the periodic tick to process replay queue - this.tick(); + // start the periodic tick to process replay queue after first replay buffer duration + setTimeout(() => this.tick(), this.getReplayBufferDurationSeconds() * 1000); } async stop() { + console.log('Stopping RealtimeHighlighterService'); + if (!this.isRunning) { + console.warn('RealtimeHighlighterService is not running'); + return; + } // don't stop replay buffer here, probably better places for it exist this.visionService.unsubscribe('event', this.onEvent.bind(this)); this.visionService.stop(); @@ -154,6 +170,9 @@ export class RealtimeHighlighterService extends Service { */ private async tick() { if (!this.saveReplayAt) { + // call this method again in 1 second. + // setTimeout instead of setInterval to avoid overlapping calls + setTimeout(() => this.tick(), 1000); return; } @@ -162,6 +181,7 @@ export class RealtimeHighlighterService extends Service { // save the replay events to file if (this.currentReplayEvents.length > 0) { console.log('Saving replay buffer'); + this.replaySavedAt = now; this.streamingService.saveReplay(); } @@ -187,35 +207,116 @@ export class RealtimeHighlighterService extends Service { const endAdjust = event.highlight.end_adjust || 0; this.saveReplayAt = Date.now() + endAdjust * 1000; + const currentTime = Date.now(); + + event.timestamp = currentTime; // use current time as timestamp this.currentReplayEvents.push(event); } private onReplayReady(path: string) { + console.log(this.getReplayBufferDurationSeconds()); const events = this.currentReplayEvents; + if (events.length === 0) { + return; + } + this.currentReplayEvents = []; - const clip: IAiClip = { - path, - aiInfo: { - inputs: events.map(event => ({ - type: event.name, - })), - score: events.reduce((acc, event) => acc + (event.highlight.score || 0), 0), + const replayBufferDuration = this.getReplayBufferDurationSeconds(); + + // absolute time in milliseconds when the replay was saved + const replaySavedAt = this.replaySavedAt; + this.replaySavedAt = null; + const replayStartedAt = replaySavedAt - replayBufferDuration * 1000; + + + const unrefinedHighlights = []; + + for (const event of events) { + const eventTime = event.timestamp; + + const relativeEventTime = eventTime - replayStartedAt; + const highlightStart = relativeEventTime - (event.highlight.start_adjust || 0) * 1000; + const highlightEnd = relativeEventTime + (event.highlight.end_adjust || 0) * 1000; + + // check if the highlight is within the replay buffer duration + if (highlightStart < 0 || highlightEnd > replayBufferDuration * 1000) { + console.warn( + `Event ${event.name} is outside of the replay buffer duration, skipping highlight creation.` + ); + continue; + } + + unrefinedHighlights.push({ + inputs: [event.name], + startTime: 0, // convert to seconds + endTime: replayBufferDuration, // convert to seconds + score: event.highlight.score || 0, + }); + } + + // merge overlapping highlights + const acceptableOffset = 5; // seconds + + const mergedHighlights: any[] = []; + for (const highlight of unrefinedHighlights) { + if (mergedHighlights.length === 0) { + mergedHighlights.push(highlight); + continue; + } + + const lastHighlight = mergedHighlights[mergedHighlights.length - 1]; + if (highlight.startTime - acceptableOffset <= lastHighlight.endTime) { + // merge highlights + lastHighlight.endTime = highlight.endTime; // extend end time + lastHighlight.score = Math.max(highlight.score, lastHighlight.score); + lastHighlight.inputs.push(...highlight.inputs); + } else { + // no overlap, push new highlight + mergedHighlights.push(highlight); + } + } + + const clips = []; + for (const highlight of mergedHighlights) { + const aiClipInfo: IAiClipInfo = { + inputs: highlight.inputs.map((input: string) => ({ type: input }) as IInput), + score: highlight.score, metadata: { - round: 0, - webcam_coordinates: undefined, + round: 0, // Placeholder, adjust as needed + webcam_coordinates: undefined, // Placeholder, adjust as needed }, - }, - enabled: true, - loaded: true, - deleted: false, - source: 'AiClip', - startTrim: 0, - endTrim: 0, - globalOrderPosition: 0, - streamInfo: {}, - }; - this.highlights.push(clip); - console.log(`New highlight added: ${clip.path}`); + }; + + const clip: INewClipData = { + path, + aiClipInfo: aiClipInfo, + startTime: 0, + endTime: this.getReplayBufferDurationSeconds(), + startTrim: highlight.startTime, + endTrim: highlight.endTime, + }; + + this.highlights.push(clip); + console.log(`New highlight added: ${clip.path}`); + console.log(this.highlights); + clips.push(clip); + } + + this.highlightsReady.next(clips); + } + + private getReplayBufferDurationSeconds(): number { + return this.settingsService.views.values.Output.RecRBTime; + } + + private setReplayBufferDurationSeconds(seconds: number) { + if (this.streamingService.state.replayBufferStatus !== EReplayBufferState.Offline) { + console.warn( + 'Replay buffer must be stopped before its settings can be changed!' + ); + return; + } + this.settingsService.setSettingsPatch({ Output: { RecRBTime: seconds } }); } } diff --git a/package.json b/package.json index f4f3c43c8dcf..915c7b275118 100644 --- a/package.json +++ b/package.json @@ -251,4 +251,4 @@ "got@^9.6.0": "11.8.5" }, "packageManager": "yarn@3.1.1" -} \ No newline at end of file +} From ca2a17285099a6812dabf73ab6839c48452546c2 Mon Sep 17 00:00:00 2001 From: ggolda Date: Wed, 11 Jun 2025 16:47:26 -0600 Subject: [PATCH 04/41] wip --- .../realtime-highlighter-service.ts | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index 17c9d7027d35..5e8670503ac8 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -112,6 +112,8 @@ export class RealtimeHighlighterService extends Service { @Inject() private settingsService: SettingsService; private visionService = new VisionService(); + private static MAX_SCORE = 5; + private isRunning = false; private highlights: INewClipData[] = []; @@ -141,6 +143,7 @@ export class RealtimeHighlighterService extends Service { this.saveReplayAt = null; this.currentReplayEvents = []; + this.highlights = []; this.visionService.subscribe('event', this.onEvent.bind(this)); this.visionService.start(); @@ -228,7 +231,9 @@ export class RealtimeHighlighterService extends Service { const replaySavedAt = this.replaySavedAt; this.replaySavedAt = null; const replayStartedAt = replaySavedAt - replayBufferDuration * 1000; - + console.log(`Replay saved at ${moment(replaySavedAt).format('YYYY-MM-DD HH:mm:ss')}`); + console.log(`Replay started at ${moment(replayStartedAt).format('YYYY-MM-DD HH:mm:ss')}`); + console.log(`Replay buffer duration: ${replayBufferDuration} seconds`); const unrefinedHighlights = []; @@ -236,9 +241,13 @@ export class RealtimeHighlighterService extends Service { const eventTime = event.timestamp; const relativeEventTime = eventTime - replayStartedAt; + console.log( + `Processing event ${event.name} at ${relativeEventTime / 1000}s relative to replay start` + ); const highlightStart = relativeEventTime - (event.highlight.start_adjust || 0) * 1000; const highlightEnd = relativeEventTime + (event.highlight.end_adjust || 0) * 1000; + console.log(`Highlight start: ${highlightStart / 1000}s, Highlight end: ${highlightEnd / 1000}s`); // check if the highlight is within the replay buffer duration if (highlightStart < 0 || highlightEnd > replayBufferDuration * 1000) { console.warn( @@ -249,12 +258,14 @@ export class RealtimeHighlighterService extends Service { unrefinedHighlights.push({ inputs: [event.name], - startTime: 0, // convert to seconds - endTime: replayBufferDuration, // convert to seconds + startTime: highlightStart / 1000, // convert to seconds + endTime: highlightEnd / 1000, // convert to seconds score: event.highlight.score || 0, }); } + console.log('Unrefined highlights:', unrefinedHighlights); + // merge overlapping highlights const acceptableOffset = 5; // seconds @@ -281,25 +292,29 @@ export class RealtimeHighlighterService extends Service { for (const highlight of mergedHighlights) { const aiClipInfo: IAiClipInfo = { inputs: highlight.inputs.map((input: string) => ({ type: input }) as IInput), - score: highlight.score, + score: Math.round(highlight.score / RealtimeHighlighterService.MAX_SCORE), metadata: { round: 0, // Placeholder, adjust as needed webcam_coordinates: undefined, // Placeholder, adjust as needed }, }; + // trim times for desktop are insanely weird, for some reason its offset between start and end + const startTrim = highlight.startTime; + const endTrim = highlight.endTime - startTrim; + const clip: INewClipData = { path, aiClipInfo: aiClipInfo, startTime: 0, endTime: this.getReplayBufferDurationSeconds(), - startTrim: highlight.startTime, - endTrim: highlight.endTime, + startTrim: startTrim, + endTrim: endTrim, }; this.highlights.push(clip); console.log(`New highlight added: ${clip.path}`); - console.log(this.highlights); + console.log(clip); clips.push(clip); } From d8ada4c1c317bd73769e176a39628a6a7fb6d6e8 Mon Sep 17 00:00:00 2001 From: ggolda Date: Wed, 11 Jun 2025 16:50:49 -0600 Subject: [PATCH 05/41] start fake vision service only after 30 seconds --- app/services/highlighter/realtime-highlighter-service.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index 5e8670503ac8..02e2db163cdf 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -146,10 +146,14 @@ export class RealtimeHighlighterService extends Service { this.highlights = []; this.visionService.subscribe('event', this.onEvent.bind(this)); - this.visionService.start(); + setTimeout(() => { + this.visionService.start(); + } + , 1000 * this.getReplayBufferDurationSeconds()); + // start the periodic tick to process replay queue after first replay buffer duration - setTimeout(() => this.tick(), this.getReplayBufferDurationSeconds() * 1000); + this.tick(); } async stop() { From bbf270c3239803964b178b16e4fc3e268433ff63 Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Thu, 12 Jun 2025 16:10:40 +0200 Subject: [PATCH 06/41] to fix negative video duration --- .../highlighter/ClipPreview.tsx | 4 ++++ .../realtime-highlighter-service.ts | 22 ++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/components-react/highlighter/ClipPreview.tsx b/app/components-react/highlighter/ClipPreview.tsx index c4e29c5cc46e..f8be3941d0c8 100644 --- a/app/components-react/highlighter/ClipPreview.tsx +++ b/app/components-react/highlighter/ClipPreview.tsx @@ -46,6 +46,10 @@ export default function ClipPreview(props: { return (
+ Duration: {JSON.stringify(v.clip.duration)}
+ CLIPS: {JSON.stringify(v.clip.startTrim)} - {JSON.stringify(v.clip.endTrim)} ={' '} + {JSON.stringify(v.clip.duration! - (v.clip.startTrim + v.clip.endTrim))} +
{!v.clip.deleted && ( { this.visionService.start(); - } - , 1000 * this.getReplayBufferDurationSeconds()); - + }, 1000 * this.getReplayBufferDurationSeconds()); // start the periodic tick to process replay queue after first replay buffer duration this.tick(); @@ -245,17 +243,17 @@ export class RealtimeHighlighterService extends Service { const eventTime = event.timestamp; const relativeEventTime = eventTime - replayStartedAt; - console.log( - `Processing event ${event.name} at ${relativeEventTime / 1000}s relative to replay start` - ); const highlightStart = relativeEventTime - (event.highlight.start_adjust || 0) * 1000; const highlightEnd = relativeEventTime + (event.highlight.end_adjust || 0) * 1000; - console.log(`Highlight start: ${highlightStart / 1000}s, Highlight end: ${highlightEnd / 1000}s`); // check if the highlight is within the replay buffer duration if (highlightStart < 0 || highlightEnd > replayBufferDuration * 1000) { console.warn( - `Event ${event.name} is outside of the replay buffer duration, skipping highlight creation.` + `Event ${ + event.name + } is outside of the replay buffer duration, skipping highlight creation. highlightStart: ${highlightStart}, highlightEnd: ${highlightEnd}, replayBufferDuration: ${ + replayBufferDuration * 1000 + } ms`, ); continue; } @@ -295,7 +293,7 @@ export class RealtimeHighlighterService extends Service { const clips = []; for (const highlight of mergedHighlights) { const aiClipInfo: IAiClipInfo = { - inputs: highlight.inputs.map((input: string) => ({ type: input }) as IInput), + inputs: highlight.inputs.map((input: string) => ({ type: input } as IInput)), score: Math.round(highlight.score / RealtimeHighlighterService.MAX_SCORE), metadata: { round: 0, // Placeholder, adjust as needed @@ -305,7 +303,7 @@ export class RealtimeHighlighterService extends Service { // trim times for desktop are insanely weird, for some reason its offset between start and end const startTrim = highlight.startTime; - const endTrim = highlight.endTime - startTrim; + const endTrim = this.getReplayBufferDurationSeconds() - highlight.endTime; const clip: INewClipData = { path, @@ -331,9 +329,7 @@ export class RealtimeHighlighterService extends Service { private setReplayBufferDurationSeconds(seconds: number) { if (this.streamingService.state.replayBufferStatus !== EReplayBufferState.Offline) { - console.warn( - 'Replay buffer must be stopped before its settings can be changed!' - ); + console.warn('Replay buffer must be stopped before its settings can be changed!'); return; } this.settingsService.setSettingsPatch({ Output: { RecRBTime: seconds } }); From acca673272aad82ac776e3767782114fe3409330 Mon Sep 17 00:00:00 2001 From: ggolda Date: Thu, 12 Jun 2025 14:33:36 -0600 Subject: [PATCH 07/41] wip --- app/services/highlighter/index.ts | 1 + .../realtime-highlighter-service.ts | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index d654d7a77753..7f9f66ebf799 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -410,6 +410,7 @@ export class HighlighterService extends PersistentStatefulService { console.log('Realtime highlights received:', highlights); + console.log(streamInfo.id); this.addAiClips(highlights, { id: streamInfo.id || '', game: streamInfo.game }); }); } else { diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index c059bdd3c89c..16eae1a3a1be 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -2,10 +2,11 @@ import { InitAfter, Inject, Service } from 'services/core'; import { EventEmitter } from 'events'; import { EReplayBufferState, StreamingService } from 'services/streaming'; import { Subject, Subscription } from 'rxjs'; -import { IAiClip, INewClipData } from './models/highlighter.models'; +import { INewClipData } from './models/highlighter.models'; import { IAiClipInfo, IInput } from './models/ai-highlighter.models'; import { SettingsService } from 'app-services'; import moment from 'moment'; +import { getVideoDuration } from './cut-highlight-clips'; /** * Just a mock class to represent a vision service events @@ -218,20 +219,21 @@ export class RealtimeHighlighterService extends Service { this.currentReplayEvents.push(event); } - private onReplayReady(path: string) { - console.log(this.getReplayBufferDurationSeconds()); + private async onReplayReady(path: string) { const events = this.currentReplayEvents; if (events.length === 0) { return; } - this.currentReplayEvents = []; - const replayBufferDuration = this.getReplayBufferDurationSeconds(); + const replayBufferDuration = + (await getVideoDuration(path)) || this.getReplayBufferDurationSeconds(); + console.log(`Replay buffer duration: ${replayBufferDuration} seconds`); // absolute time in milliseconds when the replay was saved const replaySavedAt = this.replaySavedAt; this.replaySavedAt = null; + const replayStartedAt = replaySavedAt - replayBufferDuration * 1000; console.log(`Replay saved at ${moment(replaySavedAt).format('YYYY-MM-DD HH:mm:ss')}`); console.log(`Replay started at ${moment(replayStartedAt).format('YYYY-MM-DD HH:mm:ss')}`); @@ -307,11 +309,11 @@ export class RealtimeHighlighterService extends Service { const clip: INewClipData = { path, - aiClipInfo: aiClipInfo, + aiClipInfo, startTime: 0, endTime: this.getReplayBufferDurationSeconds(), - startTrim: startTrim, - endTrim: endTrim, + startTrim, + endTrim, }; this.highlights.push(clip); From cf2d24c6bb4a30a231216083a1fd4e28e7e53c1e Mon Sep 17 00:00:00 2001 From: ggolda Date: Thu, 12 Jun 2025 16:21:08 -0600 Subject: [PATCH 08/41] wip --- app/components-react/highlighter/ClipPreview.tsx | 4 ---- app/services/highlighter/models/ai-highlighter.models.ts | 4 ++-- app/services/highlighter/realtime-highlighter-service.ts | 9 +++------ 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/app/components-react/highlighter/ClipPreview.tsx b/app/components-react/highlighter/ClipPreview.tsx index f8be3941d0c8..c4e29c5cc46e 100644 --- a/app/components-react/highlighter/ClipPreview.tsx +++ b/app/components-react/highlighter/ClipPreview.tsx @@ -46,10 +46,6 @@ export default function ClipPreview(props: { return (
- Duration: {JSON.stringify(v.clip.duration)}
- CLIPS: {JSON.stringify(v.clip.startTrim)} - {JSON.stringify(v.clip.endTrim)} ={' '} - {JSON.stringify(v.clip.duration! - (v.clip.startTrim + v.clip.endTrim))} -
{!v.clip.deleted && ( ({ type: input } as IInput)), score: Math.round(highlight.score / RealtimeHighlighterService.MAX_SCORE), - metadata: { - round: 0, // Placeholder, adjust as needed - webcam_coordinates: undefined, // Placeholder, adjust as needed - }, + metadata: {}, }; // trim times for desktop are insanely weird, for some reason its offset between start and end From 255004eb899d2c7cc2a10a86e1ce9d0bd10a92c9 Mon Sep 17 00:00:00 2001 From: ggolda Date: Thu, 12 Jun 2025 16:33:38 -0600 Subject: [PATCH 09/41] wip --- app/services/highlighter/realtime-highlighter-service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index 2ffe1737aa39..e2a15f57b0b5 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -302,13 +302,13 @@ export class RealtimeHighlighterService extends Service { // trim times for desktop are insanely weird, for some reason its offset between start and end const startTrim = highlight.startTime; - const endTrim = this.getReplayBufferDurationSeconds() - highlight.endTime; + const endTrim = replayBufferDuration - highlight.endTime; const clip: INewClipData = { path, aiClipInfo, startTime: 0, - endTime: this.getReplayBufferDurationSeconds(), + endTime: replayBufferDuration, startTrim, endTrim, }; From a527fa38eed62f96a320759e457bae40dce65687 Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Fri, 13 Jun 2025 13:48:43 +0200 Subject: [PATCH 10/41] WIP: fixes error bcs round is now optional --- app/components-react/highlighter/utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/components-react/highlighter/utils.ts b/app/components-react/highlighter/utils.ts index 6e53d6adefb2..66da04ecd484 100644 --- a/app/components-react/highlighter/utils.ts +++ b/app/components-react/highlighter/utils.ts @@ -93,13 +93,14 @@ export function aiFilterClips( ): TClip[] { const { rounds, targetDuration, includeAllEvents } = options; + // TODO: this just handles the error for now, probably not the best way. const selectedRounds = rounds.length === 1 && rounds[0] === 0 ? [ ...new Set( clips .filter(clip => clip.source === 'AiClip') - .map(clip => (clip as IAiClip).aiInfo.metadata?.round), + .map(clip => (clip as IAiClip).aiInfo.metadata?.round || 0), ), ] : rounds; From 35f92e2a6c6d4488426f3f8a88109adb986a242d Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Fri, 13 Jun 2025 13:49:03 +0200 Subject: [PATCH 11/41] added new DetectionState --- .../highlighter/models/ai-highlighter.models.ts | 1 + .../highlighter/models/highlighter.models.ts | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/app/services/highlighter/models/ai-highlighter.models.ts b/app/services/highlighter/models/ai-highlighter.models.ts index ab3f8b7b208b..202b62a28003 100644 --- a/app/services/highlighter/models/ai-highlighter.models.ts +++ b/app/services/highlighter/models/ai-highlighter.models.ts @@ -113,6 +113,7 @@ export interface IInput { export enum EAiDetectionState { INITIALIZED = 'initialized', IN_PROGRESS = 'detection-in-progress', + REALTIME_DETECTION_IN_PROGRESS = 'realtime-detection-in-progress', ERROR = 'error', FINISHED = 'detection-finished', CANCELED_BY_USER = 'detection-canceled-by-user', diff --git a/app/services/highlighter/models/highlighter.models.ts b/app/services/highlighter/models/highlighter.models.ts index 1ab48b75175c..7cf91fb32369 100644 --- a/app/services/highlighter/models/highlighter.models.ts +++ b/app/services/highlighter/models/highlighter.models.ts @@ -93,17 +93,24 @@ export interface IStreamMilestones { streamId: string; milestones: IHighlighterMilestone[]; } +type IHighlightedStreamState = + | { type: EAiDetectionState.IN_PROGRESS; progress: number } + | { type: EAiDetectionState.REALTIME_DETECTION_IN_PROGRESS; count: number } + | { + type: Exclude< + EAiDetectionState, + EAiDetectionState.IN_PROGRESS | EAiDetectionState.REALTIME_DETECTION_IN_PROGRESS + >; + }; + export interface IHighlightedStream { id: string; game: EGame; title: string; date: string; - state: { - type: EAiDetectionState; - progress: number; - }; + state: IHighlightedStreamState; abortController?: AbortController; - path: string; + path?: string; feedbackLeft?: boolean; } From 3734edf620a02bef3d79c496a631b05a7ac4a0e9 Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Fri, 13 Jun 2025 14:46:26 +0200 Subject: [PATCH 12/41] updates stream, wich updates the streamCard --- .../highlighter/StreamCard.m.less | 9 ++ .../highlighter/StreamCard.tsx | 41 ++++++++- app/services/highlighter/index.ts | 86 +++++++++++++++---- 3 files changed, 116 insertions(+), 20 deletions(-) diff --git a/app/components-react/highlighter/StreamCard.m.less b/app/components-react/highlighter/StreamCard.m.less index 0860b18049f6..7a808e1820de 100644 --- a/app/components-react/highlighter/StreamCard.m.less +++ b/app/components-react/highlighter/StreamCard.m.less @@ -96,6 +96,15 @@ z-index: 1; } +.realtime-detection-action { + display: inline-flex; + padding: 6px 11px; + align-items: center; + gap: 8px; + border-radius: 48px; + border: 1px solid #301e24; + background: #301e24; +} .delete-button { display: flex; top: 8px; diff --git a/app/components-react/highlighter/StreamCard.tsx b/app/components-react/highlighter/StreamCard.tsx index 54b5431b636d..018d7b4078fd 100644 --- a/app/components-react/highlighter/StreamCard.tsx +++ b/app/components-react/highlighter/StreamCard.tsx @@ -230,7 +230,8 @@ export default function StreamCard({

- {stream.state.type === EAiDetectionState.FINISHED ? ( + {stream.state.type === EAiDetectionState.FINISHED || + stream.state.type === EAiDetectionState.REALTIME_DETECTION_IN_PROGRESS ? ( ) : (
@@ -244,7 +245,11 @@ export default function StreamCard({ emitShowStreamClips={showStreamClips} clipsOfStreamAreLoading={clipsOfStreamAreLoading} emitRestartAiDetection={() => { - HighlighterService.actions.restartAiDetection(stream.path, stream); + if (stream.path) { + HighlighterService.actions.restartAiDetection(stream.path, stream); + } else { + console.log('TODO: Stream path is not available. Realtime dection'); + } }} emitSetView={emitSetView} emitFeedbackForm={() => { @@ -313,6 +318,38 @@ function ActionBar({ emitFeedbackForm(clips.length); }; + if (stream?.state.type === EAiDetectionState.REALTIME_DETECTION_IN_PROGRESS) { + return ( +
+
+ + + + +
+
{$t('Ai detection in progress')}
+ + +
+ ); + } + // In Progress if (stream?.state.type === EAiDetectionState.IN_PROGRESS) { return ( diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index 7f9f66ebf799..64f18c11bc28 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -372,7 +372,7 @@ export class HighlighterService extends PersistentStatefulService { this.UPDATE_HIGHLIGHTED_STREAM({ ...stream, - state: { type: EAiDetectionState.CANCELED_BY_USER, progress: 0 }, + state: { type: EAiDetectionState.CANCELED_BY_USER }, }); }); @@ -397,21 +397,37 @@ export class HighlighterService extends PersistentStatefulService { console.log('Realtime highlights received:', highlights); console.log(streamInfo.id); + console.log('Add ai clips now...'); this.addAiClips(highlights, { id: streamInfo.id || '', game: streamInfo.game }); + + console.log('Load clips now...'); + await this.loadClips(streamInfo.id); + + let count = 0; + const state = this.views.highlightedStreamsDictionary[streamInfo.id]?.state; + if (state?.type === EAiDetectionState.REALTIME_DETECTION_IN_PROGRESS) { + count = state.count; + } + + this.updateStream({ + state: { type: EAiDetectionState.REALTIME_DETECTION_IN_PROGRESS, count }, + id: streamInfo.id, + }); }); } else { this.streamingService.replayBufferFileWrite.subscribe(async clipPath => { @@ -476,26 +492,43 @@ export class HighlighterService extends PersistentStatefulService & { id: string }) { + const existingStreamInfo = this.state.highlightedStreamsDictionary[updatedStreamInfo.id]; + if (!existingStreamInfo) { + console.error(`Stream with id ${updatedStreamInfo.id} not found for update`); + return; + } + const updatedStream = { ...existingStreamInfo, ...updatedStreamInfo }; + this.UPDATE_HIGHLIGHTED_STREAM(updatedStream); } removeStream(streamId: string, deleteClipsFromSystem = true) { @@ -1421,10 +1467,14 @@ export class HighlighterService extends PersistentStatefulService { - setStreamInfo.state.progress = progress; + setStreamInfo.state = { + type: EAiDetectionState.IN_PROGRESS, + progress, + }; this.updateStream(setStreamInfo); }); From 1e72fba2a1aafd5441d065749aff9045f01ecbdf Mon Sep 17 00:00:00 2001 From: ggolda Date: Fri, 13 Jun 2025 13:32:31 -0600 Subject: [PATCH 13/41] added local vision service --- .../realtime-highlighter-service.ts | 63 +++++++++++++++++-- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index e2a15f57b0b5..3e6d3b7c0144 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -8,11 +8,58 @@ import { SettingsService } from 'app-services'; import moment from 'moment'; import { getVideoDuration } from './cut-highlight-clips'; +class LocalVisionService extends EventEmitter { + private isRunning = false; + private eventSource: EventSource | null = null; + + constructor() { + super(); + } + + subscribe(event: string | symbol, listener: (...args: any[]) => void) { + this.on(event, listener); + } + + unsubscribe(event: string | symbol, listener: (...args: any[]) => void) { + this.removeListener(event, listener); + } + + start() { + if (this.isRunning) { + console.warn('LocalVisionService is already running'); + return; + } + + this.isRunning = true; + this.eventSource = new EventSource('http://localhost:8000/events'); + this.eventSource.onmessage = (event: any) => { + const data = JSON.parse(event.data); + console.log('Received event:', data); + const events = data.events; + for (const event of events) { + console.log('Emitting event:', event); + this.emit('event', event); + } + }; + } + + stop() { + if (!this.isRunning) { + console.warn('LocalVisionService is not running'); + return; + } + + this.isRunning = false; + console.log('Stopping VisionService'); + this.eventSource?.close(); + } +} + /** * Just a mock class to represent a vision service events * that would be available when it is ready by another team. */ -class VisionService extends EventEmitter { +class MockVisionService extends EventEmitter { private timeoutId: NodeJS.Timeout | null = null; private isRunning = false; @@ -111,7 +158,7 @@ export class RealtimeHighlighterService extends Service { @Inject() private streamingService: StreamingService; @Inject() private settingsService: SettingsService; - private visionService = new VisionService(); + private visionService = new LocalVisionService(); private static MAX_SCORE = 5; @@ -147,9 +194,7 @@ export class RealtimeHighlighterService extends Service { this.highlights = []; this.visionService.subscribe('event', this.onEvent.bind(this)); - setTimeout(() => { - this.visionService.start(); - }, 1000 * this.getReplayBufferDurationSeconds()); + this.visionService.start(); // start the periodic tick to process replay queue after first replay buffer duration this.tick(); @@ -210,6 +255,8 @@ export class RealtimeHighlighterService extends Service { return; } + console.log('Received event:', event); + const endAdjust = event.highlight.end_adjust || 0; this.saveReplayAt = Date.now() + endAdjust * 1000; @@ -224,6 +271,7 @@ export class RealtimeHighlighterService extends Service { if (events.length === 0) { return; } + console.log('Current replay events:', events); this.currentReplayEvents = []; const replayBufferDuration = @@ -294,9 +342,12 @@ export class RealtimeHighlighterService extends Service { const clips = []; for (const highlight of mergedHighlights) { + // if more than 3 inputs, assign maximum score (1.0), otherwise normalize the score + const score = + highlight.inputs.length >= 3 ? 1.0 : highlight.score / RealtimeHighlighterService.MAX_SCORE; const aiClipInfo: IAiClipInfo = { inputs: highlight.inputs.map((input: string) => ({ type: input } as IInput)), - score: Math.round(highlight.score / RealtimeHighlighterService.MAX_SCORE), + score, metadata: {}, }; From 4f453427f36c2ebb7bbe78620eaa0b0c196f5623 Mon Sep 17 00:00:00 2001 From: ggolda Date: Fri, 13 Jun 2025 15:26:28 -0600 Subject: [PATCH 14/41] refactored and split into several methods --- .../realtime-highlighter-service.ts | 187 +++++++++++------- 1 file changed, 116 insertions(+), 71 deletions(-) diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index 3e6d3b7c0144..fe02e28fcdfe 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -5,7 +5,6 @@ import { Subject, Subscription } from 'rxjs'; import { INewClipData } from './models/highlighter.models'; import { IAiClipInfo, IInput } from './models/ai-highlighter.models'; import { SettingsService } from 'app-services'; -import moment from 'moment'; import { getVideoDuration } from './cut-highlight-clips'; class LocalVisionService extends EventEmitter { @@ -34,7 +33,7 @@ class LocalVisionService extends EventEmitter { this.eventSource = new EventSource('http://localhost:8000/events'); this.eventSource.onmessage = (event: any) => { const data = JSON.parse(event.data); - console.log('Received event:', data); + console.log('Received events:', data); const events = data.events; for (const event of events) { console.log('Emitting event:', event); @@ -215,6 +214,18 @@ export class RealtimeHighlighterService extends Service { this.isRunning = false; } + private getReplayBufferDurationSeconds(): number { + return this.settingsService.views.values.Output.RecRBTime; + } + + private setReplayBufferDurationSeconds(seconds: number) { + if (this.streamingService.state.replayBufferStatus !== EReplayBufferState.Offline) { + console.warn('Replay buffer must be stopped before its settings can be changed!'); + return; + } + this.settingsService.setSettingsPatch({ Output: { RecRBTime: seconds } }); + } + /** * This method is called periodically to save replay events to file at correct time * when the highlight ends. @@ -249,14 +260,15 @@ export class RealtimeHighlighterService extends Service { setTimeout(() => this.tick(), 1000); } + /** + * Fired when a new event is received from the vision service. + */ private onEvent(event: any) { // ignore events that have no highlight data if (!event.highlight) { return; } - console.log('Received event:', event); - const endAdjust = event.highlight.end_adjust || 0; this.saveReplayAt = Date.now() + endAdjust * 1000; @@ -266,80 +278,51 @@ export class RealtimeHighlighterService extends Service { this.currentReplayEvents.push(event); } + /** + * Fired when the replay buffer file is ready after the replay buffer was saved. + * + * Creates highlights from detected events in the replay buffer and notifies subscribers. + */ private async onReplayReady(path: string) { const events = this.currentReplayEvents; if (events.length === 0) { return; } - console.log('Current replay events:', events); this.currentReplayEvents = []; - const replayBufferDuration = + const replayBufferDurationSeconds = (await getVideoDuration(path)) || this.getReplayBufferDurationSeconds(); - console.log(`Replay buffer duration: ${replayBufferDuration} seconds`); // absolute time in milliseconds when the replay was saved const replaySavedAt = this.replaySavedAt; this.replaySavedAt = null; - const replayStartedAt = replaySavedAt - replayBufferDuration * 1000; - console.log(`Replay saved at ${moment(replaySavedAt).format('YYYY-MM-DD HH:mm:ss')}`); - console.log(`Replay started at ${moment(replayStartedAt).format('YYYY-MM-DD HH:mm:ss')}`); - console.log(`Replay buffer duration: ${replayBufferDuration} seconds`); - - const unrefinedHighlights = []; - - for (const event of events) { - const eventTime = event.timestamp; - - const relativeEventTime = eventTime - replayStartedAt; - const highlightStart = relativeEventTime - (event.highlight.start_adjust || 0) * 1000; - const highlightEnd = relativeEventTime + (event.highlight.end_adjust || 0) * 1000; - - // check if the highlight is within the replay buffer duration - if (highlightStart < 0 || highlightEnd > replayBufferDuration * 1000) { - console.warn( - `Event ${ - event.name - } is outside of the replay buffer duration, skipping highlight creation. highlightStart: ${highlightStart}, highlightEnd: ${highlightEnd}, replayBufferDuration: ${ - replayBufferDuration * 1000 - } ms`, - ); - continue; - } - - unrefinedHighlights.push({ - inputs: [event.name], - startTime: highlightStart / 1000, // convert to seconds - endTime: highlightEnd / 1000, // convert to seconds - score: event.highlight.score || 0, - }); - } - + const unrefinedHighlights = this.extractUnrefinedHighlights( + events, + replaySavedAt, + replayBufferDurationSeconds, + ); console.log('Unrefined highlights:', unrefinedHighlights); - // merge overlapping highlights - const acceptableOffset = 5; // seconds + const mergedHighlights: any[] = this.mergeOverlappingHighlights(unrefinedHighlights); - const mergedHighlights: any[] = []; - for (const highlight of unrefinedHighlights) { - if (mergedHighlights.length === 0) { - mergedHighlights.push(highlight); - continue; - } + const clips = this.createClipsFromHighlights( + mergedHighlights, + replayBufferDurationSeconds, + path, + ); - const lastHighlight = mergedHighlights[mergedHighlights.length - 1]; - if (highlight.startTime - acceptableOffset <= lastHighlight.endTime) { - // merge highlights - lastHighlight.endTime = highlight.endTime; // extend end time - lastHighlight.score = Math.max(highlight.score, lastHighlight.score); - lastHighlight.inputs.push(...highlight.inputs); - } else { - // no overlap, push new highlight - mergedHighlights.push(highlight); - } - } + this.highlightsReady.next(clips); + } + /** + * Creates clips from detected highlights. Several highlights can be merged into one clip + */ + private createClipsFromHighlights( + mergedHighlights: any[], + replayBufferDuration: number, + path: string, + ) { const clips = []; for (const highlight of mergedHighlights) { // if more than 3 inputs, assign maximum score (1.0), otherwise normalize the score @@ -364,24 +347,86 @@ export class RealtimeHighlighterService extends Service { endTrim, }; - this.highlights.push(clip); - console.log(`New highlight added: ${clip.path}`); - console.log(clip); clips.push(clip); } - this.highlightsReady.next(clips); + // store in a global state + this.highlights.push(...clips); + return clips; } - private getReplayBufferDurationSeconds(): number { - return this.settingsService.views.values.Output.RecRBTime; + /** + * Merges overlapping highlights based on their start and end times. + */ + private mergeOverlappingHighlights( + unrefinedHighlights: { + inputs: any[]; + startTime: number; // seconds + endTime: number; // seconds + score: any; + }[], + ) { + const acceptableOffset = 5; // seconds + const mergedHighlights: any[] = []; + for (const highlight of unrefinedHighlights) { + if (mergedHighlights.length === 0) { + mergedHighlights.push(highlight); + continue; + } + + const lastHighlight = mergedHighlights[mergedHighlights.length - 1]; + if (highlight.startTime - acceptableOffset <= lastHighlight.endTime) { + // merge highlights + lastHighlight.endTime = highlight.endTime; // extend end time + lastHighlight.score = Math.max(highlight.score, lastHighlight.score); + lastHighlight.inputs.push(...highlight.inputs); + } else { + // no overlap, push new highlight + mergedHighlights.push(highlight); + } + } + return mergedHighlights; } - private setReplayBufferDurationSeconds(seconds: number) { - if (this.streamingService.state.replayBufferStatus !== EReplayBufferState.Offline) { - console.warn('Replay buffer must be stopped before its settings can be changed!'); - return; + /** + * Attempts to find unrefined highlights from raw events from the vision service. + */ + private extractUnrefinedHighlights( + events: any[], + replaySavedAt: number, + replayBufferDurationSeconds: number, + ) { + const unrefinedHighlights = []; + + const replayStartedAt = replaySavedAt - replayBufferDurationSeconds * 1000; + + for (const event of events) { + const eventTime = event.timestamp; + + const relativeEventTime = eventTime - replayStartedAt; + const highlightStart = relativeEventTime - (event.highlight.start_adjust || 0) * 1000; + const highlightEnd = relativeEventTime + (event.highlight.end_adjust || 0) * 1000; + + // check if the highlight is within the replay buffer duration + if (highlightStart < 0 || highlightEnd > replayBufferDurationSeconds * 1000) { + console.warn( + `Event ${ + event.name + } is outside of the replay buffer duration, skipping highlight creation. highlightStart: ${highlightStart}, highlightEnd: ${highlightEnd}, replayBufferDuration: ${ + replayBufferDurationSeconds * 1000 + } ms`, + ); + continue; + } + + // need to convert all times to seconds + unrefinedHighlights.push({ + inputs: [event.name], + startTime: highlightStart / 1000, // convert to seconds + endTime: highlightEnd / 1000, // convert to seconds + score: event.highlight.score || 0, + }); } - this.settingsService.setSettingsPatch({ Output: { RecRBTime: seconds } }); + return unrefinedHighlights; } } From 5a20e7788697bec7b8f1464fa84e120493ca3641 Mon Sep 17 00:00:00 2001 From: ggolda Date: Mon, 16 Jun 2025 17:11:29 -0600 Subject: [PATCH 15/41] [EXPERIMENT] use the start time of the first detected highlight moment when calculating replay buffer start/stop times. In theory should produce better results for when many events come one after another. Previous iteration was using the highlight end moment + offset, so more replay buffers have to be created and overlap between them was possible --- .../realtime-highlighter-service.ts | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index fe02e28fcdfe..8c4312ab0552 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -269,13 +269,24 @@ export class RealtimeHighlighterService extends Service { return; } - const endAdjust = event.highlight.end_adjust || 0; - - this.saveReplayAt = Date.now() + endAdjust * 1000; const currentTime = Date.now(); - event.timestamp = currentTime; // use current time as timestamp this.currentReplayEvents.push(event); + + // replay should be recorded when it enters the window of the + // first detected event start time + // + // reported buffer durations are not always accurate, so we + // use a tolerance to avoid issues with the replay buffer length + if (this.saveReplayAt === null) { + const startAdjust = (event.highlight.start_adjust || 0) * 1000; + const reportedBufferLengthErrorTolerance = 2 * 1000; + this.saveReplayAt = + Date.now() + + this.getReplayBufferDurationSeconds() * 1000 - + startAdjust - + reportedBufferLengthErrorTolerance; + } } /** @@ -404,11 +415,15 @@ export class RealtimeHighlighterService extends Service { const eventTime = event.timestamp; const relativeEventTime = eventTime - replayStartedAt; - const highlightStart = relativeEventTime - (event.highlight.start_adjust || 0) * 1000; - const highlightEnd = relativeEventTime + (event.highlight.end_adjust || 0) * 1000; - - // check if the highlight is within the replay buffer duration - if (highlightStart < 0 || highlightEnd > replayBufferDurationSeconds * 1000) { + let highlightStart = relativeEventTime - (event.highlight.start_adjust || 0) * 1000; + let highlightEnd = relativeEventTime + (event.highlight.end_adjust || 0) * 1000; + + // add some minor error tolerance to avoid issues with the replay buffer length + const errorTolerance = 1000; // 1 second error tolerance + if ( + highlightStart < -errorTolerance || + highlightEnd > replayBufferDurationSeconds * 1000 + errorTolerance + ) { console.warn( `Event ${ event.name @@ -419,6 +434,13 @@ export class RealtimeHighlighterService extends Service { continue; } + // ensure highlight start and end times are within the replay buffer duration + // and not negative or exceeding the buffer length. + // It is possible that the event is outside of the replay buffer duration + // due to the way the replay buffer works, so we need to handle that. (actual video length can be different from the reported one) + highlightStart = Math.max(highlightStart, 0); // ensure start time is not negative + highlightEnd = Math.min(highlightEnd, replayBufferDurationSeconds * 1000); + // need to convert all times to seconds unrefinedHighlights.push({ inputs: [event.name], From 2b96e932fb4a91001fae662e2379a548b8557308 Mon Sep 17 00:00:00 2001 From: ggolda Date: Tue, 17 Jun 2025 16:10:39 -0600 Subject: [PATCH 16/41] fixed the bug when dekstop sends replay buffer event without requesting it. Fixed highlights margins and now combine several ones into the one long highlight --- app/services/highlighter/index.ts | 5 +- .../realtime-highlighter-service.ts | 54 ++++++++++++++++--- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index 64f18c11bc28..b747e433dd48 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -413,7 +413,10 @@ export class HighlighterService extends PersistentStatefulService { + console.log('VisionService state reset'); + }); } stop() { @@ -172,6 +179,9 @@ export class RealtimeHighlighterService extends Service { // events that are currently being observer in the replay buffer window // (in case there are multiple events in a row that should land in the same replay) private currentReplayEvents: any[] = []; + // sometimes Streamlabs Desktop sends weird replay buffer events + // when we didn't request them, so we need to track if we requested the replay + private replayRequested: boolean = false; async start() { console.log('Starting RealtimeHighlighterService'); @@ -206,7 +216,7 @@ export class RealtimeHighlighterService extends Service { return; } // don't stop replay buffer here, probably better places for it exist - this.visionService.unsubscribe('event', this.onEvent.bind(this)); + this.visionService.removeAllListeners(); this.visionService.stop(); this.replayBufferFileReadySubscription?.unsubscribe(); @@ -244,6 +254,7 @@ export class RealtimeHighlighterService extends Service { if (this.currentReplayEvents.length > 0) { console.log('Saving replay buffer'); this.replaySavedAt = now; + this.replayRequested = true; this.streamingService.saveReplay(); } @@ -269,8 +280,7 @@ export class RealtimeHighlighterService extends Service { return; } - const currentTime = Date.now(); - event.timestamp = currentTime; // use current time as timestamp + event.timestamp = Date.now(); this.currentReplayEvents.push(event); // replay should be recorded when it enters the window of the @@ -280,12 +290,11 @@ export class RealtimeHighlighterService extends Service { // use a tolerance to avoid issues with the replay buffer length if (this.saveReplayAt === null) { const startAdjust = (event.highlight.start_adjust || 0) * 1000; + const replayBufferDuration = this.getReplayBufferDurationSeconds() * 1000; + console.log('Buffer duration: ', replayBufferDuration); const reportedBufferLengthErrorTolerance = 2 * 1000; this.saveReplayAt = - Date.now() + - this.getReplayBufferDurationSeconds() * 1000 - - startAdjust - - reportedBufferLengthErrorTolerance; + Date.now() + replayBufferDuration - startAdjust - reportedBufferLengthErrorTolerance; } } @@ -295,6 +304,12 @@ export class RealtimeHighlighterService extends Service { * Creates highlights from detected events in the replay buffer and notifies subscribers. */ private async onReplayReady(path: string) { + if (!this.replayRequested) { + console.warn('Replay buffer file ready event received, but no replay was requested.'); + return; + } + this.replayRequested = false; + const events = this.currentReplayEvents; if (events.length === 0) { return; @@ -396,6 +411,26 @@ export class RealtimeHighlighterService extends Service { mergedHighlights.push(highlight); } } + + // for some reason only 1 highlight that points to the same file is allowed, + // so need to merge all highlights into the one big highlight + if (mergedHighlights.length > 1) { + console.log('Merging highlights into one highlight'); + const allInputs = mergedHighlights.flatMap(h => h.inputs); + const maxScore = Math.max(...mergedHighlights.map(h => h.score)); + const startTime = Math.min(...mergedHighlights.map(h => h.startTime)); + const endTime = Math.max(...mergedHighlights.map(h => h.endTime)); + + return [ + { + inputs: allInputs, + startTime, + endTime, + score: maxScore, + }, + ]; + } + return mergedHighlights; } @@ -413,13 +448,16 @@ export class RealtimeHighlighterService extends Service { for (const event of events) { const eventTime = event.timestamp; + console.log('Event time:', eventTime, 'Replay started at:', replayStartedAt); const relativeEventTime = eventTime - replayStartedAt; + console.log('Relative event time:', relativeEventTime); let highlightStart = relativeEventTime - (event.highlight.start_adjust || 0) * 1000; let highlightEnd = relativeEventTime + (event.highlight.end_adjust || 0) * 1000; + console.log(`Event ${event.name} highlight start: ${highlightStart}, end: ${highlightEnd}`); // add some minor error tolerance to avoid issues with the replay buffer length - const errorTolerance = 1000; // 1 second error tolerance + const errorTolerance = 2500; // 1 second error tolerance if ( highlightStart < -errorTolerance || highlightEnd > replayBufferDurationSeconds * 1000 + errorTolerance From 4d37a37b1774a01a0397d1677fbe645845c82fc6 Mon Sep 17 00:00:00 2001 From: ggolda Date: Wed, 18 Jun 2025 13:11:58 -0600 Subject: [PATCH 17/41] fixed the issue when replay buffer duration couldnt be set --- .../realtime-highlighter-service.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index ff85af58149a..419027c21c63 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -190,9 +190,16 @@ export class RealtimeHighlighterService extends Service { return; } - this.isRunning = true; - // start replay buffer if its not already running - this.setReplayBufferDurationSeconds(30); + const REQUIRED_REPLAY_BUFFER_DURATION_SECONDS = 30; + if (this.getReplayBufferDurationSeconds() < REQUIRED_REPLAY_BUFFER_DURATION_SECONDS) { + if (this.streamingService.state.replayBufferStatus !== EReplayBufferState.Offline) { + console.log('Stopping replay buffer to change its duration'); + // to change the replay buffer duration, it must be stopped first + this.streamingService.stopReplayBuffer(); + } + this.setReplayBufferDurationSeconds(30); + } + this.streamingService.startReplayBuffer(); this.replayBufferFileReadySubscription = this.streamingService.replayBufferFileWrite.subscribe( this.onReplayReady.bind(this), @@ -205,6 +212,7 @@ export class RealtimeHighlighterService extends Service { this.visionService.subscribe('event', this.onEvent.bind(this)); this.visionService.start(); + this.isRunning = true; // start the periodic tick to process replay queue after first replay buffer duration this.tick(); } @@ -229,10 +237,6 @@ export class RealtimeHighlighterService extends Service { } private setReplayBufferDurationSeconds(seconds: number) { - if (this.streamingService.state.replayBufferStatus !== EReplayBufferState.Offline) { - console.warn('Replay buffer must be stopped before its settings can be changed!'); - return; - } this.settingsService.setSettingsPatch({ Output: { RecRBTime: seconds } }); } From c56be84616767e182a9a35eab338689a69790a5e Mon Sep 17 00:00:00 2001 From: ggolda Date: Thu, 19 Jun 2025 13:12:30 -0600 Subject: [PATCH 18/41] added round metadata for realtime highlighter --- .../models/ai-highlighter.models.ts | 1 + .../realtime-highlighter-service.ts | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/services/highlighter/models/ai-highlighter.models.ts b/app/services/highlighter/models/ai-highlighter.models.ts index 202b62a28003..505c7d140892 100644 --- a/app/services/highlighter/models/ai-highlighter.models.ts +++ b/app/services/highlighter/models/ai-highlighter.models.ts @@ -88,6 +88,7 @@ export interface IAiClipInfo { metadata: { round?: number; webcam_coordinates?: ICoordinates; + game?: EGame; }; } diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index 419027c21c63..339d723f6d81 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -3,11 +3,13 @@ import { EventEmitter } from 'events'; import { EReplayBufferState, StreamingService } from 'services/streaming'; import { Subject, Subscription } from 'rxjs'; import { INewClipData } from './models/highlighter.models'; -import { IAiClipInfo, IInput } from './models/ai-highlighter.models'; +import { EGame, IAiClipInfo, IInput } from './models/ai-highlighter.models'; import { SettingsService } from 'app-services'; import { getVideoDuration } from './cut-highlight-clips'; class LocalVisionService extends EventEmitter { + currentGame: string | null = null; + private isRunning = false; private eventSource: EventSource | null = null; @@ -34,6 +36,9 @@ class LocalVisionService extends EventEmitter { this.eventSource.onmessage = (event: any) => { const data = JSON.parse(event.data); console.log('Received events:', data); + + this.currentGame = data.game || null; + const events = data.events; for (const event of events) { console.log('Emitting event:', event); @@ -66,6 +71,8 @@ class LocalVisionService extends EventEmitter { * that would be available when it is ready by another team. */ class MockVisionService extends EventEmitter { + currentGame: string | null = 'fortnite'; + private timeoutId: NodeJS.Timeout | null = null; private isRunning = false; @@ -160,14 +167,14 @@ class MockVisionService extends EventEmitter { @InitAfter('StreamingService') export class RealtimeHighlighterService extends Service { + private static MAX_SCORE = 5; + highlightsReady = new Subject(); @Inject() private streamingService: StreamingService; @Inject() private settingsService: SettingsService; private visionService = new LocalVisionService(); - private static MAX_SCORE = 5; - private isRunning = false; private highlights: INewClipData[] = []; @@ -183,6 +190,8 @@ export class RealtimeHighlighterService extends Service { // when we didn't request them, so we need to track if we requested the replay private replayRequested: boolean = false; + private currentRound: number = 0; + async start() { console.log('Starting RealtimeHighlighterService'); if (this.isRunning) { @@ -208,6 +217,7 @@ export class RealtimeHighlighterService extends Service { this.saveReplayAt = null; this.currentReplayEvents = []; this.highlights = []; + this.currentRound = 0; this.visionService.subscribe('event', this.onEvent.bind(this)); this.visionService.start(); @@ -279,6 +289,9 @@ export class RealtimeHighlighterService extends Service { * Fired when a new event is received from the vision service. */ private onEvent(event: any) { + if (['round_start', 'game_start'].includes(event.name)) { + this.currentRound++; + } // ignore events that have no highlight data if (!event.highlight) { return; @@ -361,7 +374,10 @@ export class RealtimeHighlighterService extends Service { const aiClipInfo: IAiClipInfo = { inputs: highlight.inputs.map((input: string) => ({ type: input } as IInput)), score, - metadata: {}, + metadata: { + round: this.currentRound, + game: this.visionService.currentGame as EGame, + }, }; // trim times for desktop are insanely weird, for some reason its offset between start and end From e03c3f2dcf194c2e1abfaa31e0a6f0208c8b7b81 Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Fri, 20 Jun 2025 11:19:10 +0200 Subject: [PATCH 19/41] realtime indicator cmoponent --- .../highlighter/RealtimeIndicator.m.less | 76 +++++++++++++++++++ .../highlighter/RealtimeIndicator.tsx | 51 +++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 app/components-react/highlighter/RealtimeIndicator.m.less create mode 100644 app/components-react/highlighter/RealtimeIndicator.tsx diff --git a/app/components-react/highlighter/RealtimeIndicator.m.less b/app/components-react/highlighter/RealtimeIndicator.m.less new file mode 100644 index 000000000000..48a37324c3f3 --- /dev/null +++ b/app/components-react/highlighter/RealtimeIndicator.m.less @@ -0,0 +1,76 @@ +.pulseWrapper { + width: 16px; + height: 16px; + margin-right: 4px; + display: flex; + align-items: center; + justify-content: center; + position: relative; +} + +.pulse { + position: absolute; + width: 16px; + height: 16px; + border-radius: 50%; + background: #f85640; + opacity: 0.3; + top: 0; + left: 0; + animation: pulse 2.4s infinite; +} + +.dot { + width: 6px; + height: 6px; + border-radius: 50%; + background: #f85640; + position: relative; + z-index: 1; +} + +@keyframes pulse { + 0% { + transform: scale(1); + opacity: 0.3; + } + 50% { + transform: scale(1.5); + opacity: 0.15; + } + 100% { + transform: scale(1); + opacity: 0.3; + } +} +.realtime-detection-action { + position: relative; + justify-content: space-between; + display: flex; + padding: 6px 11px; + padding-right: 20px; + width: 248px; + align-items: center; + gap: 8px; + border-radius: 48px; + border: 1px solid #301e24; + background: #301e24; + color: white; + border-color: #5d1e1e; + overflow: hidden; +} + +.activity-animation { + transform: translateX(200%); +} + +.background-pulse { + left: -100%; + position: absolute; + width: 200px; + height: 40px; + background: linear-gradient(90deg, #ffffff00, #5d1e1e, #ffffff00); + transition: transform 0.5s ease-in-out; +} +.realtime-cancel-button { +} diff --git a/app/components-react/highlighter/RealtimeIndicator.tsx b/app/components-react/highlighter/RealtimeIndicator.tsx new file mode 100644 index 000000000000..b9498e5e93d8 --- /dev/null +++ b/app/components-react/highlighter/RealtimeIndicator.tsx @@ -0,0 +1,51 @@ +import { Button } from 'antd'; +import React, { useState } from 'react'; +import { $t } from 'services/i18n'; +import styles from './RealtimeIndicator.m.less'; +import cx from 'classnames'; + +export default function HighlightGenerator() { + const [animateOnce, setAnimateOnce] = useState(false); + function triggerDetection() { + if (animateOnce) { + return; + } + setAnimateOnce(true); + setTimeout(() => { + setAnimateOnce(false); + }, 2000); + } + return ( +
+
+
setAnimateOnce(false)} + /> + {animateOnce ? ( +
🔫
+ ) : ( +
+
+
+
+ )} +

+ {animateOnce ? $t('Clip detected') : $t('Ai detection in progress')} +

+
+ +
+ ); +} From 7742812fae2d73ee31a6be2732df28c1798082bc Mon Sep 17 00:00:00 2001 From: ggolda Date: Fri, 20 Jun 2025 16:04:19 -0600 Subject: [PATCH 20/41] reworked instant replay widget to make it work with realtime highlighter. think how to support both versions --- .../realtime-highlighter-service.ts | 2 +- .../properties-managers/replay-manager.ts | 92 ++++++++++++++++++- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index 339d723f6d81..f8c034c17976 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -170,13 +170,13 @@ export class RealtimeHighlighterService extends Service { private static MAX_SCORE = 5; highlightsReady = new Subject(); + highlights: INewClipData[] = []; @Inject() private streamingService: StreamingService; @Inject() private settingsService: SettingsService; private visionService = new LocalVisionService(); private isRunning = false; - private highlights: INewClipData[] = []; private replayBufferFileReadySubscription: Subscription | null = null; diff --git a/app/services/sources/properties-managers/replay-manager.ts b/app/services/sources/properties-managers/replay-manager.ts index f691550109ad..1384e6c8b751 100644 --- a/app/services/sources/properties-managers/replay-manager.ts +++ b/app/services/sources/properties-managers/replay-manager.ts @@ -1,17 +1,103 @@ +import { RealtimeHighlighterService, ScenesService } from 'app-services'; import { PropertiesManager } from './properties-manager'; import { Inject } from 'services/core/injector'; import { StreamingService } from 'services/streaming'; export class ReplayManager extends PropertiesManager { @Inject() streamingService: StreamingService; + @Inject() realtimeHighlighterService: RealtimeHighlighterService; + @Inject() scenesService: ScenesService; + + private inProgress = false; + private stopAt: number | null = null; + private currentReplayIndex: number = 0; get denylist() { return ['is_local_file', 'local_file']; } init() { - this.streamingService.replayBufferFileWrite.subscribe(filePath => { - this.obsSource.update({ local_file: filePath }); - }); + console.log('ReplayManager initialized'); + setInterval(() => { + this.tick(); + }, 1000); + } + + private tick() { + const isVisible = this.isSourceVisibleInActiveScene(); + if (isVisible && this.inProgress) { + return this.keepPlaying(); + } else if (isVisible && !this.inProgress) { + return this.startPlaying(); + } else if (!isVisible && this.inProgress) { + return this.stopPlaying(); + } + } + + private keepPlaying() { + const currentTime = Date.now(); + + // if time is less than stopAt, do nothing + if (this.stopAt && currentTime < this.stopAt) { + return; + } + + // if we reached the end of the highlight, switch to the next one + const highlightsCount = this.realtimeHighlighterService.highlights.length; + const nextIndex = this.currentReplayIndex + 1; + if (nextIndex < highlightsCount) { + this.queueNextHighlight(nextIndex); + } else { + this.queueNextHighlight(0); // Loop back to the first highlight + } + } + + private startPlaying() { + if (this.realtimeHighlighterService.highlights.length === 0) { + console.log('No highlights to play'); + return; + } + + console.log('Start playing highlights'); + this.inProgress = true; + this.queueNextHighlight(0); + } + + private stopPlaying() { + this.inProgress = false; + this.stopAt = null; + this.currentReplayIndex = null; + console.log('Stop playing highlights'); + } + + /** + * Check if Instant Replay source is in currently active scene + * and visible to on the stream. + * + * One source can be in multiple scene items in different scenes, + * so we need to check all scene items that link to the source + * and check if any of them is visible in the active scene. + */ + private isSourceVisibleInActiveScene(): boolean { + const activeSceneId = this.scenesService.views.activeSceneId; + const sourceItems = this.scenesService.views.getSceneItemsBySourceId(this.obsSource.name); + for (const item of sourceItems) { + if (item.sceneId !== activeSceneId) { + continue; + } + + if (item.visible) { + return true; + } + } + return false; + } + + private queueNextHighlight(index: number) { + const highlight = this.realtimeHighlighterService.highlights[index]; + this.stopAt = Date.now() + (highlight.endTime - highlight.endTrim) * 1000; + this.obsSource.update({ local_file: highlight.path }); + this.currentReplayIndex = index; + console.log(`Queued next highlight: ${highlight.path}`); } } From 3caedd70a62f2da41945e961123516bdec68cac4 Mon Sep 17 00:00:00 2001 From: ggolda Date: Fri, 20 Jun 2025 16:08:30 -0600 Subject: [PATCH 21/41] comments added --- app/services/sources/properties-managers/replay-manager.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/app/services/sources/properties-managers/replay-manager.ts b/app/services/sources/properties-managers/replay-manager.ts index 1384e6c8b751..a10c25f62a8c 100644 --- a/app/services/sources/properties-managers/replay-manager.ts +++ b/app/services/sources/properties-managers/replay-manager.ts @@ -80,6 +80,7 @@ export class ReplayManager extends PropertiesManager { */ private isSourceVisibleInActiveScene(): boolean { const activeSceneId = this.scenesService.views.activeSceneId; + // for some reason for instant replay source, the scene id is stored inside obsSource.name const sourceItems = this.scenesService.views.getSceneItemsBySourceId(this.obsSource.name); for (const item of sourceItems) { if (item.sceneId !== activeSceneId) { From c58b2a78140f0c72b5df37ce61808027199dab82 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Mon, 23 Jun 2025 14:57:32 +0200 Subject: [PATCH 22/41] realtime highlights indicator --- .../highlighter/ClipPreviewInfo.tsx | 68 +++++---- .../RealtimeHighlightsFeed.m.less | 117 ++++++++++++++ .../RealtimeHighlightsFeed.tsx | 144 ++++++++++++++++++ .../RealtimeHighlightsIndicator.m.less | 60 ++++++++ .../RealtimeHighlightsIndicator.tsx | 14 ++ .../RealtimeHighlightsItem.tsx | 46 ++++++ app/components-react/root/StudioFooter.tsx | 7 + .../realtime-highlighter-service.ts | 49 +++++- 8 files changed, 475 insertions(+), 30 deletions(-) create mode 100644 app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.m.less create mode 100644 app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx create mode 100644 app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.m.less create mode 100644 app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.tsx create mode 100644 app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx diff --git a/app/components-react/highlighter/ClipPreviewInfo.tsx b/app/components-react/highlighter/ClipPreviewInfo.tsx index a32cf67a4dfe..4a9a11019a2b 100644 --- a/app/components-react/highlighter/ClipPreviewInfo.tsx +++ b/app/components-react/highlighter/ClipPreviewInfo.tsx @@ -1,4 +1,4 @@ -import { EGame } from 'services/highlighter/models/ai-highlighter.models'; +import { EGame, IAiClipInfo } from 'services/highlighter/models/ai-highlighter.models'; import { IAiClip } from 'services/highlighter/models/highlighter.models'; import { getConfigByGame, getEventConfig } from 'services/highlighter/models/game-config.models'; import styles from './ClipPreview.m.less'; @@ -15,33 +15,7 @@ export default function ClipPreviewInfo({ return No event data; } - const uniqueInputTypes = new Set(); - if (clip.aiInfo.inputs && Array.isArray(clip.aiInfo.inputs)) { - clip.aiInfo.inputs.forEach(input => { - if (input.type) { - uniqueInputTypes.add(input.type); - } - }); - } - - const eventDisplays = Array.from(uniqueInputTypes).map(type => { - const eventInfo = getEventConfig(game, type); - - if (eventInfo) { - return { - emoji: eventInfo.emoji, - description: eventInfo.description.singular, - type, - }; - } - - return { - emoji: '⚡', - description: type, - type, - }; - }); - + const eventDisplays = getUniqueEmojiConfigFromAiInfo(clip.aiInfo, game); return (
); } + +export interface EmojiConfig { + emoji: string; + description: string; + type: string; +} + +export function getUniqueEmojiConfigFromAiInfo(aiInfos: IAiClipInfo, game?: EGame): EmojiConfig[] { + const uniqueInputTypes = new Set(); + if (aiInfos.inputs && Array.isArray(aiInfos.inputs)) { + aiInfos.inputs.forEach(aiInput => { + if (aiInput.type) { + uniqueInputTypes.add(aiInput.type); + } + }); + } + + const eventDisplays = Array.from(uniqueInputTypes).map(type => { + if (game) { + const eventInfo = getEventConfig(game, type); + + if (eventInfo) { + return { + emoji: eventInfo.emoji, + description: eventInfo.description.singular, + type, + }; + } + } + + return { + emoji: '⚡', + description: type, + type, + }; + }); + return eventDisplays; +} diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.m.less b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.m.less new file mode 100644 index 000000000000..8bf5aa482763 --- /dev/null +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.m.less @@ -0,0 +1,117 @@ +.event-tooltip { + :global(.ant-tooltip-content) { + max-width: 320px; + } + + :global(.ant-tooltip-inner) { + padding: 12px; + background: #1f1f23; + border: 1px solid #3c3c41; + border-radius: 6px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4); + } +} + +.event-tooltip-content { + min-width: 200px; + max-width: 300px; +} + +.event-list { + display: flex; + flex-direction: column; + gap: 10px; +} + +.event-item { + padding: 10px; + background: #2d2d30; + border-radius: 6px; + border: 1px solid #404040; + transition: all 0.2s ease; + + &:hover { + background: #353538; + border-color: #4a9eff; + box-shadow: 0 2px 8px rgba(74, 158, 255, 0.1); + } +} + +.event-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 6px; +} + +.event-title { + font-weight: 600; + font-size: 12px; + color: #e4e4e7; +} + +.event-timestamp { + font-size: 10px; + color: #a1a1aa; + opacity: 0.8; +} + +.event-description { + font-size: 11px; + color: #d4d4d8; + margin-bottom: 8px; + line-height: 1.4; +} + +.event-button { + padding: 0; + height: auto; + font-size: 10px; + color: #4a9eff; + transition: color 0.2s ease; + + &:hover { + color: #60a5fa; + text-decoration: underline; + } + + &:focus { + color: #60a5fa; + } +} + +.event-footer { + margin-top: 12px; + padding-top: 10px; + border-top: 1px solid #404040; + display: flex; + justify-content: space-between; + align-items: center; +} + +.more-events-text { + font-size: 10px; + color: #a1a1aa; + opacity: 0.9; +} + +.view-all-button { + height: 26px; + font-size: 10px; + padding: 0 10px; + background: #4a9eff; + border-color: #4a9eff; + border-radius: 4px; + transition: all 0.2s ease; + + &:hover { + background: #60a5fa; + border-color: #60a5fa; + box-shadow: 0 2px 6px rgba(74, 158, 255, 0.3); + } + + &:focus { + background: #60a5fa; + border-color: #60a5fa; + } +} diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx new file mode 100644 index 000000000000..9f90e15d150e --- /dev/null +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx @@ -0,0 +1,144 @@ +import React, { useEffect, useState } from 'react'; +import { Tooltip, Button } from 'antd'; +import { TooltipPlacement } from 'antd/lib/tooltip'; +import styles from './RealtimeHighlightsFeed.m.less'; +import { IHighlight } from 'services/highlighter/models/ai-highlighter.models'; +import { HighlighterService } from 'app-services'; +import { Services } from '../../service-provider'; +import { useVuex } from '../../hooks'; +import { + EHighlighterView, + INewClipData, + TClip, +} from 'services/highlighter/models/highlighter.models'; +import { Subscription } from 'rxjs'; +import RealtimeHighlightsItem from './RealtimeHighlightsItem'; +import { useRealmObject } from '../../hooks/realm'; +import { EMenuItemKey } from 'services/side-nav'; +import Utils from 'services/utils'; + +interface IRealtimeHighlightTooltipProps { + children: React.ReactElement; + placement?: TooltipPlacement; + trigger?: + | 'hover' + | 'click' + | 'focus' + | 'contextMenu' + | Array<'hover' | 'click' | 'focus' | 'contextMenu'>; + maxEvents?: number; +} + +export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightTooltipProps) { + const { children, placement, trigger, maxEvents = 5 } = props; + const { HighlighterService, RealtimeHighlighterService, NavigationService } = Services; + + const [displayedEvents, setDisplayedEvents] = useState([]); + let hasMoreEvents = false; + const isDevMode = Utils.isDevMode(); + + const currentGame = useRealmObject(RealtimeHighlighterService.ephemeralState).game; + let realtimeHighlightSubscription: Subscription | null = null; + useEffect(() => { + // Initialize component + console.log('Initializing RealtimeEventTooltip component'); + + realtimeHighlightSubscription = RealtimeHighlighterService.highlightsReady.subscribe( + newClipData => { + // This will be called when highlights are ready + const highlights = Object.values(newClipData); + setDisplayedEvents(prevEvents => { + const updatedEvents = [...prevEvents, ...highlights]; + // Remove excess events from the beginning if we exceed maxEvents + if (updatedEvents.length > maxEvents) { + updatedEvents.splice(0, updatedEvents.length - maxEvents); + } + return updatedEvents; + }); + hasMoreEvents = highlights.length > maxEvents; + console.log('Realtime highlights are ready:', newClipData); + }, + ); + + // On unmount, unsubscribe from the realtime highlights + return () => { + realtimeHighlightSubscription?.unsubscribe(); + }; + }, []); + + useEffect(() => {}, []); + + function onViewAll() { + // Navigate to the Highlighter stream view + NavigationService.actions.navigate( + 'Highlighter', + { + view: EHighlighterView.STREAM, + }, + EMenuItemKey.Highlighter, + ); + } + + function onEventItemClick(event: INewClipData) { + console.log('Open single highlight view for event:', event); + // Navigate to specific highlight in stream + NavigationService.actions.navigate( + 'Highlighter', + { + view: EHighlighterView.CLIPS, + id: event.path, + }, + EMenuItemKey.Highlighter, + ); + } + + const tooltipContent = ( +
+
+ {displayedEvents && + displayedEvents.map(clipData => ( + + ))} +
+ + {hasMoreEvents && ( +
+ +
+ )} + + {isDevMode && ( + + )} +
+ ); + + return ( + + {children} + + ); +} diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.m.less b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.m.less new file mode 100644 index 000000000000..e69a97770835 --- /dev/null +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.m.less @@ -0,0 +1,60 @@ +.realtime-indicator { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 16px; + background: rgba(139, 69, 69, 0.8); + border-radius: 20px; + cursor: pointer; + transition: all 0.2s ease; + + &:hover { + background: rgba(139, 69, 69, 0.9); + transform: translateY(-1px); + } +} + +.fade-in { + animation: fadeIn 0.4s ease-out forwards; + opacity: 0; +} + +.status-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: #ff4d4f; + flex-shrink: 0; + animation: pulse 2s infinite; +} + +.status-text { + color: #ffffff; + font-size: 14px; + font-weight: 500; + white-space: nowrap; +} + +@keyframes fadeIn { + 0% { + opacity: 0; + } + 100% { + opacity: 1; + } +} + +@keyframes pulse { + 0% { + opacity: 1; + transform: scale(1); + } + 50% { + opacity: 0.7; + transform: scale(1.1); + } + 100% { + opacity: 1; + transform: scale(1); + } +} diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.tsx new file mode 100644 index 000000000000..0699d0f2b0e0 --- /dev/null +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.tsx @@ -0,0 +1,14 @@ +import React from 'react'; +import RealtimeHighlightsTooltip from './RealtimeHighlightsFeed'; +import styles from './RealtimeHighlightsIndicator.m.less'; + +export default function RealtimeHighlightsIndicator() { + return ( + +
+
+ Ai highlighter active +
+ + ); +} diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx new file mode 100644 index 000000000000..848de78f3855 --- /dev/null +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx @@ -0,0 +1,46 @@ +import { Button } from 'antd'; +import React from 'react'; +import { INewClipData } from 'services/highlighter/models/highlighter.models'; +import { getUniqueEmojiConfigFromAiInfo } from '../ClipPreviewInfo'; +import { EGame } from 'services/highlighter/models/ai-highlighter.models'; + +interface RealtimeHighlightItemProps { + clipData: INewClipData; + onEventItemClick: (highlight: any) => void; + game?: EGame; // Optional game prop, if needed +} + +export default function RealtimeHighlightsItem(props: RealtimeHighlightItemProps) { + const { clipData, onEventItemClick, game } = props; + const emojiDisplayConfig = getUniqueEmojiConfigFromAiInfo(clipData.aiClipInfo, game); + return ( +
+
+ {emojiDisplayConfig.map((displayConfig, index) => { + return ( + + {displayConfig.emoji} {displayConfig.description} + + ); + })} +
+ + +
+ ); +} diff --git a/app/components-react/root/StudioFooter.tsx b/app/components-react/root/StudioFooter.tsx index f28752866efd..c87dad8bd6e9 100644 --- a/app/components-react/root/StudioFooter.tsx +++ b/app/components-react/root/StudioFooter.tsx @@ -12,6 +12,8 @@ import StartStreamingButton from './StartStreamingButton'; import NotificationsArea from './NotificationsArea'; import { Tooltip } from 'antd'; import { confirmAsync } from 'components-react/modals'; +import RealtimeHighlightsIndicator from '../highlighter/realtime-highlights/RealtimeHighlightsIndicator'; +import { useRealmObject } from '../hooks/realm'; export default function StudioFooterComponent() { const { @@ -23,8 +25,12 @@ export default function StudioFooterComponent() { PerformanceService, SettingsService, UserService, + RealtimeHighlighterService, } = Services; + const realtimeHighlighterEnabled = useRealmObject(RealtimeHighlighterService.ephemeralState) + .isRunning; + const { streamingStatus, isLoggedIn, @@ -132,6 +138,7 @@ export default function StudioFooterComponent() {
+ {realtimeHighlighterEnabled && }
{isLoggedIn && }
{recordingModeEnabled && ( -
- ); + return ; } // In Progress From 6958e64def89c2ab4b17caeeaff9a0ffc4b2e99a Mon Sep 17 00:00:00 2001 From: ggolda Date: Tue, 24 Jun 2025 15:34:53 -0600 Subject: [PATCH 26/41] added webcam coordinates for realtime highlighter --- .../realtime-highlighter-service.ts | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index f8c034c17976..1b4548d75253 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -3,9 +3,10 @@ import { EventEmitter } from 'events'; import { EReplayBufferState, StreamingService } from 'services/streaming'; import { Subject, Subscription } from 'rxjs'; import { INewClipData } from './models/highlighter.models'; -import { EGame, IAiClipInfo, IInput } from './models/ai-highlighter.models'; -import { SettingsService } from 'app-services'; -import { getVideoDuration } from './cut-highlight-clips'; +import { EGame, IAiClipInfo, ICoordinates, IInput } from './models/ai-highlighter.models'; +import { ScenesService, SettingsService, SourcesService } from 'app-services'; +import { getVideoDuration, getVideoResolution } from './cut-highlight-clips'; +import { IResolution } from './models/rendering.models'; class LocalVisionService extends EventEmitter { currentGame: string | null = null; @@ -174,6 +175,8 @@ export class RealtimeHighlighterService extends Service { @Inject() private streamingService: StreamingService; @Inject() private settingsService: SettingsService; + @Inject() private scenesService: ScenesService; + @Inject() private sourcesService: SourcesService; private visionService = new LocalVisionService(); private isRunning = false; @@ -349,7 +352,7 @@ export class RealtimeHighlighterService extends Service { const mergedHighlights: any[] = this.mergeOverlappingHighlights(unrefinedHighlights); - const clips = this.createClipsFromHighlights( + const clips = await this.createClipsFromHighlights( mergedHighlights, replayBufferDurationSeconds, path, @@ -361,13 +364,14 @@ export class RealtimeHighlighterService extends Service { /** * Creates clips from detected highlights. Several highlights can be merged into one clip */ - private createClipsFromHighlights( + private async createClipsFromHighlights( mergedHighlights: any[], replayBufferDuration: number, path: string, ) { const clips = []; for (const highlight of mergedHighlights) { + const resolution = await getVideoResolution(path); // if more than 3 inputs, assign maximum score (1.0), otherwise normalize the score const score = highlight.inputs.length >= 3 ? 1.0 : highlight.score / RealtimeHighlighterService.MAX_SCORE; @@ -377,6 +381,7 @@ export class RealtimeHighlighterService extends Service { metadata: { round: this.currentRound, game: this.visionService.currentGame as EGame, + webcam_coordinates: this.findWebcamCoordinates(resolution), }, }; @@ -509,4 +514,50 @@ export class RealtimeHighlighterService extends Service { } return unrefinedHighlights; } + + private findWebcamCoordinates(videoResolution: IResolution): ICoordinates | null { + const activeSceneId = this.scenesService.views.activeSceneId; + const sources = this.sourcesService.views.getSourcesByType('dshow_input'); + if (sources.length === 0) { + return null; + } + + for (const source of sources) { + const sceneItems = this.scenesService.views.getSceneItemsBySourceId(source.sourceId); + for (const sceneItem of sceneItems) { + if (!sceneItem.visible) { + continue; + } + if (sceneItem.sceneId !== activeSceneId) { + continue; + } + + const x = sceneItem.transform.position.x; + const y = sceneItem.transform.position.y; + const width = sceneItem.width * sceneItem.transform.scale.x; + const height = sceneItem.height * sceneItem.transform.scale.y; + + const x1 = Math.max(x, 0); + const y1 = Math.max(y, 0); + const x2 = Math.min(x + width, sceneItem.width); + const y2 = Math.min(y + height, sceneItem.height); + + // convert coordinates to the video resolution coordinates from videoResolution + const scaleX = videoResolution.width / sceneItem.width; + const scaleY = videoResolution.height / sceneItem.height; + const scaledX1 = Math.round(x1 * scaleX); + const scaledY1 = Math.round(y1 * scaleY); + const scaledX2 = Math.round(x2 * scaleX); + const scaledY2 = Math.round(y2 * scaleY); + + return { + x1: scaledX1, + y1: scaledY1, + x2: scaledX2, + y2: scaledY2, + }; + } + } + return null; + } } From 32eae2fc794e4b4191f325773f345c0f6264248e Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Thu, 26 Jun 2025 13:59:34 +0200 Subject: [PATCH 27/41] styling --- .../highlighter/RealtimeIndicator.m.less | 31 ++++++--- .../highlighter/RealtimeIndicator.tsx | 32 +++++----- .../highlighter/StreamCard.tsx | 2 +- .../RealtimeHighlightsFeed.tsx | 63 ++++++++++++++++--- .../RealtimeHighlightsIndicator.tsx | 10 +-- app/components-react/pages/Highlighter.tsx | 23 ++++++- .../highlighter/models/game-config.models.ts | 2 +- .../realtime-highlighter-service.ts | 17 ++++- 8 files changed, 130 insertions(+), 50 deletions(-) diff --git a/app/components-react/highlighter/RealtimeIndicator.m.less b/app/components-react/highlighter/RealtimeIndicator.m.less index 48a37324c3f3..fa3cef252c09 100644 --- a/app/components-react/highlighter/RealtimeIndicator.m.less +++ b/app/components-react/highlighter/RealtimeIndicator.m.less @@ -17,7 +17,7 @@ opacity: 0.3; top: 0; left: 0; - animation: pulse 2.4s infinite; + animation: pulse 2.5s infinite; } .dot { @@ -31,16 +31,20 @@ @keyframes pulse { 0% { - transform: scale(1); - opacity: 0.3; + transform: scale(0.5); + opacity: 0.5; } 50% { - transform: scale(1.5); - opacity: 0.15; + transform: scale(1.4); + opacity: 0; + } + 99% { + transform: scale(1.4); + opacity: 0; } 100% { - transform: scale(1); - opacity: 0.3; + transform: scale(0.5); + opacity: 0.5; } } .realtime-detection-action { @@ -48,7 +52,7 @@ justify-content: space-between; display: flex; padding: 6px 11px; - padding-right: 20px; + padding-right: 10px; width: 248px; align-items: center; gap: 8px; @@ -72,5 +76,14 @@ background: linear-gradient(90deg, #ffffff00, #5d1e1e, #ffffff00); transition: transform 0.5s ease-in-out; } -.realtime-cancel-button { + +.emoji { + z-index: 999; + margin-right: 4px; + width: 16px; + height: 16px; + display: flex; + align-items: center; + justify-content: center; + text-align: center; } diff --git a/app/components-react/highlighter/RealtimeIndicator.tsx b/app/components-react/highlighter/RealtimeIndicator.tsx index 508e9bce40f7..ae7b4b506b49 100644 --- a/app/components-react/highlighter/RealtimeIndicator.tsx +++ b/app/components-react/highlighter/RealtimeIndicator.tsx @@ -3,34 +3,34 @@ import React, { useState, useEffect } from 'react'; import { $t } from 'services/i18n'; import styles from './RealtimeIndicator.m.less'; import cx from 'classnames'; +import { EGame } from 'services/highlighter/models/ai-highlighter.models'; +import { GAME_CONFIGS } from 'services/highlighter/models/game-config.models'; +import { TRealtimeFeedEvent } from './realtime-highlights/RealtimeHighlightsFeed'; export default function HighlightGenerator({ - emoji, + eventType, emitCancel, }: { - emoji?: string; + eventType?: TRealtimeFeedEvent; emitCancel: () => void; }) { const [animateOnce, setAnimateOnce] = useState(false); + const [emoji, setEmoji] = useState(''); // Run animation once when emoji prop changes useEffect(() => { - if (emoji) { + if (eventType) { + setEmoji(getEmojiByEventType(eventType)); setAnimateOnce(true); const timeout = setTimeout(() => setAnimateOnce(false), 2000); return () => clearTimeout(timeout); } - }, [emoji]); + }, [eventType]); - function triggerDetection() { - if (animateOnce) { - return; - } - setAnimateOnce(true); - setTimeout(() => { - setAnimateOnce(false); - }, 2000); + function getEmojiByEventType(eventType: { type: string; game: EGame }) { + return GAME_CONFIGS[eventType.game].inputTypeMap[eventType.type]?.emoji || '🤖'; } + return (
@@ -39,21 +39,19 @@ export default function HighlightGenerator({ // onAnimationEnd={() => setAnimateOnce(false)} /> {animateOnce ? ( -
🔫
+
{emoji}
) : (
)} -

- {animateOnce ? $t('Clip detected') : $t('Ai detection in progress')} -

+

{$t('Ai detection in progress')}

)} @@ -137,8 +160,28 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt trigger={trigger} overlayClassName={styles.eventTooltip} autoAdjustOverflow={false} + visible={showTooltip} > - {children} +
{ + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + timeoutRef.current = null; + } + }} + onMouseLeave={() => { + if (showTooltip) { + setShowTooltip(undefined); + } + }} + > + { + RealtimeHighlighterService.actions.stop(); + }} + /> +
); } diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.tsx index 0699d0f2b0e0..a1a4f5c17419 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsIndicator.tsx @@ -1,14 +1,8 @@ import React from 'react'; import RealtimeHighlightsTooltip from './RealtimeHighlightsFeed'; import styles from './RealtimeHighlightsIndicator.m.less'; +import RealtimeIndicator from '../RealtimeIndicator'; export default function RealtimeHighlightsIndicator() { - return ( - -
-
- Ai highlighter active -
- - ); + return ; } diff --git a/app/components-react/pages/Highlighter.tsx b/app/components-react/pages/Highlighter.tsx index 783066511171..9ceadc502aba 100644 --- a/app/components-react/pages/Highlighter.tsx +++ b/app/components-react/pages/Highlighter.tsx @@ -12,13 +12,19 @@ import StreamView from 'components-react/highlighter/StreamView'; import ClipsView from 'components-react/highlighter/ClipsView'; import UpdateModal from 'components-react/highlighter/UpdateModal'; import { EAvailableFeatures } from 'services/incremental-rollout'; +import Utils from 'services/utils'; export default function Highlighter(props: { params?: { view: string } }) { - const { HighlighterService, IncrementalRolloutService, UsageStatisticsService } = Services; + const { + HighlighterService, + IncrementalRolloutService, + UsageStatisticsService, + RealtimeHighlighterService, + } = Services; const aiHighlighterFeatureEnabled = IncrementalRolloutService.views.featureIsEnabled( EAvailableFeatures.aiHighlighter, ); - + const isDevMode = Utils.isDevMode(); const v = useVuex(() => ({ useAiHighlighter: HighlighterService.views.useAiHighlighter, })); @@ -93,6 +99,19 @@ export default function Highlighter(props: { params?: { view: string } }) { default: return ( <> + {isDevMode && ( + <> + {/* TODO: remove below + fe = fake event */} + + {/* fc = fake clip */} + + + )} {aiHighlighterFeatureEnabled && updaterModal} { diff --git a/app/services/highlighter/models/game-config.models.ts b/app/services/highlighter/models/game-config.models.ts index 6373fff46a7e..8da7b7ff835f 100644 --- a/app/services/highlighter/models/game-config.models.ts +++ b/app/services/highlighter/models/game-config.models.ts @@ -319,7 +319,7 @@ const UNSET_CONFIG: IGameConfig = { }; // Each game must have a config like and the config must be added here. -const GAME_CONFIGS: Record = { +export const GAME_CONFIGS: Record = { [EGame.FORTNITE]: FORTNITE_CONFIG, [EGame.WARZONE]: WARZONE_CONFIG, [EGame.BLACK_OPS_6]: BLACK_OPS_6_CONFIG, diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index f0d59d1edea7..677f793e0476 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -1,13 +1,14 @@ import { InitAfter, Inject, Service } from 'services/core'; import { EventEmitter } from 'events'; import { EReplayBufferState, StreamingService } from 'services/streaming'; -import { Subject, Subscription } from 'rxjs'; +import { BehaviorSubject, Subject, Subscription } from 'rxjs'; import { INewClipData } from './models/highlighter.models'; import { EGame, IAiClipInfo, IInput } from './models/ai-highlighter.models'; import { SettingsService } from 'app-services'; import { getVideoDuration } from './cut-highlight-clips'; import { ObjectSchema } from 'realm'; import { RealmObject } from '../realm'; +import { FORTNITE_CONFIG } from './models/game-config.models'; class LocalVisionService extends EventEmitter { currentGame: string | null = null; @@ -186,6 +187,7 @@ export class RealtimeHighlighterService extends Service { private static MAX_SCORE = 5; highlightsReady = new Subject(); + latestDetectedEvent = new BehaviorSubject<{ type: string; game: EGame } | null>(null); highlights: INewClipData[] = []; ephemeralState = RealtimeHighlighterEphemeralState.inject(); @@ -278,7 +280,7 @@ export class RealtimeHighlighterService extends Service { this.settingsService.setSettingsPatch({ Output: { RecRBTime: seconds } }); } - addFakeEvent() { + addFakeClip() { const clips: INewClipData[] = [ { aiClipInfo: { @@ -295,6 +297,12 @@ export class RealtimeHighlighterService extends Service { ]; this.highlightsReady.next(clips); } + + triggerFakeEvent() { + const inputTypeKeys = Object.keys(FORTNITE_CONFIG.inputTypeMap); + const randomKey = inputTypeKeys[Math.floor(Math.random() * inputTypeKeys.length)]; + this.latestDetectedEvent.next({ type: randomKey, game: EGame.FORTNITE }); + } /** * This method is called periodically to save replay events to file at correct time * when the highlight ends. @@ -342,6 +350,11 @@ export class RealtimeHighlighterService extends Service { return; } + this.latestDetectedEvent.next({ + type: event.name, + game: EGame.FORTNITE, + // game: this.visionService.currentGame as EGame, + }); event.timestamp = Date.now(); this.currentReplayEvents.push(event); From 21e30a3a81f1ec034f1858e79a5fc755f216d63a Mon Sep 17 00:00:00 2001 From: ggolda Date: Thu, 26 Jun 2025 12:25:02 -0600 Subject: [PATCH 28/41] added new source type to play highlighter reels --- app/components-react/windows/AddSource.tsx | 2 + .../windows/source-showcase/SourceGrid.tsx | 6 + .../source-showcase/useSourceShowcase.tsx | 2 + app/i18n/en-US/sources.json | 3 + .../properties-managers/highlight-manager.ts | 145 ++++++++++++++++++ .../properties-managers/replay-manager.ts | 1 - app/services/sources/sources-api.ts | 3 +- app/services/sources/sources-data.ts | 8 + app/services/sources/sources.ts | 3 + 9 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 app/services/sources/properties-managers/highlight-manager.ts diff --git a/app/components-react/windows/AddSource.tsx b/app/components-react/windows/AddSource.tsx index e9e8c31af6f6..f0cf2228f2d6 100644 --- a/app/components-react/windows/AddSource.tsx +++ b/app/components-react/windows/AddSource.tsx @@ -62,6 +62,8 @@ export default function AddSource() { let name; if (sourceAddOptions.propertiesManager === 'replay') { name = $t('Instant Replay'); + } else if (sourceAddOptions.propertiesManager === 'highlighter') { + name = $t('Highlight Reel'); } else if (sourceAddOptions.propertiesManager === 'streamlabels') { name = $t('Stream Label'); } else if (sourceAddOptions.propertiesManager === 'iconLibrary') { diff --git a/app/components-react/windows/source-showcase/SourceGrid.tsx b/app/components-react/windows/source-showcase/SourceGrid.tsx index c8de9b44ccad..b7754e109ed5 100644 --- a/app/components-react/windows/source-showcase/SourceGrid.tsx +++ b/app/components-react/windows/source-showcase/SourceGrid.tsx @@ -240,6 +240,12 @@ export default function SourceGrid(p: { activeTab: string }) { type="replay" excludeWrap={excludeWrap} /> + {designerMode && ( { + this.tick(); + }, 1000); + } + + private tick() { + const isVisible = this.isSourceVisibleInActiveScene(); + if (isVisible && this.inProgress) { + return this.keepPlaying(); + } else if (isVisible && !this.inProgress) { + return this.startPlaying(); + } else if (!isVisible && this.inProgress) { + return this.stopPlaying(); + } + } + + private keepPlaying() { + const currentTime = Date.now(); + + if (!this.stopAt) { + return; + } + + if (currentTime > this.stopAt - 500) { + this.setVolume(0.3); + } else if (currentTime > this.stopAt - 1000) { + this.setVolume(0.5); + } else if (currentTime > this.stopAt - 2000) { + this.setVolume(0.7); + } + + // if time is less than stopAt, do nothing + if (currentTime < this.stopAt) { + return; + } + + // if we reached the end of the highlight, switch to the next one + const highlightsCount = this.realtimeHighlighterService.highlights.length; + if (highlightsCount === 0) { + console.log('No highlights to play'); + return; + } + + const nextIndex = this.currentReplayIndex + 1; + if (nextIndex < highlightsCount) { + this.queueNextHighlight(nextIndex); + } else { + this.queueNextHighlight(0); // Loop back to the first highlight + } + } + + private startPlaying() { + if (this.realtimeHighlighterService.highlights.length === 0) { + console.log('No highlights to play'); + return; + } + + console.log('Start playing highlights'); + this.inProgress = true; + this.queueNextHighlight(0); + } + + private stopPlaying() { + this.inProgress = false; + this.stopAt = null; + this.currentReplayIndex = null; + console.log('Stop playing highlights'); + } + + /** + * Check if Instant Replay source is in currently active scene + * and visible to on the stream. + * + * One source can be in multiple scene items in different scenes, + * so we need to check all scene items that link to the source + * and check if any of them is visible in the active scene. + */ + private isSourceVisibleInActiveScene(): boolean { + const activeSceneId = this.scenesService.views.activeSceneId; + // for some reason for instant replay source, the scene id is stored inside obsSource.name + const sourceItems = this.scenesService.views.getSceneItemsBySourceId(this.obsSource.name); + for (const item of sourceItems) { + if (item.sceneId !== activeSceneId) { + continue; + } + if (item.visible) { + return true; + } + } + return false; + } + + private queueNextHighlight(index: number) { + // have to do this due to the bug with overlapping audio + const source = this.sourcesService.views.getSource(this.obsSource.name); + if (source) { + // return volume to normal + source.updateSettings({ deflection: 1.0 }); + console.log(`Pausing source: ${source.name}`); + source.getObsInput()?.pause(); + } + + const highlight = this.realtimeHighlighterService.highlights[index]; + this.stopAt = Date.now() + (highlight.endTime - highlight.endTrim) * 1000; + this.obsSource.update({ local_file: highlight.path }); + this.currentReplayIndex = index; + console.log(`Queued next highlight: ${highlight.path}`); + } + + private setVolume(volume: number) { + const source = this.sourcesService.views.getSource(this.obsSource.name); + if (source) { + console.log('changing volume to', volume); + source.updateSettings({ deflection: volume }); + } + } +} diff --git a/app/services/sources/properties-managers/replay-manager.ts b/app/services/sources/properties-managers/replay-manager.ts index 9a5a3809ffdb..a29f287b324d 100644 --- a/app/services/sources/properties-managers/replay-manager.ts +++ b/app/services/sources/properties-managers/replay-manager.ts @@ -119,7 +119,6 @@ export class ReplayManager extends PropertiesManager { if (item.sceneId !== activeSceneId) { continue; } - if (item.visible) { return true; } diff --git a/app/services/sources/sources-api.ts b/app/services/sources/sources-api.ts index b852cca69878..f60eb4b7af0f 100644 --- a/app/services/sources/sources-api.ts +++ b/app/services/sources/sources-api.ts @@ -151,7 +151,8 @@ export type TPropertiesManager = | 'streamlabels' | 'platformApp' | 'replay' - | 'iconLibrary'; + | 'iconLibrary' + | 'highlighter'; export interface ISourcesState { sources: Dictionary; diff --git a/app/services/sources/sources-data.ts b/app/services/sources/sources-data.ts index 77532fc2d39b..7e3bd7dd5093 100644 --- a/app/services/sources/sources-data.ts +++ b/app/services/sources/sources-data.ts @@ -291,6 +291,14 @@ export const SourceDisplayData = (): { [key: string]: ISourceDisplayData } => ({ icon: 'icon-replay-buffer', group: 'media', }, + highlighter: { + name: $t('Highlight Reel'), + shortDesc: $t('Display highlight clips'), + description: $t('Automatically display the latest clips captured with AI Highlighter.'), + demoFilename: 'media.png', + icon: 'icon-replay-buffer', + group: 'media', + }, icon_library: { name: $t('Custom Icon'), description: $t('Displays an icon from one of many selections'), diff --git a/app/services/sources/sources.ts b/app/services/sources/sources.ts index 16de72378b94..f25de4584697 100644 --- a/app/services/sources/sources.ts +++ b/app/services/sources/sources.ts @@ -42,6 +42,7 @@ import { CustomizationService } from '../customization'; import { EAvailableFeatures, IncrementalRolloutService } from '../incremental-rollout'; import { EMonitoringType, EDeinterlaceMode, EDeinterlaceFieldOrder } from '../../../obs-api'; import { GuestCamService } from 'services/guest-cam'; +import { HighlightManager } from './properties-managers/highlight-manager'; export { EDeinterlaceMode, EDeinterlaceFieldOrder } from '../../../obs-api'; @@ -57,6 +58,7 @@ export const PROPERTIES_MANAGER_TYPES = { streamlabels: StreamlabelsManager, platformApp: PlatformAppManager, replay: ReplayManager, + highlighter: HighlightManager, iconLibrary: IconLibraryManager, }; @@ -715,6 +717,7 @@ export class SourcesService extends StatefulService { let propertiesName = SourceDisplayData()[source.type].name; if (propertiesManagerType === 'replay') propertiesName = $t('Instant Replay'); + if (propertiesManagerType === 'highlighter') propertiesName = $t('Highlight Reel'); if (propertiesManagerType === 'streamlabels') propertiesName = $t('Stream Label'); // uncomment the source type to use it's React version From 3ecf9563c01733210b7a2d9d156eb7f2570f155a Mon Sep 17 00:00:00 2001 From: ggolda Date: Thu, 26 Jun 2025 15:13:49 -0600 Subject: [PATCH 29/41] reverted changes to instant replay source, should remain intact --- .../properties-managers/highlight-manager.ts | 2 + .../properties-managers/replay-manager.ts | 142 +----------------- 2 files changed, 5 insertions(+), 139 deletions(-) diff --git a/app/services/sources/properties-managers/highlight-manager.ts b/app/services/sources/properties-managers/highlight-manager.ts index 1565a9acb375..ec4cccc16cf1 100644 --- a/app/services/sources/properties-managers/highlight-manager.ts +++ b/app/services/sources/properties-managers/highlight-manager.ts @@ -24,6 +24,8 @@ export class HighlightManager extends PropertiesManager { } init() { + // reset state of the media source, sometimes it gets stuck + this.obsSource.update({ local_file: '' }); console.log('HighlightManager initialized'); // if ai highlighter is not active, preserve old behavior setInterval(() => { diff --git a/app/services/sources/properties-managers/replay-manager.ts b/app/services/sources/properties-managers/replay-manager.ts index a29f287b324d..f691550109ad 100644 --- a/app/services/sources/properties-managers/replay-manager.ts +++ b/app/services/sources/properties-managers/replay-manager.ts @@ -1,153 +1,17 @@ -import { - HighlighterService, - RealtimeHighlighterService, - ScenesService, - SourcesService, -} from 'app-services'; import { PropertiesManager } from './properties-manager'; import { Inject } from 'services/core/injector'; import { StreamingService } from 'services/streaming'; export class ReplayManager extends PropertiesManager { @Inject() streamingService: StreamingService; - @Inject() realtimeHighlighterService: RealtimeHighlighterService; - @Inject() scenesService: ScenesService; - @Inject() highlighterService: HighlighterService; - @Inject() sourcesService: SourcesService; - - private inProgress = false; - private stopAt: number | null = null; - private currentReplayIndex: number = 0; get denylist() { return ['is_local_file', 'local_file']; } init() { - console.log('ReplayManager initialized'); - // if ai highlighter is not active, preserve old behavior - if (!this.highlighterService.views.useAiHighlighter) { - console.log('Using legacy Instant Replay behavior'); - this.streamingService.replayBufferFileWrite.subscribe(filePath => { - this.obsSource.update({ local_file: filePath }); - }); - return; - } - - setInterval(() => { - this.tick(); - }, 1000); - } - - private tick() { - const isVisible = this.isSourceVisibleInActiveScene(); - if (isVisible && this.inProgress) { - return this.keepPlaying(); - } else if (isVisible && !this.inProgress) { - return this.startPlaying(); - } else if (!isVisible && this.inProgress) { - return this.stopPlaying(); - } - } - - private keepPlaying() { - const currentTime = Date.now(); - - if (!this.stopAt) { - return; - } - - if (currentTime > this.stopAt - 500) { - this.setVolume(0.3); - } else if (currentTime > this.stopAt - 1000) { - this.setVolume(0.5); - } else if (currentTime > this.stopAt - 2000) { - this.setVolume(0.7); - } - - // if time is less than stopAt, do nothing - if (currentTime < this.stopAt) { - return; - } - - // if we reached the end of the highlight, switch to the next one - const highlightsCount = this.realtimeHighlighterService.highlights.length; - if (highlightsCount === 0) { - console.log('No highlights to play'); - return; - } - - const nextIndex = this.currentReplayIndex + 1; - if (nextIndex < highlightsCount) { - this.queueNextHighlight(nextIndex); - } else { - this.queueNextHighlight(0); // Loop back to the first highlight - } - } - - private startPlaying() { - if (this.realtimeHighlighterService.highlights.length === 0) { - console.log('No highlights to play'); - return; - } - - console.log('Start playing highlights'); - this.inProgress = true; - this.queueNextHighlight(0); - } - - private stopPlaying() { - this.inProgress = false; - this.stopAt = null; - this.currentReplayIndex = null; - console.log('Stop playing highlights'); - } - - /** - * Check if Instant Replay source is in currently active scene - * and visible to on the stream. - * - * One source can be in multiple scene items in different scenes, - * so we need to check all scene items that link to the source - * and check if any of them is visible in the active scene. - */ - private isSourceVisibleInActiveScene(): boolean { - const activeSceneId = this.scenesService.views.activeSceneId; - // for some reason for instant replay source, the scene id is stored inside obsSource.name - const sourceItems = this.scenesService.views.getSceneItemsBySourceId(this.obsSource.name); - for (const item of sourceItems) { - if (item.sceneId !== activeSceneId) { - continue; - } - if (item.visible) { - return true; - } - } - return false; - } - - private queueNextHighlight(index: number) { - // have to do this due to the bug with overlapping audio - const source = this.sourcesService.views.getSource(this.obsSource.name); - if (source) { - // return volume to normal - source.updateSettings({ deflection: 1.0 }); - console.log(`Pausing source: ${source.name}`); - source.getObsInput()?.pause(); - } - - const highlight = this.realtimeHighlighterService.highlights[index]; - this.stopAt = Date.now() + (highlight.endTime - highlight.endTrim) * 1000; - this.obsSource.update({ local_file: highlight.path }); - this.currentReplayIndex = index; - console.log(`Queued next highlight: ${highlight.path}`); - } - - private setVolume(volume: number) { - const source = this.sourcesService.views.getSource(this.obsSource.name); - if (source) { - console.log('changing volume to', volume); - source.updateSettings({ deflection: volume }); - } + this.streamingService.replayBufferFileWrite.subscribe(filePath => { + this.obsSource.update({ local_file: filePath }); + }); } } From 39a97e13d9be9845c52bdefc09a7e6c234d1e987 Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Fri, 27 Jun 2025 12:53:41 +0200 Subject: [PATCH 30/41] added more styling --- .../highlighter/ClipPreviewInfo.tsx | 16 ++-- .../RealtimeHighlightsFeed.tsx | 73 +++++++++++-------- .../RealtimeHighlightsItem.m.less | 49 +++++++++++++ .../RealtimeHighlightsItem.tsx | 25 +++---- .../realtime-highlighter-service.ts | 19 +++-- 5 files changed, 128 insertions(+), 54 deletions(-) create mode 100644 app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.m.less diff --git a/app/components-react/highlighter/ClipPreviewInfo.tsx b/app/components-react/highlighter/ClipPreviewInfo.tsx index 4a9a11019a2b..88ca0d7afc81 100644 --- a/app/components-react/highlighter/ClipPreviewInfo.tsx +++ b/app/components-react/highlighter/ClipPreviewInfo.tsx @@ -36,36 +36,40 @@ export default function ClipPreviewInfo({ export interface EmojiConfig { emoji: string; + count: number; description: string; type: string; } export function getUniqueEmojiConfigFromAiInfo(aiInfos: IAiClipInfo, game?: EGame): EmojiConfig[] { - const uniqueInputTypes = new Set(); + const typeCounts: Record = {}; if (aiInfos.inputs && Array.isArray(aiInfos.inputs)) { aiInfos.inputs.forEach(aiInput => { if (aiInput.type) { - uniqueInputTypes.add(aiInput.type); + typeCounts[aiInput.type] = (typeCounts[aiInput.type] || 0) + 1; } }); } - const eventDisplays = Array.from(uniqueInputTypes).map(type => { + const uniqueInputTypes = Object.keys(typeCounts); + + const eventDisplays = uniqueInputTypes.map(type => { + const count = typeCounts[type]; if (game) { const eventInfo = getEventConfig(game, type); - if (eventInfo) { return { emoji: eventInfo.emoji, - description: eventInfo.description.singular, + description: count > 1 ? eventInfo.description.plural : eventInfo.description.singular, + count, type, }; } } - return { emoji: '⚡', description: type, + count, type, }; }); diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx index fd6a3e21d522..a42d1361d630 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx @@ -85,6 +85,10 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt useEffect(() => { if (highlightClips) { setShowTooltip(true); + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + timeoutRef.current = null; + } timeoutRef.current = setTimeout(() => { setShowTooltip(undefined); }, 5000); @@ -115,41 +119,52 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt ); } + function getItemOpacity(index: number) { + if (index === highlightClips.length - 1) return 1; + if (index === highlightClips.length - 2) return 0.75; + return 0.5; + } + const tooltipContent = (
+

Clipped highlights

+ {hasMoreEvents && ( +
+ +
+ )} + + {highlightClips.length === 0 &&

Your clipped highlights will appear here

} + {highlightClips && - highlightClips.map(clipData => ( - + highlightClips.map((clipData, index) => ( +
+ +
))}
- - {hasMoreEvents && ( -
- -
- )} - - {isDevMode && ( - - )}
); diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.m.less b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.m.less new file mode 100644 index 000000000000..e45004f62f20 --- /dev/null +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.m.less @@ -0,0 +1,49 @@ +.item-wrapper { + padding: 8px; + width: 100%; + display: flex; + background-color: #ffffff10; + gap: 4px; + align-items: center; + flex-direction: column; + border-radius: 4px; +} + +.appear-animation { + animation: fadeIn 0.4s ease-out forwards; + animation-delay: 0.5s; + opacity: 0; +} + +.status-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: #ff4d4f; + flex-shrink: 0; + animation: pulse 2s infinite; +} + +@keyframes fadeIn { + 0% { + opacity: 0; + } + 100% { + opacity: 1; + } +} + +@keyframes pulse { + 0% { + opacity: 1; + transform: scale(1); + } + 50% { + opacity: 0.7; + transform: scale(1.1); + } + 100% { + opacity: 1; + transform: scale(1); + } +} diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx index 848de78f3855..739e22013bf9 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx @@ -3,37 +3,36 @@ import React from 'react'; import { INewClipData } from 'services/highlighter/models/highlighter.models'; import { getUniqueEmojiConfigFromAiInfo } from '../ClipPreviewInfo'; import { EGame } from 'services/highlighter/models/ai-highlighter.models'; +import styles from './RealtimeHighlightsItem.m.less'; +import cx from 'classnames'; interface RealtimeHighlightItemProps { clipData: INewClipData; onEventItemClick: (highlight: any) => void; game?: EGame; // Optional game prop, if needed + latestItem: boolean; } export default function RealtimeHighlightsItem(props: RealtimeHighlightItemProps) { - const { clipData, onEventItemClick, game } = props; + const { clipData, onEventItemClick, game, latestItem } = props; const emojiDisplayConfig = getUniqueEmojiConfigFromAiInfo(clipData.aiClipInfo, game); return ( -
-
+
+
{emojiDisplayConfig.map((displayConfig, index) => { return ( - - {displayConfig.emoji} {displayConfig.description} - +

+ {displayConfig.emoji} {displayConfig.count > 1 ? displayConfig.count : ''}{' '} + {displayConfig.description} +

); })}
- {/* fc = fake clip */} - - - )} {aiHighlighterFeatureEnabled && updaterModal} { diff --git a/app/services/highlighter/realtime-highlighter-service.ts b/app/services/highlighter/realtime-highlighter-service.ts index 0b9ca9d052b1..284a3d7175fc 100644 --- a/app/services/highlighter/realtime-highlighter-service.ts +++ b/app/services/highlighter/realtime-highlighter-service.ts @@ -288,31 +288,6 @@ export class RealtimeHighlighterService extends Service { return randomKey; } - addFakeClip() { - const clips: INewClipData[] = [ - { - aiClipInfo: { - inputs: [ - { type: this.getRandomEventType() } as IInput, - { type: this.getRandomEventType() } as IInput, - { type: this.getRandomEventType() } as IInput, - ], - score: 0, - metadata: {}, - }, - path: '/Users/jankalthoefer/Desktop/streams/djnardi/djnardi-short.mp4', - startTime: 15, - endTime: 30, - startTrim: 0, - endTrim: 0, - }, - ]; - this.highlightsReady.next(clips); - } - - triggerFakeEvent() { - this.latestDetectedEvent.next({ type: this.getRandomEventType(), game: EGame.FORTNITE }); - } /** * This method is called periodically to save replay events to file at correct time * when the highlight ends. From fd520a1180c22a740d764b92d9658113ae5ecfad Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Fri, 1 Aug 2025 16:28:02 +0200 Subject: [PATCH 34/41] change to be able to open correct stream from clip --- .../RealtimeHighlightsFeed.tsx | 17 ++++++++++------- app/services/highlighter/index.ts | 2 +- .../realtime-highlighter-service.ts | 19 ++++++++++++++++--- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx index a42d1361d630..88178242cea9 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx @@ -17,6 +17,7 @@ import { useRealmObject } from '../../hooks/realm'; import { EMenuItemKey } from 'services/side-nav'; import Utils from 'services/utils'; import RealtimeIndicator from '../RealtimeIndicator'; +import { IRealtimeHighlightClipData } from 'services/highlighter/realtime-highlighter-service'; interface IRealtimeHighlightTooltipProps { placement?: TooltipPlacement; @@ -40,7 +41,7 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt const [lastEvent, setLastEvent] = useState(null); const [showTooltip, setShowTooltip] = useState(undefined); - const [highlightClips, setHighlightClips] = useState([]); + const [highlightClips, setHighlightClips] = useState([]); let hasMoreEvents = false; const isDevMode = Utils.isDevMode(); @@ -52,9 +53,9 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt console.log('Initializing RealtimeEventTooltip component'); realtimeHighlightSubscription = RealtimeHighlighterService.highlightsReady.subscribe( - newClipData => { + realtimeClipData => { // This will be called when highlights are ready - const highlights = Object.values(newClipData); + const highlights = Object.values(realtimeClipData); setHighlightClips(prevEvents => { const updatedEvents = [...prevEvents, ...highlights]; // Remove excess events from the beginning if we exceed maxEvents @@ -64,7 +65,7 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt return updatedEvents; }); hasMoreEvents = highlights.length > maxEvents; - console.log('Realtime highlights are ready:', newClipData); + console.log('Realtime highlights are ready:', realtimeClipData); }, ); @@ -106,14 +107,14 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt ); } - function onEventItemClick(event: INewClipData) { + function onEventItemClick(streamId: string) { console.log('Open single highlight view for event:', event); // Navigate to specific highlight in stream NavigationService.actions.navigate( 'Highlighter', { view: EHighlighterView.CLIPS, - id: event.path, + id: streamId, }, EMenuItemKey.Highlighter, ); @@ -159,7 +160,9 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt key={clipData.path} clipData={clipData} game={currentGame} - onEventItemClick={onEventItemClick} + onEventItemClick={() => { + onEventItemClick(clipData.streamId); + }} latestItem={highlightClips.length - 1 === index} />
diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index b747e433dd48..d00c4e7235e4 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -517,7 +517,7 @@ export class HighlighterService extends PersistentStatefulService(); + highlightsReady = new Subject(); latestDetectedEvent = new BehaviorSubject<{ type: string; game: EGame } | null>(null); highlights: INewClipData[] = []; + currentStreamId: string | null = null; ephemeralState = RealtimeHighlighterEphemeralState.inject(); @Inject() private streamingService: StreamingService; @@ -224,7 +229,9 @@ export class RealtimeHighlighterService extends Service { }); } - async start() { + async start(streamId: string) { + this.currentStreamId = streamId; + console.log('Starting RealtimeHighlighterService'); if (this.isRunning) { console.warn('RealtimeHighlighterService is already running'); @@ -271,6 +278,7 @@ export class RealtimeHighlighterService extends Service { this.replayBufferFileReadySubscription?.unsubscribe(); + this.currentStreamId = null; this.isRunning = false; } @@ -288,6 +296,9 @@ export class RealtimeHighlighterService extends Service { return randomKey; } + triggerFakeEvent() { + this.latestDetectedEvent.next({ type: this.getRandomEventType(), game: EGame.FORTNITE }); + } /** * This method is called periodically to save replay events to file at correct time * when the highlight ends. @@ -397,7 +408,9 @@ export class RealtimeHighlighterService extends Service { path, ); - this.highlightsReady.next(clips); + const clipsWithStreamId = clips.map(clip => ({ ...clip, streamId: this.currentStreamId })); + + this.highlightsReady.next(clipsWithStreamId); } /** From d9d8008088b4c7c08f4943a145f6c9a95c2834be Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Mon, 4 Aug 2025 11:29:19 +0200 Subject: [PATCH 35/41] navigation to clip view fix --- .../RealtimeHighlightsFeed.tsx | 2 +- app/components-react/pages/Highlighter.tsx | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx index 88178242cea9..d5e0ec613a3d 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx @@ -108,7 +108,7 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt } function onEventItemClick(streamId: string) { - console.log('Open single highlight view for event:', event); + console.log('Open single highlight view for event:', streamId); // Navigate to specific highlight in stream NavigationService.actions.navigate( 'Highlighter', diff --git a/app/components-react/pages/Highlighter.tsx b/app/components-react/pages/Highlighter.tsx index e9fd1b653c36..095336c1afb9 100644 --- a/app/components-react/pages/Highlighter.tsx +++ b/app/components-react/pages/Highlighter.tsx @@ -14,7 +14,7 @@ import UpdateModal from 'components-react/highlighter/UpdateModal'; import { EAvailableFeatures } from 'services/incremental-rollout'; import Utils from 'services/utils'; -export default function Highlighter(props: { params?: { view: string } }) { +export default function Highlighter(props: { params?: { view: string; id?: string } }) { const { HighlighterService, IncrementalRolloutService, @@ -35,9 +35,25 @@ export default function Highlighter(props: { params?: { view: string } }) { let initialViewState: IViewState; if (props.params?.view) { - const view = - props.params?.view === 'settings' ? EHighlighterView.SETTINGS : EHighlighterView.STREAM; - initialViewState = { view }; + console.log('Highlighter view from params:', props.params); + + switch (props.params?.view) { + case EHighlighterView.SETTINGS: + initialViewState = { view: EHighlighterView.SETTINGS }; + break; + + case EHighlighterView.STREAM: + initialViewState = { view: EHighlighterView.STREAM }; + break; + + case EHighlighterView.CLIPS: + initialViewState = { view: EHighlighterView.CLIPS, id: props.params.id }; + break; + + default: + initialViewState = { view: EHighlighterView.SETTINGS }; + break; + } } else if (streamAmount > 0 && clipsAmount > 0 && aiHighlighterFeatureEnabled) { initialViewState = { view: EHighlighterView.STREAM }; } else if (clipsAmount > 0) { From 8b266babac56deb79cb41e895322f056f2178a84 Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Mon, 4 Aug 2025 11:57:13 +0200 Subject: [PATCH 36/41] fix 0 when no round was detected --- app/components-react/highlighter/ClipPreviewInfo.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/components-react/highlighter/ClipPreviewInfo.tsx b/app/components-react/highlighter/ClipPreviewInfo.tsx index 88ca0d7afc81..8b3f6b545c58 100644 --- a/app/components-react/highlighter/ClipPreviewInfo.tsx +++ b/app/components-react/highlighter/ClipPreviewInfo.tsx @@ -27,9 +27,11 @@ export default function ClipPreviewInfo({ {eventDisplays.map((event, index) => { return {event.emoji}; })} - {clip.aiInfo.metadata?.round && ( -
{`Round: ${clip.aiInfo.metadata.round}`}
- )}{' '} + {clip.aiInfo.metadata?.round !== undefined && + clip.aiInfo.metadata?.round !== null && + clip.aiInfo.metadata?.round !== 0 && ( +
{`Round: ${clip.aiInfo.metadata.round}`}
+ )}
); } From 90b23cada641798d17f1991d7951375699a30d0f Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Mon, 4 Aug 2025 13:15:49 +0200 Subject: [PATCH 37/41] merge 1 --- .../highlighter/ClipPreviewInfo.tsx | 8 ++++---- .../RealtimeHighlightsFeed.tsx | 17 +++++++++-------- .../RealtimeHighlightsItem.tsx | 5 ++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app/components-react/highlighter/ClipPreviewInfo.tsx b/app/components-react/highlighter/ClipPreviewInfo.tsx index 8b3f6b545c58..5cba8945523a 100644 --- a/app/components-react/highlighter/ClipPreviewInfo.tsx +++ b/app/components-react/highlighter/ClipPreviewInfo.tsx @@ -15,7 +15,7 @@ export default function ClipPreviewInfo({ return No event data; } - const eventDisplays = getUniqueEmojiConfigFromAiInfo(clip.aiInfo, game); + const eventDisplays = getUniqueEmojiConfigFromAiInfo(clip.aiInfo); return (
= {}; if (aiInfos.inputs && Array.isArray(aiInfos.inputs)) { aiInfos.inputs.forEach(aiInput => { @@ -57,8 +57,8 @@ export function getUniqueEmojiConfigFromAiInfo(aiInfos: IAiClipInfo, game?: EGam const eventDisplays = uniqueInputTypes.map(type => { const count = typeCounts[type]; - if (game) { - const eventInfo = getEventConfig(game, type); + if (aiInfos.metadata?.game) { + const eventInfo = getEventConfig(aiInfos.metadata.game, type); if (eventInfo) { return { emoji: eventInfo.emoji, diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx index d5e0ec613a3d..d715d2061fb9 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx @@ -36,13 +36,13 @@ export type TRealtimeFeedEvent = { }; export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightTooltipProps) { - const { placement, trigger, maxEvents = 5 } = props; + const { placement, trigger, maxEvents = 3 } = props; const { HighlighterService, RealtimeHighlighterService, NavigationService } = Services; const [lastEvent, setLastEvent] = useState(null); const [showTooltip, setShowTooltip] = useState(undefined); - const [highlightClips, setHighlightClips] = useState([]); - let hasMoreEvents = false; + const [highlightClips, setHighlightClips] = useState([]); + const [hasMoreEvents, setHasMoreEvents] = useState(false); const isDevMode = Utils.isDevMode(); const currentGame = useRealmObject(RealtimeHighlighterService.ephemeralState).game; @@ -61,10 +61,13 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt // Remove excess events from the beginning if we exceed maxEvents if (updatedEvents.length > maxEvents) { updatedEvents.splice(0, updatedEvents.length - maxEvents); + if (hasMoreEvents === false) { + setHasMoreEvents(true); + } } return updatedEvents; }); - hasMoreEvents = highlights.length > maxEvents; + console.log('Realtime highlights are ready:', realtimeClipData); }, ); @@ -133,6 +136,7 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt {hasMoreEvents && (
diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx index 739e22013bf9..4458f9384ed0 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx @@ -9,13 +9,12 @@ import cx from 'classnames'; interface RealtimeHighlightItemProps { clipData: INewClipData; onEventItemClick: (highlight: any) => void; - game?: EGame; // Optional game prop, if needed latestItem: boolean; } export default function RealtimeHighlightsItem(props: RealtimeHighlightItemProps) { - const { clipData, onEventItemClick, game, latestItem } = props; - const emojiDisplayConfig = getUniqueEmojiConfigFromAiInfo(clipData.aiClipInfo, game); + const { clipData, onEventItemClick, latestItem } = props; + const emojiDisplayConfig = getUniqueEmojiConfigFromAiInfo(clipData.aiClipInfo); return (
From 5712c46ff1d15a06ea381d3f97a823bf874c35f1 Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Mon, 4 Aug 2025 13:17:01 +0200 Subject: [PATCH 38/41] merge 2 --- .../highlighter/StreamCard.tsx | 13 +++++-- .../RealtimeHighlightsFeed.tsx | 2 +- app/services/highlighter/index.ts | 36 +++++++++++++++---- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/app/components-react/highlighter/StreamCard.tsx b/app/components-react/highlighter/StreamCard.tsx index fac313ab3cbd..d1d4371bebc9 100644 --- a/app/components-react/highlighter/StreamCard.tsx +++ b/app/components-react/highlighter/StreamCard.tsx @@ -241,7 +241,9 @@ export default function StreamCard({ { + cancelHighlightGeneration(); + }} emitExportVideo={() => exportVideo(streamId)} emitShowStreamClips={showStreamClips} clipsOfStreamAreLoading={clipsOfStreamAreLoading} @@ -320,7 +322,14 @@ function ActionBar({ }; if (stream?.state.type === EAiDetectionState.REALTIME_DETECTION_IN_PROGRESS) { - return ; + return ( + { + emitCancelHighlightGeneration(); + }} + /> + ); } // In Progress diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx index d715d2061fb9..3805bbd39183 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx @@ -197,7 +197,7 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt { - RealtimeHighlighterService.actions.stop(); + HighlighterService.actions.stopRealtimeHighlighter(); }} />
diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index d00c4e7235e4..bd41bfbfb40a 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -137,6 +137,7 @@ export class HighlighterService extends PersistentStatefulService stream.state.type === 'detection-in-progress') + .filter(stream => stream.state.type === EAiDetectionState.IN_PROGRESS) .forEach(stream => { this.UPDATE_HIGHLIGHTED_STREAM({ ...stream, @@ -376,6 +377,16 @@ export class HighlighterService extends PersistentStatefulService stream.state.type === EAiDetectionState.REALTIME_DETECTION_IN_PROGRESS) + .forEach(stream => { + this.UPDATE_HIGHLIGHTED_STREAM({ + ...stream, + state: { type: EAiDetectionState.FINISHED }, + }); + }); + this.views.clips.forEach(c => { this.UPDATE_CLIP({ path: c.path, @@ -517,7 +528,7 @@ export class HighlighterService extends PersistentStatefulService Date: Mon, 4 Aug 2025 13:51:14 +0200 Subject: [PATCH 39/41] merge fix --- .../realtime-highlights/RealtimeHighlightsFeed.tsx | 6 ++++-- .../realtime-highlights/RealtimeHighlightsItem.tsx | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx index 3805bbd39183..7c01a1ce2ec6 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx @@ -41,7 +41,7 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt const [lastEvent, setLastEvent] = useState(null); const [showTooltip, setShowTooltip] = useState(undefined); - const [highlightClips, setHighlightClips] = useState([]); + const [highlightClips, setHighlightClips] = useState([]); const [hasMoreEvents, setHasMoreEvents] = useState(false); const isDevMode = Utils.isDevMode(); @@ -163,7 +163,9 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt { + onEventItemClick(clipData.streamId); + }} latestItem={highlightClips.length - 1 === index} />
diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx index 4458f9384ed0..e9ae95f4b125 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx @@ -8,7 +8,7 @@ import cx from 'classnames'; interface RealtimeHighlightItemProps { clipData: INewClipData; - onEventItemClick: (highlight: any) => void; + onEventItemClick: () => void; latestItem: boolean; } @@ -34,7 +34,7 @@ export default function RealtimeHighlightsItem(props: RealtimeHighlightItemProps style={{ width: '100%' }} onClick={e => { e.stopPropagation(); - onEventItemClick(clipData); + onEventItemClick(); }} > View highlight From e933dc526e06df01eaacde33e606a4318c5fe3ba Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Mon, 4 Aug 2025 15:16:41 +0200 Subject: [PATCH 40/41] rm logs + added translations removed comments + added translations --- .../highlighter/RealtimeIndicator.tsx | 2 +- app/components-react/highlighter/StreamCard.tsx | 3 ++- .../RealtimeHighlightsFeed.tsx | 10 +++------- .../RealtimeHighlightsItem.tsx | 3 ++- app/components-react/highlighter/utils.ts | 2 +- app/components-react/pages/Highlighter.tsx | 3 --- app/i18n/en-US/highlighter.json | 6 +++++- app/services/highlighter/index.ts | 12 +----------- .../highlighter/realtime-highlighter-service.ts | 14 ++++---------- .../properties-managers/highlight-manager.ts | 16 ++++++++-------- 10 files changed, 27 insertions(+), 44 deletions(-) diff --git a/app/components-react/highlighter/RealtimeIndicator.tsx b/app/components-react/highlighter/RealtimeIndicator.tsx index e82e4b3cb438..bdcbbbfc7afe 100644 --- a/app/components-react/highlighter/RealtimeIndicator.tsx +++ b/app/components-react/highlighter/RealtimeIndicator.tsx @@ -57,7 +57,7 @@ export default function HighlightGenerator({ )}

{' '} - {animateOnce ? `${description} detected` : $t('Ai detection in progress')} + {animateOnce ? `${description} detected` : $t('AI detection in progress')}

)} - {highlightClips.length === 0 &&

Your clipped highlights will appear here

} + {highlightClips.length === 0 &&

{$t('Your clipped highlights will appear here')}

} {highlightClips && highlightClips.map((clipData, index) => ( diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx index e9ae95f4b125..54f37d36b5b3 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsItem.tsx @@ -5,6 +5,7 @@ import { getUniqueEmojiConfigFromAiInfo } from '../ClipPreviewInfo'; import { EGame } from 'services/highlighter/models/ai-highlighter.models'; import styles from './RealtimeHighlightsItem.m.less'; import cx from 'classnames'; +import { $t } from 'services/i18n'; interface RealtimeHighlightItemProps { clipData: INewClipData; @@ -37,7 +38,7 @@ export default function RealtimeHighlightsItem(props: RealtimeHighlightItemProps onEventItemClick(); }} > - View highlight + {$t('View highlight')}
); diff --git a/app/components-react/highlighter/utils.ts b/app/components-react/highlighter/utils.ts index 66da04ecd484..4f6f77824590 100644 --- a/app/components-react/highlighter/utils.ts +++ b/app/components-react/highlighter/utils.ts @@ -93,7 +93,7 @@ export function aiFilterClips( ): TClip[] { const { rounds, targetDuration, includeAllEvents } = options; - // TODO: this just handles the error for now, probably not the best way. + // TODO: this just handles the error for now. Needs to be changed at one point const selectedRounds = rounds.length === 1 && rounds[0] === 0 ? [ diff --git a/app/components-react/pages/Highlighter.tsx b/app/components-react/pages/Highlighter.tsx index 095336c1afb9..42c42375eb46 100644 --- a/app/components-react/pages/Highlighter.tsx +++ b/app/components-react/pages/Highlighter.tsx @@ -24,7 +24,6 @@ export default function Highlighter(props: { params?: { view: string; id?: strin const aiHighlighterFeatureEnabled = IncrementalRolloutService.views.featureIsEnabled( EAvailableFeatures.aiHighlighter, ); - const isDevMode = Utils.isDevMode(); const v = useVuex(() => ({ useAiHighlighter: HighlighterService.views.useAiHighlighter, })); @@ -35,8 +34,6 @@ export default function Highlighter(props: { params?: { view: string; id?: strin let initialViewState: IViewState; if (props.params?.view) { - console.log('Highlighter view from params:', props.params); - switch (props.params?.view) { case EHighlighterView.SETTINGS: initialViewState = { view: EHighlighterView.SETTINGS }; diff --git a/app/i18n/en-US/highlighter.json b/app/i18n/en-US/highlighter.json index 798875896d52..93293b99db32 100644 --- a/app/i18n/en-US/highlighter.json +++ b/app/i18n/en-US/highlighter.json @@ -205,5 +205,9 @@ "Auto-generate game highlight reels of your stream": "Auto-generate game highlight reels of your stream", "Get stream highlights!": "Get stream highlights!", "Turn your gameplay into epic highlight reels": "Turn your gameplay into epic highlight reels", - "Dominate, showcase, inspire!": "Dominate, showcase, inspire!" + "Dominate, showcase, inspire!": "Dominate, showcase, inspire!", + "View all highlights": "View all highlights", + "Your clipped highlights will appear here": "Your clipped highlights will appear here", + "View highlight": "View highlight", + "AI detection in progress": "AI detection in progress" } \ No newline at end of file diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index bd41bfbfb40a..e9e9a4e41bfd 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -417,19 +417,13 @@ export class HighlighterService extends PersistentStatefulService { - console.log('Realtime highlights received:', highlights); - console.log(streamInfo.id); - console.log('Add ai clips now...'); // for some reason addAiClips adds only the first highlight for (const highlight of highlights) { this.addAiClips([highlight], { id: streamInfo.id || '', game: streamInfo.game }); } - console.log('Load clips now...'); await this.loadClips(streamInfo.id); let count = 0; @@ -445,7 +439,6 @@ export class HighlighterService extends PersistentStatefulService { - console.log('replay buffer clip received'); const streamId = streamInfo?.id || undefined; let endTime: number | undefined; @@ -455,7 +448,7 @@ export class HighlighterService extends PersistentStatefulService { const data = JSON.parse(event.data); - console.log('Received events:', data); this.currentGame = data.game || null; const events = data.events; for (const event of events) { - console.log('Emitting event:', event); this.emit('event', event); } }; @@ -53,7 +51,7 @@ class LocalVisionService extends EventEmitter { fetch('http://localhost:8000/reset_state', { method: 'POST', }).then(() => { - console.log('VisionService state reset'); + // console.log('VisionService state reset'); }); } @@ -64,7 +62,7 @@ class LocalVisionService extends EventEmitter { } this.isRunning = false; - console.log('Stopping VisionService'); + // console.log('Stopping VisionService'); this.eventSource?.close(); } } @@ -98,7 +96,7 @@ class MockVisionService extends EventEmitter { } this.isRunning = true; - console.log('Starting VisionService'); + // console.log('Starting VisionService'); this.scheduleNext(); } @@ -109,7 +107,7 @@ class MockVisionService extends EventEmitter { } this.isRunning = false; - console.log('Stopping VisionService'); + // console.log('Stopping VisionService'); if (this.timeoutId) { clearTimeout(this.timeoutId); this.timeoutId = null; @@ -121,9 +119,7 @@ class MockVisionService extends EventEmitter { const minDelay = 1 * 1000; const delay = Math.floor(Math.random() * (maxDelay - minDelay + 1)) + minDelay; - console.log('Triggering random event'); this.emitRandomEvent(); - console.log(`Next trigger in ${delay / 1000} seconds`); this.timeoutId = setTimeout(() => { this.scheduleNext(); @@ -157,7 +153,6 @@ class MockVisionService extends EventEmitter { const eventKeys = Object.keys(events) as EventKey[]; const randomEvent = eventKeys[Math.floor(Math.random() * eventKeys.length)]; - console.log(`Emitting event: ${randomEvent}`); const settings = events[randomEvent]; this.emit('event', { @@ -232,7 +227,6 @@ export class RealtimeHighlighterService extends Service { async start(streamId: string) { this.currentStreamId = streamId; - console.log('Starting RealtimeHighlighterService'); if (this.isRunning) { console.warn('RealtimeHighlighterService is already running'); return; diff --git a/app/services/sources/properties-managers/highlight-manager.ts b/app/services/sources/properties-managers/highlight-manager.ts index ec4cccc16cf1..daa93e870922 100644 --- a/app/services/sources/properties-managers/highlight-manager.ts +++ b/app/services/sources/properties-managers/highlight-manager.ts @@ -26,7 +26,7 @@ export class HighlightManager extends PropertiesManager { init() { // reset state of the media source, sometimes it gets stuck this.obsSource.update({ local_file: '' }); - console.log('HighlightManager initialized'); + // console.log('HighlightManager initialized'); // if ai highlighter is not active, preserve old behavior setInterval(() => { this.tick(); @@ -67,7 +67,7 @@ export class HighlightManager extends PropertiesManager { // if we reached the end of the highlight, switch to the next one const highlightsCount = this.realtimeHighlighterService.highlights.length; if (highlightsCount === 0) { - console.log('No highlights to play'); + // console.log('No highlights to play'); return; } @@ -81,11 +81,11 @@ export class HighlightManager extends PropertiesManager { private startPlaying() { if (this.realtimeHighlighterService.highlights.length === 0) { - console.log('No highlights to play'); + // console.log('No highlights to play'); return; } - console.log('Start playing highlights'); + // console.log('Start playing highlights'); this.inProgress = true; this.queueNextHighlight(0); } @@ -94,7 +94,7 @@ export class HighlightManager extends PropertiesManager { this.inProgress = false; this.stopAt = null; this.currentReplayIndex = null; - console.log('Stop playing highlights'); + // console.log('Stop playing highlights'); } /** @@ -126,7 +126,7 @@ export class HighlightManager extends PropertiesManager { if (source) { // return volume to normal source.updateSettings({ deflection: 1.0 }); - console.log(`Pausing source: ${source.name}`); + // console.log(`Pausing source: ${source.name}`); source.getObsInput()?.pause(); } @@ -134,13 +134,13 @@ export class HighlightManager extends PropertiesManager { this.stopAt = Date.now() + (highlight.endTime - highlight.endTrim) * 1000; this.obsSource.update({ local_file: highlight.path }); this.currentReplayIndex = index; - console.log(`Queued next highlight: ${highlight.path}`); + // console.log(`Queued next highlight: ${highlight.path}`); } private setVolume(volume: number) { const source = this.sourcesService.views.getSource(this.obsSource.name); if (source) { - console.log('changing volume to', volume); + // console.log('changing volume to', volume); source.updateSettings({ deflection: volume }); } } From 0c98ba030c797a3234871e46f7c07d8a5df68ace Mon Sep 17 00:00:00 2001 From: marvinoffers Date: Tue, 5 Aug 2025 18:03:04 +0200 Subject: [PATCH 41/41] adjusted realtimeindicator styling --- .../highlighter/RealtimeIndicator.m.less | 18 ++++++++++++---- .../highlighter/RealtimeIndicator.tsx | 21 +++++++++++++------ .../highlighter/StreamCard.tsx | 1 + .../RealtimeHighlightsFeed.tsx | 1 + 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/app/components-react/highlighter/RealtimeIndicator.m.less b/app/components-react/highlighter/RealtimeIndicator.m.less index fa3cef252c09..34d02e6b3cf7 100644 --- a/app/components-react/highlighter/RealtimeIndicator.m.less +++ b/app/components-react/highlighter/RealtimeIndicator.m.less @@ -48,22 +48,32 @@ } } .realtime-detection-action { + font-weight: 500; + letter-spacing: 0.2px; position: relative; justify-content: space-between; display: flex; - padding: 6px 11px; + padding: 4px 8px; padding-right: 10px; - width: 248px; + width: 236px; align-items: center; gap: 8px; border-radius: 48px; - border: 1px solid #301e24; background: #301e24; color: white; - border-color: #5d1e1e; overflow: hidden; } +.realtime-detection-streamcard { + background-color: var(--button); + color: var(--paragraph); + font-weight: 400; + font-size: 16px; + border-radius: 8px; + height: 40px; + width: 100%; +} + .activity-animation { transform: translateX(200%); } diff --git a/app/components-react/highlighter/RealtimeIndicator.tsx b/app/components-react/highlighter/RealtimeIndicator.tsx index bdcbbbfc7afe..804c408a5537 100644 --- a/app/components-react/highlighter/RealtimeIndicator.tsx +++ b/app/components-react/highlighter/RealtimeIndicator.tsx @@ -10,9 +10,11 @@ import { TRealtimeFeedEvent } from './realtime-highlights/RealtimeHighlightsFeed export default function HighlightGenerator({ eventType, emitCancel, + location, }: { eventType?: TRealtimeFeedEvent; emitCancel: () => void; + location: 'streamCard' | 'statusBar'; }) { const [animateOnce, setAnimateOnce] = useState(false); const [emoji, setEmoji] = useState(''); @@ -41,12 +43,19 @@ export default function HighlightGenerator({ return String(value).charAt(0).toUpperCase() + String(value).slice(1); } return ( -
+
-
setAnimateOnce(false)} - /> + {location === 'statusBar' && ( +
setAnimateOnce(false)} + /> + )} {animateOnce ? (
{emoji}
) : ( @@ -55,7 +64,7 @@ export default function HighlightGenerator({
)} -

+

{' '} {animateOnce ? `${description} detected` : $t('AI detection in progress')}

diff --git a/app/components-react/highlighter/StreamCard.tsx b/app/components-react/highlighter/StreamCard.tsx index 21ba2d32ee42..464f3d991763 100644 --- a/app/components-react/highlighter/StreamCard.tsx +++ b/app/components-react/highlighter/StreamCard.tsx @@ -325,6 +325,7 @@ function ActionBar({ if (stream?.state.type === EAiDetectionState.REALTIME_DETECTION_IN_PROGRESS) { return ( { emitCancelHighlightGeneration(); diff --git a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx index a695d07d58e9..c326181ffac5 100644 --- a/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx +++ b/app/components-react/highlighter/realtime-highlights/RealtimeHighlightsFeed.tsx @@ -193,6 +193,7 @@ export default function RealtimeHighlightsTooltip(props: IRealtimeHighlightToolt }} > { HighlighterService.actions.stopRealtimeHighlighter();