From db937f0a6026838efb15059f52cd98ec4b817a61 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 26 Mar 2025 13:20:53 +0100 Subject: [PATCH 01/34] skip download if running highlighter locally --- app/services/highlighter/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index d46602db71b8..8fb9e9f56aca 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -72,6 +72,7 @@ import { cutHighlightClips, getVideoDuration } from './cut-highlight-clips'; import { reduce } from 'lodash'; import { extractDateTimeFromPath, fileExists } from './file-utils'; import { addVerticalFilterToExportOptions } from './vertical-export'; +import Utils from '../utils'; @InitAfter('StreamingService') export class HighlighterService extends PersistentStatefulService { @@ -1167,7 +1168,7 @@ export class HighlighterService extends PersistentStatefulService Date: Wed, 26 Mar 2025 13:21:17 +0100 Subject: [PATCH 02/34] subtitle ffmpeg base --- .../highlighter/models/rendering.models.ts | 1 + .../highlighter/rendering/frame-writer.ts | 138 +++++++++++++++--- 2 files changed, 119 insertions(+), 20 deletions(-) diff --git a/app/services/highlighter/models/rendering.models.ts b/app/services/highlighter/models/rendering.models.ts index 955ee866d084..3d4571d5bdf6 100644 --- a/app/services/highlighter/models/rendering.models.ts +++ b/app/services/highlighter/models/rendering.models.ts @@ -10,6 +10,7 @@ export interface IExportOptions { height: number; preset: TPreset; complexFilter?: string; + subtitles?: {}; } // types for highlighter video operations diff --git a/app/services/highlighter/rendering/frame-writer.ts b/app/services/highlighter/rendering/frame-writer.ts index 7e8e0264fb8f..24cab22bd3dd 100644 --- a/app/services/highlighter/rendering/frame-writer.ts +++ b/app/services/highlighter/rendering/frame-writer.ts @@ -2,6 +2,8 @@ import execa from 'execa'; import { IExportOptions } from '../models/rendering.models'; import { FADE_OUT_DURATION, FFMPEG_EXE } from '../constants'; import { FrameWriteError } from './errors'; +import fs from 'fs-extra'; +import path from 'path'; export class FrameWriter { constructor( @@ -15,7 +17,7 @@ export class FrameWriter { exitPromise: Promise; - private startFfmpeg() { + private async startFfmpeg() { /* eslint-disable */ const args = [ // Video Input @@ -33,24 +35,18 @@ export class FrameWriter { '-', // Audio Input - '-i', - this.audioInput, // Input Mapping - '-map', - '0:v:0', - '-map', - '1:a:0', - - // Filters - '-af', - `afade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( - this.duration - (FADE_OUT_DURATION + 0.2), - 0, - )}`, + // '-map', + // '0:v:0', ]; + if (this.options.subtitles || true) { + console.log('adding subtitles'); - this.addVideoFilters(args); + await this.addSubtitleInput(args, this.options, 'transcription Class'); + } + this.addAudioFilters(args); + this.addVideoFilters(args, true); //!!this.options.subtitles args.push( ...[ @@ -76,6 +72,7 @@ export class FrameWriter { this.outputPath, ], ); + console.log(args.join(' ')); /* eslint-enable */ this.ffmpeg = execa(FFMPEG_EXE, args, { @@ -101,21 +98,83 @@ export class FrameWriter { console.log('ffmpeg:', data.toString()); }); } - - private addVideoFilters(args: string[]) { - const fadeFilter = `format=yuv420p,fade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( + // "subtitles='C\:\\\\Users\\\\jan\\\\Videos\\\\color.srt'" + private addVideoFilters(args: string[], addSubtitleFilter: boolean) { + // args.push( + // '-filter_complex', + // '[0:v][1:v]overlay=0:0[final];[final]format=yuv420p,fade=type=out:duration=1:start_time=4', + // ); + const subtitleFilter = addSubtitleFilter ? '[0:v][1:v]overlay=0:0[final];' : ''; + const fadeFilter = `${subtitleFilter}[final]format=yuv420p,fade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( this.duration - (FADE_OUT_DURATION + 0.2), 0, )}`; if (this.options.complexFilter) { args.push('-vf', this.options.complexFilter + `[final]${fadeFilter}`); } else { - args.push('-vf', fadeFilter); + args.push('-filter_complex', fadeFilter); } } + private addAudioFilters(args: string[]) { + args.push( + '-i', + this.audioInput, + '-map', + '2:a:0', + '-af', + `afade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( + this.duration - (FADE_OUT_DURATION + 0.2), + 0, + )}`, + ); + } + private async addSubtitleInput(args: string[], exportOptions: IExportOptions, subtitles: string) { + subtitles = testSubtitle; + const subtitleDuration = 10; + const exportWidth = exportOptions.width; + const exportHeight = exportOptions.height; + // this.outputPath + const subtitleDirectory = path.join(path.dirname(this.outputPath), 'temp_subtitles'); + if (!fs.existsSync(subtitleDirectory)) { + fs.mkdirSync(subtitleDirectory, { recursive: true }); + } + + // const subtitlePath = path.join(subtitleDirectory, 'subtitle.srt'); + // // await fs.writeFile(subtitlePath, subtitles); + // console.log('subtitle path', subtitlePath); + // // Escape backslashes in the subtitle path for ffmpeg + // const escapedSubtitlePath = subtitlePath.replace(/\\/g, '\\\\\\\\').replace(':', '\\:'); + + // console.log('escaped subtitle path', escapedSubtitlePath); + // // create subtitle pngs + + // // ffmpeg -f lavfi -i "color=color=white@0.0:s=1280x720:r=30,format=rgba,subtitles='C\:\\\\Users\\\\jan\\\\Videos\\\\color.srt':alpha=1" -t 10 "C:\Users\jan\Videos\temp_subtitles\subtitles_%04d.png" + // const subtitleArgs = [ + // '-f', + // 'lavfi', + // '-i', + // `color=color=white@0.0:s=${exportWidth}x${exportHeight}:r=30,format=rgba,subtitles=\'${escapedSubtitlePath}\':alpha=1`, + // '-c:v', + // 'png', + // // duration of the subtitles + // '-t', + // subtitleDuration.toString(), + // // temp directory for the subtitle images + // `${subtitleDirectory}\\subtitles_%04d.png`, + // '-y', + // '-loglevel', + // 'debug', + // ]; + // // -f lavfi -i "color=color=white@0.0:s=1280x720:r=30,format=rgba,subtitles='C\:\\\\Users\\\\jan\\\\Videos\\\\color.srt':alpha=1" -t 10 "C:\Users\jan\Videos\temp_subtitles\subtitles_%04d.png" + // /* eslint-enable */ + // await execa(FFMPEG_EXE, subtitleArgs); + + args.push('-i', `${subtitleDirectory}\\subtitles_%04d.png`); + } + async writeNextFrame(frameBuffer: Buffer) { - if (!this.ffmpeg) this.startFfmpeg(); + if (!this.ffmpeg) await this.startFfmpeg(); try { await new Promise((resolve, reject) => { @@ -137,3 +196,42 @@ export class FrameWriter { return this.exitPromise; } } + +const testSubtitle = ` +1 +00:00:02,000 --> 00:00:03,000 +Hi my name is Jan and this is colorful + +2 +00:00:03,000 --> 00:00:04,000 +Hi my name is Jan and this is colorful + +3 +00:00:04,000 --> 00:00:05,000 +Hi my name is Jan and this is colorful + +4 +00:00:05,000 --> 00:00:06,000 +Hi my name is Jan and this is colorful + +5 +00:00:06,000 --> 00:00:07,000 +Hi my name is Jan and this is colorful + +6 +00:00:07,000 --> 00:00:08,000 +Hi my name is Jan and this is colorful + +7 +00:00:08,000 --> 00:00:09,000 +Hi my name is Jan and this is colorful + +8 +00:00:09,000 --> 00:00:10,000 +Hi my name is Jan and this is colorful + +9 +00:00:10,000 --> 00:00:11,000 +Hi my name is Jan and this is colorful + +`; From 1c1bff330e3ab5b01f738e51a0af0f81ccc679c8 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Fri, 28 Mar 2025 10:09:23 +0100 Subject: [PATCH 03/34] subtitles base --- .../highlighter/rendering/subtitles.ts | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 app/services/highlighter/rendering/subtitles.ts diff --git a/app/services/highlighter/rendering/subtitles.ts b/app/services/highlighter/rendering/subtitles.ts new file mode 100644 index 000000000000..8a6fdfabcadb --- /dev/null +++ b/app/services/highlighter/rendering/subtitles.ts @@ -0,0 +1,105 @@ +import sharp from 'sharp'; + +const SUBTITLE_PATH = 'C:/Users/jan/Videos/temp_subtitles/'; +interface ITextItem { + text: string; + fontColor?: string; + fontSize?: number; + fontWeight?: string; + fontFamily?: string; +} +async function generateStyledTextImage(textItems: ITextItem[]) { + const width = 1280; + const height = 720; + + // Build SVG with different text styles + let svgText = ``; + + // Calculate total width to center + let totalWidth = 0; + + // First pass to calculate positions + textItems.forEach(item => { + // Approximate width for positioning - not perfect but works for basic centering + const textWidth = item.text.length * (item.fontSize || 24 * 0.6); + totalWidth += textWidth + 5; // 5px spacing + }); + + const position = 'bottom'; + const yPosition = position === 'bottom' ? height - 20 : height / 2; + + // Create a single text element with tspan children + svgText += ` + `; + + // Calculate starting x position for the first tspan + let currentX = (width - totalWidth) / 2; + + // Add tspan elements for each text item + textItems.forEach(item => { + const textWidth = item.text.length * (item.fontSize || 24 * 0.6); + + // Position tspan relative to the total text width + svgText += ` + ${item.text}`; + + currentX += textWidth + 5; // 5px spacing + }); + + // Close the text element + svgText += ` + + `; + + const buffer = await sharp({ + // Generate PNG with transparent background + create: { + width, + height, + channels: 4, + background: { r: 0, g: 0, b: 0, alpha: 0 }, + }, + }) + .composite([ + { + input: Buffer.from(svgText), + top: 0, + left: 0, + }, + ]) + .png() + .toBuffer(); + + return buffer; +} + +// Usage example +export async function saveStyledExample() { + const textItems: ITextItem[] = [ + { + text: 'Hello', + fontColor: '#ff0000', + fontSize: 32, + fontWeight: 'bold', + }, + { + text: 'World', + fontColor: '#0000ff', + fontSize: 24, + }, + ]; + + const buffer = await generateStyledTextImage(textItems); + await require('fs').promises.writeFile(SUBTITLE_PATH + 'styled-output.png', buffer); +} From ac83aa0be543765f4d8c3ac0fb7554eb4199f4e7 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Fri, 28 Mar 2025 11:31:22 +0100 Subject: [PATCH 04/34] subtitle transcriptions base --- .../highlighter/subtitles/clip-element.ts | 25 + .../highlighter/subtitles/subtitle-clip.ts | 136 +++ .../highlighter/subtitles/subtitle-clips.ts | 109 +++ .../highlighter/subtitles/subtitle-mode.ts | 14 + .../highlighter/subtitles/subtitle-utils.ts | 20 + .../highlighter/subtitles/svg-creator.ts | 286 ++++++ .../highlighter/subtitles/transcription.ts | 816 ++++++++++++++++++ app/services/highlighter/subtitles/word.ts | 168 ++++ 8 files changed, 1574 insertions(+) create mode 100644 app/services/highlighter/subtitles/clip-element.ts create mode 100644 app/services/highlighter/subtitles/subtitle-clip.ts create mode 100644 app/services/highlighter/subtitles/subtitle-clips.ts create mode 100644 app/services/highlighter/subtitles/subtitle-mode.ts create mode 100644 app/services/highlighter/subtitles/subtitle-utils.ts create mode 100644 app/services/highlighter/subtitles/svg-creator.ts create mode 100644 app/services/highlighter/subtitles/transcription.ts create mode 100644 app/services/highlighter/subtitles/word.ts diff --git a/app/services/highlighter/subtitles/clip-element.ts b/app/services/highlighter/subtitles/clip-element.ts new file mode 100644 index 000000000000..66cf2b3d26e9 --- /dev/null +++ b/app/services/highlighter/subtitles/clip-element.ts @@ -0,0 +1,25 @@ +import { Transcription } from './transcription'; + +export abstract class ClipElement { + protected _transcriptionReference: Transcription; + public startIndex: number; + public endIndex: number; + public mediaIndex: number; + + public get startTimeInEdit(): number { + return this._transcriptionReference.getStartTimeInEditAtIndex(this.startIndex); + } + public get endTimeInEdit(): number { + return this._transcriptionReference.getEndTimeInEditAtIndex(this.endIndex); + } + public get startTimeInOriginal(): number { + return this._transcriptionReference.getStartTimeInOriginalAtIndex(this.startIndex); + } + public get endTimeInOriginal(): number { + return this._transcriptionReference.getEndTimeInOriginalAtIndex(this.endIndex); + } + + constructor(transcriptionReference: Transcription) { + this._transcriptionReference = transcriptionReference; + } +} diff --git a/app/services/highlighter/subtitles/subtitle-clip.ts b/app/services/highlighter/subtitles/subtitle-clip.ts new file mode 100644 index 000000000000..969a0c2ae35f --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-clip.ts @@ -0,0 +1,136 @@ +import { SubtitleMode } from './subtitle-mode'; +import { ClipElement } from './clip-element'; +import { Transcription } from './transcription'; +import { toSrtFormat, toVttFormat } from './subtitle-utils'; + +export class SubtitleClip extends ClipElement { + _transcriptionReference: Transcription; + + public get text(): string { + return this._transcriptionReference + .sliceWords(this.startIndex, this.endIndex + 1) + .filter(word => !word.isCut && !word.isPause) + .map(word => word.text) + .join(' '); + } + + constructor( + startIndex: number, + endIndex: number, + mediaIndex: number, + _transcriptionReference: Transcription, + ) { + super(_transcriptionReference); + this.startIndex = startIndex; + this.endIndex = endIndex; + this.mediaIndex = mediaIndex; + } + + public getDynamicSubtitleClips(): SubtitleClip[] { + const dynamicSubtitleClips: SubtitleClip[] = []; + for (let index = this.startIndex; index <= this.endIndex; index++) { + const word = this._transcriptionReference.words[index]; + if (!word.isPause) { + dynamicSubtitleClips.push( + new SubtitleClip(this.startIndex, index, this.mediaIndex, this._transcriptionReference), + ); + } else { + dynamicSubtitleClips.push( + new SubtitleClip(index, index, this.mediaIndex, this._transcriptionReference), + ); + } + } + + // console.log('🚀 ~ dynamicSubtitleClips', dynamicSubtitleClips); + return dynamicSubtitleClips; + } + + // /** + // * Assembly Function for different file specification, subtitles + // */ + public assembleSrtString( + line: string, + startTime: number, + endTime: number, + index: number, + subtitleMode: SubtitleMode, + ) { + let srtStartTime = startTime; + let srtEndTime = endTime; + + let srtPreviousEndTime = + this._transcriptionReference.words[this.startIndex - 1]?.endTimeInEdit || 0; + + if (subtitleMode === SubtitleMode.dynamic) { + srtStartTime = this._transcriptionReference.words[this.endIndex].startTimeInEdit; + srtEndTime = this._transcriptionReference.words[this.endIndex].endTimeInEdit; + srtPreviousEndTime = this._transcriptionReference.words[this.endIndex - 1].endTimeInEdit; + } + let srtString = ''; + if (index > 0) { + if (srtPreviousEndTime > srtStartTime) { + const diff = srtPreviousEndTime - srtStartTime; + srtStartTime -= diff + 0.001; + } + } + if (srtStartTime === srtEndTime) { + return null; + } else if (srtStartTime > srtEndTime) { + const diff = srtEndTime - srtStartTime; + srtEndTime -= diff + 0.001; + } + + if (line) { + srtString += index + 1; + srtString += '\n'; // newLine + srtString += `${toSrtFormat(srtStartTime)} --> ${toSrtFormat(srtEndTime)}`; // Start and endtime + srtString += '\n'; // newLine + srtString += line; // actual Text + srtString += '\n'; // newLine + srtString += '\n'; // newLine + } + return srtString; + } + + assembleVttString( + line: string, + startTime: number, + endTime: number, + index: number, + subtitleMode: SubtitleMode, + ) { + let vttString = ''; + let vttStartTime = startTime; + let vttEndTime = endTime; + let vttPreviousEndTime = this._transcriptionReference.words[this.startIndex - 1]?.endTimeInEdit; + + if (subtitleMode === SubtitleMode.dynamic) { + vttStartTime = this._transcriptionReference.words[this.endIndex].startTimeInEdit; + vttEndTime = this._transcriptionReference.words[this.endIndex].endTimeInEdit; + vttPreviousEndTime = this._transcriptionReference.words[this.endIndex - 1]?.endTimeInEdit; + } + + if (index > 0) { + if (vttPreviousEndTime > vttStartTime) { + const diff = vttPreviousEndTime - vttStartTime; + vttStartTime -= diff + 0.001; + } + } + if (vttStartTime === vttEndTime) { + return null; + } else if (vttStartTime > vttEndTime) { + const diff = vttEndTime - vttStartTime; + vttEndTime -= diff + 0.001; + } + + if (line) { + vttString += '\n'; // newLine + vttString += `${toVttFormat(vttStartTime)} --> ${toVttFormat(vttEndTime)} align:middle`; // Start and endTime + vttString += '\n'; // newLine + vttString += line; // actual Text + vttString += '\n'; // newLine + vttString += '\n'; // newLine + } + return vttString; + } +} diff --git a/app/services/highlighter/subtitles/subtitle-clips.ts b/app/services/highlighter/subtitles/subtitle-clips.ts new file mode 100644 index 000000000000..8cd9b2515cdc --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-clips.ts @@ -0,0 +1,109 @@ +import { SubtitleMode } from './subtitle-mode'; +import { SubtitleClip } from './subtitle-clip'; + +export class SubtitleClips { + private _subtitleClips: SubtitleClip[]; + + public get length(): number { + return this._subtitleClips.length; + } + + constructor() { + this._subtitleClips = []; + } + + public getClipByTime(time: number, mediaIndex: number) { + return this._subtitleClips.find( + subtitleClip => + time < Math.round(subtitleClip.endTimeInOriginal * 100) / 100 && + time >= subtitleClip.startTimeInOriginal && + mediaIndex === subtitleClip.mediaIndex, + ); + } + public push(subtitleClip: SubtitleClip) { + this._subtitleClips.push(subtitleClip); + } + public forEach(callback: (subtitleClip: SubtitleClip) => void) { + for (let index = 0; index < this._subtitleClips.length; index++) { + const subtitleClip = this._subtitleClips[index]; + callback(subtitleClip); + } + } + + public convertToDynamicSubtitles(): void { + const dynamicSubtitleClips: SubtitleClip[] = []; + this._subtitleClips.forEach((subtitleClip: SubtitleClip) => { + dynamicSubtitleClips.push(...subtitleClip.getDynamicSubtitleClips()); + }); + this._subtitleClips = dynamicSubtitleClips; + } + + public getSrtString(subtitleMode: SubtitleMode) { + let srtString = ''; + let indexCounter = 0; + for (let index = 0; index < this._subtitleClips.length; index++) { + const subtitleClip = this._subtitleClips[index]; + const previousSubtitleClip = this._subtitleClips[index - 1]; + + const newLine = subtitleClip.assembleSrtString( + subtitleClip.text, + subtitleClip.startTimeInEdit, + subtitleClip.endTimeInEdit, + indexCounter, + subtitleMode, + ); + if (newLine) { + srtString += newLine; + indexCounter += 1; + } + } + return srtString; + } + public getVttString(subtitleMode: SubtitleMode) { + let vttString = 'WEBVTT\n\n'; + let indexCounter = 0; + for (let index = 0; index < this._subtitleClips.length; index++) { + const subtitleClip = this._subtitleClips[index]; + const previousSubtitleClip = this._subtitleClips[index - 1]; + + const newLine = subtitleClip.assembleVttString( + subtitleClip.text, + subtitleClip.startTimeInEdit, + subtitleClip.endTimeInEdit, + index, + subtitleMode, + ); + if (newLine) { + vttString += newLine; + indexCounter += 1; + } + } + return vttString; + } + + /** + * Returns an array with every start and end index of a subtitle clip + */ + getStartEndArray(subtitleMode: SubtitleMode): { start: number; end: number }[] { + return this._subtitleClips + .map((subtitleClip, index) => { + if (subtitleMode === SubtitleMode.static) { + return { start: subtitleClip.startIndex, end: subtitleClip.endIndex }; + } else if (subtitleMode === SubtitleMode.dynamic) { + if ( + (this._subtitleClips[index + 1]?.startIndex === + this._subtitleClips[index + 1]?.endIndex && + this._subtitleClips[index + 1]?.text.length > 0) || + this._subtitleClips[index]._transcriptionReference.words[ + this._subtitleClips[index].endIndex + ].hasSubtitlebreak === true + ) { + return { start: subtitleClip.startIndex, end: subtitleClip.endIndex }; + } + return { start: null, end: null }; + } + return { start: null, end: null }; + }) + .filter(value => value.start !== null && value.end !== null); + } +} diff --git a/app/services/highlighter/subtitles/subtitle-mode.ts b/app/services/highlighter/subtitles/subtitle-mode.ts new file mode 100644 index 000000000000..a8dd1326a7da --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-mode.ts @@ -0,0 +1,14 @@ +export enum SubtitleMode { + disabled = 'disabled', + dynamic = 'dynamic', + static = 'static', +} +export type TSubtitleHighlightStyle = 'none' | 'background' | 'opacity'; + +export interface ISubtitleConfig { + subtitleMode?: SubtitleMode; + subtitleLength?: number; + + bgColor?: string; + bg?: boolean; +} diff --git a/app/services/highlighter/subtitles/subtitle-utils.ts b/app/services/highlighter/subtitles/subtitle-utils.ts new file mode 100644 index 000000000000..f44dc752cd45 --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-utils.ts @@ -0,0 +1,20 @@ +export function removeLinebreaksFromString(str: string) { + return str.replace(/[\r\n]+/gm, ''); +} +export function roundTime(time: number): number { + return Math.round(time * 100) / 100; +} +export function toSrtFormat(time: number): string { + const o = new Date(0); + const p = new Date(time * 1000); + return new Date(p.getTime() - o.getTime()) + .toISOString() + .split('T')[1] + .split('Z')[0] + .replace('.', ','); +} +export function toVttFormat(time: number): string { + const o = new Date(0); + const p = new Date(time * 1000); + return new Date(p.getTime() - o.getTime()).toISOString().split('T')[1].split('Z')[0]; +} diff --git a/app/services/highlighter/subtitles/svg-creator.ts b/app/services/highlighter/subtitles/svg-creator.ts new file mode 100644 index 000000000000..11a48cc95405 --- /dev/null +++ b/app/services/highlighter/subtitles/svg-creator.ts @@ -0,0 +1,286 @@ +export interface IResolution { + width: number; + height: number; +} +export interface ITextStyle { + fontSize: number; + fontFamily: string; + fontColor: string; + isBold: boolean; + isItalic: boolean; +} +export class SvgCreator { + private lines: string[]; + private fontFamily: string; + private fontSize: number; + private fontColor: string; + private isBold: boolean; + private isItalic: boolean; + + private backgroundColor: string; + private backgroundAlpha: number; + private backgroundBorderRadius: number; + + private lineCount: number; + private lineWidth: number; + private lineHeight: number; + private rectHeight: number; + + private resolution: IResolution; + private subtitleHeightPositionFactor; + + private svgType: 'Subtitle' | 'CanvasText'; + + private x: number; + private y: number; + private scale: number; + private rotation: number; + private backgroundWidth: number; + private rtlLanguage = false; + + constructor( + resolution: IResolution, + textElementOptions?: ITextStyle, + scaleBackground?: boolean, + rightToLeftLanguage = false, + ) { + this.rtlLanguage = rightToLeftLanguage; + this.svgType = 'CanvasText'; + this.resolution = resolution; + this.subtitleHeightPositionFactor = this.calculateSubtitleHeightFactor(resolution); + + if (textElementOptions) { + this.isBold = textElementOptions.isBold; + this.isItalic = textElementOptions.isItalic; + this.fontSize = textElementOptions.fontSize; + this.fontFamily = textElementOptions.fontFamily; + this.fontColor = textElementOptions.fontColor; + // this.backgroundColor = textElementOptions.backgroundColor; + // this.backgroundAlpha = textElementOptions.backgroundColor === 'transparent' ? 0 : 1; + // this.backgroundBorderRadius = 2 * textElementOptions.scale; + // this.lineWidth = textElementOptions.width * textElementOptions.scale; + // this.x = textElementOptions.x; + // this.y = textElementOptions.y; + // this.rectHeight = textElementOptions.height * textElementOptions.scale; + // this.lineHeight = textElementOptions.fontSize * textElementOptions.scale; + // this.scale = textElementOptions.scale; + // this.rotation = textElementOptions.rotation; + // if (!scaleBackground) { + // this.backgroundWidth = textElementOptions.width; + // } else { + // this.backgroundWidth = this.lineWidth; + // } + } + } + + public static getProgressSquare(color: string) { + return ` + + + `; + } + public getSvgWithText(lines: string[], lineWidth: number): string { + this.lines = []; + this.lines = lines; + this.lineCount = lines.length; + + // if (this.isBold) { + // const boldFactor = 1.0; + // const lineWidthDifference = this.lineWidth * boldFactor - this.lineWidth; + // this.lineWidth *= boldFactor; + // this.x -= lineWidthDifference / 2; + // } + if (this.svgType === 'Subtitle') { + // correct line width and add margin + // 10% padding + this.lineWidth = lineWidth; + const lineHeightFactor = 1.7; + this.lineHeight = this.fontSize * lineHeightFactor - this.fontSize / 4; + this.rectHeight = lines.length * this.fontSize * lineHeightFactor + this.fontSize / 3; + } + return this.svgSkeleton; + } + + private get svgSkeleton(): string { + return ` + + + ${this.background} + ${this.textStyle} + ${this.tspans} + + `; + } + + // creates rect with width, moves it half its size to the left and 5 px down so it sits in the middle of the text + private get background(): string { + let translate; + let svgRotation = ''; + if (this.svgType === 'Subtitle') { + translate = `-${this.lineWidth / 2} -${(this.lineCount - 1) * this.lineHeight}`; + } else { + translate = '0 0'; + if (this.rotation) { + svgRotation = `rotate(${this.rotation} ${this.lineWidth / 2} ${this.rectHeight / 2})`; + } + } + let bgColor = this.backgroundColor; + let bgOpacity = this.backgroundAlpha; + // Check if color is rgba, if so: transform into rgb and alpha + if (bgColor?.includes('rgba')) { + const transformedColor = SvgCreator.transformRgba(bgColor); + bgColor = transformedColor.color; + if (bgOpacity === 1) { + // If background is available, set correct alpha + bgOpacity = transformedColor.alpha; + } + } + if (!bgColor) { + bgColor = ''; + bgOpacity = 0; + } + return ``; + } + + public static transformRgba(rgbaColor: string): { color: string; alpha: number } { + const numberRegex = /([\d.]+)/g; + const [red, green, blue, alpha] = rgbaColor.match(numberRegex); + const rgbColor = `rgb(${red},${green},${blue})`; + return { + color: rgbColor, + alpha: Number(alpha), + }; + } + + private get tspans(): string { + let tspans = ''; + let x; + let y; + if (this.svgType === 'Subtitle') { + x = 0; + y = `-${this.lineHeight * (this.lineCount - 1)}`; + } else { + x = this.lineWidth / 2; + y = (this.lineHeight / 2) * 0.25; + } + + this.lines.forEach((line, index) => { + if (index === 0 && this.lineCount > 1) { + tspans += ` + ${this.rtlLanguage === true ? '.' : ''} + ${this.convertSpecialCharacter(line)} `; + } else { + let dy; + if (this.svgType === 'Subtitle') { + dy = this.lineHeight; + } else { + if (this.lineCount > 1) { + dy = index === 0 ? this.rectHeight / 2 + 10 : this.lineHeight * 1.2; + } else { + dy = this.rectHeight / 2 + this.rectHeight * 0.15; + } + } + tspans += ` ${ + this.rtlLanguage === true ? '.' : '' + } ${this.convertSpecialCharacter(line)} `; + } + }); + return tspans; + } + private get textStyle(): string { + let svgRotation = ''; + if (this.svgType === 'CanvasText' && this.rotation) { + svgRotation = `transform="rotate(${this.rotation} ${this.lineWidth / 2} ${ + this.rectHeight / 2 + })"`; + } + let fontColor = this.fontColor; + let alpha = 1; + if (fontColor.includes('rgba')) { + const transformedColor = SvgCreator.transformRgba(fontColor); + fontColor = transformedColor.color; + alpha = transformedColor.alpha; + } + return ` + + `; + } + + private convertSpecialCharacter(line: string): string { + let correctedLine = line; + const replacementTable = [ + { element: '&', code: '&' }, // add after &, otherwise & will repace '&' from the code + { element: '<', code: '<' }, + { element: '>', code: '>' }, + { element: '\b', code: '' }, + { element: '\f', code: '' }, + { element: '\n', code: '' }, + { element: '\r', code: '' }, + { element: '\t', code: '' }, + { element: '\v', code: '' }, + // { element: "\\'", code: '' }, + // { element: '\\"', code: '' }, + // { element: '\\?', code: '' }, + { element: '\\\\', code: '' }, + ]; + for (const replacement of replacementTable) { + // correctedLine = correctedLine.replace(replacement.element, replacement.code); + correctedLine = correctedLine.replace(new RegExp(replacement.element, 'g'), replacement.code); + } + return correctedLine; + } + + calculateSubtitleHeightFactor(resolution: IResolution): number { + const aspectRatio = resolution.width / resolution.height; + if (aspectRatio > 1) { + // moved 80% of height down + return 0.85; + } else if (aspectRatio < 1) { + // portrait + // moved 90% of height down + return 0.9; + } else { + // square + // moved 90% of height down + return 0.85; + } + } + + public static getVideoBackgroundSVG(resolution: IResolution, color = '#ffffff'): string { + let transformedColor = color; + if (color.includes('rgba')) { + transformedColor = this.transformRgba(color).color; + } + const svgText = ` + + + + + `; + return svgText; + } + + public static isRTL(lines: string[]) { + try { + const rtlChars = '\u0591-\u07FF\u200F\u202B\u202E\uFB1D-\uFDFD\uFE70-\uFEFC'; + const rtlDirCheck = new RegExp('^[^' + rtlChars + ']*?[' + rtlChars + ']'); + const line = lines[0]; + return rtlDirCheck.test(line); + } catch (error: unknown) { + return false; + } + } +} diff --git a/app/services/highlighter/subtitles/transcription.ts b/app/services/highlighter/subtitles/transcription.ts new file mode 100644 index 000000000000..b9f1def7db76 --- /dev/null +++ b/app/services/highlighter/subtitles/transcription.ts @@ -0,0 +1,816 @@ +import { SubtitleMode } from './subtitle-mode'; +import { SubtitleClip } from './subtitle-clip'; +import { SubtitleClips } from './subtitle-clips'; +import { Word, IWordIndex } from './word'; +import { removeLinebreaksFromString, roundTime } from './subtitle-utils'; + +export const MAX_PAUSE_TIME = 2; //0.3 work better but creates more ; + +export interface ISpeaker { + index: number; + name: string; +} + +export interface IStartEndConfig { + startWord: number; + endWord: number; +} + +export class Transcription { + singlePauseLength = 0.25; + words: Word[] = []; + isTranslation = false; + + get duration(): number { + return this.words[this.words.length - 1].endTimeInEdit; + } + + get onlyPauses(): boolean { + return this.words.every(word => word.isPause); + } + + get onlyCuts(): boolean { + return this.words.every(word => word.isCut); + } + + // + // initialisation functions + // + clone(transcription: Transcription) { + this.words = transcription.words.map(word => new Word().clone(word)); + this.isTranslation = transcription.isTranslation; + return this; + } + + setTranslation(isTranslation: boolean) { + this.isTranslation = isTranslation; + return this; + } + + getLastWord(): Word { + return this.words[this.words.length - 1]; + } + getSubtitleLength(aspectRatio: number) { + let charactersPerRow = 70; + // Condition true for square, portrait, vertical post and pinterest + if (aspectRatio <= 1) { + charactersPerRow = 20; + } + return charactersPerRow; + } + getSpeakerIndices(skipCuts = true): number[] { + const speakers = new Set(); + for (let index = 0; index < this.words.length; index++) { + const word = this.words[index]; + if (skipCuts && word.isCut) { + continue; + } + speakers.add(word.speakerTag); + } + return [...speakers]; + } + /** + * generating abstraction of the transcription + * @param subtitleMode subtitleMode (static, dynamic) of the current project + * @param currentVideoFormat VideoFormatOption of the current project + * @param subtitleLength Characters per row of the subtitle + */ + generateSubtitleClips( + subtitleMode: SubtitleMode, + aspectRatio: number, + subtitleLength?: number, + ): SubtitleClips { + let characterCount = 0; + let inClip = false; + const subtitleClips = new SubtitleClips(); + let startIndex = 0; + + let inPauseSeries = false; + let pauseStartIndex: number = null; + let pauseDuration = 0; + const pushSubtitle = ( + startIndexToPush: number, + endIndex: number, + newStartIndex: number, + mediaIndex: number, + ) => { + subtitleClips.push(new SubtitleClip(startIndexToPush, endIndex, mediaIndex, this)); + startIndex = newStartIndex; + pauseDuration = 0; + characterCount = 0; + inClip = false; + inPauseSeries = false; + }; + + const defaultCharactersPerRow = this.getSubtitleLength(aspectRatio); + const charactersPerRow = subtitleLength || defaultCharactersPerRow; + const isCustomLength = defaultCharactersPerRow !== charactersPerRow; + const lastNonCutWord = this.getLastNonCutWord(); + + let breakOnComma = false; + + // Condition true for square, portrait, vertical post and pinterest + if (aspectRatio <= 1 && !isCustomLength) { + breakOnComma = true; + } + let previousMediaIndex = this.words[0]?.mediaIndex; + for (let index = 0; index < lastNonCutWord; index++) { + const word = this.words[index]; + const mediaIndexChanged = previousMediaIndex !== word.mediaIndex; + + // video source changed + if (mediaIndexChanged) { + pushSubtitle(startIndex, index - 1, index, previousMediaIndex); + previousMediaIndex = word.mediaIndex; + continue; + } + previousMediaIndex = word.mediaIndex; + + // + if (word.hasSubtitlebreak === true && !word.isCut && index > 0) { + pushSubtitle(startIndex, index, index + 1, word.mediaIndex); + continue; + } + // skip cuts + if (word.isCut) { + if (!inClip) { + startIndex = index + 1; + } + continue; + } + + if (word.isPause) { + pauseDuration += word.duration; + if (!inPauseSeries) { + // pause series started + //lastWordIndex = index - 1 + pauseStartIndex = index; + inPauseSeries = true; + } + } else { + // pause series ended + inPauseSeries = false; + pauseDuration = 0; + characterCount += word.text.length + 1; // + 1 because of the pause between words + if (!inClip) { + // clip started + startIndex = index; + inClip = true; + } + } + + if (inClip) { + // subtitle ended because + if ( + pauseDuration > MAX_PAUSE_TIME && + pauseStartIndex != null && + (word.hasSubtitlebreak === true || + word.hasSubtitlebreak === null || + word.hasSubtitlebreak === undefined) && + !isCustomLength + ) { + // long pause + pushSubtitle(startIndex, pauseStartIndex - 1, index + 1, word.mediaIndex); + } else if (word.hasSubtitlebreak === false) { + characterCount = 0; + pauseDuration = 0; + } else if ( + (word.hasSubtitlebreak === true || + word.hasSubtitlebreak === null || + word.hasSubtitlebreak === undefined) && + (characterCount + this.words[index + 1]?.text?.length > charactersPerRow || + ((word.isEndOfSentence || breakOnComma + ? word.hasPunctuationAndComma + : word.hasPunctuation) && + !isCustomLength) || + word.hasSubtitlebreak) + ) { + // maximum of characters was reached + // or word contains '.', ',', '?', '!', ':', ';' + // or word was the last word of the transcription + // and a subtitle clip has already started + // End subtitle clip when pause is to long + pushSubtitle(startIndex, index, index + 1, word.mediaIndex); + } + } + } + + if (startIndex < lastNonCutWord + 1) { + pushSubtitle( + startIndex, + lastNonCutWord, + lastNonCutWord + 1, + this.words[lastNonCutWord].mediaIndex, + ); + } + + if (subtitleMode === SubtitleMode.dynamic) { + subtitleClips.convertToDynamicSubtitles(); + } + // subtitleClips.forEach((sc) => { + // console.log(`[${sc.startIndex}-${sc.endIndex}]`, sc.startTimeInEdit, sc.endTimeInEdit, sc.text); + // }); + // console.log(this.words); + + return subtitleClips; + } + + calculateSelectionDuration(startEndConfig: IStartEndConfig) { + return ( + this.words[startEndConfig?.endWord]?.endTimeInEdit - + this.words[startEndConfig?.startWord]?.startTimeInEdit + ); + } + + public sumTrimtimeToIndex(index: number): number { + let sum = 0; + this.words + .slice(0, index) + .filter(word => word.isCut) + .forEach(word => { + sum += word.duration; + }); + + return sum; + } + /** + * Calculating rendered transcription with times from cutwords removed + */ + public getShiftedTranscription(): Transcription { + const shiftedTranscription = new Transcription().clone(this); + let trimTime = 0; + let mediaSourceOffset = 0; + for (let index = 0; index < shiftedTranscription.words.length; index++) { + const word = shiftedTranscription.words[index]; + if (index > 0 && word.mediaIndex !== shiftedTranscription.words[index - 1].mediaIndex) { + mediaSourceOffset = shiftedTranscription.words[index - 1].endTimeInEdit; + trimTime = 0; + } + + trimTime += word.isCut ? word.duration : 0; + word.moveForwardsFromOriginal(trimTime - mediaSourceOffset); + } + return shiftedTranscription; + } + + // + // accessing words + // + getWordByIndex(index: number): Word { + if (this.isIndexInBounds(index)) { + return this.words[index]; + } + return null; + } + + getPreviousWordByIndex( + index: number, + skipPauses?: boolean, + skipCutword?: boolean, + includeStartIndex = false, + ): IWordIndex { + const startIndex = includeStartIndex ? index : index - 1; + for (let i = startIndex; i >= 0; i--) { + const currentWord = this.words[i]; + if ((skipPauses && currentWord.isPause) || (skipCutword && currentWord.isCut)) { + continue; + } + return { index: i, word: currentWord }; + } + return null; + } + + getNextWordByIndex( + index: number, + skipPauses?: boolean, + skipCutword?: boolean, + includeStartIndex = false, + ): IWordIndex { + const startIndex = includeStartIndex ? index : index + 1; + for (let i = startIndex; i < this.words.length; i++) { + const currentWord = this.words[i]; + if ((skipPauses && currentWord?.isPause) || (skipCutword && currentWord?.isCut)) { + continue; + } + return { index: i, word: currentWord }; + } + return null; + } + + sliceWords(startIndex: number, endIndex: number): Word[] { + if (this.isIndexInBounds(startIndex)) { + return this.words.slice(startIndex, endIndex); + } + return []; + } + + // + // text related functions + // + public getText( + includeTimestamps = false, + includeSpeakers = false, + wordChunkArray?: IWordIndex[][], + speakers?: ISpeaker[], + removeLinebreaks = false, + ): string { + let words = ''; + // If the user want to have timestamps or speakers in the export loop through the paragraphs + if ((includeTimestamps || includeSpeakers) && wordChunkArray) { + const filteredParagraphs = wordChunkArray + .map(paragraphFilter => { + const noCutwords = paragraphFilter.filter(wordIndex => !wordIndex.word.isCut); + return [...noCutwords]; + }) + .filter(wordIndexArray => wordIndexArray.length); + + for (const paragraph of filteredParagraphs) { + if (includeSpeakers) { + words += speakers[paragraph[0].word.speakerTag].name; + } + if (includeSpeakers && includeTimestamps) { + words += ' - '; + } + if (includeTimestamps) { + words += `[${getFormattedTimeFromSeconds( + paragraph[0].word.startTimeInEdit, + )} - ${getFormattedTimeFromSeconds(paragraph[paragraph.length - 1].word.endTimeInEdit)}]`; + } + if (includeTimestamps || includeSpeakers) { + words += '\n'; + } + for (const wordIndex of paragraph) { + if (!wordIndex.word.isPause) { + words += wordIndex.word.text; + words += ' '; + } + if (wordIndex.word.hasLinebreak) { + words += '\n\n'; + } + } + } + } else { + for (const word of this.words) { + if (!word.isCut && !word.isPause) { + words += word.text; + words += ' '; + } + if (word.hasLinebreak) { + words += '\n\n'; + } + } + } + if (removeLinebreaks) { + words = removeLinebreaksFromString(words); + } + return words; + } + + public setLinebreak(index: number, hasLinebreak: boolean): void { + if (this.isIndexInBounds(index)) { + this.words[index].hasLinebreak = hasLinebreak; + this.words[index].hasSubtitlebreak = hasLinebreak; + } + } + setLastWordLinebreak() { + this.setLinebreak(this.words.length - 1, true); + } + public editWord(index: number, newText: string): void { + if (this.isIndexInBounds(index)) { + this.words[index].text = newText; + } + } + + public getSentencesArray(): Word[][] { + const sentences: Word[][] = []; + let singleSentence: Word[] = []; + for (const [index, word] of this.words.entries()) { + singleSentence.push(word); + if (word.isEndOfSentence || index >= this.words.length - 1) { + sentences.push(singleSentence); + singleSentence = []; + } + } + + return sentences; + } + + getCutStartEndConfig(): IStartEndConfig[] { + const onlyCutWordIndices = this.words + .map((word, index) => { + return { index, word }; + }) + .filter(wordIndex => wordIndex.word.isCut) + .map(wordIndex => wordIndex.index); + + let sequenceStart: number; + const cuts: IStartEndConfig[] = []; + for (let index = 1; index <= onlyCutWordIndices.length; index++) { + const indexA = onlyCutWordIndices[index - 1]; + const indexB = onlyCutWordIndices[index]; + + if (!sequenceStart) sequenceStart = indexA; + + if (indexA + 1 !== indexB) { + cuts.push({ startWord: sequenceStart, endWord: indexA }); + sequenceStart = indexB; + } else if (index === onlyCutWordIndices.length - 1) { + cuts.push({ startWord: sequenceStart, endWord: indexB }); + } + } + return cuts; + } + + // + // time related functions + // + public getWordMediaIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].mediaIndex; + } + return null; + } + public getStartTimeInEditAtIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].startTimeInEdit; + } + return null; + } + + public getEndTimeInEditAtIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].endTimeInEdit; + } + return null; + } + public getStartTimeInOriginalAtIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].startTimeInOriginal; + } + return null; + } + + public getEndTimeInOriginalAtIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].endTimeInOriginal; + } + return null; + } + + // gets the index at a specific timestamp from the original video + // jumps over cutwords + public getIndexAtTimeInEdit(time: number): number { + for (const [index, word] of this.words.entries()) { + if (!word.isCut && time >= word.startTimeInEdit && time < word.endTimeInEdit) { + return index; + } + } + return -1; + } + + public getNextIndexByTime(currentTime: number): number { + for (let index = 0; index < this.words.length; index++) { + const word = this.words[index]; + if (currentTime < word.endTimeInOriginal) { + return index; + } + } + // if nothing found return beginning + return 0; + } + + /** + * + * @returns duration of the edited video + */ + public getEditedDuration(): number { + let totalDuration = 0; + //TODO: refactor with reduce + this.words + .filter(word => !word.isCut) + .forEach(word => { + totalDuration += word.duration; + }); + + return totalDuration; + } + /** + * generating new Transcription with pauses before first word, between word with pauses and after last word + * @param duration + * @param skipFirstAndLastPauses + */ + public generatePauses(duration: number, skipPausesBefore = false, skipPausesAfter = false): void { + const newTranscription = new Transcription(); + + if (this.words.length > 0) { + if (!skipPausesBefore) { + this.addPausesBeforeFirstWord(this.words, duration, newTranscription); + } + for (let i = 0; i < this.words.length; i++) { + const word = this.words[i]; + const newWord = new Word().clone(word); + newTranscription.words = newTranscription.words.concat(newWord); + + // check if there is a word until next word + // create new Word with length of the pause + const pauseWords = this.getPauseWords(this.words, i, duration); + if (pauseWords) { + newTranscription.words = newTranscription.words.concat(pauseWords); + } else { + // console.log('no Pause'); + } + } + if (!skipPausesAfter) { + this.addPauseAfterLastWord(this.words, duration, newTranscription); + } + } else { + //empty word array + let currentTime = 0; + while (currentTime + this.singlePauseLength < duration) { + const newPause = new Word().initPauseWord( + currentTime, + roundTime(currentTime + this.singlePauseLength), + 0, + 0, + ); + newTranscription.words.push(newPause); + currentTime += this.singlePauseLength; + } + if (duration - currentTime !== 0) { + const newPause = new Word().initPauseWord( + currentTime, + roundTime(currentTime + duration - currentTime), + 0, + 0, + ); + newTranscription.words.push(newPause); + } + } + this.words = newTranscription.words; + } + + // + // transcription utility + // + + private isIndexInBounds(index: number): boolean { + return index >= 0 && index < this.words.length; + } + + // generate pause helper + private addPausesBeforeFirstWord( + words: Word[], + duration: number, + newTranscription: Transcription, + ) { + const pauseWords = this.getPauseWords(words, -1, duration); + if (pauseWords) { + newTranscription.words = newTranscription.words.concat(pauseWords); + } + } + addPausesBetweenStreamingChunks(words: Word[], duration: number) { + if (!words.length) { + return; + } + const pauseWords = this.getPauseWords(words, 0, duration); + if (pauseWords) { + this.words = [...pauseWords, ...this.words]; + } + } + + private addPauseAfterLastWord(words: Word[], duration: number, newTranscription: Transcription) { + const pauseWords = this.getPauseWords(words, words.length - 1, duration, true); + if (pauseWords) { + newTranscription.words = newTranscription.words.concat(pauseWords); + } + } + private getPauseWords( + words: Word[], + index: number, + duration?: number, + lastPart?: boolean, + ): Word[] { + // seconds + + if (index + 1 < words.length || lastPart) { + let firstWord: Word; + let secondWord: Word; + if (index === -1) { + // handle first Word + // check if there is a pause before the first word + firstWord = new Word().initZero(); + secondWord = new Word().clone(words[0]); + } else if (lastPart === true && index === words.length - 1) { + // handle last Word + // check if there is a pause after the last word + firstWord = new Word().clone(words[index]); + secondWord = new Word(); + secondWord.startTimeInEdit = duration; + } else { + firstWord = new Word().clone(words[index]); + secondWord = new Word().clone(words[index + 1]); + } + + const pauseTime = roundTime(secondWord.startTimeInEdit - firstWord.endTimeInEdit); + // console.log('pausetime', pauseTime); + + if (pauseTime > 0) { + const pauseCount = Math.ceil(pauseTime / this.singlePauseLength); + const pauseWords: Word[] = []; + let pauseStartTime = firstWord.endTimeInEdit; + + for (let i = 0; i < pauseCount; i++) { + let leftPauseTime = roundTime(pauseTime - i * this.singlePauseLength); + leftPauseTime = + leftPauseTime > this.singlePauseLength + ? this.singlePauseLength + : roundTime(pauseTime - i * this.singlePauseLength); + + // console.log(leftPauseTime); + + pauseWords.push( + new Word().initPauseWord( + pauseStartTime, + roundTime(pauseStartTime + leftPauseTime), + firstWord.mediaIndex, + firstWord.speakerTag ?? secondWord.speakerTag ?? 1, + ), + ); + + pauseStartTime += leftPauseTime; + } + return pauseWords; + } + // return words[index+1].startTime.time - words[index].endTime.time + } + return null; + } + /** + * Checks the sentence for the most dominant speaker and set this one for the whole sentence + */ + updateSentenceSpeaker() { + // console.log(this.words); + const sentences: Word[][] = this.getSentencesArray(); + const newWords: Word[] = []; + for (const sentence of sentences) { + const speakerTags = sentence.filter(word => !word.isPause).map(word => word.speakerTag); + const dominantSpeaker = speakerTags + .slice() + .sort( + (a, b) => + speakerTags.filter(v => v === a).length - speakerTags.filter(v => v === b).length, + ) + .pop(); + sentence.forEach(word => { + const newWord: Word = word; + newWord.speakerTag = dominantSpeaker; + newWords.push(newWord); + return newWord; + }); + } + this.words = newWords; + // console.log(newSentences.map((sentence) => sentence.words)); + } + + /** + * generates necessary speaker information + */ + public generateSpeakerInformation(): Transcription { + const SPEAKER_MIN_PERCENTAGE = 0.05; + // Check if SPEAKER_MIN_PERCENTAGE threshold is reached, if not set speaker to 1 for every word. + const allSpeakerTags = this.words + .filter(word => word.speakerTag !== null) + .map(word => word.speakerTag); + const counts: { [tag: number]: number } = {}; + for (const speakerTag of allSpeakerTags) { + counts[speakerTag] = counts[speakerTag] ? counts[speakerTag] + 1 : 1; + } + // Only check if 2 speakers, if more just take them + const speakerKeys = Object.keys(counts).map(Number); + if ( + (speakerKeys.length === 2 && + counts[speakerKeys[0]] / allSpeakerTags.length < SPEAKER_MIN_PERCENTAGE) || + counts[speakerKeys[0]] / allSpeakerTags.length > 100 - SPEAKER_MIN_PERCENTAGE + ) { + this.words = this.words.map(word => { + word.speakerTag = 1; + return word; + }); + } + + // After all the speaker stuff is done check where to set linebreaks initially (after each speaker switch) + for (let i = 1; i < this.words.length; i++) { + if ( + this.words[i - 1].speakerTag !== this.words[i].speakerTag && + this.words[i].speakerTag != null + ) { + this.words[i - 1].hasLinebreak = true; + } + } + return this; + } + + /** + * Returns the word index of the last non cut word (this includes pauses) + * @param transcription transcription + */ + getLastNonCutWord(): number { + let lastNonCutWordIndex = this.words.length - 1; + + if (this.words?.length) { + for (let i = this.words.length - 1; i >= 0; i--) { + if (!this.words[i].isCut) { + lastNonCutWordIndex = i; + break; + } + } + } + return lastNonCutWordIndex; + } + + setCutword(from: number, to: number) { + if (this.words?.length) { + for (let i = from; i < to; i++) { + this.words[i].initCutWord(); + } + } + } + + /** + * Returns pauses + * + * @param startEndConfig Searches in this start end config. Default is current selection + * + * @param pausesThreshold Ignore if more pauses than the threshold are next to each other + * @example If pausesThreshold is set to 4: 5, 6, 7 and more pauses after each other will NOT be returned + */ + public getAllPauses( + startEndConfig: IStartEndConfig = { startWord: 0, endWord: this.words.length - 1 }, + pausesThreshold = 100, + ) { + const words = this.words; + const indicesToRemove: number[] = []; + // Loop through all words + for ( + let wordIndex = startEndConfig.startWord; + wordIndex <= startEndConfig.endWord; + wordIndex++ + ) { + const currentIndices = []; + let currentCounter = 0; + // Check if current word + next words (based on currentCounter) have a pause. If so, check next word. If not, skip with outer loop + for (currentCounter; words[wordIndex + currentCounter]?.isPause; currentCounter++) { + if ( + words[wordIndex + currentCounter]?.isPause && + !words[wordIndex + currentCounter]?.isCut + ) { + currentIndices.push(wordIndex + currentCounter); + if (!words[wordIndex + currentCounter + 1]?.isPause) { + break; + } + } else { + break; + } + } + wordIndex += currentCounter; + + if (currentIndices.length < pausesThreshold && currentIndices.length > 1) { + currentIndices.map(index => indicesToRemove.push(index)); + } + } + + return indicesToRemove; + } + + /** + * Returns pauses in the current selection + */ + getConsecutivePauses( + startEndConfig: IStartEndConfig = { startWord: 0, endWord: this.words.length - 1 }, + ): number[][] { + const words = this.words; + const consecutivePausesIndices: number[][] = []; + let index = startEndConfig.startWord; + while (index <= startEndConfig.endWord) { + const currentIndices = []; + let count = 0; + let currentWord = words[index]; + while (currentWord?.isPause && !currentWord?.isCut) { + currentIndices.push(index + count); + count++; + currentWord = words[index + count]; + } + index += currentIndices.length ? currentIndices.length : 1; + if (currentIndices.length > 0) { + consecutivePausesIndices.push(currentIndices); + } + } + return consecutivePausesIndices; + } +} + +function getFormattedTimeFromSeconds(secondsInput: number) { + const date = new Date(0); + date.setSeconds(secondsInput); + return date.toISOString().substr(secondsInput >= 3600 ? 11 : 14, secondsInput >= 3600 ? 8 : 5); +} diff --git a/app/services/highlighter/subtitles/word.ts b/app/services/highlighter/subtitles/word.ts new file mode 100644 index 000000000000..bd37e7963bbd --- /dev/null +++ b/app/services/highlighter/subtitles/word.ts @@ -0,0 +1,168 @@ +import { roundTime } from './subtitle-utils'; + +export interface IWordIndex { + index: number; + word: Word; +} + +export class Word { + // properties available from firebase + text: string; + startTimeInEdit: number; + endTimeInEdit: number; + isCut = false; + isPause = false; + hasLinebreak = false; + hasSubtitlebreak?: boolean; + speakerTag?: number; + confidence?: number; + + // properties generated after loaded + mediaIndex: number; + chunkIndex: number; + startTimeInOriginal: number; + endTimeInOriginal: number; + + public get isEndOfSentence(): boolean { + const punctuation = ['.', '?', '!', '。']; + return this.text && punctuation.some(item => this.text.includes(item)); + } + + public get hasPunctuation(): boolean { + const punctuation = ['.', '?', '!', ':', '。']; + return this.text && punctuation.some(item => this.text.includes(item)); + } + + public get hasPunctuationAndComma(): boolean { + const punctuation = ['.', ',', '?', '!', ':', ';', '。']; + return this.text && punctuation.some(item => this.text.includes(item)); + } + public get isFiller(): boolean { + const fillers = ['um', 'uh', 'hmm', 'mhm', 'uh huh']; + return this.text && fillers.some(item => this.text.toLowerCase().includes(item)); + } + + public get duration(): number { + return this.endTimeInEdit - this.startTimeInEdit; + } + public get isRealWord(): boolean { + return this.isCut !== true && this.isPause !== true; + } + + public setMediaIndex(mediaIndex: number) { + this.mediaIndex = mediaIndex; + return this; + } + public setChunkIndex(chunkIndex: number) { + this.chunkIndex = chunkIndex; + return this; + } + + constructor() { + return this; + } + public clone(word: Word): Word { + this.text = word.text; + + // this.startTimeInEdit = word.startTimeInEdit; + // this.endTimeInEdit = word.endTimeInEdit; + + this.startTimeInEdit = word.startTimeInOriginal; + this.endTimeInEdit = word.endTimeInOriginal; + + this.startTimeInOriginal = word.startTimeInOriginal; + this.endTimeInOriginal = word.endTimeInOriginal; + + this.isCut = word.isCut || false; + this.isPause = word.isPause || false; + this.hasLinebreak = word.hasLinebreak || false; + this.hasSubtitlebreak = word.hasSubtitlebreak; + this.speakerTag = word.speakerTag; + this.confidence = word.confidence; + + this.mediaIndex = word.mediaIndex; + this.chunkIndex = word.chunkIndex; + return this; + } + + // start and endTime functions + + // moves word forwards - appears earlier + public moveForwardsFromOriginal(time: number) { + this.startTimeInEdit = roundTime(this.startTimeInOriginal - time); + this.endTimeInEdit = roundTime(this.endTimeInOriginal - time); + } + // moves word backwards - appears later + public moveBackwardsFromOriginal(time: number) { + this.startTimeInEdit = roundTime(this.startTimeInOriginal + time); + this.endTimeInEdit = roundTime(this.endTimeInOriginal + time); + } + + public fromTranscriptionService( + text: string, + startTime: number, + endTime: number, + speakerTag: number, + confidence: number, + ): Word { + if (text.includes('<') && text.includes('>', text.length - 2)) { + this.isPause = true; + } else { + this.text = text; + } + this.startTimeInEdit = startTime; + this.startTimeInOriginal = startTime; + this.endTimeInEdit = endTime; + this.endTimeInOriginal = endTime; + this.speakerTag = speakerTag || 0; + this.confidence = confidence; + return this; + } + + public init(startTime: number, endTime: number, mediaIndex: number, speakerTag: number) { + this.startTimeInEdit = startTime; + this.endTimeInEdit = endTime; + this.startTimeInOriginal = startTime; + this.endTimeInOriginal = endTime; + this.mediaIndex = mediaIndex; + this.speakerTag = speakerTag; + return this; + } + public initPauseWord(startTime: number, endTime: number, mediaIndex: number, speakerTag: number) { + this.isPause = true; + return this.init(startTime, endTime, mediaIndex, speakerTag); + } + public initZero() { + this.startTimeInEdit = 0; + this.endTimeInEdit = 0; + this.startTimeInOriginal = 0; + this.endTimeInOriginal = 0; + return this; + } + public initCutWord() { + this.isCut = true; + return this; + } + public initLinebreak() { + this.hasLinebreak = true; + return this; + } + + public equals(otherWord: Word): boolean { + // properties available from firebase + return ( + this.text === otherWord.text && + this.startTimeInEdit === otherWord.startTimeInEdit && + this.endTimeInEdit === otherWord.endTimeInEdit && + this.isCut === otherWord.isCut && + this.isPause === otherWord.isPause && + this.hasLinebreak === otherWord.hasLinebreak && + this.hasSubtitlebreak === otherWord.hasSubtitlebreak && + this.speakerTag === otherWord.speakerTag && + this.mediaIndex === otherWord.mediaIndex && + this.chunkIndex === otherWord.chunkIndex && + this.startTimeInOriginal === otherWord.startTimeInOriginal && + this.endTimeInOriginal === otherWord.endTimeInOriginal + ); + } +} From d4d64ac276cd359b6afba73da684d22e210315a7 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Fri, 28 Mar 2025 15:40:30 +0100 Subject: [PATCH 05/34] add sharp dependency --- electron-builder/base.config.js | 19 +- package.json | 1 + webpack.base.config.js | 1 + yarn.lock | 315 +++++++++++++++++++++++++++++++- 4 files changed, 332 insertions(+), 4 deletions(-) diff --git a/electron-builder/base.config.js b/electron-builder/base.config.js index f0d9cbbcaf70..2bd17d842692 100644 --- a/electron-builder/base.config.js +++ b/electron-builder/base.config.js @@ -34,7 +34,15 @@ const base = { allowToChangeInstallationDirectory: true, include: 'installer.nsh', }, - asarUnpack : ["**/node-libuiohook/**", "**/node-fontinfo/**", "**/font-manager/**", "**/game_overlay/**","**/color-picker/**"], + asarUnpack: [ + '**/node-libuiohook/**', + '**/node-fontinfo/**', + '**/font-manager/**', + '**/game_overlay/**', + '**/color-picker/**', + '**/node_modules/sharp/**/*', + '**/node_modules/@img/**/*', + ], publish: { provider: 'generic', url: 'https://slobs-cdn.streamlabs.com', @@ -63,12 +71,17 @@ const base = { if (fs.existsSync(signingPath)) { fs.appendFileSync(signingPath, `${config.path}\n`); } else { - cp.execSync(`logisign client --client logitech-cpg-sign-client --app streamlabs --files "${config.path}"`, { stdio: 'inherit' }); + cp.execSync( + `logisign client --client logitech-cpg-sign-client --app streamlabs --files "${config.path}"`, + { stdio: 'inherit' }, + ); } }, }, mac: { - identity: (process.env.APPLE_SLD_IDENTITY) ? process.env.APPLE_SLD_IDENTITY : 'Streamlabs LLC (UT675MBB9Q)', + identity: process.env.APPLE_SLD_IDENTITY + ? process.env.APPLE_SLD_IDENTITY + : 'Streamlabs LLC (UT675MBB9Q)', extraFiles: [ 'shared-resources/**/*', '!shared-resources/README', diff --git a/package.json b/package.json index 0d17cb96222d..e07c4e93dffc 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "realm": "12.7.1", "rimraf": "^2.6.1", "semver": "^5.5.1", + "sharp": "^0.33.5", "socket.io-client": "2.3.1", "tasklist": "^3.1.1", "uuid": "^3.0.1", diff --git a/webpack.base.config.js b/webpack.base.config.js index 495300911da8..7aa5ffddfbf7 100644 --- a/webpack.base.config.js +++ b/webpack.base.config.js @@ -102,6 +102,7 @@ module.exports = { archiver: 'require("archiver")', 'extract-zip': 'require("extract-zip")', 'fs-extra': 'require("fs-extra")', + sharp: 'commonjs sharp', }, module: { diff --git a/yarn.lock b/yarn.lock index eb641f09e4fc..e8d9ea0cdf0b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1894,6 +1894,15 @@ __metadata: languageName: node linkType: hard +"@emnapi/runtime@npm:^1.2.0": + version: 1.3.1 + resolution: "@emnapi/runtime@npm:1.3.1" + dependencies: + tslib: ^2.4.0 + checksum: 9a16ae7905a9c0e8956cf1854ef74e5087fbf36739abdba7aa6b308485aafdc993da07c19d7af104cd5f8e425121120852851bb3a0f78e2160e420a36d47f42f + languageName: node + linkType: hard + "@eslint/eslintrc@npm:^0.2.2": version: 0.2.2 resolution: "@eslint/eslintrc@npm:0.2.2" @@ -1936,6 +1945,181 @@ __metadata: languageName: node linkType: hard +"@img/sharp-darwin-arm64@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-darwin-arm64@npm:0.33.5" + dependencies: + "@img/sharp-libvips-darwin-arm64": 1.0.4 + dependenciesMeta: + "@img/sharp-libvips-darwin-arm64": + optional: true + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@img/sharp-darwin-x64@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-darwin-x64@npm:0.33.5" + dependencies: + "@img/sharp-libvips-darwin-x64": 1.0.4 + dependenciesMeta: + "@img/sharp-libvips-darwin-x64": + optional: true + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@img/sharp-libvips-darwin-arm64@npm:1.0.4": + version: 1.0.4 + resolution: "@img/sharp-libvips-darwin-arm64@npm:1.0.4" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@img/sharp-libvips-darwin-x64@npm:1.0.4": + version: 1.0.4 + resolution: "@img/sharp-libvips-darwin-x64@npm:1.0.4" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@img/sharp-libvips-linux-arm64@npm:1.0.4": + version: 1.0.4 + resolution: "@img/sharp-libvips-linux-arm64@npm:1.0.4" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@img/sharp-libvips-linux-arm@npm:1.0.5": + version: 1.0.5 + resolution: "@img/sharp-libvips-linux-arm@npm:1.0.5" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@img/sharp-libvips-linux-s390x@npm:1.0.4": + version: 1.0.4 + resolution: "@img/sharp-libvips-linux-s390x@npm:1.0.4" + conditions: os=linux & cpu=s390x + languageName: node + linkType: hard + +"@img/sharp-libvips-linux-x64@npm:1.0.4": + version: 1.0.4 + resolution: "@img/sharp-libvips-linux-x64@npm:1.0.4" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@img/sharp-libvips-linuxmusl-arm64@npm:1.0.4": + version: 1.0.4 + resolution: "@img/sharp-libvips-linuxmusl-arm64@npm:1.0.4" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@img/sharp-libvips-linuxmusl-x64@npm:1.0.4": + version: 1.0.4 + resolution: "@img/sharp-libvips-linuxmusl-x64@npm:1.0.4" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@img/sharp-linux-arm64@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-linux-arm64@npm:0.33.5" + dependencies: + "@img/sharp-libvips-linux-arm64": 1.0.4 + dependenciesMeta: + "@img/sharp-libvips-linux-arm64": + optional: true + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@img/sharp-linux-arm@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-linux-arm@npm:0.33.5" + dependencies: + "@img/sharp-libvips-linux-arm": 1.0.5 + dependenciesMeta: + "@img/sharp-libvips-linux-arm": + optional: true + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@img/sharp-linux-s390x@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-linux-s390x@npm:0.33.5" + dependencies: + "@img/sharp-libvips-linux-s390x": 1.0.4 + dependenciesMeta: + "@img/sharp-libvips-linux-s390x": + optional: true + conditions: os=linux & cpu=s390x + languageName: node + linkType: hard + +"@img/sharp-linux-x64@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-linux-x64@npm:0.33.5" + dependencies: + "@img/sharp-libvips-linux-x64": 1.0.4 + dependenciesMeta: + "@img/sharp-libvips-linux-x64": + optional: true + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@img/sharp-linuxmusl-arm64@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-linuxmusl-arm64@npm:0.33.5" + dependencies: + "@img/sharp-libvips-linuxmusl-arm64": 1.0.4 + dependenciesMeta: + "@img/sharp-libvips-linuxmusl-arm64": + optional: true + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@img/sharp-linuxmusl-x64@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-linuxmusl-x64@npm:0.33.5" + dependencies: + "@img/sharp-libvips-linuxmusl-x64": 1.0.4 + dependenciesMeta: + "@img/sharp-libvips-linuxmusl-x64": + optional: true + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@img/sharp-wasm32@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-wasm32@npm:0.33.5" + dependencies: + "@emnapi/runtime": ^1.2.0 + conditions: cpu=wasm32 + languageName: node + linkType: hard + +"@img/sharp-win32-ia32@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-win32-ia32@npm:0.33.5" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + +"@img/sharp-win32-x64@npm:0.33.5": + version: 0.33.5 + resolution: "@img/sharp-win32-x64@npm:0.33.5" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "@jridgewell/gen-mapping@npm:^0.3.0": version: 0.3.3 resolution: "@jridgewell/gen-mapping@npm:0.3.3" @@ -5349,7 +5533,7 @@ __metadata: languageName: node linkType: hard -"color-name@npm:^1.1.1, color-name@npm:~1.1.4": +"color-name@npm:^1.0.0, color-name@npm:^1.1.1, color-name@npm:~1.1.4": version: 1.1.4 resolution: "color-name@npm:1.1.4" checksum: b0445859521eb4021cd0fb0cc1a75cecf67fceecae89b63f62b201cca8d345baf8b952c966862a9d9a2632987d4f6581f0ec8d957dfacece86f0a7919316f610 @@ -5363,6 +5547,26 @@ __metadata: languageName: node linkType: hard +"color-string@npm:^1.9.0": + version: 1.9.1 + resolution: "color-string@npm:1.9.1" + dependencies: + color-name: ^1.0.0 + simple-swizzle: ^0.2.2 + checksum: c13fe7cff7885f603f49105827d621ce87f4571d78ba28ef4a3f1a104304748f620615e6bf065ecd2145d0d9dad83a3553f52bb25ede7239d18e9f81622f1cc5 + languageName: node + linkType: hard + +"color@npm:^4.2.3": + version: 4.2.3 + resolution: "color@npm:4.2.3" + dependencies: + color-convert: ^2.0.1 + color-string: ^1.9.0 + checksum: 0579629c02c631b426780038da929cca8e8d80a40158b09811a0112a107c62e10e4aad719843b791b1e658ab4e800558f2e87ca4522c8b32349d497ecb6adeb4 + languageName: node + linkType: hard + "colord@npm:^2.9.3": version: 2.9.3 resolution: "colord@npm:2.9.3" @@ -6136,6 +6340,13 @@ __metadata: languageName: node linkType: hard +"detect-libc@npm:^2.0.3": + version: 2.0.3 + resolution: "detect-libc@npm:2.0.3" + checksum: 2ba6a939ae55f189aea996ac67afceb650413c7a34726ee92c40fb0deb2400d57ef94631a8a3f052055eea7efb0f99a9b5e6ce923415daa3e68221f963cfc27d + languageName: node + linkType: hard + "detect-node@npm:^2.0.4": version: 2.0.4 resolution: "detect-node@npm:2.0.4" @@ -8466,6 +8677,13 @@ __metadata: languageName: node linkType: hard +"is-arrayish@npm:^0.3.1": + version: 0.3.2 + resolution: "is-arrayish@npm:0.3.2" + checksum: 977e64f54d91c8f169b59afcd80ff19227e9f5c791fa28fa2e5bce355cbaf6c2c356711b734656e80c9dd4a854dd7efcf7894402f1031dfc5de5d620775b4d5f + languageName: node + linkType: hard + "is-binary-path@npm:~2.1.0": version: 2.1.0 resolution: "is-binary-path@npm:2.1.0" @@ -12894,6 +13112,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:^7.6.3": + version: 7.7.1 + resolution: "semver@npm:7.7.1" + bin: + semver: bin/semver.js + checksum: 586b825d36874007c9382d9e1ad8f93888d8670040add24a28e06a910aeebd673a2eb9e3bf169c6679d9245e66efb9057e0852e70d9daa6c27372aab1dda7104 + languageName: node + linkType: hard + "serialize-error@npm:^7.0.1": version: 7.0.1 resolution: "serialize-error@npm:7.0.1" @@ -12976,6 +13203,75 @@ __metadata: languageName: node linkType: hard +"sharp@npm:^0.33.5": + version: 0.33.5 + resolution: "sharp@npm:0.33.5" + dependencies: + "@img/sharp-darwin-arm64": 0.33.5 + "@img/sharp-darwin-x64": 0.33.5 + "@img/sharp-libvips-darwin-arm64": 1.0.4 + "@img/sharp-libvips-darwin-x64": 1.0.4 + "@img/sharp-libvips-linux-arm": 1.0.5 + "@img/sharp-libvips-linux-arm64": 1.0.4 + "@img/sharp-libvips-linux-s390x": 1.0.4 + "@img/sharp-libvips-linux-x64": 1.0.4 + "@img/sharp-libvips-linuxmusl-arm64": 1.0.4 + "@img/sharp-libvips-linuxmusl-x64": 1.0.4 + "@img/sharp-linux-arm": 0.33.5 + "@img/sharp-linux-arm64": 0.33.5 + "@img/sharp-linux-s390x": 0.33.5 + "@img/sharp-linux-x64": 0.33.5 + "@img/sharp-linuxmusl-arm64": 0.33.5 + "@img/sharp-linuxmusl-x64": 0.33.5 + "@img/sharp-wasm32": 0.33.5 + "@img/sharp-win32-ia32": 0.33.5 + "@img/sharp-win32-x64": 0.33.5 + color: ^4.2.3 + detect-libc: ^2.0.3 + semver: ^7.6.3 + dependenciesMeta: + "@img/sharp-darwin-arm64": + optional: true + "@img/sharp-darwin-x64": + optional: true + "@img/sharp-libvips-darwin-arm64": + optional: true + "@img/sharp-libvips-darwin-x64": + optional: true + "@img/sharp-libvips-linux-arm": + optional: true + "@img/sharp-libvips-linux-arm64": + optional: true + "@img/sharp-libvips-linux-s390x": + optional: true + "@img/sharp-libvips-linux-x64": + optional: true + "@img/sharp-libvips-linuxmusl-arm64": + optional: true + "@img/sharp-libvips-linuxmusl-x64": + optional: true + "@img/sharp-linux-arm": + optional: true + "@img/sharp-linux-arm64": + optional: true + "@img/sharp-linux-s390x": + optional: true + "@img/sharp-linux-x64": + optional: true + "@img/sharp-linuxmusl-arm64": + optional: true + "@img/sharp-linuxmusl-x64": + optional: true + "@img/sharp-wasm32": + optional: true + "@img/sharp-win32-ia32": + optional: true + "@img/sharp-win32-x64": + optional: true + checksum: 04beae89910ac65c5f145f88de162e8466bec67705f497ace128de849c24d168993e016f33a343a1f3c30b25d2a90c3e62b017a9a0d25452371556f6cd2471e4 + languageName: node + linkType: hard + "shebang-command@npm:^2.0.0": version: 2.0.0 resolution: "shebang-command@npm:2.0.0" @@ -13056,6 +13352,15 @@ __metadata: languageName: node linkType: hard +"simple-swizzle@npm:^0.2.2": + version: 0.2.2 + resolution: "simple-swizzle@npm:0.2.2" + dependencies: + is-arrayish: ^0.3.1 + checksum: a7f3f2ab5c76c4472d5c578df892e857323e452d9f392e1b5cf74b74db66e6294a1e1b8b390b519fa1b96b5b613f2a37db6cffef52c3f1f8f3c5ea64eb2d54c0 + languageName: node + linkType: hard + "sl-vue-tree@https://github.com/stream-labs/sl-vue-tree.git": version: 1.8.3 resolution: "sl-vue-tree@https://github.com/stream-labs/sl-vue-tree.git#commit=30627b72cdac4755e82816a2bf00737a7c5806e4" @@ -13247,6 +13552,7 @@ __metadata: rxjs: 6.3.3 semver: ^5.5.1 serve-handler: 5.0.7 + sharp: ^0.33.5 shelljs: 0.8.5 signtool: 1.0.0 sl-vue-tree: "https://github.com/stream-labs/sl-vue-tree.git" @@ -14340,6 +14646,13 @@ __metadata: languageName: node linkType: hard +"tslib@npm:^2.4.0": + version: 2.8.1 + resolution: "tslib@npm:2.8.1" + checksum: e4aba30e632b8c8902b47587fd13345e2827fa639e7c3121074d5ee0880723282411a8838f830b55100cbe4517672f84a2472667d355b81e8af165a55dc6203a + languageName: node + linkType: hard + "tsutils@npm:^3.17.1": version: 3.17.1 resolution: "tsutils@npm:3.17.1" From 56b214ccf45166fac95613446081146a6cf068b6 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Fri, 28 Mar 2025 17:21:15 +0100 Subject: [PATCH 06/34] WIP Basic subtitle setup --- .../highlighter/models/rendering.models.ts | 7 +- .../highlighter/rendering/frame-writer.ts | 48 +------- .../highlighter/rendering/render-subtitle.ts | 32 ++++++ .../highlighter/rendering/start-rendering.ts | 75 ++++++++++++- .../highlighter/rendering/subtitles.ts | 105 ------------------ .../highlighter/subtitles/subtitle-clips.ts | 3 + .../highlighter/subtitles/svg-creator.ts | 10 +- 7 files changed, 123 insertions(+), 157 deletions(-) create mode 100644 app/services/highlighter/rendering/render-subtitle.ts delete mode 100644 app/services/highlighter/rendering/subtitles.ts diff --git a/app/services/highlighter/models/rendering.models.ts b/app/services/highlighter/models/rendering.models.ts index 3d4571d5bdf6..0889bce2de53 100644 --- a/app/services/highlighter/models/rendering.models.ts +++ b/app/services/highlighter/models/rendering.models.ts @@ -4,13 +4,18 @@ export type TFPS = 30 | 60; export type TResolution = 720 | 1080; export type TPreset = 'ultrafast' | 'fast' | 'slow'; +export interface ISubtitleRenderingOptions { + enabled: boolean; + directory?: string; +} + export interface IExportOptions { fps: TFPS; width: number; height: number; preset: TPreset; complexFilter?: string; - subtitles?: {}; + subtitles?: ISubtitleRenderingOptions; } // types for highlighter video operations diff --git a/app/services/highlighter/rendering/frame-writer.ts b/app/services/highlighter/rendering/frame-writer.ts index 24cab22bd3dd..f756842f6bd0 100644 --- a/app/services/highlighter/rendering/frame-writer.ts +++ b/app/services/highlighter/rendering/frame-writer.ts @@ -42,8 +42,7 @@ export class FrameWriter { ]; if (this.options.subtitles || true) { console.log('adding subtitles'); - - await this.addSubtitleInput(args, this.options, 'transcription Class'); + await this.addSubtitleInput(args, this.options); } this.addAudioFilters(args); this.addVideoFilters(args, true); //!!this.options.subtitles @@ -129,48 +128,9 @@ export class FrameWriter { )}`, ); } - private async addSubtitleInput(args: string[], exportOptions: IExportOptions, subtitles: string) { - subtitles = testSubtitle; - const subtitleDuration = 10; - const exportWidth = exportOptions.width; - const exportHeight = exportOptions.height; - // this.outputPath - const subtitleDirectory = path.join(path.dirname(this.outputPath), 'temp_subtitles'); - if (!fs.existsSync(subtitleDirectory)) { - fs.mkdirSync(subtitleDirectory, { recursive: true }); - } - - // const subtitlePath = path.join(subtitleDirectory, 'subtitle.srt'); - // // await fs.writeFile(subtitlePath, subtitles); - // console.log('subtitle path', subtitlePath); - // // Escape backslashes in the subtitle path for ffmpeg - // const escapedSubtitlePath = subtitlePath.replace(/\\/g, '\\\\\\\\').replace(':', '\\:'); - - // console.log('escaped subtitle path', escapedSubtitlePath); - // // create subtitle pngs - - // // ffmpeg -f lavfi -i "color=color=white@0.0:s=1280x720:r=30,format=rgba,subtitles='C\:\\\\Users\\\\jan\\\\Videos\\\\color.srt':alpha=1" -t 10 "C:\Users\jan\Videos\temp_subtitles\subtitles_%04d.png" - // const subtitleArgs = [ - // '-f', - // 'lavfi', - // '-i', - // `color=color=white@0.0:s=${exportWidth}x${exportHeight}:r=30,format=rgba,subtitles=\'${escapedSubtitlePath}\':alpha=1`, - // '-c:v', - // 'png', - // // duration of the subtitles - // '-t', - // subtitleDuration.toString(), - // // temp directory for the subtitle images - // `${subtitleDirectory}\\subtitles_%04d.png`, - // '-y', - // '-loglevel', - // 'debug', - // ]; - // // -f lavfi -i "color=color=white@0.0:s=1280x720:r=30,format=rgba,subtitles='C\:\\\\Users\\\\jan\\\\Videos\\\\color.srt':alpha=1" -t 10 "C:\Users\jan\Videos\temp_subtitles\subtitles_%04d.png" - // /* eslint-enable */ - // await execa(FFMPEG_EXE, subtitleArgs); - - args.push('-i', `${subtitleDirectory}\\subtitles_%04d.png`); + private async addSubtitleInput(args: string[], exportOptions: IExportOptions) { + const subtitleDirectory = exportOptions.subtitles.directory; + args.push('-framerate', '1', '-i', `${subtitleDirectory}\\subtitles_%04d.png`); } async writeNextFrame(frameBuffer: Buffer) { diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts new file mode 100644 index 000000000000..338ef4c84017 --- /dev/null +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -0,0 +1,32 @@ +import sharp from 'sharp'; +import fs from 'fs-extra'; +import { IResolution } from '../subtitles/svg-creator'; + +export async function svgToPng(svgText: string, resolution: IResolution, outputPath: string) { + try { + const buffer = await sharp({ + // Generate PNG with transparent background + create: { + width: resolution.width, + height: resolution.height, + channels: 4, + background: { r: 0, g: 0, b: 0, alpha: 0 }, + }, + }) + .composite([ + { + input: Buffer.from(svgText), + top: 0, + left: 0, + }, + ]) + .png() + .toBuffer(); + + console.log('Write svg to ', outputPath); + await fs.writeFile(outputPath, buffer); + console.log('Done'); + } catch (error: unknown) { + console.error('Error creating PNG from SVG', error); + } +} diff --git a/app/services/highlighter/rendering/start-rendering.ts b/app/services/highlighter/rendering/start-rendering.ts index 7c33b757095c..779b77c44468 100644 --- a/app/services/highlighter/rendering/start-rendering.ts +++ b/app/services/highlighter/rendering/start-rendering.ts @@ -18,7 +18,12 @@ import { $t } from '../../i18n'; import * as Sentry from '@sentry/browser'; import { sample } from 'lodash'; import { TAnalyticsEvent } from '../../usage-statistics'; - +import { Word } from '../subtitles/word'; +import { Transcription } from '../subtitles/transcription'; +import { SubtitleMode } from '../subtitles/subtitle-mode'; +import { SvgCreator } from '../subtitles/svg-creator'; +import { svgToPng } from './render-subtitle'; +import fs from 'fs-extra'; export interface IRenderingConfig { renderingClips: RenderingClip[]; isPreview: boolean; @@ -56,7 +61,7 @@ export async function startRendering( const numTransitions = renderingClips.length - 1; const transitionFrames = transitionDuration * exportOptions.fps; const totalFramesAfterTransitions = totalFrames - numTransitions * transitionFrames; - + const totalDuration = totalFramesAfterTransitions / exportOptions.fps; setExportInfo({ totalFrames: totalFramesAfterTransitions, }); @@ -73,6 +78,14 @@ export async function startRendering( fader = new AudioCrossfader(audioConcat, renderingClips, transitionDuration); await fader.export(); + exportOptions.subtitles = { enabled: true }; + // Create subtitles before audio is mixed in + if (exportOptions.subtitles?.enabled) { + const subtitleDirectory = await createSubtitlePngs(parsed, exportOptions, totalDuration); + exportOptions.subtitles.directory = subtitleDirectory; + } + // create transcriptions + if (audioInfo.musicEnabled && audioInfo.musicPath) { mixer = new AudioMixer(audioMix, [ { path: audioConcat, volume: 1, loop: false }, @@ -234,8 +247,64 @@ export async function startRendering( exporting: false, exported: !exportInfo.cancelRequested && !isPreview && !exportInfo.error, }); - + // Clean up subtitle directory if it was created + cleanupSubtitleDirectory(exportOptions); if (fader) await fader.cleanup(); if (mixer) await mixer.cleanup(); } } +const text = + "Don't even think I helped out a stroke. He made, you know, a few birdies coming in and and we held on for a one stroke victory."; +let i = 0; +const testWords = text + .split(' ') + .map(word => new Word().fromTranscriptionService(word, i++, i + 1, null, null)); + +function cleanupSubtitleDirectory(exportOptions: IExportOptions) { + if (exportOptions.subtitles?.directory) { + try { + fs.removeSync(exportOptions.subtitles.directory); + } catch (error) { + console.error('Failed to clean up subtitle directory', error); + } + } +} + +async function createSubtitlePngs( + parsed: path.ParsedPath, + exportOptions: IExportOptions, + totalDuration: number, +) { + const subtitleDirectory = path.join(parsed.dir, 'temp_subtitles'); + + if (!fs.existsSync(subtitleDirectory)) { + fs.mkdirSync(subtitleDirectory, { recursive: true }); + } + const exportResolution = { width: exportOptions.width, height: exportOptions.height }; + const transcription = new Transcription(); + transcription.words = testWords; + transcription.generatePauses(totalDuration); + const subtitleClips = transcription.generateSubtitleClips( + SubtitleMode.static, + exportOptions.width / exportOptions.height, + 20, + ); + + const svgCreator = new SvgCreator( + { width: exportOptions.width, height: exportOptions.height }, + { fontSize: 20, fontFamily: 'Arial', fontColor: 'white', isBold: false, isItalic: false }, + ); + let subtitleCounter = 0; + for (const subtitleClip of subtitleClips.clips) { + const svgString = svgCreator.getSvgWithText([subtitleClip.text], 0); + console.log(svgString); + + const pngPath = path.join( + subtitleDirectory, + `/subtitles_${String(subtitleCounter).padStart(4, '0')}.png`, + ); + await svgToPng(svgString, exportResolution, pngPath); + subtitleCounter++; + } + return subtitleDirectory; +} diff --git a/app/services/highlighter/rendering/subtitles.ts b/app/services/highlighter/rendering/subtitles.ts deleted file mode 100644 index 8a6fdfabcadb..000000000000 --- a/app/services/highlighter/rendering/subtitles.ts +++ /dev/null @@ -1,105 +0,0 @@ -import sharp from 'sharp'; - -const SUBTITLE_PATH = 'C:/Users/jan/Videos/temp_subtitles/'; -interface ITextItem { - text: string; - fontColor?: string; - fontSize?: number; - fontWeight?: string; - fontFamily?: string; -} -async function generateStyledTextImage(textItems: ITextItem[]) { - const width = 1280; - const height = 720; - - // Build SVG with different text styles - let svgText = ``; - - // Calculate total width to center - let totalWidth = 0; - - // First pass to calculate positions - textItems.forEach(item => { - // Approximate width for positioning - not perfect but works for basic centering - const textWidth = item.text.length * (item.fontSize || 24 * 0.6); - totalWidth += textWidth + 5; // 5px spacing - }); - - const position = 'bottom'; - const yPosition = position === 'bottom' ? height - 20 : height / 2; - - // Create a single text element with tspan children - svgText += ` - `; - - // Calculate starting x position for the first tspan - let currentX = (width - totalWidth) / 2; - - // Add tspan elements for each text item - textItems.forEach(item => { - const textWidth = item.text.length * (item.fontSize || 24 * 0.6); - - // Position tspan relative to the total text width - svgText += ` - ${item.text}`; - - currentX += textWidth + 5; // 5px spacing - }); - - // Close the text element - svgText += ` - - `; - - const buffer = await sharp({ - // Generate PNG with transparent background - create: { - width, - height, - channels: 4, - background: { r: 0, g: 0, b: 0, alpha: 0 }, - }, - }) - .composite([ - { - input: Buffer.from(svgText), - top: 0, - left: 0, - }, - ]) - .png() - .toBuffer(); - - return buffer; -} - -// Usage example -export async function saveStyledExample() { - const textItems: ITextItem[] = [ - { - text: 'Hello', - fontColor: '#ff0000', - fontSize: 32, - fontWeight: 'bold', - }, - { - text: 'World', - fontColor: '#0000ff', - fontSize: 24, - }, - ]; - - const buffer = await generateStyledTextImage(textItems); - await require('fs').promises.writeFile(SUBTITLE_PATH + 'styled-output.png', buffer); -} diff --git a/app/services/highlighter/subtitles/subtitle-clips.ts b/app/services/highlighter/subtitles/subtitle-clips.ts index 8cd9b2515cdc..bdb808908cb0 100644 --- a/app/services/highlighter/subtitles/subtitle-clips.ts +++ b/app/services/highlighter/subtitles/subtitle-clips.ts @@ -7,6 +7,9 @@ export class SubtitleClips { public get length(): number { return this._subtitleClips.length; } + public get clips(): SubtitleClip[] { + return this._subtitleClips; + } constructor() { this._subtitleClips = []; diff --git a/app/services/highlighter/subtitles/svg-creator.ts b/app/services/highlighter/subtitles/svg-creator.ts index 11a48cc95405..56412e3a62a6 100644 --- a/app/services/highlighter/subtitles/svg-creator.ts +++ b/app/services/highlighter/subtitles/svg-creator.ts @@ -45,7 +45,7 @@ export class SvgCreator { rightToLeftLanguage = false, ) { this.rtlLanguage = rightToLeftLanguage; - this.svgType = 'CanvasText'; + this.svgType = 'Subtitle'; this.resolution = resolution; this.subtitleHeightPositionFactor = this.calculateSubtitleHeightFactor(resolution); @@ -94,6 +94,8 @@ export class SvgCreator { // correct line width and add margin // 10% padding this.lineWidth = lineWidth; + this.x = this.resolution.width * 0.5; + this.y = this.resolution.height * 0.8; const lineHeightFactor = 1.7; this.lineHeight = this.fontSize * lineHeightFactor - this.fontSize / 4; this.rectHeight = lines.length * this.fontSize * lineHeightFactor + this.fontSize / 3; @@ -154,11 +156,11 @@ export class SvgCreator { private get tspans(): string { let tspans = ''; - let x; - let y; + let x: number; + let y: number; if (this.svgType === 'Subtitle') { x = 0; - y = `-${this.lineHeight * (this.lineCount - 1)}`; + y = Number(`-${this.lineHeight * (this.lineCount - 1)}`); } else { x = this.lineWidth / 2; y = (this.lineHeight / 2) * 0.25; From 214ac80b52cb519a539925abcfbef45086facb16 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Tue, 1 Apr 2025 10:15:00 +0200 Subject: [PATCH 07/34] subtitles V1 --- .../highlighter/ai-highlighter-updater.ts | 26 ++++- .../highlighter/ai-highlighter-utils.ts | 94 ++++++++++++++++++- app/services/highlighter/index.ts | 3 +- .../models/ai-highlighter.models.ts | 13 ++- .../highlighter/rendering/frame-writer.ts | 47 ++-------- .../highlighter/rendering/render-subtitle.ts | 92 +++++++++++++++++- .../highlighter/rendering/start-rendering.ts | 68 +++----------- .../highlighter/subtitles/transcription.ts | 8 +- 8 files changed, 239 insertions(+), 112 deletions(-) diff --git a/app/services/highlighter/ai-highlighter-updater.ts b/app/services/highlighter/ai-highlighter-updater.ts index 80bcc51b0d1f..2cfd060388bb 100644 --- a/app/services/highlighter/ai-highlighter-updater.ts +++ b/app/services/highlighter/ai-highlighter-updater.ts @@ -51,13 +51,23 @@ export class AiHighlighterUpdater { /** * Spawn the AI Highlighter process that would process the video */ - static startHighlighterProcess(videoUri: string, userId: string, milestonesPath?: string) { + static startHighlighterProcess( + videoUri: string, + userId: string, + milestonesPath?: string, + onlyTranscription = false, + ) { const runHighlighterFromRepository = Utils.getHighlighterEnvironment() === 'local'; if (runHighlighterFromRepository) { // this is for highlighter development // to run this you have to install the highlighter repository next to desktop - return AiHighlighterUpdater.startHighlighterFromRepository(videoUri, userId, milestonesPath); + return AiHighlighterUpdater.startHighlighterFromRepository( + videoUri, + userId, + milestonesPath, + onlyTranscription, + ); } const highlighterBinaryPath = path.resolve( @@ -73,7 +83,9 @@ export class AiHighlighterUpdater { } command.push('--use_sentry'); command.push('--user_id', userId); - + if (onlyTranscription) { + command.push('--only_transcription'); + } return spawn(highlighterBinaryPath, command); } @@ -81,6 +93,7 @@ export class AiHighlighterUpdater { videoUri: string, userId: string, milestonesPath?: string, + onlyTranscription = false, ) { const rootPath = '../highlighter-api/'; const command = [ @@ -100,12 +113,19 @@ export class AiHighlighterUpdater { command.push('--milestones_file'); command.push(milestonesPath); } + if (onlyTranscription) { + command.push('--only_transcription'); + } return spawn('poetry', command, { cwd: rootPath, }); } + static startTranscription(videoUri: string, userId: string) { + return this.startHighlighterProcess(videoUri, userId, undefined, true); + } + /** * Check if an update is currently in progress */ diff --git a/app/services/highlighter/ai-highlighter-utils.ts b/app/services/highlighter/ai-highlighter-utils.ts index 98adbbebf749..b46d2bcaae88 100644 --- a/app/services/highlighter/ai-highlighter-utils.ts +++ b/app/services/highlighter/ai-highlighter-utils.ts @@ -8,7 +8,10 @@ import { IHighlighterMessage, IHighlighterMilestone, IHighlighterProgressMessage, + IHighlighterTranscriptionMessage, } from './models/ai-highlighter.models'; +import { Word } from './subtitles/word'; +import { Transcription } from './subtitles/transcription'; const START_TOKEN = '>>>>'; const END_TOKEN = '<<<<'; @@ -144,7 +147,6 @@ export function getHighlightClips( }); }); } - function parseAiHighlighterMessage(messageString: string): IHighlighterMessage | string | null { try { if (messageString.includes(START_TOKEN) && messageString.includes(END_TOKEN)) { @@ -222,3 +224,93 @@ export class ProgressTracker { return interval; } } + +export function getTranscription( + videoUri: string, + userId: string, + totalDuration: number, + cancelSignal?: AbortSignal, + progressUpdate?: (progress: number) => void, +): Promise { + return new Promise((resolve, reject) => { + const childProcess: child.ChildProcess = AiHighlighterUpdater.startTranscription( + videoUri, + userId, + ); + const messageBuffer = new MessageBufferHandler(); + + if (cancelSignal) { + cancelSignal.addEventListener('abort', () => { + console.log('ending transcription process'); + messageBuffer.clear(); + kill(childProcess.pid!, 'SIGINT'); + reject(new Error('Highlight generation canceled')); + }); + } + + childProcess.stdout?.on('data', (data: Buffer) => { + const message = data.toString(); + messageBuffer.appendToBuffer(message); + + // Try to extract a complete message + const completeMessages = messageBuffer.extractCompleteMessages(); + + for (const completeMessage of completeMessages) { + // messageBuffer.clear(); + const aiHighlighterMessage = parseAiHighlighterMessage(completeMessage); + if (typeof aiHighlighterMessage === 'string' || aiHighlighterMessage instanceof String) { + console.log('message type of string', aiHighlighterMessage); + } else if (aiHighlighterMessage) { + switch (aiHighlighterMessage.type) { + case 'progress': + progressUpdate?.((aiHighlighterMessage.json as IHighlighterProgressMessage).progress); + break; + case 'transcription': { + const highlighterWords = (aiHighlighterMessage.json as IHighlighterTranscriptionMessage) + .words; + + const words = highlighterWords.map(highlighterWord => + new Word().fromTranscriptionService( + highlighterWord.text, + highlighterWord.start_time, + highlighterWord.end_time, + 0, + highlighterWord.probability, + ), + ); + const transcription = new Transcription(); + transcription.words = words; + transcription.generatePauses(totalDuration); + resolve(transcription); + break; + } + + default: + // console.log('\n\n'); + // console.log('Unrecognized message type:', aiHighlighterMessage); + // console.log('\n\n'); + break; + } + } + } + }); + + childProcess.stderr?.on('data', (data: Buffer) => { + console.log('Debug logs:', data.toString()); + }); + + childProcess.on('error', error => { + messageBuffer.clear(); + reject(new Error(`Child process threw an error. Error message: ${error.message}.`)); + }); + + childProcess.on('exit', (code, signal) => { + messageBuffer.clear(); + reject({ + message: `Child process exited with code ${code} and signal ${signal}`, + signal, + code, + }); + }); + }); +} diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index 8fb9e9f56aca..7152a8ae51dc 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -1029,6 +1029,7 @@ export class HighlighterService extends PersistentStatefulService { diff --git a/app/services/highlighter/models/ai-highlighter.models.ts b/app/services/highlighter/models/ai-highlighter.models.ts index 5e24e4609e60..1e55d221790e 100644 --- a/app/services/highlighter/models/ai-highlighter.models.ts +++ b/app/services/highlighter/models/ai-highlighter.models.ts @@ -91,14 +91,20 @@ export interface IHighlighterInput { origin: string; metadata?: IDeathMetadata | any; } - +export interface IHighlighterWord { + start_time: number; + end_time: number; + text: string; + probability: number; +} // Message export type EHighlighterMessageTypes = | 'progress' | 'inputs' | 'inputs_partial' | 'highlights' - | 'milestone'; + | 'milestone' + | 'transcription'; export interface IHighlighterMessage { type: EHighlighterMessageTypes; @@ -108,6 +114,9 @@ export interface IHighlighterMessage { export interface IHighlighterProgressMessage { progress: number; } +export interface IHighlighterTranscriptionMessage { + words: IHighlighterWord[]; +} export interface IHighlighterMilestone { name: string; diff --git a/app/services/highlighter/rendering/frame-writer.ts b/app/services/highlighter/rendering/frame-writer.ts index f756842f6bd0..3b38325fcd0e 100644 --- a/app/services/highlighter/rendering/frame-writer.ts +++ b/app/services/highlighter/rendering/frame-writer.ts @@ -4,6 +4,7 @@ import { FADE_OUT_DURATION, FFMPEG_EXE } from '../constants'; import { FrameWriteError } from './errors'; import fs from 'fs-extra'; import path from 'path'; +import { SUBTITLE_PER_SECOND } from './render-subtitle'; export class FrameWriter { constructor( @@ -130,7 +131,12 @@ export class FrameWriter { } private async addSubtitleInput(args: string[], exportOptions: IExportOptions) { const subtitleDirectory = exportOptions.subtitles.directory; - args.push('-framerate', '1', '-i', `${subtitleDirectory}\\subtitles_%04d.png`); + args.push( + '-framerate', + String(SUBTITLE_PER_SECOND), + '-i', + `${subtitleDirectory}\\subtitles_%04d.png`, + ); } async writeNextFrame(frameBuffer: Buffer) { @@ -156,42 +162,3 @@ export class FrameWriter { return this.exitPromise; } } - -const testSubtitle = ` -1 -00:00:02,000 --> 00:00:03,000 -Hi my name is Jan and this is colorful - -2 -00:00:03,000 --> 00:00:04,000 -Hi my name is Jan and this is colorful - -3 -00:00:04,000 --> 00:00:05,000 -Hi my name is Jan and this is colorful - -4 -00:00:05,000 --> 00:00:06,000 -Hi my name is Jan and this is colorful - -5 -00:00:06,000 --> 00:00:07,000 -Hi my name is Jan and this is colorful - -6 -00:00:07,000 --> 00:00:08,000 -Hi my name is Jan and this is colorful - -7 -00:00:08,000 --> 00:00:09,000 -Hi my name is Jan and this is colorful - -8 -00:00:09,000 --> 00:00:10,000 -Hi my name is Jan and this is colorful - -9 -00:00:10,000 --> 00:00:11,000 -Hi my name is Jan and this is colorful - -`; diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts index 338ef4c84017..3d6b17aa05b2 100644 --- a/app/services/highlighter/rendering/render-subtitle.ts +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -1,6 +1,12 @@ import sharp from 'sharp'; import fs from 'fs-extra'; -import { IResolution } from '../subtitles/svg-creator'; +import { IResolution, SvgCreator } from '../subtitles/svg-creator'; +import { getTranscription } from '../ai-highlighter-utils'; +import { SubtitleMode } from '../subtitles/subtitle-mode'; +import { IExportOptions } from '../models/rendering.models'; +import path from 'path'; + +export const SUBTITLE_PER_SECOND = 3; export async function svgToPng(svgText: string, resolution: IResolution, outputPath: string) { try { @@ -23,10 +29,90 @@ export async function svgToPng(svgText: string, resolution: IResolution, outputP .png() .toBuffer(); - console.log('Write svg to ', outputPath); await fs.writeFile(outputPath, buffer); - console.log('Done'); } catch (error: unknown) { console.error('Error creating PNG from SVG', error); } } + +export async function createSubtitles( + mediaPath: string, + userId: string, + parsed: path.ParsedPath, + exportOptions: IExportOptions, + totalDuration: number, + totalFrames: number, +) { + const subtitleDirectory = path.join(parsed.dir, 'temp_subtitles'); + + if (!fs.existsSync(subtitleDirectory)) { + fs.mkdirSync(subtitleDirectory, { recursive: true }); + } + + const exportResolution = { width: exportOptions.width, height: exportOptions.height }; + const svgCreator = new SvgCreator( + { width: exportOptions.width, height: exportOptions.height }, + { fontSize: 20, fontFamily: 'Arial', fontColor: 'white', isBold: false, isItalic: false }, + ); + + const transcription = await getTranscription(mediaPath, userId, totalDuration); + console.log(transcription.words); + + const subtitleClips = transcription.generateSubtitleClips( + SubtitleMode.static, + exportOptions.width / exportOptions.height, + 20, + ); + console.log('Subtitle clips', subtitleClips); + + // Create subtitles + let subtitleCounter = 0; + + const subtitleEveryNFrames = Math.floor(exportOptions.fps / SUBTITLE_PER_SECOND); + + const subtitlesToProcess = []; + for (let frameIndex = 0; frameIndex < totalFrames; frameIndex += subtitleEveryNFrames) { + // Find the appropriate subtitle clip for this frame + const timeInSeconds = frameIndex / exportOptions.fps; + const subtitleClip = subtitleClips.clips.find( + clip => timeInSeconds >= clip.startTimeInEdit && timeInSeconds <= clip.endTimeInEdit, + ); + + if (subtitleClip) { + subtitlesToProcess.push(subtitleClip.text); + } else { + subtitlesToProcess.push(''); + } + } + console.log(subtitlesToProcess); + + // // Pre-calculate all needed subtitles to avoid redundant processing + // const uniqueClips = new Map(); + // for (const { subtitleClip } of framesToProcess) { + // const key = subtitleClip.text; + // if (!uniqueClips.has(key)) { + // uniqueClips.set(key, subtitleClip); + // } + // } + + for (const subtitleText of subtitlesToProcess) { + const svgString = svgCreator.getSvgWithText([subtitleText], 0); + const pngPath = path.join( + subtitleDirectory, + `/subtitles_${String(subtitleCounter).padStart(4, '0')}.png`, + ); + await svgToPng(svgString, exportResolution, pngPath); + subtitleCounter++; + } + return subtitleDirectory; +} + +export function cleanupSubtitleDirectory(exportOptions: IExportOptions) { + if (exportOptions.subtitles?.directory) { + try { + fs.removeSync(exportOptions.subtitles.directory); + } catch (error: unknown) { + console.error('Failed to clean up subtitle directory', error); + } + } +} diff --git a/app/services/highlighter/rendering/start-rendering.ts b/app/services/highlighter/rendering/start-rendering.ts index 779b77c44468..3d6f8c4ebc05 100644 --- a/app/services/highlighter/rendering/start-rendering.ts +++ b/app/services/highlighter/rendering/start-rendering.ts @@ -22,8 +22,9 @@ import { Word } from '../subtitles/word'; import { Transcription } from '../subtitles/transcription'; import { SubtitleMode } from '../subtitles/subtitle-mode'; import { SvgCreator } from '../subtitles/svg-creator'; -import { svgToPng } from './render-subtitle'; +import { cleanupSubtitleDirectory, createSubtitles, svgToPng } from './render-subtitle'; import fs from 'fs-extra'; +import { getTranscription } from '../ai-highlighter-utils'; export interface IRenderingConfig { renderingClips: RenderingClip[]; isPreview: boolean; @@ -36,6 +37,7 @@ export interface IRenderingConfig { streamId: string | undefined; } export async function startRendering( + userId: string, renderingConfig: IRenderingConfig, handleFrame: (currentFrame: number) => void, setExportInfo: (partialExportInfo: Partial) => void, @@ -81,7 +83,14 @@ export async function startRendering( exportOptions.subtitles = { enabled: true }; // Create subtitles before audio is mixed in if (exportOptions.subtitles?.enabled) { - const subtitleDirectory = await createSubtitlePngs(parsed, exportOptions, totalDuration); + const subtitleDirectory = await createSubtitles( + audioConcat, + userId, + parsed, + exportOptions, + totalDuration, + totalFramesAfterTransitions, + ); exportOptions.subtitles.directory = subtitleDirectory; } // create transcriptions @@ -253,58 +262,3 @@ export async function startRendering( if (mixer) await mixer.cleanup(); } } -const text = - "Don't even think I helped out a stroke. He made, you know, a few birdies coming in and and we held on for a one stroke victory."; -let i = 0; -const testWords = text - .split(' ') - .map(word => new Word().fromTranscriptionService(word, i++, i + 1, null, null)); - -function cleanupSubtitleDirectory(exportOptions: IExportOptions) { - if (exportOptions.subtitles?.directory) { - try { - fs.removeSync(exportOptions.subtitles.directory); - } catch (error) { - console.error('Failed to clean up subtitle directory', error); - } - } -} - -async function createSubtitlePngs( - parsed: path.ParsedPath, - exportOptions: IExportOptions, - totalDuration: number, -) { - const subtitleDirectory = path.join(parsed.dir, 'temp_subtitles'); - - if (!fs.existsSync(subtitleDirectory)) { - fs.mkdirSync(subtitleDirectory, { recursive: true }); - } - const exportResolution = { width: exportOptions.width, height: exportOptions.height }; - const transcription = new Transcription(); - transcription.words = testWords; - transcription.generatePauses(totalDuration); - const subtitleClips = transcription.generateSubtitleClips( - SubtitleMode.static, - exportOptions.width / exportOptions.height, - 20, - ); - - const svgCreator = new SvgCreator( - { width: exportOptions.width, height: exportOptions.height }, - { fontSize: 20, fontFamily: 'Arial', fontColor: 'white', isBold: false, isItalic: false }, - ); - let subtitleCounter = 0; - for (const subtitleClip of subtitleClips.clips) { - const svgString = svgCreator.getSvgWithText([subtitleClip.text], 0); - console.log(svgString); - - const pngPath = path.join( - subtitleDirectory, - `/subtitles_${String(subtitleCounter).padStart(4, '0')}.png`, - ); - await svgToPng(svgString, exportResolution, pngPath); - subtitleCounter++; - } - return subtitleDirectory; -} diff --git a/app/services/highlighter/subtitles/transcription.ts b/app/services/highlighter/subtitles/transcription.ts index b9f1def7db76..943aa942e38a 100644 --- a/app/services/highlighter/subtitles/transcription.ts +++ b/app/services/highlighter/subtitles/transcription.ts @@ -166,8 +166,7 @@ export class Transcription { pauseStartIndex != null && (word.hasSubtitlebreak === true || word.hasSubtitlebreak === null || - word.hasSubtitlebreak === undefined) && - !isCustomLength + word.hasSubtitlebreak === undefined) ) { // long pause pushSubtitle(startIndex, pauseStartIndex - 1, index + 1, word.mediaIndex); @@ -179,10 +178,9 @@ export class Transcription { word.hasSubtitlebreak === null || word.hasSubtitlebreak === undefined) && (characterCount + this.words[index + 1]?.text?.length > charactersPerRow || - ((word.isEndOfSentence || breakOnComma + (word.isEndOfSentence || breakOnComma ? word.hasPunctuationAndComma - : word.hasPunctuation) && - !isCustomLength) || + : word.hasPunctuation) || word.hasSubtitlebreak) ) { // maximum of characters was reached From bf067dfe92ca9b680a2d9d202a4f4a04d0bcab7b Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Tue, 1 Apr 2025 16:10:08 +0200 Subject: [PATCH 08/34] vertical and subtitles fix --- .../highlighter/rendering/frame-writer.ts | 30 +++++++++++-------- .../highlighter/rendering/render-subtitle.ts | 16 ++++++---- .../highlighter/rendering/start-rendering.ts | 6 ---- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/app/services/highlighter/rendering/frame-writer.ts b/app/services/highlighter/rendering/frame-writer.ts index 3b38325fcd0e..9ae0e6a65ebc 100644 --- a/app/services/highlighter/rendering/frame-writer.ts +++ b/app/services/highlighter/rendering/frame-writer.ts @@ -98,22 +98,28 @@ export class FrameWriter { console.log('ffmpeg:', data.toString()); }); } - // "subtitles='C\:\\\\Users\\\\jan\\\\Videos\\\\color.srt'" - private addVideoFilters(args: string[], addSubtitleFilter: boolean) { - // args.push( - // '-filter_complex', - // '[0:v][1:v]overlay=0:0[final];[final]format=yuv420p,fade=type=out:duration=1:start_time=4', - // ); - const subtitleFilter = addSubtitleFilter ? '[0:v][1:v]overlay=0:0[final];' : ''; - const fadeFilter = `${subtitleFilter}[final]format=yuv420p,fade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( + private addVideoFilters(args: string[], subtitlesEnabled: boolean) { + const webcamEnabled = !!this.options.complexFilter; + + const firstInput = webcamEnabled ? '[final]' : '[0:v]'; + const output = subtitlesEnabled ? '[subtitled]' : '[final]'; + const subtitleFilter = subtitlesEnabled ? `${firstInput}[1:v]overlay=0:0[subtitled];` : ''; + + const fadeFilter = `format=yuv420p,fade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( this.duration - (FADE_OUT_DURATION + 0.2), 0, )}`; - if (this.options.complexFilter) { - args.push('-vf', this.options.complexFilter + `[final]${fadeFilter}`); - } else { - args.push('-filter_complex', fadeFilter); + args.push('-filter_complex'); + let combinedFilter = ''; + if (webcamEnabled) { + combinedFilter += this.options.complexFilter; + } + + if (subtitlesEnabled) { + combinedFilter += subtitleFilter; } + combinedFilter += output + fadeFilter; + args.push(combinedFilter); } private addAudioFilters(args: string[]) { diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts index 3d6b17aa05b2..2704811d32f4 100644 --- a/app/services/highlighter/rendering/render-subtitle.ts +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -49,11 +49,17 @@ export async function createSubtitles( fs.mkdirSync(subtitleDirectory, { recursive: true }); } - const exportResolution = { width: exportOptions.width, height: exportOptions.height }; - const svgCreator = new SvgCreator( - { width: exportOptions.width, height: exportOptions.height }, - { fontSize: 20, fontFamily: 'Arial', fontColor: 'white', isBold: false, isItalic: false }, - ); + const exportResolution = exportOptions.complexFilter + ? { width: exportOptions.height, height: exportOptions.width } + : { width: exportOptions.width, height: exportOptions.height }; + console.log('Export resolution', exportResolution); + const svgCreator = new SvgCreator(exportResolution, { + fontSize: 20, + fontFamily: 'Arial', + fontColor: 'white', + isBold: false, + isItalic: false, + }); const transcription = await getTranscription(mediaPath, userId, totalDuration); console.log(transcription.words); diff --git a/app/services/highlighter/rendering/start-rendering.ts b/app/services/highlighter/rendering/start-rendering.ts index 3d6f8c4ebc05..f4b144f0c67d 100644 --- a/app/services/highlighter/rendering/start-rendering.ts +++ b/app/services/highlighter/rendering/start-rendering.ts @@ -18,13 +18,7 @@ import { $t } from '../../i18n'; import * as Sentry from '@sentry/browser'; import { sample } from 'lodash'; import { TAnalyticsEvent } from '../../usage-statistics'; -import { Word } from '../subtitles/word'; -import { Transcription } from '../subtitles/transcription'; -import { SubtitleMode } from '../subtitles/subtitle-mode'; -import { SvgCreator } from '../subtitles/svg-creator'; import { cleanupSubtitleDirectory, createSubtitles, svgToPng } from './render-subtitle'; -import fs from 'fs-extra'; -import { getTranscription } from '../ai-highlighter-utils'; export interface IRenderingConfig { renderingClips: RenderingClip[]; isPreview: boolean; From 10ff9033780e77e4352e644bd7e10d6bbcc88f1d Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Tue, 1 Apr 2025 21:28:07 +0200 Subject: [PATCH 09/34] font test --- app/services/highlighter/constants.ts | 5 ++ .../highlighter/rendering/frame-writer.ts | 2 +- .../highlighter/rendering/render-subtitle.ts | 41 +++++---- .../highlighter/rendering/start-rendering.ts | 23 +++-- .../highlighter/subtitles/font-loader.ts | 88 +++++++++++++++++++ .../highlighter/subtitles/svg-creator.ts | 45 ++++++++-- 6 files changed, 172 insertions(+), 32 deletions(-) create mode 100644 app/services/highlighter/subtitles/font-loader.ts diff --git a/app/services/highlighter/constants.ts b/app/services/highlighter/constants.ts index c52bde01f74f..4427a83da987 100644 --- a/app/services/highlighter/constants.ts +++ b/app/services/highlighter/constants.ts @@ -20,6 +20,11 @@ export const SCRUB_WIDTH = 320; export const SCRUB_HEIGHT = 180; export const SCRUB_FRAMES = 20; export const SCRUB_SPRITE_DIRECTORY = path.join(remote.app.getPath('userData'), 'highlighter'); +export const FONT_CACHE_DIRECTORY = path.join( + remote.app.getPath('userData'), + 'highlighter', + 'fonts', +); export const FADE_OUT_DURATION = 1; diff --git a/app/services/highlighter/rendering/frame-writer.ts b/app/services/highlighter/rendering/frame-writer.ts index 9ae0e6a65ebc..fb45b293e8f9 100644 --- a/app/services/highlighter/rendering/frame-writer.ts +++ b/app/services/highlighter/rendering/frame-writer.ts @@ -42,7 +42,7 @@ export class FrameWriter { // '0:v:0', ]; if (this.options.subtitles || true) { - console.log('adding subtitles'); + console.log('adding subtitle input'); await this.addSubtitleInput(args, this.options); } this.addAudioFilters(args); diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts index 2704811d32f4..1e26d387c656 100644 --- a/app/services/highlighter/rendering/render-subtitle.ts +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -5,6 +5,8 @@ import { getTranscription } from '../ai-highlighter-utils'; import { SubtitleMode } from '../subtitles/subtitle-mode'; import { IExportOptions } from '../models/rendering.models'; import path from 'path'; +import { FontLoader } from '../subtitles/font-loader'; +import { FONT_CACHE_DIRECTORY } from '../constants'; export const SUBTITLE_PER_SECOND = 3; @@ -52,25 +54,40 @@ export async function createSubtitles( const exportResolution = exportOptions.complexFilter ? { width: exportOptions.height, height: exportOptions.width } : { width: exportOptions.width, height: exportOptions.height }; - console.log('Export resolution', exportResolution); + const fontFamily = 'Montserrat'; const svgCreator = new SvgCreator(exportResolution, { - fontSize: 20, - fontFamily: 'Arial', + fontSize: 46, + fontFamily, fontColor: 'white', - isBold: false, + isBold: true, isItalic: false, }); + // Load custom font + try { + console.log('Loading custom font:', fontFamily); + const fontLoader = FontLoader.getInstance(); + console.log(FONT_CACHE_DIRECTORY); + const fontBase64 = await fontLoader.loadGoogleFont(fontFamily); + if (fontBase64) { + console.log('Custom font loaded successfully'); + + await svgCreator.setCustomFont(fontBase64); + } else { + console.log('Custom font loading failed, using default font'); + } + } catch (error: unknown) { + console.error('Error loading custom font:', error); + // Continue with system fonts if custom font fails + } + const transcription = await getTranscription(mediaPath, userId, totalDuration); - console.log(transcription.words); const subtitleClips = transcription.generateSubtitleClips( SubtitleMode.static, exportOptions.width / exportOptions.height, 20, ); - console.log('Subtitle clips', subtitleClips); - // Create subtitles let subtitleCounter = 0; @@ -90,16 +107,6 @@ export async function createSubtitles( subtitlesToProcess.push(''); } } - console.log(subtitlesToProcess); - - // // Pre-calculate all needed subtitles to avoid redundant processing - // const uniqueClips = new Map(); - // for (const { subtitleClip } of framesToProcess) { - // const key = subtitleClip.text; - // if (!uniqueClips.has(key)) { - // uniqueClips.set(key, subtitleClip); - // } - // } for (const subtitleText of subtitlesToProcess) { const svgString = svgCreator.getSvgWithText([subtitleText], 0); diff --git a/app/services/highlighter/rendering/start-rendering.ts b/app/services/highlighter/rendering/start-rendering.ts index f4b144f0c67d..b66c30004cf3 100644 --- a/app/services/highlighter/rendering/start-rendering.ts +++ b/app/services/highlighter/rendering/start-rendering.ts @@ -77,15 +77,20 @@ export async function startRendering( exportOptions.subtitles = { enabled: true }; // Create subtitles before audio is mixed in if (exportOptions.subtitles?.enabled) { - const subtitleDirectory = await createSubtitles( - audioConcat, - userId, - parsed, - exportOptions, - totalDuration, - totalFramesAfterTransitions, - ); - exportOptions.subtitles.directory = subtitleDirectory; + try { + const subtitleDirectory = await createSubtitles( + audioConcat, + userId, + parsed, + exportOptions, + totalDuration, + totalFramesAfterTransitions, + ); + exportOptions.subtitles.directory = subtitleDirectory; + } catch (error: unknown) { + console.error('Error creating subtitles', error); + exportOptions.subtitles = { enabled: false }; + } } // create transcriptions diff --git a/app/services/highlighter/subtitles/font-loader.ts b/app/services/highlighter/subtitles/font-loader.ts new file mode 100644 index 000000000000..fabe3f405d3d --- /dev/null +++ b/app/services/highlighter/subtitles/font-loader.ts @@ -0,0 +1,88 @@ +import axios from 'axios'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as crypto from 'crypto'; +import { FONT_CACHE_DIRECTORY } from '../constants'; + +export class FontLoader { + private static instance: FontLoader; + private fontCache: Map; + private fontCacheDir: string; + + private constructor() { + this.fontCache = new Map(); + this.fontCacheDir = FONT_CACHE_DIRECTORY; + + // Ensure cache directory exists + if (!fs.existsSync(this.fontCacheDir)) { + fs.mkdirSync(this.fontCacheDir, { recursive: true }); + } + } + + public static getInstance(): FontLoader { + if (!FontLoader.instance) { + FontLoader.instance = new FontLoader(); + } + return FontLoader.instance; + } + + /** + * Loads a Google Font and returns it as a base64 string + * @param fontFamily The name of the font family to load + * @returns A base64 string representation of the font or null if loading failed + */ + public async loadGoogleFont(fontFamily: string): Promise { + try { + // Check if font is already in memory cache + if (this.fontCache.has(fontFamily)) { + return this.fontCache.get(fontFamily) || null; + } + + // Create a hash of the font name for file caching + const fontHash = crypto.createHash('md5').update(fontFamily).digest('hex'); + const cachePath = path.join(this.fontCacheDir, `${fontHash}.font`); + + // Check if font exists in file cache + if (fs.existsSync(cachePath)) { + const cachedFont = fs.readFileSync(cachePath, 'utf8'); + this.fontCache.set(fontFamily, cachedFont); + return cachedFont; + } + + // Format the font name for the Google Fonts API (replace spaces with +) + const formattedFontName = fontFamily.replace(/\s+/g, '+'); + const googleFontUrl = `https://fonts.googleapis.com/css2?family=${formattedFontName}:wght@400;700&display=swap`; + console.log(googleFontUrl); + + // Request the CSS file from Google Fonts + const response = await axios.get(googleFontUrl); + + // Extract the font URL from the CSS + const cssContent = response.data; + const fontUrlMatch = cssContent.match(/url\(([^)]+\.woff2)\)/); + console.log(fontUrlMatch); + + if (!fontUrlMatch || !fontUrlMatch[1]) { + console.error('Failed to extract font URL from CSS'); + return null; + } + + // Download the actual font file + const fontUrl = fontUrlMatch[1]; + const fontResponse = await axios.get(fontUrl, { responseType: 'arraybuffer' }); + const fontBuffer = Buffer.from(fontResponse.data); + + // Convert to base64 + const base64Font = `data:font/woff2;base64,${fontBuffer.toString('base64')}`; + + // Cache the result + this.fontCache.set(fontFamily, base64Font); + fs.writeFileSync(cachePath, base64Font); + + return base64Font; + } catch (error: unknown) { + console.error('Error loading Google Font:', error); + return null; + } + } +} diff --git a/app/services/highlighter/subtitles/svg-creator.ts b/app/services/highlighter/subtitles/svg-creator.ts index 56412e3a62a6..a7ef3f44864a 100644 --- a/app/services/highlighter/subtitles/svg-creator.ts +++ b/app/services/highlighter/subtitles/svg-creator.ts @@ -6,16 +6,19 @@ export interface ITextStyle { fontSize: number; fontFamily: string; fontColor: string; - isBold: boolean; - isItalic: boolean; + strokeColor?: string; + isBold?: boolean; + isItalic?: boolean; } export class SvgCreator { private lines: string[]; private fontFamily: string; private fontSize: number; private fontColor: string; + private strokeColor: string; private isBold: boolean; private isItalic: boolean; + private fontBase64: string; private backgroundColor: string; private backgroundAlpha: number; @@ -48,6 +51,7 @@ export class SvgCreator { this.svgType = 'Subtitle'; this.resolution = resolution; this.subtitleHeightPositionFactor = this.calculateSubtitleHeightFactor(resolution); + this.fontBase64 = null; if (textElementOptions) { this.isBold = textElementOptions.isBold; @@ -55,6 +59,7 @@ export class SvgCreator { this.fontSize = textElementOptions.fontSize; this.fontFamily = textElementOptions.fontFamily; this.fontColor = textElementOptions.fontColor; + this.strokeColor = textElementOptions.strokeColor; // this.backgroundColor = textElementOptions.backgroundColor; // this.backgroundAlpha = textElementOptions.backgroundColor === 'transparent' ? 0 : 1; // this.backgroundBorderRadius = 2 * textElementOptions.scale; @@ -72,7 +77,9 @@ export class SvgCreator { // } } } - + public async setCustomFont(fontBase64: string): Promise { + this.fontBase64 = fontBase64; + } public static getProgressSquare(color: string) { return ` @@ -104,7 +111,10 @@ export class SvgCreator { } private get svgSkeleton(): string { - return ` + return ` + ${this.fontBase64 ? this.getFontFaceStyle() : ''} ${this.background} @@ -208,7 +218,11 @@ export class SvgCreator { text-anchor:middle; fill:${fontColor || '#ffffff'}; fill-opacity:${alpha}; - stroke-opacity:0; + + stroke:${this.strokeColor || 'none'}; + stroke-opacity: ${this.strokeColor ? 1 : 0}; + stroke-width:${this.strokeColor ? 1 : 0}px; + font-family: '${this.fontFamily || 'Sans-Serif'}'; font-style: ${this.isItalic ? 'italic' : 'normal'}; font-weight: ${this.isBold ? 'bold' : 'normal'}; @@ -285,4 +299,25 @@ export class SvgCreator { return false; } } + + private getFontFaceStyle(): string { + if (!this.fontBase64) return ''; + + return ` + + `; + } } From 552eb5da67df58a172fb5f1dbdc09628164f8ad7 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Thu, 3 Apr 2025 11:25:05 +0200 Subject: [PATCH 10/34] added stroke option --- .../highlighter/rendering/render-subtitle.ts | 24 +---- .../highlighter/subtitles/svg-creator.ts | 99 +++++-------------- 2 files changed, 26 insertions(+), 97 deletions(-) diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts index 1e26d387c656..3a65707769b2 100644 --- a/app/services/highlighter/rendering/render-subtitle.ts +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -54,33 +54,15 @@ export async function createSubtitles( const exportResolution = exportOptions.complexFilter ? { width: exportOptions.height, height: exportOptions.width } : { width: exportOptions.width, height: exportOptions.height }; - const fontFamily = 'Montserrat'; const svgCreator = new SvgCreator(exportResolution, { fontSize: 46, - fontFamily, + fontFamily: 'Impact', fontColor: 'white', + strokeColor: 'black', + strokeWidth: 6, isBold: true, - isItalic: false, }); - // Load custom font - try { - console.log('Loading custom font:', fontFamily); - const fontLoader = FontLoader.getInstance(); - console.log(FONT_CACHE_DIRECTORY); - const fontBase64 = await fontLoader.loadGoogleFont(fontFamily); - if (fontBase64) { - console.log('Custom font loaded successfully'); - - await svgCreator.setCustomFont(fontBase64); - } else { - console.log('Custom font loading failed, using default font'); - } - } catch (error: unknown) { - console.error('Error loading custom font:', error); - // Continue with system fonts if custom font fails - } - const transcription = await getTranscription(mediaPath, userId, totalDuration); const subtitleClips = transcription.generateSubtitleClips( diff --git a/app/services/highlighter/subtitles/svg-creator.ts b/app/services/highlighter/subtitles/svg-creator.ts index a7ef3f44864a..55e022cc81a3 100644 --- a/app/services/highlighter/subtitles/svg-creator.ts +++ b/app/services/highlighter/subtitles/svg-creator.ts @@ -7,18 +7,14 @@ export interface ITextStyle { fontFamily: string; fontColor: string; strokeColor?: string; + strokeWidth?: number; isBold?: boolean; isItalic?: boolean; } export class SvgCreator { private lines: string[]; - private fontFamily: string; - private fontSize: number; - private fontColor: string; - private strokeColor: string; - private isBold: boolean; - private isItalic: boolean; - private fontBase64: string; + + private textStyle: ITextStyle; private backgroundColor: string; private backgroundAlpha: number; @@ -41,51 +37,23 @@ export class SvgCreator { private backgroundWidth: number; private rtlLanguage = false; - constructor( - resolution: IResolution, - textElementOptions?: ITextStyle, - scaleBackground?: boolean, - rightToLeftLanguage = false, - ) { - this.rtlLanguage = rightToLeftLanguage; + constructor(resolution: IResolution, textElementOptions?: ITextStyle) { this.svgType = 'Subtitle'; this.resolution = resolution; this.subtitleHeightPositionFactor = this.calculateSubtitleHeightFactor(resolution); - this.fontBase64 = null; if (textElementOptions) { - this.isBold = textElementOptions.isBold; - this.isItalic = textElementOptions.isItalic; - this.fontSize = textElementOptions.fontSize; - this.fontFamily = textElementOptions.fontFamily; - this.fontColor = textElementOptions.fontColor; - this.strokeColor = textElementOptions.strokeColor; - // this.backgroundColor = textElementOptions.backgroundColor; - // this.backgroundAlpha = textElementOptions.backgroundColor === 'transparent' ? 0 : 1; - // this.backgroundBorderRadius = 2 * textElementOptions.scale; - // this.lineWidth = textElementOptions.width * textElementOptions.scale; - // this.x = textElementOptions.x; - // this.y = textElementOptions.y; - // this.rectHeight = textElementOptions.height * textElementOptions.scale; - // this.lineHeight = textElementOptions.fontSize * textElementOptions.scale; - // this.scale = textElementOptions.scale; - // this.rotation = textElementOptions.rotation; - // if (!scaleBackground) { - // this.backgroundWidth = textElementOptions.width; - // } else { - // this.backgroundWidth = this.lineWidth; - // } + this.textStyle = textElementOptions; } } - public async setCustomFont(fontBase64: string): Promise { - this.fontBase64 = fontBase64; - } + public static getProgressSquare(color: string) { return ` `; } + public getSvgWithText(lines: string[], lineWidth: number): string { this.lines = []; this.lines = lines; @@ -104,21 +72,20 @@ export class SvgCreator { this.x = this.resolution.width * 0.5; this.y = this.resolution.height * 0.8; const lineHeightFactor = 1.7; - this.lineHeight = this.fontSize * lineHeightFactor - this.fontSize / 4; - this.rectHeight = lines.length * this.fontSize * lineHeightFactor + this.fontSize / 3; + this.lineHeight = this.textStyle.fontSize * lineHeightFactor - this.textStyle.fontSize / 4; + this.rectHeight = + lines.length * this.textStyle.fontSize * lineHeightFactor + this.textStyle.fontSize / 3; } return this.svgSkeleton; } private get svgSkeleton(): string { - return ` - ${this.fontBase64 ? this.getFontFaceStyle() : ''} + return ` + ${this.background} - ${this.textStyle} + ${this.textStyleELement} ${this.tspans} `; @@ -199,14 +166,14 @@ export class SvgCreator { }); return tspans; } - private get textStyle(): string { + private get textStyleELement(): string { let svgRotation = ''; if (this.svgType === 'CanvasText' && this.rotation) { svgRotation = `transform="rotate(${this.rotation} ${this.lineWidth / 2} ${ this.rectHeight / 2 })"`; } - let fontColor = this.fontColor; + let fontColor = this.textStyle.fontColor; let alpha = 1; if (fontColor.includes('rgba')) { const transformedColor = SvgCreator.transformRgba(fontColor); @@ -219,15 +186,16 @@ export class SvgCreator { fill:${fontColor || '#ffffff'}; fill-opacity:${alpha}; - stroke:${this.strokeColor || 'none'}; - stroke-opacity: ${this.strokeColor ? 1 : 0}; - stroke-width:${this.strokeColor ? 1 : 0}px; + paint-order: stroke fill; + stroke:${this.textStyle.strokeColor || 'none'}; + stroke-opacity: ${this.textStyle.strokeColor ? 1 : 0}; + stroke-width:${this.textStyle.strokeWidth ?? 0}px; - font-family: '${this.fontFamily || 'Sans-Serif'}'; - font-style: ${this.isItalic ? 'italic' : 'normal'}; - font-weight: ${this.isBold ? 'bold' : 'normal'}; + font-family: '${this.textStyle.fontFamily || 'Sans-Serif'}'; + font-style: ${this.textStyle.isItalic ? 'italic' : 'normal'}; + font-weight: ${this.textStyle.isBold ? 'bold' : 'normal'}; font-variant:normal; - font-size:${this.fontSize}px; + font-size:${this.textStyle.fontSize}px; " x="0" y="0" ${svgRotation}> `; @@ -299,25 +267,4 @@ export class SvgCreator { return false; } } - - private getFontFaceStyle(): string { - if (!this.fontBase64) return ''; - - return ` - - `; - } } From 8db3033699b7a9e372629a07d75283b87693398f Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Thu, 3 Apr 2025 11:45:09 +0200 Subject: [PATCH 11/34] transcriptionInProgress State --- app/services/highlighter/index.ts | 1 + app/services/highlighter/models/rendering.models.ts | 2 +- app/services/highlighter/rendering/start-rendering.ts | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index 1d4f6d4f4a81..0d1c1fe285c8 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -106,6 +106,7 @@ export class HighlighterService extends PersistentStatefulService Date: Fri, 4 Apr 2025 11:15:50 +0200 Subject: [PATCH 12/34] added ui --- .../highlighter/ExportModal.m.less | 12 +- .../highlighter/ExportModal.tsx | 249 +++++++++++++++++- app/services/highlighter/index.ts | 15 ++ .../highlighter/models/rendering.models.ts | 8 + .../highlighter/rendering/start-rendering.ts | 3 +- 5 files changed, 279 insertions(+), 8 deletions(-) diff --git a/app/components-react/highlighter/ExportModal.m.less b/app/components-react/highlighter/ExportModal.m.less index eddf45f03f0d..253e79aec646 100644 --- a/app/components-react/highlighter/ExportModal.m.less +++ b/app/components-react/highlighter/ExportModal.m.less @@ -132,6 +132,15 @@ height: 100%; } +.subtitle-preview { + position: absolute; + bottom: 0; + left: 0; + right: 0; + display: flex; + justify-content: center; +} + .clip-info-wrapper { display: flex; justify-content: space-between; @@ -197,7 +206,8 @@ .inner-dropdown-item { color: white; width: 100%; - height: 32px; + height: 100%; + min-height: 32px; border-radius: 4px; background-color: #232d35; display: flex; diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index 388b4eaaad44..41f6493b8984 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -4,6 +4,8 @@ import { TFPS, TResolution, TPreset, + ISubtitleOptions, + ISubtitleStyle, } from 'services/highlighter/models/rendering.models'; import { Services } from 'components-react/service-provider'; import { FileInput, TextInput, ListInput } from 'components-react/shared/inputs'; @@ -24,6 +26,7 @@ import styles from './ExportModal.m.less'; import { getCombinedClipsDuration } from './utils'; import { formatSecondsToHMS } from './ClipPreview'; import cx from 'classnames'; +import { ISubtitleConfig } from 'services/highlighter/subtitles/subtitle-mode'; type TSetting = { name: string; fps: TFPS; resolution: TResolution; preset: TPreset }; const settings: TSetting[] = [ @@ -31,6 +34,36 @@ const settings: TSetting[] = [ { name: 'Best', fps: 60, resolution: 1080, preset: 'slow' }, { name: 'Custom', fps: 30, resolution: 720, preset: 'ultrafast' }, ]; + +const subtitleSettings: ISubtitleOptions[] = [ + { + name: 'No subtitles', + enabled: false, + style: undefined, + }, + { + name: 'On', + enabled: true, + style: { + fontSize: 32, + fontFamily: 'Impact', + fontColor: '#ffffff', + strokeColor: '#000000', + strokeWidth: 5, + }, + }, + { + name: 'On', + enabled: true, + style: { + fontSize: 32, + fontFamily: 'Impact', + fontColor: '#00ff00', + strokeColor: '#ff00ff', + strokeWidth: 3, + }, + }, +]; class ExportController { get service() { return Services.HighlighterService; @@ -77,6 +110,10 @@ class ExportController { this.service.actions.setPreset(value as TPreset); } + setSubtitles(subtitleOptions: ISubtitleOptions) { + this.service.actions.setSubtitles(subtitleOptions); + } + setExport(exportFile: string) { this.service.actions.setExportFile(exportFile); } @@ -167,6 +204,7 @@ function ExportFlow({ setResolution, setFps, setPreset, + setSubtitles, fileExists, setExport, exportCurrentFile, @@ -199,6 +237,12 @@ function ExportFlow({ }; } + const [currentSubtitleSettings, setSubtitleSettings] = useState({ + name: 'Off', + enabled: false, + style: undefined, + }); + const [currentSetting, setSetting] = useState( settingMatcher({ name: 'from default', @@ -312,6 +356,17 @@ function ExportFlow({ /> )} + {currentSubtitleSettings.enabled && ( +
+ +
+ )} - setCurrentFormat(format)} - /> +
+ { + setSubtitleSettings(setting); + setSubtitles(setting); + }} + /> + setCurrentFormat(format)} + /> +
void; +}) { + const [isOpen, setIsOpen] = useState(false); + const [currentSetting, setSetting] = useState(initialSetting); + + return ( +
+ + {subtitleSettings.map(setting => { + return ( +
{ + setSetting(setting); + emitSettings(setting); + setIsOpen(false); + }} + key={setting.name} + > + {setting.enabled === false ? ( +
{setting.name}
+ ) : ( + + )} +
+ ); + })} +
+ } + trigger={['click']} + visible={isOpen} + onVisibleChange={setIsOpen} + placement="bottomCenter" + > +
setIsOpen(!isOpen)} + > +
+ +
+ +
+ +
+ ); +} + function CustomDropdownWrapper({ initialSetting, disabled, @@ -555,7 +689,7 @@ function CustomDropdownWrapper({ )} - + @@ -598,3 +732,106 @@ function OrientationToggle({ ); } + +export const SubtitlePreview = (style: ISubtitleStyle, inVideo: boolean) => { + const WIDTH = 250; + const HEIGHT = inVideo ? 60 : 250; + return ( +
+ + {/* */} + + Auto subtitles + + +
+ ); +}; + +export const SubtitleIcon = () => ( + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +); diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index 0d1c1fe285c8..15f5ed71a287 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -50,6 +50,7 @@ import { IAudioInfo, IExportInfo, IExportOptions, + ISubtitleOptions, ITransitionInfo, IVideoInfo, TFPS, @@ -116,6 +117,15 @@ export class HighlighterService extends PersistentStatefulService Date: Fri, 4 Apr 2025 11:33:52 +0200 Subject: [PATCH 13/34] fix non subtile export --- .../highlighter/rendering/frame-writer.ts | 22 ++++++++++++------- .../highlighter/rendering/start-rendering.ts | 1 - 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/app/services/highlighter/rendering/frame-writer.ts b/app/services/highlighter/rendering/frame-writer.ts index fb45b293e8f9..a04ef615a1b0 100644 --- a/app/services/highlighter/rendering/frame-writer.ts +++ b/app/services/highlighter/rendering/frame-writer.ts @@ -41,12 +41,12 @@ export class FrameWriter { // '-map', // '0:v:0', ]; - if (this.options.subtitles || true) { + if (this.options.subtitles?.enabled) { console.log('adding subtitle input'); await this.addSubtitleInput(args, this.options); } - this.addAudioFilters(args); - this.addVideoFilters(args, true); //!!this.options.subtitles + this.addAudioFilters(args, this.options.subtitles?.enabled); + this.addVideoFilters(args, this.options.subtitles?.enabled); args.push( ...[ @@ -98,36 +98,42 @@ export class FrameWriter { console.log('ffmpeg:', data.toString()); }); } - private addVideoFilters(args: string[], subtitlesEnabled: boolean) { + private addVideoFilters(args: string[], subtitlesEnabled = false) { const webcamEnabled = !!this.options.complexFilter; const firstInput = webcamEnabled ? '[final]' : '[0:v]'; const output = subtitlesEnabled ? '[subtitled]' : '[final]'; - const subtitleFilter = subtitlesEnabled ? `${firstInput}[1:v]overlay=0:0[subtitled];` : ''; const fadeFilter = `format=yuv420p,fade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( this.duration - (FADE_OUT_DURATION + 0.2), 0, )}`; args.push('-filter_complex'); + + if (!webcamEnabled && !subtitlesEnabled) { + args.push(fadeFilter); + return; + } + let combinedFilter = ''; if (webcamEnabled) { combinedFilter += this.options.complexFilter; } if (subtitlesEnabled) { - combinedFilter += subtitleFilter; + combinedFilter += `${firstInput}[1:v]overlay=0:0[subtitled];`; } + combinedFilter += output + fadeFilter; args.push(combinedFilter); } - private addAudioFilters(args: string[]) { + private addAudioFilters(args: string[], subtitlesEnabled = false) { args.push( '-i', this.audioInput, '-map', - '2:a:0', + subtitlesEnabled ? '2:a:0' : '1:a:0', '-af', `afade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( this.duration - (FADE_OUT_DURATION + 0.2), diff --git a/app/services/highlighter/rendering/start-rendering.ts b/app/services/highlighter/rendering/start-rendering.ts index 6d233b9302bb..4cc5166b9572 100644 --- a/app/services/highlighter/rendering/start-rendering.ts +++ b/app/services/highlighter/rendering/start-rendering.ts @@ -74,7 +74,6 @@ export async function startRendering( fader = new AudioCrossfader(audioConcat, renderingClips, transitionDuration); await fader.export(); - exportOptions.subtitles = { enabled: true }; // Create subtitles before audio is mixed in if (exportOptions.subtitles?.enabled) { try { From 861f40ecc74bd17f160f9acb318c2e2257ded1f3 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Fri, 4 Apr 2025 15:26:15 +0200 Subject: [PATCH 14/34] type refactoring and pass style to svgCreator --- .../highlighter/ExportModal.tsx | 106 +++++++++--------- app/services/highlighter/index.ts | 19 +--- .../highlighter/models/rendering.models.ts | 13 +-- .../highlighter/rendering/frame-writer.ts | 13 +-- .../highlighter/rendering/render-subtitle.ts | 15 +-- .../highlighter/rendering/start-rendering.ts | 11 +- .../highlighter/subtitles/subtitle-styles.ts | 27 +++++ 7 files changed, 104 insertions(+), 100 deletions(-) create mode 100644 app/services/highlighter/subtitles/subtitle-styles.ts diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index 41f6493b8984..92728978995d 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -4,7 +4,6 @@ import { TFPS, TResolution, TPreset, - ISubtitleOptions, ISubtitleStyle, } from 'services/highlighter/models/rendering.models'; import { Services } from 'components-react/service-provider'; @@ -26,42 +25,41 @@ import styles from './ExportModal.m.less'; import { getCombinedClipsDuration } from './utils'; import { formatSecondsToHMS } from './ClipPreview'; import cx from 'classnames'; -import { ISubtitleConfig } from 'services/highlighter/subtitles/subtitle-mode'; +import { SubtitleStyles } from 'services/highlighter/subtitles/subtitle-styles'; type TSetting = { name: string; fps: TFPS; resolution: TResolution; preset: TPreset }; + +interface ISubtitleItem { + name: string; + enabled: boolean; + style?: ISubtitleStyle; +} const settings: TSetting[] = [ { name: 'Standard', fps: 30, resolution: 1080, preset: 'fast' }, { name: 'Best', fps: 60, resolution: 1080, preset: 'slow' }, { name: 'Custom', fps: 30, resolution: 720, preset: 'ultrafast' }, ]; -const subtitleSettings: ISubtitleOptions[] = [ +const subtitleItems: ISubtitleItem[] = [ { name: 'No subtitles', enabled: false, style: undefined, }, { - name: 'On', + name: 'Default', + enabled: true, + style: SubtitleStyles.default, + }, + { + name: 'Thick', enabled: true, - style: { - fontSize: 32, - fontFamily: 'Impact', - fontColor: '#ffffff', - strokeColor: '#000000', - strokeWidth: 5, - }, + style: SubtitleStyles.thick, }, { - name: 'On', + name: 'Flashy', enabled: true, - style: { - fontSize: 32, - fontFamily: 'Impact', - fontColor: '#00ff00', - strokeColor: '#ff00ff', - strokeWidth: 3, - }, + style: SubtitleStyles.flashy, }, ]; class ExportController { @@ -110,8 +108,8 @@ class ExportController { this.service.actions.setPreset(value as TPreset); } - setSubtitles(subtitleOptions: ISubtitleOptions) { - this.service.actions.setSubtitles(subtitleOptions); + setSubtitles(subtitleItem: ISubtitleItem) { + this.service.actions.setSubtitleStyle(subtitleItem.style); } setExport(exportFile: string) { @@ -237,7 +235,7 @@ function ExportFlow({ }; } - const [currentSubtitleSettings, setSubtitleSettings] = useState({ + const [currentSubtitleItem, setSubtitleItem] = useState({ name: 'Off', enabled: false, style: undefined, @@ -356,14 +354,14 @@ function ExportFlow({ /> )} - {currentSubtitleSettings.enabled && ( + {currentSubtitleItem?.style && (
)} @@ -396,10 +394,10 @@ function ExportFlow({
{ - setSubtitleSettings(setting); + setSubtitleItem(setting); setSubtitles(setting); }} /> @@ -564,45 +562,47 @@ function PlatformSelect({ } function SubtitleDropdownWrapper({ - initialSetting, + initialSetting: initialItem, disabled, emitSettings, }: { - initialSetting: ISubtitleOptions; + initialSetting: ISubtitleItem; disabled: boolean; - emitSettings: (settings: ISubtitleOptions) => void; + emitSettings: (item: ISubtitleItem) => void; }) { const [isOpen, setIsOpen] = useState(false); - const [currentSetting, setSetting] = useState(initialSetting); + const [currentSetting, setSetting] = useState(initialItem); return (
- {subtitleSettings.map(setting => { + {subtitleItems.map(item => { return (
{ - setSetting(setting); - emitSettings(setting); + setSetting(item); + emitSettings(item); setIsOpen(false); }} - key={setting.name} + key={item.name} > - {setting.enabled === false ? ( -
{setting.name}
+ {item.enabled === false ? ( +
{item.name}
) : ( - + item.style && ( + + ) )}
); @@ -749,17 +749,17 @@ export const SubtitlePreview = (style: ISubtitleStyle, inVideo: boolean) => { fontFamily={style.fontFamily} fontStyle={style.isItalic ? 'italic' : 'normal'} fontWeight={style.isBold ? 'bold' : 'normal'} - fontSize={style.fontSize} + fill={style.fontColor} + fontSize="24" //{style.fontSize} textAnchor="middle" dominantBaseline="middle" paintOrder="stroke fill" - strokeWidth={style.strokeWidth} - stroke={style.strokeColor} - fill={style.fontColor} + strokeWidth={(style.strokeWidth ?? 0) + 'px'} + stroke={style.strokeColor || 'none'} x={WIDTH / 2} y={HEIGHT / 2} > - Auto subtitles + {(style.strokeWidth ?? 0) + 'px'}
diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index 15f5ed71a287..d93908dc012f 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -50,7 +50,7 @@ import { IAudioInfo, IExportInfo, IExportOptions, - ISubtitleOptions, + ISubtitleStyle, ITransitionInfo, IVideoInfo, TFPS, @@ -75,6 +75,7 @@ import { reduce } from 'lodash'; import { extractDateTimeFromPath, fileExists } from './file-utils'; import { addVerticalFilterToExportOptions } from './vertical-export'; import Utils from '../utils'; +import { SubtitleStyles } from './subtitles/subtitle-styles'; @InitAfter('StreamingService') export class HighlighterService extends PersistentStatefulService { @@ -117,15 +118,7 @@ export class HighlighterService extends PersistentStatefulService; @@ -41,12 +42,11 @@ export class FrameWriter { // '-map', // '0:v:0', ]; - if (this.options.subtitles?.enabled) { - console.log('adding subtitle input'); - await this.addSubtitleInput(args, this.options); + if (this.options.subtitleStyle && this.subtitleDirectory) { + await this.addSubtitleInput(args, this.subtitleDirectory); } - this.addAudioFilters(args, this.options.subtitles?.enabled); - this.addVideoFilters(args, this.options.subtitles?.enabled); + this.addAudioFilters(args, !!this.options.subtitleStyle); + this.addVideoFilters(args, !!this.options.subtitleStyle); args.push( ...[ @@ -141,8 +141,7 @@ export class FrameWriter { )}`, ); } - private async addSubtitleInput(args: string[], exportOptions: IExportOptions) { - const subtitleDirectory = exportOptions.subtitles.directory; + private async addSubtitleInput(args: string[], subtitleDirectory: string) { args.push( '-framerate', String(SUBTITLE_PER_SECOND), diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts index 3a65707769b2..7050c1500477 100644 --- a/app/services/highlighter/rendering/render-subtitle.ts +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -54,14 +54,7 @@ export async function createSubtitles( const exportResolution = exportOptions.complexFilter ? { width: exportOptions.height, height: exportOptions.width } : { width: exportOptions.width, height: exportOptions.height }; - const svgCreator = new SvgCreator(exportResolution, { - fontSize: 46, - fontFamily: 'Impact', - fontColor: 'white', - strokeColor: 'black', - strokeWidth: 6, - isBold: true, - }); + const svgCreator = new SvgCreator(exportResolution, exportOptions.subtitleStyle); const transcription = await getTranscription(mediaPath, userId, totalDuration); @@ -102,10 +95,10 @@ export async function createSubtitles( return subtitleDirectory; } -export function cleanupSubtitleDirectory(exportOptions: IExportOptions) { - if (exportOptions.subtitles?.directory) { +export function cleanupSubtitleDirectory(directory: string) { + if (directory) { try { - fs.removeSync(exportOptions.subtitles.directory); + fs.removeSync(directory); } catch (error: unknown) { console.error('Failed to clean up subtitle directory', error); } diff --git a/app/services/highlighter/rendering/start-rendering.ts b/app/services/highlighter/rendering/start-rendering.ts index 4cc5166b9572..5d8d936ba0ce 100644 --- a/app/services/highlighter/rendering/start-rendering.ts +++ b/app/services/highlighter/rendering/start-rendering.ts @@ -49,6 +49,7 @@ export async function startRendering( let fader: AudioCrossfader | null = null; let mixer: AudioMixer | null = null; + let subtitleDirectory: string | null = null; try { // Estimate the total number of frames to set up export info const totalFrames = renderingClips.reduce((count: number, clip) => { @@ -75,12 +76,12 @@ export async function startRendering( await fader.export(); // Create subtitles before audio is mixed in - if (exportOptions.subtitles?.enabled) { + if (exportOptions.subtitleStyle) { try { setExportInfo({ transcriptionInProgress: true, }); - const subtitleDirectory = await createSubtitles( + subtitleDirectory = await createSubtitles( audioConcat, userId, parsed, @@ -88,10 +89,9 @@ export async function startRendering( totalDuration, totalFramesAfterTransitions, ); - exportOptions.subtitles.directory = subtitleDirectory; } catch (error: unknown) { console.error('Error creating subtitles', error); - exportOptions.subtitles = { enabled: false }; + exportOptions.subtitleStyle = null; } finally { setExportInfo({ transcriptionInProgress: false, @@ -133,6 +133,7 @@ export async function startRendering( audioMix, totalFramesAfterTransitions / exportOptions.fps, exportOptions, + subtitleDirectory, ); while (true) { @@ -262,7 +263,7 @@ export async function startRendering( exported: !exportInfo.cancelRequested && !isPreview && !exportInfo.error, }); // Clean up subtitle directory if it was created - cleanupSubtitleDirectory(exportOptions); + cleanupSubtitleDirectory(subtitleDirectory); if (fader) await fader.cleanup(); if (mixer) await mixer.cleanup(); } diff --git a/app/services/highlighter/subtitles/subtitle-styles.ts b/app/services/highlighter/subtitles/subtitle-styles.ts new file mode 100644 index 000000000000..bafdefcf27e7 --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-styles.ts @@ -0,0 +1,27 @@ +import { ISubtitleStyle } from '../models/rendering.models'; + +export type SubtitleStyleName = 'default' | 'flashy' | 'thick'; + +export const SubtitleStyles: { [name in SubtitleStyleName]: ISubtitleStyle } = { + default: { + fontColor: '#FFFFFF', + fontSize: 48, + fontFamily: 'Arial', + strokeColor: '#FF000', + strokeWidth: 1, + }, + flashy: { + fontColor: '#FF00FF', + fontSize: 48, + fontFamily: 'Arial', + strokeColor: '#FF000', + strokeWidth: 2, + }, + thick: { + fontFamily: 'Impact', + fontSize: 48, + fontColor: '#FFFFFF', + strokeColor: '#FF000', + strokeWidth: 6, + }, +}; From 0427ad2b2b6f22568de740fb6ef136896844ed97 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Tue, 8 Apr 2025 12:14:55 +0200 Subject: [PATCH 15/34] fix subtitle preview and stroke --- .../highlighter/ExportModal.tsx | 39 +++++++------------ .../highlighter/subtitles/svg-creator.ts | 2 +- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index 92728978995d..17f559d6af80 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -356,13 +356,7 @@ function ExportFlow({ )} {currentSubtitleItem?.style && (
- + currentSubtitleItem.style &&
)} {item.name}
) : ( - item.style && ( - - ) + item.style && )} ); @@ -733,7 +719,7 @@ function OrientationToggle({ ); } -export const SubtitlePreview = (style: ISubtitleStyle, inVideo: boolean) => { +export const SubtitlePreview = (style: ISubtitleStyle, inVideo?: boolean) => { const WIDTH = 250; const HEIGHT = inVideo ? 60 : 250; return ( @@ -754,12 +740,13 @@ export const SubtitlePreview = (style: ISubtitleStyle, inVideo: boolean) => { textAnchor="middle" dominantBaseline="middle" paintOrder="stroke fill" - strokeWidth={(style.strokeWidth ?? 0) + 'px'} + strokeWidth={style.strokeWidth ?? 0} stroke={style.strokeColor || 'none'} + strokeOpacity={style.strokeColor ? 1 : 0} x={WIDTH / 2} y={HEIGHT / 2} > - {(style.strokeWidth ?? 0) + 'px'} + Auto subtitles @@ -768,12 +755,12 @@ export const SubtitlePreview = (style: ISubtitleStyle, inVideo: boolean) => { export const SubtitleIcon = () => ( - + - + - + ( width="14" height="10" filterUnits="userSpaceOnUse" - color-interpolation-filters="sRGB" + colorInterpolationFilters="sRGB" > - + ( width="18" height="10" filterUnits="userSpaceOnUse" - color-interpolation-filters="sRGB" + colorInterpolationFilters="sRGB" > - + Date: Tue, 8 Apr 2025 13:47:17 +0200 Subject: [PATCH 16/34] subtitle styles adjusted --- .../highlighter/ExportModal.tsx | 33 ++++++++++++------- .../highlighter/subtitles/subtitle-styles.ts | 27 +++++++++------ 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index 17f559d6af80..afb5bf5a46e2 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -26,6 +26,7 @@ import { getCombinedClipsDuration } from './utils'; import { formatSecondsToHMS } from './ClipPreview'; import cx from 'classnames'; import { SubtitleStyles } from 'services/highlighter/subtitles/subtitle-styles'; +import Utils from 'services/utils'; type TSetting = { name: string; fps: TFPS; resolution: TResolution; preset: TPreset }; @@ -57,9 +58,14 @@ const subtitleItems: ISubtitleItem[] = [ style: SubtitleStyles.thick, }, { - name: 'Flashy', + name: 'FlashyA', enabled: true, - style: SubtitleStyles.flashy, + style: SubtitleStyles.flashyA, + }, + { + name: 'FlashyB', + enabled: true, + style: SubtitleStyles.yellow, }, ]; class ExportController { @@ -217,6 +223,8 @@ function ExportFlow({ const clipsAmount = getClips(streamId).length; const clipsDuration = formatSecondsToHMS(getDuration(streamId)); + const showSubtitleSettings = ['staging', 'local'].includes(Utils.getHighlighterEnvironment()); + function settingMatcher(initialSetting: TSetting) { const matchingSetting = settings.find( setting => @@ -356,7 +364,7 @@ function ExportFlow({ )} {currentSubtitleItem?.style && (
- currentSubtitleItem.style && +
)}
- { - setSubtitleItem(setting); - setSubtitles(setting); - }} - /> + {showSubtitleSettings && ( + { + setSubtitleItem(setting); + setSubtitles(setting); + }} + /> + )} + Date: Tue, 8 Apr 2025 14:50:40 +0200 Subject: [PATCH 17/34] isTranscribing loader --- .../highlighter/ExportModal.tsx | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index afb5bf5a46e2..1372a562ee83 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -177,6 +177,7 @@ function ExportModal({ close, streamId }: { close: () => void; streamId: string return ( void; streamId: string function ExportFlow({ close, isExporting, + isTranscribing, streamId, videoName, onVideoNameChange, }: { close: () => void; isExporting: boolean; + isTranscribing: boolean; streamId: string | undefined; videoName: string; onVideoNameChange: (name: string) => void; @@ -341,27 +344,37 @@ function ExportFlow({ : { aspectRatio: '9/16' } } > - {isExporting && ( -
-

- {Math.round((exportInfo.currentFrame / exportInfo.totalFrames) * 100) || 0}% -

-

- {exportInfo.cancelRequested ? ( - {$t('Canceling...')} - ) : ( - {$t('Exporting video...')} - )} -

- -
- )} + {isExporting && + (isTranscribing ? ( +
+
+ +
+

+ {$t('Transcribing...')} +

+
+ ) : ( +
+

+ {Math.round((exportInfo.currentFrame / exportInfo.totalFrames) * 100) || 0}% +

+

+ {exportInfo.cancelRequested ? ( + {$t('Canceling...')} + ) : ( + {$t('Exporting video...')} + )} +

+ +
+ ))} {currentSubtitleItem?.style && (
From e17c6d6d5455e3a42d9c6d421955aed1b3d11ae8 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 9 Apr 2025 11:31:25 +0200 Subject: [PATCH 18/34] translation, style renaming, save to store --- .../highlighter/ExportModal.tsx | 28 ++++++++++++++----- app/i18n/en-US/highlighter.json | 3 +- app/services/highlighter/index.ts | 5 ++-- .../highlighter/subtitles/subtitle-styles.ts | 4 +-- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index 1372a562ee83..5e7a004b8af0 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -27,6 +27,7 @@ import { formatSecondsToHMS } from './ClipPreview'; import cx from 'classnames'; import { SubtitleStyles } from 'services/highlighter/subtitles/subtitle-styles'; import Utils from 'services/utils'; +import { isDeepEqual } from 'slap'; type TSetting = { name: string; fps: TFPS; resolution: TResolution; preset: TPreset }; @@ -48,9 +49,9 @@ const subtitleItems: ISubtitleItem[] = [ style: undefined, }, { - name: 'Default', + name: 'Basic', enabled: true, - style: SubtitleStyles.default, + style: SubtitleStyles.basic, }, { name: 'Thick', @@ -68,6 +69,7 @@ const subtitleItems: ISubtitleItem[] = [ style: SubtitleStyles.yellow, }, ]; + class ExportController { get service() { return Services.HighlighterService; @@ -118,6 +120,10 @@ class ExportController { this.service.actions.setSubtitleStyle(subtitleItem.style); } + getSubtitleStyle() { + return this.service.views.exportInfo.subtitleStyle; + } + setExport(exportFile: string) { this.service.actions.setExportFile(exportFile); } @@ -212,6 +218,7 @@ function ExportFlow({ setFps, setPreset, setSubtitles, + getSubtitleStyle, fileExists, setExport, exportCurrentFile, @@ -246,11 +253,9 @@ function ExportFlow({ }; } - const [currentSubtitleItem, setSubtitleItem] = useState({ - name: 'Off', - enabled: false, - style: undefined, - }); + const [currentSubtitleItem, setSubtitleItem] = useState( + findSubtitleItem(getSubtitleStyle()) || subtitleItems[0], + ); const [currentSetting, setSetting] = useState( settingMatcher({ @@ -264,6 +269,15 @@ function ExportFlow({ // Video name and export file are kept in sync const [exportFile, setExportFile] = useState(getExportFileFromVideoName(videoName)); + function findSubtitleItem(subtitleStyle: ISubtitleStyle | null) { + if (!subtitleStyle) return subtitleItems[0]; + + for (const item of subtitleItems) { + const isMatching = isDeepEqual(item.style, subtitleStyle, 0, 2); + if (isMatching) return item; + } + } + function getExportFileFromVideoName(videoName: string) { const parsed = path.parse(exportInfo.file); const sanitized = videoName.replace(/[/\\?%*:|"<>\.,;=#]/g, ''); diff --git a/app/i18n/en-US/highlighter.json b/app/i18n/en-US/highlighter.json index 0e548c29441f..80c5f92ef73b 100644 --- a/app/i18n/en-US/highlighter.json +++ b/app/i18n/en-US/highlighter.json @@ -52,6 +52,7 @@ "Rendering Frames: %{currentFrame}/%{totalFrames}": "Rendering Frames: %{currentFrame}/%{totalFrames}", "Mixing Audio:": "Mixing Audio:", "Canceling...": "Canceling...", + "Transcribing...": "Transcribing...", "Render Preview": "Render Preview", "The render preview shows a low-quality preview of the final rendered video. The final exported video will be higher resolution, framerate, and quality.": "The render preview shows a low-quality preview of the final rendered video. The final exported video will be higher resolution, framerate, and quality.", "Title": "Title", @@ -173,4 +174,4 @@ "Take a screenshot of your stream and share it here": "Take a screenshot of your stream and share it here", "Exporting video...": "Exporting video...", "%{clipsAmount} clips": "%{clipsAmount} clips" -} \ No newline at end of file +} diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index d93908dc012f..c242d53e9ad7 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -118,7 +118,7 @@ export class HighlighterService extends PersistentStatefulService Date: Wed, 9 Apr 2025 13:04:03 +0200 Subject: [PATCH 19/34] useMemo for environment --- app/components-react/highlighter/ExportModal.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index 5e7a004b8af0..c11779a3f344 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -233,7 +233,10 @@ function ExportFlow({ const clipsAmount = getClips(streamId).length; const clipsDuration = formatSecondsToHMS(getDuration(streamId)); - const showSubtitleSettings = ['staging', 'local'].includes(Utils.getHighlighterEnvironment()); + const showSubtitleSettings = useMemo( + () => ['staging', 'local'].includes(Utils.getHighlighterEnvironment()), + [], + ); function settingMatcher(initialSetting: TSetting) { const matchingSetting = settings.find( From c89279a3af488c39ee22b6b79e498598a3f9fea8 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 9 Apr 2025 13:20:01 +0200 Subject: [PATCH 20/34] cleanup --- app/services/highlighter/constants.ts | 5 -- .../highlighter/rendering/frame-writer.ts | 1 - .../highlighter/rendering/render-subtitle.ts | 2 - .../highlighter/subtitles/font-loader.ts | 88 ------------------- 4 files changed, 96 deletions(-) delete mode 100644 app/services/highlighter/subtitles/font-loader.ts diff --git a/app/services/highlighter/constants.ts b/app/services/highlighter/constants.ts index 4427a83da987..c52bde01f74f 100644 --- a/app/services/highlighter/constants.ts +++ b/app/services/highlighter/constants.ts @@ -20,11 +20,6 @@ export const SCRUB_WIDTH = 320; export const SCRUB_HEIGHT = 180; export const SCRUB_FRAMES = 20; export const SCRUB_SPRITE_DIRECTORY = path.join(remote.app.getPath('userData'), 'highlighter'); -export const FONT_CACHE_DIRECTORY = path.join( - remote.app.getPath('userData'), - 'highlighter', - 'fonts', -); export const FADE_OUT_DURATION = 1; diff --git a/app/services/highlighter/rendering/frame-writer.ts b/app/services/highlighter/rendering/frame-writer.ts index c63844242c1d..fd651fbd99ae 100644 --- a/app/services/highlighter/rendering/frame-writer.ts +++ b/app/services/highlighter/rendering/frame-writer.ts @@ -72,7 +72,6 @@ export class FrameWriter { this.outputPath, ], ); - console.log(args.join(' ')); /* eslint-enable */ this.ffmpeg = execa(FFMPEG_EXE, args, { diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts index 7050c1500477..a08b5a7be27a 100644 --- a/app/services/highlighter/rendering/render-subtitle.ts +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -5,8 +5,6 @@ import { getTranscription } from '../ai-highlighter-utils'; import { SubtitleMode } from '../subtitles/subtitle-mode'; import { IExportOptions } from '../models/rendering.models'; import path from 'path'; -import { FontLoader } from '../subtitles/font-loader'; -import { FONT_CACHE_DIRECTORY } from '../constants'; export const SUBTITLE_PER_SECOND = 3; diff --git a/app/services/highlighter/subtitles/font-loader.ts b/app/services/highlighter/subtitles/font-loader.ts deleted file mode 100644 index fabe3f405d3d..000000000000 --- a/app/services/highlighter/subtitles/font-loader.ts +++ /dev/null @@ -1,88 +0,0 @@ -import axios from 'axios'; -import * as fs from 'fs'; -import * as path from 'path'; -import * as crypto from 'crypto'; -import { FONT_CACHE_DIRECTORY } from '../constants'; - -export class FontLoader { - private static instance: FontLoader; - private fontCache: Map; - private fontCacheDir: string; - - private constructor() { - this.fontCache = new Map(); - this.fontCacheDir = FONT_CACHE_DIRECTORY; - - // Ensure cache directory exists - if (!fs.existsSync(this.fontCacheDir)) { - fs.mkdirSync(this.fontCacheDir, { recursive: true }); - } - } - - public static getInstance(): FontLoader { - if (!FontLoader.instance) { - FontLoader.instance = new FontLoader(); - } - return FontLoader.instance; - } - - /** - * Loads a Google Font and returns it as a base64 string - * @param fontFamily The name of the font family to load - * @returns A base64 string representation of the font or null if loading failed - */ - public async loadGoogleFont(fontFamily: string): Promise { - try { - // Check if font is already in memory cache - if (this.fontCache.has(fontFamily)) { - return this.fontCache.get(fontFamily) || null; - } - - // Create a hash of the font name for file caching - const fontHash = crypto.createHash('md5').update(fontFamily).digest('hex'); - const cachePath = path.join(this.fontCacheDir, `${fontHash}.font`); - - // Check if font exists in file cache - if (fs.existsSync(cachePath)) { - const cachedFont = fs.readFileSync(cachePath, 'utf8'); - this.fontCache.set(fontFamily, cachedFont); - return cachedFont; - } - - // Format the font name for the Google Fonts API (replace spaces with +) - const formattedFontName = fontFamily.replace(/\s+/g, '+'); - const googleFontUrl = `https://fonts.googleapis.com/css2?family=${formattedFontName}:wght@400;700&display=swap`; - console.log(googleFontUrl); - - // Request the CSS file from Google Fonts - const response = await axios.get(googleFontUrl); - - // Extract the font URL from the CSS - const cssContent = response.data; - const fontUrlMatch = cssContent.match(/url\(([^)]+\.woff2)\)/); - console.log(fontUrlMatch); - - if (!fontUrlMatch || !fontUrlMatch[1]) { - console.error('Failed to extract font URL from CSS'); - return null; - } - - // Download the actual font file - const fontUrl = fontUrlMatch[1]; - const fontResponse = await axios.get(fontUrl, { responseType: 'arraybuffer' }); - const fontBuffer = Buffer.from(fontResponse.data); - - // Convert to base64 - const base64Font = `data:font/woff2;base64,${fontBuffer.toString('base64')}`; - - // Cache the result - this.fontCache.set(fontFamily, base64Font); - fs.writeFileSync(cachePath, base64Font); - - return base64Font; - } catch (error: unknown) { - console.error('Error loading Google Font:', error); - return null; - } - } -} From 1518b0d9e0989f617b963758d5b9c9f4762dbf61 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 9 Apr 2025 13:33:10 +0200 Subject: [PATCH 21/34] cleanup --- app/services/highlighter/ai-highlighter-utils.ts | 8 ++------ app/services/highlighter/index.ts | 2 +- app/services/highlighter/subtitles/subtitle-clip.ts | 1 - app/services/highlighter/subtitles/transcription.ts | 11 +---------- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/app/services/highlighter/ai-highlighter-utils.ts b/app/services/highlighter/ai-highlighter-utils.ts index b46d2bcaae88..376b56088e70 100644 --- a/app/services/highlighter/ai-highlighter-utils.ts +++ b/app/services/highlighter/ai-highlighter-utils.ts @@ -119,9 +119,7 @@ export function getHighlightClips( milestoneUpdate?.(aiHighlighterMessage.json as IHighlighterMilestone); break; default: - // console.log('\n\n'); - // console.log('Unrecognized message type:', aiHighlighterMessage); - // console.log('\n\n'); + // ('Unrecognized message type:', aiHighlighterMessage); break; } } @@ -286,9 +284,7 @@ export function getTranscription( } default: - // console.log('\n\n'); - // console.log('Unrecognized message type:', aiHighlighterMessage); - // console.log('\n\n'); + // ('Unrecognized message type:', aiHighlighterMessage); break; } } diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index c242d53e9ad7..af33ae77cc39 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -118,7 +118,7 @@ export class HighlighterService extends PersistentStatefulService { - // console.log(`[${sc.startIndex}-${sc.endIndex}]`, sc.startTimeInEdit, sc.endTimeInEdit, sc.text); - // }); - // console.log(this.words); - return subtitleClips; } @@ -515,7 +510,7 @@ export class Transcription { if (pauseWords) { newTranscription.words = newTranscription.words.concat(pauseWords); } else { - // console.log('no Pause'); + // No Pause } } if (!skipPausesAfter) { @@ -610,7 +605,6 @@ export class Transcription { } const pauseTime = roundTime(secondWord.startTimeInEdit - firstWord.endTimeInEdit); - // console.log('pausetime', pauseTime); if (pauseTime > 0) { const pauseCount = Math.ceil(pauseTime / this.singlePauseLength); @@ -624,7 +618,6 @@ export class Transcription { ? this.singlePauseLength : roundTime(pauseTime - i * this.singlePauseLength); - // console.log(leftPauseTime); pauseWords.push( new Word().initPauseWord( @@ -647,7 +640,6 @@ export class Transcription { * Checks the sentence for the most dominant speaker and set this one for the whole sentence */ updateSentenceSpeaker() { - // console.log(this.words); const sentences: Word[][] = this.getSentencesArray(); const newWords: Word[] = []; for (const sentence of sentences) { @@ -667,7 +659,6 @@ export class Transcription { }); } this.words = newWords; - // console.log(newSentences.map((sentence) => sentence.words)); } /** From 89a8cbaa193429f6034d52df1a40f08a924655f5 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Thu, 10 Apr 2025 11:56:36 +0200 Subject: [PATCH 22/34] fontSize adjustment --- .../highlighter/ExportModal.tsx | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index c11779a3f344..76b2d4874e2e 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -394,7 +394,11 @@ function ExportFlow({ ))} {currentSubtitleItem?.style && (
- +
)} {item.name}
) : ( - item.style && + item.style && )}
); @@ -760,9 +764,20 @@ function OrientationToggle({ ); } -export const SubtitlePreview = (style: ISubtitleStyle, inVideo?: boolean) => { +function SubtitlePreview({ + svgStyle, + inPreview, + orientation, +}: { + svgStyle: ISubtitleStyle; + inPreview?: boolean; + orientation?: TOrientation; +}) { const WIDTH = 250; - const HEIGHT = inVideo ? 60 : 250; + const HEIGHT = 60; + const formattedFontSize = orientation === 'horizontal' ? 22 : 14; + const fontSize = inPreview ? formattedFontSize : 24; + return (
{ > {/* */} @@ -792,7 +807,7 @@ export const SubtitlePreview = (style: ISubtitleStyle, inVideo?: boolean) => {
); -}; +} export const SubtitleIcon = () => ( From 549dc48f6b61bf8dc2083be169955eca8dfadb32 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Tue, 15 Apr 2025 13:28:54 +0200 Subject: [PATCH 23/34] add version check for subtitles --- app/components-react/highlighter/ExportModal.tsx | 10 ++++++++-- app/services/highlighter/index.ts | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index 76b2d4874e2e..ed559b67a1bc 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -146,6 +146,9 @@ class ExportController { async fileExists(exportFile: string) { return await fileExists(exportFile); } + isHighlighterAfterVersion(version: string) { + return this.service.isHighlighterVersionAfter(version); + } } export const ExportModalCtx = React.createContext(null); @@ -209,7 +212,7 @@ function ExportFlow({ videoName: string; onVideoNameChange: (name: string) => void; }) { - const { UsageStatisticsService } = Services; + const { UsageStatisticsService, HighlighterService } = Services; const { exportInfo, cancelExport, @@ -226,6 +229,7 @@ function ExportFlow({ getClips, getDuration, getClipThumbnail, + isHighlighterAfterVersion, } = useController(ExportModalCtx); const [currentFormat, setCurrentFormat] = useState(EOrientation.HORIZONTAL); @@ -234,7 +238,9 @@ function ExportFlow({ const clipsDuration = formatSecondsToHMS(getDuration(streamId)); const showSubtitleSettings = useMemo( - () => ['staging', 'local'].includes(Utils.getHighlighterEnvironment()), + () => + ['staging', 'local'].includes(Utils.getHighlighterEnvironment()) && + isHighlighterAfterVersion('0.0.53'), [], ); diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index af33ae77cc39..e935e5a57b1b 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -1154,6 +1154,21 @@ export class HighlighterService extends PersistentStatefulService [1, 2, 3]) + const currentVersion = this.state.highlighterVersion; + return currentVersion > checkVersion; + } + setAiHighlighter(state: boolean) { this.SET_USE_AI_HIGHLIGHTER(state); this.usageStatisticsService.recordAnalyticsEvent('AIHighlighter', { From 2183f671743b62c9932a087146d7197e81b9f450 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 16 Apr 2025 10:42:27 +0200 Subject: [PATCH 24/34] text changes --- app/components-react/highlighter/ExportModal.tsx | 5 +++-- app/i18n/en-US/highlighter.json | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index ed559b67a1bc..e72a23f1ca7f 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -374,7 +374,7 @@ function ExportFlow({

- {$t('Transcribing...')} + {$t('Generating subtitles...')}

) : ( @@ -629,6 +629,7 @@ function SubtitleDropdownWrapper({ className={`${styles.innerDropdownItem} ${ item.name === currentSetting.name ? styles.active : '' }`} + style={{ display: 'flex', justifyContent: 'center' }} onClick={() => { setSetting(item); emitSettings(item); @@ -637,7 +638,7 @@ function SubtitleDropdownWrapper({ key={item.name} > {item.enabled === false ? ( -
{item.name}
+
{item.name}
) : ( item.style && )} diff --git a/app/i18n/en-US/highlighter.json b/app/i18n/en-US/highlighter.json index 80c5f92ef73b..91d8d6ec9e41 100644 --- a/app/i18n/en-US/highlighter.json +++ b/app/i18n/en-US/highlighter.json @@ -52,7 +52,7 @@ "Rendering Frames: %{currentFrame}/%{totalFrames}": "Rendering Frames: %{currentFrame}/%{totalFrames}", "Mixing Audio:": "Mixing Audio:", "Canceling...": "Canceling...", - "Transcribing...": "Transcribing...", + "Generating subtitles...": "Generating subtitles...", "Render Preview": "Render Preview", "The render preview shows a low-quality preview of the final rendered video. The final exported video will be higher resolution, framerate, and quality.": "The render preview shows a low-quality preview of the final rendered video. The final exported video will be higher resolution, framerate, and quality.", "Title": "Title", From 6ff0aafa04ca0296a0e90c94cc12afb53b66f6a8 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 23 Apr 2025 15:15:26 +0200 Subject: [PATCH 25/34] cleanup --- app/services/highlighter/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index cd10efec0597..f4bdb36e30ab 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -1189,7 +1189,6 @@ export class HighlighterService extends PersistentStatefulService [1, 2, 3]) const currentVersion = this.state.highlighterVersion; return currentVersion > checkVersion; } From 175074ade222b571b652d31ebfde59e1ff9229ab Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Tue, 13 May 2025 11:34:25 +0200 Subject: [PATCH 26/34] merge fix --- app/components-react/highlighter/Export/ExportModal.tsx | 9 --------- app/services/highlighter/index.ts | 1 - 2 files changed, 10 deletions(-) diff --git a/app/components-react/highlighter/Export/ExportModal.tsx b/app/components-react/highlighter/Export/ExportModal.tsx index 3718ad79ba05..085b9947d683 100644 --- a/app/components-react/highlighter/Export/ExportModal.tsx +++ b/app/components-react/highlighter/Export/ExportModal.tsx @@ -279,15 +279,6 @@ function ExportFlow({ findSubtitleItem(getSubtitleStyle()) || subtitleItems[0], ); - const [currentSetting, setSetting] = useState( - settingMatcher({ - name: 'from default', - fps: exportInfo.fps, - resolution: exportInfo.resolution, - preset: exportInfo.preset, - }), - ); - const [currentSetting, setSetting] = useState(null); const [isLoadingResolution, setIsLoadingResolution] = useState(true); diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index dad7e20456a1..83d17f662f86 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -77,7 +77,6 @@ import { cutHighlightClips, getVideoDuration } from './cut-highlight-clips'; import { reduce } from 'lodash'; import { extractDateTimeFromPath, fileExists } from './file-utils'; import { addVerticalFilterToExportOptions } from './vertical-export'; -import Utils from '../utils'; import { SubtitleStyles } from './subtitles/subtitle-styles'; import { isGameSupported } from './models/game-config.models'; import Utils from 'services/utils'; From 0b64fca0157723e9d75ca5ee67ea84f089b5ba93 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 21 May 2025 12:14:20 +0200 Subject: [PATCH 27/34] linting fix --- app/services/highlighter/ai-highlighter-updater.ts | 2 +- app/services/highlighter/models/rendering.models.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/highlighter/ai-highlighter-updater.ts b/app/services/highlighter/ai-highlighter-updater.ts index 98395be05469..2acd9e1b1c6d 100644 --- a/app/services/highlighter/ai-highlighter-updater.ts +++ b/app/services/highlighter/ai-highlighter-updater.ts @@ -135,7 +135,7 @@ export class AiHighlighterUpdater { } static startTranscription(videoUri: string, userId: string) { - return this.startHighlighterProcess(videoUri, userId, undefined, null, true); + return this.startHighlighterProcess(videoUri, userId, undefined, undefined, true); } /** diff --git a/app/services/highlighter/models/rendering.models.ts b/app/services/highlighter/models/rendering.models.ts index d70cc51afc0b..b88faf7aa716 100644 --- a/app/services/highlighter/models/rendering.models.ts +++ b/app/services/highlighter/models/rendering.models.ts @@ -17,7 +17,7 @@ export interface IExportOptions { height: number; preset: TPreset; complexFilter?: string; - subtitleStyle?: ISubtitleStyle; + subtitleStyle?: ISubtitleStyle | null; } // types for highlighter video operations From 6b2777466a2161c6f042c73e3aa864408349732d Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Mon, 26 May 2025 10:54:44 +0200 Subject: [PATCH 28/34] sharp windows only --- package.json | 6 +++--- yarn.lock | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index d2a808c9a3c1..87c2846e1ddb 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,6 @@ "realm": "12.7.1", "rimraf": "^2.6.1", "semver": "^5.5.1", - "sharp": "^0.33.5", "socket.io-client": "2.3.1", "tasklist": "^3.1.1", "uuid": "^3.0.1", @@ -243,7 +242,8 @@ "zustand": "^4.4.1" }, "optionalDependencies": { - "node-win32-np": "1.0.6" + "node-win32-np": "1.0.6", + "sharp": "^0.33.5" }, "resolutions": { "minimist": "1.2.6", @@ -252,4 +252,4 @@ "got@^9.6.0": "11.8.5" }, "packageManager": "yarn@3.1.1" -} \ No newline at end of file +} diff --git a/yarn.lock b/yarn.lock index 99182905eff4..bef7d35c425e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13605,6 +13605,8 @@ __metadata: dependenciesMeta: node-win32-np: optional: true + sharp: + optional: true languageName: unknown linkType: soft From 51df70498eee56506d3aea4a88596e11ba063331 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Tue, 3 Jun 2025 18:25:15 +0200 Subject: [PATCH 29/34] strange linebreak linting issue --- app/services/highlighter/subtitles/transcription.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/services/highlighter/subtitles/transcription.ts b/app/services/highlighter/subtitles/transcription.ts index f529324d684e..5d65388647f3 100644 --- a/app/services/highlighter/subtitles/transcription.ts +++ b/app/services/highlighter/subtitles/transcription.ts @@ -618,7 +618,6 @@ export class Transcription { ? this.singlePauseLength : roundTime(pauseTime - i * this.singlePauseLength); - pauseWords.push( new Word().initPauseWord( pauseStartTime, From e7b57464b93883eaa61ed7c4fc408bd6e8872bca Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 2 Jul 2025 13:40:30 +0200 Subject: [PATCH 30/34] update sharp library --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 87c2846e1ddb..9c565e8822bf 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Streamlabs streaming software", "author": "General Workings, Inc.", "license": "GPL-3.0", - "version": "1.18.3", + "version": "1.28.3", "main": "main.js", "scripts": { "compile": "yarn clear && yarn compile:updater && yarn webpack-cli --progress --config ./webpack.dev.config.js", @@ -243,7 +243,7 @@ }, "optionalDependencies": { "node-win32-np": "1.0.6", - "sharp": "^0.33.5" + "sharp": "^0.34.2" }, "resolutions": { "minimist": "1.2.6", From a89813a62e16544f373dbe0443ebce4bb66d4ac9 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 2 Jul 2025 13:41:07 +0200 Subject: [PATCH 31/34] dynamic import of sharp with error handling --- .../highlighter/rendering/render-subtitle.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts index a08b5a7be27a..ad95aa9545df 100644 --- a/app/services/highlighter/rendering/render-subtitle.ts +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -1,4 +1,3 @@ -import sharp from 'sharp'; import fs from 'fs-extra'; import { IResolution, SvgCreator } from '../subtitles/svg-creator'; import { getTranscription } from '../ai-highlighter-utils'; @@ -8,7 +7,18 @@ import path from 'path'; export const SUBTITLE_PER_SECOND = 3; +let sharp: any = null; + export async function svgToPng(svgText: string, resolution: IResolution, outputPath: string) { + try { + if (!sharp) { + // Import sharp dynamically to avoid issues with the main process + sharp = (await import('sharp')).default; + } + } catch (error: unknown) { + console.error('Error importing sharp:', error); + throw new Error('Sharp library is not available. Please ensure it is installed.'); + } try { const buffer = await sharp({ // Generate PNG with transparent background From fc490338e988f56bbfc2d38727311b02c9fc7c9f Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 2 Jul 2025 15:38:41 +0200 Subject: [PATCH 32/34] sharp update lock --- yarn.lock | 230 +++++++++++++++++++++++++++++------------------------- 1 file changed, 125 insertions(+), 105 deletions(-) diff --git a/yarn.lock b/yarn.lock index bef7d35c425e..9ee9b99dbd21 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1894,12 +1894,12 @@ __metadata: languageName: node linkType: hard -"@emnapi/runtime@npm:^1.2.0": - version: 1.3.1 - resolution: "@emnapi/runtime@npm:1.3.1" +"@emnapi/runtime@npm:^1.4.3": + version: 1.4.3 + resolution: "@emnapi/runtime@npm:1.4.3" dependencies: tslib: ^2.4.0 - checksum: 9a16ae7905a9c0e8956cf1854ef74e5087fbf36739abdba7aa6b308485aafdc993da07c19d7af104cd5f8e425121120852851bb3a0f78e2160e420a36d47f42f + checksum: ff2074809638ed878e476ece370c6eae7e6257bf029a581bb7a290488d8f2a08c420a65988c7f03bfc6bb689218f0cd995d2f935bd182150b357fc2341142f4f languageName: node linkType: hard @@ -1945,11 +1945,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-darwin-arm64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-darwin-arm64@npm:0.33.5" +"@img/sharp-darwin-arm64@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-darwin-arm64@npm:0.34.2" dependencies: - "@img/sharp-libvips-darwin-arm64": 1.0.4 + "@img/sharp-libvips-darwin-arm64": 1.1.0 dependenciesMeta: "@img/sharp-libvips-darwin-arm64": optional: true @@ -1957,11 +1957,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-darwin-x64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-darwin-x64@npm:0.33.5" +"@img/sharp-darwin-x64@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-darwin-x64@npm:0.34.2" dependencies: - "@img/sharp-libvips-darwin-x64": 1.0.4 + "@img/sharp-libvips-darwin-x64": 1.1.0 dependenciesMeta: "@img/sharp-libvips-darwin-x64": optional: true @@ -1969,67 +1969,74 @@ __metadata: languageName: node linkType: hard -"@img/sharp-libvips-darwin-arm64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-darwin-arm64@npm:1.0.4" +"@img/sharp-libvips-darwin-arm64@npm:1.1.0": + version: 1.1.0 + resolution: "@img/sharp-libvips-darwin-arm64@npm:1.1.0" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@img/sharp-libvips-darwin-x64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-darwin-x64@npm:1.0.4" +"@img/sharp-libvips-darwin-x64@npm:1.1.0": + version: 1.1.0 + resolution: "@img/sharp-libvips-darwin-x64@npm:1.1.0" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@img/sharp-libvips-linux-arm64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linux-arm64@npm:1.0.4" +"@img/sharp-libvips-linux-arm64@npm:1.1.0": + version: 1.1.0 + resolution: "@img/sharp-libvips-linux-arm64@npm:1.1.0" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@img/sharp-libvips-linux-arm@npm:1.0.5": - version: 1.0.5 - resolution: "@img/sharp-libvips-linux-arm@npm:1.0.5" +"@img/sharp-libvips-linux-arm@npm:1.1.0": + version: 1.1.0 + resolution: "@img/sharp-libvips-linux-arm@npm:1.1.0" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@img/sharp-libvips-linux-s390x@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linux-s390x@npm:1.0.4" +"@img/sharp-libvips-linux-ppc64@npm:1.1.0": + version: 1.1.0 + resolution: "@img/sharp-libvips-linux-ppc64@npm:1.1.0" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + +"@img/sharp-libvips-linux-s390x@npm:1.1.0": + version: 1.1.0 + resolution: "@img/sharp-libvips-linux-s390x@npm:1.1.0" conditions: os=linux & cpu=s390x languageName: node linkType: hard -"@img/sharp-libvips-linux-x64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linux-x64@npm:1.0.4" +"@img/sharp-libvips-linux-x64@npm:1.1.0": + version: 1.1.0 + resolution: "@img/sharp-libvips-linux-x64@npm:1.1.0" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@img/sharp-libvips-linuxmusl-arm64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linuxmusl-arm64@npm:1.0.4" +"@img/sharp-libvips-linuxmusl-arm64@npm:1.1.0": + version: 1.1.0 + resolution: "@img/sharp-libvips-linuxmusl-arm64@npm:1.1.0" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@img/sharp-libvips-linuxmusl-x64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linuxmusl-x64@npm:1.0.4" +"@img/sharp-libvips-linuxmusl-x64@npm:1.1.0": + version: 1.1.0 + resolution: "@img/sharp-libvips-linuxmusl-x64@npm:1.1.0" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@img/sharp-linux-arm64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linux-arm64@npm:0.33.5" +"@img/sharp-linux-arm64@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-linux-arm64@npm:0.34.2" dependencies: - "@img/sharp-libvips-linux-arm64": 1.0.4 + "@img/sharp-libvips-linux-arm64": 1.1.0 dependenciesMeta: "@img/sharp-libvips-linux-arm64": optional: true @@ -2037,11 +2044,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linux-arm@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linux-arm@npm:0.33.5" +"@img/sharp-linux-arm@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-linux-arm@npm:0.34.2" dependencies: - "@img/sharp-libvips-linux-arm": 1.0.5 + "@img/sharp-libvips-linux-arm": 1.1.0 dependenciesMeta: "@img/sharp-libvips-linux-arm": optional: true @@ -2049,11 +2056,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linux-s390x@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linux-s390x@npm:0.33.5" +"@img/sharp-linux-s390x@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-linux-s390x@npm:0.34.2" dependencies: - "@img/sharp-libvips-linux-s390x": 1.0.4 + "@img/sharp-libvips-linux-s390x": 1.1.0 dependenciesMeta: "@img/sharp-libvips-linux-s390x": optional: true @@ -2061,11 +2068,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linux-x64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linux-x64@npm:0.33.5" +"@img/sharp-linux-x64@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-linux-x64@npm:0.34.2" dependencies: - "@img/sharp-libvips-linux-x64": 1.0.4 + "@img/sharp-libvips-linux-x64": 1.1.0 dependenciesMeta: "@img/sharp-libvips-linux-x64": optional: true @@ -2073,11 +2080,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linuxmusl-arm64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linuxmusl-arm64@npm:0.33.5" +"@img/sharp-linuxmusl-arm64@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-linuxmusl-arm64@npm:0.34.2" dependencies: - "@img/sharp-libvips-linuxmusl-arm64": 1.0.4 + "@img/sharp-libvips-linuxmusl-arm64": 1.1.0 dependenciesMeta: "@img/sharp-libvips-linuxmusl-arm64": optional: true @@ -2085,11 +2092,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linuxmusl-x64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linuxmusl-x64@npm:0.33.5" +"@img/sharp-linuxmusl-x64@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-linuxmusl-x64@npm:0.34.2" dependencies: - "@img/sharp-libvips-linuxmusl-x64": 1.0.4 + "@img/sharp-libvips-linuxmusl-x64": 1.1.0 dependenciesMeta: "@img/sharp-libvips-linuxmusl-x64": optional: true @@ -2097,25 +2104,32 @@ __metadata: languageName: node linkType: hard -"@img/sharp-wasm32@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-wasm32@npm:0.33.5" +"@img/sharp-wasm32@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-wasm32@npm:0.34.2" dependencies: - "@emnapi/runtime": ^1.2.0 + "@emnapi/runtime": ^1.4.3 conditions: cpu=wasm32 languageName: node linkType: hard -"@img/sharp-win32-ia32@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-win32-ia32@npm:0.33.5" +"@img/sharp-win32-arm64@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-win32-arm64@npm:0.34.2" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@img/sharp-win32-ia32@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-win32-ia32@npm:0.34.2" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@img/sharp-win32-x64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-win32-x64@npm:0.33.5" +"@img/sharp-win32-x64@npm:0.34.2": + version: 0.34.2 + resolution: "@img/sharp-win32-x64@npm:0.34.2" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -6340,10 +6354,10 @@ __metadata: languageName: node linkType: hard -"detect-libc@npm:^2.0.3": - version: 2.0.3 - resolution: "detect-libc@npm:2.0.3" - checksum: 2ba6a939ae55f189aea996ac67afceb650413c7a34726ee92c40fb0deb2400d57ef94631a8a3f052055eea7efb0f99a9b5e6ce923415daa3e68221f963cfc27d +"detect-libc@npm:^2.0.4": + version: 2.0.4 + resolution: "detect-libc@npm:2.0.4" + checksum: 3d186b7d4e16965e10e21db596c78a4e131f9eee69c0081d13b85e6a61d7448d3ba23fe7997648022bdfa3b0eb4cc3c289a44c8188df949445a20852689abef6 languageName: node linkType: hard @@ -13112,12 +13126,12 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.6.3": - version: 7.7.1 - resolution: "semver@npm:7.7.1" +"semver@npm:^7.7.2": + version: 7.7.2 + resolution: "semver@npm:7.7.2" bin: semver: bin/semver.js - checksum: 586b825d36874007c9382d9e1ad8f93888d8670040add24a28e06a910aeebd673a2eb9e3bf169c6679d9245e66efb9057e0852e70d9daa6c27372aab1dda7104 + checksum: dd94ba8f1cbc903d8eeb4dd8bf19f46b3deb14262b6717d0de3c804b594058ae785ef2e4b46c5c3b58733c99c83339068203002f9e37cfe44f7e2cc5e3d2f621 languageName: node linkType: hard @@ -13203,32 +13217,34 @@ __metadata: languageName: node linkType: hard -"sharp@npm:^0.33.5": - version: 0.33.5 - resolution: "sharp@npm:0.33.5" - dependencies: - "@img/sharp-darwin-arm64": 0.33.5 - "@img/sharp-darwin-x64": 0.33.5 - "@img/sharp-libvips-darwin-arm64": 1.0.4 - "@img/sharp-libvips-darwin-x64": 1.0.4 - "@img/sharp-libvips-linux-arm": 1.0.5 - "@img/sharp-libvips-linux-arm64": 1.0.4 - "@img/sharp-libvips-linux-s390x": 1.0.4 - "@img/sharp-libvips-linux-x64": 1.0.4 - "@img/sharp-libvips-linuxmusl-arm64": 1.0.4 - "@img/sharp-libvips-linuxmusl-x64": 1.0.4 - "@img/sharp-linux-arm": 0.33.5 - "@img/sharp-linux-arm64": 0.33.5 - "@img/sharp-linux-s390x": 0.33.5 - "@img/sharp-linux-x64": 0.33.5 - "@img/sharp-linuxmusl-arm64": 0.33.5 - "@img/sharp-linuxmusl-x64": 0.33.5 - "@img/sharp-wasm32": 0.33.5 - "@img/sharp-win32-ia32": 0.33.5 - "@img/sharp-win32-x64": 0.33.5 +"sharp@npm:^0.34.2": + version: 0.34.2 + resolution: "sharp@npm:0.34.2" + dependencies: + "@img/sharp-darwin-arm64": 0.34.2 + "@img/sharp-darwin-x64": 0.34.2 + "@img/sharp-libvips-darwin-arm64": 1.1.0 + "@img/sharp-libvips-darwin-x64": 1.1.0 + "@img/sharp-libvips-linux-arm": 1.1.0 + "@img/sharp-libvips-linux-arm64": 1.1.0 + "@img/sharp-libvips-linux-ppc64": 1.1.0 + "@img/sharp-libvips-linux-s390x": 1.1.0 + "@img/sharp-libvips-linux-x64": 1.1.0 + "@img/sharp-libvips-linuxmusl-arm64": 1.1.0 + "@img/sharp-libvips-linuxmusl-x64": 1.1.0 + "@img/sharp-linux-arm": 0.34.2 + "@img/sharp-linux-arm64": 0.34.2 + "@img/sharp-linux-s390x": 0.34.2 + "@img/sharp-linux-x64": 0.34.2 + "@img/sharp-linuxmusl-arm64": 0.34.2 + "@img/sharp-linuxmusl-x64": 0.34.2 + "@img/sharp-wasm32": 0.34.2 + "@img/sharp-win32-arm64": 0.34.2 + "@img/sharp-win32-ia32": 0.34.2 + "@img/sharp-win32-x64": 0.34.2 color: ^4.2.3 - detect-libc: ^2.0.3 - semver: ^7.6.3 + detect-libc: ^2.0.4 + semver: ^7.7.2 dependenciesMeta: "@img/sharp-darwin-arm64": optional: true @@ -13242,6 +13258,8 @@ __metadata: optional: true "@img/sharp-libvips-linux-arm64": optional: true + "@img/sharp-libvips-linux-ppc64": + optional: true "@img/sharp-libvips-linux-s390x": optional: true "@img/sharp-libvips-linux-x64": @@ -13264,11 +13282,13 @@ __metadata: optional: true "@img/sharp-wasm32": optional: true + "@img/sharp-win32-arm64": + optional: true "@img/sharp-win32-ia32": optional: true "@img/sharp-win32-x64": optional: true - checksum: 04beae89910ac65c5f145f88de162e8466bec67705f497ace128de849c24d168993e016f33a343a1f3c30b25d2a90c3e62b017a9a0d25452371556f6cd2471e4 + checksum: beb34afe75cc6492fc7e6331efebfa11a0f92bf0f54ac850bf4c93ab48ab4152103cf096a892802bacca7c8102b721312b098bfdda16a4bf6c95716dabb28a16 languageName: node linkType: hard @@ -13552,7 +13572,7 @@ __metadata: rxjs: 6.3.3 semver: ^5.5.1 serve-handler: 5.0.7 - sharp: ^0.33.5 + sharp: ^0.34.2 shelljs: 0.8.5 signtool: 1.0.0 sl-vue-tree: "https://github.com/streamlabs/sl-vue-tree.git" From d2c4b726bf64464c2ecd9765c7070bfc54e459b5 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 2 Jul 2025 15:39:48 +0200 Subject: [PATCH 33/34] remove staging filter --- app/components-react/highlighter/Export/ExportModal.tsx | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/components-react/highlighter/Export/ExportModal.tsx b/app/components-react/highlighter/Export/ExportModal.tsx index f051ef35fc3a..de64637f981f 100644 --- a/app/components-react/highlighter/Export/ExportModal.tsx +++ b/app/components-react/highlighter/Export/ExportModal.tsx @@ -253,12 +253,7 @@ function ExportFlow({ }; }, [streamId]); - const showSubtitleSettings = useMemo( - () => - ['staging', 'local'].includes(Utils.getHighlighterEnvironment()) && - isHighlighterAfterVersion('0.0.53'), - [], - ); + const showSubtitleSettings = useMemo(() => isHighlighterAfterVersion('0.0.53'), []); function settingMatcher(initialSetting: TSetting) { const matchingSetting = settings.find( From 345b609103a982c479ce87c6f0c9afb74731a957 Mon Sep 17 00:00:00 2001 From: jankalthoefer Date: Wed, 2 Jul 2025 18:33:12 +0200 Subject: [PATCH 34/34] throw error if png creation failed --- app/services/highlighter/rendering/render-subtitle.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts index ad95aa9545df..c65da9c7b980 100644 --- a/app/services/highlighter/rendering/render-subtitle.ts +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -42,6 +42,7 @@ export async function svgToPng(svgText: string, resolution: IResolution, outputP await fs.writeFile(outputPath, buffer); } catch (error: unknown) { console.error('Error creating PNG from SVG', error); + throw new Error('Failed to create PNG from SVG'); } }