diff --git a/app/components-react/highlighter/Export/ExportModal.m.less b/app/components-react/highlighter/Export/ExportModal.m.less index f40d6ce68c14..72aabc621d3b 100644 --- a/app/components-react/highlighter/Export/ExportModal.m.less +++ b/app/components-react/highlighter/Export/ExportModal.m.less @@ -140,6 +140,15 @@ height: 100%; } +.subtitle-preview { + position: absolute; + bottom: 0; + left: 0; + right: 0; + display: flex; + justify-content: center; +} + .clip-info-wrapper { display: flex; justify-content: space-between; @@ -205,7 +214,8 @@ .inner-dropdown-item { color: white; width: 100%; - height: 32px; + height: 100%; + min-height: 32px; border-radius: 4px; background-color: #232d35; display: flex; diff --git a/app/components-react/highlighter/Export/ExportModal.tsx b/app/components-react/highlighter/Export/ExportModal.tsx index 8a9720adb73c..de64637f981f 100644 --- a/app/components-react/highlighter/Export/ExportModal.tsx +++ b/app/components-react/highlighter/Export/ExportModal.tsx @@ -4,6 +4,7 @@ import { TFPS, TResolution, TPreset, + ISubtitleStyle, } from 'services/highlighter/models/rendering.models'; import { Services } from 'components-react/service-provider'; import { FileInput, TextInput, ListInput } from 'components-react/shared/inputs'; @@ -25,15 +26,53 @@ import { getCombinedClipsDuration } from '../utils'; import { formatSecondsToHMS } from '../ClipPreview'; import PlatformSelect from './Platform'; import cx from 'classnames'; +import { SubtitleStyles } from 'services/highlighter/subtitles/subtitle-styles'; +import Utils from 'services/utils'; +import { isDeepEqual } from 'slap'; import { getVideoResolution } from 'services/highlighter/cut-highlight-clips'; type TSetting = { name: string; fps: TFPS; resolution: TResolution; preset: TPreset }; + +interface ISubtitleItem { + name: string; + enabled: boolean; + style?: ISubtitleStyle; +} const settings: TSetting[] = [ { name: 'Standard', fps: 30, resolution: 1080, preset: 'medium' }, { name: 'Best', fps: 60, resolution: 1080, preset: 'slow' }, { name: 'Fast', fps: 30, resolution: 720, preset: 'fast' }, { name: 'Custom', fps: 30, resolution: 720, preset: 'medium' }, ]; + +const subtitleItems: ISubtitleItem[] = [ + { + name: 'No subtitles', + enabled: false, + style: undefined, + }, + { + name: 'Basic', + enabled: true, + style: SubtitleStyles.basic, + }, + { + name: 'Thick', + enabled: true, + style: SubtitleStyles.thick, + }, + { + name: 'FlashyA', + enabled: true, + style: SubtitleStyles.flashyA, + }, + { + name: 'FlashyB', + enabled: true, + style: SubtitleStyles.yellow, + }, +]; + class ExportController { get service() { return Services.HighlighterService; @@ -86,6 +125,14 @@ class ExportController { this.service.actions.setPreset(value as TPreset); } + setSubtitles(subtitleItem: ISubtitleItem) { + this.service.actions.setSubtitleStyle(subtitleItem.style); + } + + getSubtitleStyle() { + return this.service.views.exportInfo.subtitleStyle; + } + setExport(exportFile: string) { this.service.actions.setExportFile(exportFile); } @@ -108,6 +155,9 @@ class ExportController { async fileExists(exportFile: string) { return await fileExists(exportFile); } + isHighlighterAfterVersion(version: string) { + return this.service.isHighlighterVersionAfter(version); + } } export const ExportModalCtx = React.createContext(null); @@ -145,6 +195,7 @@ function ExportModal({ close, streamId }: { close: () => void; streamId: string return ( void; streamId: string function ExportFlow({ close, isExporting, + isTranscribing, streamId, videoName, onVideoNameChange, }: { close: () => void; isExporting: boolean; + isTranscribing: boolean; streamId: string | undefined; videoName: string; onVideoNameChange: (name: string) => void; @@ -176,12 +229,15 @@ function ExportFlow({ setResolution, setFps, setPreset, + setSubtitles, + getSubtitleStyle, fileExists, setExport, exportCurrentFile, getStreamTitle, getClips, getDuration, + isHighlighterAfterVersion, getClipResolution, } = useController(ExportModalCtx); @@ -197,6 +253,8 @@ function ExportFlow({ }; }, [streamId]); + const showSubtitleSettings = useMemo(() => isHighlighterAfterVersion('0.0.53'), []); + function settingMatcher(initialSetting: TSetting) { const matchingSetting = settings.find( setting => @@ -215,6 +273,10 @@ function ExportFlow({ }; } + const [currentSubtitleItem, setSubtitleItem] = useState( + findSubtitleItem(getSubtitleStyle()) || subtitleItems[0], + ); + const [currentSetting, setSetting] = useState(null); const [isLoadingResolution, setIsLoadingResolution] = useState(true); @@ -262,6 +324,15 @@ function ExportFlow({ // Video name and export file are kept in sync const [exportFile, setExportFile] = useState(getExportFileFromVideoName(videoName)); + function findSubtitleItem(subtitleStyle: ISubtitleStyle | null) { + if (!subtitleStyle) return subtitleItems[0]; + + for (const item of subtitleItems) { + const isMatching = isDeepEqual(item.style, subtitleStyle, 0, 2); + if (isMatching) return item; + } + } + function getExportFileFromVideoName(videoName: string) { const parsed = path.parse(exportInfo.file); const sanitized = videoName.replace(/[/\\?%*:|"<>\.,;=#]/g, ''); @@ -359,24 +430,43 @@ function ExportFlow({ : { aspectRatio: '9/16' } } > - {isExporting && ( -
-

- {Math.round((exportInfo.currentFrame / exportInfo.totalFrames) * 100) || 0}% -

-

- {exportInfo.cancelRequested ? ( - {$t('Canceling...')} - ) : ( - {$t('Exporting video...')} - )} -

- +
+ +
+

+ {$t('Generating subtitles...')} +

+
+ ) : ( +
+

+ {Math.round((exportInfo.currentFrame / exportInfo.totalFrames) * 100) || 0}% +

+

+ {exportInfo.cancelRequested ? ( + {$t('Canceling...')} + ) : ( + {$t('Exporting video...')} + )} +

+ +
+ ))} + {currentSubtitleItem?.style && ( +
+
)} @@ -406,11 +496,24 @@ function ExportFlow({ {duration} | {$t('%{clipsAmount} clips', { clipsAmount: amount })}

- setCurrentFormat(format)} - /> +
+ {showSubtitleSettings && ( + { + setSubtitleItem(setting); + setSubtitles(setting); + }} + /> + )} + + setCurrentFormat(format)} + /> +
void; +}) { + const [isOpen, setIsOpen] = useState(false); + const [currentSetting, setSetting] = useState(initialItem); + + return ( +
+ + {subtitleItems.map(item => { + return ( +
{ + setSetting(item); + emitSettings(item); + setIsOpen(false); + }} + key={item.name} + > + {item.enabled === false ? ( +
{item.name}
+ ) : ( + item.style && + )} +
+ ); + })} +
+ } + trigger={['click']} + visible={isOpen} + onVisibleChange={setIsOpen} + placement="bottomCenter" + > +
setIsOpen(!isOpen)} + > +
+ +
+ +
+ +
+ ); +} + function CustomDropdownWrapper({ initialSetting, disabled, @@ -579,7 +746,7 @@ function CustomDropdownWrapper({ )} - + @@ -622,3 +789,118 @@ function OrientationToggle({ ); } + +function SubtitlePreview({ + svgStyle, + inPreview, + orientation, +}: { + svgStyle: ISubtitleStyle; + inPreview?: boolean; + orientation?: TOrientation; +}) { + const WIDTH = 250; + const HEIGHT = 60; + const formattedFontSize = orientation === 'horizontal' ? 22 : 14; + const fontSize = inPreview ? formattedFontSize : 24; + + return ( +
+ + {/* */} + + Auto subtitles + + +
+ ); +} + +export const SubtitleIcon = () => ( + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +); diff --git a/app/i18n/en-US/highlighter.json b/app/i18n/en-US/highlighter.json index e68a1048f7b0..f381c4f3dca9 100644 --- a/app/i18n/en-US/highlighter.json +++ b/app/i18n/en-US/highlighter.json @@ -52,6 +52,7 @@ "Rendering Frames: %{currentFrame}/%{totalFrames}": "Rendering Frames: %{currentFrame}/%{totalFrames}", "Mixing Audio:": "Mixing Audio:", "Canceling...": "Canceling...", + "Generating subtitles...": "Generating subtitles...", "Render Preview": "Render Preview", "The render preview shows a low-quality preview of the final rendered video. The final exported video will be higher resolution, framerate, and quality.": "The render preview shows a low-quality preview of the final rendered video. The final exported video will be higher resolution, framerate, and quality.", "Title": "Title", @@ -210,4 +211,4 @@ "Export Markers": "Export Markers", "Timeline starts from 01:00:00 (default)": "Timeline starts from 01:00:00 (default)", "Export full highlight duration as marker range": "Export full highlight duration as marker range" -} \ No newline at end of file +} diff --git a/app/services/highlighter/ai-highlighter-updater.ts b/app/services/highlighter/ai-highlighter-updater.ts index 5ae490164201..f9e115c722a9 100644 --- a/app/services/highlighter/ai-highlighter-updater.ts +++ b/app/services/highlighter/ai-highlighter-updater.ts @@ -59,6 +59,7 @@ export class AiHighlighterUpdater { userId: string, milestonesPath?: string, game?: string, + onlyTranscription = false, ) { const runHighlighterFromRepository = Utils.getHighlighterEnvironment() === 'local'; @@ -70,6 +71,7 @@ export class AiHighlighterUpdater { userId, milestonesPath, game, + onlyTranscription, ); } @@ -90,7 +92,9 @@ export class AiHighlighterUpdater { } command.push('--use_sentry'); command.push('--user_id', userId); - + if (onlyTranscription) { + command.push('--only_transcription'); + } return spawn(highlighterBinaryPath, command); } @@ -99,6 +103,7 @@ export class AiHighlighterUpdater { userId: string, milestonesPath?: string, game?: string, + onlyTranscription = false, ) { const rootPath = '../highlighter-api/'; const command = [ @@ -118,6 +123,9 @@ export class AiHighlighterUpdater { command.push('--milestones_file'); command.push(milestonesPath); } + if (onlyTranscription) { + command.push('--only_transcription'); + } if (game) { command.push('--game'); @@ -129,6 +137,10 @@ export class AiHighlighterUpdater { }); } + static startTranscription(videoUri: string, userId: string) { + return this.startHighlighterProcess(videoUri, userId, undefined, undefined, true); + } + /** * Check if an update is currently in progress */ diff --git a/app/services/highlighter/ai-highlighter-utils.ts b/app/services/highlighter/ai-highlighter-utils.ts index 6400a44c9814..4ccdf40171bd 100644 --- a/app/services/highlighter/ai-highlighter-utils.ts +++ b/app/services/highlighter/ai-highlighter-utils.ts @@ -9,7 +9,10 @@ import { IHighlighterMessage, IHighlighterMilestone, IHighlighterProgressMessage, + IHighlighterTranscriptionMessage, } from './models/ai-highlighter.models'; +import { Word } from './subtitles/word'; +import { Transcription } from './subtitles/transcription'; const START_TOKEN = '>>>>'; const END_TOKEN = '<<<<'; @@ -119,9 +122,7 @@ export function getHighlightClips( milestoneUpdate?.(aiHighlighterMessage.json as IHighlighterMilestone); break; default: - // console.log('\n\n'); - // console.log('Unrecognized message type:', aiHighlighterMessage); - // console.log('\n\n'); + // ('Unrecognized message type:', aiHighlighterMessage); break; } } @@ -147,7 +148,6 @@ export function getHighlightClips( }); }); } - function parseAiHighlighterMessage(messageString: string): IHighlighterMessage | string | null { try { if (messageString.includes(START_TOKEN) && messageString.includes(END_TOKEN)) { @@ -225,3 +225,91 @@ export class ProgressTracker { return interval; } } + +export function getTranscription( + videoUri: string, + userId: string, + totalDuration: number, + cancelSignal?: AbortSignal, + progressUpdate?: (progress: number) => void, +): Promise { + return new Promise((resolve, reject) => { + const childProcess: child.ChildProcess = AiHighlighterUpdater.startTranscription( + videoUri, + userId, + ); + const messageBuffer = new MessageBufferHandler(); + + if (cancelSignal) { + cancelSignal.addEventListener('abort', () => { + console.log('ending transcription process'); + messageBuffer.clear(); + kill(childProcess.pid!, 'SIGINT'); + reject(new Error('Highlight generation canceled')); + }); + } + + childProcess.stdout?.on('data', (data: Buffer) => { + const message = data.toString(); + messageBuffer.appendToBuffer(message); + + // Try to extract a complete message + const completeMessages = messageBuffer.extractCompleteMessages(); + + for (const completeMessage of completeMessages) { + // messageBuffer.clear(); + const aiHighlighterMessage = parseAiHighlighterMessage(completeMessage); + if (typeof aiHighlighterMessage === 'string' || aiHighlighterMessage instanceof String) { + console.log('message type of string', aiHighlighterMessage); + } else if (aiHighlighterMessage) { + switch (aiHighlighterMessage.type) { + case 'progress': + progressUpdate?.((aiHighlighterMessage.json as IHighlighterProgressMessage).progress); + break; + case 'transcription': { + const highlighterWords = (aiHighlighterMessage.json as IHighlighterTranscriptionMessage) + .words; + + const words = highlighterWords.map(highlighterWord => + new Word().fromTranscriptionService( + highlighterWord.text, + highlighterWord.start_time, + highlighterWord.end_time, + 0, + highlighterWord.probability, + ), + ); + const transcription = new Transcription(); + transcription.words = words; + transcription.generatePauses(totalDuration); + resolve(transcription); + break; + } + + default: + // ('Unrecognized message type:', aiHighlighterMessage); + break; + } + } + } + }); + + childProcess.stderr?.on('data', (data: Buffer) => { + console.log('Debug logs:', data.toString()); + }); + + childProcess.on('error', error => { + messageBuffer.clear(); + reject(new Error(`Child process threw an error. Error message: ${error.message}.`)); + }); + + childProcess.on('exit', (code, signal) => { + messageBuffer.clear(); + reject({ + message: `Child process exited with code ${code} and signal ${signal}`, + signal, + code, + }); + }); + }); +} diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index c7e5d3140ea6..861eb195b31b 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -53,6 +53,7 @@ import { IAudioInfo, IExportInfo, IExportOptions, + ISubtitleStyle, ITransitionInfo, IVideoInfo, TFPS, @@ -76,6 +77,7 @@ import { cutHighlightClips, getVideoDuration } from './cut-highlight-clips'; import { reduce } from 'lodash'; import { extractDateTimeFromPath, fileExists } from './file-utils'; import { addVerticalFilterToExportOptions } from './vertical-export'; +import { SubtitleStyles } from './subtitles/subtitle-styles'; import { isGameSupported } from './models/game-config.models'; import Utils from 'services/utils'; import { getOS, OS } from '../../util/operating-systems'; @@ -111,6 +113,7 @@ export class HighlighterService extends PersistentStatefulService checkVersion; + } + setAiHighlighter(state: boolean) { this.SET_USE_AI_HIGHLIGHTER(state); this.usageStatisticsService.recordAnalyticsEvent('AIHighlighter', { @@ -1299,7 +1324,7 @@ export class HighlighterService extends PersistentStatefulService { diff --git a/app/services/highlighter/models/ai-highlighter.models.ts b/app/services/highlighter/models/ai-highlighter.models.ts index 0f90f8633ce2..201132c821f6 100644 --- a/app/services/highlighter/models/ai-highlighter.models.ts +++ b/app/services/highlighter/models/ai-highlighter.models.ts @@ -126,14 +126,20 @@ export interface IHighlighterInput { origin: string; metadata?: IDeathMetadata | any; } - +export interface IHighlighterWord { + start_time: number; + end_time: number; + text: string; + probability: number; +} // Message export type EHighlighterMessageTypes = | 'progress' | 'inputs' | 'inputs_partial' | 'highlights' - | 'milestone'; + | 'milestone' + | 'transcription'; export interface IHighlighterMessage { type: EHighlighterMessageTypes; @@ -143,6 +149,9 @@ export interface IHighlighterMessage { export interface IHighlighterProgressMessage { progress: number; } +export interface IHighlighterTranscriptionMessage { + words: IHighlighterWord[]; +} export interface IHighlighterMilestone { name: string; diff --git a/app/services/highlighter/models/rendering.models.ts b/app/services/highlighter/models/rendering.models.ts index c343b3b5155c..46b22883c37d 100644 --- a/app/services/highlighter/models/rendering.models.ts +++ b/app/services/highlighter/models/rendering.models.ts @@ -1,9 +1,11 @@ import { $t } from '../../i18n'; +import { ITextStyle } from '../subtitles/svg-creator'; export type TFPS = 30 | 60; export type TResolution = 720 | 1080; export type TPreset = 'ultrafast' | 'fast' | 'slow' | 'medium'; +export interface ISubtitleStyle extends ITextStyle {} export interface IResolution { width: number; height: number; @@ -15,6 +17,7 @@ export interface IExportOptions { height: number; preset: TPreset; complexFilter?: string; + subtitleStyle?: ISubtitleStyle | null; } // types for highlighter video operations @@ -31,7 +34,7 @@ export interface IExportInfo { cancelRequested: boolean; file: string; previewFile: string; - + transcriptionInProgress: boolean; /** * Whether the export finished successfully. * Will be set to false whenever something changes @@ -42,6 +45,7 @@ export interface IExportInfo { fps: TFPS; resolution: TResolution; preset: TPreset; + subtitleStyle: ISubtitleStyle | null; } // Capitalization is not consistent because it matches with the diff --git a/app/services/highlighter/rendering/frame-writer.ts b/app/services/highlighter/rendering/frame-writer.ts index ab49d64dd416..1b7f0713a1d2 100644 --- a/app/services/highlighter/rendering/frame-writer.ts +++ b/app/services/highlighter/rendering/frame-writer.ts @@ -2,6 +2,9 @@ import execa from 'execa'; import { IExportOptions } from '../models/rendering.models'; import { FADE_OUT_DURATION, FFMPEG_EXE } from '../constants'; import { FrameWriteError } from './errors'; +import fs from 'fs-extra'; +import path from 'path'; +import { SUBTITLE_PER_SECOND } from './render-subtitle'; export class FrameWriter { constructor( @@ -9,13 +12,14 @@ export class FrameWriter { public readonly audioInput: string, public readonly duration: number, public readonly options: IExportOptions, + public readonly subtitleDirectory: string | null, ) {} private ffmpeg: execa.ExecaChildProcess; exitPromise: Promise; - private startFfmpeg() { + private async startFfmpeg() { /* eslint-disable */ const args = [ // Video Input @@ -33,24 +37,16 @@ export class FrameWriter { '-', // Audio Input - '-i', - this.audioInput, // Input Mapping - '-map', - '0:v:0', - '-map', - '1:a:0', - - // Filters - '-af', - `afade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( - this.duration - (FADE_OUT_DURATION + 0.2), - 0, - )}`, + // '-map', + // '0:v:0', ]; - - this.addVideoFilters(args); + if (this.options.subtitleStyle && this.subtitleDirectory) { + await this.addSubtitleInput(args, this.subtitleDirectory); + } + this.addAudioFilters(args, !!this.options.subtitleStyle); + this.addVideoFilters(args, !!this.options.subtitleStyle); const crf = this.options.preset === 'slow' ? '18' : '21'; @@ -103,21 +99,60 @@ export class FrameWriter { console.log('ffmpeg:', data.toString()); }); } + private addVideoFilters(args: string[], subtitlesEnabled = false) { + const webcamEnabled = !!this.options.complexFilter; + + const firstInput = webcamEnabled ? '[final]' : '[0:v]'; + const output = subtitlesEnabled ? '[subtitled]' : '[final]'; - private addVideoFilters(args: string[]) { const fadeFilter = `format=yuv420p,fade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( this.duration - (FADE_OUT_DURATION + 0.2), 0, )}`; - if (this.options.complexFilter) { - args.push('-vf', this.options.complexFilter + `[final]${fadeFilter}`); - } else { - args.push('-vf', fadeFilter); + args.push('-filter_complex'); + + if (!webcamEnabled && !subtitlesEnabled) { + args.push(fadeFilter); + return; + } + + let combinedFilter = ''; + if (webcamEnabled) { + combinedFilter += this.options.complexFilter; + } + + if (subtitlesEnabled) { + combinedFilter += `${firstInput}[1:v]overlay=0:0[subtitled];`; } + + combinedFilter += output + fadeFilter; + args.push(combinedFilter); + } + + private addAudioFilters(args: string[], subtitlesEnabled = false) { + args.push( + '-i', + this.audioInput, + '-map', + subtitlesEnabled ? '2:a:0' : '1:a:0', + '-af', + `afade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( + this.duration - (FADE_OUT_DURATION + 0.2), + 0, + )}`, + ); + } + private async addSubtitleInput(args: string[], subtitleDirectory: string) { + args.push( + '-framerate', + String(SUBTITLE_PER_SECOND), + '-i', + `${subtitleDirectory}\\subtitles_%04d.png`, + ); } async writeNextFrame(frameBuffer: Buffer) { - if (!this.ffmpeg) this.startFfmpeg(); + if (!this.ffmpeg) await this.startFfmpeg(); try { await new Promise((resolve, reject) => { diff --git a/app/services/highlighter/rendering/render-subtitle.ts b/app/services/highlighter/rendering/render-subtitle.ts new file mode 100644 index 000000000000..c65da9c7b980 --- /dev/null +++ b/app/services/highlighter/rendering/render-subtitle.ts @@ -0,0 +1,115 @@ +import fs from 'fs-extra'; +import { IResolution, SvgCreator } from '../subtitles/svg-creator'; +import { getTranscription } from '../ai-highlighter-utils'; +import { SubtitleMode } from '../subtitles/subtitle-mode'; +import { IExportOptions } from '../models/rendering.models'; +import path from 'path'; + +export const SUBTITLE_PER_SECOND = 3; + +let sharp: any = null; + +export async function svgToPng(svgText: string, resolution: IResolution, outputPath: string) { + try { + if (!sharp) { + // Import sharp dynamically to avoid issues with the main process + sharp = (await import('sharp')).default; + } + } catch (error: unknown) { + console.error('Error importing sharp:', error); + throw new Error('Sharp library is not available. Please ensure it is installed.'); + } + try { + const buffer = await sharp({ + // Generate PNG with transparent background + create: { + width: resolution.width, + height: resolution.height, + channels: 4, + background: { r: 0, g: 0, b: 0, alpha: 0 }, + }, + }) + .composite([ + { + input: Buffer.from(svgText), + top: 0, + left: 0, + }, + ]) + .png() + .toBuffer(); + + await fs.writeFile(outputPath, buffer); + } catch (error: unknown) { + console.error('Error creating PNG from SVG', error); + throw new Error('Failed to create PNG from SVG'); + } +} + +export async function createSubtitles( + mediaPath: string, + userId: string, + parsed: path.ParsedPath, + exportOptions: IExportOptions, + totalDuration: number, + totalFrames: number, +) { + const subtitleDirectory = path.join(parsed.dir, 'temp_subtitles'); + + if (!fs.existsSync(subtitleDirectory)) { + fs.mkdirSync(subtitleDirectory, { recursive: true }); + } + + const exportResolution = exportOptions.complexFilter + ? { width: exportOptions.height, height: exportOptions.width } + : { width: exportOptions.width, height: exportOptions.height }; + const svgCreator = new SvgCreator(exportResolution, exportOptions.subtitleStyle); + + const transcription = await getTranscription(mediaPath, userId, totalDuration); + + const subtitleClips = transcription.generateSubtitleClips( + SubtitleMode.static, + exportOptions.width / exportOptions.height, + 20, + ); + // Create subtitles + let subtitleCounter = 0; + + const subtitleEveryNFrames = Math.floor(exportOptions.fps / SUBTITLE_PER_SECOND); + + const subtitlesToProcess = []; + for (let frameIndex = 0; frameIndex < totalFrames; frameIndex += subtitleEveryNFrames) { + // Find the appropriate subtitle clip for this frame + const timeInSeconds = frameIndex / exportOptions.fps; + const subtitleClip = subtitleClips.clips.find( + clip => timeInSeconds >= clip.startTimeInEdit && timeInSeconds <= clip.endTimeInEdit, + ); + + if (subtitleClip) { + subtitlesToProcess.push(subtitleClip.text); + } else { + subtitlesToProcess.push(''); + } + } + + for (const subtitleText of subtitlesToProcess) { + const svgString = svgCreator.getSvgWithText([subtitleText], 0); + const pngPath = path.join( + subtitleDirectory, + `/subtitles_${String(subtitleCounter).padStart(4, '0')}.png`, + ); + await svgToPng(svgString, exportResolution, pngPath); + subtitleCounter++; + } + return subtitleDirectory; +} + +export function cleanupSubtitleDirectory(directory: string) { + if (directory) { + try { + fs.removeSync(directory); + } catch (error: unknown) { + console.error('Failed to clean up subtitle directory', error); + } + } +} diff --git a/app/services/highlighter/rendering/start-rendering.ts b/app/services/highlighter/rendering/start-rendering.ts index 7c33b757095c..5d8d936ba0ce 100644 --- a/app/services/highlighter/rendering/start-rendering.ts +++ b/app/services/highlighter/rendering/start-rendering.ts @@ -18,7 +18,7 @@ import { $t } from '../../i18n'; import * as Sentry from '@sentry/browser'; import { sample } from 'lodash'; import { TAnalyticsEvent } from '../../usage-statistics'; - +import { cleanupSubtitleDirectory, createSubtitles, svgToPng } from './render-subtitle'; export interface IRenderingConfig { renderingClips: RenderingClip[]; isPreview: boolean; @@ -31,6 +31,7 @@ export interface IRenderingConfig { streamId: string | undefined; } export async function startRendering( + userId: string, renderingConfig: IRenderingConfig, handleFrame: (currentFrame: number) => void, setExportInfo: (partialExportInfo: Partial) => void, @@ -48,6 +49,7 @@ export async function startRendering( let fader: AudioCrossfader | null = null; let mixer: AudioMixer | null = null; + let subtitleDirectory: string | null = null; try { // Estimate the total number of frames to set up export info const totalFrames = renderingClips.reduce((count: number, clip) => { @@ -56,7 +58,7 @@ export async function startRendering( const numTransitions = renderingClips.length - 1; const transitionFrames = transitionDuration * exportOptions.fps; const totalFramesAfterTransitions = totalFrames - numTransitions * transitionFrames; - + const totalDuration = totalFramesAfterTransitions / exportOptions.fps; setExportInfo({ totalFrames: totalFramesAfterTransitions, }); @@ -73,6 +75,31 @@ export async function startRendering( fader = new AudioCrossfader(audioConcat, renderingClips, transitionDuration); await fader.export(); + // Create subtitles before audio is mixed in + if (exportOptions.subtitleStyle) { + try { + setExportInfo({ + transcriptionInProgress: true, + }); + subtitleDirectory = await createSubtitles( + audioConcat, + userId, + parsed, + exportOptions, + totalDuration, + totalFramesAfterTransitions, + ); + } catch (error: unknown) { + console.error('Error creating subtitles', error); + exportOptions.subtitleStyle = null; + } finally { + setExportInfo({ + transcriptionInProgress: false, + }); + } + } + // create transcriptions + if (audioInfo.musicEnabled && audioInfo.musicPath) { mixer = new AudioMixer(audioMix, [ { path: audioConcat, volume: 1, loop: false }, @@ -106,6 +133,7 @@ export async function startRendering( audioMix, totalFramesAfterTransitions / exportOptions.fps, exportOptions, + subtitleDirectory, ); while (true) { @@ -234,7 +262,8 @@ export async function startRendering( exporting: false, exported: !exportInfo.cancelRequested && !isPreview && !exportInfo.error, }); - + // Clean up subtitle directory if it was created + cleanupSubtitleDirectory(subtitleDirectory); if (fader) await fader.cleanup(); if (mixer) await mixer.cleanup(); } diff --git a/app/services/highlighter/subtitles/clip-element.ts b/app/services/highlighter/subtitles/clip-element.ts new file mode 100644 index 000000000000..66cf2b3d26e9 --- /dev/null +++ b/app/services/highlighter/subtitles/clip-element.ts @@ -0,0 +1,25 @@ +import { Transcription } from './transcription'; + +export abstract class ClipElement { + protected _transcriptionReference: Transcription; + public startIndex: number; + public endIndex: number; + public mediaIndex: number; + + public get startTimeInEdit(): number { + return this._transcriptionReference.getStartTimeInEditAtIndex(this.startIndex); + } + public get endTimeInEdit(): number { + return this._transcriptionReference.getEndTimeInEditAtIndex(this.endIndex); + } + public get startTimeInOriginal(): number { + return this._transcriptionReference.getStartTimeInOriginalAtIndex(this.startIndex); + } + public get endTimeInOriginal(): number { + return this._transcriptionReference.getEndTimeInOriginalAtIndex(this.endIndex); + } + + constructor(transcriptionReference: Transcription) { + this._transcriptionReference = transcriptionReference; + } +} diff --git a/app/services/highlighter/subtitles/subtitle-clip.ts b/app/services/highlighter/subtitles/subtitle-clip.ts new file mode 100644 index 000000000000..0448c751c3b5 --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-clip.ts @@ -0,0 +1,135 @@ +import { SubtitleMode } from './subtitle-mode'; +import { ClipElement } from './clip-element'; +import { Transcription } from './transcription'; +import { toSrtFormat, toVttFormat } from './subtitle-utils'; + +export class SubtitleClip extends ClipElement { + _transcriptionReference: Transcription; + + public get text(): string { + return this._transcriptionReference + .sliceWords(this.startIndex, this.endIndex + 1) + .filter(word => !word.isCut && !word.isPause) + .map(word => word.text) + .join(' '); + } + + constructor( + startIndex: number, + endIndex: number, + mediaIndex: number, + _transcriptionReference: Transcription, + ) { + super(_transcriptionReference); + this.startIndex = startIndex; + this.endIndex = endIndex; + this.mediaIndex = mediaIndex; + } + + public getDynamicSubtitleClips(): SubtitleClip[] { + const dynamicSubtitleClips: SubtitleClip[] = []; + for (let index = this.startIndex; index <= this.endIndex; index++) { + const word = this._transcriptionReference.words[index]; + if (!word.isPause) { + dynamicSubtitleClips.push( + new SubtitleClip(this.startIndex, index, this.mediaIndex, this._transcriptionReference), + ); + } else { + dynamicSubtitleClips.push( + new SubtitleClip(index, index, this.mediaIndex, this._transcriptionReference), + ); + } + } + + return dynamicSubtitleClips; + } + + // /** + // * Assembly Function for different file specification, subtitles + // */ + public assembleSrtString( + line: string, + startTime: number, + endTime: number, + index: number, + subtitleMode: SubtitleMode, + ) { + let srtStartTime = startTime; + let srtEndTime = endTime; + + let srtPreviousEndTime = + this._transcriptionReference.words[this.startIndex - 1]?.endTimeInEdit || 0; + + if (subtitleMode === SubtitleMode.dynamic) { + srtStartTime = this._transcriptionReference.words[this.endIndex].startTimeInEdit; + srtEndTime = this._transcriptionReference.words[this.endIndex].endTimeInEdit; + srtPreviousEndTime = this._transcriptionReference.words[this.endIndex - 1].endTimeInEdit; + } + let srtString = ''; + if (index > 0) { + if (srtPreviousEndTime > srtStartTime) { + const diff = srtPreviousEndTime - srtStartTime; + srtStartTime -= diff + 0.001; + } + } + if (srtStartTime === srtEndTime) { + return null; + } else if (srtStartTime > srtEndTime) { + const diff = srtEndTime - srtStartTime; + srtEndTime -= diff + 0.001; + } + + if (line) { + srtString += index + 1; + srtString += '\n'; // newLine + srtString += `${toSrtFormat(srtStartTime)} --> ${toSrtFormat(srtEndTime)}`; // Start and endtime + srtString += '\n'; // newLine + srtString += line; // actual Text + srtString += '\n'; // newLine + srtString += '\n'; // newLine + } + return srtString; + } + + assembleVttString( + line: string, + startTime: number, + endTime: number, + index: number, + subtitleMode: SubtitleMode, + ) { + let vttString = ''; + let vttStartTime = startTime; + let vttEndTime = endTime; + let vttPreviousEndTime = this._transcriptionReference.words[this.startIndex - 1]?.endTimeInEdit; + + if (subtitleMode === SubtitleMode.dynamic) { + vttStartTime = this._transcriptionReference.words[this.endIndex].startTimeInEdit; + vttEndTime = this._transcriptionReference.words[this.endIndex].endTimeInEdit; + vttPreviousEndTime = this._transcriptionReference.words[this.endIndex - 1]?.endTimeInEdit; + } + + if (index > 0) { + if (vttPreviousEndTime > vttStartTime) { + const diff = vttPreviousEndTime - vttStartTime; + vttStartTime -= diff + 0.001; + } + } + if (vttStartTime === vttEndTime) { + return null; + } else if (vttStartTime > vttEndTime) { + const diff = vttEndTime - vttStartTime; + vttEndTime -= diff + 0.001; + } + + if (line) { + vttString += '\n'; // newLine + vttString += `${toVttFormat(vttStartTime)} --> ${toVttFormat(vttEndTime)} align:middle`; // Start and endTime + vttString += '\n'; // newLine + vttString += line; // actual Text + vttString += '\n'; // newLine + vttString += '\n'; // newLine + } + return vttString; + } +} diff --git a/app/services/highlighter/subtitles/subtitle-clips.ts b/app/services/highlighter/subtitles/subtitle-clips.ts new file mode 100644 index 000000000000..bdb808908cb0 --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-clips.ts @@ -0,0 +1,112 @@ +import { SubtitleMode } from './subtitle-mode'; +import { SubtitleClip } from './subtitle-clip'; + +export class SubtitleClips { + private _subtitleClips: SubtitleClip[]; + + public get length(): number { + return this._subtitleClips.length; + } + public get clips(): SubtitleClip[] { + return this._subtitleClips; + } + + constructor() { + this._subtitleClips = []; + } + + public getClipByTime(time: number, mediaIndex: number) { + return this._subtitleClips.find( + subtitleClip => + time < Math.round(subtitleClip.endTimeInOriginal * 100) / 100 && + time >= subtitleClip.startTimeInOriginal && + mediaIndex === subtitleClip.mediaIndex, + ); + } + public push(subtitleClip: SubtitleClip) { + this._subtitleClips.push(subtitleClip); + } + public forEach(callback: (subtitleClip: SubtitleClip) => void) { + for (let index = 0; index < this._subtitleClips.length; index++) { + const subtitleClip = this._subtitleClips[index]; + callback(subtitleClip); + } + } + + public convertToDynamicSubtitles(): void { + const dynamicSubtitleClips: SubtitleClip[] = []; + this._subtitleClips.forEach((subtitleClip: SubtitleClip) => { + dynamicSubtitleClips.push(...subtitleClip.getDynamicSubtitleClips()); + }); + this._subtitleClips = dynamicSubtitleClips; + } + + public getSrtString(subtitleMode: SubtitleMode) { + let srtString = ''; + let indexCounter = 0; + for (let index = 0; index < this._subtitleClips.length; index++) { + const subtitleClip = this._subtitleClips[index]; + const previousSubtitleClip = this._subtitleClips[index - 1]; + + const newLine = subtitleClip.assembleSrtString( + subtitleClip.text, + subtitleClip.startTimeInEdit, + subtitleClip.endTimeInEdit, + indexCounter, + subtitleMode, + ); + if (newLine) { + srtString += newLine; + indexCounter += 1; + } + } + return srtString; + } + public getVttString(subtitleMode: SubtitleMode) { + let vttString = 'WEBVTT\n\n'; + let indexCounter = 0; + for (let index = 0; index < this._subtitleClips.length; index++) { + const subtitleClip = this._subtitleClips[index]; + const previousSubtitleClip = this._subtitleClips[index - 1]; + + const newLine = subtitleClip.assembleVttString( + subtitleClip.text, + subtitleClip.startTimeInEdit, + subtitleClip.endTimeInEdit, + index, + subtitleMode, + ); + if (newLine) { + vttString += newLine; + indexCounter += 1; + } + } + return vttString; + } + + /** + * Returns an array with every start and end index of a subtitle clip + */ + getStartEndArray(subtitleMode: SubtitleMode): { start: number; end: number }[] { + return this._subtitleClips + .map((subtitleClip, index) => { + if (subtitleMode === SubtitleMode.static) { + return { start: subtitleClip.startIndex, end: subtitleClip.endIndex }; + } else if (subtitleMode === SubtitleMode.dynamic) { + if ( + (this._subtitleClips[index + 1]?.startIndex === + this._subtitleClips[index + 1]?.endIndex && + this._subtitleClips[index + 1]?.text.length > 0) || + this._subtitleClips[index]._transcriptionReference.words[ + this._subtitleClips[index].endIndex + ].hasSubtitlebreak === true + ) { + return { start: subtitleClip.startIndex, end: subtitleClip.endIndex }; + } + return { start: null, end: null }; + } + return { start: null, end: null }; + }) + .filter(value => value.start !== null && value.end !== null); + } +} diff --git a/app/services/highlighter/subtitles/subtitle-mode.ts b/app/services/highlighter/subtitles/subtitle-mode.ts new file mode 100644 index 000000000000..a8dd1326a7da --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-mode.ts @@ -0,0 +1,14 @@ +export enum SubtitleMode { + disabled = 'disabled', + dynamic = 'dynamic', + static = 'static', +} +export type TSubtitleHighlightStyle = 'none' | 'background' | 'opacity'; + +export interface ISubtitleConfig { + subtitleMode?: SubtitleMode; + subtitleLength?: number; + + bgColor?: string; + bg?: boolean; +} diff --git a/app/services/highlighter/subtitles/subtitle-styles.ts b/app/services/highlighter/subtitles/subtitle-styles.ts new file mode 100644 index 000000000000..e5fe723511d0 --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-styles.ts @@ -0,0 +1,34 @@ +import { ISubtitleStyle } from '../models/rendering.models'; + +export type SubtitleStyleName = 'basic' | 'thick' | 'flashyA' | 'yellow'; + +export const SubtitleStyles: { [name in SubtitleStyleName]: ISubtitleStyle } = { + basic: { + fontColor: '#FFFFFF', + fontSize: 48, + fontFamily: 'Arial', + strokeColor: '#000000', + strokeWidth: 2, + }, + thick: { + fontFamily: 'Impact', + fontSize: 48, + fontColor: '#FFFFFF', + strokeColor: '#000000', + strokeWidth: 6, + }, + flashyA: { + fontColor: '#FFFFFF', + fontSize: 48, + fontFamily: 'Showcard Gothic', + strokeColor: '#000000', + strokeWidth: 6, + }, + yellow: { + fontColor: '#f6f154', + fontSize: 48, + fontFamily: 'Arial', + strokeColor: '#000000', + strokeWidth: 6, + }, +}; diff --git a/app/services/highlighter/subtitles/subtitle-utils.ts b/app/services/highlighter/subtitles/subtitle-utils.ts new file mode 100644 index 000000000000..f44dc752cd45 --- /dev/null +++ b/app/services/highlighter/subtitles/subtitle-utils.ts @@ -0,0 +1,20 @@ +export function removeLinebreaksFromString(str: string) { + return str.replace(/[\r\n]+/gm, ''); +} +export function roundTime(time: number): number { + return Math.round(time * 100) / 100; +} +export function toSrtFormat(time: number): string { + const o = new Date(0); + const p = new Date(time * 1000); + return new Date(p.getTime() - o.getTime()) + .toISOString() + .split('T')[1] + .split('Z')[0] + .replace('.', ','); +} +export function toVttFormat(time: number): string { + const o = new Date(0); + const p = new Date(time * 1000); + return new Date(p.getTime() - o.getTime()).toISOString().split('T')[1].split('Z')[0]; +} diff --git a/app/services/highlighter/subtitles/svg-creator.ts b/app/services/highlighter/subtitles/svg-creator.ts new file mode 100644 index 000000000000..6a95ae321330 --- /dev/null +++ b/app/services/highlighter/subtitles/svg-creator.ts @@ -0,0 +1,270 @@ +export interface IResolution { + width: number; + height: number; +} +export interface ITextStyle { + fontSize: number; + fontFamily: string; + fontColor: string; + strokeColor?: string; + strokeWidth?: number; + isBold?: boolean; + isItalic?: boolean; +} +export class SvgCreator { + private lines: string[]; + + private textStyle: ITextStyle; + + private backgroundColor: string; + private backgroundAlpha: number; + private backgroundBorderRadius: number; + + private lineCount: number; + private lineWidth: number; + private lineHeight: number; + private rectHeight: number; + + private resolution: IResolution; + private subtitleHeightPositionFactor; + + private svgType: 'Subtitle' | 'CanvasText'; + + private x: number; + private y: number; + private scale: number; + private rotation: number; + private backgroundWidth: number; + private rtlLanguage = false; + + constructor(resolution: IResolution, textElementOptions?: ITextStyle) { + this.svgType = 'Subtitle'; + this.resolution = resolution; + this.subtitleHeightPositionFactor = this.calculateSubtitleHeightFactor(resolution); + + if (textElementOptions) { + this.textStyle = textElementOptions; + } + } + + public static getProgressSquare(color: string) { + return ` + + + `; + } + + public getSvgWithText(lines: string[], lineWidth: number): string { + this.lines = []; + this.lines = lines; + this.lineCount = lines.length; + + // if (this.isBold) { + // const boldFactor = 1.0; + // const lineWidthDifference = this.lineWidth * boldFactor - this.lineWidth; + // this.lineWidth *= boldFactor; + // this.x -= lineWidthDifference / 2; + // } + if (this.svgType === 'Subtitle') { + // correct line width and add margin + // 10% padding + this.lineWidth = lineWidth; + this.x = this.resolution.width * 0.5; + this.y = this.resolution.height * 0.8; + const lineHeightFactor = 1.7; + this.lineHeight = this.textStyle.fontSize * lineHeightFactor - this.textStyle.fontSize / 4; + this.rectHeight = + lines.length * this.textStyle.fontSize * lineHeightFactor + this.textStyle.fontSize / 3; + } + return this.svgSkeleton; + } + + private get svgSkeleton(): string { + return ` + + + + ${this.background} + ${this.textStyleELement} + ${this.tspans} + + `; + } + + // creates rect with width, moves it half its size to the left and 5 px down so it sits in the middle of the text + private get background(): string { + let translate; + let svgRotation = ''; + if (this.svgType === 'Subtitle') { + translate = `-${this.lineWidth / 2} -${(this.lineCount - 1) * this.lineHeight}`; + } else { + translate = '0 0'; + if (this.rotation) { + svgRotation = `rotate(${this.rotation} ${this.lineWidth / 2} ${this.rectHeight / 2})`; + } + } + let bgColor = this.backgroundColor; + let bgOpacity = this.backgroundAlpha; + // Check if color is rgba, if so: transform into rgb and alpha + if (bgColor?.includes('rgba')) { + const transformedColor = SvgCreator.transformRgba(bgColor); + bgColor = transformedColor.color; + if (bgOpacity === 1) { + // If background is available, set correct alpha + bgOpacity = transformedColor.alpha; + } + } + if (!bgColor) { + bgColor = ''; + bgOpacity = 0; + } + return ``; + } + + public static transformRgba(rgbaColor: string): { color: string; alpha: number } { + const numberRegex = /([\d.]+)/g; + const [red, green, blue, alpha] = rgbaColor.match(numberRegex); + const rgbColor = `rgb(${red},${green},${blue})`; + return { + color: rgbColor, + alpha: Number(alpha), + }; + } + + private get tspans(): string { + let tspans = ''; + let x: number; + let y: number; + if (this.svgType === 'Subtitle') { + x = 0; + y = Number(`-${this.lineHeight * (this.lineCount - 1)}`); + } else { + x = this.lineWidth / 2; + y = (this.lineHeight / 2) * 0.25; + } + + this.lines.forEach((line, index) => { + if (index === 0 && this.lineCount > 1) { + tspans += ` + ${this.rtlLanguage === true ? '.' : ''} + ${this.convertSpecialCharacter(line)} `; + } else { + let dy; + if (this.svgType === 'Subtitle') { + dy = this.lineHeight; + } else { + if (this.lineCount > 1) { + dy = index === 0 ? this.rectHeight / 2 + 10 : this.lineHeight * 1.2; + } else { + dy = this.rectHeight / 2 + this.rectHeight * 0.15; + } + } + tspans += ` ${ + this.rtlLanguage === true ? '.' : '' + } ${this.convertSpecialCharacter(line)} `; + } + }); + return tspans; + } + private get textStyleELement(): string { + let svgRotation = ''; + if (this.svgType === 'CanvasText' && this.rotation) { + svgRotation = `transform="rotate(${this.rotation} ${this.lineWidth / 2} ${ + this.rectHeight / 2 + })"`; + } + let fontColor = this.textStyle.fontColor; + let alpha = 1; + if (fontColor.includes('rgba')) { + const transformedColor = SvgCreator.transformRgba(fontColor); + fontColor = transformedColor.color; + alpha = transformedColor.alpha; + } + return ` + + `; + } + + private convertSpecialCharacter(line: string): string { + let correctedLine = line; + const replacementTable = [ + { element: '&', code: '&' }, // add after &, otherwise & will repace '&' from the code + { element: '<', code: '<' }, + { element: '>', code: '>' }, + { element: '\b', code: '' }, + { element: '\f', code: '' }, + { element: '\n', code: '' }, + { element: '\r', code: '' }, + { element: '\t', code: '' }, + { element: '\v', code: '' }, + // { element: "\\'", code: '' }, + // { element: '\\"', code: '' }, + // { element: '\\?', code: '' }, + { element: '\\\\', code: '' }, + ]; + for (const replacement of replacementTable) { + // correctedLine = correctedLine.replace(replacement.element, replacement.code); + correctedLine = correctedLine.replace(new RegExp(replacement.element, 'g'), replacement.code); + } + return correctedLine; + } + + calculateSubtitleHeightFactor(resolution: IResolution): number { + const aspectRatio = resolution.width / resolution.height; + if (aspectRatio > 1) { + // moved 80% of height down + return 0.85; + } else if (aspectRatio < 1) { + // portrait + // moved 90% of height down + return 0.9; + } else { + // square + // moved 90% of height down + return 0.85; + } + } + + public static getVideoBackgroundSVG(resolution: IResolution, color = '#ffffff'): string { + let transformedColor = color; + if (color.includes('rgba')) { + transformedColor = this.transformRgba(color).color; + } + const svgText = ` + + + + + `; + return svgText; + } + + public static isRTL(lines: string[]) { + try { + const rtlChars = '\u0591-\u07FF\u200F\u202B\u202E\uFB1D-\uFDFD\uFE70-\uFEFC'; + const rtlDirCheck = new RegExp('^[^' + rtlChars + ']*?[' + rtlChars + ']'); + const line = lines[0]; + return rtlDirCheck.test(line); + } catch (error: unknown) { + return false; + } + } +} diff --git a/app/services/highlighter/subtitles/transcription.ts b/app/services/highlighter/subtitles/transcription.ts new file mode 100644 index 000000000000..5d65388647f3 --- /dev/null +++ b/app/services/highlighter/subtitles/transcription.ts @@ -0,0 +1,804 @@ +import { SubtitleMode } from './subtitle-mode'; +import { SubtitleClip } from './subtitle-clip'; +import { SubtitleClips } from './subtitle-clips'; +import { Word, IWordIndex } from './word'; +import { removeLinebreaksFromString, roundTime } from './subtitle-utils'; + +export const MAX_PAUSE_TIME = 2; //0.3 work better but creates more ; + +export interface ISpeaker { + index: number; + name: string; +} + +export interface IStartEndConfig { + startWord: number; + endWord: number; +} + +export class Transcription { + singlePauseLength = 0.25; + words: Word[] = []; + isTranslation = false; + + get duration(): number { + return this.words[this.words.length - 1].endTimeInEdit; + } + + get onlyPauses(): boolean { + return this.words.every(word => word.isPause); + } + + get onlyCuts(): boolean { + return this.words.every(word => word.isCut); + } + + // + // initialisation functions + // + clone(transcription: Transcription) { + this.words = transcription.words.map(word => new Word().clone(word)); + this.isTranslation = transcription.isTranslation; + return this; + } + + setTranslation(isTranslation: boolean) { + this.isTranslation = isTranslation; + return this; + } + + getLastWord(): Word { + return this.words[this.words.length - 1]; + } + getSubtitleLength(aspectRatio: number) { + let charactersPerRow = 70; + // Condition true for square, portrait, vertical post and pinterest + if (aspectRatio <= 1) { + charactersPerRow = 20; + } + return charactersPerRow; + } + getSpeakerIndices(skipCuts = true): number[] { + const speakers = new Set(); + for (let index = 0; index < this.words.length; index++) { + const word = this.words[index]; + if (skipCuts && word.isCut) { + continue; + } + speakers.add(word.speakerTag); + } + return [...speakers]; + } + /** + * generating abstraction of the transcription + * @param subtitleMode subtitleMode (static, dynamic) of the current project + * @param currentVideoFormat VideoFormatOption of the current project + * @param subtitleLength Characters per row of the subtitle + */ + generateSubtitleClips( + subtitleMode: SubtitleMode, + aspectRatio: number, + subtitleLength?: number, + ): SubtitleClips { + let characterCount = 0; + let inClip = false; + const subtitleClips = new SubtitleClips(); + let startIndex = 0; + + let inPauseSeries = false; + let pauseStartIndex: number = null; + let pauseDuration = 0; + const pushSubtitle = ( + startIndexToPush: number, + endIndex: number, + newStartIndex: number, + mediaIndex: number, + ) => { + subtitleClips.push(new SubtitleClip(startIndexToPush, endIndex, mediaIndex, this)); + startIndex = newStartIndex; + pauseDuration = 0; + characterCount = 0; + inClip = false; + inPauseSeries = false; + }; + + const defaultCharactersPerRow = this.getSubtitleLength(aspectRatio); + const charactersPerRow = subtitleLength || defaultCharactersPerRow; + const isCustomLength = defaultCharactersPerRow !== charactersPerRow; + const lastNonCutWord = this.getLastNonCutWord(); + + let breakOnComma = false; + + // Condition true for square, portrait, vertical post and pinterest + if (aspectRatio <= 1 && !isCustomLength) { + breakOnComma = true; + } + let previousMediaIndex = this.words[0]?.mediaIndex; + for (let index = 0; index < lastNonCutWord; index++) { + const word = this.words[index]; + const mediaIndexChanged = previousMediaIndex !== word.mediaIndex; + + // video source changed + if (mediaIndexChanged) { + pushSubtitle(startIndex, index - 1, index, previousMediaIndex); + previousMediaIndex = word.mediaIndex; + continue; + } + previousMediaIndex = word.mediaIndex; + + // + if (word.hasSubtitlebreak === true && !word.isCut && index > 0) { + pushSubtitle(startIndex, index, index + 1, word.mediaIndex); + continue; + } + // skip cuts + if (word.isCut) { + if (!inClip) { + startIndex = index + 1; + } + continue; + } + + if (word.isPause) { + pauseDuration += word.duration; + if (!inPauseSeries) { + // pause series started + //lastWordIndex = index - 1 + pauseStartIndex = index; + inPauseSeries = true; + } + } else { + // pause series ended + inPauseSeries = false; + pauseDuration = 0; + characterCount += word.text.length + 1; // + 1 because of the pause between words + if (!inClip) { + // clip started + startIndex = index; + inClip = true; + } + } + + if (inClip) { + // subtitle ended because + if ( + pauseDuration > MAX_PAUSE_TIME && + pauseStartIndex != null && + (word.hasSubtitlebreak === true || + word.hasSubtitlebreak === null || + word.hasSubtitlebreak === undefined) + ) { + // long pause + pushSubtitle(startIndex, pauseStartIndex - 1, index + 1, word.mediaIndex); + } else if (word.hasSubtitlebreak === false) { + characterCount = 0; + pauseDuration = 0; + } else if ( + (word.hasSubtitlebreak === true || + word.hasSubtitlebreak === null || + word.hasSubtitlebreak === undefined) && + (characterCount + this.words[index + 1]?.text?.length > charactersPerRow || + (word.isEndOfSentence || breakOnComma + ? word.hasPunctuationAndComma + : word.hasPunctuation) || + word.hasSubtitlebreak) + ) { + // maximum of characters was reached + // or word contains '.', ',', '?', '!', ':', ';' + // or word was the last word of the transcription + // and a subtitle clip has already started + // End subtitle clip when pause is to long + pushSubtitle(startIndex, index, index + 1, word.mediaIndex); + } + } + } + + if (startIndex < lastNonCutWord + 1) { + pushSubtitle( + startIndex, + lastNonCutWord, + lastNonCutWord + 1, + this.words[lastNonCutWord].mediaIndex, + ); + } + + if (subtitleMode === SubtitleMode.dynamic) { + subtitleClips.convertToDynamicSubtitles(); + } + return subtitleClips; + } + + calculateSelectionDuration(startEndConfig: IStartEndConfig) { + return ( + this.words[startEndConfig?.endWord]?.endTimeInEdit - + this.words[startEndConfig?.startWord]?.startTimeInEdit + ); + } + + public sumTrimtimeToIndex(index: number): number { + let sum = 0; + this.words + .slice(0, index) + .filter(word => word.isCut) + .forEach(word => { + sum += word.duration; + }); + + return sum; + } + /** + * Calculating rendered transcription with times from cutwords removed + */ + public getShiftedTranscription(): Transcription { + const shiftedTranscription = new Transcription().clone(this); + let trimTime = 0; + let mediaSourceOffset = 0; + for (let index = 0; index < shiftedTranscription.words.length; index++) { + const word = shiftedTranscription.words[index]; + if (index > 0 && word.mediaIndex !== shiftedTranscription.words[index - 1].mediaIndex) { + mediaSourceOffset = shiftedTranscription.words[index - 1].endTimeInEdit; + trimTime = 0; + } + + trimTime += word.isCut ? word.duration : 0; + word.moveForwardsFromOriginal(trimTime - mediaSourceOffset); + } + return shiftedTranscription; + } + + // + // accessing words + // + getWordByIndex(index: number): Word { + if (this.isIndexInBounds(index)) { + return this.words[index]; + } + return null; + } + + getPreviousWordByIndex( + index: number, + skipPauses?: boolean, + skipCutword?: boolean, + includeStartIndex = false, + ): IWordIndex { + const startIndex = includeStartIndex ? index : index - 1; + for (let i = startIndex; i >= 0; i--) { + const currentWord = this.words[i]; + if ((skipPauses && currentWord.isPause) || (skipCutword && currentWord.isCut)) { + continue; + } + return { index: i, word: currentWord }; + } + return null; + } + + getNextWordByIndex( + index: number, + skipPauses?: boolean, + skipCutword?: boolean, + includeStartIndex = false, + ): IWordIndex { + const startIndex = includeStartIndex ? index : index + 1; + for (let i = startIndex; i < this.words.length; i++) { + const currentWord = this.words[i]; + if ((skipPauses && currentWord?.isPause) || (skipCutword && currentWord?.isCut)) { + continue; + } + return { index: i, word: currentWord }; + } + return null; + } + + sliceWords(startIndex: number, endIndex: number): Word[] { + if (this.isIndexInBounds(startIndex)) { + return this.words.slice(startIndex, endIndex); + } + return []; + } + + // + // text related functions + // + public getText( + includeTimestamps = false, + includeSpeakers = false, + wordChunkArray?: IWordIndex[][], + speakers?: ISpeaker[], + removeLinebreaks = false, + ): string { + let words = ''; + // If the user want to have timestamps or speakers in the export loop through the paragraphs + if ((includeTimestamps || includeSpeakers) && wordChunkArray) { + const filteredParagraphs = wordChunkArray + .map(paragraphFilter => { + const noCutwords = paragraphFilter.filter(wordIndex => !wordIndex.word.isCut); + return [...noCutwords]; + }) + .filter(wordIndexArray => wordIndexArray.length); + + for (const paragraph of filteredParagraphs) { + if (includeSpeakers) { + words += speakers[paragraph[0].word.speakerTag].name; + } + if (includeSpeakers && includeTimestamps) { + words += ' - '; + } + if (includeTimestamps) { + words += `[${getFormattedTimeFromSeconds( + paragraph[0].word.startTimeInEdit, + )} - ${getFormattedTimeFromSeconds(paragraph[paragraph.length - 1].word.endTimeInEdit)}]`; + } + if (includeTimestamps || includeSpeakers) { + words += '\n'; + } + for (const wordIndex of paragraph) { + if (!wordIndex.word.isPause) { + words += wordIndex.word.text; + words += ' '; + } + if (wordIndex.word.hasLinebreak) { + words += '\n\n'; + } + } + } + } else { + for (const word of this.words) { + if (!word.isCut && !word.isPause) { + words += word.text; + words += ' '; + } + if (word.hasLinebreak) { + words += '\n\n'; + } + } + } + if (removeLinebreaks) { + words = removeLinebreaksFromString(words); + } + return words; + } + + public setLinebreak(index: number, hasLinebreak: boolean): void { + if (this.isIndexInBounds(index)) { + this.words[index].hasLinebreak = hasLinebreak; + this.words[index].hasSubtitlebreak = hasLinebreak; + } + } + setLastWordLinebreak() { + this.setLinebreak(this.words.length - 1, true); + } + public editWord(index: number, newText: string): void { + if (this.isIndexInBounds(index)) { + this.words[index].text = newText; + } + } + + public getSentencesArray(): Word[][] { + const sentences: Word[][] = []; + let singleSentence: Word[] = []; + for (const [index, word] of this.words.entries()) { + singleSentence.push(word); + if (word.isEndOfSentence || index >= this.words.length - 1) { + sentences.push(singleSentence); + singleSentence = []; + } + } + + return sentences; + } + + getCutStartEndConfig(): IStartEndConfig[] { + const onlyCutWordIndices = this.words + .map((word, index) => { + return { index, word }; + }) + .filter(wordIndex => wordIndex.word.isCut) + .map(wordIndex => wordIndex.index); + + let sequenceStart: number; + const cuts: IStartEndConfig[] = []; + for (let index = 1; index <= onlyCutWordIndices.length; index++) { + const indexA = onlyCutWordIndices[index - 1]; + const indexB = onlyCutWordIndices[index]; + + if (!sequenceStart) sequenceStart = indexA; + + if (indexA + 1 !== indexB) { + cuts.push({ startWord: sequenceStart, endWord: indexA }); + sequenceStart = indexB; + } else if (index === onlyCutWordIndices.length - 1) { + cuts.push({ startWord: sequenceStart, endWord: indexB }); + } + } + return cuts; + } + + // + // time related functions + // + public getWordMediaIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].mediaIndex; + } + return null; + } + public getStartTimeInEditAtIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].startTimeInEdit; + } + return null; + } + + public getEndTimeInEditAtIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].endTimeInEdit; + } + return null; + } + public getStartTimeInOriginalAtIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].startTimeInOriginal; + } + return null; + } + + public getEndTimeInOriginalAtIndex(index: number): number { + if (this.isIndexInBounds(index)) { + return this.words[index].endTimeInOriginal; + } + return null; + } + + // gets the index at a specific timestamp from the original video + // jumps over cutwords + public getIndexAtTimeInEdit(time: number): number { + for (const [index, word] of this.words.entries()) { + if (!word.isCut && time >= word.startTimeInEdit && time < word.endTimeInEdit) { + return index; + } + } + return -1; + } + + public getNextIndexByTime(currentTime: number): number { + for (let index = 0; index < this.words.length; index++) { + const word = this.words[index]; + if (currentTime < word.endTimeInOriginal) { + return index; + } + } + // if nothing found return beginning + return 0; + } + + /** + * + * @returns duration of the edited video + */ + public getEditedDuration(): number { + let totalDuration = 0; + //TODO: refactor with reduce + this.words + .filter(word => !word.isCut) + .forEach(word => { + totalDuration += word.duration; + }); + + return totalDuration; + } + /** + * generating new Transcription with pauses before first word, between word with pauses and after last word + * @param duration + * @param skipFirstAndLastPauses + */ + public generatePauses(duration: number, skipPausesBefore = false, skipPausesAfter = false): void { + const newTranscription = new Transcription(); + + if (this.words.length > 0) { + if (!skipPausesBefore) { + this.addPausesBeforeFirstWord(this.words, duration, newTranscription); + } + for (let i = 0; i < this.words.length; i++) { + const word = this.words[i]; + const newWord = new Word().clone(word); + newTranscription.words = newTranscription.words.concat(newWord); + + // check if there is a word until next word + // create new Word with length of the pause + const pauseWords = this.getPauseWords(this.words, i, duration); + if (pauseWords) { + newTranscription.words = newTranscription.words.concat(pauseWords); + } else { + // No Pause + } + } + if (!skipPausesAfter) { + this.addPauseAfterLastWord(this.words, duration, newTranscription); + } + } else { + //empty word array + let currentTime = 0; + while (currentTime + this.singlePauseLength < duration) { + const newPause = new Word().initPauseWord( + currentTime, + roundTime(currentTime + this.singlePauseLength), + 0, + 0, + ); + newTranscription.words.push(newPause); + currentTime += this.singlePauseLength; + } + if (duration - currentTime !== 0) { + const newPause = new Word().initPauseWord( + currentTime, + roundTime(currentTime + duration - currentTime), + 0, + 0, + ); + newTranscription.words.push(newPause); + } + } + this.words = newTranscription.words; + } + + // + // transcription utility + // + + private isIndexInBounds(index: number): boolean { + return index >= 0 && index < this.words.length; + } + + // generate pause helper + private addPausesBeforeFirstWord( + words: Word[], + duration: number, + newTranscription: Transcription, + ) { + const pauseWords = this.getPauseWords(words, -1, duration); + if (pauseWords) { + newTranscription.words = newTranscription.words.concat(pauseWords); + } + } + addPausesBetweenStreamingChunks(words: Word[], duration: number) { + if (!words.length) { + return; + } + const pauseWords = this.getPauseWords(words, 0, duration); + if (pauseWords) { + this.words = [...pauseWords, ...this.words]; + } + } + + private addPauseAfterLastWord(words: Word[], duration: number, newTranscription: Transcription) { + const pauseWords = this.getPauseWords(words, words.length - 1, duration, true); + if (pauseWords) { + newTranscription.words = newTranscription.words.concat(pauseWords); + } + } + private getPauseWords( + words: Word[], + index: number, + duration?: number, + lastPart?: boolean, + ): Word[] { + // seconds + + if (index + 1 < words.length || lastPart) { + let firstWord: Word; + let secondWord: Word; + if (index === -1) { + // handle first Word + // check if there is a pause before the first word + firstWord = new Word().initZero(); + secondWord = new Word().clone(words[0]); + } else if (lastPart === true && index === words.length - 1) { + // handle last Word + // check if there is a pause after the last word + firstWord = new Word().clone(words[index]); + secondWord = new Word(); + secondWord.startTimeInEdit = duration; + } else { + firstWord = new Word().clone(words[index]); + secondWord = new Word().clone(words[index + 1]); + } + + const pauseTime = roundTime(secondWord.startTimeInEdit - firstWord.endTimeInEdit); + + if (pauseTime > 0) { + const pauseCount = Math.ceil(pauseTime / this.singlePauseLength); + const pauseWords: Word[] = []; + let pauseStartTime = firstWord.endTimeInEdit; + + for (let i = 0; i < pauseCount; i++) { + let leftPauseTime = roundTime(pauseTime - i * this.singlePauseLength); + leftPauseTime = + leftPauseTime > this.singlePauseLength + ? this.singlePauseLength + : roundTime(pauseTime - i * this.singlePauseLength); + + pauseWords.push( + new Word().initPauseWord( + pauseStartTime, + roundTime(pauseStartTime + leftPauseTime), + firstWord.mediaIndex, + firstWord.speakerTag ?? secondWord.speakerTag ?? 1, + ), + ); + + pauseStartTime += leftPauseTime; + } + return pauseWords; + } + // return words[index+1].startTime.time - words[index].endTime.time + } + return null; + } + /** + * Checks the sentence for the most dominant speaker and set this one for the whole sentence + */ + updateSentenceSpeaker() { + const sentences: Word[][] = this.getSentencesArray(); + const newWords: Word[] = []; + for (const sentence of sentences) { + const speakerTags = sentence.filter(word => !word.isPause).map(word => word.speakerTag); + const dominantSpeaker = speakerTags + .slice() + .sort( + (a, b) => + speakerTags.filter(v => v === a).length - speakerTags.filter(v => v === b).length, + ) + .pop(); + sentence.forEach(word => { + const newWord: Word = word; + newWord.speakerTag = dominantSpeaker; + newWords.push(newWord); + return newWord; + }); + } + this.words = newWords; + } + + /** + * generates necessary speaker information + */ + public generateSpeakerInformation(): Transcription { + const SPEAKER_MIN_PERCENTAGE = 0.05; + // Check if SPEAKER_MIN_PERCENTAGE threshold is reached, if not set speaker to 1 for every word. + const allSpeakerTags = this.words + .filter(word => word.speakerTag !== null) + .map(word => word.speakerTag); + const counts: { [tag: number]: number } = {}; + for (const speakerTag of allSpeakerTags) { + counts[speakerTag] = counts[speakerTag] ? counts[speakerTag] + 1 : 1; + } + // Only check if 2 speakers, if more just take them + const speakerKeys = Object.keys(counts).map(Number); + if ( + (speakerKeys.length === 2 && + counts[speakerKeys[0]] / allSpeakerTags.length < SPEAKER_MIN_PERCENTAGE) || + counts[speakerKeys[0]] / allSpeakerTags.length > 100 - SPEAKER_MIN_PERCENTAGE + ) { + this.words = this.words.map(word => { + word.speakerTag = 1; + return word; + }); + } + + // After all the speaker stuff is done check where to set linebreaks initially (after each speaker switch) + for (let i = 1; i < this.words.length; i++) { + if ( + this.words[i - 1].speakerTag !== this.words[i].speakerTag && + this.words[i].speakerTag != null + ) { + this.words[i - 1].hasLinebreak = true; + } + } + return this; + } + + /** + * Returns the word index of the last non cut word (this includes pauses) + * @param transcription transcription + */ + getLastNonCutWord(): number { + let lastNonCutWordIndex = this.words.length - 1; + + if (this.words?.length) { + for (let i = this.words.length - 1; i >= 0; i--) { + if (!this.words[i].isCut) { + lastNonCutWordIndex = i; + break; + } + } + } + return lastNonCutWordIndex; + } + + setCutword(from: number, to: number) { + if (this.words?.length) { + for (let i = from; i < to; i++) { + this.words[i].initCutWord(); + } + } + } + + /** + * Returns pauses + * + * @param startEndConfig Searches in this start end config. Default is current selection + * + * @param pausesThreshold Ignore if more pauses than the threshold are next to each other + * @example If pausesThreshold is set to 4: 5, 6, 7 and more pauses after each other will NOT be returned + */ + public getAllPauses( + startEndConfig: IStartEndConfig = { startWord: 0, endWord: this.words.length - 1 }, + pausesThreshold = 100, + ) { + const words = this.words; + const indicesToRemove: number[] = []; + // Loop through all words + for ( + let wordIndex = startEndConfig.startWord; + wordIndex <= startEndConfig.endWord; + wordIndex++ + ) { + const currentIndices = []; + let currentCounter = 0; + // Check if current word + next words (based on currentCounter) have a pause. If so, check next word. If not, skip with outer loop + for (currentCounter; words[wordIndex + currentCounter]?.isPause; currentCounter++) { + if ( + words[wordIndex + currentCounter]?.isPause && + !words[wordIndex + currentCounter]?.isCut + ) { + currentIndices.push(wordIndex + currentCounter); + if (!words[wordIndex + currentCounter + 1]?.isPause) { + break; + } + } else { + break; + } + } + wordIndex += currentCounter; + + if (currentIndices.length < pausesThreshold && currentIndices.length > 1) { + currentIndices.map(index => indicesToRemove.push(index)); + } + } + + return indicesToRemove; + } + + /** + * Returns pauses in the current selection + */ + getConsecutivePauses( + startEndConfig: IStartEndConfig = { startWord: 0, endWord: this.words.length - 1 }, + ): number[][] { + const words = this.words; + const consecutivePausesIndices: number[][] = []; + let index = startEndConfig.startWord; + while (index <= startEndConfig.endWord) { + const currentIndices = []; + let count = 0; + let currentWord = words[index]; + while (currentWord?.isPause && !currentWord?.isCut) { + currentIndices.push(index + count); + count++; + currentWord = words[index + count]; + } + index += currentIndices.length ? currentIndices.length : 1; + if (currentIndices.length > 0) { + consecutivePausesIndices.push(currentIndices); + } + } + return consecutivePausesIndices; + } +} + +function getFormattedTimeFromSeconds(secondsInput: number) { + const date = new Date(0); + date.setSeconds(secondsInput); + return date.toISOString().substr(secondsInput >= 3600 ? 11 : 14, secondsInput >= 3600 ? 8 : 5); +} diff --git a/app/services/highlighter/subtitles/word.ts b/app/services/highlighter/subtitles/word.ts new file mode 100644 index 000000000000..bd37e7963bbd --- /dev/null +++ b/app/services/highlighter/subtitles/word.ts @@ -0,0 +1,168 @@ +import { roundTime } from './subtitle-utils'; + +export interface IWordIndex { + index: number; + word: Word; +} + +export class Word { + // properties available from firebase + text: string; + startTimeInEdit: number; + endTimeInEdit: number; + isCut = false; + isPause = false; + hasLinebreak = false; + hasSubtitlebreak?: boolean; + speakerTag?: number; + confidence?: number; + + // properties generated after loaded + mediaIndex: number; + chunkIndex: number; + startTimeInOriginal: number; + endTimeInOriginal: number; + + public get isEndOfSentence(): boolean { + const punctuation = ['.', '?', '!', '。']; + return this.text && punctuation.some(item => this.text.includes(item)); + } + + public get hasPunctuation(): boolean { + const punctuation = ['.', '?', '!', ':', '。']; + return this.text && punctuation.some(item => this.text.includes(item)); + } + + public get hasPunctuationAndComma(): boolean { + const punctuation = ['.', ',', '?', '!', ':', ';', '。']; + return this.text && punctuation.some(item => this.text.includes(item)); + } + public get isFiller(): boolean { + const fillers = ['um', 'uh', 'hmm', 'mhm', 'uh huh']; + return this.text && fillers.some(item => this.text.toLowerCase().includes(item)); + } + + public get duration(): number { + return this.endTimeInEdit - this.startTimeInEdit; + } + public get isRealWord(): boolean { + return this.isCut !== true && this.isPause !== true; + } + + public setMediaIndex(mediaIndex: number) { + this.mediaIndex = mediaIndex; + return this; + } + public setChunkIndex(chunkIndex: number) { + this.chunkIndex = chunkIndex; + return this; + } + + constructor() { + return this; + } + public clone(word: Word): Word { + this.text = word.text; + + // this.startTimeInEdit = word.startTimeInEdit; + // this.endTimeInEdit = word.endTimeInEdit; + + this.startTimeInEdit = word.startTimeInOriginal; + this.endTimeInEdit = word.endTimeInOriginal; + + this.startTimeInOriginal = word.startTimeInOriginal; + this.endTimeInOriginal = word.endTimeInOriginal; + + this.isCut = word.isCut || false; + this.isPause = word.isPause || false; + this.hasLinebreak = word.hasLinebreak || false; + this.hasSubtitlebreak = word.hasSubtitlebreak; + this.speakerTag = word.speakerTag; + this.confidence = word.confidence; + + this.mediaIndex = word.mediaIndex; + this.chunkIndex = word.chunkIndex; + return this; + } + + // start and endTime functions + + // moves word forwards - appears earlier + public moveForwardsFromOriginal(time: number) { + this.startTimeInEdit = roundTime(this.startTimeInOriginal - time); + this.endTimeInEdit = roundTime(this.endTimeInOriginal - time); + } + // moves word backwards - appears later + public moveBackwardsFromOriginal(time: number) { + this.startTimeInEdit = roundTime(this.startTimeInOriginal + time); + this.endTimeInEdit = roundTime(this.endTimeInOriginal + time); + } + + public fromTranscriptionService( + text: string, + startTime: number, + endTime: number, + speakerTag: number, + confidence: number, + ): Word { + if (text.includes('<') && text.includes('>', text.length - 2)) { + this.isPause = true; + } else { + this.text = text; + } + this.startTimeInEdit = startTime; + this.startTimeInOriginal = startTime; + this.endTimeInEdit = endTime; + this.endTimeInOriginal = endTime; + this.speakerTag = speakerTag || 0; + this.confidence = confidence; + return this; + } + + public init(startTime: number, endTime: number, mediaIndex: number, speakerTag: number) { + this.startTimeInEdit = startTime; + this.endTimeInEdit = endTime; + this.startTimeInOriginal = startTime; + this.endTimeInOriginal = endTime; + this.mediaIndex = mediaIndex; + this.speakerTag = speakerTag; + return this; + } + public initPauseWord(startTime: number, endTime: number, mediaIndex: number, speakerTag: number) { + this.isPause = true; + return this.init(startTime, endTime, mediaIndex, speakerTag); + } + public initZero() { + this.startTimeInEdit = 0; + this.endTimeInEdit = 0; + this.startTimeInOriginal = 0; + this.endTimeInOriginal = 0; + return this; + } + public initCutWord() { + this.isCut = true; + return this; + } + public initLinebreak() { + this.hasLinebreak = true; + return this; + } + + public equals(otherWord: Word): boolean { + // properties available from firebase + return ( + this.text === otherWord.text && + this.startTimeInEdit === otherWord.startTimeInEdit && + this.endTimeInEdit === otherWord.endTimeInEdit && + this.isCut === otherWord.isCut && + this.isPause === otherWord.isPause && + this.hasLinebreak === otherWord.hasLinebreak && + this.hasSubtitlebreak === otherWord.hasSubtitlebreak && + this.speakerTag === otherWord.speakerTag && + this.mediaIndex === otherWord.mediaIndex && + this.chunkIndex === otherWord.chunkIndex && + this.startTimeInOriginal === otherWord.startTimeInOriginal && + this.endTimeInOriginal === otherWord.endTimeInOriginal + ); + } +} diff --git a/electron-builder/base.config.js b/electron-builder/base.config.js index 78336c5d548a..5ae634ec8c9e 100644 --- a/electron-builder/base.config.js +++ b/electron-builder/base.config.js @@ -35,7 +35,15 @@ const base = { allowToChangeInstallationDirectory: true, include: 'installer.nsh', }, - asarUnpack : ["**/node-libuiohook/**", "**/node-fontinfo/**", "**/font-manager/**", "**/game_overlay/**","**/color-picker/**"], + asarUnpack: [ + '**/node-libuiohook/**', + '**/node-fontinfo/**', + '**/font-manager/**', + '**/game_overlay/**', + '**/color-picker/**', + '**/node_modules/sharp/**/*', + '**/node_modules/@img/**/*', + ], publish: { provider: 'generic', url: 'https://slobs-cdn.streamlabs.com', @@ -64,12 +72,17 @@ const base = { if (fs.existsSync(signingPath)) { fs.appendFileSync(signingPath, `${config.path}\n`); } else { - cp.execSync(`logisign client --client logitech-cpg-sign-client --app streamlabs --files "${config.path}"`, { stdio: 'inherit' }); + cp.execSync( + `logisign client --client logitech-cpg-sign-client --app streamlabs --files "${config.path}"`, + { stdio: 'inherit' }, + ); } }, }, mac: { - identity: (process.env.APPLE_SLD_IDENTITY) ? process.env.APPLE_SLD_IDENTITY : 'Streamlabs LLC (UT675MBB9Q)', + identity: process.env.APPLE_SLD_IDENTITY + ? process.env.APPLE_SLD_IDENTITY + : 'Streamlabs LLC (UT675MBB9Q)', extraFiles: [ 'shared-resources/**/*', '!shared-resources/README', diff --git a/package.json b/package.json index 018440e7e6c2..5f2c56beba1f 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,6 @@ "realm": "12.7.1", "rimraf": "^2.6.1", "semver": "^5.5.1", - "sharp": "^0.33.5", "socket.io-client": "2.3.1", "tasklist": "^3.1.1", "uuid": "^3.0.1", @@ -248,7 +247,8 @@ "zustand": "^4.4.1" }, "optionalDependencies": { - "node-win32-np": "1.0.6" + "node-win32-np": "1.0.6", + "sharp": "^0.34.2" }, "resolutions": { "minimist": "1.2.6", diff --git a/webpack.base.config.js b/webpack.base.config.js index 495300911da8..7aa5ffddfbf7 100644 --- a/webpack.base.config.js +++ b/webpack.base.config.js @@ -102,6 +102,7 @@ module.exports = { archiver: 'require("archiver")', 'extract-zip': 'require("extract-zip")', 'fs-extra': 'require("fs-extra")', + sharp: 'commonjs sharp', }, module: { diff --git a/yarn.lock b/yarn.lock index 40403eebfaf6..442b9f0b915a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1921,12 +1921,12 @@ __metadata: languageName: node linkType: hard -"@emnapi/runtime@npm:^1.2.0": - version: 1.3.1 - resolution: "@emnapi/runtime@npm:1.3.1" +"@emnapi/runtime@npm:^1.4.4": + version: 1.5.0 + resolution: "@emnapi/runtime@npm:1.5.0" dependencies: tslib: ^2.4.0 - checksum: 9a16ae7905a9c0e8956cf1854ef74e5087fbf36739abdba7aa6b308485aafdc993da07c19d7af104cd5f8e425121120852851bb3a0f78e2160e420a36d47f42f + checksum: 03b23bdc0bb72bce4d8967ca29d623c2599af18977975c10532577db2ec89a57d97d2c76c5c4bde856c7c29302b9f7af357e921c42bd952bdda206972185819a languageName: node linkType: hard @@ -2014,11 +2014,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-darwin-arm64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-darwin-arm64@npm:0.33.5" +"@img/sharp-darwin-arm64@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-darwin-arm64@npm:0.34.3" dependencies: - "@img/sharp-libvips-darwin-arm64": 1.0.4 + "@img/sharp-libvips-darwin-arm64": 1.2.0 dependenciesMeta: "@img/sharp-libvips-darwin-arm64": optional: true @@ -2026,11 +2026,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-darwin-x64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-darwin-x64@npm:0.33.5" +"@img/sharp-darwin-x64@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-darwin-x64@npm:0.34.3" dependencies: - "@img/sharp-libvips-darwin-x64": 1.0.4 + "@img/sharp-libvips-darwin-x64": 1.2.0 dependenciesMeta: "@img/sharp-libvips-darwin-x64": optional: true @@ -2038,67 +2038,74 @@ __metadata: languageName: node linkType: hard -"@img/sharp-libvips-darwin-arm64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-darwin-arm64@npm:1.0.4" +"@img/sharp-libvips-darwin-arm64@npm:1.2.0": + version: 1.2.0 + resolution: "@img/sharp-libvips-darwin-arm64@npm:1.2.0" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@img/sharp-libvips-darwin-x64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-darwin-x64@npm:1.0.4" +"@img/sharp-libvips-darwin-x64@npm:1.2.0": + version: 1.2.0 + resolution: "@img/sharp-libvips-darwin-x64@npm:1.2.0" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@img/sharp-libvips-linux-arm64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linux-arm64@npm:1.0.4" +"@img/sharp-libvips-linux-arm64@npm:1.2.0": + version: 1.2.0 + resolution: "@img/sharp-libvips-linux-arm64@npm:1.2.0" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@img/sharp-libvips-linux-arm@npm:1.0.5": - version: 1.0.5 - resolution: "@img/sharp-libvips-linux-arm@npm:1.0.5" +"@img/sharp-libvips-linux-arm@npm:1.2.0": + version: 1.2.0 + resolution: "@img/sharp-libvips-linux-arm@npm:1.2.0" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@img/sharp-libvips-linux-s390x@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linux-s390x@npm:1.0.4" +"@img/sharp-libvips-linux-ppc64@npm:1.2.0": + version: 1.2.0 + resolution: "@img/sharp-libvips-linux-ppc64@npm:1.2.0" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + +"@img/sharp-libvips-linux-s390x@npm:1.2.0": + version: 1.2.0 + resolution: "@img/sharp-libvips-linux-s390x@npm:1.2.0" conditions: os=linux & cpu=s390x languageName: node linkType: hard -"@img/sharp-libvips-linux-x64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linux-x64@npm:1.0.4" +"@img/sharp-libvips-linux-x64@npm:1.2.0": + version: 1.2.0 + resolution: "@img/sharp-libvips-linux-x64@npm:1.2.0" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@img/sharp-libvips-linuxmusl-arm64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linuxmusl-arm64@npm:1.0.4" +"@img/sharp-libvips-linuxmusl-arm64@npm:1.2.0": + version: 1.2.0 + resolution: "@img/sharp-libvips-linuxmusl-arm64@npm:1.2.0" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@img/sharp-libvips-linuxmusl-x64@npm:1.0.4": - version: 1.0.4 - resolution: "@img/sharp-libvips-linuxmusl-x64@npm:1.0.4" +"@img/sharp-libvips-linuxmusl-x64@npm:1.2.0": + version: 1.2.0 + resolution: "@img/sharp-libvips-linuxmusl-x64@npm:1.2.0" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@img/sharp-linux-arm64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linux-arm64@npm:0.33.5" +"@img/sharp-linux-arm64@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-linux-arm64@npm:0.34.3" dependencies: - "@img/sharp-libvips-linux-arm64": 1.0.4 + "@img/sharp-libvips-linux-arm64": 1.2.0 dependenciesMeta: "@img/sharp-libvips-linux-arm64": optional: true @@ -2106,11 +2113,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linux-arm@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linux-arm@npm:0.33.5" +"@img/sharp-linux-arm@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-linux-arm@npm:0.34.3" dependencies: - "@img/sharp-libvips-linux-arm": 1.0.5 + "@img/sharp-libvips-linux-arm": 1.2.0 dependenciesMeta: "@img/sharp-libvips-linux-arm": optional: true @@ -2118,11 +2125,23 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linux-s390x@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linux-s390x@npm:0.33.5" +"@img/sharp-linux-ppc64@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-linux-ppc64@npm:0.34.3" + dependencies: + "@img/sharp-libvips-linux-ppc64": 1.2.0 + dependenciesMeta: + "@img/sharp-libvips-linux-ppc64": + optional: true + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + +"@img/sharp-linux-s390x@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-linux-s390x@npm:0.34.3" dependencies: - "@img/sharp-libvips-linux-s390x": 1.0.4 + "@img/sharp-libvips-linux-s390x": 1.2.0 dependenciesMeta: "@img/sharp-libvips-linux-s390x": optional: true @@ -2130,11 +2149,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linux-x64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linux-x64@npm:0.33.5" +"@img/sharp-linux-x64@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-linux-x64@npm:0.34.3" dependencies: - "@img/sharp-libvips-linux-x64": 1.0.4 + "@img/sharp-libvips-linux-x64": 1.2.0 dependenciesMeta: "@img/sharp-libvips-linux-x64": optional: true @@ -2142,11 +2161,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linuxmusl-arm64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linuxmusl-arm64@npm:0.33.5" +"@img/sharp-linuxmusl-arm64@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-linuxmusl-arm64@npm:0.34.3" dependencies: - "@img/sharp-libvips-linuxmusl-arm64": 1.0.4 + "@img/sharp-libvips-linuxmusl-arm64": 1.2.0 dependenciesMeta: "@img/sharp-libvips-linuxmusl-arm64": optional: true @@ -2154,11 +2173,11 @@ __metadata: languageName: node linkType: hard -"@img/sharp-linuxmusl-x64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-linuxmusl-x64@npm:0.33.5" +"@img/sharp-linuxmusl-x64@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-linuxmusl-x64@npm:0.34.3" dependencies: - "@img/sharp-libvips-linuxmusl-x64": 1.0.4 + "@img/sharp-libvips-linuxmusl-x64": 1.2.0 dependenciesMeta: "@img/sharp-libvips-linuxmusl-x64": optional: true @@ -2166,25 +2185,32 @@ __metadata: languageName: node linkType: hard -"@img/sharp-wasm32@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-wasm32@npm:0.33.5" +"@img/sharp-wasm32@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-wasm32@npm:0.34.3" dependencies: - "@emnapi/runtime": ^1.2.0 + "@emnapi/runtime": ^1.4.4 conditions: cpu=wasm32 languageName: node linkType: hard -"@img/sharp-win32-ia32@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-win32-ia32@npm:0.33.5" +"@img/sharp-win32-arm64@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-win32-arm64@npm:0.34.3" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@img/sharp-win32-ia32@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-win32-ia32@npm:0.34.3" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@img/sharp-win32-x64@npm:0.33.5": - version: 0.33.5 - resolution: "@img/sharp-win32-x64@npm:0.33.5" +"@img/sharp-win32-x64@npm:0.34.3": + version: 0.34.3 + resolution: "@img/sharp-win32-x64@npm:0.34.3" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -6653,10 +6679,10 @@ __metadata: languageName: node linkType: hard -"detect-libc@npm:^2.0.3": - version: 2.0.3 - resolution: "detect-libc@npm:2.0.3" - checksum: 2ba6a939ae55f189aea996ac67afceb650413c7a34726ee92c40fb0deb2400d57ef94631a8a3f052055eea7efb0f99a9b5e6ce923415daa3e68221f963cfc27d +"detect-libc@npm:^2.0.4": + version: 2.0.4 + resolution: "detect-libc@npm:2.0.4" + checksum: 3d186b7d4e16965e10e21db596c78a4e131f9eee69c0081d13b85e6a61d7448d3ba23fe7997648022bdfa3b0eb4cc3c289a44c8188df949445a20852689abef6 languageName: node linkType: hard @@ -13874,12 +13900,12 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.6.3": - version: 7.6.3 - resolution: "semver@npm:7.6.3" +"semver@npm:^7.7.2": + version: 7.7.2 + resolution: "semver@npm:7.7.2" bin: semver: bin/semver.js - checksum: 4110ec5d015c9438f322257b1c51fe30276e5f766a3f64c09edd1d7ea7118ecbc3f379f3b69032bacf13116dc7abc4ad8ce0d7e2bd642e26b0d271b56b61a7d8 + checksum: dd94ba8f1cbc903d8eeb4dd8bf19f46b3deb14262b6717d0de3c804b594058ae785ef2e4b46c5c3b58733c99c83339068203002f9e37cfe44f7e2cc5e3d2f621 languageName: node linkType: hard @@ -13965,32 +13991,35 @@ __metadata: languageName: node linkType: hard -"sharp@npm:^0.33.5": - version: 0.33.5 - resolution: "sharp@npm:0.33.5" - dependencies: - "@img/sharp-darwin-arm64": 0.33.5 - "@img/sharp-darwin-x64": 0.33.5 - "@img/sharp-libvips-darwin-arm64": 1.0.4 - "@img/sharp-libvips-darwin-x64": 1.0.4 - "@img/sharp-libvips-linux-arm": 1.0.5 - "@img/sharp-libvips-linux-arm64": 1.0.4 - "@img/sharp-libvips-linux-s390x": 1.0.4 - "@img/sharp-libvips-linux-x64": 1.0.4 - "@img/sharp-libvips-linuxmusl-arm64": 1.0.4 - "@img/sharp-libvips-linuxmusl-x64": 1.0.4 - "@img/sharp-linux-arm": 0.33.5 - "@img/sharp-linux-arm64": 0.33.5 - "@img/sharp-linux-s390x": 0.33.5 - "@img/sharp-linux-x64": 0.33.5 - "@img/sharp-linuxmusl-arm64": 0.33.5 - "@img/sharp-linuxmusl-x64": 0.33.5 - "@img/sharp-wasm32": 0.33.5 - "@img/sharp-win32-ia32": 0.33.5 - "@img/sharp-win32-x64": 0.33.5 +"sharp@npm:^0.34.2": + version: 0.34.3 + resolution: "sharp@npm:0.34.3" + dependencies: + "@img/sharp-darwin-arm64": 0.34.3 + "@img/sharp-darwin-x64": 0.34.3 + "@img/sharp-libvips-darwin-arm64": 1.2.0 + "@img/sharp-libvips-darwin-x64": 1.2.0 + "@img/sharp-libvips-linux-arm": 1.2.0 + "@img/sharp-libvips-linux-arm64": 1.2.0 + "@img/sharp-libvips-linux-ppc64": 1.2.0 + "@img/sharp-libvips-linux-s390x": 1.2.0 + "@img/sharp-libvips-linux-x64": 1.2.0 + "@img/sharp-libvips-linuxmusl-arm64": 1.2.0 + "@img/sharp-libvips-linuxmusl-x64": 1.2.0 + "@img/sharp-linux-arm": 0.34.3 + "@img/sharp-linux-arm64": 0.34.3 + "@img/sharp-linux-ppc64": 0.34.3 + "@img/sharp-linux-s390x": 0.34.3 + "@img/sharp-linux-x64": 0.34.3 + "@img/sharp-linuxmusl-arm64": 0.34.3 + "@img/sharp-linuxmusl-x64": 0.34.3 + "@img/sharp-wasm32": 0.34.3 + "@img/sharp-win32-arm64": 0.34.3 + "@img/sharp-win32-ia32": 0.34.3 + "@img/sharp-win32-x64": 0.34.3 color: ^4.2.3 - detect-libc: ^2.0.3 - semver: ^7.6.3 + detect-libc: ^2.0.4 + semver: ^7.7.2 dependenciesMeta: "@img/sharp-darwin-arm64": optional: true @@ -14004,6 +14033,8 @@ __metadata: optional: true "@img/sharp-libvips-linux-arm64": optional: true + "@img/sharp-libvips-linux-ppc64": + optional: true "@img/sharp-libvips-linux-s390x": optional: true "@img/sharp-libvips-linux-x64": @@ -14016,6 +14047,8 @@ __metadata: optional: true "@img/sharp-linux-arm64": optional: true + "@img/sharp-linux-ppc64": + optional: true "@img/sharp-linux-s390x": optional: true "@img/sharp-linux-x64": @@ -14026,11 +14059,13 @@ __metadata: optional: true "@img/sharp-wasm32": optional: true + "@img/sharp-win32-arm64": + optional: true "@img/sharp-win32-ia32": optional: true "@img/sharp-win32-x64": optional: true - checksum: 04beae89910ac65c5f145f88de162e8466bec67705f497ace128de849c24d168993e016f33a343a1f3c30b25d2a90c3e62b017a9a0d25452371556f6cd2471e4 + checksum: f1d70751ab15080957f1a5db5583d00b002e644878e3e9a6d00bdbb31255f6e9df218ef82d96d3588b83023681f35bcf03e265a44e214b11306ef3fb439ac04d languageName: node linkType: hard @@ -14318,7 +14353,7 @@ __metadata: rxjs: 6.3.3 semver: ^5.5.1 serve-handler: 5.0.7 - sharp: ^0.33.5 + sharp: ^0.34.2 shelljs: 0.8.5 signtool: 1.0.0 sl-vue-tree: "https://github.com/streamlabs/sl-vue-tree.git" @@ -14371,6 +14406,8 @@ __metadata: dependenciesMeta: node-win32-np: optional: true + sharp: + optional: true languageName: unknown linkType: soft