From e782cc21b553e41cc42a1a31b145f0a3988599b6 Mon Sep 17 00:00:00 2001 From: Carbrex <95964955+Carbrex@users.noreply.github.com> Date: Sat, 31 Aug 2024 01:53:52 +0530 Subject: [PATCH] Lazy load mic.ts --- ui/coordinateTrainer/src/coordinateTrainer.ts | 22 +- ui/round/src/ctrl.ts | 6 +- ui/site/package.json | 3 +- ui/site/src/mic.ts | 470 +++++++++--------- ui/site/src/site.ts | 2 - 5 files changed, 254 insertions(+), 249 deletions(-) diff --git a/ui/coordinateTrainer/src/coordinateTrainer.ts b/ui/coordinateTrainer/src/coordinateTrainer.ts index b62f36eb9e0a0..cdc8b43263753 100644 --- a/ui/coordinateTrainer/src/coordinateTrainer.ts +++ b/ui/coordinateTrainer/src/coordinateTrainer.ts @@ -14,16 +14,18 @@ import CoordinateTrainerCtrl from './ctrl'; const patch = init([classModule, attributesModule, propsModule, eventListenersModule, styleModule]); export function initModule(config: CoordinateTrainerConfig) { - const ctrl = new CoordinateTrainerCtrl(config, redraw); - const element = document.getElementById('trainer')!; - element.innerHTML = ''; - const inner = document.createElement('div'); - element.appendChild(inner); - let vnode = patch(inner, view(ctrl)); + site.asset.loadEsm('mic').then(() => { + const ctrl = new CoordinateTrainerCtrl(config, redraw); + const element = document.getElementById('trainer')!; + element.innerHTML = ''; + const inner = document.createElement('div'); + element.appendChild(inner); + let vnode = patch(inner, view(ctrl)); - function redraw() { - vnode = patch(vnode, view(ctrl)); - } + function redraw() { + vnode = patch(vnode, view(ctrl)); + } - menuHover(); + menuHover(); + }); } diff --git a/ui/round/src/ctrl.ts b/ui/round/src/ctrl.ts index 6b280b90dd587..257f47aa3da6e 100644 --- a/ui/round/src/ctrl.ts +++ b/ui/round/src/ctrl.ts @@ -838,8 +838,10 @@ export default class RoundController implements MoveRootCtrl { this.keyboardMove.update(up); } if (this.data.pref.voiceMove) { - if (this.voiceMove) this.voiceMove.update(up); - else this.voiceMove = makeVoiceMove(this, up); + site.asset.loadEsm('mic').then(() => { + if (this.voiceMove) this.voiceMove.update(up); + else this.voiceMove = makeVoiceMove(this, up); + }); } if (this.keyboardMove || this.voiceMove) requestAnimationFrame(() => this.redraw()); site.pubsub.on('board.change', (is3d: boolean) => { diff --git a/ui/site/package.json b/ui/site/package.json index 04035db03b4d9..5ff78343f1474 100644 --- a/ui/site/package.json +++ b/ui/site/package.json @@ -19,7 +19,8 @@ "src/site.ts", "src/site.tvEmbed.ts", "src/site.puzzleEmbed.ts", - "src/site.lpvEmbed.ts" + "src/site.lpvEmbed.ts", + "src/mic.ts" ], "sync": { "node_modules/dialog-polyfill/dist/dialog-polyfill.esm.js": "public/npm" diff --git a/ui/site/src/mic.ts b/ui/site/src/mic.ts index dcdbc80156ebd..db8991bb4536b 100644 --- a/ui/site/src/mic.ts +++ b/ui/site/src/mic.ts @@ -35,269 +35,271 @@ class RecNode implements Selectable { } } -export const mic = new (class implements Voice.Microphone { - language = 'en'; - - audioCtx: AudioContext | undefined; - mediaStream: MediaStream; - micSource: AudioNode; - vosk: VoskModule; - - deviceId = storedStringProp('voice.micDeviceId', 'default'); - deviceIds?: string[]; - - recs = new Selector(); - recId = 'default'; - ctrl: Voice.Listener; - download?: XMLHttpRequest; - broadcastTimeout?: number; - voskStatus = ''; - busy = false; - interrupt = false; - paused = 0; - - get lang() { - return this.language; - } - - setLang(lang: string) { - if (lang === this.language) return; - this.stop(); - this.language = lang; - } +export function initModule(): void { + site.mic = new (class implements Voice.Microphone { + language = 'en'; + + audioCtx: AudioContext | undefined; + mediaStream: MediaStream; + micSource: AudioNode; + vosk: VoskModule; + + deviceId = storedStringProp('voice.micDeviceId', 'default'); + deviceIds?: string[]; + + recs = new Selector(); + recId = 'default'; + ctrl: Voice.Listener; + download?: XMLHttpRequest; + broadcastTimeout?: number; + voskStatus = ''; + busy = false; + interrupt = false; + paused = 0; + + get lang() { + return this.language; + } - async getMics() { - return navigator.mediaDevices - .enumerateDevices() - .then(d => d.filter(d => d.kind == 'audioinput' && d.label)); - } + setLang(lang: string) { + if (lang === this.language) return; + this.stop(); + this.language = lang; + } - get micId() { - return this.deviceId(); - } + async getMics() { + return navigator.mediaDevices + .enumerateDevices() + .then(d => d.filter(d => d.kind == 'audioinput' && d.label)); + } - setMic(id: string) { - const listening = this.isListening; - site.mic.stop(); - this.deviceId(id); - this.recs.close(); - this.audioCtx?.close(); - this.audioCtx = undefined; - if (listening) this.start(); - } + get micId() { + return this.deviceId(); + } - setController(ctrl: Voice.Listener) { - this.ctrl = ctrl; - this.ctrl('', 'status'); // hello - } + setMic(id: string) { + const listening = this.isListening; + site.mic.stop(); + this.deviceId(id); + this.recs.close(); + this.audioCtx?.close(); + this.audioCtx = undefined; + if (listening) this.start(); + } - addListener(listener: Voice.Listener, also: { recId?: string; listenerId?: string } = {}) { - const recId = also.recId ?? 'default'; - if (!this.recs.group.has(recId)) throw `No recognizer for '${recId}'`; - this.recs.group.get(recId)!.listenerMap.set(also.listenerId ?? recId, listener); - } + setController(ctrl: Voice.Listener) { + this.ctrl = ctrl; + this.ctrl('', 'status'); // hello + } - removeListener(listenerId: string) { - this.recs.group.forEach(v => v.listenerMap.delete(listenerId)); - } + addListener(listener: Voice.Listener, also: { recId?: string; listenerId?: string } = {}) { + const recId = also.recId ?? 'default'; + if (!this.recs.group.has(recId)) throw `No recognizer for '${recId}'`; + this.recs.group.get(recId)!.listenerMap.set(also.listenerId ?? recId, listener); + } - initRecognizer( - words: string[], - also: { - recId?: string; - partial?: boolean; - listener?: Voice.Listener; - listenerId?: string; - } = {}, - ) { - if (words.length === 0) { - this.recs.delete(also.recId); - return; + removeListener(listenerId: string) { + this.recs.group.forEach(v => v.listenerMap.delete(listenerId)); } - const recId = also.recId ?? 'default'; - const rec = new RecNode(words.slice(), also.partial === true); - if (this.vosk?.isLoaded(this.lang)) this.initKaldi(recId, rec); - this.recs.set(recId, rec); - if (also.listener) this.addListener(also.listener, { recId, listenerId: also.listenerId }); - } - setRecognizer(recId = 'default') { - this.recId = recId; - if (!this.isListening) return; - this.recs.select(recId); - this.vosk?.select(recId); - } + initRecognizer( + words: string[], + also: { + recId?: string; + partial?: boolean; + listener?: Voice.Listener; + listenerId?: string; + } = {}, + ) { + if (words.length === 0) { + this.recs.delete(also.recId); + return; + } + const recId = also.recId ?? 'default'; + const rec = new RecNode(words.slice(), also.partial === true); + if (this.vosk?.isLoaded(this.lang)) this.initKaldi(recId, rec); + this.recs.set(recId, rec); + if (also.listener) this.addListener(also.listener, { recId, listenerId: also.listenerId }); + } - async start(listen = true): Promise { - try { - if (listen && this.isListening && this.recId === this.recs.key) return; - this.busy = true; - await this.initModel(); - if (!this.busy) throw ''; - for (const [recId, rec] of this.recs.group) this.initKaldi(recId, rec); - this.recs.select(listen && this.recId); - this.vosk?.select(listen && this.recId); - this.micTrack!.enabled = listen; - this.busy = false; - this.broadcast(listen ? 'Listening...' : '', 'start'); - } catch (e: any) { - if (e instanceof DOMException && e.name === 'NotAllowedError') this.stop(['No permission', 'error']); - else this.stop([e.toString(), 'error']); - if (e !== '') throw e; + setRecognizer(recId = 'default') { + this.recId = recId; + if (!this.isListening) return; + this.recs.select(recId); + this.vosk?.select(recId); } - } - stop(reason: [string, Voice.MsgType] = ['', 'stop']) { - if (this.micTrack) this.micTrack.enabled = false; - this.download?.abort(); - this.download = undefined; - this.busy = false; - this.recs.select(false); - this.vosk?.select(false); - this.broadcast(...reason); - } + async start(listen = true): Promise { + try { + if (listen && this.isListening && this.recId === this.recs.key) return; + this.busy = true; + await this.initModel(); + if (!this.busy) throw ''; + for (const [recId, rec] of this.recs.group) this.initKaldi(recId, rec); + this.recs.select(listen && this.recId); + this.vosk?.select(listen && this.recId); + this.micTrack!.enabled = listen; + this.busy = false; + this.broadcast(listen ? 'Listening...' : '', 'start'); + } catch (e: any) { + if (e instanceof DOMException && e.name === 'NotAllowedError') this.stop(['No permission', 'error']); + else this.stop([e.toString(), 'error']); + if (e !== '') throw e; + } + } - // pause/resume use a counter so calls must be balanced. - // short duration interruptions, use start/stop otherwise - pause() { - if (++this.paused !== 1 || !this.micTrack?.enabled) return; - this.micTrack.enabled = false; - } + stop(reason: [string, Voice.MsgType] = ['', 'stop']) { + if (this.micTrack) this.micTrack.enabled = false; + this.download?.abort(); + this.download = undefined; + this.busy = false; + this.recs.select(false); + this.vosk?.select(false); + this.broadcast(...reason); + } - resume() { - this.paused = Math.min(this.paused - 1, 0); - if (this.paused !== 0 || this.micTrack === undefined) return; - this.micTrack.enabled = !!this.recs.selected; - } + // pause/resume use a counter so calls must be balanced. + // short duration interruptions, use start/stop otherwise + pause() { + if (++this.paused !== 1 || !this.micTrack?.enabled) return; + this.micTrack.enabled = false; + } - get isListening(): boolean { - return !!this.recs.selected && !!(this.micTrack?.enabled || this.paused) && !this.isBusy; - } + resume() { + this.paused = Math.min(this.paused - 1, 0); + if (this.paused !== 0 || this.micTrack === undefined) return; + this.micTrack.enabled = !!this.recs.selected; + } - get isBusy(): boolean { - return this.busy; - } + get isListening(): boolean { + return !!this.recs.selected && !!(this.micTrack?.enabled || this.paused) && !this.isBusy; + } - get status(): string { - return this.voskStatus; - } + get isBusy(): boolean { + return this.busy; + } - stopPropagation() { - this.interrupt = true; - } + get status(): string { + return this.voskStatus; + } - /*private*/ get micTrack(): MediaStreamTrack | undefined { - return this.mediaStream?.getAudioTracks()[0]; - } + stopPropagation() { + this.interrupt = true; + } - /*private*/ initKaldi(recId: string, rec: RecNode) { - if (rec.node) return; - rec.node = this.vosk?.initRecognizer({ - recId: recId, - audioCtx: this.audioCtx!, - partial: rec.partial, - words: rec.words, - broadcast: this.broadcast.bind(this), - }); - } + /*private*/ get micTrack(): MediaStreamTrack | undefined { + return this.mediaStream?.getAudioTracks()[0]; + } - /*private*/ async initModel(): Promise { - if (this.vosk?.isLoaded(this.lang)) { - await this.initAudio(); - return; + /*private*/ initKaldi(recId: string, rec: RecNode) { + if (rec.node) return; + rec.node = this.vosk?.initRecognizer({ + recId: recId, + audioCtx: this.audioCtx!, + partial: rec.partial, + words: rec.words, + broadcast: this.broadcast.bind(this), + }); } - this.broadcast('Loading...'); - const modelUrl = site.asset.url(models.get(this.lang)!, { version: false }); - const downloadAsync = this.downloadModel(`/vosk/${modelUrl.replace(/[\W]/g, '_')}`); - const audioAsync = this.initAudio(); + /*private*/ async initModel(): Promise { + if (this.vosk?.isLoaded(this.lang)) { + await this.initAudio(); + return; + } + this.broadcast('Loading...'); - this.vosk ??= await site.asset.loadEsm('voice.vosk'); + const modelUrl = site.asset.url(models.get(this.lang)!, { version: false }); + const downloadAsync = this.downloadModel(`/vosk/${modelUrl.replace(/[\W]/g, '_')}`); + const audioAsync = this.initAudio(); - await downloadAsync; - await this.vosk.initModel(modelUrl, this.lang); - await audioAsync; - } + this.vosk ??= await site.asset.loadEsm('voice.vosk'); - /*private*/ async initAudio(): Promise { - if (this.audioCtx?.state === 'suspended') await this.audioCtx.resume(); - if (this.audioCtx?.state === 'running') return; - else if (this.audioCtx) throw `Error ${this.audioCtx.state}`; - this.mediaStream = await navigator.mediaDevices.getUserMedia({ - video: false, - audio: { - sampleRate: { ideal: 16000 }, - echoCancellation: { ideal: true }, - noiseSuppression: { ideal: true }, - deviceId: this.micId, - }, - }); - this.audioCtx = new AudioContext({ - sampleRate: this.mediaStream.getAudioTracks()[0].getSettings().sampleRate, - }); - this.micSource = this.audioCtx.createMediaStreamSource(this.mediaStream); - this.recs.ctx = { vosk: this.vosk, source: this.micSource, ctx: this.audioCtx }; - } + await downloadAsync; + await this.vosk.initModel(modelUrl, this.lang); + await audioAsync; + } - /*private*/ broadcast(text: string, msgType: Voice.MsgType = 'status', forMs = 0) { - this.ctrl?.call(this, text, msgType); - if (msgType === 'status' || msgType === 'full') window.clearTimeout(this.broadcastTimeout); - this.voskStatus = text; - for (const li of this.recs.get(this.recId)?.listeners ?? []) { - if (!this.interrupt) li(text, msgType); + /*private*/ async initAudio(): Promise { + if (this.audioCtx?.state === 'suspended') await this.audioCtx.resume(); + if (this.audioCtx?.state === 'running') return; + else if (this.audioCtx) throw `Error ${this.audioCtx.state}`; + this.mediaStream = await navigator.mediaDevices.getUserMedia({ + video: false, + audio: { + sampleRate: { ideal: 16000 }, + echoCancellation: { ideal: true }, + noiseSuppression: { ideal: true }, + deviceId: this.micId, + }, + }); + this.audioCtx = new AudioContext({ + sampleRate: this.mediaStream.getAudioTracks()[0].getSettings().sampleRate, + }); + this.micSource = this.audioCtx.createMediaStreamSource(this.mediaStream); + this.recs.ctx = { vosk: this.vosk, source: this.micSource, ctx: this.audioCtx }; } - this.interrupt = false; - this.broadcastTimeout = forMs > 0 ? window.setTimeout(() => this.broadcast(''), forMs) : undefined; - } - /*private*/ async downloadModel(emscriptenPath: string): Promise { - const voskStore = await objectStorage({ - db: '/vosk', - store: 'FILE_DATA', - version: 21, - upgrade: (_, idbStore?: IDBObjectStore) => { - // make emscripten fs happy - idbStore?.createIndex('timestamp', 'timestamp', { unique: false }); - }, - }); - if ((await voskStore.count(`${emscriptenPath}/extracted.ok`)) > 0) return; - const modelBlob: ArrayBuffer | undefined = await new Promise((resolve, reject) => { - this.download = new XMLHttpRequest(); - this.download.open('GET', site.asset.url(models.get(this.lang)!, { version: false }), true); - this.download.responseType = 'arraybuffer'; - this.download.onerror = _ => reject('Failed. See console'); - this.download.onabort = _ => reject('Aborted'); - this.download.onprogress = (e: ProgressEvent) => { - this.broadcast( - e.total <= 0 - ? 'Downloading...' - : `Downloaded ${Math.round((100 * e.loaded) / e.total)}% of ${Math.round(e.total / 1000000)}MB`, - ); - }; - - this.download.onload = _ => { - if (this.download?.status !== 200) reject(`${this.download?.status} Failed`); - else resolve(this.download?.response); - }; - this.download.send(); - }); - this.broadcast('Extracting...'); - const now = new Date(); - await voskStore.put(emscriptenPath, { timestamp: now, mode: 16877 }); - await voskStore.put(`${emscriptenPath}/downloaded.ok`, { - contents: new Uint8Array([]), - timestamp: now, - mode: 33206, - }); - await voskStore.remove(`${emscriptenPath}/downloaded.tar.gz`); - await voskStore.put(`${emscriptenPath}/downloaded.tar.gz`, { - contents: new Uint8Array(modelBlob!), - timestamp: now, - mode: 33188, - }); - voskStore.txn('readwrite').objectStore('FILE_DATA').index('timestamp'); - } -})(); + /*private*/ broadcast(text: string, msgType: Voice.MsgType = 'status', forMs = 0) { + this.ctrl?.call(this, text, msgType); + if (msgType === 'status' || msgType === 'full') window.clearTimeout(this.broadcastTimeout); + this.voskStatus = text; + for (const li of this.recs.get(this.recId)?.listeners ?? []) { + if (!this.interrupt) li(text, msgType); + } + this.interrupt = false; + this.broadcastTimeout = forMs > 0 ? window.setTimeout(() => this.broadcast(''), forMs) : undefined; + } + + /*private*/ async downloadModel(emscriptenPath: string): Promise { + const voskStore = await objectStorage({ + db: '/vosk', + store: 'FILE_DATA', + version: 21, + upgrade: (_, idbStore?: IDBObjectStore) => { + // make emscripten fs happy + idbStore?.createIndex('timestamp', 'timestamp', { unique: false }); + }, + }); + if ((await voskStore.count(`${emscriptenPath}/extracted.ok`)) > 0) return; + const modelBlob: ArrayBuffer | undefined = await new Promise((resolve, reject) => { + this.download = new XMLHttpRequest(); + this.download.open('GET', site.asset.url(models.get(this.lang)!, { version: false }), true); + this.download.responseType = 'arraybuffer'; + this.download.onerror = _ => reject('Failed. See console'); + this.download.onabort = _ => reject('Aborted'); + this.download.onprogress = (e: ProgressEvent) => { + this.broadcast( + e.total <= 0 + ? 'Downloading...' + : `Downloaded ${Math.round((100 * e.loaded) / e.total)}% of ${Math.round(e.total / 1000000)}MB`, + ); + }; + + this.download.onload = _ => { + if (this.download?.status !== 200) reject(`${this.download?.status} Failed`); + else resolve(this.download?.response); + }; + this.download.send(); + }); + this.broadcast('Extracting...'); + const now = new Date(); + await voskStore.put(emscriptenPath, { timestamp: now, mode: 16877 }); + await voskStore.put(`${emscriptenPath}/downloaded.ok`, { + contents: new Uint8Array([]), + timestamp: now, + mode: 33206, + }); + await voskStore.remove(`${emscriptenPath}/downloaded.tar.gz`); + await voskStore.put(`${emscriptenPath}/downloaded.tar.gz`, { + contents: new Uint8Array(modelBlob!), + timestamp: now, + mode: 33188, + }); + voskStore.txn('readwrite').objectStore('FILE_DATA').index('timestamp'); + } + })(); +} const models = new Map([ ['ca', 'lifat/vosk/model-ca-0.4.tar.gz'], diff --git a/ui/site/src/site.ts b/ui/site/src/site.ts index 63dc21990d558..46ed4a38f2269 100644 --- a/ui/site/src/site.ts +++ b/ui/site/src/site.ts @@ -15,7 +15,6 @@ import { unload, redirect, reload } from './reload'; import announce from './announce'; import { trans } from './trans'; import sound from './sound'; -import { mic } from './mic'; import * as miniBoard from 'common/miniBoard'; import * as miniGame from './miniGame'; import { format as timeago, formatter as dateFormat, displayLocale } from './timeago'; @@ -46,7 +45,6 @@ s.watchers = watchers; s.announce = announce; s.trans = trans; s.sound = sound; -s.mic = mic; s.miniBoard = miniBoard; s.miniGame = miniGame; s.timeago = timeago;