From c5e3363621e040e1a3a338066d49da7437646e5a Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sun, 4 Aug 2024 11:04:13 -0500 Subject: [PATCH 1/2] temp save --- src/lib/common/AudioGallery.svelte | 5 ++- src/lib/common/MessageFileGallery.svelte | 4 +++ .../common/audio-player/AudioPlayer.svelte | 7 +++- .../common/audio-player/AudioSpeaker.svelte | 15 +++++++-- src/lib/common/audio-player/store.js | 32 ++++++++++++++----- src/lib/helpers/types/types.js | 8 ++++- src/routes/+layout.svelte | 1 + .../[conversationId]/chat-box.svelte | 7 +++- .../[conversationId]/conv-dialogs.svelte | 1 + 9 files changed, 65 insertions(+), 15 deletions(-) diff --git a/src/lib/common/AudioGallery.svelte b/src/lib/common/AudioGallery.svelte index 864a71a6..4f7a50d3 100644 --- a/src/lib/common/AudioGallery.svelte +++ b/src/lib/common/AudioGallery.svelte @@ -4,6 +4,9 @@ /** @type {import('$types').AudioFileModel[]} */ export let audios = []; + /** @type {string} */ + export let id; + /** @type {string} */ export let containerClasses = ""; @@ -20,6 +23,6 @@ class="{disableDefaultStyles ? '' : 'audio-gallery-list'} {containerClasses}" style={`${containerStyles}`} > - + {/if} diff --git a/src/lib/common/MessageFileGallery.svelte b/src/lib/common/MessageFileGallery.svelte index a91733c1..92caef5d 100644 --- a/src/lib/common/MessageFileGallery.svelte +++ b/src/lib/common/MessageFileGallery.svelte @@ -13,6 +13,9 @@ /** @type {string} */ export let galleryStyles = ''; + /** @type {string} */ + export let messageId; + /** @type {() => Promise} */ export let fetchFiles = () => Promise.resolve([]); @@ -56,6 +59,7 @@ files={textFiles} /> { dispatch("destroy"); + clearAudioInstantce(id); }); const init = () => { const audioPlayer = document.createElement("audio"); - initPlayer(audioPlayer, dispatch); + initPlayer({ id: id, player: audioPlayer }, dispatch); isShowList = !propsBool($$props, "list_folded") && $audioList.length > 1; volume = Math.max(volume, 0); volume = Math.min(volume, 1); diff --git a/src/lib/common/audio-player/AudioSpeaker.svelte b/src/lib/common/audio-player/AudioSpeaker.svelte index 12c579a3..b7bdac91 100644 --- a/src/lib/common/audio-player/AudioSpeaker.svelte +++ b/src/lib/common/audio-player/AudioSpeaker.svelte @@ -1,10 +1,13 @@ diff --git a/src/lib/common/audio-player/store.js b/src/lib/common/audio-player/store.js index 0a10d335..147241c6 100644 --- a/src/lib/common/audio-player/store.js +++ b/src/lib/common/audio-player/store.js @@ -2,19 +2,19 @@ import { derived, writable } from "svelte/store"; import { secondToTime } from "./utils"; import { SPEECH_VOICES } from "$lib/services/web-speech"; -/** @type {HTMLAudioElement[]} */ -export const instances = []; +/** @type {import('$types').AudioModel[]} */ +export const audioInstances = []; /** @type {import('$types').SpeechModel[]} */ export const speechInstances = []; /** - * @param {HTMLAudioElement} player + * @param {import('$types').AudioModel} audio * @param {(name: string, detail?: any) => void} dispatch */ -export function initPlayer(player, dispatch) { - instances.push(player); - bindAudioEvent(player, dispatch); +export function initPlayer(audio, dispatch) { + audioInstances.push(audio); + bindAudioEvent(audio.player, dispatch); } /** @param {import('$types').SpeechModel} speech */ @@ -26,9 +26,25 @@ export function initSpeech(speech) { speechInstances.push(speech); } +/** @param {string} id */ +export function clearAudioInstantce(id) { + const foundAudioIdx = audioInstances.findIndex(x => x.id === id); + if (foundAudioIdx > -1) { + audioInstances.splice(foundAudioIdx); + } +} + +/** @param {string} id */ +export function clearSpeakerInstantce(id) { + const foundSpeechIdx = speechInstances.findIndex(x => x.id === id); + if (foundSpeechIdx > -1) { + speechInstances.splice(foundSpeechIdx); + } +} + export function stopAll() { - if (instances?.length > 0) { - instances.forEach(player => player.pause()); + if (audioInstances?.length > 0) { + audioInstances.forEach(audio => audio.player?.pause()); } if (speechInstances?.length > 0) { speechInstances.forEach(sp => sp.stop()); diff --git a/src/lib/helpers/types/types.js b/src/lib/helpers/types/types.js index cad87173..a315d178 100644 --- a/src/lib/helpers/types/types.js +++ b/src/lib/helpers/types/types.js @@ -236,12 +236,18 @@ // Speech /** * @typedef {Object} SpeechModel - * @property {string} [id] + * @property {string} id * @property {SpeechSynthesis} synth * @property {SpeechSynthesisUtterance} utterThis * @property {() => void} stop */ +/** + * @typedef {Object} AudioModel + * @property {string} id + * @property {HTMLAudioElement} player + */ + /** * @interface diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 09d5beea..678b4039 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -17,6 +17,7 @@ */ let isLoading; onMount(() => { + window?.speechSynthesis?.cancel(); const subscribe = loaderStore.subscribe(value => { isLoading = value; }); diff --git a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte index 5697170b..33765455 100644 --- a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte +++ b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte @@ -1084,6 +1084,7 @@ {/if} {#if !!message.is_chat_message || !!message.has_message_files} getConversationFiles(params.conversationId, message.message_id, FileSourceType.User)} /> @@ -1111,9 +1112,13 @@
- + {#if !!message.is_chat_message || !!message.has_message_files} getConversationFiles(params.conversationId, message.message_id, FileSourceType.Bot)} /> diff --git a/src/routes/page/conversation/[conversationId]/conv-dialogs.svelte b/src/routes/page/conversation/[conversationId]/conv-dialogs.svelte index 614303a6..250f0664 100644 --- a/src/routes/page/conversation/[conversationId]/conv-dialogs.svelte +++ b/src/routes/page/conversation/[conversationId]/conv-dialogs.svelte @@ -71,6 +71,7 @@

{#if !!dialog.has_message_files} getConversationFiles(conversation.id, dialog.message_id, showInRight(dialog) ? FileSourceType.User : FileSourceType.Bot)} /> From 4cd74f664be13ccdccb262aec41e4afdb1315af9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sun, 4 Aug 2024 23:06:14 -0500 Subject: [PATCH 2/2] refine speaker and audio ondestroy --- .../common/audio-player/AudioSpeaker.svelte | 8 ++++++-- src/lib/common/audio-player/store.js | 18 ++++++++++++------ src/lib/helpers/types/types.js | 1 + .../[agentId]/[conversationId]/chat-box.svelte | 2 +- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/lib/common/audio-player/AudioSpeaker.svelte b/src/lib/common/audio-player/AudioSpeaker.svelte index b7bdac91..601f6f6b 100644 --- a/src/lib/common/audio-player/AudioSpeaker.svelte +++ b/src/lib/common/audio-player/AudioSpeaker.svelte @@ -36,7 +36,8 @@ id: id, synth: window?.speechSynthesis, utterThis: utterThis, - stop: () => stop() + stop: () => stop(), + isSpeaking: () => isSpeaking() }; initSpeech(speech); }); @@ -68,8 +69,11 @@ } } + const isSpeaking = () => { + return speaking; + } + onDestroy(() => { - console.log('inside audio speaker ondestroy ', id); clearSpeakerInstantce(id); }); diff --git a/src/lib/common/audio-player/store.js b/src/lib/common/audio-player/store.js index 147241c6..6a308bcc 100644 --- a/src/lib/common/audio-player/store.js +++ b/src/lib/common/audio-player/store.js @@ -28,17 +28,23 @@ export function initSpeech(speech) { /** @param {string} id */ export function clearAudioInstantce(id) { - const foundAudioIdx = audioInstances.findIndex(x => x.id === id); - if (foundAudioIdx > -1) { - audioInstances.splice(foundAudioIdx); + const foundIdx = audioInstances.findIndex(x => x.id === id); + if (foundIdx > -1) { + if (!audioInstances[foundIdx].player?.paused) { + audioInstances[foundIdx].player?.pause(); + } + audioInstances.splice(foundIdx, 1); } } /** @param {string} id */ export function clearSpeakerInstantce(id) { - const foundSpeechIdx = speechInstances.findIndex(x => x.id === id); - if (foundSpeechIdx > -1) { - speechInstances.splice(foundSpeechIdx); + const foundIdx = speechInstances.findIndex(x => x.id === id); + if (foundIdx > -1) { + if (speechInstances[foundIdx].isSpeaking()) { + speechInstances[foundIdx].stop(); + } + speechInstances.splice(foundIdx, 1); } } diff --git a/src/lib/helpers/types/types.js b/src/lib/helpers/types/types.js index a315d178..46ab37c0 100644 --- a/src/lib/helpers/types/types.js +++ b/src/lib/helpers/types/types.js @@ -240,6 +240,7 @@ * @property {SpeechSynthesis} synth * @property {SpeechSynthesisUtterance} utterThis * @property {() => void} stop + * @property {() => boolean} isSpeaking */ /** diff --git a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte index 33765455..b546d9cb 100644 --- a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte +++ b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte @@ -601,7 +601,7 @@ * @param {any[]} files */ function buildFilePayload(files) { - if (!files) return ''; + if (!files || files.length === 0) return ''; const excelCount = files.filter(x => isExcel(x.file_type || x.file_name)).length; const pdfCount = files.filter(x => isPdf(x.file_type || x.file_name)).length;