From c5e3363621e040e1a3a338066d49da7437646e5a Mon Sep 17 00:00:00 2001
From: Jicheng Lu <franklujc@gmail.com>
Date: Sun, 4 Aug 2024 11:04:13 -0500
Subject: [PATCH 1/2] temp save
---
src/lib/common/AudioGallery.svelte | 5 ++-
src/lib/common/MessageFileGallery.svelte | 4 +++
.../common/audio-player/AudioPlayer.svelte | 7 +++-
.../common/audio-player/AudioSpeaker.svelte | 15 +++++++--
src/lib/common/audio-player/store.js | 32 ++++++++++++++-----
src/lib/helpers/types/types.js | 8 ++++-
src/routes/+layout.svelte | 1 +
.../[conversationId]/chat-box.svelte | 7 +++-
.../[conversationId]/conv-dialogs.svelte | 1 +
9 files changed, 65 insertions(+), 15 deletions(-)
diff --git a/src/lib/common/AudioGallery.svelte b/src/lib/common/AudioGallery.svelte
index 864a71a6..4f7a50d3 100644
--- a/src/lib/common/AudioGallery.svelte
+++ b/src/lib/common/AudioGallery.svelte
@@ -4,6 +4,9 @@
/** @type {import('$types').AudioFileModel[]} */
export let audios = [];
+ /** @type {string} */
+ export let id;
+
/** @type {string} */
export let containerClasses = "";
@@ -20,6 +23,6 @@
class="{disableDefaultStyles ? '' : 'audio-gallery-list'} {containerClasses}"
style={`${containerStyles}`}
>
- <AudioPlayer audio={audios} />
+ <AudioPlayer id={id} audio={audios} />
</div>
{/if}
diff --git a/src/lib/common/MessageFileGallery.svelte b/src/lib/common/MessageFileGallery.svelte
index a91733c1..92caef5d 100644
--- a/src/lib/common/MessageFileGallery.svelte
+++ b/src/lib/common/MessageFileGallery.svelte
@@ -13,6 +13,9 @@
/** @type {string} */
export let galleryStyles = '';
+ /** @type {string} */
+ export let messageId;
+
/** @type {() => Promise<any>} */
export let fetchFiles = () => Promise.resolve([]);
@@ -56,6 +59,7 @@
files={textFiles}
/>
<AudioGallery
+ id={messageId}
containerClasses={galleryClasses}
containerStyles={galleryStyles}
audios={audioFiles}
diff --git a/src/lib/common/audio-player/AudioPlayer.svelte b/src/lib/common/audio-player/AudioPlayer.svelte
index 64ed2e2a..b1820913 100644
--- a/src/lib/common/audio-player/AudioPlayer.svelte
+++ b/src/lib/common/audio-player/AudioPlayer.svelte
@@ -2,6 +2,7 @@
import {
initPlayer,
stopAll,
+ clearAudioInstantce,
useAudioStore
} from "./store";
import { volumeEventHandlers, progressEventHandlers } from "./handlers";
@@ -46,6 +47,9 @@
/** @type {import('$types').AudioFileModel[]} */
export let audio;
+ /** @type {string} */
+ export let id;
+
/** @type {"list" | "random"} */
export let order = 'list';
@@ -182,11 +186,12 @@
onDestroy(() => {
dispatch("destroy");
+ clearAudioInstantce(id);
});
const init = () => {
const audioPlayer = document.createElement("audio");
- initPlayer(audioPlayer, dispatch);
+ initPlayer({ id: id, player: audioPlayer }, dispatch);
isShowList = !propsBool($$props, "list_folded") && $audioList.length > 1;
volume = Math.max(volume, 0);
volume = Math.min(volume, 1);
diff --git a/src/lib/common/audio-player/AudioSpeaker.svelte b/src/lib/common/audio-player/AudioSpeaker.svelte
index 12c579a3..b7bdac91 100644
--- a/src/lib/common/audio-player/AudioSpeaker.svelte
+++ b/src/lib/common/audio-player/AudioSpeaker.svelte
@@ -1,10 +1,13 @@
<script>
- import { initSpeech, stopAll } from "$lib/common/audio-player/store";
- import { onMount } from "svelte";
+ import { initSpeech, stopAll, clearSpeakerInstantce } from "$lib/common/audio-player/store";
+ import { onMount, onDestroy } from "svelte";
/** @type {string} */
export let text;
+ /** @type {string} */
+ export let id;
+
/** @type {boolean} */
export let mutex = true;
@@ -30,6 +33,7 @@
utterThis.onend = (e) => { stop(); };
speech = {
+ id: id,
synth: window?.speechSynthesis,
utterThis: utterThis,
stop: () => stop()
@@ -59,10 +63,15 @@
const stop = () => {
speaking = false;
- if (speech?.synth) {
+ if (speech?.synth && speech.synth.speaking) {
speech.synth.cancel();
}
}
+
+ onDestroy(() => {
+ console.log('inside audio speaker ondestroy ', id);
+ clearSpeakerInstantce(id);
+ });
</script>
<!-- svelte-ignore a11y-click-events-have-key-events -->
diff --git a/src/lib/common/audio-player/store.js b/src/lib/common/audio-player/store.js
index 0a10d335..147241c6 100644
--- a/src/lib/common/audio-player/store.js
+++ b/src/lib/common/audio-player/store.js
@@ -2,19 +2,19 @@ import { derived, writable } from "svelte/store";
import { secondToTime } from "./utils";
import { SPEECH_VOICES } from "$lib/services/web-speech";
-/** @type {HTMLAudioElement[]} */
-export const instances = [];
+/** @type {import('$types').AudioModel[]} */
+export const audioInstances = [];
/** @type {import('$types').SpeechModel[]} */
export const speechInstances = [];
/**
- * @param {HTMLAudioElement} player
+ * @param {import('$types').AudioModel} audio
* @param {(name: string, detail?: any) => void} dispatch
*/
-export function initPlayer(player, dispatch) {
- instances.push(player);
- bindAudioEvent(player, dispatch);
+export function initPlayer(audio, dispatch) {
+ audioInstances.push(audio);
+ bindAudioEvent(audio.player, dispatch);
}
/** @param {import('$types').SpeechModel} speech */
@@ -26,9 +26,25 @@ export function initSpeech(speech) {
speechInstances.push(speech);
}
+/** @param {string} id */
+export function clearAudioInstantce(id) {
+ const foundAudioIdx = audioInstances.findIndex(x => x.id === id);
+ if (foundAudioIdx > -1) {
+ audioInstances.splice(foundAudioIdx);
+ }
+}
+
+/** @param {string} id */
+export function clearSpeakerInstantce(id) {
+ const foundSpeechIdx = speechInstances.findIndex(x => x.id === id);
+ if (foundSpeechIdx > -1) {
+ speechInstances.splice(foundSpeechIdx);
+ }
+}
+
export function stopAll() {
- if (instances?.length > 0) {
- instances.forEach(player => player.pause());
+ if (audioInstances?.length > 0) {
+ audioInstances.forEach(audio => audio.player?.pause());
}
if (speechInstances?.length > 0) {
speechInstances.forEach(sp => sp.stop());
diff --git a/src/lib/helpers/types/types.js b/src/lib/helpers/types/types.js
index cad87173..a315d178 100644
--- a/src/lib/helpers/types/types.js
+++ b/src/lib/helpers/types/types.js
@@ -236,12 +236,18 @@
// Speech
/**
* @typedef {Object} SpeechModel
- * @property {string} [id]
+ * @property {string} id
* @property {SpeechSynthesis} synth
* @property {SpeechSynthesisUtterance} utterThis
* @property {() => void} stop
*/
+/**
+ * @typedef {Object} AudioModel
+ * @property {string} id
+ * @property {HTMLAudioElement} player
+ */
+
/**
* @interface
diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte
index 09d5beea..678b4039 100644
--- a/src/routes/+layout.svelte
+++ b/src/routes/+layout.svelte
@@ -17,6 +17,7 @@
*/
let isLoading;
onMount(() => {
+ window?.speechSynthesis?.cancel();
const subscribe = loaderStore.subscribe(value => {
isLoading = value;
});
diff --git a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte
index 5697170b..33765455 100644
--- a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte
+++ b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte
@@ -1084,6 +1084,7 @@
{/if}
{#if !!message.is_chat_message || !!message.has_message_files}
<MessageFileGallery
+ messageId={message?.message_id}
galleryStyles={'justify-content: flex-end;'}
fetchFiles={() => getConversationFiles(params.conversationId, message.message_id, FileSourceType.User)}
/>
@@ -1111,9 +1112,13 @@
</div>
<div class="msg-container">
<RcMessage message={message} />
- <AudioSpeaker text={message?.rich_content?.message?.text || message?.text} />
+ <AudioSpeaker
+ id={message?.message_id}
+ text={message?.rich_content?.message?.text || message?.text}
+ />
{#if !!message.is_chat_message || !!message.has_message_files}
<MessageFileGallery
+ messageId={message?.message_id}
galleryStyles={'justify-content: flex-start;'}
fetchFiles={() => getConversationFiles(params.conversationId, message.message_id, FileSourceType.Bot)}
/>
diff --git a/src/routes/page/conversation/[conversationId]/conv-dialogs.svelte b/src/routes/page/conversation/[conversationId]/conv-dialogs.svelte
index 614303a6..250f0664 100644
--- a/src/routes/page/conversation/[conversationId]/conv-dialogs.svelte
+++ b/src/routes/page/conversation/[conversationId]/conv-dialogs.svelte
@@ -71,6 +71,7 @@
</p>
{#if !!dialog.has_message_files}
<MessageFileGallery
+ messageId={dialog?.message_id}
galleryClasses={'dialog-file-display'}
fetchFiles={() => getConversationFiles(conversation.id, dialog.message_id, showInRight(dialog) ? FileSourceType.User : FileSourceType.Bot)}
/>
From 4cd74f664be13ccdccb262aec41e4afdb1315af9 Mon Sep 17 00:00:00 2001
From: Jicheng Lu <franklujc@gmail.com>
Date: Sun, 4 Aug 2024 23:06:14 -0500
Subject: [PATCH 2/2] refine speaker and audio ondestroy
---
.../common/audio-player/AudioSpeaker.svelte | 8 ++++++--
src/lib/common/audio-player/store.js | 18 ++++++++++++------
src/lib/helpers/types/types.js | 1 +
.../[agentId]/[conversationId]/chat-box.svelte | 2 +-
4 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/src/lib/common/audio-player/AudioSpeaker.svelte b/src/lib/common/audio-player/AudioSpeaker.svelte
index b7bdac91..601f6f6b 100644
--- a/src/lib/common/audio-player/AudioSpeaker.svelte
+++ b/src/lib/common/audio-player/AudioSpeaker.svelte
@@ -36,7 +36,8 @@
id: id,
synth: window?.speechSynthesis,
utterThis: utterThis,
- stop: () => stop()
+ stop: () => stop(),
+ isSpeaking: () => isSpeaking()
};
initSpeech(speech);
});
@@ -68,8 +69,11 @@
}
}
+ const isSpeaking = () => {
+ return speaking;
+ }
+
onDestroy(() => {
- console.log('inside audio speaker ondestroy ', id);
clearSpeakerInstantce(id);
});
</script>
diff --git a/src/lib/common/audio-player/store.js b/src/lib/common/audio-player/store.js
index 147241c6..6a308bcc 100644
--- a/src/lib/common/audio-player/store.js
+++ b/src/lib/common/audio-player/store.js
@@ -28,17 +28,23 @@ export function initSpeech(speech) {
/** @param {string} id */
export function clearAudioInstantce(id) {
- const foundAudioIdx = audioInstances.findIndex(x => x.id === id);
- if (foundAudioIdx > -1) {
- audioInstances.splice(foundAudioIdx);
+ const foundIdx = audioInstances.findIndex(x => x.id === id);
+ if (foundIdx > -1) {
+ if (!audioInstances[foundIdx].player?.paused) {
+ audioInstances[foundIdx].player?.pause();
+ }
+ audioInstances.splice(foundIdx, 1);
}
}
/** @param {string} id */
export function clearSpeakerInstantce(id) {
- const foundSpeechIdx = speechInstances.findIndex(x => x.id === id);
- if (foundSpeechIdx > -1) {
- speechInstances.splice(foundSpeechIdx);
+ const foundIdx = speechInstances.findIndex(x => x.id === id);
+ if (foundIdx > -1) {
+ if (speechInstances[foundIdx].isSpeaking()) {
+ speechInstances[foundIdx].stop();
+ }
+ speechInstances.splice(foundIdx, 1);
}
}
diff --git a/src/lib/helpers/types/types.js b/src/lib/helpers/types/types.js
index a315d178..46ab37c0 100644
--- a/src/lib/helpers/types/types.js
+++ b/src/lib/helpers/types/types.js
@@ -240,6 +240,7 @@
* @property {SpeechSynthesis} synth
* @property {SpeechSynthesisUtterance} utterThis
* @property {() => void} stop
+ * @property {() => boolean} isSpeaking
*/
/**
diff --git a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte
index 33765455..b546d9cb 100644
--- a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte
+++ b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte
@@ -601,7 +601,7 @@
* @param {any[]} files
*/
function buildFilePayload(files) {
- if (!files) return '';
+ if (!files || files.length === 0) return '';
const excelCount = files.filter(x => isExcel(x.file_type || x.file_name)).length;
const pdfCount = files.filter(x => isPdf(x.file_type || x.file_name)).length;