From a7d1f0014f98ffbe184c6cd843fd173f26a2fb10 Mon Sep 17 00:00:00 2001 From: Jared Drueco Date: Sun, 7 Jan 2024 01:36:55 -0700 Subject: [PATCH 1/5] bugfix: double mount render --- frontend/src/views/patient/PatientHome.jsx | 24 +++++++++---------- .../views/patient/components/Exercises.jsx | 8 +++---- .../src/views/patient/components/VoiceAI.jsx | 15 ++++++------ 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/frontend/src/views/patient/PatientHome.jsx b/frontend/src/views/patient/PatientHome.jsx index 5af51b2..75235c2 100644 --- a/frontend/src/views/patient/PatientHome.jsx +++ b/frontend/src/views/patient/PatientHome.jsx @@ -1,9 +1,6 @@ -import Navbar from "./components/Navbar"; -import RecordButton from "./components/RecordButton"; -// import Ai3D from './components/Ai3D'; -import Conversation from './components/Conversation'; +import Navbar from './components/Navbar'; import Exercises from './components/Exercises'; -import "./styles.css"; +import './styles.css'; import { useState, useEffect, useCallback } from 'react'; import VoiceAI from './components/VoiceAI'; import axios from 'axios'; @@ -33,13 +30,18 @@ const PatientHome = () => { const response = await axios.get( `http://localhost:8080/conversation/start?${queryParams.toString()}` ); - setConvo((prevConvo) => ({ ...prevConvo, gpt: response.data.reply })); + setConvo((prevConvo) => { + if (prevConvo.gpt === null) { + return { ...prevConvo, gpt: response.data.reply }; + } + return prevConvo; + }); } catch (error) { console.error('Error fetching conversation start:', error); } }; startConversation(); - }, []); + }, []); return (
@@ -54,11 +56,9 @@ const PatientHome = () => { {/* */}

{convo.user}

-

- {convo.gpt !== null - ? convo.gpt - : } -

+
+ {convo.gpt !== null ? convo.gpt : } +
{ instructions: ["Step 1 for Card 5", "Step 2 for Card 5", "Step 3 for Card 5"] }, { - id: 5, - title: "Card 5", + id: 6, + title: "Card 6", description: "Description for Card 5", imageUrl: glutesImage, // Placeholder image instructions: ["Step 1 for Card 5", "Step 2 for Card 5", "Step 3 for Card 5"] }, { - id: 5, - title: "Card 5", + id: 7, + title: "Card 7", description: "Description for Card 5", imageUrl: glutesImage, // Placeholder image instructions: ["Step 1 for Card 5", "Step 2 for Card 5", "Step 3 for Card 5"] diff --git a/frontend/src/views/patient/components/VoiceAI.jsx b/frontend/src/views/patient/components/VoiceAI.jsx index e53ef72..f16173c 100644 --- a/frontend/src/views/patient/components/VoiceAI.jsx +++ b/frontend/src/views/patient/components/VoiceAI.jsx @@ -6,7 +6,6 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { const [isRecording, setIsRecording] = useState(false); const [mediaStream, setMediaStream] = useState(null); const [mediaRecorder, setMediaRecorder] = useState(null); - const [isConvoStarted, setIsConvoStarted] = useState(false); const [speechRecognition, setSpeechRecognition] = useState(null); useEffect(() => { @@ -31,13 +30,13 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { const startRecording = async () => { const queryParams = new URLSearchParams({ patient: 'demo', practitioner: 'demo' }); - if (!isConvoStarted) { - // Start a new conversation - const gptResponse = await axios.get(`http://localhost:8080/conversation/start?${queryParams.toString()}`); - setIsConvoStarted(true); - console.log(gptResponse.data.reply); // TODO: speak/display the AI response here - updateGptResponse(gptResponse.data.reply); - } + // if (!isConvoStarted) { + // // Start a new conversation + // const gptResponse = await axios.get(`http://localhost:8080/conversation/start?${queryParams.toString()}`); + // setIsConvoStarted(true); + // console.log(gptResponse.data.reply); // TODO: speak/display the AI response here + // updateGptResponse(gptResponse.data.reply); + // } // Start recording audio const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); From 18f803822611b78d179abd27eb2924b11f4a0e38 Mon Sep 17 00:00:00 2001 From: Jared Drueco Date: Sun, 7 Jan 2024 03:03:40 -0700 Subject: [PATCH 2/5] a real real real real real nice ui change --- backend/server/src/conversation/views.py | 16 +---- frontend/src/views/patient/PatientHome.jsx | 70 ++++++++++--------- .../src/views/patient/components/Skeleton.jsx | 4 +- .../src/views/patient/components/VoiceAI.jsx | 37 ++++------ frontend/src/views/patient/styles.css | 18 +++++ 5 files changed, 70 insertions(+), 75 deletions(-) diff --git a/backend/server/src/conversation/views.py b/backend/server/src/conversation/views.py index d95a2ec..769bece 100644 --- a/backend/server/src/conversation/views.py +++ b/backend/server/src/conversation/views.py @@ -28,7 +28,6 @@ def start(): return jsonify({"reply": greeting}), 200 - @conversation_blueprint.route("/send_message", methods=["POST"]) def send_message(): practitioner = request.args.get("practitioner") @@ -38,33 +37,22 @@ def send_message(): ) if "conversation_id" not in session: return jsonify({"reply": "Please start a conversation first"}), 400 - conversation_id = session["conversation_id"] conversation = Conversation( user_doc_ref=user_doc_ref, conversaton_id=conversation_id ) - # Store audio in a temp file - message = request.json.get("message") - - # Generate a reply using the Conversation object - reply = conversation.generate_reply(message) - return jsonify({"reply": reply}), 200 - -@conversation_blueprint.route("/transcribe", methods=["POST"]) -def transcribe(): - # Store audio in a temp file audio = request.files["audioFile"] temp_audio_path = os.path.join(tempfile.gettempdir(), "received_audio.wav") audio.save(temp_audio_path) - # Transcribe the audio message = Conversation.transcribe(str(temp_audio_path)) os.remove(temp_audio_path) # Generate a reply using the Conversation object - return jsonify({"user_msg": message}), 200 + reply = conversation.generate_reply(message) + return jsonify({"reply": reply}), 200 @conversation_blueprint.route('/end', methods=['POST']) def end(): diff --git a/frontend/src/views/patient/PatientHome.jsx b/frontend/src/views/patient/PatientHome.jsx index 75235c2..836f995 100644 --- a/frontend/src/views/patient/PatientHome.jsx +++ b/frontend/src/views/patient/PatientHome.jsx @@ -41,47 +41,49 @@ const PatientHome = () => { } }; startConversation(); - }, []); + }, []); return ( -
- -
-
-
-
-

Welcome Back

-
John
-
- {/* */} -
-

{convo.user}

-
- {convo.gpt !== null ? convo.gpt : } +
+
+ {/* */} +
+
+
+
+

Welcome Back

+
John
+
+ {/* */} +
+

{convo.user}

+
+ {convo.gpt !== null ? convo.gpt : } +
+ + + + +
+
+
+
-
- - -
-
-
-
-
+
+
+
-
-
- + {/* TODO: finish button that calls conversation/end */}
- {/* TODO: finish button that calls conversation/end */}
); }; diff --git a/frontend/src/views/patient/components/Skeleton.jsx b/frontend/src/views/patient/components/Skeleton.jsx index f196edb..5c3ff41 100644 --- a/frontend/src/views/patient/components/Skeleton.jsx +++ b/frontend/src/views/patient/components/Skeleton.jsx @@ -1,9 +1,9 @@ const Skeleton = () => { return ( -
-
+
+
); }; diff --git a/frontend/src/views/patient/components/VoiceAI.jsx b/frontend/src/views/patient/components/VoiceAI.jsx index f16173c..eec0385 100644 --- a/frontend/src/views/patient/components/VoiceAI.jsx +++ b/frontend/src/views/patient/components/VoiceAI.jsx @@ -15,11 +15,14 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { recognition.continuous = true; recognition.interimResults = true; + let accumulatedTranscript = ''; + recognition.onresult = (event) => { - // Only use this for real-time display, not for sending to the server - const latestResult = event.results[event.resultIndex]; - const latestTranscript = latestResult[0].transcript.trim(); - updateUserMessage(latestTranscript); + accumulatedTranscript = ''; + for (let i = 0; i < event.results.length; i++) { + accumulatedTranscript += event.results[i][0].transcript.trim() + ' '; + } + updateUserMessage(accumulatedTranscript); }; setSpeechRecognition(recognition); @@ -30,13 +33,6 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { const startRecording = async () => { const queryParams = new URLSearchParams({ patient: 'demo', practitioner: 'demo' }); - // if (!isConvoStarted) { - // // Start a new conversation - // const gptResponse = await axios.get(`http://localhost:8080/conversation/start?${queryParams.toString()}`); - // setIsConvoStarted(true); - // console.log(gptResponse.data.reply); // TODO: speak/display the AI response here - // updateGptResponse(gptResponse.data.reply); - // } // Start recording audio const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); @@ -50,21 +46,17 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { }; recorder.onstop = async () => { + updateGptResponse(null); // Process and send the audio data to the server for transcription const audioBlob = new Blob(chunks, { type: 'audio/wav' }); const formData = new FormData(); formData.append('audioFile', audioBlob, 'recorded_audio.wav'); - const userMessage = await axios.post(`http://localhost:8080/conversation/transcribe`, formData); - updateUserMessage(userMessage.data.user_msg); // Update with the final, reliable transcription - console.log(userMessage); - - // Fetch GPT response - const gptResponse = await axios.post( + const response = await axios.post( `http://localhost:8080/conversation/send_message?${queryParams.toString()}`, - { "message": userMessage.data.user_msg } + formData ); - updateGptResponse(gptResponse.data.reply); + updateGptResponse(response.data.reply); }; recorder.start(); @@ -101,14 +93,9 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { return (
- {isRecording ? ( -

Recording...

- ) : ( -

Click Start Recording to begin recording.

- )}
-
-
+
diff --git a/frontend/src/views/patient/components/VoiceAI.jsx b/frontend/src/views/patient/components/VoiceAI.jsx index eec0385..acb1682 100644 --- a/frontend/src/views/patient/components/VoiceAI.jsx +++ b/frontend/src/views/patient/components/VoiceAI.jsx @@ -1,5 +1,9 @@ import { useState, useEffect, useRef } from 'react'; import axios from 'axios'; +import gsap from 'gsap'; +import React, { Suspense } from 'react'; + +const Spline = React.lazy(() => import('@splinetool/react-spline')); const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { const sphere = useRef(); @@ -9,7 +13,8 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { const [speechRecognition, setSpeechRecognition] = useState(null); useEffect(() => { - const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + const SpeechRecognition = + window.SpeechRecognition || window.webkitSpeechRecognition; if (SpeechRecognition) { const recognition = new SpeechRecognition(); recognition.continuous = true; @@ -27,12 +32,15 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { setSpeechRecognition(recognition); } else { - console.warn("Speech recognition not supported in this browser."); + console.warn('Speech recognition not supported in this browser.'); } }, [updateUserMessage]); const startRecording = async () => { - const queryParams = new URLSearchParams({ patient: 'demo', practitioner: 'demo' }); + const queryParams = new URLSearchParams({ + patient: 'demo', + practitioner: 'demo', + }); // Start recording audio const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); @@ -76,33 +84,58 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { }; function onLoad(spline) { - spline.setZoom(1); - const obj = spline.findObjectById('ec9f2de1-4a48-4948-a32f-653838ab50ec'); - sphere.current = obj + spline.setZoom(0.1); + const obj = spline.findObjectById('f5f3b334-53b6-4337-8497-c6815ba02c98'); + sphere.current = obj; } const triggerStart = () => { startRecording(); - // sphere.current.emitEvent('start', 'Sphere'); - } + console.log(sphere.current.scale); + gsap.to(sphere.current.scale, { + duration: 3, + x: 1.5, + y: 1.5, + z: 1.5, + ease: 'power3.out', + }); + }; const triggerEnd = () => { stopRecording(); - // sphere.current.emitEvent('mouseHover', 'Sphere'); - } + gsap.to(sphere.current.scale, { + duration: 2, + x: 1, + y: 1, + z: 1, + ease: 'power3.out', + }); + }; return (
}> + {/* */} + {/* */} + */} + scene="https://prod.spline.design/Omn4EqepHAUv5XKP/scene.splinecode" + /> +
); From 8919d6c6b7a42603a18326a9dea8c08fd6e35fdf Mon Sep 17 00:00:00 2001 From: Jared Drueco Date: Sun, 7 Jan 2024 04:49:46 -0700 Subject: [PATCH 4/5] end session --- frontend/src/views/patient/PatientHome.jsx | 32 ++++++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/patient/PatientHome.jsx b/frontend/src/views/patient/PatientHome.jsx index 7ccda0b..6118f00 100644 --- a/frontend/src/views/patient/PatientHome.jsx +++ b/frontend/src/views/patient/PatientHome.jsx @@ -5,8 +5,11 @@ import { useState, useEffect, useCallback } from 'react'; import VoiceAI from './components/VoiceAI'; import axios from 'axios'; import Skeleton from './components/Skeleton'; +import { LogOut } from 'lucide-react'; +import { useNavigate } from 'react-router-dom'; const PatientHome = () => { + const navigate = useNavigate(); const [convo, setConvo] = useState({ user: null, gpt: null, @@ -43,6 +46,21 @@ const PatientHome = () => { startConversation(); }, []); + const handleEndSession = async () => { + try { + await axios.post('http://localhost:8080/conversation/end', {}, { + // TODO: what are thooooose + params: new URLSearchParams({ + patient: 'demo', + practitioner: 'demo', + }) + }); + navigate('/') + } catch (error) { + console.error('Error ending conversation:', error); + } + }; + return (
@@ -50,9 +68,17 @@ const PatientHome = () => {
-
-

Welcome Back

-
John
+
+
+

Welcome Back

+
John
+
+
{/* */}
From 47ada2a7cc8caec90e188f290916d7fd604ffabb Mon Sep 17 00:00:00 2001 From: Jared Drueco Date: Sun, 7 Jan 2024 06:36:26 -0700 Subject: [PATCH 5/5] fix voice model size --- frontend/src/views/patient/components/VoiceAI.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/patient/components/VoiceAI.jsx b/frontend/src/views/patient/components/VoiceAI.jsx index acb1682..b277e62 100644 --- a/frontend/src/views/patient/components/VoiceAI.jsx +++ b/frontend/src/views/patient/components/VoiceAI.jsx @@ -116,14 +116,14 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => {
}> +
}> {/*