From 02db8d07d9f87c488f4e1076c4bea880a81b517b Mon Sep 17 00:00:00 2001 From: Jared Drueco Date: Sat, 6 Jan 2024 22:00:08 -0700 Subject: [PATCH 01/34] show user message on patient home page --- backend/server/src/conversation/views.py | 13 +- frontend/package-lock.json | 34 +++- frontend/package.json | 3 +- frontend/src/views/patient/PatientHome.jsx | 52 ++++-- .../src/views/patient/components/Ai3D.jsx | 17 -- .../views/patient/components/Conversation.jsx | 4 +- .../views/patient/components/ExerciseCard.jsx | 11 +- .../src/views/patient/components/VoiceAI.jsx | 158 ++++++++++++++++++ .../practitioner/PractitionerDashboard.jsx | 2 +- 9 files changed, 240 insertions(+), 54 deletions(-) delete mode 100644 frontend/src/views/patient/components/Ai3D.jsx create mode 100644 frontend/src/views/patient/components/VoiceAI.jsx diff --git a/backend/server/src/conversation/views.py b/backend/server/src/conversation/views.py index 219428d..c3f29b9 100644 --- a/backend/server/src/conversation/views.py +++ b/backend/server/src/conversation/views.py @@ -44,6 +44,16 @@ def send_message(): user_doc_ref=user_doc_ref, conversaton_id=conversation_id ) + # Store audio in a temp file + message = request.json.get("message") + + # Generate a reply using the Conversation object + reply = conversation.generate_reply(message) + return jsonify({"reply": reply}), 200 + +@conversation_blueprint.route("/transcribe", methods=["POST"]) +def transcribe(): + # Store audio in a temp file audio = request.files["audioFile"] temp_audio_path = os.path.join(tempfile.gettempdir(), "received_audio.wav") @@ -54,5 +64,4 @@ def send_message(): os.remove(temp_audio_path) # Generate a reply using the Conversation object - reply = conversation.generate_reply(message) - return jsonify({"reply": reply}), 200 + return jsonify({"user_msg": message}), 200 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c7b5e81..f9f0d66 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -14,7 +14,8 @@ "firebase": "^10.7.1", "react": "^18.2.0", "react-dom": "^18.2.0", - "react-router-dom": "^6.21.1" + "react-router-dom": "^6.21.1", + "typewriter-effect": "^2.21.0" }, "devDependencies": { "@types/react": "^18.2.43", @@ -4352,7 +4353,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -4600,6 +4600,11 @@ "node": "14 || >=16.14" } }, + "node_modules/performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" + }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -4812,7 +4817,6 @@ "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", - "dev": true, "dependencies": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", @@ -4876,6 +4880,14 @@ } ] }, + "node_modules/raf": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/raf/-/raf-3.4.1.tgz", + "integrity": "sha512-Sq4CW4QhwOHE8ucn6J34MqtZCeWFP2aQSmrlroYgqAV1PjStIhJXxYuTgUIfkEk7zTLjmIjLmU5q+fbD1NnOJA==", + "dependencies": { + "performance-now": "^2.1.0" + } + }, "node_modules/react": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", @@ -4902,8 +4914,7 @@ "node_modules/react-is": { "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", - "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", - "dev": true + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" }, "node_modules/react-merge-refs": { "version": "2.1.1", @@ -5717,6 +5728,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typewriter-effect": { + "version": "2.21.0", + "resolved": "https://registry.npmjs.org/typewriter-effect/-/typewriter-effect-2.21.0.tgz", + "integrity": "sha512-Y3VL1fuJpUBj0gS4OTXBLzy1gnYTYaBuVuuO99tGNyTkkub5CXi+b/hsV7Og9fp6HlhogOwWJwgq7iXI5sQlEg==", + "dependencies": { + "prop-types": "^15.8.1", + "raf": "^3.4.1" + }, + "peerDependencies": { + "react": "^17.x || ^18.x", + "react-dom": "^17.x || ^18.x" + } + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 35c9d48..eba7d72 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,7 +16,8 @@ "firebase": "^10.7.1", "react": "^18.2.0", "react-dom": "^18.2.0", - "react-router-dom": "^6.21.1" + "react-router-dom": "^6.21.1", + "typewriter-effect": "^2.21.0" }, "devDependencies": { "@types/react": "^18.2.43", diff --git a/frontend/src/views/patient/PatientHome.jsx b/frontend/src/views/patient/PatientHome.jsx index 48a9de5..205863c 100644 --- a/frontend/src/views/patient/PatientHome.jsx +++ b/frontend/src/views/patient/PatientHome.jsx @@ -1,10 +1,24 @@ -import Navbar from "./components/Navbar"; -import RecordButton from "./components/RecordButton"; -import Ai3D from './components/Ai3D'; +import { useState, useCallback } from 'react'; +import Navbar from './components/Navbar'; import Conversation from './components/Conversation'; import Exercises from './components/Exercises'; +import VoiceAI from './components/VoiceAI'; +import Typewriter from 'typewriter-effect'; const PatientHome = () => { + const [convo, setConvo] = useState({ + user: 'init user msg', + gpt: 'init gpt reply', + }); + + const updateUserMessage = useCallback((newMessage) => { + setConvo((prevConvo) => ({ ...prevConvo, user: newMessage })); + }, []); + + const updateGptResponse = useCallback((newResponse) => { + setConvo((prevConvo) => ({ ...prevConvo, gpt: newResponse })); + }, []); + const messages = [ { sender: 'patient', @@ -27,33 +41,37 @@ const PatientHome = () => { return (
-
+

Welcome Back

John
- -
- - -
+ {/* */} +
+

{convo.user}

+

{convo.gpt}

+
+
+ + +
-
-
- -
+
); diff --git a/frontend/src/views/patient/components/Ai3D.jsx b/frontend/src/views/patient/components/Ai3D.jsx deleted file mode 100644 index 8bd4a10..0000000 --- a/frontend/src/views/patient/components/Ai3D.jsx +++ /dev/null @@ -1,17 +0,0 @@ -import Spline from '@splinetool/react-spline'; - -const Ai3D = () => { - function onLoad(spline) { - spline.setZoom(1); - } - - return ( - - ); -}; - -export default Ai3D; diff --git a/frontend/src/views/patient/components/Conversation.jsx b/frontend/src/views/patient/components/Conversation.jsx index 9760fdf..17bda4f 100644 --- a/frontend/src/views/patient/components/Conversation.jsx +++ b/frontend/src/views/patient/components/Conversation.jsx @@ -4,11 +4,11 @@ const Conversation = ({ messages }) => { const endOfMessagesRef = useRef(null); useEffect(() => { - endOfMessagesRef.current?.scrollIntoView({ behavior: 'smooth' }); + // endOfMessagesRef.current?.scrollIntoView({ behavior: 'smooth' }); }, [messages]); return ( -
+
{messages.map((message, index) => (

{message.text}

diff --git a/frontend/src/views/patient/components/ExerciseCard.jsx b/frontend/src/views/patient/components/ExerciseCard.jsx index 59fb90d..c9be5c0 100644 --- a/frontend/src/views/patient/components/ExerciseCard.jsx +++ b/frontend/src/views/patient/components/ExerciseCard.jsx @@ -1,15 +1,8 @@ - const ExerciseCard = () => { return ( -
-
- Shoes -
+
-

Shoes!

+

Card title!

If a dog chews shoes whose shoes does he choose?

diff --git a/frontend/src/views/patient/components/VoiceAI.jsx b/frontend/src/views/patient/components/VoiceAI.jsx new file mode 100644 index 0000000..d39d1c3 --- /dev/null +++ b/frontend/src/views/patient/components/VoiceAI.jsx @@ -0,0 +1,158 @@ +import { useState, useEffect, useRef } from 'react'; +import Spline from '@splinetool/react-spline'; +import axios from 'axios'; + +const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { + const sphere = useRef(); + const [isRecording, setIsRecording] = useState(false); + const [mediaStream, setMediaStream] = useState(null); + const [mediaRecorder, setMediaRecorder] = useState(null); + const [isConvoStarted, setIsConvoStarted] = useState(false); + + const userPromptRef = useRef(""); + + const [speechRecognition, setSpeechRecognition] = useState(null); + +// useEffect(() => { +// const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; +// if (SpeechRecognition) { +// const recognition = new SpeechRecognition(); +// recognition.continuous = true; +// recognition.interimResults = true; + +// recognition.onresult = (event) => { +// const latestResult = event.results[event.resultIndex]; +// const latestTranscript = latestResult[0].transcript.trim(); + +// if (latestResult.isFinal) { +// userPromptRef.current += ` ${latestTranscript}`; // Update the ref for final results +// } + +// console.log(latestTranscript); // Log each word as it is spoken +// }; + +// setSpeechRecognition(recognition); +// } else { +// console.warn("Speech recognition not supported in this browser."); +// } +// }, [updateUserPrompt]); + + const startRecording = async () => { + // FIXME: use actual IDs for people here + const queryParams = new URLSearchParams(); + queryParams.set('patient', 'demo'); + queryParams.set('practitioner', 'demo'); + + if (!isConvoStarted) { + const response = await axios.get( + `http://localhost:8080/conversation/start?${queryParams.toString()}` + ); + setIsConvoStarted(true); + // TODO: speak/display the AI response here + console.log(response.data.reply); + // updateUserMessage(latestTranscript); + + userPromptRef.current = ""; + speechRecognition?.start(); + } + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const recorder = new MediaRecorder(stream); + const chunks = []; // Array to store audio chunks + + recorder.ondataavailable = (event) => { + if (event.data.size > 0) { + chunks.push(event.data); + } + }; + + recorder.onstop = async () => { + // Combine all audio chunks into a single Blob + const audioBlob = new Blob(chunks, { type: 'audio/wav' }); + + // Create a FormData object and append the audio file + const formData = new FormData(); + formData.append('audioFile', audioBlob, 'recorded_audio.wav'); + + // Send the FormData to the server using a fetch or XMLHttpRequest + const userMessage = await axios.post( + `http://localhost:8080/conversation/transcribe`, + formData + ); + + // update user message display + updateUserMessage(userMessage.data.user_msg) + console.log(userMessage) + + const gptResponse = await axios.post( + `http://localhost:8080/conversation/send_message?${queryParams.toString()}`, + { + "message": userMessage.data.user_msg, + } + ); + + // update + updateGptResponse(gptResponse.data.reply) + + // TODO: speak/display the AI response here + speechRecognition?.stop(); + }; + + recorder.start(); + + setMediaStream(stream); + setMediaRecorder(recorder); + + // Toggle recording state + setIsRecording(true); + }; + + const stopRecording = () => { + if (mediaRecorder && mediaStream) { + mediaRecorder.stop(); + mediaStream.getTracks().forEach((track) => track.stop()); + } + + // Toggle recording state + setIsRecording(false); + }; + + function onLoad(spline) { + spline.setZoom(1); + const obj = spline.findObjectById('ec9f2de1-4a48-4948-a32f-653838ab50ec'); + sphere.current = obj + } + + const triggerStart = () => { + startRecording(); + // sphere.current.emitEvent('start', 'Sphere'); + } + + const triggerEnd = () => { + stopRecording(); + // sphere.current.emitEvent('mouseHover', 'Sphere'); + } + + return ( +
+ {isRecording ? ( +

Recording...

+ ) : ( +

Click Start Recording to begin recording.

+ )} + +
+ ); +}; + +export default VoiceAI; diff --git a/frontend/src/views/practitioner/PractitionerDashboard.jsx b/frontend/src/views/practitioner/PractitionerDashboard.jsx index b694567..3fceb6c 100644 --- a/frontend/src/views/practitioner/PractitionerDashboard.jsx +++ b/frontend/src/views/practitioner/PractitionerDashboard.jsx @@ -4,7 +4,7 @@ const PractitionerDashboard = () => {
Practitioner Dashboard
-
+

Welcome, Practitioner

Access patient records, set exercise routines, and view progress reports. From e6bf59a0e05ea820ae76bd482b96a0b93ab16893 Mon Sep 17 00:00:00 2001 From: Jared Drueco Date: Sat, 6 Jan 2024 23:23:11 -0700 Subject: [PATCH 02/34] faster response for user message --- backend/server/src/conversation/views.py | 15 +++ frontend/.eslintrc.cjs | 1 + frontend/src/views/patient/PatientHome.jsx | 30 +---- .../src/views/patient/components/VoiceAI.jsx | 105 ++++++------------ 4 files changed, 58 insertions(+), 93 deletions(-) diff --git a/backend/server/src/conversation/views.py b/backend/server/src/conversation/views.py index c3f29b9..d95a2ec 100644 --- a/backend/server/src/conversation/views.py +++ b/backend/server/src/conversation/views.py @@ -65,3 +65,18 @@ def transcribe(): # Generate a reply using the Conversation object return jsonify({"user_msg": message}), 200 + +@conversation_blueprint.route('/end', methods=['POST']) +def end(): + if 'conversation_id' not in session: + return jsonify({'reply': 'No Conversation to end'}), 200 + conversation_id = session.pop('conversation_id') + practitioner = request.args.get('practitioner') + patient = request.args.get('patient') + + user_doc_ref = users_ref.document(practitioner).collection("patients").document(patient) + + # Retrieve the Conversation object based on the conversation ID + conversation = Conversation(user_doc_ref, conversaton_id=conversation_id) + summary = conversation.end_conversation() + return jsonify({'reply': summary}), 200 \ No newline at end of file diff --git a/frontend/.eslintrc.cjs b/frontend/.eslintrc.cjs index 4dcb439..cd9ba0d 100644 --- a/frontend/.eslintrc.cjs +++ b/frontend/.eslintrc.cjs @@ -16,5 +16,6 @@ module.exports = { 'warn', { allowConstantExport: true }, ], + "react/prop-types": "off" }, } diff --git a/frontend/src/views/patient/PatientHome.jsx b/frontend/src/views/patient/PatientHome.jsx index 205863c..8299aec 100644 --- a/frontend/src/views/patient/PatientHome.jsx +++ b/frontend/src/views/patient/PatientHome.jsx @@ -7,8 +7,8 @@ import Typewriter from 'typewriter-effect'; const PatientHome = () => { const [convo, setConvo] = useState({ - user: 'init user msg', - gpt: 'init gpt reply', + user: null, + gpt: "Let's talk!", }); const updateUserMessage = useCallback((newMessage) => { @@ -19,25 +19,6 @@ const PatientHome = () => { setConvo((prevConvo) => ({ ...prevConvo, gpt: newResponse })); }, []); - const messages = [ - { - sender: 'patient', - text: 'Hi there, and I was hoping to try something new today. Can you guide me through a specific exercise from the set you provided?', - }, - { - sender: 'ai', - text: "Of course! I'm glad to hear you've been keeping up with your exercises. Which one were you thinking of trying, or do you have a specific area you'd like to focus on today?", - }, - { - sender: 'patient', - text: 'Hi there, and I was hoping to try something new today. Can you guide me through a specific exercise from the set you provided?', - }, - { - sender: 'ai', - text: "Of course! I'm glad to hear you've been keeping up with your exercises. Which one were you thinking of trying, or do you have a specific area you'd like to focus on today?", - }, - ]; - return (

@@ -49,9 +30,9 @@ const PatientHome = () => {
John
{/* */} -
-

{convo.user}

-

{convo.gpt}

+
+

{convo.user}

+

{convo.gpt}

{ updateGptResponse={updateGptResponse} />
+ {/* TODO: finish button that calls conversation/end */}
); }; diff --git a/frontend/src/views/patient/components/VoiceAI.jsx b/frontend/src/views/patient/components/VoiceAI.jsx index d39d1c3..e53ef72 100644 --- a/frontend/src/views/patient/components/VoiceAI.jsx +++ b/frontend/src/views/patient/components/VoiceAI.jsx @@ -1,5 +1,4 @@ import { useState, useEffect, useRef } from 'react'; -import Spline from '@splinetool/react-spline'; import axios from 'axios'; const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { @@ -8,57 +7,42 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { const [mediaStream, setMediaStream] = useState(null); const [mediaRecorder, setMediaRecorder] = useState(null); const [isConvoStarted, setIsConvoStarted] = useState(false); - - const userPromptRef = useRef(""); - const [speechRecognition, setSpeechRecognition] = useState(null); -// useEffect(() => { -// const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; -// if (SpeechRecognition) { -// const recognition = new SpeechRecognition(); -// recognition.continuous = true; -// recognition.interimResults = true; - -// recognition.onresult = (event) => { -// const latestResult = event.results[event.resultIndex]; -// const latestTranscript = latestResult[0].transcript.trim(); - -// if (latestResult.isFinal) { -// userPromptRef.current += ` ${latestTranscript}`; // Update the ref for final results -// } - -// console.log(latestTranscript); // Log each word as it is spoken -// }; - -// setSpeechRecognition(recognition); -// } else { -// console.warn("Speech recognition not supported in this browser."); -// } -// }, [updateUserPrompt]); + useEffect(() => { + const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + if (SpeechRecognition) { + const recognition = new SpeechRecognition(); + recognition.continuous = true; + recognition.interimResults = true; + + recognition.onresult = (event) => { + // Only use this for real-time display, not for sending to the server + const latestResult = event.results[event.resultIndex]; + const latestTranscript = latestResult[0].transcript.trim(); + updateUserMessage(latestTranscript); + }; + + setSpeechRecognition(recognition); + } else { + console.warn("Speech recognition not supported in this browser."); + } + }, [updateUserMessage]); const startRecording = async () => { - // FIXME: use actual IDs for people here - const queryParams = new URLSearchParams(); - queryParams.set('patient', 'demo'); - queryParams.set('practitioner', 'demo'); - + const queryParams = new URLSearchParams({ patient: 'demo', practitioner: 'demo' }); if (!isConvoStarted) { - const response = await axios.get( - `http://localhost:8080/conversation/start?${queryParams.toString()}` - ); + // Start a new conversation + const gptResponse = await axios.get(`http://localhost:8080/conversation/start?${queryParams.toString()}`); setIsConvoStarted(true); - // TODO: speak/display the AI response here - console.log(response.data.reply); - // updateUserMessage(latestTranscript); - - userPromptRef.current = ""; - speechRecognition?.start(); + console.log(gptResponse.data.reply); // TODO: speak/display the AI response here + updateGptResponse(gptResponse.data.reply); } + // Start recording audio const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); const recorder = new MediaRecorder(stream); - const chunks = []; // Array to store audio chunks + const chunks = []; recorder.ondataavailable = (event) => { if (event.data.size > 0) { @@ -67,43 +51,27 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { }; recorder.onstop = async () => { - // Combine all audio chunks into a single Blob + // Process and send the audio data to the server for transcription const audioBlob = new Blob(chunks, { type: 'audio/wav' }); - - // Create a FormData object and append the audio file const formData = new FormData(); formData.append('audioFile', audioBlob, 'recorded_audio.wav'); - // Send the FormData to the server using a fetch or XMLHttpRequest - const userMessage = await axios.post( - `http://localhost:8080/conversation/transcribe`, - formData - ); - - // update user message display - updateUserMessage(userMessage.data.user_msg) - console.log(userMessage) + const userMessage = await axios.post(`http://localhost:8080/conversation/transcribe`, formData); + updateUserMessage(userMessage.data.user_msg); // Update with the final, reliable transcription + console.log(userMessage); + // Fetch GPT response const gptResponse = await axios.post( `http://localhost:8080/conversation/send_message?${queryParams.toString()}`, - { - "message": userMessage.data.user_msg, - } + { "message": userMessage.data.user_msg } ); - - // update - updateGptResponse(gptResponse.data.reply) - - // TODO: speak/display the AI response here - speechRecognition?.stop(); + updateGptResponse(gptResponse.data.reply); }; recorder.start(); - setMediaStream(stream); setMediaRecorder(recorder); - - // Toggle recording state + speechRecognition?.start(); setIsRecording(true); }; @@ -112,8 +80,7 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { mediaRecorder.stop(); mediaStream.getTracks().forEach((track) => track.stop()); } - - // Toggle recording state + speechRecognition?.stop(); setIsRecording(false); }; @@ -144,7 +111,7 @@ const VoiceAI = ({ updateUserMessage, updateGptResponse }) => { onClick={isRecording ? triggerEnd : triggerStart} className="absolute bottom-8 left-1/2 w-32 h-32 transform -translate-x-1/2"> {/*
*/} -
+
{/* Date: Sat, 6 Jan 2024 23:46:44 -0700 Subject: [PATCH 03/34] first fetch of the convo --- frontend/src/views/patient/PatientHome.jsx | 32 ++++++++++++++++--- .../src/views/patient/components/Skeleton.jsx | 11 +++++++ 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 frontend/src/views/patient/components/Skeleton.jsx diff --git a/frontend/src/views/patient/PatientHome.jsx b/frontend/src/views/patient/PatientHome.jsx index 8299aec..551985a 100644 --- a/frontend/src/views/patient/PatientHome.jsx +++ b/frontend/src/views/patient/PatientHome.jsx @@ -1,14 +1,14 @@ -import { useState, useCallback } from 'react'; +import { useState, useEffect, useCallback } from 'react'; import Navbar from './components/Navbar'; -import Conversation from './components/Conversation'; import Exercises from './components/Exercises'; import VoiceAI from './components/VoiceAI'; -import Typewriter from 'typewriter-effect'; +import axios from 'axios'; +import Skeleton from './components/Skeleton'; const PatientHome = () => { const [convo, setConvo] = useState({ user: null, - gpt: "Let's talk!", + gpt: null, }); const updateUserMessage = useCallback((newMessage) => { @@ -19,6 +19,24 @@ const PatientHome = () => { setConvo((prevConvo) => ({ ...prevConvo, gpt: newResponse })); }, []); + useEffect(() => { + const startConversation = async () => { + const queryParams = new URLSearchParams({ + patient: 'demo', + practitioner: 'demo', + }); + try { + const response = await axios.get( + `http://localhost:8080/conversation/start?${queryParams.toString()}` + ); + setConvo((prevConvo) => ({ ...prevConvo, gpt: response.data.reply })); + } catch (error) { + console.error('Error fetching conversation start:', error); + } + }; + startConversation(); + }, []); + return (
@@ -32,7 +50,11 @@ const PatientHome = () => { {/* */}

{convo.user}

-

{convo.gpt}

+

+ {convo.gpt !== null + ? convo.gpt + : } +

{ + return ( +
+
+
+
+
+ ); +}; + +export default Skeleton; From 146501ddaa08440fe8d504a5a0d69bdcd322dd82 Mon Sep 17 00:00:00 2001 From: Owen Cooke Date: Sat, 6 Jan 2024 20:28:01 -0700 Subject: [PATCH 04/34] add basic practitioner dash --- frontend/package-lock.json | 9 ++ frontend/package.json | 1 + .../practitioner/PractitionerDashboard.jsx | 100 +++++++++++++++--- .../views/practitioner/PractitionerLogin.jsx | 81 ++++++++------ .../views/practitioner/components/Navbar.jsx | 58 ++++++++++ .../components/NewPatientModal.jsx | 98 +++++++++++++++++ 6 files changed, 304 insertions(+), 43 deletions(-) create mode 100644 frontend/src/views/practitioner/components/Navbar.jsx create mode 100644 frontend/src/views/practitioner/components/NewPatientModal.jsx diff --git a/frontend/package-lock.json b/frontend/package-lock.json index f9f0d66..546a0b3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -12,6 +12,7 @@ "@splinetool/runtime": "^1.0.18", "axios": "^1.6.5", "firebase": "^10.7.1", + "lucide-react": "^0.307.0", "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.21.1", @@ -4222,6 +4223,14 @@ "yallist": "^3.0.2" } }, + "node_modules/lucide-react": { + "version": "0.307.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.307.0.tgz", + "integrity": "sha512-+vZ+vUiWPZTMnLHURg4aoIaz6NHOWXVVcVd8iLROu1k4LbyjcnHIKmbjXHCmulz7XAYLWRVXzhJJgIr+Aq3vOg==", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index eba7d72..b194e1a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,6 +14,7 @@ "@splinetool/runtime": "^1.0.18", "axios": "^1.6.5", "firebase": "^10.7.1", + "lucide-react": "^0.307.0", "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.21.1", diff --git a/frontend/src/views/practitioner/PractitionerDashboard.jsx b/frontend/src/views/practitioner/PractitionerDashboard.jsx index 3fceb6c..2b8bf7c 100644 --- a/frontend/src/views/practitioner/PractitionerDashboard.jsx +++ b/frontend/src/views/practitioner/PractitionerDashboard.jsx @@ -1,17 +1,91 @@ +import Navbar from "./components/Navbar"; +import { UserRound, Dumbbell, Plus } from "lucide-react"; +import { auth, db } from "../../../firebaseConfig"; +import { useEffect, useState } from "react"; +import NewPatientModal from "./components/NewPatientModal"; + const PractitionerDashboard = () => { - return ( -
-
- Practitioner Dashboard -
-
-

Welcome, Practitioner

-

- Access patient records, set exercise routines, and view progress reports. -

-
+ const [patients, setPatients] = useState([]); + + useEffect(() => { + const fetchData = async () => { + try { + const patientsRef = db + .collection("practitioners") + .doc(auth.currentUser.uid) + .collection("patients"); + + // Get all patient info + const snapshot = await patientsRef.get(); + setPatients( + snapshot.docs.map((doc) => ({ + id: doc.id, + ...doc.data(), + })) + ); + } catch (error) { + // Handle errors here + console.error("Error fetching data:", error); + } + }; + + fetchData(); + }, []); + + return ( +
+ +
+
+ + +
+
+
+

Patients

+ +
+ + + + + + + + + + + + {patients.map((patient, idx) => ( + + + + + + + + ))} + +
IDNameAgeEmailLast Login
{patient.id}{patient.name}{patient.age}{patient.email}{patient.lastLogin || "Never"}
- ); +
+ +
+ ); }; -export default PractitionerDashboard; \ No newline at end of file +export default PractitionerDashboard; diff --git a/frontend/src/views/practitioner/PractitionerLogin.jsx b/frontend/src/views/practitioner/PractitionerLogin.jsx index 7b79d44..b9889ec 100644 --- a/frontend/src/views/practitioner/PractitionerLogin.jsx +++ b/frontend/src/views/practitioner/PractitionerLogin.jsx @@ -1,27 +1,25 @@ -import { auth } from '../../../firebaseConfig'; +import { auth } from "../../../firebaseConfig"; const PractitionerLogin = () => { const handleLogin = async (event) => { - event.preventDefault(); - const { email, password } = event.target.elements; + event.preventDefault(); + const { email, password } = event.target.elements; - try { - // Authenticate user - await auth.signInWithEmailAndPassword(email.value, password.value); - - // TODO: Redirect to "/practitioner/dashboard" - // Redirect to "/patient/home" as placeholder - window.location.href = "/patient/home"; - } catch (error) { - console.error("Login Error: ", error); - alert("Failed to login. Please check your credentials."); - } - }; + try { + // Authenticate user + await auth.signInWithEmailAndPassword(email.value, password.value); + auth.onAuthStateChanged( + () => (window.location.href = "/practitioner/dashboard") + ); + } catch (error) { + console.error("Login Error: ", error); + alert("Failed to login. Please check your credentials."); + } + }; return (
-
@@ -34,25 +32,45 @@ const PractitionerLogin = () => { MobilityMate
- -
Welcome back!
-
Please login in to your account.
+ +
+ Welcome back! +
+
+ Please login in to your account. +
- - + +
- - + +
-
@@ -67,11 +85,14 @@ const PractitionerLogin = () => {
- +
- Support For All + Support For All
-
); diff --git a/frontend/src/views/practitioner/components/Navbar.jsx b/frontend/src/views/practitioner/components/Navbar.jsx new file mode 100644 index 0000000..477c7dc --- /dev/null +++ b/frontend/src/views/practitioner/components/Navbar.jsx @@ -0,0 +1,58 @@ +const Navbar = () => { + return ( +
+ +
+
+
+
+ 8 Items + Subtotal: $999 +
+ +
+
+
+
+
+
+
+ Tailwind CSS Navbar component +
+
+ +
+
+
+ ); +}; + +export default Navbar; diff --git a/frontend/src/views/practitioner/components/NewPatientModal.jsx b/frontend/src/views/practitioner/components/NewPatientModal.jsx new file mode 100644 index 0000000..1cabe55 --- /dev/null +++ b/frontend/src/views/practitioner/components/NewPatientModal.jsx @@ -0,0 +1,98 @@ +import { useState } from "react"; +import { auth, db } from "../../../../firebaseConfig"; + +const NewPatientModal = () => { + const [formData, setFormData] = useState({ + name: "", + email: "", + age: "", + }); + + const handleChange = (e) => { + const { name, value } = e.target; + setFormData((prevData) => ({ ...prevData, [name]: value })); + }; + + const handleSubmit = async (e) => { + e.preventDefault(); + + try { + // Add the new patient data to Firestore + await db + .collection(`practitioners/${auth.currentUser.uid}/patients`) + .add(formData); + handleClose(); + } catch (error) { + console.error("Error adding new patient:", error); + } + }; + + const handleClose = () => { + document.getElementById("new_patient_modal").close(); + }; + + return ( + +
+

Register a New Patient

+ {/*

Press ESC key or click the button below to close

*/} +
+ +
+ +
+
+ +
+
+ +
+
+ + +
+ +
+
+ + ); +}; + +export default NewPatientModal; From e235615b7ba953f6c86e59c8a6a2a60b836dc885 Mon Sep 17 00:00:00 2001 From: Owen Cooke Date: Sat, 6 Jan 2024 20:37:48 -0700 Subject: [PATCH 05/34] fix auth user problem --- frontend/firebaseConfig.js | 22 ++++++++++++++----- .../practitioner/PractitionerDashboard.jsx | 5 +++-- .../components/NewPatientModal.jsx | 5 +++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/frontend/firebaseConfig.js b/frontend/firebaseConfig.js index c716d70..8f5be1d 100644 --- a/frontend/firebaseConfig.js +++ b/frontend/firebaseConfig.js @@ -1,7 +1,6 @@ -import firebase from 'firebase/compat/app'; -import 'firebase/compat/firestore'; -import 'firebase/compat/auth'; - +import firebase from "firebase/compat/app"; +import "firebase/compat/firestore"; +import "firebase/compat/auth"; const firebaseConfig = { apiKey: "AIzaSyCdjuKe_DkpM9PT71WMfl7l3SVpCeZfD5c", @@ -9,7 +8,7 @@ const firebaseConfig = { projectId: "mobilitymate-a8b53", storageBucket: "mobilitymate-a8b53.appspot.com", messagingSenderId: "911524752185", - appId: "1:911524752185:web:928da1ee8a528a348356e8" + appId: "1:911524752185:web:928da1ee8a528a348356e8", }; if (!firebase.apps.length) { @@ -20,3 +19,16 @@ if (!firebase.apps.length) { export const db = firebase.firestore(); export const auth = firebase.auth(); +export async function getCurrentUser() { + return new Promise((resolve, reject) => { + if (auth.currentUser) { + resolve(auth.currentUser); + return; + } + // The user is not found, hence listen to the change + const removeListener = auth.onAuthStateChanged((user) => { + removeListener(); + resolve(user); + }, reject); + }); +} diff --git a/frontend/src/views/practitioner/PractitionerDashboard.jsx b/frontend/src/views/practitioner/PractitionerDashboard.jsx index 2b8bf7c..08e7e8a 100644 --- a/frontend/src/views/practitioner/PractitionerDashboard.jsx +++ b/frontend/src/views/practitioner/PractitionerDashboard.jsx @@ -1,6 +1,6 @@ import Navbar from "./components/Navbar"; import { UserRound, Dumbbell, Plus } from "lucide-react"; -import { auth, db } from "../../../firebaseConfig"; +import { db, getCurrentUser } from "../../../firebaseConfig"; import { useEffect, useState } from "react"; import NewPatientModal from "./components/NewPatientModal"; @@ -9,10 +9,11 @@ const PractitionerDashboard = () => { useEffect(() => { const fetchData = async () => { + const currentUser = await getCurrentUser(); try { const patientsRef = db .collection("practitioners") - .doc(auth.currentUser.uid) + .doc(currentUser.uid) .collection("patients"); // Get all patient info diff --git a/frontend/src/views/practitioner/components/NewPatientModal.jsx b/frontend/src/views/practitioner/components/NewPatientModal.jsx index 1cabe55..6969c34 100644 --- a/frontend/src/views/practitioner/components/NewPatientModal.jsx +++ b/frontend/src/views/practitioner/components/NewPatientModal.jsx @@ -1,5 +1,5 @@ import { useState } from "react"; -import { auth, db } from "../../../../firebaseConfig"; +import { getCurrentUser, db } from "../../../../firebaseConfig"; const NewPatientModal = () => { const [formData, setFormData] = useState({ @@ -18,8 +18,9 @@ const NewPatientModal = () => { try { // Add the new patient data to Firestore + const currentUser = await getCurrentUser(); await db - .collection(`practitioners/${auth.currentUser.uid}/patients`) + .collection(`practitioners/${currentUser.uid}/patients`) .add(formData); handleClose(); } catch (error) { From e84b86cb5dcf0a453c9c7429148d7e79cbf578d0 Mon Sep 17 00:00:00 2001 From: Owen Cooke Date: Sat, 6 Jan 2024 21:18:11 -0700 Subject: [PATCH 06/34] add snapshot listener and clear form --- .../practitioner/PractitionerDashboard.jsx | 21 ++--- .../components/NewPatientModal.jsx | 78 ++++++++++--------- 2 files changed, 54 insertions(+), 45 deletions(-) diff --git a/frontend/src/views/practitioner/PractitionerDashboard.jsx b/frontend/src/views/practitioner/PractitionerDashboard.jsx index 08e7e8a..9278a6b 100644 --- a/frontend/src/views/practitioner/PractitionerDashboard.jsx +++ b/frontend/src/views/practitioner/PractitionerDashboard.jsx @@ -16,14 +16,17 @@ const PractitionerDashboard = () => { .doc(currentUser.uid) .collection("patients"); - // Get all patient info - const snapshot = await patientsRef.get(); - setPatients( - snapshot.docs.map((doc) => ({ - id: doc.id, - ...doc.data(), - })) - ); + // Use snapshot listener for real-time updates + const unsubscribe = patientsRef.onSnapshot((snapshot) => { + setPatients( + snapshot.docs.map((doc) => ({ + id: doc.id, + ...doc.data(), + })) + ); + }); + + return () => unsubscribe(); } catch (error) { // Handle errors here console.error("Error fetching data:", error); @@ -37,7 +40,7 @@ const PractitionerDashboard = () => {
-
+
-
+
+
diff --git a/frontend/src/views/patient/components/ExerciseCard.jsx b/frontend/src/views/patient/components/ExerciseCard.jsx index c9be5c0..5f5fff3 100644 --- a/frontend/src/views/patient/components/ExerciseCard.jsx +++ b/frontend/src/views/patient/components/ExerciseCard.jsx @@ -1,15 +1,51 @@ -const ExerciseCard = () => { +import PropTypes from 'prop-types'; +import { useEffect, useState } from 'react'; + +const ExerciseCard = ({ title, description, imageUrl, instructions, onClick, isExpanded }) => { + const [startAnimation, setStartAnimation] = useState(false); + + useEffect(() => { + if (isExpanded) { + + const timeoutId = window.setTimeout(() => { + setStartAnimation(true); + }, 100); // Delay in milliseconds + + return () => window.clearTimeout(timeoutId); + } + }, [isExpanded]); + + const animationClass = startAnimation ? 'start-animation' : ''; + return ( -
-
-

Card title!

-

If a dog chews shoes whose shoes does he choose?

-
- +
+
+
+ {title} +
+
+

{title}

+

{description}

+ {isExpanded && ( +
+ {instructions.map((step, index) => ( +

{step}

+ ))} +
+ )}
); }; -export default ExerciseCard; +ExerciseCard.propTypes = { + title: PropTypes.string.isRequired, + description: PropTypes.string.isRequired, + imageUrl: PropTypes.string.isRequired, + instructions: PropTypes.arrayOf(PropTypes.string).isRequired, + onClick: PropTypes.func.isRequired, + isExpanded: PropTypes.bool.isRequired, +}; + +export default ExerciseCard; \ No newline at end of file diff --git a/frontend/src/views/patient/components/Exercises.jsx b/frontend/src/views/patient/components/Exercises.jsx index b27457c..da73c51 100644 --- a/frontend/src/views/patient/components/Exercises.jsx +++ b/frontend/src/views/patient/components/Exercises.jsx @@ -1,10 +1,113 @@ -import Card from "./ExerciseCard"; +import { useState } from 'react'; +// import ExerciseCard from "./ExerciseCard"; +// import PropTypes from 'prop-types'; +import '../styles.css'; +import glutesImage from '../../../assets/raises.webp'; +import circleImage from '../../../assets/circs.png'; +import squatImage from '../../../assets/squat.png' +import quartSquatImage from '../../../assets/quartSquat.png' +import singleLegGluteImage from '../../../assets/singleLegGlute.png'; const Exercises = () => { + // const [expandedCard, setExpandedCard] = useState(null); + const [selectedCard, setSelectedCard] = useState(null); + + const cards = [ + { + id: 1, + title: "Glutes", + description: "Glute Bridges good for glutes", + imageUrl: glutesImage, + instructions: ["Step 1 for Card 1", "Step 2 for Card 1", "Step 3 for Card 1"] + }, + { + id: 2, + title: "Circs", + description: "Single Leg Circles good for glutes", + imageUrl: circleImage, + instructions: ["Step 1 for Card 2", "Step 2 for Card 2", "Step 3 for Card 2"] + }, + { + id: 3, + title: "Squats", + description: "Description for Card 3", + imageUrl: squatImage, // Placeholder image + instructions: ["Step 1 for Card 3", "Step 2 for Card 3", "Step 3 for Card 3"] + }, + { + id: 4, + title: "Card 4", + description: "Description for Card 4", + imageUrl: quartSquatImage, // Placeholder image + instructions: ["Step 1 for Card 4", "Step 2 for Card 4", "Step 3 for Card 4"] + }, + { + id: 5, + title: "Card 5", + description: "Description for Card 5", + imageUrl: singleLegGluteImage, // Placeholder image + instructions: ["Step 1 for Card 5", "Step 2 for Card 5", "Step 3 for Card 5"] + }, + { + id: 5, + title: "Card 5", + description: "Description for Card 5", + imageUrl: glutesImage, // Placeholder image + instructions: ["Step 1 for Card 5", "Step 2 for Card 5", "Step 3 for Card 5"] + }, + { + id: 5, + title: "Card 5", + description: "Description for Card 5", + imageUrl: glutesImage, // Placeholder image + instructions: ["Step 1 for Card 5", "Step 2 for Card 5", "Step 3 for Card 5"] + }, +]; + +const handleCardClick = (card) => { + setSelectedCard(card); + }; + + const closeModal = () => { + setSelectedCard(null); + }; + + const renderCard = (card) => ( +
+
handleCardClick(card)}> + {card.title} +
+

{card.title}

+

{card.description}

+
+
+
+ ); + + const renderModal = (card) => ( + +
+ {card.title} +
+ +

{card.title}

+

{card.description}

+
    + {card.instructions.map((step, index) => ( +
  1. {step}
    2. + ))} +
+
+
+ + ); + return ( -
- - +
+
+ {cards.map(renderCard)} +
+ {selectedCard && renderModal(selectedCard)}
); }; diff --git a/frontend/src/views/patient/components/Phone.jsx b/frontend/src/views/patient/components/Phone.jsx new file mode 100644 index 0000000..4c1ba78 --- /dev/null +++ b/frontend/src/views/patient/components/Phone.jsx @@ -0,0 +1,22 @@ + +const Phone = () => { + return ( +
+
+ Shoes +
+
+

Phone!

+

I broke my phone, whose fault is it?

+
+ +
+
+
+ ); +}; + +export default Phone; diff --git a/frontend/src/views/patient/styles.css b/frontend/src/views/patient/styles.css new file mode 100644 index 0000000..20e4ec6 --- /dev/null +++ b/frontend/src/views/patient/styles.css @@ -0,0 +1,106 @@ +.exercise-container { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + height: 100%; + overflow: hidden; +} + +.carousel { + width: 80%; + max-height: 75%; + overflow-y: auto; +} + +.carousel-item { + display: flex; + flex-direction: column; + align-items: center; + padding: 1rem; +} + +.card { + background: #f5f5f5; + border-radius: 0.5rem; + box-shadow: 0 2px 4px rgba(0,0,0,0.2); + cursor: pointer; + overflow: hidden; + transition: 0.3s; + width: 100%; + margin-bottom: 1rem; +} + +.card:hover { + transform: translateY(-5px); + box-shadow: 0 4px 6px rgba(0,0,0,0.2); +} + +.card img { + width: 100%; + height: auto; + object-fit: cover; +} + +.card-body { + padding: 1rem; +} + +.card-title { + font-size: 1.5rem; + margin-bottom: 0.5rem; +} + +.card-description { + font-size: 1rem; + color: #333; + margin-bottom: 1rem; +} + +.card-steps { + list-style-type: decimal; + padding-left: 1.5rem; +} + +.text-content { + padding: 1rem; +} + +.close-btn { + border: none; + background: transparent; + font-size: 1.5rem; + position: absolute; + top: 1rem; + right: 1rem; + cursor: pointer; +} + +.btn { + align-self: center; + padding: 0.5rem 1rem; +} + +@keyframes fadeIn { + from { opacity: 0; } + to { opacity: 1; } +} + +@media (max-width: 768px) { + .modal { + max-width: 90%; + } + + .modal-box { + flex-direction: column; + } + + .modal-image { + max-width: 100%; + margin-bottom: 1rem; + } + + .text-content { + max-width: 100%; + } +} From 63173ddad876d3f895bb8887cfa47b52871d6d47 Mon Sep 17 00:00:00 2001 From: Jared Drueco Date: Sun, 7 Jan 2024 00:14:43 -0700 Subject: [PATCH 34/34] fix merge conflicts --- frontend/src/views/patient/PatientHome.jsx | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/frontend/src/views/patient/PatientHome.jsx b/frontend/src/views/patient/PatientHome.jsx index f649b1b..5af51b2 100644 --- a/frontend/src/views/patient/PatientHome.jsx +++ b/frontend/src/views/patient/PatientHome.jsx @@ -44,11 +44,7 @@ const PatientHome = () => { return (
-<<<<<<< HEAD -
-=======
->>>>>>> main
@@ -80,16 +76,10 @@ const PatientHome = () => {
-<<<<<<< HEAD -======= -
- {/* */} -
->>>>>>> main
{/* TODO: finish button that calls conversation/end */}