Welcome to Your Home Page
-- Here you can find your daily routines, track your progress, and manage your health journey. -
-diff --git a/.gitignore b/.gitignore index fdac2da..5b8dcd7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,11 @@ /venv *.env -*.pyc \ No newline at end of file +*.pyc +__pycache__/ +service.json +settings.py +.vscode +.venv +backend/settings.py +.vscode +__pycache__ diff --git a/backend/.firebaserc b/backend/conversation/__init__.py similarity index 100% rename from backend/.firebaserc rename to backend/conversation/__init__.py diff --git a/backend/conversation/views.py b/backend/conversation/views.py new file mode 100644 index 0000000..5b1f8ef --- /dev/null +++ b/backend/conversation/views.py @@ -0,0 +1,31 @@ +from flask import Flask, request, jsonify, session, Blueprint +from firebase_admin import firestore +from .conversation import Conversation # Assuming your Conversation class is defined in 'conversation.py' +from firebase_admin import firestore + + +conversation_blueprint = Blueprint('conversation', __name__) + +db = firestore.client() +users_ref = db.collection('practitioners') + +@conversation_blueprint.route('/send_message', methods=['POST']) +def send_message(): + practitioner = request.args.get('practitioner') + patient = request.args.get('patient') + + user_doc_ref = users_ref.document(practitioner).collection("patients").document(patient) + if 'conversation_id' not in session: + conversation = Conversation(user_doc_ref=user_doc_ref) + session['conversation_id'] = conversation.get_conversation_id() + else: + conversation_id = session['conversation_id'] + conversation = Conversation(user_doc_ref=user_doc_ref, conversaton_id=conversation_id) + + # TODO: transcribe if it is audio + message = request.json.get('message') + + # Generate a reply using the Conversation object + reply = conversation.generate_reply(message) + + return jsonify({'reply': reply}), 200 \ No newline at end of file diff --git a/backend/firebase.json b/backend/firebase.json deleted file mode 100644 index e69de29..0000000 diff --git a/backend/server/src/app.py b/backend/server/src/app.py index 595d96d..50ea1fa 100644 --- a/backend/server/src/app.py +++ b/backend/server/src/app.py @@ -4,9 +4,21 @@ import os from constants import * import time +import firebase_admin +from firebase_admin import credentials +from flask_cors import CORS load_dotenv() app = Flask(__name__) +app.secret_key = os.getenv("SECRET_KEY") +cred = credentials.Certificate("service.json") +firebase_admin.initialize_app(cred) + +from conversation.views import conversation_blueprint + +# Register the conversation Blueprint +app.register_blueprint(conversation_blueprint, url_prefix="/conversation") +CORS(app) # Load Flask-Mail config from .env app.config["MAIL_SERVER"] = os.getenv("MAIL_SERVER") diff --git a/backend/server/src/conversation/conversation.py b/backend/server/src/conversation/conversation.py new file mode 100644 index 0000000..a823c37 --- /dev/null +++ b/backend/server/src/conversation/conversation.py @@ -0,0 +1,87 @@ +import whisper +import datetime +import os +from openai import OpenAI + +client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) +model = whisper.load_model("base") + + +class Conversation: + prompt = "You are a physiotherapist. You will now be connected to a patient (the user). You are to ask them about these following exercises one at a time to see how they are progressing. Please maintain a professional tone and keep your questions succint. Your goal is to quickly acquire all the required information from the patient. Use your professional knowledge to help inform your questions relative to each exercise. Here are the exercises:" + + def __init__(self, user_doc_ref, conversaton_id=None): + if conversaton_id: + self.conversation_ref = user_doc_ref.collection("conversations").document( + conversaton_id + ) + self.history = self.conversation_ref.get().get("history") + else: + self.history = [] + _, self.conversation_ref = user_doc_ref.collection("conversations").add( + {"date": datetime.datetime.now(), "history": self.history} + ) + self.engineer_prompt(user_doc_ref) + + def get_conversation_id(self): + return self.conversation_ref.id + + @staticmethod + def transcribe(audio_file): + return model.transcribe(audio_file, fp16=False).get("text") + + def request_reply(self): + return client.chat.completions.create( + model="gpt-3.5-turbo", messages=self.history, temperature=0.5 + ) + + def generate_reply(self, text): + self.history.append({"role": "user", "content": text}) + self.conversation_ref.update({"history": self.history}) + response = self.request_reply() + self.history.append( + { + "role": "assistant", + "content": response.choices[0].message.content.strip(), + } + ) + self.conversation_ref.update({"history": self.history}) + return response.choices[0].message.content.strip() + + def generate_greeting(self): + self.history.append({"role": "system", "content": self.prompt}) + response = self.request_reply() + self.history.append( + { + "role": "assistant", + "content": response.choices[0].message.content.strip(), + } + ) + self.conversation_ref.update({"history": self.history}) + return response.choices[0].message.content.strip() + + def engineer_prompt(self, user_doc_ref): + exercises = user_doc_ref.get().get("exercises") + if exercises: + for exercise in exercises: + self.prompt += "\n" + exercise + + self.prompt += f". When necessary, please refer the patient to contact their physiotherapist who's name is: {user_doc_ref.parent.parent.get().get('name')}." + + def end_conversation(self): + print(self.history) + self.history.append( + { + "role": "system", + "content": "Summarize the entire conversation between the user and the assistant for a physiotherapist focus on the answers the user game. Give me only the summary, no other text.", + } + ) + response = client.chat.completions.create( + model="gpt-3.5-turbo", messages=self.history, temperature=0.5 + ) + self.summary = response.choices[0].message.content.strip() + + # Add the summary to the conversation collection + self.conversation_ref.update({"summary": self.summary}) + + return self.summary diff --git a/backend/server/src/conversation/views.py b/backend/server/src/conversation/views.py new file mode 100644 index 0000000..219428d --- /dev/null +++ b/backend/server/src/conversation/views.py @@ -0,0 +1,58 @@ +from flask import request, jsonify, Blueprint +from firebase_admin import firestore +from .conversation import ( + Conversation, +) +from firebase_admin import firestore +import tempfile +import os + +conversation_blueprint = Blueprint("conversation", __name__) +db = firestore.client() +users_ref = db.collection("practitioners") + +session = {} + + +@conversation_blueprint.route("/start") +def start(): + practitioner = request.args.get("practitioner") + patient = request.args.get("patient") + user_doc_ref = ( + users_ref.document(practitioner).collection("patients").document(patient) + ) + + conversation = Conversation(user_doc_ref=user_doc_ref) + greeting = conversation.generate_greeting() + session["conversation_id"] = conversation.get_conversation_id() + + return jsonify({"reply": greeting}), 200 + + +@conversation_blueprint.route("/send_message", methods=["POST"]) +def send_message(): + practitioner = request.args.get("practitioner") + patient = request.args.get("patient") + user_doc_ref = ( + users_ref.document(practitioner).collection("patients").document(patient) + ) + if "conversation_id" not in session: + return jsonify({"reply": "Please start a conversation first"}), 400 + + conversation_id = session["conversation_id"] + conversation = Conversation( + user_doc_ref=user_doc_ref, conversaton_id=conversation_id + ) + + # Store audio in a temp file + audio = request.files["audioFile"] + temp_audio_path = os.path.join(tempfile.gettempdir(), "received_audio.wav") + audio.save(temp_audio_path) + + # Transcribe the audio + message = Conversation.transcribe(str(temp_audio_path)) + os.remove(temp_audio_path) + + # Generate a reply using the Conversation object + reply = conversation.generate_reply(message) + return jsonify({"reply": reply}), 200 diff --git a/backend/server/src/requirements.txt b/backend/server/src/requirements.txt new file mode 100644 index 0000000..0fda4df --- /dev/null +++ b/backend/server/src/requirements.txt @@ -0,0 +1,88 @@ +annotated-types==0.6.0 +anyio==4.2.0 +bidict==0.22.1 +blinker==1.7.0 +CacheControl==0.13.1 +cachelib==0.10.2 +cachetools==5.3.2 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cryptography==41.0.7 +distro==1.9.0 +exceptiongroup==1.2.0 +filelock==3.13.1 +firebase-admin==6.3.0 +Flask==3.0.0 +Flask-Cors==4.0.0 +Flask-Mail==0.9.1 +fsspec==2023.12.2 +google-api-core==2.15.0 +google-api-python-client==2.112.0 +google-auth==2.26.1 +google-auth-httplib2==0.2.0 +google-cloud-core==2.4.1 +google-cloud-firestore==2.14.0 +google-cloud-storage==2.14.0 +google-crc32c==1.5.0 +google-resumable-media==2.7.0 +googleapis-common-protos==1.62.0 +grpcio==1.60.0 +grpcio-status==1.60.0 +h11==0.14.0 +httpcore==1.0.2 +httplib2==0.22.0 +httpx==0.26.0 +idna==3.6 +itsdangerous==2.1.2 +Jinja2==3.1.2 +llvmlite==0.41.1 +MarkupSafe==2.1.3 +more-itertools==10.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +networkx==3.2.1 +numba==0.58.1 +numpy==1.26.3 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +openai==1.6.1 +openai-whisper==20231117 +proto-plus==1.23.0 +protobuf==4.25.1 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 +pycparser==2.21 +pydantic==2.5.3 +pydantic_core==2.14.6 +PyJWT==2.8.0 +pyparsing==3.1.1 +python-dotenv==1.0.0 +python-engineio==4.8.2 +python-socketio==5.10.0 +regex==2023.12.25 +requests==2.31.0 +rsa==4.9 +simple-websocket==1.0.0 +sniffio==1.3.0 +sympy==1.12 +tiktoken==0.5.2 +torch==2.1.2 +tqdm==4.66.1 +triton==2.1.0 +typing_extensions==4.9.0 +uritemplate==4.1.1 +urllib3==2.1.0 +Werkzeug==3.0.1 +wsproto==1.2.0 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index d2e9552..9d0caf7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.0", "dependencies": { "firebase": "^10.7.1", + "axios": "^1.6.5", "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.21.1" @@ -2067,6 +2068,11 @@ "has-symbols": "^1.0.3" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "node_modules/autoprefixer": { "version": "10.4.16", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.16.tgz", @@ -2116,6 +2122,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axios": { + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.5.tgz", + "integrity": "sha512-Ii012v05KEVuUoFWmMW/UQv9aRIc3ZwkWDcM+h5Il8izZCtRVpDUfwpoFf7eOtajT3QiGR4yDUx7lPqHJULgbg==", + "dependencies": { + "follow-redirects": "^1.15.4", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2382,6 +2398,17 @@ "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -2527,6 +2554,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/didyoumean": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", @@ -3206,6 +3241,25 @@ "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", + "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -3231,6 +3285,19 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fraction.js": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", @@ -4147,6 +4214,25 @@ "node": ">=8.6" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -4714,7 +4800,11 @@ }, "engines": { "node": ">=12.0.0" - } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "node_modules/punycode": { "version": "2.3.1", @@ -6053,4 +6143,5 @@ } } } -} + } +} \ No newline at end of file diff --git a/frontend/package.json b/frontend/package.json index ee8bd00..2e8b260 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "firebase": "^10.7.1", + "axios": "^1.6.5", "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.21.1" diff --git a/frontend/src/views/patient/PatientHome.jsx b/frontend/src/views/patient/PatientHome.jsx index a92d01b..f3b155d 100644 --- a/frontend/src/views/patient/PatientHome.jsx +++ b/frontend/src/views/patient/PatientHome.jsx @@ -1,17 +1,22 @@ -import Navbar from './components/Navbar' +import Navbar from "./components/Navbar"; +import RecordButton from "./components/RecordButton"; const PatientHome = () => { - return ( -
- Here you can find your daily routines, track your progress, and manage your health journey. -
-+ Here you can find your daily routines, track your progress, and manage + your health journey. +
+Recording...
+ ) : ( +Click Start Recording to begin recording.
+ )} + + +