VibeSync.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "22957828-ad78-4610-be23-99433c43b395",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install the dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "ba965e63-f8c9-478a-97d3-97ea5d670841",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: librosa in c:\\users\\dell\\anaconda3\\lib\\site-packages (0.10.2.post1)Note: you may need to restart the kernel to use updated packages.\n",
      "\n",
      "Requirement already satisfied: soundfile in c:\\users\\dell\\anaconda3\\lib\\site-packages (0.12.1)\n",
      "Requirement already satisfied: pyaudio in c:\\users\\dell\\anaconda3\\lib\\site-packages (0.2.14)\n",
      "Requirement already satisfied: audioread>=2.1.9 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (3.0.1)\n",
      "Requirement already satisfied: numpy!=1.22.0,!=1.22.1,!=1.22.2,>=1.20.3 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (1.26.4)\n",
      "Requirement already satisfied: scipy>=1.2.0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (1.11.4)\n",
      "Requirement already satisfied: scikit-learn>=0.20.0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (1.2.2)\n",
      "Requirement already satisfied: joblib>=0.14 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (1.2.0)\n",
      "Requirement already satisfied: decorator>=4.3.0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (5.1.1)\n",
      "Requirement already satisfied: numba>=0.51.0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (0.59.0)\n",
      "Requirement already satisfied: pooch>=1.1 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (1.8.2)\n",
      "Requirement already satisfied: soxr>=0.3.2 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (0.5.0.post1)\n",
      "Requirement already satisfied: typing-extensions>=4.1.1 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (4.9.0)\n",
      "Requirement already satisfied: lazy-loader>=0.1 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (0.3)\n",
      "Requirement already satisfied: msgpack>=1.0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from librosa) (1.0.3)\n",
      "Requirement already satisfied: cffi>=1.0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from soundfile) (1.16.0)\n",
      "Requirement already satisfied: pycparser in c:\\users\\dell\\anaconda3\\lib\\site-packages (from cffi>=1.0->soundfile) (2.21)\n",
      "Requirement already satisfied: llvmlite<0.43,>=0.42.0dev0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from numba>=0.51.0->librosa) (0.42.0)\n",
      "Requirement already satisfied: platformdirs>=2.5.0 in c:\\users\\dell\\appdata\\roaming\\python\\python311\\site-packages (from pooch>=1.1->librosa) (4.3.6)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\dell\\appdata\\roaming\\python\\python311\\site-packages (from pooch>=1.1->librosa) (24.1)\n",
      "Requirement already satisfied: requests>=2.19.0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from pooch>=1.1->librosa) (2.31.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from scikit-learn>=0.20.0->librosa) (2.2.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from requests>=2.19.0->pooch>=1.1->librosa) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from requests>=2.19.0->pooch>=1.1->librosa) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from requests>=2.19.0->pooch>=1.1->librosa) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from requests>=2.19.0->pooch>=1.1->librosa) (2024.2.2)\n"
     ]
    }
   ],
   "source": [
    "pip install librosa soundfile pyaudio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "4e2d9a2f-aacf-44ce-a45e-7e90a09e241c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make the necessary imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "12fd78d2-fed2-45a8-80b1-4f3949d5d27c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import librosa\n",
    "import soundfile\n",
    "import os, glob, pickle\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b688e909-c2a2-4992-90b0-1343020b9efc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature Extraction ( Check sample data processing folder to know the concepts )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "31cedb0b-40c4-4aa6-abe3-98313cf28539",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_feature(file_name, mfcc, chroma, mel):\n",
    "    with soundfile.SoundFile(file_name) as sound_file:\n",
    "        X = sound_file.read(dtype=\"float32\")\n",
    "        sample_rate = sound_file.samplerate\n",
    "        result = np.array([])\n",
    "        if chroma:\n",
    "            stft = np.abs(librosa.stft(X))\n",
    "        if mfcc:\n",
    "            mfccs = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=40).T, axis=0)\n",
    "            result = np.hstack((result, mfccs))\n",
    "        if chroma:\n",
    "            chroma = np.mean(librosa.feature.chroma_stft(S=stft, sr=sample_rate).T, axis=0)\n",
    "            result = np.hstack((result, chroma))\n",
    "        if mel:\n",
    "            mel_spec = librosa.feature.melspectrogram(y=X, sr=sample_rate)\n",
    "            mel = np.mean(mel_spec.T, axis=0)\n",
    "            result = np.hstack((result, mel))\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9257a026-7c1a-4293-9763-48ef160148b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define dictionary for the emotions in the RAVDESS dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "7e06f256-9afc-4f47-8098-950ff6362f29",
   "metadata": {},
   "outputs": [],
   "source": [
    "emotions={\n",
    "  '01':'neutral',\n",
    "  '02':'calm',\n",
    "  '03':'happy',\n",
    "  '04':'sad',\n",
    "  '05':'angry',\n",
    "  '06':'fearful',\n",
    "  '07':'disgust',\n",
    "  '08':'surprised'\n",
    "}\n",
    "# Emotions to observe\n",
    "observed_emotions=['neutral', 'happy', 'sad', 'angry','fearful', 'disgust']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "dcd93ae4-e408-4d91-9c82-547f1824d4c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the data and extract features for each sound file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "9b2cef77-63cb-4d16-96f6-0818145fa48b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(test_size=0.2):\n",
    "    x,y=[],[]\n",
    "    for file in glob.glob(r\"E:\\Downloads\\speech-emotion-recognition-ravdess-data\\Actor_*\\*.wav\"):\n",
    "        file_name=os.path.basename(file)\n",
    "        emotion=emotions[file_name.split(\"-\")[2]]\n",
    "        if emotion not in observed_emotions:\n",
    "            continue\n",
    "        feature=extract_feature(file, mfcc=True, chroma=True, mel=True)\n",
    "        x.append(feature)\n",
    "        y.append(emotion)\n",
    "    return train_test_split(np.array(x), y, test_size=test_size, random_state=9) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "8b981edd-5c12-4ff4-9cc7-76015d154f4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "4de4970c-cddf-4b46-8a1d-570640318b96",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test = load_data(test_size=0.21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "c0586d3c-5215-4600-b545-0165d88ad85d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# No. of samples in training and testing set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "0fe1bb2c-8c90-4e2b-8570-766b1393a1a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(834, 222)\n"
     ]
    }
   ],
   "source": [
    "print((x_train.shape[0], x_test.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "f4950f02-6d7f-494d-98e1-6a5ae183e2e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# No. of features extracted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "1e7f752b-33f3-41e3-969e-058f962ca56d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features extracted: 180\n"
     ]
    }
   ],
   "source": [
    "print(f'Features extracted: {x_train.shape[1]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "46ee55b8-f7e5-48ee-9e39-7138b03d2777",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the Multi Layer Perceptron Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "4af09c89-6ec0-4b78-9f9b-c8c5c0d0ef07",
   "metadata": {},
   "outputs": [],
   "source": [
    "model=MLPClassifier(alpha=0.01, batch_size=256, epsilon=1e-08, hidden_layer_sizes=(300,), learning_rate='adaptive', max_iter=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "696ed088-28f1-4796-9d4e-be31105f704b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "739477d9-b842-4b33-8432-b02148eda273",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-17 {color: black;background-color: white;}#sk-container-id-17 pre{padding: 0;}#sk-container-id-17 div.sk-toggleable {background-color: white;}#sk-container-id-17 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-17 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-17 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-17 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-17 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-17 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-17 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-17 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-17 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-17 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-17 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-17 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-17 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-17 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-17 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-17 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-17 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-17 div.sk-item {position: relative;z-index: 1;}#sk-container-id-17 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-17 div.sk-item::before, #sk-container-id-17 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-17 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-17 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-17 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-17 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-17 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-17 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-17 div.sk-label-container {text-align: center;}#sk-container-id-17 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-17 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-17\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MLPClassifier(alpha=0.01, batch_size=256, hidden_layer_sizes=(300,),\n",
       "              learning_rate=&#x27;adaptive&#x27;, max_iter=500)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-17\" type=\"checkbox\" checked><label for=\"sk-estimator-id-17\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MLPClassifier</label><div class=\"sk-toggleable__content\"><pre>MLPClassifier(alpha=0.01, batch_size=256, hidden_layer_sizes=(300,),\n",
       "              learning_rate=&#x27;adaptive&#x27;, max_iter=500)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "MLPClassifier(alpha=0.01, batch_size=256, hidden_layer_sizes=(300,),\n",
       "              learning_rate='adaptive', max_iter=500)"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "f92bbcf3-d547-47b2-af16-4f5671ff062b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict for the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "fe709425-24bb-4f00-80b0-aa39c4a295a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred=model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "f3eba3a0-8da4-47ce-a689-58569e9d3042",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate the accuracy of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "aa468279-8444-4b8c-91e5-d030703532f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 68.02%\n"
     ]
    }
   ],
   "source": [
    "accuracy=accuracy_score(y_true=y_test, y_pred=y_pred)\n",
    "\n",
    "# Print the accuracy\n",
    "print(\"Accuracy: {:.2f}%\".format(accuracy*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "6b720225-2b15-4dc7-aa57-3a7fa541aa44",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to predict emotion from custom audio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "342c50df-7e89-457f-94cd-f4695a3f36b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_emotion_from_audio(file_path):\n",
    "    # Extract features from the custom audio\n",
    "    features = extract_feature(file_path, mfcc=True, chroma=True, mel=True)\n",
    "\n",
    "    # Reshape the features for prediction (make it 2D array: 1 sample, n_features)\n",
    "    features = features.reshape(1, -1)\n",
    "\n",
    "    # Predict using the trained model\n",
    "    emotion = model.predict(features)\n",
    "\n",
    "    return emotion[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "bd806f64-e687-4ef3-9626-cd2e49df0645",
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_file = r\"E:\\Downloads\\speech-emotion-recognition-ravdess-data\\Actor_24\\03-01-07-01-02-02-24.wav\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "939588ba-74bb-4a56-afd2-e689591a7887",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted Emotion: disgust\n"
     ]
    }
   ],
   "source": [
    "predicted_emotion = predict_emotion_from_audio(audio_file)\n",
    "print(f\"Predicted Emotion: {predicted_emotion}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d53b312f-8f6a-4892-b6ee-0fca9c209eeb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}