classification_futurama.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "33fa760c",
   "metadata": {},
   "source": [
    "# External Validation: An Episode of Futurama"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14ba5f64",
   "metadata": {},
   "source": [
    "As an external validation exercise we will try to automatically detect gunshots in the futurama episode <a href=https://www.imdb.com/title/tt1630889/> \"Law and Oracle\".<a/>\n",
    "\n",
    "Due to copyright issues, we can not provide the audio data itself. We provide only the features extracted from that data, in the file \"extracted_features_futurama.csv\". To obtain these features, we split the audio data for the episode into fragments each approximately 5 seconds in length, then applied the feature extraction procedures described in a previous notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0cb11489",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import required packages\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from datetime import timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c395dd68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 260 entries, 0 to 259\n",
      "Columns: 390 entries, eq_0 to hits_ratio\n",
      "dtypes: float64(389), int64(1)\n",
      "memory usage: 794.2 KB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eq_0</th>\n",
       "      <th>eq_10</th>\n",
       "      <th>eq_20</th>\n",
       "      <th>eq_30</th>\n",
       "      <th>eq_40</th>\n",
       "      <th>eq_60</th>\n",
       "      <th>eq_80</th>\n",
       "      <th>eq_120</th>\n",
       "      <th>eq_160</th>\n",
       "      <th>eq_230</th>\n",
       "      <th>...</th>\n",
       "      <th>roll_128.logbands_mw_20.12</th>\n",
       "      <th>roll_128.logbands_mw_20.13</th>\n",
       "      <th>roll_128.logbands_mw_20.14</th>\n",
       "      <th>roll_128.logbands_mw_20.15</th>\n",
       "      <th>roll_128.logbands_mw_20.16</th>\n",
       "      <th>roll_128.logbands_mw_20.17</th>\n",
       "      <th>roll_128.logbands_mw_20.18</th>\n",
       "      <th>roll_128.logbands_mw_20.19</th>\n",
       "      <th>power_ratio</th>\n",
       "      <th>hits_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.197786</td>\n",
       "      <td>0.475224</td>\n",
       "      <td>0.640533</td>\n",
       "      <td>0.877277</td>\n",
       "      <td>1.040508</td>\n",
       "      <td>1.037066</td>\n",
       "      <td>1.244018</td>\n",
       "      <td>1.316525</td>\n",
       "      <td>1.339426</td>\n",
       "      <td>1.265299</td>\n",
       "      <td>...</td>\n",
       "      <td>0.934274</td>\n",
       "      <td>0.900729</td>\n",
       "      <td>0.862285</td>\n",
       "      <td>0.812940</td>\n",
       "      <td>0.730737</td>\n",
       "      <td>0.598143</td>\n",
       "      <td>0.409162</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.331745</td>\n",
       "      <td>-1.518403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.010537</td>\n",
       "      <td>1.511049</td>\n",
       "      <td>1.229768</td>\n",
       "      <td>2.022973</td>\n",
       "      <td>2.413715</td>\n",
       "      <td>2.187437</td>\n",
       "      <td>2.187408</td>\n",
       "      <td>2.046839</td>\n",
       "      <td>1.960684</td>\n",
       "      <td>1.721527</td>\n",
       "      <td>...</td>\n",
       "      <td>0.911348</td>\n",
       "      <td>0.875964</td>\n",
       "      <td>0.831477</td>\n",
       "      <td>0.783590</td>\n",
       "      <td>0.703153</td>\n",
       "      <td>0.586626</td>\n",
       "      <td>0.408304</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.259233</td>\n",
       "      <td>-0.858878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.038472</td>\n",
       "      <td>1.468064</td>\n",
       "      <td>1.274101</td>\n",
       "      <td>2.044264</td>\n",
       "      <td>2.430573</td>\n",
       "      <td>2.155454</td>\n",
       "      <td>2.176625</td>\n",
       "      <td>2.008776</td>\n",
       "      <td>1.843365</td>\n",
       "      <td>1.628750</td>\n",
       "      <td>...</td>\n",
       "      <td>0.917274</td>\n",
       "      <td>0.890996</td>\n",
       "      <td>0.837766</td>\n",
       "      <td>0.787540</td>\n",
       "      <td>0.686259</td>\n",
       "      <td>0.553205</td>\n",
       "      <td>0.376001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.411571</td>\n",
       "      <td>-1.072553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.947900</td>\n",
       "      <td>1.200151</td>\n",
       "      <td>1.109876</td>\n",
       "      <td>1.496561</td>\n",
       "      <td>1.800022</td>\n",
       "      <td>2.082368</td>\n",
       "      <td>1.825926</td>\n",
       "      <td>1.802636</td>\n",
       "      <td>1.699902</td>\n",
       "      <td>1.569342</td>\n",
       "      <td>...</td>\n",
       "      <td>0.928391</td>\n",
       "      <td>0.891245</td>\n",
       "      <td>0.851558</td>\n",
       "      <td>0.805795</td>\n",
       "      <td>0.721267</td>\n",
       "      <td>0.588290</td>\n",
       "      <td>0.399301</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.045720</td>\n",
       "      <td>-1.097423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.296087</td>\n",
       "      <td>1.254715</td>\n",
       "      <td>1.371302</td>\n",
       "      <td>1.391663</td>\n",
       "      <td>1.602395</td>\n",
       "      <td>1.804000</td>\n",
       "      <td>1.734784</td>\n",
       "      <td>1.708646</td>\n",
       "      <td>1.770531</td>\n",
       "      <td>1.559705</td>\n",
       "      <td>...</td>\n",
       "      <td>0.933297</td>\n",
       "      <td>0.898499</td>\n",
       "      <td>0.847763</td>\n",
       "      <td>0.790472</td>\n",
       "      <td>0.698572</td>\n",
       "      <td>0.551165</td>\n",
       "      <td>0.356246</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.699165</td>\n",
       "      <td>-1.169000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 390 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       eq_0     eq_10     eq_20     eq_30     eq_40     eq_60     eq_80  \\\n",
       "0  0.197786  0.475224  0.640533  0.877277  1.040508  1.037066  1.244018   \n",
       "1  1.010537  1.511049  1.229768  2.022973  2.413715  2.187437  2.187408   \n",
       "2  1.038472  1.468064  1.274101  2.044264  2.430573  2.155454  2.176625   \n",
       "3  0.947900  1.200151  1.109876  1.496561  1.800022  2.082368  1.825926   \n",
       "4  1.296087  1.254715  1.371302  1.391663  1.602395  1.804000  1.734784   \n",
       "\n",
       "     eq_120    eq_160    eq_230  ...  roll_128.logbands_mw_20.12  \\\n",
       "0  1.316525  1.339426  1.265299  ...                    0.934274   \n",
       "1  2.046839  1.960684  1.721527  ...                    0.911348   \n",
       "2  2.008776  1.843365  1.628750  ...                    0.917274   \n",
       "3  1.802636  1.699902  1.569342  ...                    0.928391   \n",
       "4  1.708646  1.770531  1.559705  ...                    0.933297   \n",
       "\n",
       "   roll_128.logbands_mw_20.13  roll_128.logbands_mw_20.14  \\\n",
       "0                    0.900729                    0.862285   \n",
       "1                    0.875964                    0.831477   \n",
       "2                    0.890996                    0.837766   \n",
       "3                    0.891245                    0.851558   \n",
       "4                    0.898499                    0.847763   \n",
       "\n",
       "   roll_128.logbands_mw_20.15  roll_128.logbands_mw_20.16  \\\n",
       "0                    0.812940                    0.730737   \n",
       "1                    0.783590                    0.703153   \n",
       "2                    0.787540                    0.686259   \n",
       "3                    0.805795                    0.721267   \n",
       "4                    0.790472                    0.698572   \n",
       "\n",
       "   roll_128.logbands_mw_20.17  roll_128.logbands_mw_20.18  \\\n",
       "0                    0.598143                    0.409162   \n",
       "1                    0.586626                    0.408304   \n",
       "2                    0.553205                    0.376001   \n",
       "3                    0.588290                    0.399301   \n",
       "4                    0.551165                    0.356246   \n",
       "\n",
       "   roll_128.logbands_mw_20.19  power_ratio  hits_ratio  \n",
       "0                         0.0     1.331745   -1.518403  \n",
       "1                         0.0    -0.259233   -0.858878  \n",
       "2                         0.0    -0.411571   -1.072553  \n",
       "3                         0.0    -0.045720   -1.097423  \n",
       "4                         0.0    -0.699165   -1.169000  \n",
       "\n",
       "[5 rows x 390 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the features\n",
    "# Note: These include only features described in a previous notebook as Feature Set 1 and 2, as 3 was not used in training the final classifiers\n",
    "\n",
    "features_df = pd.read_csv('large_data/extracted_features_futurama.csv', index_col=0)\n",
    "features_df.info()\n",
    "features_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "89466a4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a list of the keys of features used to train the final classifiers in the previous notebook\n",
    "\n",
    "classify_keys = ['eq_0', 'eq_10', 'eq_20', 'eq_30', 'eq_40', 'eq_60', 'eq_80', 'eq_120', 'eq_160', 'eq_230', 'eq_300', 'eq_450', 'eq_600', 'eq_900', 'eq_1200', 'eq_1800', 'eq_2400', 'eq_3700', 'eq_5000', 'eq_7500', 'eq_10000', 'eq_15000', 'eq_20000', 'crestfactor', 'harmonic_power', 'percussive_power', 'harmonic_hits', 'percussive_hits', 'power_ratio', 'hits_ratio', 'roll_32.spec_flatness.median', 'roll_32.spec_flatness.iqr', 'roll_32.spec_centroid.median', 'roll_32.spec_bandwidth.median', 'roll_32.spec_bandwidth.std', 'roll_32.y_mw_zcr', 'roll_64.spec_flatness.median', 'roll_64.spec_flatness.iqr', 'roll_64.spec_centroid.median', 'roll_64.spec_bandwidth.median', 'roll_64.spec_bandwidth.std', 'roll_64.y_mw_zcr', 'roll_128.spec_flatness.median', 'roll_128.spec_flatness.iqr', 'roll_128.spec_centroid.median', 'roll_128.spec_bandwidth.median', 'roll_128.spec_bandwidth.std', 'roll_128.y_mw_zcr', 'roll_64.logbands_mw_5.1', 'roll_64.logbands_mw_5.2', 'roll_64.logbands_mw_5.3', 'roll_64.logbands_mw_10.1', 'roll_64.logbands_mw_10.2', 'roll_64.logbands_mw_10.3', 'roll_64.logbands_mw_10.4', 'roll_64.logbands_mw_10.5', 'roll_64.logbands_mw_10.6', 'roll_64.logbands_mw_10.7', 'roll_64.logbands_mw_10.8', 'roll_64.logbands_mw_20.0', 'roll_64.logbands_mw_20.2', 'roll_64.logbands_mw_20.3', 'roll_64.logbands_mw_20.4', 'roll_64.logbands_mw_20.5', 'roll_64.logbands_mw_20.6', 'roll_64.logbands_mw_20.7', 'roll_64.logbands_mw_20.8', 'roll_64.logbands_mw_20.9', 'roll_64.logbands_mw_20.10', 'roll_64.logbands_mw_20.11', 'roll_64.logbands_mw_20.12', 'roll_64.logbands_mw_20.13', 'roll_64.logbands_mw_20.14', 'roll_64.logbands_mw_20.15', 'roll_64.logbands_mw_20.16', 'roll_64.logbands_mw_20.17', 'roll_64.logbands_mw_20.18']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f539c0a",
   "metadata": {},
   "source": [
    "We then import the pickled (i.e. saved) models for the logistic regression and random forest (These are not included on Github but can be generated at the end of the previous notebook). From here we will pass the data extracted from the futurama episode through the prediction function and analyze which clips, by index, are identified as gunshots."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0aeda46b",
   "metadata": {},
   "outputs": [],
   "source": [
    "forest = pickle.load(open('fullfeaturesRobustScaledRandomForest.pkl','rb'))\n",
    "logistic = pickle.load(open('fullfeaturesRobustScaledlogistic.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9c7b32c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = features_df[classify_keys].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8ad528dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_pred = forest.predict(X_test)\n",
    "logistic_pred = logistic.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6d710864",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 7 7 9 9 9 2 2 9 9 2 2 2 3 3 2 9 9 2 6 6 2 9 2 3 2 6 2 9 9 9 9 9 9 9 2 2\n",
      " 9 9 9 9 9 9 9 9 2 2 9 9 9 9 9 9 6 9 9 2 3 2 9 9 2 3 3 2 9 9 2 9 9 2 2 3 2\n",
      " 9 9 9 9 9 6 8 9 9 6 9 9 9 9 9 6 7 9 2 9 9 2 6 3 9 9 9 3 9 2 9 9 7 7 9 2 2\n",
      " 2 6 9 9 2 3 9 2 9 9 9 2 2 2 2 2 2 2 9 2 2 6 9 4 2 9 3 2 2 2 9 2 9 9 9 9 2\n",
      " 2 9 6 9 9 9 9 2 9 9 9 2 9 9 2 9 9 9 2 9 9 2 9 9 3 2 2 9 9 9 3 2 9 2 9 9 9\n",
      " 9 2 6 2 9 2 2 3 2 2 9 9 2 9 9 3 2 9 2 9 9 9 6 2 7 4 7 3 9 2 2 2 2 9 2 9 9\n",
      " 9 9 2 9 2 2 9 2 9 6 9 9 2 9 2 2 2 2 2 2 2 9 2 9 9 9 9 2 2 9 2 7 7 7 7 7 9\n",
      " 9]\n",
      "[4 9 5 4 6 0 2 2 9 4 2 9 2 3 2 3 8 8 2 3 2 2 9 2 2 8 3 3 2 6 9 0 2 9 9 2 2\n",
      " 9 9 9 9 2 9 2 9 9 8 9 9 4 9 9 9 6 9 9 9 8 2 3 9 8 2 3 2 2 9 8 9 9 2 9 3 2\n",
      " 9 9 9 9 9 9 9 5 9 8 9 9 9 9 9 9 1 9 2 8 9 9 6 8 8 8 9 3 9 3 9 9 7 4 3 3 2\n",
      " 2 2 9 9 2 2 9 2 9 3 9 2 8 8 8 2 9 2 9 2 2 3 6 2 6 9 2 2 2 2 9 2 3 2 9 6 2\n",
      " 2 9 6 9 9 9 9 8 9 9 2 2 9 9 9 3 9 9 3 9 9 8 9 9 2 2 2 2 2 6 2 3 3 8 9 9 9\n",
      " 2 2 8 2 3 2 2 2 2 2 9 2 2 2 2 2 3 9 9 9 2 8 2 2 3 9 0 3 6 9 2 2 2 9 8 9 2\n",
      " 3 9 8 8 2 2 9 8 2 3 9 6 8 2 2 2 8 2 2 9 2 3 8 6 9 9 9 2 8 9 2 4 9 9 5 5 9\n",
      " 3]\n"
     ]
    }
   ],
   "source": [
    "# Print the resulting predicted labels for each ~5 second interval\n",
    "print(forest_pred)\n",
    "print(logistic_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "212c0ab1",
   "metadata": {},
   "source": [
    "The models return a predicted class for each 5 second interval, as they were trained to do. Of course, most of these labels can not possibly be correct, as most sounds in the episode do not belong to the 10 sound classes the models were trained on. Nevertheless, we can estimate when in the episode gun_shot predictions were made. In our presentation slides, we compare these predictions with the episode's true content."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "24b9c36c",
   "metadata": {},
   "outputs": [],
   "source": [
    "desired_class = 6 # Recall from previous notebooks that classID for gun_shots is 6\n",
    "\n",
    "where_forest = np.where(forest_pred == desired_class)[0]\n",
    "where_logistic = np.where(logistic_pred == desired_class)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c05e949f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using the Random Forest model, possible gun shots were detected at approximate times:\n",
      "0:01:35s\n",
      "0:01:40s\n",
      "0:02:10s\n",
      "0:04:25s\n",
      "0:06:35s\n",
      "0:06:55s\n",
      "0:07:25s\n",
      "0:08:00s\n",
      "0:09:20s\n",
      "0:11:00s\n",
      "0:12:30s\n",
      "0:15:35s\n",
      "0:17:15s\n",
      "0:19:15s\n"
     ]
    }
   ],
   "source": [
    "print('Using the Random Forest model, possible gun shots were detected at approximate times:')\n",
    "for index in where_forest:\n",
    "    print(f'{timedelta(seconds=int(index)*5)}s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "85c22d45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using the Logistic Regression model, possible gun shots were detected at approximate times:\n",
      "0:00:20s\n",
      "0:02:25s\n",
      "0:04:25s\n",
      "0:08:00s\n",
      "0:11:05s\n",
      "0:11:15s\n",
      "0:12:10s\n",
      "0:12:30s\n",
      "0:14:45s\n",
      "0:17:45s\n",
      "0:19:25s\n",
      "0:20:25s\n"
     ]
    }
   ],
   "source": [
    "print('Using the Logistic Regression model, possible gun shots were detected at approximate times:')\n",
    "for index in where_logistic:\n",
    "    print(f'{timedelta(seconds=int(index)*5)}s')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}