preprocess_data_interview_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This file will use trained models in IEMOCAP database and apply to all  small frame(about 4 seconds) of each interview record (3-5 minutes). The array with type 'Input' described below will be dumped to files eventually**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from sklearn.gaussian_process import GaussianProcessClassifier\n",
    "from sklearn.gaussian_process.kernels import RBF\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "import sys\n",
    "sys.path.insert(0, \"../\")\n",
    "from acousticFeatures import getAllFeatures\n",
    "import parselmouth \n",
    "import numpy as np\n",
    "from pydub import AudioSegment\n",
    "from IPython.display import Audio\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from pandas import DataFrame, read_csv\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd \n",
    "from os.path import isfile\n",
    "\n",
    "\n",
    "class Input:\n",
    "    def __init__(self, code=None, excited=None, valance=None, arouse=None):\n",
    "        self.__code = code #name file\n",
    "        self.__excited = excited # array of predicted excited value on each frame \n",
    "        self.__valance = valance # array of predicted valance value on each frame\n",
    "        self.__arouse = arouse #array of predicted arouse value on each frame\n",
    "    \n",
    "    def get_code(self):\n",
    "        return self.__code\n",
    "    \n",
    "    def get_excited(self):\n",
    "        return self.__excited\n",
    "    \n",
    "    def get_valance(self):\n",
    "        return self.__valance\n",
    "    \n",
    "    def get_arouse(self):\n",
    "        return self.__arouse\n",
    "\n",
    "        \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Path input and labels.\n",
    "AUDIO_FILES_PATH = \"auditary_emotion_recognition/data_interview/Audio/Audio\"\n",
    "AUDIO_LABEL_PATH = \"auditary_emotion_recognition/data_interview/Labels/turker_scores_full_interview.csv\"\n",
    "\n",
    "try:\n",
    "    #Load max, min, mean, std of acoustic features.\n",
    "    max_acoustic_features = pickle.load(open('processed-data/max_acoustic_features.obj', 'rb'))\n",
    "    min_acoustic_features = pickle.load(open('processed-data/min_acoustic_features.obj', 'rb'))\n",
    "    mean_acoustic_features = pickle.load(open('processed-data/mean_acoustic_features.obj', 'rb'))\n",
    "    std_acoustic_features = pickle.load(open('processed-data/std_acoustic_features.obj', 'rb'))\n",
    "\n",
    "    #Load model\n",
    "    model_classification_excited = pickle.load(open(\"model/model_classification_excited.sav\", 'rb'))\n",
    "    model_regression_valance = pickle.load(open(\"model/model_regression_valance.sav\", 'rb'))\n",
    "    model_regression_arouse = pickle.load(open(\"model/model_regression_arouse.sav\", 'rb'))\n",
    "except:\n",
    "    raise (\"Getting error when loading statistic of features or trained models!!!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_input(code=None, frame_length=4000, frame_overlap=2000):\n",
    "    file_path = AUDIO_FILES_PATH + \"/\" + code.upper() +\".wav\"\n",
    "    if not isfile(file_path):\n",
    "        print(\"File path is invalid\")\n",
    "    sound = AudioSegment.from_file(file_path)\n",
    "    \n",
    "    input = []\n",
    "    s = 0 # start frame\n",
    "    e = frame_length # end frame\n",
    "    while (True):\n",
    "        chunk = sound[s:e]\n",
    "        left, right = chunk.split_to_mono()\n",
    "        sound_frame = parselmouth.Sound(left.get_array_of_samples(), sound.frame_rate)\n",
    "\n",
    "        # Get acoustic features and normalize them\n",
    "        acoustic_features = np.array(getAllFeatures(sound_frame))\n",
    "        if(acoustic_features is not None and len(acoustic_features) > 10):\n",
    "            normalized_acoustic_features = (acoustic_features - min_acoustic_features) / (max_acoustic_features - min_acoustic_features)\n",
    "            input.append(normalized_acoustic_features)\n",
    "        \n",
    "        # Update start frame and end frame\n",
    "        s = e - frame_overlap\n",
    "        e = s + frame_length\n",
    "        \n",
    "        if (e > sound.duration_seconds * 1000):\n",
    "            break\n",
    "    \n",
    "    # Fill Nan values by mean values of columns\n",
    "    input = np.array(input)\n",
    "    col_mean = np.nanmean(input, axis=0)\n",
    "    inds = np.where(np.isnan(input))\n",
    "    input[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "    # Predict by frames.\n",
    "    excited = np.array(model_classification_excited.predict(np.array(input)))\n",
    "    valance = np.array(model_regression_valance.predict(np.array(input)))\n",
    "    arouse = np.array(model_regression_arouse.predict(np.array(input)))\n",
    "    \n",
    "    return  Input(code, excited, valance, arouse)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create input\n",
    "input = []\n",
    "df = pd.read_csv(\"auditary_emotion_recognition/data_interview/Labels/turker_scores_full_interview.csv\")\n",
    "for key in df.Participant.unique():\n",
    "    input.append(create_input(key))\n",
    "    print(\"Finish {}\".format(key))\n",
    "\n",
    "pickle.dump(input, open('processed-data/input-interview-frameLengh4-overlap2.obj', 'wb'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python3.6",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}