preprocess niue segmented.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rishi\\AppData\\Local\\Temp\\ipykernel_11612\\1479078273.py:1: DeprecationWarning: \n",
      "Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),\n",
      "(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)\n",
      "but was not found to be installed on your system.\n",
      "If this would cause problems for you,\n",
      "please provide us feedback at https://github.com/pandas-dev/pandas/issues/54466\n",
      "        \n",
      "  import pandas as pd\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import librosa\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(r\"C:\\Users\\rishi\\OneDrive - Monash University\\Documents\\Monash\\Sonorus\\Sonorus - nuie data\\Data\\niue outcomes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Record_ID</th>\n",
       "      <th>File_name</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>065150_M.WAV</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>064736_A.WAV</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>064926_P.WAV</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>065025_T.WAV</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>070521_M.WAV</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Record_ID     File_name  Outcome\n",
       "0         1  065150_M.WAV        0\n",
       "1         1  064736_A.WAV        0\n",
       "2         1  064926_P.WAV        0\n",
       "3         1  065025_T.WAV        0\n",
       "4         2  070521_M.WAV        0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Record_ID</th>\n",
       "      <th>File_name</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>855</th>\n",
       "      <td>90a</td>\n",
       "      <td>124458_P.WAV</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>856</th>\n",
       "      <td>90b</td>\n",
       "      <td>140843_A.WAV</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>90b</td>\n",
       "      <td>140936_P.WAV</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858</th>\n",
       "      <td>90b</td>\n",
       "      <td>141122_M.WAV</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>859</th>\n",
       "      <td>90b</td>\n",
       "      <td>141031_T.WAV</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Record_ID     File_name  Outcome\n",
       "855       90a  124458_P.WAV        1\n",
       "856       90b  140843_A.WAV        1\n",
       "857       90b  140936_P.WAV        1\n",
       "858       90b  141122_M.WAV        1\n",
       "859       90b  141031_T.WAV        1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "228 10a\n"
     ]
    }
   ],
   "source": [
    "print(df.iloc[804, 0], df.iloc[805, 0])\n",
    "#the file loading method uptill 804 and after 804 index is different.\n",
    "#a represents first recording, and b represents second recording. for some ids the data is recorded twice."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pad signal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_signal(data, max_len):\n",
    "    #pad_length - 1422490 = max_length of the signals in the dataset\n",
    "    #pad_length after spectrogram = 355712 = max length of the spectrograms\n",
    "    new_arr = np.zeros(max_len)\n",
    "    if len(data) > max_len:\n",
    "        new_arr = data[:max_len]\n",
    "    else:\n",
    "        new_arr[:len(data)] = data\n",
    "    return new_arr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess first recording"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rishi\\AppData\\Local\\Temp\\ipykernel_11612\\3350238616.py:11: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  audio_file, sr = librosa.load(file_path, sr=None)\n",
      "c:\\Users\\rishi\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\librosa\\core\\audio.py:183: FutureWarning: librosa.core.audio.__audioread_load\n",
      "\tDeprecated as of librosa version 0.10.0.\n",
      "\tIt will be removed in librosa version 1.0.\n",
      "  y, sr_native = __audioread_load(path, offset, duration, dtype)\n"
     ]
    }
   ],
   "source": [
    "specs = []\n",
    "outcome = []\n",
    "for i in range(805): #\n",
    "\n",
    "    ##get file path\n",
    "    try: \n",
    "        file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-001\\\\Recor\\\\\"\n",
    "        file_path += df.loc[i, \"Record_ID\"] + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "        #load file\n",
    "        audio_file, sr = librosa.load(file_path, sr=None)\n",
    "    except:\n",
    "        file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-002\\\\Recor\\\\\"\n",
    "        file_path += df.loc[i, \"Record_ID\"] + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "        #load file\n",
    "        audio_file, sr = librosa.load(file_path, sr=None)\n",
    "\n",
    "    \n",
    "    #get 5 second segments of the audio\n",
    "    for j in range(0, len(audio_file), 5*sr):\n",
    "        segment = audio_file[j:j+5*sr]\n",
    "        #padding - for the last segment since it might be less than 5 seconds\n",
    "        padded_audio = pad_signal(segment, 5*sr)\n",
    "        #spectrogram\n",
    "        spec = librosa.feature.melspectrogram(y=padded_audio, sr=sr)\n",
    "        specs.append(spec)\n",
    "\n",
    "    Number_of_segments = len(audio_file)/(5*sr)\n",
    "    for j in range(int(Number_of_segments)):\n",
    "        #outcome\n",
    "        if df.loc[i, \"Outcome\"] == 0: #normal\n",
    "            outcome.append(np.array([1, 0])) #normal = 1, abnormal = 0  \n",
    "        else:    \n",
    "            outcome.append(np.array([0, 1])) #normal = 0, abnormal = 1\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7245 7245 (128, 40)\n"
     ]
    }
   ],
   "source": [
    "print(len(specs), len(outcome), specs[0].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess second recording"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rishi\\AppData\\Local\\Temp\\ipykernel_11612\\917655890.py:25: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  audio_file, sr = librosa.load(file_path, sr=None)\n",
      "C:\\Users\\rishi\\AppData\\Local\\Temp\\ipykernel_11612\\917655890.py:33: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  audio_file, sr = librosa.load(file_path, sr=None)\n",
      "C:\\Users\\rishi\\AppData\\Local\\Temp\\ipykernel_11612\\917655890.py:42: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  audio_file, sr = librosa.load(file_path, sr=None)\n",
      "C:\\Users\\rishi\\AppData\\Local\\Temp\\ipykernel_11612\\917655890.py:50: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  audio_file, sr = librosa.load(file_path, sr=None)\n",
      "C:\\Users\\rishi\\AppData\\Local\\Temp\\ipykernel_11612\\917655890.py:58: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  audio_file, sr = librosa.load(file_path, sr=None)\n",
      "C:\\Users\\rishi\\AppData\\Local\\Temp\\ipykernel_11612\\917655890.py:9: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  audio_file, sr = librosa.load(file_path, sr=None)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for i in range(805, 860):\n",
    "    #------a------------------\n",
    "    if df.loc[i, \"Record_ID\"][-1] == 'a':\n",
    "        try:\n",
    "            file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-001\\\\Recor\\\\\"\n",
    "            file_path += df.loc[i, \"Record_ID\"][:-1] + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "            #load file\n",
    "            audio_file, sr = librosa.load(file_path, sr=None)\n",
    "        except:\n",
    "            file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-002\\\\Recor\\\\\"\n",
    "            file_path += df.loc[i, \"Record_ID\"][:-1] + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "            #load file\n",
    "            audio_file, sr = librosa.load(file_path, sr=None)\n",
    "\n",
    "    #------b------------------\n",
    "    else:\n",
    "        try:\n",
    "            #---------2nd scan under 002----------------\n",
    "            file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-002\\\\Recor\\\\\"\n",
    "            file_path += df.loc[i, \"Record_ID\"][:-1] + \"\\\\\" + \"2nd scan\" + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "            #load file\n",
    "            audio_file, sr = librosa.load(file_path, sr=None)\n",
    "            #--------2nd test under 002----------------\n",
    "        except:\n",
    "            try:\n",
    "                file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-002\\\\Recor\\\\\"\n",
    "                file_path += df.loc[i, \"Record_ID\"][:-1] + \"\\\\\" + \"2nd test\" + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "                #load file\n",
    "                audio_file, sr = librosa.load(file_path, sr=None)\n",
    "            \n",
    "            except:\n",
    "                try: \n",
    "                    #--------second test under 002----------------\n",
    "                    file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-002\\\\Recor\\\\\"\n",
    "                    file_path += df.loc[i, \"Record_ID\"][:-1] + \"\\\\\" + \"second test\" + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "                    #load file\n",
    "                    audio_file, sr = librosa.load(file_path, sr=None)\n",
    "                    #--------2nd tets under 002----------------\n",
    "                except:\n",
    "                    try:\n",
    "                        file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-002\\\\Recor\\\\\"\n",
    "                        file_path += df.loc[i, \"Record_ID\"][:-1] + \"\\\\\" + \"2nd tets\" + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "                        #load file\n",
    "                        audio_file, sr = librosa.load(file_path, sr=None)\n",
    "                    #--------2nd scan but under 001----------------\n",
    "                    except:\n",
    "                        try:\n",
    "                            file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-001\\\\Recor\\\\\"\n",
    "                            file_path += df.loc[i, \"Record_ID\"][:-1] + \"\\\\\" + \"2nd scan\" + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "                            #load file\n",
    "                            audio_file, sr = librosa.load(file_path, sr=None)\n",
    "                        except:\n",
    "                            \n",
    "                            #--------2nd test but under 001----------------\n",
    "                            file_path = \"C:\\\\Users\\\\rishi\\\\OneDrive - Monash University\\\\Documents\\\\Monash\\\\Sonorus\\\\Sonorus - nuie data\\\\Data\\\\Recor-20240305T001003Z-001\\\\Recor\\\\\"\n",
    "                            file_path += df.loc[i, \"Record_ID\"][:-1] + \"\\\\\" + \"2nd test\" + \"\\\\\" + df.loc[i, \"File_name\"]\n",
    "\n",
    "                            #load file\n",
    "                            audio_file, sr = librosa.load(file_path, sr=None)\n",
    "                            \n",
    "    \n",
    "    #get 5 second segments of the audio\n",
    "    for j in range(0, len(audio_file), 5*sr):\n",
    "        segment = audio_file[j:j+5*sr]\n",
    "        #padding - for the last segment since it might be less than 5 seconds\n",
    "        padded_audio = pad_signal(segment, 5*sr)\n",
    "        #spectrogram\n",
    "        spec = librosa.feature.melspectrogram(y=padded_audio, sr=sr)\n",
    "        specs.append(spec)\n",
    "\n",
    "    Number_of_segments = len(audio_file)/(5*sr)\n",
    "    for j in range(int(Number_of_segments)):\n",
    "        #outcome\n",
    "        if df.loc[i, \"Outcome\"] == 0: #normal\n",
    "            outcome.append(np.array([1, 0])) #normal = 1, abnormal = 0  \n",
    "        else:    \n",
    "            outcome.append(np.array([0, 1])) #normal = 0, abnormal = 1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7740 7740 (128, 40)\n"
     ]
    }
   ],
   "source": [
    "print(len(specs), len(outcome), specs[0].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "specs = np.array(specs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7740, 128, 40)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "specs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(r\"C:\\Users\\rishi\\OneDrive - Monash University\\Documents\\Monash\\Sonorus\\Sonorus - nuie data\\preprocessed data\\specs segmented.npy\", specs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "outcome = np.array(outcome)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7740, 2)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outcome.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(r\"C:\\Users\\rishi\\OneDrive - Monash University\\Documents\\Monash\\Sonorus\\Sonorus - nuie data\\preprocessed data\\outcome segmented.npy\", outcome)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}