RainforestNN-Test-Full.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import torch\n",
    "import torchaudio\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torchvision.models as vmodels\n",
    "import torchaudio.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "import glob\n",
    "import soundfile as sf\n",
    "from scipy.interpolate import interp1d\n",
    "import IPython.display as ipd\n",
    "import sys\n",
    "import time\n",
    "import pprint\n",
    "from nnutils import reshape_mel_spec, RainforestNetV2, Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'EPOCHS': 50, 'TRAIN_BATCH_SIZE': 4, 'LR': 0.0001, 'RAND_NOISE_UPPER': 0.1, 'RAND_NOISE_LOWER': 0.05, 'N_MELS': 224, 'SAMPLE_RATE': 48000, 'N_FFT': 5000, 'F_MIN': 90.0, 'F_MAX': 14000.0}\n"
     ]
    }
   ],
   "source": [
    "# Hyper parameters\n",
    "params = Params()\n",
    "print (params.__to_dict__())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:  True\n"
     ]
    }
   ],
   "source": [
    "# CUDA\n",
    "is_cuda = True and torch.cuda.is_available()\n",
    "device = torch.device('cuda') if is_cuda else torch.device('cpu')\n",
    "\n",
    "print ('cuda: ', is_cuda)\n",
    "\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_files = glob.glob( 'data/test/*.flac' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "test_res = RainforestNetV2()\n",
    "test_res.load_state_dict(torch.load('models/rainforest1608843907.662235.pt'))\n",
    "test_res.eval()\n",
    "print (test_res.training)\n",
    "test_res = test_res.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_files.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.set_printoptions(sci_mode=False)\n",
    "np.set_printoptions(suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0a0c280372b45b0aa1bc203d4c48f9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=1992.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "-------- DONE --------\n",
      "start time: 1608844067.9482539\n",
      "end time: 1608844213.5595324\n",
      "\n",
      "testing duration: 145.61127853393555\n"
     ]
    }
   ],
   "source": [
    "results = pd.DataFrame()\n",
    "print (device)\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "mel_trans = transforms.MelSpectrogram(sample_rate=params.SAMPLE_RATE, n_mels=params.N_MELS, n_fft=params.N_FFT, f_min=params.F_MIN, f_max=params.F_MAX).to(device)\n",
    "\n",
    "for f in tqdm(range(len(test_files))):\n",
    "    #ex = test_files.iloc[idx]\n",
    "    path = test_files[f]\n",
    "    waveform, sample_rate = torchaudio.load(path)\n",
    "    waveform = waveform.to(device)\n",
    "    \n",
    "    mel_spec = mel_trans(waveform)\n",
    "    \n",
    "    res = torch.zeros(24).to(device)\n",
    "    with torch.no_grad():\n",
    "        mel_spec = mel_spec.repeat(3, 1, 1)\n",
    "        mel_spec = mel_spec.unsqueeze(0)\n",
    "        mel_spec = mel_spec.to(device)\n",
    "\n",
    "        output = test_res(mel_spec)[0]\n",
    "        res = res+output\n",
    "    \n",
    "    res = res.to(torch.device('cpu')).numpy()    \n",
    "    results = results.append({ 'result': res }, ignore_index=True)\n",
    "\n",
    "end_time = time.time()\n",
    "print ('\\n-------- DONE --------')\n",
    "print ('start time: {}'.format(start_time))\n",
    "print ('end time: {}\\n'.format(end_time))\n",
    "\n",
    "duration = end_time-start_time\n",
    "print ('testing duration: {}'.format(duration))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0.078973874, 0.06721929, 0.078984186, 0.61286...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.05860769, 0.068104684, 0.062475156, 0.55882...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0.085184984, 0.081337556, 0.085454985, 0.1360...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0.9629003, 0.09849737, 0.07585755, 0.06292099...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.07171287, 0.07314431, 0.07082009, 0.0855429...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              result\n",
       "0  [0.078973874, 0.06721929, 0.078984186, 0.61286...\n",
       "1  [0.05860769, 0.068104684, 0.062475156, 0.55882...\n",
       "2  [0.085184984, 0.081337556, 0.085454985, 0.1360...\n",
       "3  [0.9629003, 0.09849737, 0.07585755, 0.06292099...\n",
       "4  [0.07171287, 0.07314431, 0.07082009, 0.0855429..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.set_printoptions(suppress=True, threshold=sys.maxsize)\n",
    "results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame()\n",
    "\n",
    "for idx in range(len(results)):\n",
    "    ex = results.iloc[idx]\n",
    "    recording_id = test_files[idx].replace('.flac', '').replace('data/test\\\\', '')\n",
    "    result = ex.result\n",
    "    \n",
    "    species = 0\n",
    "    o = {}\n",
    "    for r in result:\n",
    "        s = 's{}'.format(species)\n",
    "        o[s] = r\n",
    "        species += 1 \n",
    "    o[\"recording_id\"] = recording_id\n",
    "    submission = submission.append(o, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['recording_id', 's0', 's1', 's2', 's3', 's4', 's5', 's6', 's7', 's8', 's9', 's10', 's11', 's12', 's13', 's14', 's15', 's16', 's17', 's18', 's19', 's20', 's21', 's22', 's23']\n",
    "submission = submission[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>recording_id</th>\n",
       "      <th>s0</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>s6</th>\n",
       "      <th>s7</th>\n",
       "      <th>s8</th>\n",
       "      <th>...</th>\n",
       "      <th>s14</th>\n",
       "      <th>s15</th>\n",
       "      <th>s16</th>\n",
       "      <th>s17</th>\n",
       "      <th>s18</th>\n",
       "      <th>s19</th>\n",
       "      <th>s20</th>\n",
       "      <th>s21</th>\n",
       "      <th>s22</th>\n",
       "      <th>s23</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000316da7</td>\n",
       "      <td>0.078974</td>\n",
       "      <td>0.067219</td>\n",
       "      <td>0.078984</td>\n",
       "      <td>0.612862</td>\n",
       "      <td>0.069095</td>\n",
       "      <td>0.163462</td>\n",
       "      <td>0.238944</td>\n",
       "      <td>0.051775</td>\n",
       "      <td>0.077974</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053447</td>\n",
       "      <td>0.089859</td>\n",
       "      <td>0.081821</td>\n",
       "      <td>0.123917</td>\n",
       "      <td>0.072684</td>\n",
       "      <td>0.114141</td>\n",
       "      <td>0.089841</td>\n",
       "      <td>0.074515</td>\n",
       "      <td>0.091755</td>\n",
       "      <td>0.087553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>003bc2cb2</td>\n",
       "      <td>0.058608</td>\n",
       "      <td>0.068105</td>\n",
       "      <td>0.062475</td>\n",
       "      <td>0.558821</td>\n",
       "      <td>0.066234</td>\n",
       "      <td>0.064979</td>\n",
       "      <td>0.141251</td>\n",
       "      <td>0.086257</td>\n",
       "      <td>0.071711</td>\n",
       "      <td>...</td>\n",
       "      <td>0.047209</td>\n",
       "      <td>0.051390</td>\n",
       "      <td>0.366547</td>\n",
       "      <td>0.090765</td>\n",
       "      <td>0.076027</td>\n",
       "      <td>0.100275</td>\n",
       "      <td>0.084809</td>\n",
       "      <td>0.051803</td>\n",
       "      <td>0.074159</td>\n",
       "      <td>0.072912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0061c037e</td>\n",
       "      <td>0.085185</td>\n",
       "      <td>0.081338</td>\n",
       "      <td>0.085455</td>\n",
       "      <td>0.136067</td>\n",
       "      <td>0.085988</td>\n",
       "      <td>0.375832</td>\n",
       "      <td>0.352599</td>\n",
       "      <td>0.088961</td>\n",
       "      <td>0.081987</td>\n",
       "      <td>...</td>\n",
       "      <td>0.061683</td>\n",
       "      <td>0.095715</td>\n",
       "      <td>0.088121</td>\n",
       "      <td>0.131480</td>\n",
       "      <td>0.082336</td>\n",
       "      <td>0.168728</td>\n",
       "      <td>0.102916</td>\n",
       "      <td>0.096038</td>\n",
       "      <td>0.137964</td>\n",
       "      <td>0.141130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>010eb14d3</td>\n",
       "      <td>0.962900</td>\n",
       "      <td>0.098497</td>\n",
       "      <td>0.075858</td>\n",
       "      <td>0.062921</td>\n",
       "      <td>0.068998</td>\n",
       "      <td>0.071244</td>\n",
       "      <td>0.100396</td>\n",
       "      <td>0.074007</td>\n",
       "      <td>0.075570</td>\n",
       "      <td>...</td>\n",
       "      <td>0.077534</td>\n",
       "      <td>0.079532</td>\n",
       "      <td>0.084595</td>\n",
       "      <td>0.079223</td>\n",
       "      <td>0.257452</td>\n",
       "      <td>0.079426</td>\n",
       "      <td>0.080237</td>\n",
       "      <td>0.073081</td>\n",
       "      <td>0.068508</td>\n",
       "      <td>0.078710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>011318064</td>\n",
       "      <td>0.071713</td>\n",
       "      <td>0.073144</td>\n",
       "      <td>0.070820</td>\n",
       "      <td>0.085543</td>\n",
       "      <td>0.070030</td>\n",
       "      <td>0.060925</td>\n",
       "      <td>0.091752</td>\n",
       "      <td>0.066602</td>\n",
       "      <td>0.064507</td>\n",
       "      <td>...</td>\n",
       "      <td>0.954288</td>\n",
       "      <td>0.068642</td>\n",
       "      <td>0.084105</td>\n",
       "      <td>0.067214</td>\n",
       "      <td>0.065180</td>\n",
       "      <td>0.069332</td>\n",
       "      <td>0.064883</td>\n",
       "      <td>0.060741</td>\n",
       "      <td>0.082403</td>\n",
       "      <td>0.068262</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  recording_id        s0        s1        s2        s3        s4        s5  \\\n",
       "0    000316da7  0.078974  0.067219  0.078984  0.612862  0.069095  0.163462   \n",
       "1    003bc2cb2  0.058608  0.068105  0.062475  0.558821  0.066234  0.064979   \n",
       "2    0061c037e  0.085185  0.081338  0.085455  0.136067  0.085988  0.375832   \n",
       "3    010eb14d3  0.962900  0.098497  0.075858  0.062921  0.068998  0.071244   \n",
       "4    011318064  0.071713  0.073144  0.070820  0.085543  0.070030  0.060925   \n",
       "\n",
       "         s6        s7        s8  ...       s14       s15       s16       s17  \\\n",
       "0  0.238944  0.051775  0.077974  ...  0.053447  0.089859  0.081821  0.123917   \n",
       "1  0.141251  0.086257  0.071711  ...  0.047209  0.051390  0.366547  0.090765   \n",
       "2  0.352599  0.088961  0.081987  ...  0.061683  0.095715  0.088121  0.131480   \n",
       "3  0.100396  0.074007  0.075570  ...  0.077534  0.079532  0.084595  0.079223   \n",
       "4  0.091752  0.066602  0.064507  ...  0.954288  0.068642  0.084105  0.067214   \n",
       "\n",
       "        s18       s19       s20       s21       s22       s23  \n",
       "0  0.072684  0.114141  0.089841  0.074515  0.091755  0.087553  \n",
       "1  0.076027  0.100275  0.084809  0.051803  0.074159  0.072912  \n",
       "2  0.082336  0.168728  0.102916  0.096038  0.137964  0.141130  \n",
       "3  0.257452  0.079426  0.080237  0.073081  0.068508  0.078710  \n",
       "4  0.065180  0.069332  0.064883  0.060741  0.082403  0.068262  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(suppress=True, threshold=sys.maxsize)\n",
    "submission.to_csv(\"submissions/submission-{}.csv\".format(end_time), index=False, float_format='%.5f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rainforest_proj",
   "language": "python",
   "name": "rainforest_proj"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}