inference_dev.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import time\n",
    "from torchvision.models import *\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make models from str\n",
    "model_name = \"resnet18\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeTime(model, input_size=[1, 3, 224, 224], device='cuda', FP16=False):\n",
    "    inputs = torch.randn(input_size)\n",
    "    if device == 'cuda':\n",
    "        model = model.cuda()\n",
    "        inputs = inputs.cuda()\n",
    "    if FP16:\n",
    "        model = model.half()\n",
    "        inputs = inputs.half()\n",
    "\n",
    "    model.eval()\n",
    "\n",
    "    i = 0\n",
    "    time_spent = []\n",
    "    while i < 200:\n",
    "        start_time = time.time()\n",
    "        with torch.no_grad():\n",
    "            _ = model(inputs)\n",
    "\n",
    "        if device == 'cuda':\n",
    "            torch.cuda.synchronize() # wait for cuda to finish (cuda is asynchronous!)\n",
    "        if i != 0:\n",
    "            time_spent.append(time.time() - start_time)\n",
    "        i += 1\n",
    "    print('Avg execution time (ms): {:.3f}'.format(np.mean(time_spent)))\n",
    "    return np.mean(time_spent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "modellist = [\"resnet18\", \"resnet34\", \"resnet50\", \"resnet101\", \"resnet152\", \\\n",
    " \"resnext50_32x4d\", \"resnext101_32x8d\", \"mnasnet1_0\", \"squeezenet1_0\", \"densenet121\", \"densenet169\", \"inception_v3\"]\n",
    "\n",
    "modellist = [\"resnet18\", \"resnet34\", \"resnet50\", \"resnet101\", \"resnet152\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model: resnet18\n",
      "Avg execution time (ms): 0.008\n",
      "Avg execution time (ms): 0.019\n",
      "Avg execution time (ms): 0.046\n",
      "Avg execution time (ms): 0.040\n"
     ]
    }
   ],
   "source": [
    "for i, model_name in enumerate(modellist):\n",
    "    batchlist = [1, 4, 8, 16, 32]\n",
    "    imsize = [128, 256, 512]\n",
    "    runtimes = []\n",
    "    \n",
    "    # define model\n",
    "    print(\"model: {}\".format(model_name))\n",
    "    mdl = globals()[model_name]\n",
    "    model = mdl()\n",
    "    \n",
    "    for batch in batchlist:        \n",
    "        runtimes.append(computeTime(model, input_size=[batch, 3, 256, 256], device=\"cuda\", FP16=False)/batch)\n",
    "\n",
    "    if i == 0:\n",
    "        dfbatch = pd.DataFrame({model_name: runtimes},\n",
    "                         index = batchlist)\n",
    "    else:\n",
    "        dfbatch[model_name] = runtimes\n",
    "        \n",
    "    runtimes = []\n",
    "    for isize in imsize:\n",
    "        print(\"model: {}\".format(model_name))\n",
    "        mdl = globals()[model_name]\n",
    "        model = mdl()\n",
    "        runtimes.append(computeTime(model, input_size=[1, 3, isize, isize], device=\"cuda\", FP16=False))\n",
    "\n",
    "    if i == 0:\n",
    "        dfimsize = pd.DataFrame({model_name: runtimes},\n",
    "                         index = imsize)\n",
    "    else:\n",
    "        dfimsize[model_name] = runtimes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>resnet18</th>\n",
       "      <th>resnet34</th>\n",
       "      <th>resnet50</th>\n",
       "      <th>resnet101</th>\n",
       "      <th>resnet152</th>\n",
       "      <th>resnext50_32x4d</th>\n",
       "      <th>resnext101_32x8d</th>\n",
       "      <th>mnasnet1_0</th>\n",
       "      <th>squeezenet1_0</th>\n",
       "      <th>densenet121</th>\n",
       "      <th>densenet169</th>\n",
       "      <th>inception_v3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cuda FP32</th>\n",
       "      <td>0.006895</td>\n",
       "      <td>0.013512</td>\n",
       "      <td>0.016632</td>\n",
       "      <td>0.032939</td>\n",
       "      <td>0.048400</td>\n",
       "      <td>0.033309</td>\n",
       "      <td>0.118709</td>\n",
       "      <td>0.007704</td>\n",
       "      <td>0.004120</td>\n",
       "      <td>0.025061</td>\n",
       "      <td>0.037639</td>\n",
       "      <td>0.027673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cuda FP16</th>\n",
       "      <td>0.007969</td>\n",
       "      <td>0.015316</td>\n",
       "      <td>0.017940</td>\n",
       "      <td>0.035898</td>\n",
       "      <td>0.052364</td>\n",
       "      <td>0.033756</td>\n",
       "      <td>0.114106</td>\n",
       "      <td>0.007777</td>\n",
       "      <td>0.003966</td>\n",
       "      <td>0.022630</td>\n",
       "      <td>0.034838</td>\n",
       "      <td>0.030210</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           resnet18  resnet34  resnet50  resnet101  resnet152  \\\n",
       "cuda FP32  0.006895  0.013512  0.016632   0.032939   0.048400   \n",
       "cuda FP16  0.007969  0.015316  0.017940   0.035898   0.052364   \n",
       "\n",
       "           resnext50_32x4d  resnext101_32x8d  mnasnet1_0  squeezenet1_0  \\\n",
       "cuda FP32         0.033309          0.118709    0.007704       0.004120   \n",
       "cuda FP16         0.033756          0.114106    0.007777       0.003966   \n",
       "\n",
       "           densenet121  densenet169  inception_v3  \n",
       "cuda FP32     0.025061     0.037639      0.027673  \n",
       "cuda FP16     0.022630     0.034838      0.030210  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}