Testing the Model -d -c.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6468ee9d-8a03-42ba-a5e4-5ce9694aaca6",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "914fa10d-d3be-49bb-b2df-e3e9930763fe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:17.419168Z",
     "iopub.status.busy": "2021-06-29T23:50:17.419168Z",
     "iopub.status.idle": "2021-06-29T23:50:22.220528Z",
     "shell.execute_reply": "2021-06-29T23:50:22.218526Z",
     "shell.execute_reply.started": "2021-06-29T23:50:17.419168Z"
    },
    "id": "330dfc9b-4e7e-4262-91c0-0baec3158880",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "import os\n",
    "import sys\n",
    "from datetime import datetime\n",
    "import time\n",
    "from pprint import pprint\n",
    "from contextlib import redirect_stdout\n",
    "import re\n",
    "import io\n",
    "\n",
    "# import itertools\n",
    "from natsort import natsorted\n",
    "from icecream import ic\n",
    "# import logging  # https://gist.github.com/joshbode/58fac7ababc700f51e2a9ecdebe563ad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5e80c9d3-adba-4600-9e59-27ddc0211936",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:22.223531Z",
     "iopub.status.busy": "2021-06-29T23:50:22.222530Z",
     "iopub.status.idle": "2021-06-29T23:50:28.517005Z",
     "shell.execute_reply": "2021-06-29T23:50:28.516003Z",
     "shell.execute_reply.started": "2021-06-29T23:50:22.223531Z"
    },
    "id": "e039f65a-4b32-405a-a4ae-95afdc2e8eb1",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "# import matplotlib.image as mpimg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b1d6eaf2-46ce-4bf2-8b37-9435129517bc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:28.520004Z",
     "iopub.status.busy": "2021-06-29T23:50:28.520004Z",
     "iopub.status.idle": "2021-06-29T23:50:28.533006Z",
     "shell.execute_reply": "2021-06-29T23:50:28.532009Z",
     "shell.execute_reply.started": "2021-06-29T23:50:28.520004Z"
    },
    "id": "e17e0443-a109-4fb8-a5f1-d25e2d4d53a8",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import imgaug as ia\n",
    "# from imgaug import augmenters as iaa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "42f6f662-4bf4-479f-8f5b-f038a9f11aaa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:28.535005Z",
     "iopub.status.busy": "2021-06-29T23:50:28.535005Z",
     "iopub.status.idle": "2021-06-29T23:50:28.596008Z",
     "shell.execute_reply": "2021-06-29T23:50:28.595009Z",
     "shell.execute_reply.started": "2021-06-29T23:50:28.535005Z"
    },
    "id": "iQBpiIhqjVk6",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# attention\n",
    "am_I_using_colab = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7f8dd5b1-d863-4b21-962e-8116485e2266",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:28.597011Z",
     "iopub.status.busy": "2021-06-29T23:50:28.597011Z",
     "iopub.status.idle": "2021-06-29T23:50:28.628011Z",
     "shell.execute_reply": "2021-06-29T23:50:28.627010Z",
     "shell.execute_reply.started": "2021-06-29T23:50:28.597011Z"
    },
    "id": "4d9436d0-fb2f-496d-840f-c5a0a4e1eac9",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# if am_I_using_colab:\n",
    "#     !pip install -U tensorflow-addons\n",
    "    \n",
    "# import tensorflow_addons as tfa\n",
    "# import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "05172f42-8dfb-4095-85bf-30f1e686072f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:28.630011Z",
     "iopub.status.busy": "2021-06-29T23:50:28.630011Z",
     "iopub.status.idle": "2021-06-29T23:50:28.660014Z",
     "shell.execute_reply": "2021-06-29T23:50:28.659014Z",
     "shell.execute_reply.started": "2021-06-29T23:50:28.630011Z"
    },
    "id": "3dd459d4-e3e1-4601-aa6e-7d4a01a0e594",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# plt.ioff()  # interactive mode off\n",
    "# plt.ion()  # interactive mode on\n",
    "\n",
    "# %pylab inline\n",
    "\n",
    "np.random.seed(1728)\n",
    "# ia.random.seed(1728)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a5c26251-dd33-4889-8ab6-a67ba0c75229",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:28.662015Z",
     "iopub.status.busy": "2021-06-29T23:50:28.662015Z",
     "iopub.status.idle": "2021-06-29T23:50:28.708016Z",
     "shell.execute_reply": "2021-06-29T23:50:28.707015Z",
     "shell.execute_reply.started": "2021-06-29T23:50:28.662015Z"
    },
    "id": "S04tKXrZieAG",
    "outputId": "128332fe-95fd-4a94-91d1-88de817b672c",
    "tags": []
   },
   "outputs": [],
   "source": [
    "if am_I_using_colab:\n",
    "    ###\n",
    "    from google.colab import drive\n",
    "    drive.mount('/content/drive')\n",
    "\n",
    "    print(os.getcwd())\n",
    "    print(os.listdir())\n",
    "\n",
    "    ###\n",
    "    temp_dir = '/content/temp'\n",
    "    try:\n",
    "        os.mkdir(temp_dir)\n",
    "    except:\n",
    "        pass\n",
    "    print(os.listdir(temp_dir))\n",
    "\n",
    "    ###\n",
    "    dataset_filepath = f'{temp_dir:s}/dataset_v1_test.7z'\n",
    "    import gdown\n",
    "    gdown.download(\n",
    "        r'https://drive.google.com/uc?id=1PPZ7WgKVsPjvfkQN33IItxOIfm2RjOBY',\n",
    "        output=dataset_filepath,\n",
    "        quiet=False,\n",
    "        )\n",
    "\n",
    "    ###\n",
    "    !pip install py7zr\n",
    "    import py7zr\n",
    "    with py7zr.SevenZipFile(dataset_filepath, 'r') as archive:\n",
    "        archive.extractall(path=\"./dataset_v1_test\")\n",
    "\n",
    "    ###\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1020d8af-f630-4c51-a75e-a567350e7422",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:28.710016Z",
     "iopub.status.busy": "2021-06-29T23:50:28.709017Z",
     "iopub.status.idle": "2021-06-29T23:50:28.899037Z",
     "shell.execute_reply": "2021-06-29T23:50:28.898033Z",
     "shell.execute_reply.started": "2021-06-29T23:50:28.710016Z"
    },
    "id": "5a98c5bd-a334-447c-89ea-b7080052ceb2",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n",
      "(155.01 ms) == (  0s)\n",
      "-----\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'  0s'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "def mulai_hitung_waktu():\n",
    "    global waktu_mulai\n",
    "    waktu_mulai = time.time()\n",
    "\n",
    "def cetak_lama_waktu(to='ms'):\n",
    "    global waktu_mulai\n",
    "    now = time.time()\n",
    "    hasil_detik = abs(waktu_mulai - now)\n",
    "    hasil_milidetik = hasil_detik * 1000\n",
    "    \n",
    "    menit = hasil_detik / 60\n",
    "    detik = hasil_detik % 60\n",
    "    \n",
    "    \n",
    "    if to == 's':\n",
    "        waktu = f\"{int(hasil_detik):3d}s\"\n",
    "    elif to == 'ms':\n",
    "        waktu = f\"{int(menit):3d}m:{int(detik):2d}s\" # :{int(hasil_milidetik%1000):3d}ms\n",
    "        \n",
    "    print('-----\\n(%.2f ms) == (%s)\\n-----' % (hasil_milidetik, waktu))\n",
    "    return waktu\n",
    "    del waktu_mulai\n",
    "\n",
    "\n",
    "# cara pakai\n",
    "# ----------\n",
    "mulai_hitung_waktu()  ###\n",
    "\n",
    "time.sleep(150/1000)  # time-consuming computing here\n",
    "\n",
    "cetak_lama_waktu('s')  ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7092f693-e2a3-49d4-982f-145d3a3fed96",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ec964260-d13f-4253-8b90-3e6b76a4de1e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:28.903034Z",
     "iopub.status.busy": "2021-06-29T23:50:28.902035Z",
     "iopub.status.idle": "2021-06-29T23:50:29.988123Z",
     "shell.execute_reply": "2021-06-29T23:50:29.987116Z",
     "shell.execute_reply.started": "2021-06-29T23:50:28.903034Z"
    },
    "id": "530dc6d4-fdac-4a36-bcc1-829b7788007f",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available: 1 Physical GPUs, 1 Logical GPU\n"
     ]
    }
   ],
   "source": [
    "gpus = tf.config.list_physical_devices('GPU')\n",
    "\n",
    "if gpus:\n",
    "    # Restrict TensorFlow to only use the first GPU\n",
    "    try:\n",
    "        tf.config.experimental.set_visible_devices(gpus[0], 'GPU')\n",
    "        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        print(\"Num GPUs Available: %d Physical GPUs, %d Logical GPU\" % (len(gpus), len(logical_gpus)))\n",
    "    except RuntimeError as e:\n",
    "        # Visible devices must be set before GPUs have been initialized\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "074116b2-7b33-4ed4-a8f8-07d1be48b187",
   "metadata": {
    "id": "n_eonvAiKs1J"
   },
   "source": [
    "## Setup Directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a331ed47-dc1e-453e-8d19-d8f7aafae383",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:29.991116Z",
     "iopub.status.busy": "2021-06-29T23:50:29.991116Z",
     "iopub.status.idle": "2021-06-29T23:50:30.052117Z",
     "shell.execute_reply": "2021-06-29T23:50:30.051116Z",
     "shell.execute_reply.started": "2021-06-29T23:50:29.991116Z"
    },
    "id": "6187d76c-a4e2-4d62-b29f-b2e57ae5b86f",
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current work dir:\n",
      "D:\\Model testing\n",
      "\n",
      "listdir WORK_DIR:\n",
      "2\n",
      "['results', 'weights and models']\n"
     ]
    }
   ],
   "source": [
    "if am_I_using_colab:\n",
    "    WORK_DIR = '/content'\n",
    "    TRAIN_SET_PATH = f'{WORK_DIR}/dataset_v1'\n",
    "    # TEST_SET_PATH = f''\n",
    "\n",
    "    DRIVE_DIR = '/content/drive/MyDrive/MyNotebook/MySkripsi'\n",
    "    CHECKPOINTS_DIR = f'{WORK_DIR}/checkpoints'\n",
    "    FINAL_EPOCH_DIR = f'{WORK_DIR}/weights and models'\n",
    "\n",
    "else:\n",
    "    WORK_DIR = r'D:\\Model testing'\n",
    "    TRAIN_SET_PATH = r'D:\\MaskTheFace\\datasets\\_temp\\_v1'\n",
    "    TEST_SET_PATH = r'D:\\MaskTheFace\\datasets\\_temp\\_v1 - test only'\n",
    "\n",
    "WEIGHTS_AND_MODELS_DIR = fr'{WORK_DIR}\\weights and models'\n",
    "TEST_RESULTS_DIR = fr'{WORK_DIR}\\results'\n",
    "\n",
    "directories = [\n",
    "    WEIGHTS_AND_MODELS_DIR,\n",
    "    TEST_RESULTS_DIR\n",
    "]\n",
    "\n",
    "# Set working directory\n",
    "os.chdir(WORK_DIR)\n",
    "\n",
    "print('current work dir:')\n",
    "print(os.getcwd())\n",
    "print()\n",
    "print('listdir WORK_DIR:')\n",
    "print(len(os.listdir('./')))\n",
    "pprint(natsorted(os.listdir('./')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7f0674a8-cfa0-4b8e-8e64-849d88a10cf9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.054118Z",
     "iopub.status.busy": "2021-06-29T23:50:30.053122Z",
     "iopub.status.idle": "2021-06-29T23:50:30.099121Z",
     "shell.execute_reply": "2021-06-29T23:50:30.098120Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.053122Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "listdir WEIGHTS_AND_MODELS_DIR:\n",
      "63\n",
      "['lenet5_actual_32-16',\n",
      " 'lenet5_actual_32-32',\n",
      " 'lenet5_actual_32-64',\n",
      " 'lenet5_actual_rm-32-16',\n",
      " 'lenet5_actual_rm-32-32',\n",
      " 'lenet5_actual_rm-32-64',\n",
      " 'lenet5_actual_rm-64-16',\n",
      " 'lenet5_actual_rm-64-32',\n",
      " 'lenet5_actual_rm-64-64',\n",
      " 'lenet5_actual_rm-96-16',\n",
      " 'lenet5_actual_rm-96-32',\n",
      " 'lenet5_actual_rm-96-64',\n",
      " 'lenet5_actual_rm-128-16',\n",
      " 'lenet5_actual_rm-128-32',\n",
      " 'lenet5_actual_rm-128-64',\n",
      " 'mobilenetv2_1.00_32-16-12',\n",
      " 'mobilenetv2_1.00_32-16-fc12',\n",
      " 'mobilenetv2_1.00_32-32-12',\n",
      " 'mobilenetv2_1.00_32-32-fc12',\n",
      " 'mobilenetv2_1.00_32-64-12',\n",
      " 'mobilenetv2_1.00_32-64-fc12',\n",
      " 'mobilenetv2_1.00_32-imagenet224-16-12',\n",
      " 'mobilenetv2_1.00_32-imagenet224-16-fc12',\n",
      " 'mobilenetv2_1.00_32-imagenet224-32-12',\n",
      " 'mobilenetv2_1.00_32-imagenet224-32-fc12',\n",
      " 'mobilenetv2_1.00_32-imagenet224-64-12',\n",
      " 'mobilenetv2_1.00_32-imagenet224-64-fc12',\n",
      " 'mobilenetv2_1.00_64-16-12',\n",
      " 'mobilenetv2_1.00_64-16-fc12',\n",
      " 'mobilenetv2_1.00_64-32-12',\n",
      " 'mobilenetv2_1.00_64-32-fc12',\n",
      " 'mobilenetv2_1.00_64-64-12',\n",
      " 'mobilenetv2_1.00_64-64-fc12',\n",
      " 'mobilenetv2_1.00_64-imagenet224-16-12',\n",
      " 'mobilenetv2_1.00_64-imagenet224-16-fc12',\n",
      " 'mobilenetv2_1.00_64-imagenet224-32-12',\n",
      " 'mobilenetv2_1.00_64-imagenet224-32-fc12',\n",
      " 'mobilenetv2_1.00_64-imagenet224-64-12',\n",
      " 'mobilenetv2_1.00_64-imagenet224-64-fc12',\n",
      " 'mobilenetv2_1.00_96-16-12',\n",
      " 'mobilenetv2_1.00_96-16-fc12',\n",
      " 'mobilenetv2_1.00_96-32-12',\n",
      " 'mobilenetv2_1.00_96-32-fc12',\n",
      " 'mobilenetv2_1.00_96-64-12',\n",
      " 'mobilenetv2_1.00_96-64-fc12',\n",
      " 'mobilenetv2_1.00_96-imagenet96-16-12',\n",
      " 'mobilenetv2_1.00_96-imagenet96-16-fc12',\n",
      " 'mobilenetv2_1.00_96-imagenet96-32-12',\n",
      " 'mobilenetv2_1.00_96-imagenet96-32-fc12',\n",
      " 'mobilenetv2_1.00_96-imagenet96-64-12',\n",
      " 'mobilenetv2_1.00_96-imagenet96-64-fc12',\n",
      " 'mobilenetv2_1.00_128-16-12',\n",
      " 'mobilenetv2_1.00_128-16-fc12',\n",
      " 'mobilenetv2_1.00_128-32-12',\n",
      " 'mobilenetv2_1.00_128-32-fc12',\n",
      " 'mobilenetv2_1.00_128-64-12',\n",
      " 'mobilenetv2_1.00_128-64-fc12',\n",
      " 'mobilenetv2_1.00_128-imagenet128-16-12',\n",
      " 'mobilenetv2_1.00_128-imagenet128-16-fc12',\n",
      " 'mobilenetv2_1.00_128-imagenet128-32-12',\n",
      " 'mobilenetv2_1.00_128-imagenet128-32-fc12',\n",
      " 'mobilenetv2_1.00_128-imagenet128-64-12',\n",
      " 'mobilenetv2_1.00_128-imagenet128-64-fc12']\n"
     ]
    }
   ],
   "source": [
    "print('listdir WEIGHTS_AND_MODELS_DIR:')\n",
    "model_names = natsorted(os.listdir(WEIGHTS_AND_MODELS_DIR))\n",
    "\n",
    "print(len(model_names))\n",
    "pprint(model_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "eb82f420-18b8-4085-a6f0-30db0b68fb0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.100123Z",
     "iopub.status.busy": "2021-06-29T23:50:30.100123Z",
     "iopub.status.idle": "2021-06-29T23:50:30.130130Z",
     "shell.execute_reply": "2021-06-29T23:50:30.129128Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.100123Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "listdir TEST_RESULTS_DIR:\n",
      "7\n",
      "['lenet-5 versi asli (035).evaluate_turbo',\n",
      " 'lenet-5 versi asli (035).predict_turbo',\n",
      " 'lenet-5 versi asli.evaluate_turbo',\n",
      " 'lenet-5 versi asli.predict_turbo',\n",
      " 'lenet-5 versi modifikasi (035).evaluate_turbo',\n",
      " 'lenet-5 versi modifikasi.evaluate_turbo',\n",
      " 'lenet-5 versi modifikasi.predict_turbo']\n"
     ]
    }
   ],
   "source": [
    "print('listdir TEST_RESULTS_DIR:')\n",
    "print(len(os.listdir(TEST_RESULTS_DIR)))\n",
    "pprint(natsorted(os.listdir(TEST_RESULTS_DIR)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "633a57ac-4f99-44d1-ad28-267480692891",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eae40f73-260f-4e91-8c49-35c04cf4981e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.131124Z",
     "iopub.status.busy": "2021-06-29T23:50:30.131124Z",
     "iopub.status.idle": "2021-06-29T23:50:30.162130Z",
     "shell.execute_reply": "2021-06-29T23:50:30.161128Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.131124Z"
    },
    "id": "f209a709-9670-4587-9fa6-6ca058507fd2",
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 names\n",
      "{'Andhika': 18, 'Ardiyan': 18, 'Artik': 18, 'Ballya': 18, 'Bina': 18, 'Buyung': 18, 'Kresna': 18, 'Mhartian': 18, 'Raihan A': 18, 'Syifa': 18, 'Taufik': 18, 'Yandi': 18}\n",
      "max: 18\n",
      "min: 18\n",
      "total: 216\n",
      "\n",
      "test images: 216\n"
     ]
    }
   ],
   "source": [
    "names = os.listdir(TEST_SET_PATH)\n",
    "\n",
    "length_dict = {}\n",
    "for name in names:\n",
    "    samples = f'{TEST_SET_PATH:s}/{name:s}'\n",
    "    length_dict[name] = len(os.listdir(samples))\n",
    "\n",
    "n_min = min(length_dict.values())\n",
    "n_max = max(length_dict.values())\n",
    "n_sum = sum(length_dict.values())\n",
    "\n",
    "print(f'{len(names):d} names')\n",
    "print(length_dict)\n",
    "print(f\"max: {n_min:d}\")\n",
    "print(f\"min: {n_max:d}\")\n",
    "print(f\"total: {n_sum:d}\")\n",
    "print()\n",
    "\n",
    "total_test = n_sum\n",
    "print(f'test images: {total_test:d}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b6f789e-a0c4-4eb4-9862-16bc708d686c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "94721388-bee2-4a6f-b749-52c8bb1a54da",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "## Define Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de7b006b-6014-4420-8fd8-6b4019bb7839",
   "metadata": {},
   "source": [
    "### create_lenet_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3a3aa364-2ec6-4257-94c7-35f70055e65e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.164128Z",
     "iopub.status.busy": "2021-06-29T23:50:30.163128Z",
     "iopub.status.idle": "2021-06-29T23:50:30.194128Z",
     "shell.execute_reply": "2021-06-29T23:50:30.193132Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.164128Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_lenet_name(input_size, activation, pooling, batch_size):\n",
    "    # argument validation\n",
    "    if input_size in [32, 64, 96, 128] and activation in ['tanh', 'relu'] and pooling in ['average', 'max']:\n",
    "        pass\n",
    "    else:\n",
    "        raise ValueError\n",
    "    \n",
    "    name = f'lenet5_actual_'\n",
    "    name = f'{name}' if activation=='tanh' else f'{name}r'\n",
    "    name = f'{name}' if pooling=='average' else f'{name}m-'\n",
    "    name = f'{name}{input_size:d}-{batch_size}'\n",
    "    \n",
    "    return name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e3635db-e9c8-4ab1-9378-fbd8d9fb52aa",
   "metadata": {},
   "source": [
    "### create_mobilenetv2_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a59a7265-9bb3-4d73-97e9-71932ec70817",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.196130Z",
     "iopub.status.busy": "2021-06-29T23:50:30.195131Z",
     "iopub.status.idle": "2021-06-29T23:50:30.226130Z",
     "shell.execute_reply": "2021-06-29T23:50:30.225134Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.196130Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_mobilenetv2_name(input_size, batch_size, weights, dense):\n",
    "    # argument validation\n",
    "    if input_size in [32, 64, 96, 128] and batch_size in [16, 32, 64] and weights in [None, 'imagenet'] and dense in [False, True]:\n",
    "        pass\n",
    "    else:\n",
    "        raise ValueError\n",
    "    \n",
    "    if weights == 'imagenet':\n",
    "        if input_size in [96, 128, 192, 224]:\n",
    "            name = f'mobilenetv2_1.00_{input_size:d}-imagenet{input_size:d}-{batch_size:d}'\n",
    "        else:\n",
    "            name = f'mobilenetv2_1.00_{input_size:d}-imagenet224-{batch_size:d}'\n",
    "    else:\n",
    "        name = f'mobilenetv2_1.00_{input_size:d}-{batch_size:d}'\n",
    "\n",
    "    if bool(dense) == True:\n",
    "        name = f'{name:s}-fc12'\n",
    "    else:\n",
    "        name = f'{name:s}-12'\n",
    "    \n",
    "    return name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f49f85fc-15df-472b-a3dc-831720396296",
   "metadata": {},
   "source": [
    "### load_full_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "02fe055d-406c-4291-9b29-1697ccc7151f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.228134Z",
     "iopub.status.busy": "2021-06-29T23:50:30.228134Z",
     "iopub.status.idle": "2021-06-29T23:50:30.258135Z",
     "shell.execute_reply": "2021-06-29T23:50:30.257135Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.228134Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_full_model(model_name, re_epoch=\"\", debug=False):\n",
    "    global WEIGHTS_AND_MODELS_DIR    \n",
    "    \n",
    "    # ic.disable() if debug==False else ic.enable()\n",
    "        \n",
    "    model_dir = f\"{WEIGHTS_AND_MODELS_DIR}/{model_name}\"\n",
    "    model_filename = [file for file in os.listdir(model_dir) if file.startswith(('model',))][0]\n",
    "    \n",
    "    if re_epoch==\"\":\n",
    "        weight_filename = [file for file in os.listdir(model_dir) if file.endswith(('hdf5',))][0]\n",
    "    else:\n",
    "        _temp = [file for file in os.listdir(model_dir) if re.findall(\"(weights.[0-9]+)\", file)]\n",
    "        weight_filename = [file for file in _temp if re.findall(f\"({re_epoch})\", file)][0]\n",
    "            \n",
    "    model_fullpath = f\"{model_dir}/{model_filename}\"\n",
    "    weight_fullpath = f\"{model_dir}/{weight_filename}\"\n",
    "    \n",
    "    model = keras.models.load_model(model_fullpath, compile=True)\n",
    "    model.load_weights(weight_fullpath)\n",
    "    \n",
    "    if debug:\n",
    "        # print(model_filename)\n",
    "        # print(weight_filename)\n",
    "        ic(weight_filename)\n",
    "        # print(f\"loaded: {weight_filename}\")\n",
    "    time.sleep(250/1000)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fe34776b-fe84-4da5-be15-81188b0d82d2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.260140Z",
     "iopub.status.busy": "2021-06-29T23:50:30.260140Z",
     "iopub.status.idle": "2021-06-29T23:50:30.319137Z",
     "shell.execute_reply": "2021-06-29T23:50:30.318141Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.260140Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['lenet5_actual_32-16.weights.030_0.0833_2.5426.hdf5',\n",
      " 'lenet5_actual_32-16.weights.035.hdf5']\n",
      "['lenet5_actual_32-32.weights.035.hdf5',\n",
      " 'lenet5_actual_32-32.weights.050_0.9630_0.1092.hdf5']\n",
      "['lenet5_actual_32-64.weights.035.hdf5',\n",
      " 'lenet5_actual_32-64.weights.043_0.9815_0.0609.hdf5']\n",
      "['lenet5_actual_rm-32-16.weights.028_0.9726_0.1022.hdf5',\n",
      " 'lenet5_actual_rm-32-16.weights.035.hdf5']\n",
      "['lenet5_actual_rm-32-32.weights.035_0.9886_0.0437.hdf5',\n",
      " 'lenet5_actual_rm-32-32.weights.049_0.9901_0.0414.hdf5']\n",
      "['lenet5_actual_rm-32-64.weights.035_0.9955_0.0180.hdf5',\n",
      " 'lenet5_actual_rm-32-64.weights.036_0.9945_0.0180.hdf5']\n",
      "['lenet5_actual_rm-64-16.weights.033_0.0833_2.4852.hdf5',\n",
      " 'lenet5_actual_rm-64-16.weights.035.hdf5']\n",
      "['lenet5_actual_rm-64-32.weights.006_0.0833_2.4851.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.035.hdf5']\n",
      "['lenet5_actual_rm-64-64.weights.029_0.9960_0.0183.hdf5',\n",
      " 'lenet5_actual_rm-64-64.weights.035.hdf5']\n",
      "['lenet5_actual_rm-96-16.weights.035.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.038_0.0833_2.4853.hdf5']\n",
      "['lenet5_actual_rm-96-32.weights.035.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.040_0.0833_2.4852.hdf5']\n",
      "['lenet5_actual_rm-96-64.weights.035_0.9943_0.0189.hdf5']\n",
      "['lenet5_actual_rm-128-16.weights.031_0.0833_2.4853.hdf5',\n",
      " 'lenet5_actual_rm-128-16.weights.035.hdf5']\n",
      "['lenet5_actual_rm-128-32.weights.014_0.9874_0.0473.hdf5',\n",
      " 'lenet5_actual_rm-128-32.weights.035.hdf5']\n",
      "['lenet5_actual_rm-128-64.weights.029_0.9907_0.0348.hdf5',\n",
      " 'lenet5_actual_rm-128-64.weights.035.hdf5']\n",
      "['mobilenetv2_1.00_32-16-12.weights.045_0.9954_0.0185.hdf5']\n",
      "['mobilenetv2_1.00_32-16-fc12.weights.049_0.7533_0.5465.hdf5']\n",
      "['mobilenetv2_1.00_32-32-12.weights.046_0.9947_0.0173.hdf5']\n",
      "['mobilenetv2_1.00_32-32-fc12.weights.049_0.9678_0.1378.hdf5']\n",
      "['mobilenetv2_1.00_32-64-12.weights.050_0.9953_0.0159.hdf5']\n",
      "['mobilenetv2_1.00_32-64-fc12.weights.050_0.9899_0.0389.hdf5']\n",
      "['mobilenetv2_1.00_32-imagenet224-16-12.weights.025_0.2841_2.0599.hdf5']\n",
      "['mobilenetv2_1.00_32-imagenet224-16-fc12.weights.047_0.2988_2.0399.hdf5']\n",
      "['mobilenetv2_1.00_32-imagenet224-32-12.weights.047_0.2966_2.0507.hdf5']\n",
      "['mobilenetv2_1.00_32-imagenet224-32-fc12.weights.042_0.2943_2.0200.hdf5']\n",
      "['mobilenetv2_1.00_32-imagenet224-64-12.weights.031_0.2817_2.0544.hdf5']\n",
      "['mobilenetv2_1.00_32-imagenet224-64-fc12.weights.050_0.3070_2.0004.hdf5']\n",
      "['mobilenetv2_1.00_64-16-12.weights.048_0.9962_0.0103.hdf5']\n",
      "['mobilenetv2_1.00_64-16-fc12.weights.043_0.9813_0.0794.hdf5']\n",
      "['mobilenetv2_1.00_64-32-12.weights.046_0.9988_0.0041.hdf5']\n",
      "['mobilenetv2_1.00_64-32-fc12.weights.048_0.9936_0.0256.hdf5']\n",
      "['mobilenetv2_1.00_64-64-12.weights.038_0.9990_0.0039.hdf5']\n",
      "['mobilenetv2_1.00_64-64-fc12.weights.049_0.9942_0.0268.hdf5']\n",
      "['mobilenetv2_1.00_64-imagenet224-16-12.weights.026_0.9636_0.2933.hdf5']\n",
      "['mobilenetv2_1.00_64-imagenet224-16-fc12.weights.034_0.9420_0.2607.hdf5']\n",
      "['mobilenetv2_1.00_64-imagenet224-32-12.weights.048_0.9711_0.2074.hdf5']\n",
      "['mobilenetv2_1.00_64-imagenet224-32-fc12.weights.039_0.9602_0.1616.hdf5']\n",
      "['mobilenetv2_1.00_64-imagenet224-64-12.weights.026_0.9638_0.1562.hdf5']\n",
      "['mobilenetv2_1.00_64-imagenet224-64-fc12.weights.035_0.9661_0.1243.hdf5']\n",
      "['mobilenetv2_1.00_96-16-12.weights.046_0.9987_0.0042.hdf5']\n",
      "['mobilenetv2_1.00_96-16-fc12.weights.050_0.8985_0.1913.hdf5']\n",
      "['mobilenetv2_1.00_96-32-12.weights.045_0.9987_0.0027.hdf5']\n",
      "['mobilenetv2_1.00_96-32-fc12.weights.040_0.9922_0.0273.hdf5']\n",
      "['mobilenetv2_1.00_96-64-12.weights.031_0.9995_0.0023.hdf5']\n",
      "['mobilenetv2_1.00_96-64-fc12.weights.050_0.9955_0.0152.hdf5']\n",
      "['mobilenetv2_1.00_96-imagenet96-16-12.weights.037_0.9975_0.0125.hdf5']\n",
      "['mobilenetv2_1.00_96-imagenet96-16-fc12.weights.049_0.9902_0.0360.hdf5']\n",
      "['mobilenetv2_1.00_96-imagenet96-32-12.weights.040_0.9975_0.0103.hdf5']\n",
      "['mobilenetv2_1.00_96-imagenet96-32-fc12.weights.036_0.9954_0.0167.hdf5']\n",
      "['mobilenetv2_1.00_96-imagenet96-64-12.weights.011_0.9978_0.0074.hdf5']\n",
      "['mobilenetv2_1.00_96-imagenet96-64-fc12.weights.035_0.9963_0.0128.hdf5']\n",
      "['mobilenetv2_1.00_128-16-12.weights.043_0.9998_0.0004.hdf5']\n",
      "['mobilenetv2_1.00_128-16-fc12.weights.049_0.9808_0.0826.hdf5']\n",
      "['mobilenetv2_1.00_128-32-12.weights.048_0.9995_0.0012.hdf5']\n",
      "['mobilenetv2_1.00_128-32-fc12.weights.045_0.9859_0.0628.hdf5']\n",
      "['mobilenetv2_1.00_128-64-12.weights.044_0.9995_0.0015.hdf5']\n",
      "['mobilenetv2_1.00_128-64-fc12.weights.035_0.9990_0.0062.hdf5']\n",
      "['mobilenetv2_1.00_128-imagenet128-16-12.weights.041_0.9995_0.0031.hdf5']\n",
      "['mobilenetv2_1.00_128-imagenet128-16-fc12.weights.019_0.9955_0.0284.hdf5']\n",
      "['mobilenetv2_1.00_128-imagenet128-32-12.weights.045_0.9992_0.0037.hdf5']\n",
      "['mobilenetv2_1.00_128-imagenet128-32-fc12.weights.029_0.9950_0.0180.hdf5']\n",
      "['mobilenetv2_1.00_128-imagenet128-64-12.weights.049_0.9992_0.0044.hdf5']\n",
      "['mobilenetv2_1.00_128-imagenet128-64-fc12.weights.019_0.9973_0.0096.hdf5']\n"
     ]
    }
   ],
   "source": [
    "_dirs = []\n",
    "re_epoch = \"35\"\n",
    "for model_name in model_names:\n",
    "    model_dir = f\"{WEIGHTS_AND_MODELS_DIR}/{model_name}\"\n",
    "    _temp = [file for file in os.listdir(model_dir) if re.findall(\"(weights.[0-9]+)\", file)]\n",
    "    pprint(\n",
    "        _temp\n",
    "    )\n",
    "\n",
    "del _dirs, re_epoch, model_dir, _temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7934aee5-00f2-48b3-ad1b-02745321b7f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.321143Z",
     "iopub.status.busy": "2021-06-29T23:50:30.321143Z",
     "iopub.status.idle": "2021-06-29T23:50:30.335141Z",
     "shell.execute_reply": "2021-06-29T23:50:30.334144Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.321143Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lenet5_actual_32-16.weights.030_0.0833_2.5426.hdf5'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_dir = f\"{WEIGHTS_AND_MODELS_DIR}/{model_names[0]}\"\n",
    "\n",
    "[file for file in os.listdir(model_dir) if file.endswith(('hdf5',))][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bcbc170-8a7b-4188-8653-9f00a977fbe1",
   "metadata": {},
   "source": [
    "### create_test_datagen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e3f8e929-db9a-469d-ad37-7898295f6286",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.337139Z",
     "iopub.status.busy": "2021-06-29T23:50:30.336141Z",
     "iopub.status.idle": "2021-06-29T23:50:30.367143Z",
     "shell.execute_reply": "2021-06-29T23:50:30.366148Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.337139Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_test_datagen(input_size, batch_size=1, shuffle=False):\n",
    "    ### create test generator\n",
    "    image_generator = keras.preprocessing.image.ImageDataGenerator(\n",
    "        rescale=1./255,\n",
    "        rotation_range=5,\n",
    "        # width_shift_range=1.05,\n",
    "        # height_shift_range=1.05,\n",
    "        brightness_range=(0.75, 1.25),\n",
    "        # shear_range = 0.075,\n",
    "        zoom_range=0.1,\n",
    "        fill_mode = 'nearest',\n",
    "        horizontal_flip=True,\n",
    "        # preprocessing_function=seq.augment_image\n",
    "    )\n",
    "    # https://stackoverflow.com/questions/63650807/disable-the-keras-found-images-belonging-to-classes-message\n",
    "    with redirect_stdout(None):\n",
    "        test_data_gen = image_generator.flow_from_directory(\n",
    "            seed=1728,\n",
    "            batch_size=batch_size,\n",
    "            directory=TEST_SET_PATH,\n",
    "            shuffle=shuffle,\n",
    "            target_size=(input_size, input_size),\n",
    "            interpolation='bicubic',\n",
    "            class_mode='categorical')\n",
    "    \n",
    "    # print('generator with size (%d, %d) and batch size %d created' % (input_size, input_size, batch_size))\n",
    "    \n",
    "    return test_data_gen"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "900b49fa-4c67-4a05-9b69-c63ae341117d",
   "metadata": {},
   "source": [
    "### model_evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1f958c88-b959-4637-b685-079fda9e6190",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.369141Z",
     "iopub.status.busy": "2021-06-29T23:50:30.368145Z",
     "iopub.status.idle": "2021-06-29T23:50:30.399144Z",
     "shell.execute_reply": "2021-06-29T23:50:30.398145Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.369141Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def model_evaluate(nama_model, input_size, re_epoch=\"\", verbose=1, debug=False):\n",
    "    if nama_model in model_names:\n",
    "        pass\n",
    "    else:\n",
    "        raise ValueError\n",
    "    \n",
    "    # load model\n",
    "    model = load_full_model(model_name=nama_model, re_epoch=re_epoch, debug=debug)\n",
    "    \n",
    "    # create data generator\n",
    "    test_data_gen = create_test_datagen(input_size=input_size)\n",
    "    \n",
    "    # start evaluate model\n",
    "    mulai_hitung_waktu()  ###\n",
    "    \n",
    "    test_history = model.evaluate(\n",
    "        test_data_gen,\n",
    "        batch_size=1,\n",
    "        verbose=verbose,\n",
    "        steps=len(test_data_gen.filenames),\n",
    "        return_dict=True)\n",
    "\n",
    "    cetak_lama_waktu('s')  ###\n",
    "    \n",
    "    print()\n",
    "    dict_1 = {'name' : nama_model}\n",
    "    dict_2 = {**dict_1, **test_history}\n",
    "    return model, dict_2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bde442f1-de9d-4f65-9ffb-8a5674e5f8b3",
   "metadata": {},
   "source": [
    "### model_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ff46edc6-84b9-4254-9ea5-7a5cad20e8be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.401143Z",
     "iopub.status.busy": "2021-06-29T23:50:30.401143Z",
     "iopub.status.idle": "2021-06-29T23:50:30.431146Z",
     "shell.execute_reply": "2021-06-29T23:50:30.430145Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.401143Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def model_predict(nama_model, input_size, re_epoch=\"\", verbose=1, debug=False):\n",
    "    if nama_model in model_names:\n",
    "        pass\n",
    "    else:\n",
    "        raise ValueError\n",
    "    \n",
    "    # load model\n",
    "    model = load_full_model(model_name=nama_model, re_epoch=re_epoch, debug=debug)\n",
    "    \n",
    "    # create data generator\n",
    "    test_data_gen = create_test_datagen(input_size=input_size)\n",
    "    \n",
    "    # start evaluate model\n",
    "    mulai_hitung_waktu()  ###\n",
    "    \n",
    "    y_pred = model.predict(\n",
    "        test_data_gen,\n",
    "        batch_size=1,\n",
    "        verbose=verbose,\n",
    "        steps=len(test_data_gen.filenames)\n",
    "    )\n",
    "\n",
    "    cetak_lama_waktu('s')  ###\n",
    "    \n",
    "    print()\n",
    "    return model, np.argmax(y_pred, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79e79458-890d-4b0e-a750-e9f192bae502",
   "metadata": {},
   "source": [
    "### print_to_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c546119c-9e2b-442a-b389-0885797955e6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.432147Z",
     "iopub.status.busy": "2021-06-29T23:50:30.432147Z",
     "iopub.status.idle": "2021-06-29T23:50:30.479166Z",
     "shell.execute_reply": "2021-06-29T23:50:30.478147Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.432147Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def print_to_file(filename, string, mode='w'):\n",
    "    global TEST_RESULTS_DIR\n",
    "    filepath = f\"{TEST_RESULTS_DIR}/{filename}.txt\"\n",
    "    \n",
    "    with open(filepath, mode) as file, redirect_stdout(file):\n",
    "        print(string)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d2ec110-a8ff-4e5d-89de-299ed60a52ea",
   "metadata": {},
   "source": [
    "## Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6e6aadd6-3a37-484c-9e3c-b5a3dad5cba8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.481151Z",
     "iopub.status.busy": "2021-06-29T23:50:30.480152Z",
     "iopub.status.idle": "2021-06-29T23:50:30.511153Z",
     "shell.execute_reply": "2021-06-29T23:50:30.510156Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.481151Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# LeNet-5 and MobileNetV2\n",
    "_input_size = [32, 64, 96, 128]\n",
    "_batch_size = [16, 32, 64]\n",
    "\n",
    "# MobileNetV2 only\n",
    "_weights = [None, 'imagenet']\n",
    "_dense = [False, True]\n",
    "\n",
    "\n",
    "## metrics\n",
    "metrics = ['name', 'loss', 'categorical_accuracy', 'top-3', 'ROC-AUC', 'PR-AUC', 'precision', 'recall', 'TP', 'TN', 'FP', 'FN']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce650c24-f191-4d95-8e66-ee6e916c2e82",
   "metadata": {},
   "source": [
    "# PostPreparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b4f4cddc-f5b3-4655-97ad-3692a9d9ca20",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:50:30.513152Z",
     "iopub.status.busy": "2021-06-29T23:50:30.512159Z",
     "iopub.status.idle": "2021-06-29T23:50:30.559159Z",
     "shell.execute_reply": "2021-06-29T23:50:30.558155Z",
     "shell.execute_reply.started": "2021-06-29T23:50:30.513152Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ic.configureOutput(prefix=\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9792a53-c214-4938-a754-654dc811b899",
   "metadata": {},
   "source": [
    "# LeNet-5"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58a8e9df-95be-4a6b-9102-3da3da47b381",
   "metadata": {},
   "source": [
    "## Asli - model_evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "216e8379-0af8-46ad-8117-f444c792d6c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:36:46.244845Z",
     "iopub.status.busy": "2021-06-29T23:36:46.244845Z",
     "iopub.status.idle": "2021-06-29T23:38:25.767461Z",
     "shell.execute_reply": "2021-06-29T23:38:25.766467Z",
     "shell.execute_reply.started": "2021-06-29T23:36:46.244845Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_32-16.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 32s 149ms/step - loss: 2.6034 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(36560.94 ms) == ( 36s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_32-32.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step - loss: 0.3653 - categorical_accuracy: 0.8796 - top-3: 0.9769 - ROC-AUC: 0.9910 - PR-AUC: 0.9551 - precision: 0.8920 - recall: 0.8796 - TP: 190.0000 - TN: 2353.0000 - FP: 23.0000 - FN: 26.0000\n",
      "-----\n",
      "(30796.33 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_32-64.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step - loss: 0.2751 - categorical_accuracy: 0.8889 - top-3: 0.9954 - ROC-AUC: 0.9946 - PR-AUC: 0.9634 - precision: 0.9095 - recall: 0.8843 - TP: 191.0000 - TN: 2357.0000 - FP: 19.0000 - FN: 25.0000\n",
      "-----\n",
      "(30148.20 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lenet5_asli_df = pd.DataFrame(columns=metrics)\n",
    "lenet5_asli_output = \"\"\n",
    "\n",
    "nama_file = 'lenet-5 versi asli (035)' + '.evaluate_turbo'\n",
    "df_savepath = f'{TEST_RESULTS_DIR}/{nama_file}/history.{nama_file}.csv'\n",
    "ic(nama_file)\n",
    "\n",
    "i = 0\n",
    "for batch_size in _batch_size:\n",
    "    \n",
    "    # versi asli only\n",
    "    input_size = 32\n",
    "    activation = 'tanh'\n",
    "    pooling = 'average'\n",
    "    \n",
    "    kwargs = {\n",
    "        'input_size' : input_size,\n",
    "        'activation' : activation,\n",
    "        'pooling' : pooling,\n",
    "        'batch_size' : batch_size\n",
    "    }\n",
    "    nama_model = create_lenet_name(**kwargs)\n",
    "    \n",
    "    with io.StringIO() as buf, redirect_stdout(buf):\n",
    "        _, dict_2 = model_evaluate(nama_model=nama_model, input_size=input_size, re_epoch=\"35\", debug=True)\n",
    "        lenet5_asli_output += buf.getvalue()\n",
    "        _output = buf.getvalue()\n",
    "    print(_output, end='')\n",
    "    \n",
    "    lenet5_asli_df.loc[i] = list(dict_2.values())\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "57a02750-5928-412c-baf0-c955650ff62f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:38:25.769470Z",
     "iopub.status.busy": "2021-06-29T23:38:25.768462Z",
     "iopub.status.idle": "2021-06-29T23:38:25.814468Z",
     "shell.execute_reply": "2021-06-29T23:38:25.814468Z",
     "shell.execute_reply.started": "2021-06-29T23:38:25.769470Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>loss</th>\n",
       "      <th>categorical_accuracy</th>\n",
       "      <th>top-3</th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>PR-AUC</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>TP</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>waktu</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>lenet5_actual_32-16</td>\n",
       "      <td>2.603427</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>149ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lenet5_actual_32-32</td>\n",
       "      <td>0.365282</td>\n",
       "      <td>0.879630</td>\n",
       "      <td>0.976852</td>\n",
       "      <td>0.990967</td>\n",
       "      <td>0.955130</td>\n",
       "      <td>0.892019</td>\n",
       "      <td>0.879630</td>\n",
       "      <td>190.0</td>\n",
       "      <td>2353.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>lenet5_actual_32-64</td>\n",
       "      <td>0.275105</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.995370</td>\n",
       "      <td>0.994631</td>\n",
       "      <td>0.963402</td>\n",
       "      <td>0.909524</td>\n",
       "      <td>0.884259</td>\n",
       "      <td>191.0</td>\n",
       "      <td>2357.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  name      loss  categorical_accuracy     top-3   ROC-AUC  \\\n",
       "0  lenet5_actual_32-16  2.603427              0.083333  0.250000  0.500000   \n",
       "1  lenet5_actual_32-32  0.365282              0.879630  0.976852  0.990967   \n",
       "2  lenet5_actual_32-64  0.275105              0.888889  0.995370  0.994631   \n",
       "\n",
       "     PR-AUC  precision    recall     TP      TN    FP     FN  waktu  \n",
       "0  0.083333   0.000000  0.000000    0.0  2376.0   0.0  216.0  149ms  \n",
       "1  0.955130   0.892019  0.879630  190.0  2353.0  23.0   26.0  130ms  \n",
       "2  0.963402   0.909524  0.884259  191.0  2357.0  19.0   25.0  129ms  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_df = lenet5_asli_df\n",
    "\n",
    "waktu = re.findall(\"([0-9]+ms)\", lenet5_asli_output)\n",
    "test_df['waktu'] = waktu\n",
    "\n",
    "display(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "ced8a5aa-1e28-49c9-8e20-12963cfa8576",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:38:25.816469Z",
     "iopub.status.busy": "2021-06-29T23:38:25.816469Z",
     "iopub.status.idle": "2021-06-29T23:38:25.895471Z",
     "shell.execute_reply": "2021-06-29T23:38:25.894473Z",
     "shell.execute_reply.started": "2021-06-29T23:38:25.816469Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    os.mkdir(f'{TEST_RESULTS_DIR}/{nama_file}')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "# save to csv: \n",
    "with open(df_savepath, mode='w') as f:\n",
    "    test_df.to_csv(\n",
    "        f,\n",
    "        header=True,\n",
    "        index=False,\n",
    "        line_terminator='\\n'\n",
    "        )\n",
    "    \n",
    "# print_to_file(f\"verbose.{nama_file}\", lenet5_asli_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "697afa67-0d50-4a87-aa0e-3e8be8d9e02f",
   "metadata": {},
   "source": [
    "## Asli - model_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cff9243b-3cfe-4754-ac0b-5f9e4c069f8b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:51:04.044680Z",
     "iopub.status.busy": "2021-06-29T23:51:04.043679Z",
     "iopub.status.idle": "2021-06-29T23:52:31.473518Z",
     "shell.execute_reply": "2021-06-29T23:52:31.472521Z",
     "shell.execute_reply.started": "2021-06-29T23:51:04.044680Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "nama_file: 'lenet-5 versi asli (035).predict_turbo'\n",
      "weight_filename: 'lenet5_actual_32-16.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 124ms/step\n",
      "-----\n",
      "(29010.61 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_32-32.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 120ms/step\n",
      "-----\n",
      "(26373.41 ms) == ( 26s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_32-64.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(26789.09 ms) == ( 26s)\n",
      "-----\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lenet5_asli_df = pd.DataFrame(columns=['name'])\n",
    "lenet5_asli_output = \"\"\n",
    "\n",
    "nama_file = 'lenet-5 versi asli (035)' + '.predict_turbo'\n",
    "df_savepath = f'{TEST_RESULTS_DIR}/{nama_file}/history.{nama_file}.csv'\n",
    "ic(nama_file)\n",
    "\n",
    "try:\n",
    "    os.mkdir(f'{TEST_RESULTS_DIR}/{nama_file}')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "i = 0\n",
    "for batch_size in _batch_size:\n",
    "    \n",
    "    # versi asli only\n",
    "    input_size = 32\n",
    "    activation = 'tanh'\n",
    "    pooling = 'average'\n",
    "    \n",
    "    kwargs = {\n",
    "        'input_size' : input_size,\n",
    "        'activation' : activation,\n",
    "        'pooling' : pooling,\n",
    "        'batch_size' : batch_size\n",
    "    }\n",
    "    nama_model = create_lenet_name(**kwargs)\n",
    "    \n",
    "    ### start evaluate\n",
    "    with io.StringIO() as buf, redirect_stdout(buf):\n",
    "        _, y_pred = model_predict(nama_model=nama_model, input_size=input_size, re_epoch=\"35\", debug=True)\n",
    "        lenet5_asli_output += buf.getvalue()\n",
    "        _output = buf.getvalue()\n",
    "    print(_output, end='')\n",
    "    \n",
    "    ### create confusion matrix & classification report\n",
    "    test_data_gen = create_test_datagen(input_size=input_size)\n",
    "    \n",
    "    cm = confusion_matrix(\n",
    "        y_true=test_data_gen.classes,\n",
    "        y_pred=y_pred,\n",
    "        labels=[item for item in test_data_gen.class_indices.values()]\n",
    "    )\n",
    "    \n",
    "    cr = classification_report(\n",
    "        y_true=test_data_gen.classes,\n",
    "        y_pred=y_pred,\n",
    "        labels=[item for item in test_data_gen.class_indices.values()],\n",
    "        target_names=[item for item in test_data_gen.class_indices.keys()],\n",
    "        digits=2,\n",
    "        output_dict=False,\n",
    "        zero_division=0\n",
    "    )\n",
    "    \n",
    "    print_to_file(rf\"{nama_file}/{nama_model} cm-cr\" + \"_turbo\", f\"{cm}\\n\\n\\n{cr}\")\n",
    "    \n",
    "    lenet5_asli_df.loc[i] = nama_model\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e828a1a9-049b-4a07-b8e1-64e2f87f04c2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:52:31.475516Z",
     "iopub.status.busy": "2021-06-29T23:52:31.475516Z",
     "iopub.status.idle": "2021-06-29T23:52:31.505513Z",
     "shell.execute_reply": "2021-06-29T23:52:31.504519Z",
     "shell.execute_reply.started": "2021-06-29T23:52:31.475516Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>waktu</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>lenet5_actual_32-16</td>\n",
       "      <td>124ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lenet5_actual_32-32</td>\n",
       "      <td>120ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>lenet5_actual_32-64</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  name  waktu\n",
       "0  lenet5_actual_32-16  124ms\n",
       "1  lenet5_actual_32-32  120ms\n",
       "2  lenet5_actual_32-64  122ms"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_df = lenet5_asli_df\n",
    "\n",
    "waktu = re.findall(\"([0-9]+ms)\", lenet5_asli_output)\n",
    "test_df['waktu'] = waktu\n",
    "\n",
    "display(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7caa2630-3d30-4ba0-b7ae-a2b168c2ac0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:52:31.506519Z",
     "iopub.status.busy": "2021-06-29T23:52:31.506519Z",
     "iopub.status.idle": "2021-06-29T23:52:32.096572Z",
     "shell.execute_reply": "2021-06-29T23:52:32.095572Z",
     "shell.execute_reply.started": "2021-06-29T23:52:31.506519Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# save to csv: \n",
    "with open(df_savepath, mode='w') as f:\n",
    "    test_df.to_csv(\n",
    "        f,\n",
    "        header=True,\n",
    "        index=False,\n",
    "        line_terminator='\\n'\n",
    "        )\n",
    "    \n",
    "# print_to_file(f\"verbose.{nama_file}\", lenet5_asli_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe079ad3-27bf-4f5d-84c9-a33ca89388ec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbf157c9-bf18-45b2-87dd-cdd3ec6ab5d4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5b1a87f0-fa25-47bd-8ed2-32ae6c92b591",
   "metadata": {},
   "source": [
    "## Eksperimen - model_evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "e8ae2818-a2e7-47b9-91e0-352b2ecf3fcb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:41:15.463188Z",
     "iopub.status.busy": "2021-06-29T23:41:15.462195Z",
     "iopub.status.idle": "2021-06-29T23:47:34.705576Z",
     "shell.execute_reply": "2021-06-29T23:47:34.704571Z",
     "shell.execute_reply.started": "2021-06-29T23:41:15.463188Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "nama_file: 'lenet-5 versi modifikasi (035).evaluate_turbo'\n",
      "weight_filename: 'lenet5_actual_rm-32-16.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 31s 144ms/step - loss: 2.4855 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(33156.93 ms) == ( 33s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-32-32.weights.035_0.9886_0.0437.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 132ms/step - loss: 1.5309 - categorical_accuracy: 0.8889 - top-3: 0.9630 - ROC-AUC: 0.9566 - PR-AUC: 0.8584 - precision: 0.8962 - recall: 0.8796 - TP: 190.0000 - TN: 2354.0000 - FP: 22.0000 - FN: 26.0000\n",
      "-----\n",
      "(30671.36 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-32-64.weights.035_0.9955_0.0180.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 127ms/step - loss: 0.2816 - categorical_accuracy: 0.9769 - top-3: 1.0000 - ROC-AUC: 0.9869 - PR-AUC: 0.9561 - precision: 0.9766 - recall: 0.9676 - TP: 209.0000 - TN: 2371.0000 - FP: 5.0000 - FN: 7.0000  \n",
      "-----\n",
      "(29498.80 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-64-16.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 128ms/step - loss: 2.4858 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(29376.15 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-64-32.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 31s 142ms/step - loss: 2.4853 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(32438.44 ms) == ( 32s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-64-64.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step - loss: 0.1902 - categorical_accuracy: 0.9676 - top-3: 0.9861 - ROC-AUC: 0.9947 - PR-AUC: 0.9839 - precision: 0.9721 - recall: 0.9676 - TP: 209.0000 - TN: 2370.0000 - FP: 6.0000 - FN: 7.0000  \n",
      "-----\n",
      "(30172.27 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-96-16.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 29s 132ms/step - loss: 2.4853 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(30500.58 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-96-32.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 29s 133ms/step - loss: 2.4855 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(30639.76 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-96-64.weights.035_0.9943_0.0189.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 30s 138ms/step - loss: 0.1265 - categorical_accuracy: 0.9537 - top-3: 0.9861 - ROC-AUC: 0.9971 - PR-AUC: 0.9914 - precision: 0.9717 - recall: 0.9537 - TP: 206.0000 - TN: 2370.0000 - FP: 6.0000 - FN: 10.0000 \n",
      "-----\n",
      "(32216.72 ms) == ( 32s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-128-16.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step - loss: 2.4859 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(29855.25 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-128-32.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 131ms/step - loss: 0.1921 - categorical_accuracy: 0.9676 - top-3: 0.9954 - ROC-AUC: 0.9918 - PR-AUC: 0.9704 - precision: 0.9717 - recall: 0.9537 - TP: 206.0000 - TN: 2370.0000 - FP: 6.0000 - FN: 10.0000 \n",
      "-----\n",
      "(29996.43 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-128-64.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 131ms/step - loss: 0.1222 - categorical_accuracy: 0.9583 - top-3: 1.0000 - ROC-AUC: 0.9970 - PR-AUC: 0.9873 - precision: 0.9579 - recall: 0.9491 - TP: 205.0000 - TN: 2367.0000 - FP: 9.0000 - FN: 11.0000\n",
      "-----\n",
      "(29936.89 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lenet5_modifikasi_df = pd.DataFrame(columns=metrics)\n",
    "lenet5_modifikasi_output = \"\"\n",
    "\n",
    "nama_file = 'lenet-5 versi modifikasi (035)' + '.evaluate_turbo'\n",
    "df_savepath = f'{TEST_RESULTS_DIR}/{nama_file}/history.{nama_file}.csv'\n",
    "ic(nama_file)\n",
    "\n",
    "i = 0\n",
    "for input_size in _input_size:\n",
    "    for batch_size in _batch_size:\n",
    "    \n",
    "        # versi modifikasi only\n",
    "        activation = 'relu'\n",
    "        pooling = 'max'\n",
    "\n",
    "        kwargs = {\n",
    "            'input_size' : input_size,\n",
    "            'activation' : activation,\n",
    "            'pooling' : pooling,\n",
    "            'batch_size' : batch_size\n",
    "        }\n",
    "        nama_model = create_lenet_name(**kwargs)\n",
    "\n",
    "        with io.StringIO() as buf, redirect_stdout(buf):\n",
    "            _, dict_2 = model_evaluate(nama_model=nama_model, input_size=input_size, re_epoch=\"35\", debug=True)\n",
    "            lenet5_modifikasi_output += buf.getvalue()\n",
    "            _output = buf.getvalue()\n",
    "        print(_output, end='')\n",
    "\n",
    "        lenet5_modifikasi_df.loc[i] = list(dict_2.values())\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "8862d1e6-181b-443e-9072-8666aa516fba",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:47:34.708569Z",
     "iopub.status.busy": "2021-06-29T23:47:34.707572Z",
     "iopub.status.idle": "2021-06-29T23:47:34.768573Z",
     "shell.execute_reply": "2021-06-29T23:47:34.767583Z",
     "shell.execute_reply.started": "2021-06-29T23:47:34.708569Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>loss</th>\n",
       "      <th>categorical_accuracy</th>\n",
       "      <th>top-3</th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>PR-AUC</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>TP</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>waktu</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>lenet5_actual_rm-32-16</td>\n",
       "      <td>2.485466</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>144ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lenet5_actual_rm-32-32</td>\n",
       "      <td>1.530854</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.962963</td>\n",
       "      <td>0.956581</td>\n",
       "      <td>0.858425</td>\n",
       "      <td>0.896226</td>\n",
       "      <td>0.879630</td>\n",
       "      <td>190.0</td>\n",
       "      <td>2354.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>132ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>lenet5_actual_rm-32-64</td>\n",
       "      <td>0.281593</td>\n",
       "      <td>0.976852</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.986941</td>\n",
       "      <td>0.956051</td>\n",
       "      <td>0.976636</td>\n",
       "      <td>0.967593</td>\n",
       "      <td>209.0</td>\n",
       "      <td>2371.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>lenet5_actual_rm-64-16</td>\n",
       "      <td>2.485776</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>128ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>lenet5_actual_rm-64-32</td>\n",
       "      <td>2.485272</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>142ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>lenet5_actual_rm-64-64</td>\n",
       "      <td>0.190170</td>\n",
       "      <td>0.967593</td>\n",
       "      <td>0.986111</td>\n",
       "      <td>0.994651</td>\n",
       "      <td>0.983927</td>\n",
       "      <td>0.972093</td>\n",
       "      <td>0.967593</td>\n",
       "      <td>209.0</td>\n",
       "      <td>2370.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>lenet5_actual_rm-96-16</td>\n",
       "      <td>2.485327</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>132ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>lenet5_actual_rm-96-32</td>\n",
       "      <td>2.485512</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>133ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>lenet5_actual_rm-96-64</td>\n",
       "      <td>0.126490</td>\n",
       "      <td>0.953704</td>\n",
       "      <td>0.986111</td>\n",
       "      <td>0.997106</td>\n",
       "      <td>0.991385</td>\n",
       "      <td>0.971698</td>\n",
       "      <td>0.953704</td>\n",
       "      <td>206.0</td>\n",
       "      <td>2370.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>138ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>lenet5_actual_rm-128-16</td>\n",
       "      <td>2.485947</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>lenet5_actual_rm-128-32</td>\n",
       "      <td>0.192064</td>\n",
       "      <td>0.967593</td>\n",
       "      <td>0.995370</td>\n",
       "      <td>0.991766</td>\n",
       "      <td>0.970381</td>\n",
       "      <td>0.971698</td>\n",
       "      <td>0.953704</td>\n",
       "      <td>206.0</td>\n",
       "      <td>2370.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>131ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>lenet5_actual_rm-128-64</td>\n",
       "      <td>0.122221</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997043</td>\n",
       "      <td>0.987345</td>\n",
       "      <td>0.957944</td>\n",
       "      <td>0.949074</td>\n",
       "      <td>205.0</td>\n",
       "      <td>2367.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>131ms</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       name      loss  categorical_accuracy     top-3  \\\n",
       "0    lenet5_actual_rm-32-16  2.485466              0.083333  0.250000   \n",
       "1    lenet5_actual_rm-32-32  1.530854              0.888889  0.962963   \n",
       "2    lenet5_actual_rm-32-64  0.281593              0.976852  1.000000   \n",
       "3    lenet5_actual_rm-64-16  2.485776              0.083333  0.250000   \n",
       "4    lenet5_actual_rm-64-32  2.485272              0.083333  0.250000   \n",
       "5    lenet5_actual_rm-64-64  0.190170              0.967593  0.986111   \n",
       "6    lenet5_actual_rm-96-16  2.485327              0.083333  0.250000   \n",
       "7    lenet5_actual_rm-96-32  2.485512              0.083333  0.250000   \n",
       "8    lenet5_actual_rm-96-64  0.126490              0.953704  0.986111   \n",
       "9   lenet5_actual_rm-128-16  2.485947              0.083333  0.250000   \n",
       "10  lenet5_actual_rm-128-32  0.192064              0.967593  0.995370   \n",
       "11  lenet5_actual_rm-128-64  0.122221              0.958333  1.000000   \n",
       "\n",
       "     ROC-AUC    PR-AUC  precision    recall     TP      TN    FP     FN  waktu  \n",
       "0   0.500000  0.083333   0.000000  0.000000    0.0  2376.0   0.0  216.0  144ms  \n",
       "1   0.956581  0.858425   0.896226  0.879630  190.0  2354.0  22.0   26.0  132ms  \n",
       "2   0.986941  0.956051   0.976636  0.967593  209.0  2371.0   5.0    7.0  127ms  \n",
       "3   0.500000  0.083333   0.000000  0.000000    0.0  2376.0   0.0  216.0  128ms  \n",
       "4   0.500000  0.083333   0.000000  0.000000    0.0  2376.0   0.0  216.0  142ms  \n",
       "5   0.994651  0.983927   0.972093  0.967593  209.0  2370.0   6.0    7.0  130ms  \n",
       "6   0.500000  0.083333   0.000000  0.000000    0.0  2376.0   0.0  216.0  132ms  \n",
       "7   0.500000  0.083333   0.000000  0.000000    0.0  2376.0   0.0  216.0  133ms  \n",
       "8   0.997106  0.991385   0.971698  0.953704  206.0  2370.0   6.0   10.0  138ms  \n",
       "9   0.500000  0.083333   0.000000  0.000000    0.0  2376.0   0.0  216.0  129ms  \n",
       "10  0.991766  0.970381   0.971698  0.953704  206.0  2370.0   6.0   10.0  131ms  \n",
       "11  0.997043  0.987345   0.957944  0.949074  205.0  2367.0   9.0   11.0  131ms  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_df = lenet5_modifikasi_df\n",
    "\n",
    "waktu = re.findall(\"([0-9]+ms)\", lenet5_modifikasi_output)\n",
    "test_df['waktu'] = waktu\n",
    "\n",
    "display(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "509a98bf-5ef6-4974-af98-fed7693682a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:47:34.771578Z",
     "iopub.status.busy": "2021-06-29T23:47:34.770575Z",
     "iopub.status.idle": "2021-06-29T23:47:35.056598Z",
     "shell.execute_reply": "2021-06-29T23:47:35.055600Z",
     "shell.execute_reply.started": "2021-06-29T23:47:34.771578Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    os.mkdir(f'{TEST_RESULTS_DIR}/{nama_file}')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "# save to csv: \n",
    "with open(df_savepath, mode='w') as f:\n",
    "    test_df.to_csv(\n",
    "        f,\n",
    "        header=True,\n",
    "        index=False,\n",
    "        line_terminator='\\n'\n",
    "        )\n",
    "    \n",
    "# print_to_file(f\"verbose.{nama_file}\", lenet5_modifikasi_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "652d9d7b-241d-46c8-bc18-8510ca5c50ac",
   "metadata": {},
   "source": [
    "## Eksperimen - model_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "cf4122f8-62b0-4b30-9e92-a38a174163cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:52:32.097575Z",
     "iopub.status.busy": "2021-06-29T23:52:32.097575Z",
     "iopub.status.idle": "2021-06-29T23:58:08.701753Z",
     "shell.execute_reply": "2021-06-29T23:58:08.700754Z",
     "shell.execute_reply.started": "2021-06-29T23:52:32.097575Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "nama_file: 'lenet-5 versi modifikasi (035).predict_turbo'\n",
      "weight_filename: 'lenet5_actual_rm-32-16.weights.035.hdf5'\n",
      "weight_filename"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 120ms/step\n",
      "-----\n",
      "(26349.08 ms) == ( 26s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      ": 'lenet5_actual_rm-32-32.weights.035_0.9886_0.0437.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(26837.02 ms) == ( 26s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-32-64.weights.035_0.9955_0.0180.hdf5'\n",
      "weight_filename"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 125ms/step\n",
      "-----\n",
      "(27513.53 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      ": 'lenet5_actual_rm-64-16.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(26816.56 ms) == ( 26s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-64-32.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(26791.06 ms) == ( 26s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-64-64.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(26827.54 ms) == ( 26s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-96-16.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 30s 139ms/step\n",
      "-----\n",
      "(30518.99 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-96-32.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step\n",
      "-----\n",
      "(28254.16 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-96-64.weights.035_0.9943_0.0189.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step\n",
      "-----\n",
      "(26986.88 ms) == ( 26s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-128-16.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 127ms/step\n",
      "-----\n",
      "(27809.22 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-128-32.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 127ms/step\n",
      "-----\n",
      "(27844.29 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'lenet5_actual_rm-128-64.weights.035.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step\n",
      "-----\n",
      "(27672.36 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lenet5_modifikasi_df = pd.DataFrame(columns=['name'])\n",
    "lenet5_modifikasi_output = \"\"\n",
    "\n",
    "nama_file = 'lenet-5 versi modifikasi (035)' + '.predict_turbo'\n",
    "df_savepath = f'{TEST_RESULTS_DIR}/{nama_file}/history.{nama_file}.csv'\n",
    "ic(nama_file)\n",
    "\n",
    "try:\n",
    "    os.mkdir(f'{TEST_RESULTS_DIR}/{nama_file}')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "i = 0\n",
    "for input_size in _input_size:\n",
    "    for batch_size in _batch_size:\n",
    "    \n",
    "        # versi modifikasi only\n",
    "        activation = 'relu'\n",
    "        pooling = 'max'\n",
    "    \n",
    "        kwargs = {\n",
    "            'input_size' : input_size,\n",
    "            'activation' : activation,\n",
    "            'pooling' : pooling,\n",
    "            'batch_size' : batch_size\n",
    "        }\n",
    "        nama_model = create_lenet_name(**kwargs)\n",
    "\n",
    "        ### start evaluate\n",
    "        with io.StringIO() as buf, redirect_stdout(buf):\n",
    "            _, y_pred = model_predict(nama_model=nama_model, input_size=input_size, re_epoch=\"35\", debug=True)\n",
    "            lenet5_modifikasi_output += buf.getvalue()\n",
    "            _output = buf.getvalue()\n",
    "        print(_output, end='')\n",
    "\n",
    "        ### create confusion matrix & classification report\n",
    "        test_data_gen = create_test_datagen(input_size=input_size)\n",
    "\n",
    "        cm = confusion_matrix(\n",
    "            y_true=test_data_gen.classes,\n",
    "            y_pred=y_pred,\n",
    "            labels=[item for item in test_data_gen.class_indices.values()]\n",
    "        )\n",
    "\n",
    "        cr = classification_report(\n",
    "            y_true=test_data_gen.classes,\n",
    "            y_pred=y_pred,\n",
    "            labels=[item for item in test_data_gen.class_indices.values()],\n",
    "            target_names=[item for item in test_data_gen.class_indices.keys()],\n",
    "            digits=2,\n",
    "            output_dict=False,\n",
    "            zero_division=0\n",
    "        )\n",
    "\n",
    "        print_to_file(rf\"{nama_file}/{nama_model} cm-cr\" + \"_turbo\", f\"{cm}\\n\\n\\n{cr}\")\n",
    "\n",
    "        lenet5_modifikasi_df.loc[i] = nama_model\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b8c23535-40e7-48b3-b810-c7e8dd4f2042",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:58:08.702753Z",
     "iopub.status.busy": "2021-06-29T23:58:08.702753Z",
     "iopub.status.idle": "2021-06-29T23:58:08.733752Z",
     "shell.execute_reply": "2021-06-29T23:58:08.732753Z",
     "shell.execute_reply.started": "2021-06-29T23:58:08.702753Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>waktu</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>lenet5_actual_rm-32-16</td>\n",
       "      <td>120ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lenet5_actual_rm-32-32</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>lenet5_actual_rm-32-64</td>\n",
       "      <td>125ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>lenet5_actual_rm-64-16</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>lenet5_actual_rm-64-32</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>lenet5_actual_rm-64-64</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>lenet5_actual_rm-96-16</td>\n",
       "      <td>139ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>lenet5_actual_rm-96-32</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>lenet5_actual_rm-96-64</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>lenet5_actual_rm-128-16</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>lenet5_actual_rm-128-32</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>lenet5_actual_rm-128-64</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       name  waktu\n",
       "0    lenet5_actual_rm-32-16  120ms\n",
       "1    lenet5_actual_rm-32-32  122ms\n",
       "2    lenet5_actual_rm-32-64  125ms\n",
       "3    lenet5_actual_rm-64-16  122ms\n",
       "4    lenet5_actual_rm-64-32  122ms\n",
       "5    lenet5_actual_rm-64-64  122ms\n",
       "6    lenet5_actual_rm-96-16  139ms\n",
       "7    lenet5_actual_rm-96-32  129ms\n",
       "8    lenet5_actual_rm-96-64  123ms\n",
       "9   lenet5_actual_rm-128-16  127ms\n",
       "10  lenet5_actual_rm-128-32  127ms\n",
       "11  lenet5_actual_rm-128-64  126ms"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_df = lenet5_modifikasi_df\n",
    "\n",
    "waktu = re.findall(\"([0-9]+ms)\", lenet5_modifikasi_output)\n",
    "test_df['waktu'] = waktu\n",
    "\n",
    "display(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3327dc00-29bd-45d0-8de7-15fefce10827",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T23:58:08.734754Z",
     "iopub.status.busy": "2021-06-29T23:58:08.734754Z",
     "iopub.status.idle": "2021-06-29T23:58:08.812759Z",
     "shell.execute_reply": "2021-06-29T23:58:08.811762Z",
     "shell.execute_reply.started": "2021-06-29T23:58:08.734754Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# save to csv: \n",
    "with open(df_savepath, mode='w') as f:\n",
    "    test_df.to_csv(\n",
    "        f,\n",
    "        header=True,\n",
    "        index=False,\n",
    "        line_terminator='\\n'\n",
    "        )\n",
    "    \n",
    "# print_to_file(f\"verbose.{nama_file}\", lenet5_modifikasi_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe1bc6ff-2451-4128-8656-967ddb08dc2a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c81c22c4-ca7b-4251-9534-e9d421ebf808",
   "metadata": {},
   "source": [
    "# MobileNetV2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9756d453-f0cd-4fd0-bad8-83ce594f8f72",
   "metadata": {},
   "source": [
    "## model_evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "d9ffa6b2-ae78-4627-a66f-4329b5104324",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-30T00:30:23.466820Z",
     "iopub.status.busy": "2021-06-30T00:30:23.465820Z",
     "iopub.status.idle": "2021-06-30T00:57:48.913417Z",
     "shell.execute_reply": "2021-06-30T00:57:48.912416Z",
     "shell.execute_reply.started": "2021-06-30T00:30:23.465820Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "nama_file: 'mobilenetv2.evaluate_turbo'\n",
      "weight_filename: 'mobilenetv2_1.00_32-16-12.weights.045_0.9954_0.0185.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 127ms/step - loss: 2.9348 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(30792.44 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-16-fc12.weights.049_0.7533_0.5465.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 30s 139ms/step - loss: 4.7159 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(33404.55 ms) == ( 33s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-32-12.weights.046_0.9947_0.0173.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step - loss: 2.5232 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(31196.46 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-32-fc12.weights.049_0.9678_0.1378.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 29s 132ms/step - loss: 6.9353 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0833 - recall: 0.0833 - TP: 18.0000 - TN: 2178.0000 - FP: 198.0000 - FN: 198.0000\n",
      "-----\n",
      "(31432.47 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-64-12.weights.050_0.9953_0.0159.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step - loss: 2.7379 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(29518.92 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-64-fc12.weights.050_0.9899_0.0389.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 127ms/step - loss: 6.4178 - categorical_accuracy: 0.0833 - top-3: 0.2500 - ROC-AUC: 0.5000 - PR-AUC: 0.0833 - precision: 0.0833 - recall: 0.0833 - TP: 18.0000 - TN: 2178.0000 - FP: 198.0000 - FN: 198.0000           \n",
      "-----\n",
      "(30395.14 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-16-12.weights.025_0.2841_2.0599.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step - loss: 2.2715 - categorical_accuracy: 0.2731 - top-3: 0.4676 - ROC-AUC: 0.6763 - PR-AUC: 0.2297 - precision: 0.5909 - recall: 0.0602 - TP: 13.0000 - TN: 2367.0000 - FP: 9.0000 - FN: 203.0000     \n",
      "-----\n",
      "(29189.41 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-16-fc12.weights.047_0.2988_2.0399.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 127ms/step - loss: 2.2262 - categorical_accuracy: 0.2407 - top-3: 0.4537 - ROC-AUC: 0.6889 - PR-AUC: 0.2402 - precision: 0.7222 - recall: 0.0602 - TP: 13.0000 - TN: 2371.0000 - FP: 5.0000 - FN: 203.0000     \n",
      "-----\n",
      "(31339.41 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-32-12.weights.047_0.2966_2.0507.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step - loss: 2.2459 - categorical_accuracy: 0.2917 - top-3: 0.4398 - ROC-AUC: 0.6742 - PR-AUC: 0.2369 - precision: 0.7619 - recall: 0.0741 - TP: 16.0000 - TN: 2371.0000 - FP: 5.0000 - FN: 200.0000     \n",
      "-----\n",
      "(29906.06 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-32-fc12.weights.042_0.2943_2.0200.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step - loss: 2.2973 - categorical_accuracy: 0.2685 - top-3: 0.4306 - ROC-AUC: 0.6862 - PR-AUC: 0.2691 - precision: 0.8000 - recall: 0.0926 - TP: 20.0000 - TN: 2371.0000 - FP: 5.0000 - FN: 196.0000     \n",
      "-----\n",
      "(32001.31 ms) == ( 32s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-64-12.weights.031_0.2817_2.0544.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step - loss: 2.2672 - categorical_accuracy: 0.2778 - top-3: 0.4167 - ROC-AUC: 0.6606 - PR-AUC: 0.2344 - precision: 0.7500 - recall: 0.0694 - TP: 15.0000 - TN: 2371.0000 - FP: 5.0000 - FN: 201.0000     \n",
      "-----\n",
      "(29228.20 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-64-fc12.weights.050_0.3070_2.0004.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 123ms/step - loss: 2.2255 - categorical_accuracy: 0.2639 - top-3: 0.4861 - ROC-AUC: 0.7001 - PR-AUC: 0.2489 - precision: 0.5400 - recall: 0.1250 - TP: 27.0000 - TN: 2353.0000 - FP: 23.0000 - FN: 189.000000  \n",
      "-----\n",
      "(29113.78 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-16-12.weights.048_0.9962_0.0103.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step - loss: 1.8560 - categorical_accuracy: 0.4815 - top-3: 0.6620 - ROC-AUC: 0.8645 - PR-AUC: 0.5259 - precision: 0.6203 - recall: 0.4537 - TP: 98.0000 - TN: 2316.0000 - FP: 60.0000 - FN: 118.0000         0\n",
      "-----\n",
      "(29827.99 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-16-fc12.weights.043_0.9813_0.0794.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 124ms/step - loss: 3.2050 - categorical_accuracy: 0.1898 - top-3: 0.3657 - ROC-AUC: 0.6385 - PR-AUC: 0.1398 - precision: 0.2330 - recall: 0.1111 - TP: 24.0000 - TN: 2297.0000 - FP: 79.0000 - FN: 192.0000    0    0\n",
      "-----\n",
      "(29624.19 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-32-12.weights.046_0.9988_0.0041.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 125ms/step - loss: 3.3224 - categorical_accuracy: 0.2917 - top-3: 0.4537 - ROC-AUC: 0.6528 - PR-AUC: 0.2939 - precision: 0.4623 - recall: 0.2269 - TP: 49.0000 - TN: 2319.0000 - FP: 57.0000 - FN: 167.0000  00      \n",
      "-----\n",
      "(29734.78 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-32-fc12.weights.048_0.9936_0.0256.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 124ms/step - loss: 3.0709 - categorical_accuracy: 0.2639 - top-3: 0.5556 - ROC-AUC: 0.6558 - PR-AUC: 0.1903 - precision: 0.2857 - recall: 0.0463 - TP: 10.0000 - TN: 2351.0000 - FP: 25.0000 - FN: 206.0000  000   \n",
      "-----\n",
      "(30508.29 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-64-12.weights.038_0.9990_0.0039.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 125ms/step - loss: 2.6091 - categorical_accuracy: 0.2269 - top-3: 0.4769 - ROC-AUC: 0.6481 - PR-AUC: 0.1827 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 2376.0000 - FP: 0.0000e+00 - FN: 216.0000\n",
      "-----\n",
      "(29812.86 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-64-fc12.weights.049_0.9942_0.0268.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 124ms/step - loss: 3.8957 - categorical_accuracy: 0.0833 - top-3: 0.2222 - ROC-AUC: 0.5202 - PR-AUC: 0.0839 - precision: 0.1154 - recall: 0.0833 - TP: 18.0000 - TN: 2238.0000 - FP: 138.0000 - FN: 198.0000         \n",
      "-----\n",
      "(29640.99 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-16-12.weights.026_0.9636_0.2933.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step - loss: 6.5548 - categorical_accuracy: 0.8426 - top-3: 0.9722 - ROC-AUC: 0.9212 - PR-AUC: 0.7581 - precision: 0.8426 - recall: 0.8426 - TP: 182.0000 - TN: 2342.0000 - FP: 34.0000 - FN: 34.0000\n",
      "-----\n",
      "(30326.28 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-16-fc12.weights.034_0.9420_0.2607.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step - loss: 1.0577 - categorical_accuracy: 0.8333 - top-3: 0.9352 - ROC-AUC: 0.9443 - PR-AUC: 0.8294 - precision: 0.8483 - recall: 0.8287 - TP: 179.0000 - TN: 2344.0000 - FP: 32.0000 - FN: 37.0000\n",
      "-----\n",
      "(29210.94 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-32-12.weights.048_0.9711_0.2074.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step - loss: 2.2710 - categorical_accuracy: 0.8981 - top-3: 0.9907 - ROC-AUC: 0.9539 - PR-AUC: 0.8515 - precision: 0.8981 - recall: 0.8981 - TP: 194.0000 - TN: 2354.0000 - FP: 22.0000 - FN: 22.0000\n",
      "-----\n",
      "(29860.20 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-32-fc12.weights.039_0.9602_0.1616.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 125ms/step - loss: 0.7422 - categorical_accuracy: 0.8241 - top-3: 0.9074 - ROC-AUC: 0.9680 - PR-AUC: 0.9039 - precision: 0.8878 - recall: 0.8056 - TP: 174.0000 - TN: 2354.0000 - FP: 22.0000 - FN: 42.0000\n",
      "-----\n",
      "(29693.38 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-64-12.weights.026_0.9638_0.1562.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step - loss: 1.4764 - categorical_accuracy: 0.8981 - top-3: 1.0000 - ROC-AUC: 0.9563 - PR-AUC: 0.8596 - precision: 0.8981 - recall: 0.8981 - TP: 194.0000 - TN: 2354.0000 - FP: 22.0000 - FN: 22.0000\n",
      "-----\n",
      "(30444.98 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-64-fc12.weights.035_0.9661_0.1243.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step - loss: 1.5870 - categorical_accuracy: 0.8750 - top-3: 0.9398 - ROC-AUC: 0.9751 - PR-AUC: 0.9223 - precision: 0.9255 - recall: 0.8056 - TP: 174.0000 - TN: 2362.0000 - FP: 14.0000 - FN: 42.0000\n",
      "-----\n",
      "(29262.20 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-16-12.weights.046_0.9987_0.0042.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step - loss: 1.2864 - categorical_accuracy: 0.5880 - top-3: 0.8426 - ROC-AUC: 0.9239 - PR-AUC: 0.6654 - precision: 0.6230 - recall: 0.5509 - TP: 119.0000 - TN: 2304.0000 - FP: 72.0000 - FN: 97.0000\n",
      "-----\n",
      "(30304.43 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-16-fc12.weights.050_0.8985_0.1913.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 128ms/step - loss: 17.4358 - categorical_accuracy: 0.2037 - top-3: 0.3935 - ROC-AUC: 0.5509 - PR-AUC: 0.1148 - precision: 0.2037 - recall: 0.2037 - TP: 44.0000 - TN: 2204.0000 - FP: 172.0000 - FN: 172.0000\n",
      "-----\n",
      "(31821.78 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-32-12.weights.045_0.9987_0.0027.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 29s 133ms/step - loss: 2.7892 - categorical_accuracy: 0.3009 - top-3: 0.5463 - ROC-AUC: 0.7404 - PR-AUC: 0.3200 - precision: 0.3289 - recall: 0.2315 - TP: 50.0000 - TN: 2274.0000 - FP: 102.0000 - FN: 166.0000 \n",
      "-----\n",
      "(31519.38 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-32-fc12.weights.040_0.9922_0.0273.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step - loss: 3.4230 - categorical_accuracy: 0.1065 - top-3: 0.4213 - ROC-AUC: 0.6184 - PR-AUC: 0.1273 - precision: 0.1565 - recall: 0.0833 - TP: 18.0000 - TN: 2279.0000 - FP: 97.0000 - FN: 198.0000    00  \n",
      "-----\n",
      "(31192.16 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-64-12.weights.031_0.9995_0.0023.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 125ms/step - loss: 3.8575 - categorical_accuracy: 0.1667 - top-3: 0.2500 - ROC-AUC: 0.6064 - PR-AUC: 0.1616 - precision: 0.2857 - recall: 0.1667 - TP: 36.0000 - TN: 2286.0000 - FP: 90.0000 - FN: 180.0000          00\n",
      "-----\n",
      "(31268.35 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-64-fc12.weights.050_0.9955_0.0152.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 128ms/step - loss: 7.0707 - categorical_accuracy: 0.0926 - top-3: 0.2731 - ROC-AUC: 0.5132 - PR-AUC: 0.0902 - precision: 0.0952 - recall: 0.0833 - TP: 18.0000 - TN: 2205.0000 - FP: 171.0000 - FN: 198.0000     \n",
      "-----\n",
      "(30681.01 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-16-12.weights.037_0.9975_0.0125.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step - loss: 0.2803 - categorical_accuracy: 0.9722 - top-3: 0.9954 - ROC-AUC: 0.9900 - PR-AUC: 0.9712 - precision: 0.9722 - recall: 0.9722 - TP: 210.0000 - TN: 2370.0000 - FP: 6.0000 - FN: 6.0000    \n",
      "-----\n",
      "(29840.00 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-16-fc12.weights.049_0.9902_0.0360.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step - loss: 0.2582 - categorical_accuracy: 0.9352 - top-3: 0.9676 - ROC-AUC: 0.9930 - PR-AUC: 0.9788 - precision: 0.9897 - recall: 0.8889 - TP: 192.0000 - TN: 2374.0000 - FP: 2.0000 - FN: 24.0000\n",
      "-----\n",
      "(29805.17 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-32-12.weights.040_0.9975_0.0103.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 128ms/step - loss: 0.7938 - categorical_accuracy: 0.9444 - top-3: 0.9907 - ROC-AUC: 0.9772 - PR-AUC: 0.9291 - precision: 0.9444 - recall: 0.9444 - TP: 204.0000 - TN: 2364.0000 - FP: 12.0000 - FN: 12.0000  \n",
      "-----\n",
      "(30218.28 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-32-fc12.weights.036_0.9954_0.0167.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step - loss: 5.7281 - categorical_accuracy: 0.1343 - top-3: 0.2685 - ROC-AUC: 0.5065 - PR-AUC: 0.0873 - precision: 0.0914 - recall: 0.0787 - TP: 17.0000 - TN: 2207.0000 - FP: 169.0000 - FN: 199.0000   00    \n",
      "-----\n",
      "(30654.80 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-64-12.weights.011_0.9978_0.0074.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 127ms/step - loss: 0.2561 - categorical_accuracy: 0.9491 - top-3: 0.9954 - ROC-AUC: 0.9919 - PR-AUC: 0.9727 - precision: 0.9488 - recall: 0.9444 - TP: 204.0000 - TN: 2365.0000 - FP: 11.0000 - FN: 12.0000\n",
      "-----\n",
      "(30043.66 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-64-fc12.weights.035_0.9963_0.0128.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step - loss: 0.5530 - categorical_accuracy: 0.9213 - top-3: 0.9815 - ROC-AUC: 0.9871 - PR-AUC: 0.9473 - precision: 0.9415 - recall: 0.8935 - TP: 193.0000 - TN: 2364.0000 - FP: 12.0000 - FN: 23.0000\n",
      "-----\n",
      "(29803.68 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-16-12.weights.043_0.9998_0.0004.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 127ms/step - loss: 0.9346 - categorical_accuracy: 0.6944 - top-3: 0.9306 - ROC-AUC: 0.9637 - PR-AUC: 0.8021 - precision: 0.7337 - recall: 0.6759 - TP: 146.0000 - TN: 2323.0000 - FP: 53.0000 - FN: 70.0000\n",
      "-----\n",
      "(30545.64 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-16-fc12.weights.049_0.9808_0.0826.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step - loss: 3.2924 - categorical_accuracy: 0.3519 - top-3: 0.6065 - ROC-AUC: 0.7066 - PR-AUC: 0.2244 - precision: 0.3598 - recall: 0.2731 - TP: 59.0000 - TN: 2271.0000 - FP: 105.0000 - FN: 157.0000\n",
      "-----\n",
      "(30674.31 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-32-12.weights.048_0.9995_0.0012.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step - loss: 2.8330 - categorical_accuracy: 0.2454 - top-3: 0.5324 - ROC-AUC: 0.7186 - PR-AUC: 0.2947 - precision: 0.2840 - recall: 0.2130 - TP: 46.0000 - TN: 2260.0000 - FP: 116.0000 - FN: 170.0000 00 0\n",
      "-----\n",
      "(30639.48 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-32-fc12.weights.045_0.9859_0.0628.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step - loss: 3.6286 - categorical_accuracy: 0.2315 - top-3: 0.4954 - ROC-AUC: 0.6459 - PR-AUC: 0.1746 - precision: 0.2930 - recall: 0.2130 - TP: 46.0000 - TN: 2265.0000 - FP: 111.0000 - FN: 170.0000 0   \n",
      "-----\n",
      "(30831.49 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-64-12.weights.044_0.9995_0.0015.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step - loss: 3.8492 - categorical_accuracy: 0.1019 - top-3: 0.3472 - ROC-AUC: 0.6104 - PR-AUC: 0.1453 - precision: 0.1143 - recall: 0.0926 - TP: 20.0000 - TN: 2221.0000 - FP: 155.0000 - FN: 196.0000 00 0\n",
      "-----\n",
      "(30765.35 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-64-fc12.weights.035_0.9990_0.0062.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step - loss: 3.6210 - categorical_accuracy: 0.1204 - top-3: 0.3380 - ROC-AUC: 0.5542 - PR-AUC: 0.1153 - precision: 0.2326 - recall: 0.0926 - TP: 20.0000 - TN: 2310.0000 - FP: 66.0000 - FN: 196.0000  00  00  \n",
      "-----\n",
      "(30924.36 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-16-12.weights.041_0.9995_0.0031.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step - loss: 1.8095 - categorical_accuracy: 0.9028 - top-3: 0.9815 - ROC-AUC: 0.9542 - PR-AUC: 0.8564 - precision: 0.9028 - recall: 0.9028 - TP: 195.0000 - TN: 2355.0000 - FP: 21.0000 - FN: 21.0000\n",
      "-----\n",
      "(30558.80 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-16-fc12.weights.019_0.9955_0.0284.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 29s 134ms/step - loss: 0.5300 - categorical_accuracy: 0.9120 - top-3: 0.9491 - ROC-AUC: 0.9837 - PR-AUC: 0.9404 - precision: 0.9242 - recall: 0.9028 - TP: 195.0000 - TN: 2360.0000 - FP: 16.0000 - FN: 21.0000\n",
      "-----\n",
      "(31501.64 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-32-12.weights.045_0.9992_0.0037.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 30s 138ms/step - loss: 0.9410 - categorical_accuracy: 0.9167 - top-3: 0.9769 - ROC-AUC: 0.9765 - PR-AUC: 0.9217 - precision: 0.9167 - recall: 0.9167 - TP: 198.0000 - TN: 2358.0000 - FP: 18.0000 - FN: 18.0000\n",
      "-----\n",
      "(33196.00 ms) == ( 33s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-32-fc12.weights.029_0.9950_0.0180.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 132ms/step - loss: 1.1129 - categorical_accuracy: 0.8889 - top-3: 0.9120 - ROC-AUC: 0.9541 - PR-AUC: 0.8847 - precision: 0.9078 - recall: 0.8657 - TP: 187.0000 - TN: 2357.0000 - FP: 19.0000 - FN: 29.0000         0\n",
      "-----\n",
      "(30997.52 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-64-12.weights.049_0.9992_0.0044.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 131ms/step - loss: 0.7685 - categorical_accuracy: 0.9306 - top-3: 0.9861 - ROC-AUC: 0.9721 - PR-AUC: 0.9176 - precision: 0.9306 - recall: 0.9306 - TP: 201.0000 - TN: 2361.0000 - FP: 15.0000 - FN: 15.0000\n",
      "-----\n",
      "(30789.32 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-64-fc12.weights.019_0.9973_0.0096.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step - loss: 0.5204 - categorical_accuracy: 0.8935 - top-3: 0.9259 - ROC-AUC: 0.9781 - PR-AUC: 0.9407 - precision: 0.9310 - recall: 0.8750 - TP: 189.0000 - TN: 2362.0000 - FP: 14.0000 - FN: 27.0000\n",
      "-----\n",
      "(32779.71 ms) == ( 32s)\n",
      "-----\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mobilenetv2_df = pd.DataFrame(columns=metrics)\n",
    "mobilenetv2_output = \"\"\n",
    "\n",
    "nama_file = 'mobilenetv2' + '.evaluate_turbo'\n",
    "df_savepath = f'{TEST_RESULTS_DIR}/{nama_file}/history.{nama_file}.csv'\n",
    "ic(nama_file)\n",
    "\n",
    "i = 0\n",
    "for input_size in _input_size:\n",
    "    for weights in _weights:\n",
    "        for batch_size in _batch_size:\n",
    "            for dense in _dense:\n",
    "\n",
    "                kwargs = {\n",
    "                    'input_size' : input_size,\n",
    "                    'batch_size' : batch_size,\n",
    "                    'weights' : weights,\n",
    "                    'dense' : dense\n",
    "                }\n",
    "                nama_model = create_mobilenetv2_name(**kwargs)\n",
    "    \n",
    "                with io.StringIO() as buf, redirect_stdout(buf):\n",
    "                    _, dict_2 = model_evaluate(nama_model=nama_model, input_size=input_size, debug=True)\n",
    "                    mobilenetv2_output += buf.getvalue()\n",
    "                    _output = buf.getvalue()\n",
    "                print(_output, end='')\n",
    "\n",
    "                mobilenetv2_df.loc[i] = list(dict_2.values())\n",
    "                i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "8a84a824-a229-4d19-a549-5e64b6470f83",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-30T00:57:48.915415Z",
     "iopub.status.busy": "2021-06-30T00:57:48.915415Z",
     "iopub.status.idle": "2021-06-30T00:57:49.088432Z",
     "shell.execute_reply": "2021-06-30T00:57:49.087430Z",
     "shell.execute_reply.started": "2021-06-30T00:57:48.915415Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>loss</th>\n",
       "      <th>categorical_accuracy</th>\n",
       "      <th>top-3</th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>PR-AUC</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>TP</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>waktu</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mobilenetv2_1.00_32-16-12</td>\n",
       "      <td>2.934757</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mobilenetv2_1.00_32-16-fc12</td>\n",
       "      <td>4.715878</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>139ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mobilenetv2_1.00_32-32-12</td>\n",
       "      <td>2.523243</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mobilenetv2_1.00_32-32-fc12</td>\n",
       "      <td>6.935299</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2178.0</td>\n",
       "      <td>198.0</td>\n",
       "      <td>198.0</td>\n",
       "      <td>132ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mobilenetv2_1.00_32-64-12</td>\n",
       "      <td>2.737893</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>mobilenetv2_1.00_32-64-fc12</td>\n",
       "      <td>6.417768</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2178.0</td>\n",
       "      <td>198.0</td>\n",
       "      <td>198.0</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-16-12</td>\n",
       "      <td>2.271506</td>\n",
       "      <td>0.273148</td>\n",
       "      <td>0.467593</td>\n",
       "      <td>0.676328</td>\n",
       "      <td>0.229739</td>\n",
       "      <td>0.590909</td>\n",
       "      <td>0.060185</td>\n",
       "      <td>13.0</td>\n",
       "      <td>2367.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>203.0</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-16-fc12</td>\n",
       "      <td>2.226222</td>\n",
       "      <td>0.240741</td>\n",
       "      <td>0.453704</td>\n",
       "      <td>0.688892</td>\n",
       "      <td>0.240246</td>\n",
       "      <td>0.722222</td>\n",
       "      <td>0.060185</td>\n",
       "      <td>13.0</td>\n",
       "      <td>2371.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>203.0</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-32-12</td>\n",
       "      <td>2.245855</td>\n",
       "      <td>0.291667</td>\n",
       "      <td>0.439815</td>\n",
       "      <td>0.674189</td>\n",
       "      <td>0.236907</td>\n",
       "      <td>0.761905</td>\n",
       "      <td>0.074074</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2371.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-32-fc12</td>\n",
       "      <td>2.297338</td>\n",
       "      <td>0.268519</td>\n",
       "      <td>0.430556</td>\n",
       "      <td>0.686186</td>\n",
       "      <td>0.269113</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.092593</td>\n",
       "      <td>20.0</td>\n",
       "      <td>2371.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>196.0</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-64-12</td>\n",
       "      <td>2.267226</td>\n",
       "      <td>0.277778</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.660576</td>\n",
       "      <td>0.234417</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.069444</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2371.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-64-fc12</td>\n",
       "      <td>2.225513</td>\n",
       "      <td>0.263889</td>\n",
       "      <td>0.486111</td>\n",
       "      <td>0.700086</td>\n",
       "      <td>0.248923</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>27.0</td>\n",
       "      <td>2353.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mobilenetv2_1.00_64-16-12</td>\n",
       "      <td>1.855950</td>\n",
       "      <td>0.481481</td>\n",
       "      <td>0.662037</td>\n",
       "      <td>0.864486</td>\n",
       "      <td>0.525939</td>\n",
       "      <td>0.620253</td>\n",
       "      <td>0.453704</td>\n",
       "      <td>98.0</td>\n",
       "      <td>2316.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>mobilenetv2_1.00_64-16-fc12</td>\n",
       "      <td>3.205030</td>\n",
       "      <td>0.189815</td>\n",
       "      <td>0.365741</td>\n",
       "      <td>0.638470</td>\n",
       "      <td>0.139784</td>\n",
       "      <td>0.233010</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>24.0</td>\n",
       "      <td>2297.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>124ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>mobilenetv2_1.00_64-32-12</td>\n",
       "      <td>3.322379</td>\n",
       "      <td>0.291667</td>\n",
       "      <td>0.453704</td>\n",
       "      <td>0.652830</td>\n",
       "      <td>0.293865</td>\n",
       "      <td>0.462264</td>\n",
       "      <td>0.226852</td>\n",
       "      <td>49.0</td>\n",
       "      <td>2319.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>167.0</td>\n",
       "      <td>125ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>mobilenetv2_1.00_64-32-fc12</td>\n",
       "      <td>3.070916</td>\n",
       "      <td>0.263889</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.655800</td>\n",
       "      <td>0.190305</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.046296</td>\n",
       "      <td>10.0</td>\n",
       "      <td>2351.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>124ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>mobilenetv2_1.00_64-64-12</td>\n",
       "      <td>2.609080</td>\n",
       "      <td>0.226852</td>\n",
       "      <td>0.476852</td>\n",
       "      <td>0.648145</td>\n",
       "      <td>0.182672</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>125ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>mobilenetv2_1.00_64-64-fc12</td>\n",
       "      <td>3.895672</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.520247</td>\n",
       "      <td>0.083870</td>\n",
       "      <td>0.115385</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2238.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>198.0</td>\n",
       "      <td>124ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-16-12</td>\n",
       "      <td>6.554763</td>\n",
       "      <td>0.842593</td>\n",
       "      <td>0.972222</td>\n",
       "      <td>0.921231</td>\n",
       "      <td>0.758111</td>\n",
       "      <td>0.842593</td>\n",
       "      <td>0.842593</td>\n",
       "      <td>182.0</td>\n",
       "      <td>2342.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-16-fc12</td>\n",
       "      <td>1.057743</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.935185</td>\n",
       "      <td>0.944260</td>\n",
       "      <td>0.829389</td>\n",
       "      <td>0.848341</td>\n",
       "      <td>0.828704</td>\n",
       "      <td>179.0</td>\n",
       "      <td>2344.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-32-12</td>\n",
       "      <td>2.270993</td>\n",
       "      <td>0.898148</td>\n",
       "      <td>0.990741</td>\n",
       "      <td>0.953927</td>\n",
       "      <td>0.851473</td>\n",
       "      <td>0.898148</td>\n",
       "      <td>0.898148</td>\n",
       "      <td>194.0</td>\n",
       "      <td>2354.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-32-fc12</td>\n",
       "      <td>0.742205</td>\n",
       "      <td>0.824074</td>\n",
       "      <td>0.907407</td>\n",
       "      <td>0.968001</td>\n",
       "      <td>0.903871</td>\n",
       "      <td>0.887755</td>\n",
       "      <td>0.805556</td>\n",
       "      <td>174.0</td>\n",
       "      <td>2354.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>125ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-64-12</td>\n",
       "      <td>1.476430</td>\n",
       "      <td>0.898148</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.956282</td>\n",
       "      <td>0.859644</td>\n",
       "      <td>0.898148</td>\n",
       "      <td>0.898148</td>\n",
       "      <td>194.0</td>\n",
       "      <td>2354.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-64-fc12</td>\n",
       "      <td>1.586980</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.939815</td>\n",
       "      <td>0.975063</td>\n",
       "      <td>0.922255</td>\n",
       "      <td>0.925532</td>\n",
       "      <td>0.805556</td>\n",
       "      <td>174.0</td>\n",
       "      <td>2362.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>mobilenetv2_1.00_96-16-12</td>\n",
       "      <td>1.286382</td>\n",
       "      <td>0.587963</td>\n",
       "      <td>0.842593</td>\n",
       "      <td>0.923893</td>\n",
       "      <td>0.665407</td>\n",
       "      <td>0.623037</td>\n",
       "      <td>0.550926</td>\n",
       "      <td>119.0</td>\n",
       "      <td>2304.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>mobilenetv2_1.00_96-16-fc12</td>\n",
       "      <td>17.435793</td>\n",
       "      <td>0.203704</td>\n",
       "      <td>0.393519</td>\n",
       "      <td>0.550936</td>\n",
       "      <td>0.114797</td>\n",
       "      <td>0.203704</td>\n",
       "      <td>0.203704</td>\n",
       "      <td>44.0</td>\n",
       "      <td>2204.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>128ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>mobilenetv2_1.00_96-32-12</td>\n",
       "      <td>2.789221</td>\n",
       "      <td>0.300926</td>\n",
       "      <td>0.546296</td>\n",
       "      <td>0.740425</td>\n",
       "      <td>0.320038</td>\n",
       "      <td>0.328947</td>\n",
       "      <td>0.231481</td>\n",
       "      <td>50.0</td>\n",
       "      <td>2274.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>166.0</td>\n",
       "      <td>133ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>mobilenetv2_1.00_96-32-fc12</td>\n",
       "      <td>3.423041</td>\n",
       "      <td>0.106481</td>\n",
       "      <td>0.421296</td>\n",
       "      <td>0.618422</td>\n",
       "      <td>0.127292</td>\n",
       "      <td>0.156522</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2279.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>198.0</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>mobilenetv2_1.00_96-64-12</td>\n",
       "      <td>3.857547</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.606441</td>\n",
       "      <td>0.161637</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>36.0</td>\n",
       "      <td>2286.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>125ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>mobilenetv2_1.00_96-64-fc12</td>\n",
       "      <td>7.070718</td>\n",
       "      <td>0.092593</td>\n",
       "      <td>0.273148</td>\n",
       "      <td>0.513169</td>\n",
       "      <td>0.090236</td>\n",
       "      <td>0.095238</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2205.0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>198.0</td>\n",
       "      <td>128ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-16-12</td>\n",
       "      <td>0.280338</td>\n",
       "      <td>0.972222</td>\n",
       "      <td>0.995370</td>\n",
       "      <td>0.990032</td>\n",
       "      <td>0.971218</td>\n",
       "      <td>0.972222</td>\n",
       "      <td>0.972222</td>\n",
       "      <td>210.0</td>\n",
       "      <td>2370.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-16-fc12</td>\n",
       "      <td>0.258153</td>\n",
       "      <td>0.935185</td>\n",
       "      <td>0.967593</td>\n",
       "      <td>0.992987</td>\n",
       "      <td>0.978815</td>\n",
       "      <td>0.989691</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>192.0</td>\n",
       "      <td>2374.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-32-12</td>\n",
       "      <td>0.793756</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.990741</td>\n",
       "      <td>0.977209</td>\n",
       "      <td>0.929119</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>204.0</td>\n",
       "      <td>2364.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>128ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-32-fc12</td>\n",
       "      <td>5.728146</td>\n",
       "      <td>0.134259</td>\n",
       "      <td>0.268519</td>\n",
       "      <td>0.506452</td>\n",
       "      <td>0.087326</td>\n",
       "      <td>0.091398</td>\n",
       "      <td>0.078704</td>\n",
       "      <td>17.0</td>\n",
       "      <td>2207.0</td>\n",
       "      <td>169.0</td>\n",
       "      <td>199.0</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-64-12</td>\n",
       "      <td>0.256071</td>\n",
       "      <td>0.949074</td>\n",
       "      <td>0.995370</td>\n",
       "      <td>0.991898</td>\n",
       "      <td>0.972698</td>\n",
       "      <td>0.948837</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>204.0</td>\n",
       "      <td>2365.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-64-fc12</td>\n",
       "      <td>0.553042</td>\n",
       "      <td>0.921296</td>\n",
       "      <td>0.981481</td>\n",
       "      <td>0.987137</td>\n",
       "      <td>0.947295</td>\n",
       "      <td>0.941463</td>\n",
       "      <td>0.893519</td>\n",
       "      <td>193.0</td>\n",
       "      <td>2364.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>mobilenetv2_1.00_128-16-12</td>\n",
       "      <td>0.934586</td>\n",
       "      <td>0.694444</td>\n",
       "      <td>0.930556</td>\n",
       "      <td>0.963663</td>\n",
       "      <td>0.802126</td>\n",
       "      <td>0.733668</td>\n",
       "      <td>0.675926</td>\n",
       "      <td>146.0</td>\n",
       "      <td>2323.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>mobilenetv2_1.00_128-16-fc12</td>\n",
       "      <td>3.292364</td>\n",
       "      <td>0.351852</td>\n",
       "      <td>0.606481</td>\n",
       "      <td>0.706648</td>\n",
       "      <td>0.224381</td>\n",
       "      <td>0.359756</td>\n",
       "      <td>0.273148</td>\n",
       "      <td>59.0</td>\n",
       "      <td>2271.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>mobilenetv2_1.00_128-32-12</td>\n",
       "      <td>2.832994</td>\n",
       "      <td>0.245370</td>\n",
       "      <td>0.532407</td>\n",
       "      <td>0.718585</td>\n",
       "      <td>0.294687</td>\n",
       "      <td>0.283951</td>\n",
       "      <td>0.212963</td>\n",
       "      <td>46.0</td>\n",
       "      <td>2260.0</td>\n",
       "      <td>116.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>mobilenetv2_1.00_128-32-fc12</td>\n",
       "      <td>3.628558</td>\n",
       "      <td>0.231481</td>\n",
       "      <td>0.495370</td>\n",
       "      <td>0.645853</td>\n",
       "      <td>0.174620</td>\n",
       "      <td>0.292994</td>\n",
       "      <td>0.212963</td>\n",
       "      <td>46.0</td>\n",
       "      <td>2265.0</td>\n",
       "      <td>111.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>mobilenetv2_1.00_128-64-12</td>\n",
       "      <td>3.849226</td>\n",
       "      <td>0.101852</td>\n",
       "      <td>0.347222</td>\n",
       "      <td>0.610386</td>\n",
       "      <td>0.145296</td>\n",
       "      <td>0.114286</td>\n",
       "      <td>0.092593</td>\n",
       "      <td>20.0</td>\n",
       "      <td>2221.0</td>\n",
       "      <td>155.0</td>\n",
       "      <td>196.0</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>mobilenetv2_1.00_128-64-fc12</td>\n",
       "      <td>3.620984</td>\n",
       "      <td>0.120370</td>\n",
       "      <td>0.337963</td>\n",
       "      <td>0.554220</td>\n",
       "      <td>0.115301</td>\n",
       "      <td>0.232558</td>\n",
       "      <td>0.092593</td>\n",
       "      <td>20.0</td>\n",
       "      <td>2310.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>196.0</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-16-12</td>\n",
       "      <td>1.809454</td>\n",
       "      <td>0.902778</td>\n",
       "      <td>0.981481</td>\n",
       "      <td>0.954202</td>\n",
       "      <td>0.856368</td>\n",
       "      <td>0.902778</td>\n",
       "      <td>0.902778</td>\n",
       "      <td>195.0</td>\n",
       "      <td>2355.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-16-fc12</td>\n",
       "      <td>0.530010</td>\n",
       "      <td>0.912037</td>\n",
       "      <td>0.949074</td>\n",
       "      <td>0.983702</td>\n",
       "      <td>0.940441</td>\n",
       "      <td>0.924171</td>\n",
       "      <td>0.902778</td>\n",
       "      <td>195.0</td>\n",
       "      <td>2360.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>134ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-32-12</td>\n",
       "      <td>0.940998</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.976852</td>\n",
       "      <td>0.976513</td>\n",
       "      <td>0.921660</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>198.0</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>138ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-32-fc12</td>\n",
       "      <td>1.112879</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.912037</td>\n",
       "      <td>0.954095</td>\n",
       "      <td>0.884705</td>\n",
       "      <td>0.907767</td>\n",
       "      <td>0.865741</td>\n",
       "      <td>187.0</td>\n",
       "      <td>2357.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>132ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-64-12</td>\n",
       "      <td>0.768508</td>\n",
       "      <td>0.930556</td>\n",
       "      <td>0.986111</td>\n",
       "      <td>0.972091</td>\n",
       "      <td>0.917565</td>\n",
       "      <td>0.930556</td>\n",
       "      <td>0.930556</td>\n",
       "      <td>201.0</td>\n",
       "      <td>2361.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>131ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-64-fc12</td>\n",
       "      <td>0.520367</td>\n",
       "      <td>0.893519</td>\n",
       "      <td>0.925926</td>\n",
       "      <td>0.978113</td>\n",
       "      <td>0.940686</td>\n",
       "      <td>0.931035</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>189.0</td>\n",
       "      <td>2362.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        name       loss  categorical_accuracy  \\\n",
       "0                  mobilenetv2_1.00_32-16-12   2.934757              0.083333   \n",
       "1                mobilenetv2_1.00_32-16-fc12   4.715878              0.083333   \n",
       "2                  mobilenetv2_1.00_32-32-12   2.523243              0.083333   \n",
       "3                mobilenetv2_1.00_32-32-fc12   6.935299              0.083333   \n",
       "4                  mobilenetv2_1.00_32-64-12   2.737893              0.083333   \n",
       "5                mobilenetv2_1.00_32-64-fc12   6.417768              0.083333   \n",
       "6      mobilenetv2_1.00_32-imagenet224-16-12   2.271506              0.273148   \n",
       "7    mobilenetv2_1.00_32-imagenet224-16-fc12   2.226222              0.240741   \n",
       "8      mobilenetv2_1.00_32-imagenet224-32-12   2.245855              0.291667   \n",
       "9    mobilenetv2_1.00_32-imagenet224-32-fc12   2.297338              0.268519   \n",
       "10     mobilenetv2_1.00_32-imagenet224-64-12   2.267226              0.277778   \n",
       "11   mobilenetv2_1.00_32-imagenet224-64-fc12   2.225513              0.263889   \n",
       "12                 mobilenetv2_1.00_64-16-12   1.855950              0.481481   \n",
       "13               mobilenetv2_1.00_64-16-fc12   3.205030              0.189815   \n",
       "14                 mobilenetv2_1.00_64-32-12   3.322379              0.291667   \n",
       "15               mobilenetv2_1.00_64-32-fc12   3.070916              0.263889   \n",
       "16                 mobilenetv2_1.00_64-64-12   2.609080              0.226852   \n",
       "17               mobilenetv2_1.00_64-64-fc12   3.895672              0.083333   \n",
       "18     mobilenetv2_1.00_64-imagenet224-16-12   6.554763              0.842593   \n",
       "19   mobilenetv2_1.00_64-imagenet224-16-fc12   1.057743              0.833333   \n",
       "20     mobilenetv2_1.00_64-imagenet224-32-12   2.270993              0.898148   \n",
       "21   mobilenetv2_1.00_64-imagenet224-32-fc12   0.742205              0.824074   \n",
       "22     mobilenetv2_1.00_64-imagenet224-64-12   1.476430              0.898148   \n",
       "23   mobilenetv2_1.00_64-imagenet224-64-fc12   1.586980              0.875000   \n",
       "24                 mobilenetv2_1.00_96-16-12   1.286382              0.587963   \n",
       "25               mobilenetv2_1.00_96-16-fc12  17.435793              0.203704   \n",
       "26                 mobilenetv2_1.00_96-32-12   2.789221              0.300926   \n",
       "27               mobilenetv2_1.00_96-32-fc12   3.423041              0.106481   \n",
       "28                 mobilenetv2_1.00_96-64-12   3.857547              0.166667   \n",
       "29               mobilenetv2_1.00_96-64-fc12   7.070718              0.092593   \n",
       "30      mobilenetv2_1.00_96-imagenet96-16-12   0.280338              0.972222   \n",
       "31    mobilenetv2_1.00_96-imagenet96-16-fc12   0.258153              0.935185   \n",
       "32      mobilenetv2_1.00_96-imagenet96-32-12   0.793756              0.944444   \n",
       "33    mobilenetv2_1.00_96-imagenet96-32-fc12   5.728146              0.134259   \n",
       "34      mobilenetv2_1.00_96-imagenet96-64-12   0.256071              0.949074   \n",
       "35    mobilenetv2_1.00_96-imagenet96-64-fc12   0.553042              0.921296   \n",
       "36                mobilenetv2_1.00_128-16-12   0.934586              0.694444   \n",
       "37              mobilenetv2_1.00_128-16-fc12   3.292364              0.351852   \n",
       "38                mobilenetv2_1.00_128-32-12   2.832994              0.245370   \n",
       "39              mobilenetv2_1.00_128-32-fc12   3.628558              0.231481   \n",
       "40                mobilenetv2_1.00_128-64-12   3.849226              0.101852   \n",
       "41              mobilenetv2_1.00_128-64-fc12   3.620984              0.120370   \n",
       "42    mobilenetv2_1.00_128-imagenet128-16-12   1.809454              0.902778   \n",
       "43  mobilenetv2_1.00_128-imagenet128-16-fc12   0.530010              0.912037   \n",
       "44    mobilenetv2_1.00_128-imagenet128-32-12   0.940998              0.916667   \n",
       "45  mobilenetv2_1.00_128-imagenet128-32-fc12   1.112879              0.888889   \n",
       "46    mobilenetv2_1.00_128-imagenet128-64-12   0.768508              0.930556   \n",
       "47  mobilenetv2_1.00_128-imagenet128-64-fc12   0.520367              0.893519   \n",
       "\n",
       "       top-3   ROC-AUC    PR-AUC  precision    recall     TP      TN     FP  \\\n",
       "0   0.250000  0.500000  0.083333   0.000000  0.000000    0.0  2376.0    0.0   \n",
       "1   0.250000  0.500000  0.083333   0.000000  0.000000    0.0  2376.0    0.0   \n",
       "2   0.250000  0.500000  0.083333   0.000000  0.000000    0.0  2376.0    0.0   \n",
       "3   0.250000  0.500000  0.083333   0.083333  0.083333   18.0  2178.0  198.0   \n",
       "4   0.250000  0.500000  0.083333   0.000000  0.000000    0.0  2376.0    0.0   \n",
       "5   0.250000  0.500000  0.083333   0.083333  0.083333   18.0  2178.0  198.0   \n",
       "6   0.467593  0.676328  0.229739   0.590909  0.060185   13.0  2367.0    9.0   \n",
       "7   0.453704  0.688892  0.240246   0.722222  0.060185   13.0  2371.0    5.0   \n",
       "8   0.439815  0.674189  0.236907   0.761905  0.074074   16.0  2371.0    5.0   \n",
       "9   0.430556  0.686186  0.269113   0.800000  0.092593   20.0  2371.0    5.0   \n",
       "10  0.416667  0.660576  0.234417   0.750000  0.069444   15.0  2371.0    5.0   \n",
       "11  0.486111  0.700086  0.248923   0.540000  0.125000   27.0  2353.0   23.0   \n",
       "12  0.662037  0.864486  0.525939   0.620253  0.453704   98.0  2316.0   60.0   \n",
       "13  0.365741  0.638470  0.139784   0.233010  0.111111   24.0  2297.0   79.0   \n",
       "14  0.453704  0.652830  0.293865   0.462264  0.226852   49.0  2319.0   57.0   \n",
       "15  0.555556  0.655800  0.190305   0.285714  0.046296   10.0  2351.0   25.0   \n",
       "16  0.476852  0.648145  0.182672   0.000000  0.000000    0.0  2376.0    0.0   \n",
       "17  0.222222  0.520247  0.083870   0.115385  0.083333   18.0  2238.0  138.0   \n",
       "18  0.972222  0.921231  0.758111   0.842593  0.842593  182.0  2342.0   34.0   \n",
       "19  0.935185  0.944260  0.829389   0.848341  0.828704  179.0  2344.0   32.0   \n",
       "20  0.990741  0.953927  0.851473   0.898148  0.898148  194.0  2354.0   22.0   \n",
       "21  0.907407  0.968001  0.903871   0.887755  0.805556  174.0  2354.0   22.0   \n",
       "22  1.000000  0.956282  0.859644   0.898148  0.898148  194.0  2354.0   22.0   \n",
       "23  0.939815  0.975063  0.922255   0.925532  0.805556  174.0  2362.0   14.0   \n",
       "24  0.842593  0.923893  0.665407   0.623037  0.550926  119.0  2304.0   72.0   \n",
       "25  0.393519  0.550936  0.114797   0.203704  0.203704   44.0  2204.0  172.0   \n",
       "26  0.546296  0.740425  0.320038   0.328947  0.231481   50.0  2274.0  102.0   \n",
       "27  0.421296  0.618422  0.127292   0.156522  0.083333   18.0  2279.0   97.0   \n",
       "28  0.250000  0.606441  0.161637   0.285714  0.166667   36.0  2286.0   90.0   \n",
       "29  0.273148  0.513169  0.090236   0.095238  0.083333   18.0  2205.0  171.0   \n",
       "30  0.995370  0.990032  0.971218   0.972222  0.972222  210.0  2370.0    6.0   \n",
       "31  0.967593  0.992987  0.978815   0.989691  0.888889  192.0  2374.0    2.0   \n",
       "32  0.990741  0.977209  0.929119   0.944444  0.944444  204.0  2364.0   12.0   \n",
       "33  0.268519  0.506452  0.087326   0.091398  0.078704   17.0  2207.0  169.0   \n",
       "34  0.995370  0.991898  0.972698   0.948837  0.944444  204.0  2365.0   11.0   \n",
       "35  0.981481  0.987137  0.947295   0.941463  0.893519  193.0  2364.0   12.0   \n",
       "36  0.930556  0.963663  0.802126   0.733668  0.675926  146.0  2323.0   53.0   \n",
       "37  0.606481  0.706648  0.224381   0.359756  0.273148   59.0  2271.0  105.0   \n",
       "38  0.532407  0.718585  0.294687   0.283951  0.212963   46.0  2260.0  116.0   \n",
       "39  0.495370  0.645853  0.174620   0.292994  0.212963   46.0  2265.0  111.0   \n",
       "40  0.347222  0.610386  0.145296   0.114286  0.092593   20.0  2221.0  155.0   \n",
       "41  0.337963  0.554220  0.115301   0.232558  0.092593   20.0  2310.0   66.0   \n",
       "42  0.981481  0.954202  0.856368   0.902778  0.902778  195.0  2355.0   21.0   \n",
       "43  0.949074  0.983702  0.940441   0.924171  0.902778  195.0  2360.0   16.0   \n",
       "44  0.976852  0.976513  0.921660   0.916667  0.916667  198.0  2358.0   18.0   \n",
       "45  0.912037  0.954095  0.884705   0.907767  0.865741  187.0  2357.0   19.0   \n",
       "46  0.986111  0.972091  0.917565   0.930556  0.930556  201.0  2361.0   15.0   \n",
       "47  0.925926  0.978113  0.940686   0.931035  0.875000  189.0  2362.0   14.0   \n",
       "\n",
       "       FN  waktu  \n",
       "0   216.0  127ms  \n",
       "1   216.0  139ms  \n",
       "2   216.0  126ms  \n",
       "3   198.0  132ms  \n",
       "4   216.0  123ms  \n",
       "5   198.0  127ms  \n",
       "6   203.0  123ms  \n",
       "7   203.0  127ms  \n",
       "8   200.0  126ms  \n",
       "9   196.0  129ms  \n",
       "10  201.0  123ms  \n",
       "11  189.0  123ms  \n",
       "12  118.0  123ms  \n",
       "13  192.0  124ms  \n",
       "14  167.0  125ms  \n",
       "15  206.0  124ms  \n",
       "16  216.0  125ms  \n",
       "17  198.0  124ms  \n",
       "18   34.0  123ms  \n",
       "19   37.0  123ms  \n",
       "20   22.0  126ms  \n",
       "21   42.0  125ms  \n",
       "22   22.0  123ms  \n",
       "23   42.0  123ms  \n",
       "24   97.0  126ms  \n",
       "25  172.0  128ms  \n",
       "26  166.0  133ms  \n",
       "27  198.0  129ms  \n",
       "28  180.0  125ms  \n",
       "29  198.0  128ms  \n",
       "30    6.0  126ms  \n",
       "31   24.0  126ms  \n",
       "32   12.0  128ms  \n",
       "33  199.0  129ms  \n",
       "34   12.0  127ms  \n",
       "35   23.0  126ms  \n",
       "36   70.0  127ms  \n",
       "37  157.0  129ms  \n",
       "38  170.0  129ms  \n",
       "39  170.0  130ms  \n",
       "40  196.0  129ms  \n",
       "41  196.0  130ms  \n",
       "42   21.0  130ms  \n",
       "43   21.0  134ms  \n",
       "44   18.0  138ms  \n",
       "45   29.0  132ms  \n",
       "46   15.0  131ms  \n",
       "47   27.0  130ms  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_df = mobilenetv2_df\n",
    "\n",
    "waktu = re.findall(\"([0-9]+ms)\", mobilenetv2_output)\n",
    "test_df['waktu'] = waktu\n",
    "\n",
    "display(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "106de02e-faf0-4d5a-8d9d-d44d34c84540",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-30T00:57:49.090433Z",
     "iopub.status.busy": "2021-06-30T00:57:49.090433Z",
     "iopub.status.idle": "2021-06-30T00:57:49.152433Z",
     "shell.execute_reply": "2021-06-30T00:57:49.151436Z",
     "shell.execute_reply.started": "2021-06-30T00:57:49.090433Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    os.mkdir(f'{TEST_RESULTS_DIR}/{nama_file}')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "# save to csv: \n",
    "with open(df_savepath, mode='w') as f:\n",
    "    test_df.to_csv(\n",
    "        f,\n",
    "        header=True,\n",
    "        index=False,\n",
    "        line_terminator='\\n'\n",
    "        )\n",
    "    \n",
    "# print_to_file(f\"verbose.{nama_file}\", mobilenetv2_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfbc1cc3-c9fd-483a-9e02-d0e5a6f8d60d",
   "metadata": {},
   "source": [
    "## model_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b924b92d-1cc8-4fb5-87b2-1353a24ef071",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-30T00:57:49.369449Z",
     "iopub.status.busy": "2021-06-30T00:57:49.368453Z",
     "iopub.status.idle": "2021-06-30T01:25:04.124316Z",
     "shell.execute_reply": "2021-06-30T01:25:04.123318Z",
     "shell.execute_reply.started": "2021-06-30T00:57:49.369449Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "nama_file: 'mobilenetv2.predict_turbo'\n",
      "weight_filename: 'mobilenetv2_1.00_32-16-12.weights.045_0.9954_0.0185.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(27870.10 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-16-fc12.weights.049_0.7533_0.5465.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step\n",
      "-----\n",
      "(28050.81 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-32-12.weights.046_0.9947_0.0173.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step\n",
      "-----\n",
      "(27915.10 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-32-fc12.weights.049_0.9678_0.1378.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step\n",
      "-----\n",
      "(28076.21 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-64-12.weights.050_0.9953_0.0159.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 121ms/step\n",
      "-----\n",
      "(27549.78 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-64-fc12.weights.050_0.9899_0.0389.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(27870.50 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-16-12.weights.025_0.2841_2.0599.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(29647.80 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-16-fc12.weights.047_0.2988_2.0399.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step\n",
      "-----\n",
      "(27775.09 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-32-12.weights.047_0.2966_2.0507.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 121ms/step\n",
      "-----\n",
      "(27434.89 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-32-fc12.weights.042_0.2943_2.0200.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(27657.42 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-64-12.weights.031_0.2817_2.0544.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 26s 122ms/step\n",
      "-----\n",
      "(27679.34 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_32-imagenet224-64-fc12.weights.050_0.3070_2.0004.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step\n",
      "-----\n",
      "(27805.72 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-16-12.weights.048_0.9962_0.0103.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 123ms/step\n",
      "-----\n",
      "(27953.51 ms) == ( 27s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-16-fc12.weights.043_0.9813_0.0794.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 125ms/step\n",
      "-----\n",
      "(28419.31 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-32-12.weights.046_0.9988_0.0041.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 132ms/step\n",
      "-----\n",
      "(29895.25 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-32-fc12.weights.048_0.9936_0.0256.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step\n",
      "-----\n",
      "(29733.97 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-64-12.weights.038_0.9990_0.0039.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 125ms/step\n",
      "-----\n",
      "(28479.67 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-64-fc12.weights.049_0.9942_0.0268.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 125ms/step\n",
      "-----\n",
      "(28538.15 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-16-12.weights.026_0.9636_0.2933.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step\n",
      "-----\n",
      "(28479.25 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-16-fc12.weights.034_0.9420_0.2607.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 125ms/step\n",
      "-----\n",
      "(28189.05 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-32-12.weights.048_0.9711_0.2074.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step\n",
      "-----\n",
      "(28323.11 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-32-fc12.weights.039_0.9602_0.1616.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 127ms/step\n",
      "-----\n",
      "(28597.16 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-64-12.weights.026_0.9638_0.1562.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step\n",
      "-----\n",
      "(28354.50 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-64-fc12.weights.035_0.9661_0.1243.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 126ms/step\n",
      "-----\n",
      "(28441.15 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-16-12.weights.046_0.9987_0.0042.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 30s 141ms/step\n",
      "-----\n",
      "(31850.04 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-16-fc12.weights.050_0.8985_0.1913.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 30s 138ms/step\n",
      "-----\n",
      "(31312.30 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-32-12.weights.045_0.9987_0.0027.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 29s 136ms/step\n",
      "-----\n",
      "(30974.04 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-32-fc12.weights.040_0.9922_0.0273.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 128ms/step\n",
      "-----\n",
      "(29060.45 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-64-12.weights.031_0.9995_0.0023.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 128ms/step\n",
      "-----\n",
      "(31695.20 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-64-fc12.weights.050_0.9955_0.0152.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 128ms/step\n",
      "-----\n",
      "(29144.66 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-16-12.weights.037_0.9975_0.0125.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 128ms/step\n",
      "-----\n",
      "(28922.44 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-16-fc12.weights.049_0.9902_0.0360.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step\n",
      "-----\n",
      "(29253.89 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-32-12.weights.040_0.9975_0.0103.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 27s 127ms/step\n",
      "-----\n",
      "(28642.88 ms) == ( 28s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-32-fc12.weights.036_0.9954_0.0167.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 30s 138ms/step\n",
      "-----\n",
      "(31424.41 ms) == ( 31s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-64-12.weights.011_0.9978_0.0074.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 32s 146ms/step\n",
      "-----\n",
      "(33175.44 ms) == ( 33s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_96-imagenet96-64-fc12.weights.035_0.9963_0.0128.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 45s 207ms/step\n",
      "-----\n",
      "(46493.60 ms) == ( 46s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-16-12.weights.043_0.9998_0.0004.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 38s 176ms/step\n",
      "-----\n",
      "(39745.61 ms) == ( 39s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-16-fc12.weights.049_0.9808_0.0826.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 34s 156ms/step\n",
      "-----\n",
      "(35391.34 ms) == ( 35s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-32-12.weights.048_0.9995_0.0012.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 52s 240ms/step\n",
      "-----\n",
      "(54724.40 ms) == ( 54s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-32-fc12.weights.045_0.9859_0.0628.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 56s 261ms/step\n",
      "-----\n",
      "(60350.18 ms) == ( 60s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-64-12.weights.044_0.9995_0.0015.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 32s 148ms/step\n",
      "-----\n",
      "(33332.69 ms) == ( 33s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-64-fc12.weights.035_0.9990_0.0062.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 29s 133ms/step\n",
      "-----\n",
      "(30184.19 ms) == ( 30s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-16-12.weights.041_0.9995_0.0031.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 131ms/step\n",
      "-----\n",
      "(29479.11 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-16-fc12.weights.019_0.9955_0.0284.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step\n",
      "-----\n",
      "(29411.82 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-32-12.weights.045_0.9992_0.0037.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 129ms/step\n",
      "-----\n",
      "(29171.01 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-32-fc12.weights.029_0.9950_0.0180.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 131ms/step\n",
      "-----\n",
      "(29559.22 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-64-12.weights.049_0.9992_0.0044.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 130ms/step\n",
      "-----\n",
      "(29276.45 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_128-imagenet128-64-fc12.weights.019_0.9973_0.0096.hdf5'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 28s 131ms/step\n",
      "-----\n",
      "(29505.22 ms) == ( 29s)\n",
      "-----\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mobilenetv2_df = pd.DataFrame(columns='name')\n",
    "mobilenetv2_output = \"\"\n",
    "\n",
    "nama_file = 'mobilenetv2' + '.predict_turbo'\n",
    "df_savepath = f'{TEST_RESULTS_DIR}/{nama_file}/history.{nama_file}.csv'\n",
    "ic(nama_file)\n",
    "\n",
    "try:\n",
    "    os.mkdir(f'{TEST_RESULTS_DIR}/{nama_file}')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "i = 0\n",
    "for input_size in _input_size:\n",
    "    for weights in _weights:\n",
    "        for batch_size in _batch_size:\n",
    "            for dense in _dense:\n",
    "\n",
    "                kwargs = {\n",
    "                    'input_size' : input_size,\n",
    "                    'batch_size' : batch_size,\n",
    "                    'weights' : weights,\n",
    "                    'dense' : dense\n",
    "                }\n",
    "                nama_model = create_mobilenetv2_name(**kwargs)\n",
    "    \n",
    "                ### start evaluate\n",
    "                with io.StringIO() as buf, redirect_stdout(buf):\n",
    "                    _, y_pred = model_predict(nama_model=nama_model, input_size=input_size, debug=True)\n",
    "                    mobilenetv2_output += buf.getvalue()\n",
    "                    _output = buf.getvalue()\n",
    "                print(_output, end='')\n",
    "\n",
    "                ### create confusion matrix & classification report\n",
    "                test_data_gen = create_test_datagen(input_size=input_size)\n",
    "\n",
    "                cm = confusion_matrix(\n",
    "                    y_true=test_data_gen.classes,\n",
    "                    y_pred=y_pred,\n",
    "                    labels=[item for item in test_data_gen.class_indices.values()]\n",
    "                )\n",
    "\n",
    "                cr = classification_report(\n",
    "                    y_true=test_data_gen.classes,\n",
    "                    y_pred=y_pred,\n",
    "                    labels=[item for item in test_data_gen.class_indices.values()],\n",
    "                    target_names=[item for item in test_data_gen.class_indices.keys()],\n",
    "                    digits=2,\n",
    "                    output_dict=False,\n",
    "                    zero_division=0\n",
    "                )\n",
    "\n",
    "                print_to_file(rf\"{nama_file}/{nama_model} cm-cr\" + \"_turbo\", f\"{cm}\\n\\n\\n{cr}\")\n",
    "\n",
    "                mobilenetv2_df.loc[i] = nama_model\n",
    "                i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "b3cbc498-624d-4c67-a3ba-d0850d8c68cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-30T05:57:45.714468Z",
     "iopub.status.busy": "2021-06-30T05:57:45.714468Z",
     "iopub.status.idle": "2021-06-30T05:57:45.734465Z",
     "shell.execute_reply": "2021-06-30T05:57:45.733467Z",
     "shell.execute_reply.started": "2021-06-30T05:57:45.714468Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "mobilenetv2_df = pd.DataFrame(columns=['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "07a08eab-8fc5-4a36-bea0-1ca4347c77d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-30T05:57:46.299513Z",
     "iopub.status.busy": "2021-06-30T05:57:46.298505Z",
     "iopub.status.idle": "2021-06-30T05:57:46.413516Z",
     "shell.execute_reply": "2021-06-30T05:57:46.412518Z",
     "shell.execute_reply.started": "2021-06-30T05:57:46.299513Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for input_size in _input_size:\n",
    "    for weights in _weights:\n",
    "        for batch_size in _batch_size:\n",
    "            for dense in _dense:\n",
    "                \n",
    "                kwargs = {\n",
    "                    'input_size' : input_size,\n",
    "                    'batch_size' : batch_size,\n",
    "                    'weights' : weights,\n",
    "                    'dense' : dense\n",
    "                }\n",
    "                nama_model = create_mobilenetv2_name(**kwargs)\n",
    "                \n",
    "                mobilenetv2_df.loc[i] = nama_model\n",
    "                i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "c2a12c0d-e102-4c7a-8178-2ba710d879a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-30T05:59:19.103324Z",
     "iopub.status.busy": "2021-06-30T05:59:19.103324Z",
     "iopub.status.idle": "2021-06-30T05:59:19.152327Z",
     "shell.execute_reply": "2021-06-30T05:59:19.151336Z",
     "shell.execute_reply.started": "2021-06-30T05:59:19.103324Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>waktu</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>mobilenetv2_1.00_32-16-12</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>mobilenetv2_1.00_32-16-fc12</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>mobilenetv2_1.00_32-32-12</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>mobilenetv2_1.00_32-32-fc12</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>mobilenetv2_1.00_32-64-12</td>\n",
       "      <td>121ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>mobilenetv2_1.00_32-64-fc12</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-16-12</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-16-fc12</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-32-12</td>\n",
       "      <td>121ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-32-fc12</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-64-12</td>\n",
       "      <td>122ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>mobilenetv2_1.00_32-imagenet224-64-fc12</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>mobilenetv2_1.00_64-16-12</td>\n",
       "      <td>123ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>mobilenetv2_1.00_64-16-fc12</td>\n",
       "      <td>125ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>mobilenetv2_1.00_64-32-12</td>\n",
       "      <td>132ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>mobilenetv2_1.00_64-32-fc12</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>mobilenetv2_1.00_64-64-12</td>\n",
       "      <td>125ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>mobilenetv2_1.00_64-64-fc12</td>\n",
       "      <td>125ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-16-12</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-16-fc12</td>\n",
       "      <td>125ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-32-12</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-32-fc12</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-64-12</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>mobilenetv2_1.00_64-imagenet224-64-fc12</td>\n",
       "      <td>126ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>mobilenetv2_1.00_96-16-12</td>\n",
       "      <td>141ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>mobilenetv2_1.00_96-16-fc12</td>\n",
       "      <td>138ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>mobilenetv2_1.00_96-32-12</td>\n",
       "      <td>136ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>mobilenetv2_1.00_96-32-fc12</td>\n",
       "      <td>128ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>mobilenetv2_1.00_96-64-12</td>\n",
       "      <td>128ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>mobilenetv2_1.00_96-64-fc12</td>\n",
       "      <td>128ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-16-12</td>\n",
       "      <td>128ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-16-fc12</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-32-12</td>\n",
       "      <td>127ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-32-fc12</td>\n",
       "      <td>138ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-64-12</td>\n",
       "      <td>146ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>mobilenetv2_1.00_96-imagenet96-64-fc12</td>\n",
       "      <td>207ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>mobilenetv2_1.00_128-16-12</td>\n",
       "      <td>176ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>mobilenetv2_1.00_128-16-fc12</td>\n",
       "      <td>156ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>mobilenetv2_1.00_128-32-12</td>\n",
       "      <td>240ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>mobilenetv2_1.00_128-32-fc12</td>\n",
       "      <td>261ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>mobilenetv2_1.00_128-64-12</td>\n",
       "      <td>148ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>mobilenetv2_1.00_128-64-fc12</td>\n",
       "      <td>133ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-16-12</td>\n",
       "      <td>131ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-16-fc12</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-32-12</td>\n",
       "      <td>129ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-32-fc12</td>\n",
       "      <td>131ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-64-12</td>\n",
       "      <td>130ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>mobilenetv2_1.00_128-imagenet128-64-fc12</td>\n",
       "      <td>131ms</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         name  waktu\n",
       "96                  mobilenetv2_1.00_32-16-12  122ms\n",
       "97                mobilenetv2_1.00_32-16-fc12  123ms\n",
       "98                  mobilenetv2_1.00_32-32-12  123ms\n",
       "99                mobilenetv2_1.00_32-32-fc12  123ms\n",
       "100                 mobilenetv2_1.00_32-64-12  121ms\n",
       "101               mobilenetv2_1.00_32-64-fc12  122ms\n",
       "102     mobilenetv2_1.00_32-imagenet224-16-12  122ms\n",
       "103   mobilenetv2_1.00_32-imagenet224-16-fc12  123ms\n",
       "104     mobilenetv2_1.00_32-imagenet224-32-12  121ms\n",
       "105   mobilenetv2_1.00_32-imagenet224-32-fc12  122ms\n",
       "106     mobilenetv2_1.00_32-imagenet224-64-12  122ms\n",
       "107   mobilenetv2_1.00_32-imagenet224-64-fc12  123ms\n",
       "108                 mobilenetv2_1.00_64-16-12  123ms\n",
       "109               mobilenetv2_1.00_64-16-fc12  125ms\n",
       "110                 mobilenetv2_1.00_64-32-12  132ms\n",
       "111               mobilenetv2_1.00_64-32-fc12  130ms\n",
       "112                 mobilenetv2_1.00_64-64-12  125ms\n",
       "113               mobilenetv2_1.00_64-64-fc12  125ms\n",
       "114     mobilenetv2_1.00_64-imagenet224-16-12  126ms\n",
       "115   mobilenetv2_1.00_64-imagenet224-16-fc12  125ms\n",
       "116     mobilenetv2_1.00_64-imagenet224-32-12  126ms\n",
       "117   mobilenetv2_1.00_64-imagenet224-32-fc12  127ms\n",
       "118     mobilenetv2_1.00_64-imagenet224-64-12  126ms\n",
       "119   mobilenetv2_1.00_64-imagenet224-64-fc12  126ms\n",
       "120                 mobilenetv2_1.00_96-16-12  141ms\n",
       "121               mobilenetv2_1.00_96-16-fc12  138ms\n",
       "122                 mobilenetv2_1.00_96-32-12  136ms\n",
       "123               mobilenetv2_1.00_96-32-fc12  128ms\n",
       "124                 mobilenetv2_1.00_96-64-12  128ms\n",
       "125               mobilenetv2_1.00_96-64-fc12  128ms\n",
       "126      mobilenetv2_1.00_96-imagenet96-16-12  128ms\n",
       "127    mobilenetv2_1.00_96-imagenet96-16-fc12  130ms\n",
       "128      mobilenetv2_1.00_96-imagenet96-32-12  127ms\n",
       "129    mobilenetv2_1.00_96-imagenet96-32-fc12  138ms\n",
       "130      mobilenetv2_1.00_96-imagenet96-64-12  146ms\n",
       "131    mobilenetv2_1.00_96-imagenet96-64-fc12  207ms\n",
       "132                mobilenetv2_1.00_128-16-12  176ms\n",
       "133              mobilenetv2_1.00_128-16-fc12  156ms\n",
       "134                mobilenetv2_1.00_128-32-12  240ms\n",
       "135              mobilenetv2_1.00_128-32-fc12  261ms\n",
       "136                mobilenetv2_1.00_128-64-12  148ms\n",
       "137              mobilenetv2_1.00_128-64-fc12  133ms\n",
       "138    mobilenetv2_1.00_128-imagenet128-16-12  131ms\n",
       "139  mobilenetv2_1.00_128-imagenet128-16-fc12  130ms\n",
       "140    mobilenetv2_1.00_128-imagenet128-32-12  129ms\n",
       "141  mobilenetv2_1.00_128-imagenet128-32-fc12  131ms\n",
       "142    mobilenetv2_1.00_128-imagenet128-64-12  130ms\n",
       "143  mobilenetv2_1.00_128-imagenet128-64-fc12  131ms"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_df = mobilenetv2_df\n",
    "\n",
    "waktu = re.findall(\"([0-9]+ms)\", mobilenetv2_output)\n",
    "test_df['waktu'] = waktu\n",
    "\n",
    "display(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "c145dce1-7fd7-4d54-b932-55a4f0cc1e84",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-30T05:59:25.910869Z",
     "iopub.status.busy": "2021-06-30T05:59:25.910869Z",
     "iopub.status.idle": "2021-06-30T05:59:25.932869Z",
     "shell.execute_reply": "2021-06-30T05:59:25.931876Z",
     "shell.execute_reply.started": "2021-06-30T05:59:25.910869Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# save to csv: \n",
    "with open(df_savepath, mode='w') as f:\n",
    "    test_df.to_csv(\n",
    "        f,\n",
    "        header=True,\n",
    "        index=False,\n",
    "        line_terminator='\\n'\n",
    "        )\n",
    "    \n",
    "# print_to_file(f\"verbose.{nama_file}\", mobilenetv2_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf9725cc-a9b6-4ff3-a25b-46218d7fd0a9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74fbe9c6-bcc2-4873-8083-478c3073431b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c820683-615e-4214-8674-be66497eb201",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "202767b5-faad-4ad4-8562-4aef92023e9e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3fd2df16-f6a8-4667-9492-6836e038288a",
   "metadata": {
    "tags": []
   },
   "source": [
    "# ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "82467d8a-dc15-4a57-8806-e029738b36ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-28T13:47:11.996364Z",
     "iopub.status.busy": "2021-06-28T13:47:11.996364Z",
     "iopub.status.idle": "2021-06-28T13:47:12.025366Z",
     "shell.execute_reply": "2021-06-28T13:47:12.024366Z",
     "shell.execute_reply.started": "2021-06-28T13:47:11.996364Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "64c54721-5468-4571-ba95-46efc027500c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-28T13:59:31.884449Z",
     "iopub.status.busy": "2021-06-28T13:59:31.883450Z",
     "iopub.status.idle": "2021-06-28T13:59:33.565578Z",
     "shell.execute_reply": "2021-06-28T13:59:33.564582Z",
     "shell.execute_reply.started": "2021-06-28T13:59:31.884449Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAD8CAYAAAA45tAbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABQMElEQVR4nO2dd5xVxfmHn+8WmvReFRAEFRUVsSuIChobsSsqNtTEGPOLGmOMikZFjRqNLdiwoqJoLAgSFLAgAtIVlLIisPSyIG137/v748zC3cuWi9y+8/A5n3vOzJw5c2Yv573vzJzvKzPD4/F4PJ5kk5XsBng8Ho/HA94geTwejydF8AbJ4/F4PCmBN0gej8fjSQm8QfJ4PB5PSuANksfj8XhSAm+QPB6Pp4oj6QVJKyTNKidfkh6XNE/SDEmHhOX1kTTX5d0alt5Q0mhJP7rPBpW1wxskj8fj8QwB+lSQfwrQ0W0DgKcBJGUDT7r8/YALJe3nzrkVGGNmHYEx7rhCvEHyeDyeKo6ZjQfWVFDkTOBlC/gaqC+pBdAdmGdmC8xsG/CGK1tyzktu/yXgrMrakfMr2++JoHDVAi954fF4oiK3cXvtbh3RPnOqNdn7GgKvpoTBZjZ4Fy/XCvg57HixSysr/XC338zM8gHMLF9S08ou4g2Sx+PxZDDO+OyqAYqkLANqFaT/KrxB8ng8nnQkVJzIqy0G2oQdtwaWAtXKSQdYLqmF845aACsqu4ifQ6qC3H7fIxz3mws4q9+1yW5KxuL7OP5U+T4uLopuiw3vA5e61XZHAOvdcNwkoKOkdpKqARe4siXnXOb2LwP+W9lFUs4gSeorySR13sXzekj60O3fJemmcsp9FVm+qnHWqSfxzCP/SHYzMhrfx/GnqvexWSiqLRokDQUmAJ0kLZZ0paRrJZVY+xHAAmAe8Czwu6ANVgRcD4wCvgfeMrPZ7pxBwEmSfgROcscVkopDdhcCXxBY2rtiXbmZHRXrOtONbl0PYEn+8mQ3I6PxfRx/qnwfh6IzNtFgZhdWkm/A78vJG0FgsCLTVwO9dqUdKeUhSaoNHA1cSWCQSjyZsZLeljRH0muS5PL6uLQvgN9GVLefO2+BpBvCrrGxjOseJmmqpPaSTpc00R3/T1KzuN2wx+Px/FosFN2WRqSah3QWMNLMfpC0Juxt4IOB/Qkmy74EjpY0mcB1PIHAjXwzoq7OQE+gDjBX0tNmVhh5QUlHAf8GzjSzRZLWAkeYmUm6CrgF+HOsb9Tj8Xh2i8QuakgIKeUhEQzXveH233DHAN+Y2WILBkSnAW0JDM5CM/vRuZOvRtT1kZltNbNVBKs7yvJ09iVYDnm6mS1yaa2BUZJmAjcTGMIykTRA0mRJk597eegu3qrH4/HsBt5Dih+SGhF4O10kGZBNsJ59BLA1rGgxO9pd0Xr38s4JJx+oQeCBlSxV/DfwiJm9L6kHFcxjha/v9y/GejyeRGKxW0GXMqSSh3QOgTTFXmbW1szaAAuBY8opPwdoJ2lvd1zhpFw5rAN+A9znjA9APWCJ279s51PSn5vvHMTF1/yJvEWL6XVWP975YFSym5Rx+D6OP1W+j0Oh6LY0ImU8JAKDErks8B3gOmB+ZGEz2yJpAPCRpFUEK/O67OpFzWy5pNOBjyVdQeARDZO0BPgaaLerdaY6Dw2sVOPQs5v4Po4/Vb6P02w4LhoUTL94dhc/ZOfxeKIlFlp2W+eMi+qZU73z8bt9rUSRSh6Sx+PxeKIlAz0kb5A8Ho8nHcnARQ3eIHk8Hk86kmYLFqLBGySPx+NJQ8wy78VYb5A8Ho8nHfFzSJ5M4Pb7HmH8l9/QsEF93nv1mWQ3JyPxfRx/qnwfZ+CQXSq9GLsTuxqKQtIQSee4/eck7RffFqYnVV22PxH4Po4/Vb6PM1A6KKUNEqVDUZRCUnZFJ5rZVWb2Xbwals5063oA9erWSXYzMhrfx/GnyvdxcWF0WxqRsgapglAUn0l6HZjpohc+Iek7SR8BTcPOHyupmws09WhY+tWSHnH770maImm2U30oKbNR0r2Spkv62oeg8Hg8KUcGSgelrEEiLBQFEB6KojvwNzPbD+gLdAIOAK4Gygq+9wZwhqRcd3w58KLbv8LMDgW6ATc4gVeAPYCvzewgYLyreye82rfH40kaMRyyc7Hl5kqaJ2knTSZJDSS9K2mGpG8kdXHpnSRNC9sKJN3o8u6StCQs79TK2pHKixouBP7l9ktCUXxEEIpioUs/DhhqwfrHpZI+jazEzH5x6adJ+h7INbOZLvsGSX3dfhugI7Aa2AaUhDefQhB+dye82rfH40kaMfJ+3PTHkwTPucXAJEnvR0x53AZMM7O+bk7/SaCXmc0FuobVswR4N+y8R83sn9G2JSUNUiWhKH6JKB6NIXiOoEPn4Lwjp+59InCkmW2SNJYgFAVAoe0Q+SsvdIXH4/Ekj9gNx3UH5pnZAgBJbwBnAuEGaT/gfgAzmyOpraRmZhYeQ74XMN/Mfvq1DUnVIbtoQ1GMBy6QlC2pBUGE2J0ws4kEHtBFQMnYWj1grTNGnYEj4nEjqUiVl+1PAL6P409V72MrLoxqC59acNuAiKpaAT+HHS92aeFMB34LIKk7sBdBMNNwLmDH87WE690w3wuSGlR2T6n6yz/aUBTvEnhSM4EfgHEV1PkW0NXM1rrjkcC1kmYAcwlCTVQJqrxsfwLwfRx/qnwfRzk/FD61UA5lqYFHjjwNAh6TNI3geTsV2C6mJ6kacAbw17BzngbucXXdAzwMXFFRW1PSIJlZjzLSHgcej0gz4Poo6zgGeDQsfytwSjnn1g7bfxt4O7qWezweT4KI3ZDdYoIRpBJasyOCNgBmVkCwIAxJIhixWhhW5BTg2/AhvPB9Sc+yY16+XFJ1yC5mSKov6Qdgs5mNSXZ7PB6PJybEbpXdJKCjpHbO07kAeD+8gHuOVnOHVwHjnZEq4UIihuvcNEoJfYFZlTUkJT2kWGJm64B9kt0Oj8fjiSkx8pDMrEjS9cAoggVkL5jZbEnXuvxngH2BlyUVEyx2uLLkfEm1CFboXRNR9YOSuhIM2eWVkb8TGW+QPB6PJyOJoSyQmY0gWMUcnvZM2P4Egtdiyjp3E9CojPRLdrUd3iB5PB5POlLkA/R5MoAqr5KcAHwfx58q38dpJpwaDRm7qKEypXA3Sfe7sOOWkt52+/0lPZGotiaaKq+SnAB8H8efKt/HXssurahMKbw+sN0gmdlSMzsnYa1LIlVeJTkB+D6OP1W+j334ifQgGqVwghe99naifw85KYydliVK+o2kCZIaJ/IePB6Pp0Iy0EPK1Dmks3BK4ZIilcK7mNlCSW3dflcAd1wKJ7z6f8CpYQoPHo/Hk3zSzPuJhoz0kAiG695w+yVK4VBaKbwyegJ/AX5TnjHy4Sc8Hk/SKCqKbksjMs5D2kWl8IpYALQneKl2clkFfPgJj8eTNCzzHjmZ6CFFqxS+AahoRvQnAnXblyXtH5+mJoeqrpKcCHwfx58q38cZOIckyzAr6+IaDTKzkWFpN+CUws3stLD014EDgY8JAk59aGZdJPUHupnZ9ZIOBl4DTjezcKXxUngPyePxREtu4/ZlKWzvEptf+3tUz5yaF9+z29dKFBlnkJKFN0gejydaYmKQXv1bdAap371pY5Aybg7J4/F4qgTFxcluQczxBsmTNmx7dmCym5DxZB12ZLKbUCXIPfHa3a8kzeaHosEbJI/H40lHvEHyeDweT0rgX4z1eDweTypgIYtqiwZJfSTNlTRP0q1l5DeQ9K6kGZK+kdQlLC9P0kwnwzY5LL2hpNGSfnSfDSprh/eQqiBVXrY/BmS3P4BqJ14MWVkUTRtH4dcflS5QoxbVT72KrAZNsaJCtn70HLZqyY58iRr9B2Ib17J12KOJbXwa8eXsPB58eyyhUIi+R3fhipO7l8ov2LSFO1/9hMUr11MtN5uB/U6mQ8vG5C1fwy3P74g3t2T1eq77zZH0O+GQyEukLzEasnNi008SRH1dDEyS9L6ZfRdW7DZgmpn1dREUngR6heX3NLNVEVXfCowxs0HOyN1KoH5TLmnpIUkqdtZ4uqRvJR0VxTkb3WeZIqpViSov27+7SFQ7+VK2vPUwmwf/lez9jkCNWpYqknvk6YRWLGLz87ez9YPBVD/p4lL5Od1OxlYvTWSr047iUIj73/qUJ39/FsP/fhkjJ89lfv7qUmWeG/kNnVo1YdjfLuEfl/bhwWFjAWjbrCFv3daPt27rx9BbL6JGbg4nHNQhCXcRR4qLo9sqpzswz8wWmNk2Arm1MyPK7AeMATCzOUBbSc0qqfdM4CW3/xKBxmiFpKVBAjabWVczOwj4K3B/shuUTlR52f7dJKtle0Jrl2PrVkKomOLvJ5KzT+lf3lmNW1KcNxsAW5OP6jWBWnUBUJ0G5HQ4iMLp4xLe9nRiVt4y2jSpT+vG9cnNyab3oZ0YO6P0u+kLlq3h8E57AtCueUOWrilgdUFphbCJc3+mdZN6tGxUN2FtTwhRKjWEa266bUBETa2An8OOF7u0cKYTKNcgqTuwF9Da5RnwiaQpEXU3M7N8APfZtLJbSleDFE5dYC0EYSckjXFe00xJkVa+FJI+l9Q17PhLSQdK6i7pK0lT3Wen+N6CJ51Q7QZYwZrtx7ZhDapTeng8tOJncjp1AyCrRXtUrxFZdRsCUO3Ei9n22VsZqUUWS1as20jzBjt+ODWrX5sV6zaWKrNPq8aMmT4PgJl5y8hfU8DyiDKjJs/llEPLjNOZ3kRpkMxssJl1C9sGR9RU1ouzkV/OQUADSdOAPwBTgRLl1qPN7BDgFOD3ko77tbeUrgapphuymwM8B9zj0rcAfV3n9AQellTRW8rPAf0BJO0DVDezGcAc4DgzOxi4A7ivrJO92ncVpayvVIRxKZzwIdTYgxpX3E1utxMJLf8JQsVkdzgI21RAaFleYtqaxpRlriP/O19x8mEUbNrCefe9yhtjp9KpdVOys3Y81gqLihk3cz4nHdIxzq1NAmbRbZWzGGgTdtwaKDWebGYFZna5C9dzKdCEQCMUM1vqPlcA7xIMAQIsl9QCwH2uqKwh6bqoYXNYHKMjCQRQuxBY+vuchQ4RuJ3NgGXl1DMM+Lukm4ErgCEuvR7wkqSOBP8vcss62at9V01swxrkvB0A1WmIbVxXutC2LWz76LnthzWv+yehdSvJ3fcIsjscTM32B0JOLqpek+qnX8PWD/6ToNanD83q12bZ2g3bj5ev20iTenuUKlO7ZnXuvqQ3AGbGqXe8QKuwobkvZufRuU1TGtUtfV5GELv3kCYBHSW1A5YQBDW9KLyApPrAJjfHdBUw3swKJO0BZJnZBrd/MnC3O+194DIC7+oy4L+VNSRdDdJ2zKwkmmsT4FT3eaiZFUrKA2pUcO4mSaMJJt/OA7q5rHuAz9yKkrbA2PjdgSfdCC1dSFaDZqheY2zDWrL3PZyt70esVqxeCwq3QqiYnIOOp/jnH2DbFgrHDaNw3DAAsvbsTO7hp3hjVA7779WcRSvWsmTVeprWr82oKXO5r/8ppcoUbNpCzWq55OZkM/yrWRzaoRW1a1bfnj9yyhz6dMvA4TqAKJd0V4aZFUm6HhhFEK7nBTObLelal/8MsC/BD/9i4DuCaNwQ/OB/13muOcDrYcLWg4C3JF0JLALOrawtaW+Q3BLEbGA1gWezwhmjngQTb5XxHPAB8LmZlUwM1CP4pQBuSC+TuPnOQUyaOoN16wrodVY/fnflJZx9eu9kNyt9sBDbRr9CjQtuBmVRNGM8tmoJOQf3BKBo6mdkNW5B9dMGgIUIrVrK1hHPJ7nR6UdOdha3nncC1z05nFDIOPPI/enQsjHDPp8OwLnHHsTCZWu4/eVRZGeJ9s0bcVe/k7afv3lbIV/PWcTtF56YrFuILzHUsjOzEQQx48LTngnbnwDsNO5pZguAg8qpczWll4ZXSlqqfTsrPbPkELjNzD5yntIHBENs04CjgVPMLE/SRjOr7TyeD80s/MWuOcCNJZbdDQO+BKwEPgUuMbO2FbXJD9nFH69lF3+8ll1iqHnitbutwP3L/ZdF9czZ468vebXveGJm2eWkrwLK/B9lZrXdZx4QboxaEizu+CSs7ASCSLEl/H23G+3xeDyxJEZDdqlEuq6yiwmSLgUmAn8zy0BhKI/Hk7lYKLotjUhLDylWmNnLwMvJbocnOqpdfWeym+DxpA4Z6CFVaYPk8Xg8aUuRD9Dn8Xg8nlQgzYbjosEbpCqIV/uOP76P40+V7+MMHLLL2EUN5SmCS2op6e1kty+ZeLXv+OP7OP5U9T62UCiqLZ3IWINEOYrgZrbUzM5JbtOSi1f7jj++j+NPle/jkEW3pRGZbJDCCVcE3x4PSVJ/ScMljXRRDR8sOUHS0044dbYk/0amx+NJLTLQIGXyHFJNJ5VeA2gBnFBOua7AwcBWYK6kf5vZzwTvJq1x0RTHSDrQKYF7PB5P8omhdFCqkMkeUsmQXWegD4EwYFkSGmPMbL2ZbSEQDSzRvztP0rcEcT/2J4iYWAoffsLj8SQLC1lUWzqRyR7SdiIUwSPZGrZfDOQ4GfabgMPMbK2kIZShGu7DT3g8nqSRZsYmGjLZQ9pOhCJ4NNQFfgHWu7jxp1RSPq24+c5BXHzNn8hbtJheZ/XjnQ9GJbtJGYfv4/hT5fs4yoix6UQme0glc0gQKIJfZmbFFQeQDTCz6ZKmArOBBcCXcWtlEnho4K3JbkLG4/s4/lT5Ps5ADyljDVIFiuB5OLVvMxvCjiixmNlpYfv949k+j8fj2S0y0CBViSE7j8fjyTSsOBTVFg2S+kiaK2mepJ1cT0kNJL0raYakbyR1celtJH0m6Xv3iswfw865S9ISJ1AwTdKplbUjYz2kRFP48v3JbkLGk3vpX5PdhIynaPr/kt2EKkFurwG7X0mMPCT3asuTwEnAYmCSpPfN7LuwYrcB08ysr5uTf5IgGmwR8Gcz+1ZSHWCKpNFh5z5qZv+Mti3eQ/J4PJ40JIbLvrsD88xsgZltA94Azowosx8wBsDM5gBtJTUzs3wz+9albwC+B1r92nvyBsnj8XjSkSiVGsLfl3RbpHvWCvg57HgxOxuV6cBvASR1J3hfs3V4AUltCUQGJoYlX++G+V6Q1KCyW/IGyePxeNKRUHSbmQ02s25h2+CImspaehzpWg0CGriVy38gEAwo2l6BVBt4B7jRzApc8tPA3gRqOPnAw5Xdkp9DykCy2u5PtR4XQFYWRTM/p2jSyNIFqteiWu/+ZNVrghUXsm3UEGz1UgBqXHk/FG6BkGGhYra+fm8S7iD9qfKhEWLEl7MX8uCwzwiZ0feoLlzR+/BS+QWbtnDnK6NYvHId1XJzGHhJbzq0bEze8jXc8vyH28stWbWe6047in4nHJroW4gbVhSzd4wWA23CjlsDS0tdKzAylwM4xZuFbkNSLoExes3Mhoeds7xkX9KzwIdUQsoaJEnFwEwC610MXG9mXyW3VWmARLUTLmLrO49iG9ZS4+K/UTx/OrYmf3uR3MNPJbTiZ7a9/xRq0JxqvS5i69uPbM/f8tbDsGVjMlqfMZx16klcdPYZ3HZP1PO5ngiKQyHuf3MMz9xwDs3q1+HiB17j+AM7sHeLRtvLPDdyIp1aN+HRa85k4bLV3P/mpwz+47m0bdaQt267dHs9J9/2H044qGOybiU+xO6d10lAR6dQswS4ALgovICk+sAmN8d0FTDezAqccXoe+N7MHok4p4WZlTx4+gKzKmtIKg/ZlRk+wlMxWc3bYetWYutXQaiYojmTyN67a+kyDVsQWvQ9ALZ2GarbCGpVYRn/OFDlQyPEgFl5y2jTpD6tG9cnNyeb3od2Yuz0eaXKLMhfzeGd9gSgXfNGLF29ntUFv5QqM3HOIlo3rk/LRnUT1vZEEKtFDWZWBFwPjCJYlPCWmc2WdK2ka12xfYHZkuYQKNeULO8+GrgEOKGM5d0PSpopaQbQE/hTZW1JWQ8pgvDwET2Am0peYpX0BDCZYFLuejPr69JPAq4zs99K2mhmtV36OcBpZtbfadQVAN2A5sAtZva2pCzgCeB4Arc0C3jBzFI+sJ9q18c2rNl+bBvXktWiXakyoZWLye54CKGl88hq3hbVbYRqN8A2bQCgxtk3AlA4YxzFMz9PWNs9nnBWrNtI8wY7jHqzBnWYmZdfqsw+rZswZto8Du7Qmpl5+eSvKWD5uo00qrvH9jKjpszhlG6dE9buhBFDVSAzGwGMiEh7Jmx/ArCTi2lmX1D2HBRmdsmutiOVPaSaztrOAZ4D7qmk/KfAvpJKBFQvB16M4jotgGOA0wgm7iBYTdIWOIDAPT2yrBPDV6+8MGFOFJdKBGV8NyJ+JBVO+hhVr0WNfneQ0/UEQit+3q55tfWNQWx57R9sGf4YuV17ktUqw4Y5PGmD7TSvvvO3+4qTu1OwaQvn3fcyb4ydSqfWTcnO2lGqsKiYcTPmc9Ih+8S5tYnHq30nls1m1hVA0pEE4SO6lFfYzEzSK0A/SS8SGJFLo7jOe2YWAr5zQqoQGKhhLn2ZpM/KueZ2te9Nj1ydEn9527gW1Wm4/Vi1G2Ab15UutG0L2z4Zsv2wxpX3YwWrgvN/WR8kbt5A8bypZDVvR2jJj3FutcezM83q12HZ2g3bj5ev3UCTerVLlaldszp3X9oHADPj1L8/R6tG9bbnfzF7IZ3bNCvlMWUM6aWbGhWp7CFtx7mLJeEjiijd7vCwEC8C/YALCQxKybJEK6c8lA4/oYjPtCO0LA/Vb4rqNoasbHI6H0bxgumlC1WvCVmB1F/2AccGBmfbFsipBrnVgzI51cjaaz9Cq5ck+A48noD992rOohXrWLJqPYVFxYyaMpfjD9y7VJmCTVsoLAoC1Q3/ciaHdmhN7ZrVt+ePnDyHPodl4HAdYEXRbelEKntI24kIH/ETsJ+k6gTGpRfwBYCZLZW0FLidQAajhOWS9gXmEqz22EDFfAFcJuklAiPYA3g9ZjcUTyzEts9ep/rZN4JE0awvsdVLyTnweACKZowjq2ELqvW5AixEaHU+2z55CQDtUZfqZ/wuqEfZFM2ZSChvdpJuJL25+c5BTJo6g3XrCuh1Vj9+d+UlnH1672Q3K63Iyc7i1vNP4Lon3iEUCnHmkV3o0LIxw8YHP7DOPe4gFi5bw+0vfUx2lmjfvBF3XbKjjzdvK+TrOT9x+0UnlXeJtMYy0EOSWUqMNO1E2LJvCDyW28zsI5f3IIG0xY/ANuB9p9yNpAsIXs46Iqyuc4AHCBY+zAJqhy1q+LBksULJ4ge3qOEp4DjgB6A68IiZjS6vvakyZJfJeC27+OO17BJDzV4DdnsUZlXv46N65jQeNS5tRnxS1kMqL3yEy7sFuKWc7GOAZyPKvw3stEIuMsREyUo8MwtJusnMNkpqBHzDDuPo8Xg8SScTPaSUNUi/BklTCCK9/jkG1X3oXgarBtxjZstiUKfH4/HEBG+QUhwzi5kuiJn12JXy//nX5lhd2lMO10ezZtKzW2x94Y1kN6FKUDMG4SesOG1G4qImowySx+PxVBW8h+TxeDyelMBC3kPypAF7HX8gPe66hKzsLGa9MZZJT31QKr9anZqc8th11GnZiKycbCb/ZwTfDRsPQPW6tTjpwatotE9rzIzRNz9L/rfzyrqMpwK82ndsyDnwMGpecj1kZbFt7Ai2fjC0VL5q1abmgFvIbtYCKyxk0+AHCS3OQw2bUOu6W8mq1xDM2Prph2wbNbycq6Qn3kNKIhF6dKcCjwG9zGxRcluWWihLnPCPyxh+8SA25K/hog/uZv7oKaz5cYea/EGXnsTqH5fw3yseoWbDOvQf+xBz3vuSUGExPe66hLyxM/jw2sfJys0mN+wlQ0/0eLXvGKAsavb/I7/cfzOhNSupc8/TFH77FaElP20vUv3MiyleNI9N/7qDrBZtXPmbIFTMlteeoTjvR6hRkzr/eIaiWVNKnZvumGWeh5QWSg3hSOoF/BvoE2mMJKWNgY0Xzbvuzbq85axftJJQYTFzP/iavU+OXOthVNujJgC5e9Rgy7pfCBWFqFa7Jq26d2LWG2MBCBUWs7VgU2JvIEPwat+7T/benQktX0JoZT4UF7Ht60/JPfSo0mVa7UXRrG8BCOX/TFaT5qhuA2zdmsAYAWzZTGjpIrIaNE70LcQVC0W3pRNpZZAkHUvwjtFvzGy+Sxsi6RGnN/eApL0ljZQ0RdLnTuUBSedKmiVpuqTxLq2/pOGu/I/uhduSaz3thFNnSxqYhNv9VdRu3oANS3eofW/MX0PtZqUjB08bMpqGHVoyYPITXPLJ/Yy96xUwo96eTdi8ZgMnPzyAi0f8gxMfuIoc7yF5kkRWw8aEVq/Yfhxas4qsBk1KlSleNJ/cw44FILt9Z7IaNyOrYWnDk9W4Gdl7daBo/vfxb3QCCRUrqi2dSCeDVB34L3CWmUVKa+8DnGhmfyYQO/2DWwJ+E4HiAsAdQG8XX+mMsHO7AucTKHufL6kkcuLfzKwbcCBwvKQD43BPsUc7fwEjxTjaHn8AK7/7icHdrufVPn+j592XUq12TbJysmnapS0zXhnDa6feTtHmrRz2u9MT1HCPJ5KylOtLf5m3fDAU7VGHOvcNpnrvvhTn/YiFincUqF6DWjcOZPMrT8HmzPL2LaSotnQinQxSIfAVcGUZecPMrNjFdT8KGOZiv/+HILwEwJfAEElXE+jilTDGzNab2RbgO2Avl36epG8JYsfvD+wXedHw8BMTNqaGIvbG/DXUablD7bt2i4b8smJtqTL7nXs880ZOBmD9T8tZ//NKGuzdgg35a9iQv4Zl0+YD8OOIb2japW3C2u7xhBNas5KsRk23H2c1bExo3arShTZvYvPgB9lw2wA2PX0/WXXrE1rp3mHPzmaPGwdS+OX/KJyceXG9YmmQJPWRNFfSPEm3lpHfQNK7kmZI+iY88kJ550pqKGm0G30aLalBZL2RpJNBCgHnAYdJui0iryREZBawzkWaLdn2BTCzawlEV9sA05wkEJRW+y4Gclwo35sIFk0cCHzEzirhmNlgM+tmZt2OrJ0acYOWTV9Ag3bNqdumCVm52XQ6/QgWjP62VJkNS1fR5uj9AajVuC4N927B+kUr2LRyPRvz19CgfWDD2xy9P2t+9GrfnuRQvGAOWc1bkdWkOWTnUO2IEyicMqFUGdXaA7KDqeNqPX9D0ZwZ2z2hWlffTGjJIrZ+nPJxNX8VZtFtlSEpG3iSIBLsfsCFkiJ/gN8GTHPPw0sJFpVVdu6tBD/4OwJj3HGFpNUiADPbJOk04HNJy83s+Yj8AkkLJZ1rZsNcvPcDzWy6pL3NbCIwUdLpBIapPOoSGLn1LkbSKcDY+NxVbLHiEJ/+/SV++8otKDuL2W+OY/UPSziw3wkAzHj1UyY+/h69H76GSz65HwSf3/8mW9ZuBOCzO17ilMevIys3h/WLVvDJTYOTeTtpi1f7jgGhEJuH/Js9/vIAZGWzbdzHhJbkUa1XMIy8bcwHZLXci1rX3QqhEMVLfmLz4IcAyN6nC9WOPZniRfOpc1/wHd785vMUTZ+YtNuJNTEcjusOzDOzBQCS3iAQr/4urMx+wP0AZjZHUlv3bGxfwblnEkRKAHiJ4Bn6l4oaklYGCcDM1kjqA4yXtKqMIhcDT0u6HcgF3gCmAw9J6kgwMD3GpXUt5xrTJU0FZgMLCIb70oa8z6Yz5LPSMZBmvPrp9v1flq9jeL8Hyjx35XeLeP20O+LavqrAQwMr/THoiYKi6RPZEGFEto3Z8V5d8bzv2PDnnTWlin+YxbqLT4h7+5JJtMu+JQ0AwrWKBrvgoiW0IoiEUMJi4PCIaqYTRNL+QlJ3gqmN1pWc28zM8oO2Wr6kplRC2hikkneQ3P7PQDt3+N+IcguBPmWc/9syqh3itpIyp4Xt99+d9no8Hk88KY5yBV14ZOtyKKuiyMG+QcBjbm5+JsHcelGU50ZN2hgkj8fj8ewghi/GLqb0FEZrYGl4ATMrAC4HcFMhC91Wq4Jzl0tq4byjFsAKKsEbpBhxzY01k90Ej2e3qX7FBclugidKYjiHNAno6BZzLQEuAC4KL+BC8Wwys23AVcB4N2df0bnvA5cReFeXETGaVRbeIHk8Hk8aEqtg32ZWJOl6YBTBKzEvmNlsSde6/GeAfYGXXSTv73Cv35R3rqt6EPCWpCuBRcC5lbXFGySPx+NJQ2L50quZjQBGRKQ9E7Y/ASjz3ZayznXpq4Feu9IOb5A8Ho8nDSkOpdNrpNHhDVIGktV2f6r1uACysiia+TlFk0aWLlC9FtV69yerXhOsuJBto4Zgq4N5yBpX3g+FWyBkWKiYra/fm4Q7SH98+InY8OXshTw47DNCZvQ9qgtX9C69Grlg0xbufGUUi1euo1puDgMv6U2Hlo3JW76GW57/cHu5JavWc91pR9HvhJgFlU46sRqySyWSamIlmaRXwo5zJK2U9KE7vkvSTbt5jRsl1Qo7HuEm6DITiWonXMTWdx9jy5A7yOncHTVsUapI7uGnElrxM1teGci2j1+gWs/SE9lb3nqYLa/e7Y3RbnDWqSfxzCP/SHYz0priUIj73xzDk9f/luF/78/IyXOZn7+6VJnnRk6kU+smDLv9Mv5xWR8eHPYZAG2bNeSt2y7lrdsuZeit/ahRLYcTDkoNNZVYETJFtaUTyfb5fgG6SCpZonYSwUqNmOBkLW4kWJoIgJmdambrYnWNVCOreTts3Ups/SoIFVM0ZxLZe3ctXaZhC0KLAuVjW7sM1W0EtXyohFjiw0/sPrPyltGmSX1aN65Pbk42vQ/txNjppYNFLshfzeGd9gSgXfNGLF29ntUFv5QqM3HOIlo3rk/LRnUT1vZEYKaotnQi2QYJ4GPgN27/QmBoRP5+ksZKWiDphpJESe+5EBOz3ZvIJekbJd0taSLwN6Al8JkLT4GkPEmNo6jjXheq4msnkZEWqHZ9bMOO8BO2cS2qU79UmdDKxWR3PASArOZtUd1GqPYO3cMaZ99IjYtvJ/uAYxPSZo+nLFas20jzBjuMerMGdVixfmOpMvu0bsKYaYGRmpmXT/6aApavK11m1JQ5nNKtc/wbnGBipWWXSqSCQXoDuEBSDYJQD5FiU52B3gR6S3dKynXpV7gQE92AG8LEUvcAZpnZ4WZ2N8FLWj3NrGcZ166ojq9dqIrxwNVlNTxc7fuFCZERMZJFWZL9pQ8LJ32MqteiRr87yOl6AqEVP0MoiOS19Y1BbHntH2wZ/hi5XXuS1Sqzhjk86YOV8cJ/5Lf7ipO7U7BpC+fd9zJvjJ1Kp9ZNyc7aUaqwqJhxM+Zz0iH7xLm1iScTh+ySvqjBzGZIakvgHe20dBD4yMy2AlslrQCaEbxZfIOkvq5MG4IliasJFLvfifLy5dWxDSiZEZ1CMJRYVtu3S3JseuTqlPgtEnhEO8JPqHYDbOO60oW2bWHbJ0O2H9a48n6sIJAFtF/WB4mbN1A8bypZzdsRWpIaoTU8VYtm9euwbO2G7cfL126gSb3apcrUrlmduy8NlMLMjFP//hytGtXbnv/F7IV0btOMRnX3SEyjE0gmrrJLlTt6H/gnOw/XQdnhIXoAJwJHOi9mKjvCQ2wxs2IqoZI6Cs22O7vFpIDhjpbQsjxUvymq2xiyssnpfBjFC0oLrVK9JmQFIaGyDzg2MDjbtkBONch1EWJzqpG1136EVvvwE57ksP9ezVm0Yh1LVq2nsKiYUVPmcvyBe5cqU7BpC4VFwX/34V/O5NAOrakdFuV45OQ59Dks84brIBj4iGZLJ1LlQfsCsN7MZjpDURn1gLUuHEVn4IgKym4A6gCRyuC7Ukf6YCG2ffY61c++ESSKZn2JrV5KzoHHA1A0YxxZDVtQrc8VYCFCq/PZ9slLAGiPulQ/43dBPcqmaM5EQnmzy7mQpyJ8+IndJyc7i1vPP4HrnniHUCjEmUd2oUPLxgwbH/zAOve4g1i4bA23v/Qx2VmiffNG3HXJjj7evK2Qr+f8xO0XlTnAkfak23BcNMiSOOslaWO4irdL6wHcZGanSboL2Ghm/3R5s4DTgHzgPQLp87lAE+AuMxsbWaekPwC/B/LNrKekPII5ow3R1CHpHOC0ytS/U2XILpPJvfSvyW5CxlM0/X/JbkKVoGavAbttTb5sfk5Uz5yjl72dNpYrqR5SpDFyaWNxwfDM7K6IvC5hh6dEU6eZ/Rv4d9hx212pw8zeBjIz5KTH40lbQsluQBxIlSE7j8fj8ewCVmYoovTGG6QY4YeTPJlAzkEnJrsJnigpysA5JG+QPB6PJw3xHpLH4/F4UgI/h+TJCLwSdfzxfRx/qnofZ6KHlGy172JJ0yTNkvRBZSrckrpJetzt77YSeCXX+q+kCfGqP5l4Jer44/s4/lT1Pg5FuUWDpD6S5kqaJ+nWMvLruWf0dKf9eblL7+Se4SVbgaQbXd5dkpaE5Z1aWTuSrdSw2cy6uuXcawjeFyoXM5tsZjdUVCYWOMN4CFDfxYrPKLwSdfzxfRx/qnofF6OotspwURGeJHgNZj/gQkn7RRT7PfCdU7XpATwsqZqZzXXP8K7AocAm4N2w8x4tyXeRZSsk2QYpnAkEL6kiqbukryRNdZ+dXHqPklhJjl+jBB6NivfZwAc44ddY36jH4/HsLiFFt0VBd2CemS0ws20Ez70zI8oYUEeSgNoEDkRRRJlewHwz++nX3lNKGCRnoXsRaNoBzAGOM7ODgTuA+8o59dcogVeq4s2OMBhD3b7H4/GkFCEU1RYelcBtAyKqagX8HHa82KWF8wSwL0H0hJnAH80sckTwAnbWI71e0gxJL0hqQCUk2yDVlDSNQGG7ITDapdcDhjmpoEeB/cs5/yMz22pmq4ASJXAIjNB04Gt2qHjDzirebSMrdF5TB+ALM/sBKJLUJbKcK7v9D/3cy2Xpwno8Hk98iFZc1cwGm1m3sG1wRFVl+VGRskS9gWkE8eW6Ak9I2h7xUFI14AxgWNg5TwN7u/L5wMOV3VOyDdJmN/a4F1CNHXNI9wCfubml09mhwh3JriqBR6PifT7QAFjodO/aUs6wXfgf+qpLvSPl8XgSRwwXNSwm+OFeQmsCTyicy4HhFjAPWEgwQlXCKcC3Zra8JMHMlptZsfOkniUYyaqQZBskAMxsPXADcJMbdqvHjlDm/Xexut1V8b4Q6GNmbZ3u3aFk2DzSzXcO4uJr/kTeosX0Oqsf73wwKtlNyjh8H8efqt7HISmqLQomAR0ltXOezgXsmD4pYRHBtErJKFInYEFY/k7RviW1CDvsC8yqrCEp8x6SmU11w2wXAA8CL0n6P+DTXaxqJHCtpBkEKt5fR3uiCxS4Z/g5ZrbQLWU83Mwio9mmJQ8N3GlVpyfG+D6OP1W9jysN+hYlZlYk6XpgFJANvGBmsyVd6/KfIRi1GiJpJsEQ31/cVAmSahEEMb0mouoHJXUlGP7LKyN/J5IafiKTKFy1wHekx+OJitzG7Xf7rdahLS+O6plz4dLX0uYN2pTxkDwej8cTPaEMVGrwBsnj8XjSkEwckvEGyePxeNKQKF96TSu8QfJ4PJ40xKt9ezwejyclKPYekicTqOqy/YnA93H8qep9nIkeUkq8GBtLJP3NiarOcJLnh1dQdkRJyAtJN0j6XtJrCWtskqjqsv2JwPdx/KnqfRzL8BOpQkZ5SJKOBE4DDjGzrZIaE0gSlYmZhcfn+B1wipktjHMzk063rgewJH955QU9vxrfx/GnqvexZeCQXaZ5SC2AVWa2FcC9SbyvpO3xOSSdJGm428+T1FjSM0B74H1Jfyov/IXH4/GkCpnoIWWaQfoEaCPpB0lPSTqeQHpoX0lNXJnLgRfDTzKzawnEBHua2aNEGf7Cq317PJ5kURzllk5k1JCdmW2UdChwLNATeBO4FXgF6CfpReBI4NJKqqpHoKXXkeD9s9yyCjkZ98HgpYM8Hk9i8e8hpQFmVgyMBcY6IcDLCET9PgC2AMPMLDLSYSQl4S/6OsHVsXFrsMfj8fwK0m04LhoyashOUifn1ZTQFfjJzJYSDMndDgyJoqrdCX+R8lR12f5E4Ps4/lT1Ps7EOaSMUvt2w3X/BuoTxHufBwwws1WSLgBuNLMjwsrnAd1cfvj+kcBLwEqCOahLXGykcvFDdh6PJ1piofb9zz37RfXMuWnRq2kzuJdRQ3ZmNgU4qpzsYwiiFoaXb1vO/gRgn7Cif49ZIz0ejycGZOIcUkYN2ZWHpCnAgcCryW6Lx+PxxIJYrrKT1EfSXEnzJO0U+VBSPUkfSJruhAcuD8vLkzTTCRFMDktvKGm0pB/dZ4PK2pFRHlJ5mNmhyW6DZ/ep2fLYZDch43m34XHJbkKV4PRlu/+aSChGASgkZQNPEkR9XQxMkvS+mX0XVuz3wHdmdrp7hWaupNfMbJvL71kSQTaMW4ExZjbIGblbgb9U1JYq4SF5PB5PphHDRQ3dgXlmtsAZmDeAMyPKGFBHkoDawBqCefqKOJNgLh73eVZlDfEGyePxeNIQi3ILf4HfbQMiqmoF/Bx2vNilhfMEsC/BauWZwB/NrMTeGfCJpCkRdTczs3wA99m0snuqEkN2ntJUdZXkWND75B488sjdZGdl8cKLQ3nwoSdL5f/5/67lwgt/C0BOTjb7du5I85YHsnbtOv54w9VcccWFmBmzZs3hyqv+j61btybjNlKeJj0Poss9l6LsLBa99hnznni/VH5OnZoc/OTvqdmqMVk52cx/+kN+fmMcAO0HnMKeF5+AmbHh+5+ZduMzhLYWJuM24kK0S7rDX+Avh7KWR0SOB/YGpgEnAHsDoyV9bmYFwNFmtlRSU5c+x8zGR9m8UqSthySpkZtEmyZpmaQlYcflCqqWUc9DbpLuIUnXSrrUpY+V1C1+d5A8qrpK8u6SlZXF44/dy2mn9+OAg3py/vlnse++HUuVefiRZ+h22Ml0O+xkbr99EOPHf83ateto2bI51//+Cg4/4lS6HtyL7Oxszj8vcnTEA0CWOOD+y5l40QN8dtxNtOx7FLX3Kf3Dve3lJ7PxhyWM73UrX/32bva7sx/KzaZG8wa0u6oP43vfxrget6DsLFqedWSSbiQ+FMmi2qJgMdAm7Lg1gScUzuXAcAuYBywEOgO49zwxsxXAuwRDgADLJbUAcJ8rKmtI2hokM1ttZl3NrCvwDPBoyXHYRFs0XEOgDn6zmT1jZi/HpcEpRLeuB1Cvbp1kNyNt6X7Ywcyfn8fChYsoLCzkrbf+yxmn9y63/Pnnn8kbb763/TgnJ4eaNWuQnZ1NrZo1yc9floBWpx8NDu7ALwuXsWnRCqywmKXvTaB574jfiAY5tWsCkL1HDQrXbcSKAt9B2dlk16iGsrPIrlmNrcvWJvoW4kq0Q3ZRMAnoKKmd+zF/AfB+RJlFQC8ASc2ATsACSXtIquPS9wBOBma5c94nUMrBff63soakrUEqC0lXS5rklia+I6mWSx8i6Zywchvd5/vAHsBESedLukvSTRF1Zkl6SZJ3KTwAtGzVnJ8X7/gBuXhJPi1bNi+zbM2aNeh9cg+GvzsCgKVLl/HIo8+wcP43LF40lfUFBYz+368a3ch4arRowOalq7cfb8lfTY0WpVcOL3xhFLU7tuSk6U/R47MHmfX3l8GMLcvWMv/pDzlxyhOcNONpCgs2sXLczETfQlyJ1aIGJ6V2PTAK+B54y8xmuxGja12xe4CjnBzbGOAvblVdM+ALSdOBb4CPzGykO2cQcJKkHwlW8A2qrC0ZZZAIXMrDzOwggo69sqLCZnYGsNl5VW+WUSQHeA34wcxuj31zPelIsNCoNOUpnpx22sl8NWEya9euA6B+/XqccXpvOuxzBG32OoQ99qjFRRf9Np7NTV/K6OfIn/xNex5IwayfGH3Q7xjX61YOuK8/ObVrkltvD5r36caY7jcw+qDfkVOrOq3OPiYx7U4QISyqLRrMbISZ7WNme5vZvS7tGTN7xu0vNbOTzewAM+tiZq+69AVmdpDb9i851+WtNrNeZtbRfa6prB2ZZpC6SPrcWfGLgf13s77/ALPCOzkcH36iarJkcT5tWrfcfty6VQvyywkUd/55Z5QaruvV61gW5i1i1ao1FBUV8e57H3PkERk5VbnbbFm6hpotG20/rtGiEVsiht3aXNCD/BHfALApbzmbFq2kdseWND6uC5sWrWDb6g1YUTH5IybR8LB9yCRiOGSXMmSaQRoCXG9mBwADgRouvQh3r24dfbSLHr4CekqqUVammQ02s25m1u2qSy/crYZ70odJk6fRoUM72rZtQ25uLueddyYffPjJTuXq1q3Dcccewfvv7xD9/HnREg4//BBq1gy+Uif0PIY5c35MWNvTiXXT5rNH++bU3LMJys2m5VlHsuyTKaXKbF6yisbHdgGgWuN67LF3Czb9tILNi1fR4NCOZNcM/qs3PrYLG35cstM10plMFFfNtGXfdYB8SbkEHlLJNzAPOBR4i+BlrTLjG5XB88BxwDBJfaMIW5EW3HznICZNncG6dQX0Oqsfv7vyEs6uYFLeU5ri4mL+eOPtjPjodbKzshjy0pt8990PDLj6EgAGP/sKAGedeQqj/zeeTZs2bz/3m0lTGT78IyZ9M4qioiKmTZvNs8+9lpT7SHWsOMSs24ZwxNC/ouwsfh46lo1zF7PXpScC8NPL/+OHR97l4Meu5fjPHgCJ7/8xlG1rNrBtzQaWfjiR4z65j1BxiIKZeSx6ZUyS7yi2FKed/1M5GaH2LekuYCPwC3AL8BPBy1t1zKy/WxXyXwIvaQzwBzOr7c7dGLZ/F7DRzP4paSxwk5lNljSQQGz14rCXwUrh1b7jj5cOij9eOigxnL5s6G5Lo/6x7QVRPXMey3sjbWRYM8JDMrO7wg6fLiN/OXBEWNJfw/Jql1WPmfUI278zNi31eDye2GAZ6CFlhEHyeDyeqka6zQ9FgzdInrRh89LPk90EjydliJXadyrhDZLH4/GkIZlnjrxB8ng8nrSkKANNkjdIHo/Hk4b4RQ2ejMCHn4g/vo/jT1Xv40xc1JD2Sg0K+ELSKWFp50kaWdF5ldTZX9ITbn97SIpMwYefiD++j+NPVe9ji/JfOpH2HpKZmVOkHSbpMyAbuBfoE6P6M+6nV7euB7CkHO01T2zwfRx/qnofZ6KHlPYGCcDMZkn6APgLQTiJV4FXJNUENgOXm9lcSf2BM4BaBFEP3zWzWwAkXU7wwmw+8AOw1aXfhVNvSOhNeTweTwUUZ4DKTiRpP2QXxkDgIuAU4F/AcWZ2MHAHcF9Yua7A+cABwPmS2rhohgOBowniduwXzQW92rfH40kWsQw/kSpkhIcEYGa/SHqTQNOuLjBEUkeC5frhYqpjzGw9gKTvgL2AxsBYM1vp0t8k0K6r7JrbY9V7LTuPx5NIYjk/JKkP8BjBlMdzZjYoIr8ewcjTngR2459m9qKkNsDLQHOCUcTBZvaYO+cu4GpgpavmNjMbUVE7MsYgOUoU1+8BPjOzvpLaAmPDymwN2y9mRx94g+LxeNKGWM0hScoGniQYHVoMTJL0vpl9F1bs98B3Zna6pCbAXEmvEYT2+bOZfetCmU+RNDrs3Ed3Zbojk4bswqnHjtAT/aMoPxHoIamRC11xbrwalgrcfOcgLr7mT+QtWkyvs/rxzgejKj/Js0v4Po4/Vb2PYzhk1x2Y56K/bgPeIAjTE44BdVw8udrAGqDIzPLN7FsAM9tAEKm71a+9p0zzkEp4EHhJ0v8Bn1ZW2MzynXs5gWBRw7cErmtG8tDAW5PdhIzH93H8qep9HMMhu1bAz2HHi4HDI8o8AbwPLCWIO3d+ZCgeNxp1MMEP/BKud6/NTCbwpEqH/I0gI+IhpQJ+Dsnj8URLbuP2ux2jqO+ep0f1zHnv5w+vAQaEJQ12898ASDoX6G1mV7njS4DuZvaHsDLnECz6+j+CFcqjgYPMrMDl1wbGAfea2XCX1gxYReBd3QO0MLMrKmprpnpIHo/Hk9FEu4IufPFVOSwG2oQdtybwhMK5HBhkgQczT9JCoDPwjZvmeAd4rcQYuetuf0lM0rPAh5W11RukGFE8d0Kym+DxeNKE3Mbtd7uOGL4YOwnoKKkdwdz7BQSv0ISzCOgFfO48n07AAjen9DzwvZk9En6CpBZmlu8O+wKzKmuIN0gej8eThsRqDsnMiiRdD4wimDt/wcxmOwWcErWaewhepZkJCPiLma2SdAxwCTBT0jRXZcny7gcldSUYsssDrqmsLd4geTweTxoSy5denQEZEZH2TNj+UuDkMs77gsBAlVXnJbvaDm+QMpAvZ87jgddHEbIQfY89mCt/c0yp/IJfNnPHC++zeOVaquXmMPDyM+jYuikAr3zyNcPHT0WCjq2acveVZ1I9139NIvF9nBh8P5dPJi5Iy9T3kKosxaEQ9736MU/96SLe/cfvGDlxNvOXrCxV5rmPvqDzns15++5rufeqs3hwaCCMvnxtAa//7xuG3nEVw++5jlDIGDmx0mHfKofv48Tg+7liirGotnQi4w2SpMz5SRQFsxYsoU3TBrRu2oDcnGz6HL4/Y6fNLVVmwdKVdN+3HQDtWjRm6ar1rF6/EYDi4hBbtxVRVBxi87ZCmtSvk/B7SHV8HycG388Vk4ladkk1SJLekzRF0mxJA1xaH0nfSpouaYxLqy3pRUkzJc2QdLZL3xhW1zmShrj9IZIeceEoHpDUXdJXkqa6z06uXLakf4bV+wdJvSS9G1bvSZKGkyasWLeB5g3rbT9u2qAuy9duKFVmnzbNGPPt9wDMXLCE/NXrWL62gGYN6nJZnyPpffO/OPFPj1CnVnWO6rJ3QtufDvg+Tgy+nyvGzKLa0olke0hXmNmhQDfgBrec8FngbDM7iB0SPn8H1pvZAWZ2IFGoLxCIo55oZn8G5lC2+vcAoB1wsKv3NVf3vk6vCYL19y/u7o0mirK+f4qYcrzi1GMo+GUL5935H4aO+YbOe7YgOyuLgl8289nUuYx44AZGP/InNm8t5MMJMxLT8DTC93Fi8P1cMZnoISV7OOsGSX3dfhsCAzHezBYCmNkal3ciwdp4XHqF8hOOYWZW7PbrEUgJRap/nwg8Y2ZF4deT9ArQT9KLwJFAmRFjnVc3AOCJm6/gyjNPiKJZ8aVZgzosW7N++/GKtQU0jRiqqF2zOvdcGUhVmRmn3vI4rZo04KtZ82nVuD4N6+4BQK9DOjN93mJOO/LAxN1AGuD7ODH4fq6YdIsGGw1J85Ak9SAwCEc6b2gqMJ2yVbdVTnp4Wo2IvF/C9kvUv7sAp4eVLa/eF4F+wIUEhq2orHsws8Fm1s3MuqWCMQLYv10rFi1fw+KVayksKmbkxNkc37V0JI2CTVsoLAps9fDxUzlkn72oXbM6zRvWZcaCJWzeWoiZMfH7hbRr0TgZt5HS+D5ODL6fK6bYLKotnUimh1QPWGtmmyR1Bo4AqgPHS2pnZgslNXReyyfA9cCNAJIaOC9puaR9gbkEbwJvKOtClK/+/QlwraSx7uWwhma2xsyWSloK3E4gyZ425GRn8dd+p3DdI68RChlnHdOVDq2a8tZnkwE4r2c3Fi5dye3P/ZesLNG+ZRMGXn46AAfu3ZqTuu3LBQMHk52dRec9m3PO8Yck83ZSEt/HicH3c8Wk23BcNCRNXFVSdeA9AqXZuUAT4C6gJsEcTxawwsxOcsJ9TwKHEsQwGmhmw53g3wMESrWzgNpm1t8tbvjQzN521zoSeIkgUNSnwCVm1tatwHsQ6AMUAs+a2RPunAuAG83siGjuZ8uXr2Xet8Pj8cSFGkdfvNviqke26hnVM2fCks92+1qJwqt9l4OkJ4CpZvZ8NOW9QfJ4PNESC4N0RMseUT1zvl46Nm0MUrIXNaQkkqYQzEH9Odlt8Xg8nrLIxCE7b5DKwC1F3zVyq8WhJZ5SFG5Ldgsyn1AMNaQ9cSUTV9l5g+TxeDxpSLFl3o8Hb5A8Ho8nDcnE+X9vkDwejycN8XNInrTgyxk/8MArIwiFQvTtcShXnn58qfyCXzZzx7PDWbxiTSDZf9Vv6dimGQCvfPwlw8dNQUDHNs24++rfUr1abhlXqdr4sAiJ4ctZ83lg6ChCIaPvsV258tSjS+UX/LKZO4Z8yOIVJf18Gh1blfTzRIZ/MRUhOrZuwt2Xn5FR/ZyJc0jJ1rLzxJjiUIj7XvqAp26+lHcfuIGRE2Yyf8mKUmWee38cnfdswdv3/YF7rzmHB1/9CIDlawp4/ZMJDL37OoYPuiGQ7P96ZjJuI6XxYRESQ3EoxH2vfcxTN17Iu/dcy8hvZjN/aUQ/j/iSzm2a8fbAAdx75Rk8OPQTwPXzp98w9PYrGX73NUE/fzM7GbcRN0JmUW3R4ESt50qaJ+nWMvLrSfrAiV7PlnR5ZedKaihptKQf3WeDytqR1gZJUnay25BqzJq/mDbNGtG6aUNyc3Loc8QBjJ3yfakyC5asoPv+7QFo17IJS1et3SHZHwqxdVshRcXFgWR/g8yS7I8FPixCYpi1cCltmjakdRPXz933Z+y0H0qVWbB0Fd33bQu4fl69roJ+rp3oW4grFuW/ynDP0SeBU4D9gAsl7RdR7PfAd07mrQfwsKRqlZx7KzDGzDoCY9xxhcTNIElqK2mOpOckzZL0mqQTJX3pLGZ3Vy7q0BAuPU/SHZK+AM6VdKErM0vSA+W05Q5Jk1yZwQrYV9I3Ee2d4fZPdW3/QtLjkj6MVz/FmhVrC0pL9jesy/K1BaXK7LNnc8ZM/g6AmfMXk79qPcvXrKdZw7pcduox9L7xn5z4hweoU7M6Rx3QMaHtTwd8WITEsGLtBpo3qLv9uGmDOmX0c1PGfBv8GAj6eT3L124I+rn3kfT+y+Oc+Od/Bd/l/TOrn4stFNUWBd2BeWa2wMy2AW8AZ0aUMaCOJAG1gTVAUSXnnkmgkIP7PKuyhsTbQ+oAPAYcCHQGLgKOAW4CbnNldiU0RAlbzOwYYDyBdNAJQFfgMElnldGOJ8zsMCeuWhM4zcy+B6pJau/KnA+8JakG8B/gFHeNJmXUBwRq35ImS5r8/Lv/i7ZP4krZkv2lX9S+4vTjKPhlM+f97QmGjp5A573CJPunfM+IR/7M6Mf/Ekj2fzktMQ1PI3xYhMRQ1q/7nfr5lKOD7/LAZxn66SQ679mc7GzXz9PmMmLQ9Yz+5x9dP2fW8HO0Q3bhzym3DYioqhWB/FoJi11aOE8A+wJLgZnAH80sVMm5zcwsH8B9Nq3snuI9w7fQzGYCSJpN4L6ZpJlAW1dml0JDON50n4cBY81spbvGa8BxBBp54fSUdAtQC2gIzAY+AN4CzgMGERik8wkM54KSEBjAUFyIiUjMbDAwGGDLN8NSYoaxWcO6pSX715Ql2V+DewacDTjJ/v97mFZNG/DVjHm0atJgh2T/Yfsx/cdFnHZ014S1Px3wYRESQ7MGdVkW5t2vWLuh7H6+4gzA9fOtT9CqcX2+mu36uU5YP89fzGlHHpC4G4gz0S5qCH9OlUNZ0kKRlfcGphH8+N8bGC3p8yjPjZp4e0hbw/ZDYcchdhjDXQ0NATtCS1Sq0eQ8nqeAc8zsAIIAgCXXeBM4T9I+gJnZj9HUmcrs374Vi5atZvGKNRQWFTHy65kcf0jnUmUKftlMYVEQUWP42Mkc0qkttWvWoHmjesyYv5jNW7cFkv2z59OuZbkOYpXFh0VIDPu3bVm6n7+ZzfEHVdDPn0/lkH32dP1cL+P7OYaLGhYTxKMroTWBJxTO5cBwC5gHLCT48V7RucsltQBwn6VXV5VBKqyB3KXQEBHnTgQek9QYWEsQv+jfEWVKjM8qpxp+DvA2gJnNl1RMEJG2xOuaA7SX1NbM8gi8prQhJzubv156Gtc99BKhUIizjjuUDq2b8daYYLrsvF7dA8n+/7wTSPa3asrAq4IYiQd2aMNJh+3PBX9/iuysLDq3bcE5PQ9L5u2kJD4sQmLIyc7irxf14bp/DQ2+y0d3pUOrJrw1dgoA5/U4lIX5q7j9+f+SlZVF+xaNGdj/NAAObN+Kkw7dlwvueS74Lu/ZjHOOOziZtxNzYrjsexLQUVI7gmfxBQTTK+EsAnoBnyuI7N0JWACsq+Dc94HLCEagLgP+W1lD4qb2LaktQQiILu54iDt+OzxvV0NDSMoDupnZKlfvRcBfCTybEWZ2Sxlt+QdBR+URjHf+ZGZ3ubybgIeAds4AIel0l7YK+IZgLPTiiu43VYbsMhqvZRd/vJZdQqhx7CW7PRKzV6MDo3rm/LR6RjQjSacC/wKygRfM7F5J1wKY2TOSWgJDgBYEz9pBZvZqeee69EYE0yJ7Ehi0c8twKkq3IxPlJ3YXSbXNbKNbUfIk8KOZPVrROd4gJQBvkOKPN0gJIRYGac+GB0T1zFm0ZmbaTEOk9XtIceRqSdMIFj/UI1h15/F4PClDCItqSydSYQ4p5XDeUIUekcfj8SSTTBzd8gbJ4/F40pBoZYHSCW+QPB6PJw3JRHFVb5AyEK/2HX+82ndi8Grf5ZOJAfriqWX3XBkCfZFlhkg6p4z0tm45965es8z6qhJe7Tv+eLXvxODVvivGzKLa0om4GSQzu8rMvvuVp7dl5xezPFHg1b7jj1f7Tgxe7btiYhl+IlWo0CBJukXSDW7/UUmfuv1ekkpeijpZ0gRJ30oa5tQQkDRWUje3f6WkH1zas5KeCLvMcU7le0GYdzMIOFbSNEl/csrfDznF7hmSrnH1StITkr6T9BHliPdJutqdO13SO5JqKYjvkScpy5WpJelnSbmSDnPXmeCumzY/Yb3ad/zxat+Jwat9V0xV9JDGA8e6/W5AbUm5BIrdnzvJntuBE83sEGAy8H/hFbg3fP8OHAGcRKB/FE4LV99pBIYIgrgZn5tZV7cE+0pgvZkdRiCoerWTquhLIGFxAHA1cFQ59zHcqX0fBHwPXGlm64HpQMkEy+nAKDMrBF4ErjWzI4HiSvoopfBq3/HHq30nBq/2XTFV8T2kKcChkuoQCKN+S2CYjgVuIDAy+wFfuodeNWBCRB3dgXElkhGShgHhConvORnz75xGUlmcDBwY5kHVAzoSKHsPNbNiYGmJB1cGXZx8UH2CWB6jXPqbBFp1nxFICz0lqT5Qx8y+cmVeJzCWO+Fk3AcAPHHrAK7se2I5l08cXu07/ni178Tg1b4rJt28n2io0ENy3kIegdLrV8DnQE8C+fHvCTSNRjtPpquZ7WdmV0ZUU5lsRbgieHllBfwh7DrtzOyTkmZWUj8EGkzXO7XvgewQXH0fOEVSQ+BQAi29qGU2zGywmXUzs26pYIzAq30nAq/2nRi82nfFxDBAX8oQzRrI8QQB9a4gCMz0CDDFxTX6GnhSUgczmyepFtDazMJnHr8BHlUQT30DcLarpyI2AOE/hUYB10n61MwKXbiIJa5t10h6mWD+qCeBRxNJHSDfDTde7M7F6dV9QxBE8EPnaa2VtEHSEWb2NYHnlDZ4te/449W+E4NX+66YdFuwEA2ViqtK6gWMBOqb2S+SfiAInPeIyz+BIGprdXfK7Wb2vqSxwE1mVhKh8CaCOBnfA2vM7G/hCuCuro1mVjJPNRJoTODdPAb8g2CeRwTK4GcBBQThJk4ASozgqyX1hd3DdcAtwE8ExrCOmfV3eecAw4AeZjbOpR1OEDfpF2AsQUTb0i9ARODFVROAF1eNP15cNSHEQly1Ro09o3rmbNmyKG3EVROi9h2mnp0DvEsgUf5u3C/8Kylpr9u/FWhhZn+s6BxvkBKAN0jxxxukhBALg1S9Rpuonjlbt/ycNgYpUa8t3yXpRIK5m0/YOcR4qvEbSX8l6J+fKB040OPxeJJOJi5qSIhBMrObEnGdWGFmb7IjgqzH4/GkHFVyDsmTuUgaYGaDk92OTMb3cfzxfZw5+AB9VZsByW5AFcD3cfzxfZwheIPk8Xg8npTAGySPx+PxpATeIFVt/Lh7/PF9HH98H2cIflGDx+PxeFIC7yF5PB6PJyXwBsnj8Xg8KYE3SFUUSd0kPV5BfktJb5eX74kNkvqXBKyUdJektHqJ/Nci6QZJ37uAmRMkba0q9+4pn0RJB3nijKRsp1YeFWY2mSCgYnn5S4Fzysuv6igIACYXy8uz6/wOOIVAwHgvArHkhCEpx8yKEnlNT+V4DykNkNRW0hxJL7nQ6m+7kOt5ku6Q9AVwbgXh5A9zYeKnS/pGUh1JPSR96PKPd+Hip0ma6vLbloRul1RD0ouSZrr8ni69v6ThkkZK+lHSg0nrpATg+uR7SU8RBKv8u6RJ7m8yMKzcpS5tuqRXXNrpkia6/vtfBcEoMx5JzwDtCeKRXWxmk4DCSs7Z6Tvq0m9x38vpkga5tK6SvnZ/g3dd6BskjZV0n6RxwB8lHSppnKQpkkZJahHXG/dUTrRx2f2WvA1oSxCI8Gh3/AJBOI884BaX1pggPtQe7vgvwB0EUXwXAIe59LoEnnEPgtAfAB+E1V3b5bcFZrm0PwMvuv3OwCICodz+ru567vgnoE2y+yvOf4cQQaTkkwmWG4vgh92HBBGM9wfmAo3dOQ3dZwN2rGq9CnjY7fcHnnD7dxGEbEn6vSagL/NK+iiaey/nO3oKQeDQWhF9PQM43u3fDfzL7Y8FnnL7ue7cJu74fIIoBEnvm6q8+SG79OFnM/vS7b9KEEIedojAlhdOvhOQb8GvUMysAMCVKeFL4BFJrwHDzWxxRP4xBHGnMLM5kn5iRxj6MWa23tX5HcHwy8+xuOEU5Scz+1rSPwmM0lSXXhvoCBwEvG1mqwDMbI3Lbw286X6FVwMWJrbZaU9Z39ETCX4obYKgryXVI4jdNs6d9xJBvLMSSv6/dAK6AKPddz0byE/AfXgqwA/ZpQ+RL4yVHP/iPssLJ68yzi1dkdkggl/tNYGvJXWOKFJRPJXwEPTFZP68ZHh/3x/W3x3M7HnK7+9/E3hCBwDXEHiUnnKQ9PuwIbqW5XxHK/1ul0H432922N/vADM7OXZ34Pk1eIOUPuwp6Ui3fyHwRUT+18DRkjoAuDmmfYA5QEtJh7n0OgoCJW5H0t5mNtPMHiBY6BBpkMYThH7H1bknwbBUVWYUcEXYPF0rSU2BMcB5khq59IaufD1gidu/LNGNTTfM7MkwY7G0nO/oJwR/g1oQ9LXz1tdKOtZVdQkwroxLzAWalPyfkpQraf+435inQjL912wm8T1wmaT/AD8CTwN/KMk0s5WS+gNDJYWHk/9B0vnAvyXVBDYDJ0bUfaNbqFAMfAd8DIRP8D4FPCNpJlAE9DezrRHDelUKM/tE0r7ABNcPG4F+ZjZb0r3AOEnFBEN6/QnmSIZJWkLw46FdUhqeYkhqTmBg6gIhSTcC+5UMLYex03fUfQe7ApMlbQNGALcRGPxnnKFaAFweeV0z2ybpHOBxN8yXA/wLmB37u/REi5cOSgMktSVYgNAl2W3xeDyeeOGH7Dwej8eTEngPyePxeDwpgfeQPB6Px5MSeIPk8Xg8npTAGySPx+PxpATeIHk8Ho8nJfAGyePxeDwpwf8DzGoKszarcvQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.heatmap(pd.DataFrame(cr).iloc[:-1, :].T, annot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "845640a1-5d14-48e8-89e5-e7a913d3950e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e1f4a911-7fbf-49c2-86c0-15973154221b",
   "metadata": {},
   "source": [
    "# ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "06e2a67c-60ac-4c87-b4fa-2a550925b7a2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T07:06:44.152506Z",
     "iopub.status.busy": "2021-06-29T07:06:44.152506Z",
     "iopub.status.idle": "2021-06-29T07:06:44.163505Z",
     "shell.execute_reply": "2021-06-29T07:06:44.162506Z",
     "shell.execute_reply.started": "2021-06-29T07:06:44.152506Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mobilenetv2_1.00_64-imagenet224-32-12\n"
     ]
    }
   ],
   "source": [
    "kwargs = {\n",
    "    'input_size' : 64,\n",
    "    'batch_size' : 32,\n",
    "    'weights' : 'imagenet',\n",
    "    'dense' : False\n",
    "}\n",
    "nama_model = create_mobilenetv2_name(**kwargs)\n",
    "print(nama_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "17107780-6a72-49e8-89d3-34a2e21b7df8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T07:06:45.029606Z",
     "iopub.status.busy": "2021-06-29T07:06:45.029606Z",
     "iopub.status.idle": "2021-06-29T07:06:46.915737Z",
     "shell.execute_reply": "2021-06-29T07:06:46.914745Z",
     "shell.execute_reply.started": "2021-06-29T07:06:45.029606Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "weight_filename: 'mobilenetv2_1.00_64-imagenet224-32-12.weights.048_0.9711_0.2074.hdf5'\n"
     ]
    }
   ],
   "source": [
    "model = load_full_model(nama_model, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "e32942ae-8c10-412f-bea4-6f14ca848649",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T07:06:46.916741Z",
     "iopub.status.busy": "2021-06-29T07:06:46.916741Z",
     "iopub.status.idle": "2021-06-29T07:06:46.963742Z",
     "shell.execute_reply": "2021-06-29T07:06:46.962742Z",
     "shell.execute_reply.started": "2021-06-29T07:06:46.916741Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_data_gen = create_test_datagen(kwargs['input_size'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "e5173ab9-5851-439e-acd0-9197c98acc82",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-29T07:06:46.965741Z",
     "iopub.status.busy": "2021-06-29T07:06:46.965741Z",
     "iopub.status.idle": "2021-06-29T07:07:18.290413Z",
     "shell.execute_reply": "2021-06-29T07:07:18.289407Z",
     "shell.execute_reply.started": "2021-06-29T07:06:46.965741Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216/216 [==============================] - 29s 133ms/step - loss: 2.2710 - categorical_accuracy: 0.8981 - top-3: 0.9907 - ROC-AUC: 0.9539 - PR-AUC: 0.8515 - precision: 0.8981 - recall: 0.8981 - TP: 194.0000 - TN: 2354.0000 - FP: 22.0000 - FN: 22.0000\n"
     ]
    }
   ],
   "source": [
    "history_dict = model.evaluate(\n",
    "    test_data_gen,\n",
    "    batch_size=1,\n",
    "    verbose=1,\n",
    "    steps=len(test_data_gen.filenames),\n",
    "    # callbacks=callbacks,\n",
    "    # max_queue_size=10,\n",
    "    # workers=0,\n",
    "    return_dict=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "044a99f2-901e-4443-9331-bc7e1693bfa1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc-autonumbering": true,
  "toc-showmarkdowntxt": false,
  "toc-showtags": false
 },
 "nbformat": 4,
 "nbformat_minor": 5
}