LeNet-5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "06eea33e-2aff-4340-9aa8-9fb662d92596",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:32.118190Z",
     "iopub.status.busy": "2021-06-19T13:46:32.118190Z",
     "iopub.status.idle": "2021-06-19T13:46:41.205207Z",
     "shell.execute_reply": "2021-06-19T13:46:41.204204Z",
     "shell.execute_reply.started": "2021-06-19T13:46:32.118190Z"
    },
    "id": "e039f65a-4b32-405a-a4ae-95afdc2e8eb1",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "# import matplotlib.image as mpimg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4de0baaf-cdeb-4683-9b38-d2a682e11272",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:41.206203Z",
     "iopub.status.busy": "2021-06-19T13:46:41.206203Z",
     "iopub.status.idle": "2021-06-19T13:46:41.221204Z",
     "shell.execute_reply": "2021-06-19T13:46:41.220208Z",
     "shell.execute_reply.started": "2021-06-19T13:46:41.206203Z"
    },
    "id": "e17e0443-a109-4fb8-a5f1-d25e2d4d53a8",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import imgaug as ia\n",
    "# from imgaug import augmenters as iaa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "943378c3-5072-4f00-a770-c1f7f579f040",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:41.223207Z",
     "iopub.status.busy": "2021-06-19T13:46:41.223207Z",
     "iopub.status.idle": "2021-06-19T13:46:41.285213Z",
     "shell.execute_reply": "2021-06-19T13:46:41.284215Z",
     "shell.execute_reply.started": "2021-06-19T13:46:41.223207Z"
    },
    "id": "330dfc9b-4e7e-4262-91c0-0baec3158880",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "from datetime import datetime\n",
    "import time\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ceaa741-4ab0-4268-9229-86e204fa5f82",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bedbe96d-8dc9-4df4-abc7-52687b59150c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8622bc7-2e2b-4d95-a309-94d847f49580",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7246e482-215a-48b9-88f8-1db94f586a4f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:41.287209Z",
     "iopub.status.busy": "2021-06-19T13:46:41.286214Z",
     "iopub.status.idle": "2021-06-19T13:46:41.317216Z",
     "shell.execute_reply": "2021-06-19T13:46:41.316212Z",
     "shell.execute_reply.started": "2021-06-19T13:46:41.287209Z"
    },
    "id": "iQBpiIhqjVk6",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# attention! Change according to environment, if local machine set to False\n",
    "am_I_using_colab = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8971753f-f1d3-4f0b-b108-55ca85f7dca1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "905e105f-bd5f-4912-9391-56035fd26e1d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea2ac1c5-30f0-4dde-9bc4-0836c3042ae6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "678ee0e1-ddfd-4b0b-ad8b-466cde78be84",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "51bae2d3-9ea5-4c70-b9d2-c2e357220c02",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:41.319215Z",
     "iopub.status.busy": "2021-06-19T13:46:41.318215Z",
     "iopub.status.idle": "2021-06-19T13:46:41.349221Z",
     "shell.execute_reply": "2021-06-19T13:46:41.348219Z",
     "shell.execute_reply.started": "2021-06-19T13:46:41.318215Z"
    },
    "id": "4d9436d0-fb2f-496d-840f-c5a0a4e1eac9",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# if am_I_using_colab:\n",
    "#     !pip install -U tensorflow-addons\n",
    "    \n",
    "# import tensorflow_addons as tfa\n",
    "# import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c0b70f6c-2cf6-4397-8ec0-14242579d21f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:41.350227Z",
     "iopub.status.busy": "2021-06-19T13:46:41.350227Z",
     "iopub.status.idle": "2021-06-19T13:46:41.395219Z",
     "shell.execute_reply": "2021-06-19T13:46:41.395219Z",
     "shell.execute_reply.started": "2021-06-19T13:46:41.350227Z"
    },
    "id": "3dd459d4-e3e1-4601-aa6e-7d4a01a0e594",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# plt.ioff()  # interactive mode off\n",
    "# plt.ion()  # interactive mode on\n",
    "\n",
    "# %pylab inline\n",
    "\n",
    "np.random.seed(1728)\n",
    "# ia.random.seed(1728)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b608180f-96bb-4453-900e-179d4e7440a7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:41.397226Z",
     "iopub.status.busy": "2021-06-19T13:46:41.396228Z",
     "iopub.status.idle": "2021-06-19T13:46:41.428224Z",
     "shell.execute_reply": "2021-06-19T13:46:41.427227Z",
     "shell.execute_reply.started": "2021-06-19T13:46:41.397226Z"
    },
    "id": "S04tKXrZieAG",
    "outputId": "128332fe-95fd-4a94-91d1-88de817b672c",
    "tags": []
   },
   "outputs": [],
   "source": [
    "if am_I_using_colab:\n",
    "    ###\n",
    "    from google.colab import drive\n",
    "    drive.mount('/content/drive')\n",
    "\n",
    "    print(os.getcwd())\n",
    "    print(os.listdir())\n",
    "\n",
    "    ###\n",
    "    temp_dir = '/content/temp'\n",
    "    try:\n",
    "        os.mkdir(temp_dir)\n",
    "    except:\n",
    "        pass\n",
    "    print(os.listdir(temp_dir))\n",
    "\n",
    "    ###\n",
    "    dataset_filepath = f'{temp_dir:s}/dataset_v1.7z'\n",
    "    import gdown\n",
    "    gdown.download(\n",
    "        r'https://drive.google.com/uc?id=15kAUrJvaJy54xoXIwPZqtkBQGccjOR5k',\n",
    "        output=dataset_filepath,\n",
    "        quiet=False,\n",
    "        )\n",
    "\n",
    "    ###\n",
    "    !pip install py7zr\n",
    "    import py7zr\n",
    "    with py7zr.SevenZipFile(dataset_filepath, 'r') as archive:\n",
    "        archive.extractall(path=\"./dataset_v1\")\n",
    "\n",
    "    ###\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "41dcc745-154e-4d26-ac81-9c669740b00f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:41.429222Z",
     "iopub.status.busy": "2021-06-19T13:46:41.429222Z",
     "iopub.status.idle": "2021-06-19T13:46:41.572234Z",
     "shell.execute_reply": "2021-06-19T13:46:41.571235Z",
     "shell.execute_reply.started": "2021-06-19T13:46:41.429222Z"
    },
    "id": "5a98c5bd-a334-447c-89ea-b7080052ceb2",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n",
      "(105.01 ms) == (0m:0s)\n",
      "-----\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "def mulai_hitung_waktu():\n",
    "    global waktu_mulai\n",
    "    waktu_mulai = time.time()\n",
    "\n",
    "def cetak_lama_waktu():\n",
    "    global waktu_mulai\n",
    "    hasil_detik = abs(waktu_mulai - time.time())\n",
    "    hasil_milidetik = hasil_detik * 1000\n",
    "    \n",
    "    menit = hasil_detik / 60\n",
    "    detik = hasil_detik % 60\n",
    "    \n",
    "    menit_detik = str(int(menit)) + 'm' + ':' + str(int(detik)) + 's'\n",
    "    \n",
    "    print('-----\\n(%.2f ms) == (%s)\\n-----' % (hasil_milidetik, menit_detik))\n",
    "    del waktu_mulai\n",
    "\n",
    "\n",
    "# cara pakai\n",
    "# ----------\n",
    "mulai_hitung_waktu()  ###\n",
    "\n",
    "time.sleep(100/1000)  # time-consuming computing here\n",
    "\n",
    "cetak_lama_waktu()  ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f09d7019-49c7-42a9-8531-2a4d3da4c9e0",
   "metadata": {
    "id": "d78a19da-d950-4997-9909-955a088a8e1b"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f1dedcc4-0c5d-4501-8cc9-01b3567fc5be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:41.575233Z",
     "iopub.status.busy": "2021-06-19T13:46:41.574233Z",
     "iopub.status.idle": "2021-06-19T13:46:42.567311Z",
     "shell.execute_reply": "2021-06-19T13:46:42.566307Z",
     "shell.execute_reply.started": "2021-06-19T13:46:41.575233Z"
    },
    "id": "530dc6d4-fdac-4a36-bcc1-829b7788007f",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available: 1 Physical GPUs, 1 Logical GPU\n"
     ]
    }
   ],
   "source": [
    "gpus = tf.config.list_physical_devices('GPU')\n",
    "\n",
    "if gpus:\n",
    "    # Restrict TensorFlow to only use the first GPU\n",
    "    try:\n",
    "        tf.config.experimental.set_visible_devices(gpus[0], 'GPU')\n",
    "        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        print(\"Num GPUs Available: %d Physical GPUs, %d Logical GPU\" % (len(gpus), len(logical_gpus)))\n",
    "    except RuntimeError as e:\n",
    "        # Visible devices must be set before GPUs have been initialized\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bd2a7f7-6e68-4659-8c82-140832e873c8",
   "metadata": {
    "id": "n_eonvAiKs1J"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9fcc14ba-0eb8-46f6-9d44-912e8b8f35fa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:42.569310Z",
     "iopub.status.busy": "2021-06-19T13:46:42.569310Z",
     "iopub.status.idle": "2021-06-19T13:46:42.694316Z",
     "shell.execute_reply": "2021-06-19T13:46:42.694316Z",
     "shell.execute_reply.started": "2021-06-19T13:46:42.569310Z"
    },
    "id": "6187d76c-a4e2-4d62-b29f-b2e57ae5b86f",
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_checkpoints\n",
      "success\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_weights and models\n",
      "success\n",
      "\n",
      "current work dir:\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi\n",
      "\n",
      "listdir WORK_DIR:\n",
      "['.ipynb_checkpoints',\n",
      " '.~imgaug seed.ipynb',\n",
      " 'bulk image resizing.ipynb',\n",
      " 'imgaug seed.ipynb',\n",
      " 'imgaug seed_edit.ipynb',\n",
      " 'imgaug.ipynb',\n",
      " 'imgaug.jupyterlab-workspace',\n",
      " 'LeNet-5.ipynb',\n",
      " 'MobileNetV2.ipynb',\n",
      " 'MobileNetV2_drive.ipynb',\n",
      " 'MobileNetV2_drive_consecutive.ipynb',\n",
      " 'model.png',\n",
      " 'Testing the Model -d -c.ipynb',\n",
      " '_checkpoints',\n",
      " '_weights and models']\n"
     ]
    }
   ],
   "source": [
    "if am_I_using_colab:\n",
    "    WORK_DIR = '/content'\n",
    "    TRAIN_SET_PATH = f'{WORK_DIR}/dataset_v1'\n",
    "    # TEST_SET_PATH = f''\n",
    "\n",
    "    DRIVE_DIR = '/content/drive/MyDrive/MyNotebook/MySkripsi'\n",
    "    CHECKPOINTS_DIR = f'{WORK_DIR}/checkpoints'\n",
    "    FINAL_EPOCH_DIR = f'{WORK_DIR}/weights and models'\n",
    "\n",
    "else:\n",
    "    WORK_DIR = r'D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi'\n",
    "    TRAIN_SET_PATH = r'D:\\MaskTheFace\\datasets\\_temp\\_v1'\n",
    "    TEST_SET_PATH = r'D:\\MaskTheFace\\datasets\\_temp\\_v1 - test only'\n",
    "\n",
    "    CHECKPOINTS_DIR = fr'{WORK_DIR}/_checkpoints'\n",
    "    FINAL_EPOCH_DIR = fr'{WORK_DIR}/_weights and models'\n",
    "\n",
    "directories = [\n",
    "    CHECKPOINTS_DIR,\n",
    "    FINAL_EPOCH_DIR\n",
    "]\n",
    "\n",
    "def is_dir_error(directories):\n",
    "    for _dir in directories:\n",
    "        print(_dir)\n",
    "        try:\n",
    "            os.mkdir(_dir)\n",
    "            print('success')\n",
    "        except FileExistsError:\n",
    "            print('!!! exist')\n",
    "            continue\n",
    "        except:\n",
    "            print('~~~ error')\n",
    "            return True\n",
    "    print()\n",
    "    return None\n",
    "\n",
    "\n",
    "is_dir_error(directories)\n",
    "\n",
    "# Set working directory\n",
    "os.chdir(WORK_DIR)\n",
    "\n",
    "print('current work dir:')\n",
    "print(os.getcwd())\n",
    "print()\n",
    "print('listdir WORK_DIR:')\n",
    "pprint(os.listdir('./'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6133ce35-953d-4be7-923a-2900946d2887",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c1893073-7594-4ccf-b066-c8de6cbe18df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:42.696317Z",
     "iopub.status.busy": "2021-06-19T13:46:42.696317Z",
     "iopub.status.idle": "2021-06-19T13:46:42.743323Z",
     "shell.execute_reply": "2021-06-19T13:46:42.742327Z",
     "shell.execute_reply.started": "2021-06-19T13:46:42.696317Z"
    },
    "id": "f209a709-9670-4587-9fa6-6ca058507fd2",
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 names\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'Andhika': 1260,\n",
       " 'Ardiyan': 1260,\n",
       " 'Artik': 1260,\n",
       " 'Ballya': 1260,\n",
       " 'Bina': 1260,\n",
       " 'Buyung': 1260,\n",
       " 'Kresna': 1260,\n",
       " 'Mhartian': 1260,\n",
       " 'RaihanA': 1260,\n",
       " 'Syifa': 1260,\n",
       " 'Taufik': 1260,\n",
       " 'Yandi': 1260}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max: 1260\n",
      "min: 1260\n",
      "total: 15120\n"
     ]
    }
   ],
   "source": [
    "names = os.listdir(TRAIN_SET_PATH)\n",
    "\n",
    "length_dict = {}\n",
    "for name in names:\n",
    "    samples = f'{TRAIN_SET_PATH:s}/{name:s}'\n",
    "    length_dict[name] = len(os.listdir(samples))\n",
    "\n",
    "n_min = min(length_dict.values())\n",
    "n_max = max(length_dict.values())\n",
    "n_sum = sum(length_dict.values())\n",
    "\n",
    "print(f'{len(names):d} names')\n",
    "display(length_dict)\n",
    "print(f\"max: {n_min:d}\")\n",
    "print(f\"min: {n_max:d}\")\n",
    "print(f\"total: {n_sum:d}\")\n",
    "\n",
    "# del n_min, n_max, n_sum, length_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "78af7167-189e-47f8-aabe-95cadf243600",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:42.744320Z",
     "iopub.status.busy": "2021-06-19T13:46:42.744320Z",
     "iopub.status.idle": "2021-06-19T13:46:42.759322Z",
     "shell.execute_reply": "2021-06-19T13:46:42.758321Z",
     "shell.execute_reply.started": "2021-06-19T13:46:42.744320Z"
    },
    "id": "7655a9f1-f6bf-4b82-ab75-bae523e19cc6",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train images: 9072\n",
      "validation images: 6048\n"
     ]
    }
   ],
   "source": [
    "validation_split = 0.4\n",
    "total_train = round(n_sum * (1 - validation_split))\n",
    "total_val = n_sum - total_train\n",
    "\n",
    "print(f'train images: {total_train:d}')\n",
    "print(f'validation images: {total_val:d}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d8e0ace9-357a-4225-a3c8-547a04a71ac1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:42.760320Z",
     "iopub.status.busy": "2021-06-19T13:46:42.760320Z",
     "iopub.status.idle": "2021-06-19T13:46:42.791328Z",
     "shell.execute_reply": "2021-06-19T13:46:42.790326Z",
     "shell.execute_reply.started": "2021-06-19T13:46:42.760320Z"
    }
   },
   "outputs": [],
   "source": [
    "def do_rsync():\n",
    "    if am_I_using_colab:\n",
    "        !rsync -aruvzh --no-inc-recursive --info=progress2 \"/content/checkpoints\" \"/content/drive/MyDrive/MyNotebook/MySkripsi\"\n",
    "        print()\n",
    "        !rsync -aruvzh --no-inc-recursive --info=progress2 \"/content/weights and models\" \"/content/drive/MyDrive/MyNotebook/MySkripsi\"\n",
    "    else:\n",
    "        !rsync -aruvzh --no-inc-recursive --info=progress2 \"./_checkpoints\" \"/cygdrive/d/MaskTheFace/datasets/_temp\"\n",
    "        print()\n",
    "        !rsync -aruvzh --no-inc-recursive --info=progress2 \"./_weights and models\" \"/cygdrive/d/MaskTheFace/datasets/_temp\"\n",
    "    print()\n",
    "    print('rsync operation completed')\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "358984f8-dafa-496e-b37a-e05602f5ac24",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:42.793327Z",
     "iopub.status.busy": "2021-06-19T13:46:42.793327Z",
     "iopub.status.idle": "2021-06-19T13:46:42.823325Z",
     "shell.execute_reply": "2021-06-19T13:46:42.822325Z",
     "shell.execute_reply.started": "2021-06-19T13:46:42.793327Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def new_lenet5_actual(input_size=32, batch_size=32, activation='tanh', pooling='average'):\n",
    "    # argument validation\n",
    "    if activation in ['tanh', 'relu'] and pooling in ['average', 'max']:\n",
    "        pass\n",
    "    else:\n",
    "        raise ValueError\n",
    "        \n",
    "    inputs = keras.Input(shape=(input_size, input_size, 3), name='input')\n",
    "    \n",
    "    x = keras.layers.Conv2D(6, kernel_size=(5,5), strides=1, activation=activation, padding='valid', name='C1')(inputs)  # 1st\n",
    "    x = keras.layers.AveragePooling2D(name='S2')(x) if pooling=='average' else keras.layers.MaxPooling2D(name='S2')(x)   # 2nd\n",
    "    \n",
    "    x = keras.layers.Conv2D(16, kernel_size=(5,5), strides=1, activation=activation, padding='valid', name='C3')(x)      # 3rd\n",
    "    x = keras.layers.AveragePooling2D(name='S4')(x) if pooling=='average' else keras.layers.MaxPooling2D(name='S4')(x)   # 4th\n",
    "\n",
    "    x = keras.layers.Flatten(name='flatten')(x)\n",
    "    x = keras.layers.Dense(120, activation=activation, name='C5')(x)  # 5th\n",
    "    x = keras.layers.Dense(84, activation=activation, name='F6')(x)   # 6th\n",
    "    outputs = keras.layers.Dense(12, activation='softmax', name='OUTPUT')(x)  # 7th\n",
    "        \n",
    "    name = f'lenet5_actual_'\n",
    "    name = f'{name}t' if activation=='tanh' else f'{name}r'\n",
    "    name = f'{name}a' if pooling=='average' else f'{name}m'\n",
    "    name = f'{name}-{input_size:d}-{batch_size}'\n",
    "    \n",
    "    model = keras.Model(inputs=inputs, outputs=outputs, name=name)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "57500c50-a80c-4fd8-a99d-1e1311d9be4c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:42.824330Z",
     "iopub.status.busy": "2021-06-19T13:46:42.824330Z",
     "iopub.status.idle": "2021-06-19T13:46:43.601388Z",
     "shell.execute_reply": "2021-06-19T13:46:43.600389Z",
     "shell.execute_reply.started": "2021-06-19T13:46:42.824330Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"lenet5_actual_rm-32-32\"\n",
      "__________________________________________________________________________________________\n",
      "Layer (type)                                          Output Shape           Param #  \n",
      "==========================================================================================\n",
      "input (InputLayer)                                    [(None, 32, 32, 3)]    0        \n",
      "__________________________________________________________________________________________\n",
      "C1 (Conv2D)                                           (None, 28, 28, 6)      456      \n",
      "__________________________________________________________________________________________\n",
      "S2 (MaxPooling2D)                                     (None, 14, 14, 6)      0        \n",
      "__________________________________________________________________________________________\n",
      "C3 (Conv2D)                                           (None, 10, 10, 16)     2416     \n",
      "__________________________________________________________________________________________\n",
      "S4 (MaxPooling2D)                                     (None, 5, 5, 16)       0        \n",
      "__________________________________________________________________________________________\n",
      "flatten (Flatten)                                     (None, 400)            0        \n",
      "__________________________________________________________________________________________\n",
      "C5 (Dense)                                            (None, 120)            48120    \n",
      "__________________________________________________________________________________________\n",
      "F6 (Dense)                                            (None, 84)             10164    \n",
      "__________________________________________________________________________________________\n",
      "OUTPUT (Dense)                                        (None, 12)             1020     \n",
      "==========================================================================================\n",
      "Total params: 62,176\n",
      "Trainable params: 62,176\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARkAAALPCAYAAABWn7S4AAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3dsU7rvv//8Xf/OhJEqhg6duhCx15BByauBFWi3AF3QDdGWLi0w8BQ6SzonE4gIfU/fL7hlwbbcRK/Ezt5PqQKSFrHaZtXbccpk+PxeBQAUPL/+q4AgGEjZACoImQA6Do6PD09HUWEGzdu3Ky3x8dHV4wcf4nD379/5f7+Xh4eHlx3Q6R2u538+/dvsK/f0PcvBbvdTv78+eO8D90lAKoIGQCqCBkAqggZAKqcA78Yr8lkIkOZDD6ZTL5/L+9Tvp+u+zQtP/YyTa9xm/JtCBkYaQdM1yFm2lYxYMoHsm/dbI9Ntcz892LYtEV3CaMUKuRsZbQpu+syQwaKCSEDo+IbL/99Mpl4Lbfdt3h/08+u2D69Q5c7pjJdCBn8YAuS4/H4/clnW54rv4lNB3X5ZwyaHoAaQdlVmdqtGUIGP7gCoWp5ytp8wpuCtm25XZephZABoIqQARQNYUylLUIGrfg2u22DvF0P+rpohkGo/dQoUxvzZPCDKRDKk9byn8VlpnGb8v185mpoc01Es01Os9WvfKCXA8A2nhJLmfk6zeefkMEPrjecafap7/19fu+TqR5Vg6RN5rTEVGYXgnSXYmq25adXu6pT19tDGL5nWFIZU2lz6l07gIKETBdT0H11cWrVNB8klk/jrvQ1ka4p0weBT9BovK6xlGnrHoZ+TekuoZGUQtW3Ozc2tu5haMG7S1VTzU3r6kxBL//uWzfN+vjUwVZmuQ6u+1TtDxCj1iHjMwW9al2dKeim+1fVTbs+VXXIyzdtw/bT9DhTnYHYtQ4Z3wOwj7GLGOrTpvxyS2as4z9IG2MyHbB1a4rzNUyDcCHCZLfbyW63a11OzIa+f7G7v793ridkFJm6X12Powz5X9rwL1H6l78GLp1fVuA7N8F0X40DVKs+vvMP6szXAFLUuiXjOwW9OJ5g+oT3nYJe/r3v+lSdbTKFiE/XqPw4W/2B2LUOmSZzEHyWu84mNZ33oFEf39ZKExpjNEDXuAq7Z7RKMHSdDfzG1tTvuz6MsXTHdBlIcZ2paxqi/NjLtF1W0LR8m85CJoZgKeq7Pn1vP4RQAd1F0JvKt00fqFMf11hdimUWxz1DobuEUQoVbHXH+WIsU7tVzTwZiEh189z2CWc66+eaH1SnHC22T+/Q5Y6pTBdaMrBeXyViP8NlutaqXFaTckz37VLTA7CvOVwhytRuzRAyCGII11W1+YQ3BWvbcrsuUwshA0AVIQMoGsKYSluEDGqxXbZh+r1NOX3QDINQ+6ZRpjbOLsFrkNZ0HVf+XTeua76alKP9Se2aiGabnGarT/lANw2E2x4TQ5n5Os3nm5CBiFR/ipvO/jSZz+FTTl9dAdN2qwZJmz4HsZTZBbpLGCXfMyypjKm0OfWuHUCEDIKo+sqLvuVdsiKfoNE4AGMp09Y9DP0a0l1CEDGf7fDtvo2NrXsYGi0ZAKoIGQCqCBkAuo4OT09PRxHhxo0bN+vt8fHRFSPHyXHMI1+oLbUp7egf3SUAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqfvVdAcTt6+tL3t7eTpa9vr5+/75YLOTXL95GsOM/SMLp+flZ7u7u5OLiQkT+C508VA6Hg7y8vMjNzU2fVUTk+AiC0/X1tWRZJofD4ce6LMvk6uqqh1ohJYzJwGm5XMpisTCuWywWslwuO64RUkPIoNJms5Esy06WZVkmm82mpxohJYzJoNJ+v5fLy0v5+Pj4XnZ+fi6/f/+W+XzeY82QAloyqDSfz2W1Wp0sW61WBAy8EDLwst1uZTqdiojIdDqV7Xbbc42QCrpL8PL+/i7z+Vw+Pz/l7OxM9vu9zGazvquFBNCSgZfZbCbr9VpERNbrNQEDb4QMvN3e3p78BLwclT09PR1FhBs3bhHeHh8ftSPgqD7j9+/fv3J/fy8PDw/amxqd3W4n//79G+xzO/T969tut5M/f/6ob4fuEgBVhAwAVYQMAFWEDABVfNXDSE0mEzkObB5mcZ8mk8n38jr76Xpc7GXG+poSMiOl/Wbs+g1fDpjitn3r4npcCmUej8cog4buEpIX6sByldG0/D7KLLZyYkDIjFTxjZj/PplMvJbb7lu8v+mnBlPAhAgcjRZBKmWGRsiMkC1Iik1u2/Kc68DOfy//7EOTg1AjFLssM7bWDCEzQq5AqFqekqaf8qZQbVtuH2XGgpABoIqQAWoY67hKG4QMnHyb4rZB3j6b8pphEGq/NMqMDfNkRsgUCPmbvbyuuMw0blO+n2n+huYntat803LbvhTXFcsurzMFQUxl5utjahkRMiPkO3fD9UY33d/n9y6ZtusaKG06pyWmMmMUXciUT5PW+ZQKVXZotingJqm9gWJQ57VMZUylaZmxtWJEIgsZU3Pbdj+tskOrOwU8ljdJeZ5M7HyDRmNfYikz1tcqmoHfOjM3m7RgNGaF+mgy9yGGAcA8CGN809qkVFcNse5/NCFjU+eJq3tw2rphpmn0xfLLM2JdfzcRS9AAIUQfMnW0TfJiV6Z8oNum2dum0Mf6qQJ0bVAhoylEeMTaZwY0RTXwm6piq6brINntdrLb7TrbXh+Gvn99ur+/V99G9C0Z7a8J6Eqd8Klz3/v7+5Mu3pBuDw8Pg96/vm9d/auZaELmePw52BmqVaBZto86U8fpUmFoououlcPAdrDZ5nC4DlCfsl33KW+zaiDY9Dif5QQMhiaqkBHxO8hcQdJV2baQaloeMFTRdJcADFN0LRmgKdvYV51Wo+txMZVZfrytux4DQgZWod60Xbz5ywHjO1ZnKyPmMm33jzVo6C4heSHPQoamdcDb9tl0JrVvhMxI2K6pKi8zXYtVXGd6TNNyQujq4lfTGcW+yiye4YwtUEwImRFwXZNlO0DLP22n7OuWY7qvppDdB43uSNMyba9DscxYEDLwYgqL2LUJBNd3/TQ9gEOWmdLrQMgAHspX3IdoKWiUGSNCBijROEMT41mfrhAyOFE1UOv7aas94Ouj6bwT3+vM+iwzJcyTGQGfQVrTpK787IVpLkabcroajHVdS2a7f/GnaTvFv/sqs+r1jK3VRMiMRJ3ruqoGeV1l+ZTT1QFQdx5J02vfui7Tp9yY0F1C8uoMmqYy3tK0zNhaMSKEDDy5mv0x8A2aVGb1DiVgROguwVOMb96yFOqoKdb9pyUDQBUhA0AVIQNA11HZ09PTUUS4ceMW4e3x8VE7Ao6T4zHS0SJEKdYzGIgX3SUAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqfvVdAcTt6+tL3t7eTpa9vr5+/75YLOTXL95GsOM/SMLp+flZ7u7u5OLiQkT+C508VA6Hg7y8vMjNzU2fVUTk+AiC0/X1tWRZJofD4ce6LMvk6uqqh1ohJYzJwGm5XMpisTCuWywWslwuO64RUkPIoNJms5Esy06WZVkmm82mpxohJYzJoNJ+v5fLy0v5+Pj4XnZ+fi6/f/+W+XzeY82QAloyqDSfz2W1Wp0sW61WBAy8EDLwst1uZTqdiojIdDqV7Xbbc42QCrpL8PL+/i7z+Vw+Pz/l7OxM9vu9zGazvquFBNCSgZfZbCbr9VpERNbrNQEDb4QMvN3e3p78BHw4u0uTyaTLugBIlGvUpXLGL0M2wzeZTAb9Og99//pW1RihuwRAFSEDQBUhA0AVIQNAFV/1gMaGOqCa71dxQLPuftoeG1uZIcuwIWTQmHbA9BFixYApH8i+dbE9NrYyq8oOhe4S8D+hDi6NYNQKW9M+l1txbREyaKz4Rsx/n0wmP96gtnWuv8s/y7+HVj7YtIKiuF+hAq1pmcXWkOZzS8igEVvAHI/HH2982zrTJ2j5d+0D30fI7oOpi9NnmabXpVhmCIQMGqkKBJ91KWgbCK6xlBjK7OJ1IWSAjhTDIVRLQaPM0AgZQInG2bEUpw0QMuiEzyesabDX97Fa2sw7yR8bqv4aZXaBeTJoxHb2x3a2wzbgW17umrOh+SluGzy1LSvvh2l9OQhcE/z6KLPqsaGea0IGjbjegK43ddVy19mkProJtsFsV0ui6XPTdZlV5YYSLGRMp7+aTn32uW/badC2Jz/Uk67xCWzaZ+39GBPfU8GpjLW0Pa0dSpCQcQWKbX1VMvv2OZs+GXWax3WZmrRt2Z5Dzf0IIfTkM20+QZPKjN4YAkZEaeA3VDPXFjaxv2FjeMPEcjozD8KYX6+ylOoamsa+q43J+Fa2q66Ea5Crbrl11uf3KXedbINypmAI3VoDuhQkZFwHjc9jfcp23c+3K1H3oKvq5vl0A13T712/28oDUhOsJVNuPfh8qjdherzP+I5r7MK3rCaqTtE2QfggJcG7S13MaTCpOv1nqw8H639iGL/RNPT9i5nK2aW29yuragH4nt2KvQXgM27UprtXte2hiv11T11VgAc7u1S1oRBTopu8WWyzTmNUHEuqCpg6zzfQp8r/IFmnhVI1vbnI1a3ymersOnNk4rpP1aCy6351Jsi56mOrn6vMpvtUNvRAGvr+9a3yxEyIkEEYfU2qG/rrPPT961vV88tV2ABUETIRybs/xRufwN0rzmfKb23Kqbuu6zLzx7bZVxeuwo7MEEMl5KzurrqObSdGphYwmpNAackA/xPy4NKYjKpRpm1ya8gWDSGDWkzNatff5Z/lZW3KCal8sA2xRVlWbLVpTu0gZOCtPI/HdgmJ6WAtn4ovltWknC7F3N0LdZmO75SKJggZdKrPsGhiyAEjktg34wFws13uEluZoREyQAc0Jln2MXGzCbpLCM42SGsa/G1bjrYYD9rU0JKBN9f1aVXfm2M6e2Ma8K1bTiiu7xuyXVvnc81bnev/ui6z6nrDUM8zIYNaXG882ylg15s8RDmaTNv0vSC3zvpYywyB7hLwP76nbWM9UxSqTGb8Imnak+ra6uvTP5YyNcKO7hI6lcJAagp11KKx77RkAKgiZACoImQAqKock4l1gA5hDf11Hvr+xcwZMmMeAINZrFPXES+6SwBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFRV/gdJjNvX15e8vb2dLHt9ff3+fbFYyK9fvI1gNzny7wDh8Pz8LHd3d3JxcSEi/4VOHiqHw0FeXl7k5uamzyoicnwEwen6+lqyLJPD4fBjXZZlcnV11UOtkBLGZOC0XC5lsVgY1y0WC1kulx3XCKkhZFBps9lIlmUny7Isk81m01ONkBLGZFBpv9/L5eWlfHx8fC87Pz+X379/y3w+77FmSAEtGVSaz+eyWq1Olq1WKwIGXggZeNlutzKdTkVEZDqdyna77blGSAXdJXh5f3+X+Xwun5+fcnZ2Jvv9XmazWd/VQgJoycDLbDaT9XotIiLr9ZqAgTdCBt5ub29PfgI+VLpLk8kkdJEAOqAxeqI245ehHh2TyWTQz+3Q9y9mWo0DuksAVBEyAFQRMgBUETIAVBEyIzGGM37FfZxMJt+3NuU0Wd9Vmfnj8sfG+hoTMiOhfcam7zd48axU/nt+q1O31AIm30cRqb2vXSFkkLyQp71d5TTdjkaZtsfFGDSEzEiUuxL5z/Ib0rbO9bepud7VG910sA19nk2+z027g10jZEbAFjDl7oRrnetALjbXTev7FKqVozFJsE2ZptcoF1trhpAZgapA8FmXoqEGjEharxFfJA54MHUr2x7oGmXGiJABKpQP/BBhoFFmrOgu4Qef/rxtbkZMYwFDPWhTQ8iMgOmMkevMUPHMRXk8p7y8uL48V6ULrkFO0/KqMzJNJrZ1XWbxdTC1gGJrFdFdGgHXG862zme562xSDG/yJvNIfOpt2tcYy4xFFCFj+hQtf0IW14csW4tpuyYpvVliVed1jfFMUcgyY2vFiEQQMq5Asa33fRKrytZiq3PsTdvyPJmU+AaNxn7FUmasr1uvYzI+szXbTlYKUVZddbYT08Sp4vU+KUq13qHEuv9RDvzanqwQLRNT2cVBtPLy4nrb/W2Pr1OnWIIGCC3KkDExHYQhZ3M2mWJv+wng/yQTMk0u22+7veLPpmLtJwNd6X3gN1XFgcaug2ToXauh79/YRBkymgdtl4GgdVp1yC0jWn790Qr3XrtLpu5PqDeZZtk+fGe/clBh6HpvyZTDwDSPpHz/4rqq2axVE/ls9ykP/JrmkLjmvpTrbptaTsBg6HoPGZFm096r1rW9j+8cG5/HAmOWzNklAGkiZJC0UBMiXVdsx15mueyq+3Utiu4S4hFqILqLAW3bFIK627aFQQpl2srqY2qFDS0ZJCnkARTLBY5Ny7Q9F7FcrkLIDJipae76u+qsWJtyQjJ9ao9VsTUXQ6CYEDIDZbsmy3XWrHwphc/1Wz7ldCmWLkKXTK9PLobWDCEDq9Qu/BxjwIjE//oQMgBUETIAVBEyI2cbpDUN/rYtR1vs3YaxYp7MQLmu2yoP4FbN37AN+NYtJxTXNWO268hc9XBdl1bcZmpllu/bF0JmwHyv+/I5HeyzvO/Tyk3nijTZ5xTKjAXdJSTJ99Ssxif5mMtsgpCBVYzXwRS1+fRvu93Yy4wlYEToLsEhljepSwp17ENMzwstGQCqCBkAqggZAKrUxmRiHSwcgqE/t0Pfv7FRCZmYBp0QVkxnLZAGuksAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUETIAVBEyAFQRMgBUqf0HSQzD19eXvL29nSx7fX39/n2xWMivX7yNYDc58u8A4fD8/Cx3d3dycXEhIv+FTh4qh8NBXl5e5Obmps8qInJ8BMHp+vpasiyTw+HwY12WZXJ1ddVDrZASxmTgtFwuZbFYGNctFgtZLpcd1wipIWRQabPZSJZlJ8uyLJPNZtNTjZASxmRQab/fy+XlpXx8fHwvOz8/l9+/f8t8Pu+xZkgBLRlUms/nslqtTpatVisCBl4IGXjZbrcynU5FRGQ6ncp2u+25RkgF3SV4eX9/l/l8Lp+fn3J2dib7/V5ms1nf1UICaMnAy2w2k/V6LSIi6/WagIE3Qgbebm9vT34CXo4FT09PRxHhxo0bt8a3x8fHYqwcT2b8/v37V+7v7+Xh4UEwXLvdTv79+zfY13no+xez3W4nf/78OVlGdwmAKkIGgCpCBoAqQgaAKkIGtUwmk76rEFxxnyaTyfetTTnl5SmUWS7b574++D4Z1KI9QXwymahvw7a98rbr1MUVBimUaSvreDy2fk1oyWC0QgaaRjB2Xabt+ciDpilCBrWUuxb5T5/ltvuWm+Yhm+o2pgNqzJfx5c9H0y6YCyEDb7YgOR6PP96g5eU514Gd/17+2Yeuu20xsL1mIu1aM4QMvLkCoWp5SsYYMCJ6rxkhA0AVIQNAFSEDFb79d9sgb5/zccbYVdLEPBl4MwVC8axEcV1xmWncpnw/07wPzbERV/mm5bZ9Ka+3zTMp/p1amab710HIwJvrTVbnjV5e5/N7l5rMFfHd1xTLbKtxyJRTr86ngk95Te/ju42iUE+ua/Zo0/JyxXJNaOLXV6e1pNGqGkuZjULG1Ly13c9XnfPwTXfY9qYKHQj5ttqWZ+pCaO5DCLbmeKx8g2YIM3qbCPE6Bhn4bdOCMZVlCpvY37Sh61a3vLZTv0PJgzDm16ospbp2LcRzE2xMps4FWiG7J+Xtm7pxTepoe4xveaauk22swhQObQKagwYxaRQyrgNGxB0kPgeAz8Hi252oe+C5rnD1vfrVNf2+aZlAqhq3ZMoth5ADnmV1L2YrBovrdKRPWU1UnaKti+BBylp3l2yfzkVaB0nVqTrbdjlg//tW+d1u13c1VA19/2J1f39/8nejgd+qg7s8+BdyALhYB9cgY3kcJFZ5HfNbm7NGde57f3//47Uayu3h4WHQ+xfzzfRvaBqfXerq4G3SCjKNycTKFZau1qGtHCA2rcZkmp65Md3XNEBa3kb5PqZuWXk7xcdU1aFqv1zrbNtx1ccWILZ627ZTrgsQk8Znl4o/fe5r+7vOct+y6t6nzmPq1t/nfq4WTNPtALHgKmwAqrhAsmembh+tlXjYurV1XqOmr2/b94WrRRxyikUVQiYCQwuVUG/avgezQ06abLofTR/nc2a2uA3N55ruEmDQd1C23b6rBWMbF9Q6C0tLBk5VZ9NsZ8h8r91qUo62ujPMfcprUkbTx7nKs70OmmjJwKo8h8c1JlE+41gOjfK1ZXXLMd23L3XCzvb8aT2uiu11yLep0ZohZKCqznSHFLRpTTU9iEMe/H28DoQMAFWEDABVhAyC8LmsIkQ5fWs7ANzHmaa+cXYJVj6DtKZJXaYryl1nNXzL6epgc80bcX0/keuaPNN6jceV7+O6FrDO5SxtEDJwqnrTmc7+NLnuyqecGD7N684xqdrn0I+rejwDv0Ak6pzR6bobpNHiYMYvkmX7aooU+AZN15cMpBQwInSXoCyGLk4bqdffh/Y+0pIBoIqQAaCKkAGg61jw9PR0FBFu3Lhxa3x7fHwsxspxchzDyBaCSX32KbpHdwmAKkIGgCpCBoAqQgaAKkIGgCpCBoAqQgaAKkIGgCpCBoAqQgaAKkIGgCpCBoAqQgaAKkIGgCpCBoAqQgaAKkIGgCpCBoAqQgaAKkIGgCpCBoAqQgaAKkIGgCpCBoCqX31XAHH7+vqSt7e3k2Wvr6/fvy8WC/n1i7cR7PgPknB6fn6Wu7s7ubi4EJH/QicPlcPhIC8vL3Jzc9NnFRE5PoLgdH19LVmWyeFw+LEuyzK5urrqoVZICWMycFoul7JYLIzrFouFLJfLjmuE1BAyqLTZbCTLspNlWZbJZrPpqUZICWMyqLTf7+Xy8lI+Pj6+l52fn8vv379lPp/3WDOkgJYMKs3nc1mtVifLVqsVAQMvhAy8bLdbmU6nIiIynU5lu932XCOkgu4SvLy/v8t8PpfPz085OzuT/X4vs9ms72ohAbRk4GU2m8l6vRYRkfV6TcDAGyEDb7e3tyc/AR/q3aXJZKJZPICWtEdMOpnxy7BPPyaTyaCf+6HvXxe6aATQXQKgipABoIqQAaCKkAGgiq96wA9DHVDN96s42Fl3P8sDpb6Pb/q4/LGu++dl5/sW22tHyOAH7TdpHwdCMWCK225Sl6Z1b/K4qrM/5frHGDR0lzB4oQ66puW02X5VC8a0vtxa6xshgx+Kb9D898lk8uONa1vn+rv8s/x7aKZP+rblNalv08e5ysvDJKZAMSFkcMIWMMfj8eQT0rWufCCbDvKQB35TdVsY+X7WbSk0fVwV03Nf3GYs4UPI4ERVIPisS0HbLlTTgzjkwZ/Kc0/IAFBFyABQRcigFZ+mv2mw1/exWkIMAHd9pilVzJPBCdvZn/Kgb3FMxjTgW17ump+ieeDZ5o3YlpX3w7S+fJ+mj6vzWNt8GFe5sYQZIYMTrjembZ3PctfZpD4OhibzS1z72eRxIR6bguhCps2Uby0+b4KYPjlwyncWbB9dII33TWzvxajGZOo+OV318Ytdg+Kt7nZjGpNoyzbOEiufU8ddXi4Q4rEmsQWMSGQhk4vtSbJJfa5IG6awjV1KdW0qxn2MJmSqpq+bpqq7fvo+tsn0dp+L1rqoL5CCaMZkyv1m26h6eSp1+SyH6wpb0xmTcpkuvqdru6ovkIJoQsYmxHyGcnlND9JyALru01TI+gIxiD5kRNoNLmocnCFaPE3Lrmvo3auh798QRB8yri5EjGKr75BbQLTw2uvi/RnNwG8bvqeG+z7gc6nVF2ijk/8g6bMJ2wFlm99gms5umqJdntzn+rv8WFsZVeu7qq/W5LJUDH3/utDFcxhNyCC8oT/3Q9+/LnTxHA6iuwQgXoQMYFCeANlmfMxnwqVrXepjc4QMGgv15o/tIMq7EMWJlD7XPtnKMpVtKtO2rum2Y0HIAAUhxyhClpVy0BAyEBF7M932t+naK9c1YU3K6Vo5FBhUDoOQgbOZXmQ6AG3XmjUtx3T/GNRtlWictUm1NUPIIIghf+1FDAGTsugvKwBSZBvwHSNCBgisHCZjDhgRukuowTRIW/7b9zt3qsqJyZgDIgRaMnBeE2War+H6oi3bgG+dcvr65Ld9d49tWf6YJtso/u2zzlaPFBAyEBH/f73hcxbIZ7mrnNgOJFN96pzpqbN/se17CHSXgALf8Oi6VZFqK0aEkEEgtnGWFPkEDQHjj+4Sgkj5IDCJaX9iqksTtGQAqCJkAKgiZACo6mRMZgiDgaka+nM/9P0bAvWQSX3QCqdSP9OB7tFdAqCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoKqT/yCJdH19fcnb29vJstfX1+/fF4uF/PrF2wh2kyP/DhAOz8/Pcnd3JxcXFyLyX+jkoXI4HOTl5UVubm76rCIix0cQnK6vryXLMjkcDj/WZVkmV1dXPdQKKWFMBk7L5VIWi4Vx3WKxkOVy2XGNkBpCBpU2m41kWXayLMsy2Ww2PdUIKWFMBpX2+71cXl7Kx8fH97Lz83P5/fu3zOfzHmuGFNCSQaX5fC6r1epk2Wq1ImDghZCBl+12K9PpVEREptOpbFBGViMAAA24SURBVLfbnmuEVNBdgpf393eZz+fy+fkpZ2dnst/vZTab9V0tJICWDLzMZjNZr9ciIrJerwkYeCNk4O329vbkJ+DlmKCnp6ejiHDjNqrb4+Nj34deI0nO+P3796/c39/Lw8ND31WJ2m63k3///g32eRr6/hXtdjv58+dP39VohO4SAFWEDABVhAwAVYQMAFWEDIwmk0nfVQiuuE+TyeT71rYsnzLL64b4/NokeXYJ+o7KE8Enk4n6NmzbK2+7bl1sIWIr07au6+egL7RkMHghD+aQZeVBM3SEDIzKXYv8p89y233LXYUuug6mUBhD6yEmhAx+sAXJ8Xj8/vS1Lc+5Duz89/LPPtRpmWh0b8bQmiFk8IMrEKqWp6TvgBkLBn4BT6YuIcFTjZABPJTDhIDxR3cJrfiOJ9gGefscjyAkukFLBj+YAqE4t6O4rrjMNG5Tvl/X80Vc5ZuW2/bFdzvFv33WjaFFRMjgB9ebvnyA1Lm/z+9dMm3X92yP7bF17j8WowiZ8qeI6xOseD+f8nwfg/7UaS112bIYQytGZAQhY2qi2/i+4K6gGsObRuTn/JnY+QYNARPeoAd+fWd7hnrBxzCxKpdPwEvpQImprjHVRdugQ8bGFjS2C9/qlm2bYu87Jd/2GNdyIFaD7y75cI32t/3EcXXXbFfqln93lTWmT0SkiZAp0TilahpU9t1G1QA1EDtCpgNt5ly0vYJ4t9vJbrer9ZjUDH3/cvf3931XoZFRjsl0/dUCTZiubG7i/v7+ZJB2SLeHh4dB7195X1M16JAxHaRVIdAmJHwe6xMadafqAzEbfHepHDSmEHCtd03c85nAZ9p+eY6Jac6Jqbvksy9AbAYfMiLVB6NrvS042my/7t9ttg30bdDdJQD9I2QwGqbvHG4yrmV7jKvM8roxjaeNorsEHaHOpHUxqdA22bHu9l0BYyuzj6+4iAktGQxeyIPZVk6T8k1nP4eIlgxExH6GrXyWyzUloHy2rG05IfheJAs9tGTwfSDmN9dp8vzv8s/yafim5Zjuq6nvLssYWjOEDIIwhUXstAKm7+CKDSEDQBUhAwREK+YnQga1mOZ52H5vU04XQoeB7XuAxo6QwffgY34zDdKW19nOBJnuW7eckAeoa2DVNWnOxjaZzvVNh67ndwwtH05hQ0TqXd9VNcjre+2VrZyuDjqfC1p9HuNa7rt+yGjJYPDqnCbusmUxhlaMCCGDQGK/Jsc3aAiY8OguIYgUDpiY6hhTXbTRkgGgipABoIqQAaDrmKCnp6ejiHDjNqrb4+Nj34deI5PjcUQjUGhtTGdFEAbdJQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKoIGQCqCBkAqggZAKp+9V0BxO3r60ve3t5Olr2+vn7/vlgs5Ncv3kaw4z9Iwun5+Vnu7u7k4uJCRP4LnTxUDoeDvLy8yM3NTZ9VROT4CILT9fW1ZFkmh8Phx7osy+Tq6qqHWiEljMnAablcymKxMK5bLBayXC47rhFSQ8ig0mazkSzLTpZlWSabzaanGiEljMmg0n6/l8vLS/n4+Phedn5+Lr9//5b5fN5jzZACWjKoNJ/PZbVanSxbrVYEDLwQMvCy3W5lOp2KiMh0OpXtdttzjZAKukvw8v7+LvP5XD4/P+Xs7Ez2+73MZrO+q4UE0JKBl9lsJuv1WkRE1us1AQNvhAy83d7envwEfCTbXZpMJn1XAehUoodq2jN+U33SuzKZTAb9HA19/4pS/lCluwRAFSEDQBUhA0AVIQNAFSGDH1IeZHTJ92symXzf2pRTXmYr07RuqM+xSdJnl6BD+4xNH2eF8m2Wt123LrYQsZVpW2eqy1DRksHghTyYTeU0LTsPmqEjZPCDqVlv6grY1rn+Lv8s/x5aOWDG0HKIDSGDE7aAKTbxq9aVD2TTQR7DgR9Dd2UMrRlCBieqAsFnXQq0AiaG4IoNIQNAFSEDBEIrxoyQQSs+4wmmwV7fx2oJHQbl09b4P4QMTpjOGLnODOUDl6azOOXlrrkkWmwDq65Jcy6us2OmxxefB1NLZwytHybj4YTrDW9b57PcdTapj4PMNphdFTJN5skMPUSqDDpkXG8Y0wtvOwXrU+bY30gx851d23WrYgytGJGBh4ztzeUzNbxJmWN4w+TK82Ri5xM0BIyOQYeMjUZAjOlaFJE0W24x1Tmmumgb1cCvqwXjmjbvy9Snd12BW+eqXddyIGajaMn4nDEwXTXb9tPGdAVu1TZtZ13aXj0M9GUUIVN1qlTzYLWd0vQd/3GVBaRgFCGTi+VUadX9fS409DX0YBr6/g3BqEJGW6gujO0Lj9qUNURj6jKmHKajGvjV5DsPw6cc3+0BKRh0S8ZnLkf5jFDV4KrtOhyfGaTl72Mx/V0s2zRN37U9IEaDDhnfA7HOdPkmYyxt/m6zbSAGdJcAqCJkMErlCZFNxriqHtvV9xjHjpBBY6EOnK4PwOJ4WP79xHW/a7fqsbb5UWNEyGBUujjtbdvGWIOGkIGI2K+xsv1d58ubmpYTWvngbxM2tjOD+ImQgbXp7zrzVb6+q3zANS3HdP8uNAkK0yUiPl8nMbbWDCGDIEJdVNqHpi0R11wn/J9Bz5MBtJSv1rdN0iR4CBkgmLF/W6IN3SV4c31am35vU06XCAJdtGTgvC6qPIBbNdBpG/CtU45WC6DO9zPbBq3LZRX/9jHG1g0hAxHxv2bK5yyQz3JXOV0fhE3ntFTVc2xhYkN3CaPiewpZo8UxxlaMCCGDQGzjLDEK0Uqpa6wBI0J3CYGkdgDF0CUbC1oyAFQRMgBUETIAVCU9JpPCIGPfhv4cDX3/hiDZkBnzQFqfxnyWBM3QXQKgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaCKkAGgipABoIqQAaAq2X/uhm58fX3J29vbybLX19fv3xeLhfz6xdsIdpMj/w4QDs/Pz3J3dycXFxci8l/o5KFyOBzk5eVFbm5u+qwiIsdHEJyur68lyzI5HA4/1mVZJldXVz3UCilhTAZOy+VSFouFcd1isZDlctlxjZAaQgaVNpuNZFl2sizLMtlsNj3VCClhTAaV9vu9XF5eysfHx/ey8/Nz+f37t8zn8x5rhhTQkkGl+Xwuq9XqZNlqtSJg4IWQgZftdivT6VRERKbTqWy3255rhFTQXYKX9/d3mc/n8vn5KWdnZ7Lf72U2m/VdLSSAlgy8zGYzWa/XIiKyXq8JGHgjZODt9vb25Cfg5dixp6eno4hw48ath9vj42PXh/yx8xm/f//+lfv7e3l4eOh60whot9vJv3//Bvs6DnH/drud/Pnzp/Pt0l0CoIqQAaCKkAGgipABoIqQQecmk0nfVQiuuE+TyeT7VreMqseVt5MCQgadOypPMu/64JtMJt/7lP+e33zr4vO48rI65feJkAFaKAZMH9tJIWgIGXTO1OQvdxNsy233Ld7f9FOD6cBvGjjFsOgquLpCyKBTtiApdhNsy3OuAzv/vfyzD3XDIt/P8mOqyom9NUPIoFOuQKhanpImrZH8MeUgTvl5ECFkgCgUw8QUNKYuYSr4bwVAxOp2nWJESwbRqnP613T/Pj/xUwsCTYQMOmU6a2T7WRwINo3blNcV72Ma3wjNVb5tnovt/sX9qdtaib11Q3cJnao6S5IrBo3P/X1+71KTOS0+dY05TGyiDxnTLEfbetObNF9e9eK6PmFCb8vFpx6Ih+20s4lGiyP2VoxI5N2l8lRr0wFc5zRoXkb59/L9bdsLtS0Xn3oMXWpnUXxfnzEGjEjEIRN6GrXtxdB4kTS2NaagKYZrKvqoayrPT7QhM3RNg9I2xd53Sr7tMa7lQBvRj8kMVYhPoXJrzzZT1PW7q6xUPikRN0KmpGqgOTa2y/+bnKmgBQMNhExJ7KFSVre+xS5X2yuId7ud7Ha7Wo9JzdD27/7+vvNtMiaTkFBdmFBnrO7v73+cBRvK7eHhYXD719e/d4k2ZGwHgWv2Z9X9tOrVxfiFzzZ8QqPuVH2grai7S6YD2nagle9ru5/ti4HqfGFQ2225tmObI2K6r6ke5W2a6mDqLvnuE1BX1CEjUu/N7nNfV0jV0WZbmvWo+3ebbQM+ou0uARgGQgYIwPUlU03LKS+3TbqMHSGDKIU6gLo4EE2THfNbne27AsZUZt3y+0LIAC2EPLPYZJwuhaCJfuAX6av6iozyWa/i+nILwXRmrEk5IdimU+AULRmocnUfbAdo+Wf5NHzTckz31dTV9V+xt2YIGUTPFBax0wqYFC9cJWQAqCJkgESk2IoRIWQQEdMlFU3mhVSV04XQYWD7HqAUEDJQlQ9K5jfbxa3FdbYzQab71i0n5AHqGnB1fSOhje26NZ9vOoy5hcMpbKjzueC0/HvTOSNV5XR1MPpc0OrzGNfyVNCSAVqoc/pYo8UReytGhJBBAmzdiFj4Bs0YA0aE7hISkMKB1EcdU3heRGjJAFBGyABQRcgA0HXs2NPT01FEuHHj1sPt8fGx60P+ODkeExk9ApAkuksAVBEyAFQRMgBU/X8JgGoJTNfIbwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test = new_lenet5_actual(input_size=32, batch_size=32, activation='relu', pooling='max')\n",
    "test.summary(line_length=90, positions=[.60, .86, .96, 1.])\n",
    "\n",
    "model_viz = tf.keras.utils.plot_model(\n",
    "    test,\n",
    "    # to_file=saved_model_name + '.png',\n",
    "    show_shapes=True,\n",
    "    show_layer_names=True,\n",
    "    rankdir='TB',  # 'TB' (top-bottom)  'LR' (left-right)\n",
    "    expand_nested=True,\n",
    "    dpi=72\n",
    "    )\n",
    "\n",
    "display(model_viz)\n",
    "\n",
    "del test, model_viz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a6579b3-30a6-4a3e-8f33-417cbcb4dae9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "40666579-e240-4c5f-be42-7154000cb4c1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:43.602392Z",
     "iopub.status.busy": "2021-06-19T13:46:43.602392Z",
     "iopub.status.idle": "2021-06-19T13:46:43.631390Z",
     "shell.execute_reply": "2021-06-19T13:46:43.630392Z",
     "shell.execute_reply.started": "2021-06-19T13:46:43.602392Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def consecutiveModelTraining(\n",
    "    input_size,  # [32, 64, 96, 128] or divisible by 32\n",
    "    batch_size,  # [16, 32, 64] or power of 2\n",
    "    activation,  # ['tanh', 'relu']\n",
    "    pooling):    # ['average', 'max']\n",
    "    \n",
    "    \n",
    "    ### create train/val generator\n",
    "    image_generator = keras.preprocessing.image.ImageDataGenerator(\n",
    "        rescale=1./255,\n",
    "        rotation_range=5,\n",
    "        width_shift_range=1.05,\n",
    "        height_shift_range=1.05,\n",
    "        brightness_range=(0.75, 1.25),\n",
    "        shear_range = 0.075,\n",
    "        zoom_range=0.2,\n",
    "        fill_mode = 'nearest',\n",
    "        horizontal_flip=True,\n",
    "        # vertical_flip=True,\n",
    "        validation_split=validation_split,\n",
    "        # preprocessing_function=seq.augment_image\n",
    "    )\n",
    "    train_data_gen = image_generator.flow_from_directory(\n",
    "        seed=1728,\n",
    "        subset='training',\n",
    "        batch_size=batch_size,\n",
    "        directory=TRAIN_SET_PATH,\n",
    "        shuffle=True,\n",
    "        target_size=(input_size, input_size),\n",
    "        interpolation='bicubic',\n",
    "        class_mode='categorical')\n",
    "\n",
    "    val_data_gen = image_generator.flow_from_directory(\n",
    "        seed=1728,\n",
    "        subset='validation',\n",
    "        batch_size=batch_size,\n",
    "        directory=TRAIN_SET_PATH,\n",
    "        shuffle=True,\n",
    "        target_size=(input_size, input_size),\n",
    "        interpolation='bicubic',\n",
    "        class_mode='categorical')\n",
    "    \n",
    "    \n",
    "    ### create model\n",
    "    model = new_lenet5_actual(input_size=input_size, batch_size=batch_size, activation=activation, pooling=pooling)\n",
    "    print()\n",
    "    print()\n",
    "    \n",
    "    ### create save dir for model callbacks and post-train\n",
    "    dir1 = f'{CHECKPOINTS_DIR}/{model.name}'\n",
    "    dir2 = f'{FINAL_EPOCH_DIR}/{model.name}'\n",
    "    if is_dir_error([dir1, dir2]):\n",
    "        return\n",
    "    \n",
    "\n",
    "    ### model optimizer, loss fn, metrics\n",
    "    # Optimizer's scheduler: ExponentialDecay\n",
    "    lr_schedule = keras.optimizers.schedules.ExponentialDecay(\n",
    "        initial_learning_rate=1e-2,\n",
    "        decay_steps=10000,\n",
    "        decay_rate=0.9)\n",
    "    \n",
    "    optimizer = keras.optimizers.Adam(learning_rate=lr_schedule)\n",
    "    loss_fn = 'categorical_crossentropy'\n",
    "    metrics=[\n",
    "        'categorical_accuracy',\n",
    "        keras.metrics.TopKCategoricalAccuracy(k=3, name='top-3'),\n",
    "        keras.metrics.AUC(name='ROC-AUC', curve='ROC'),\n",
    "        keras.metrics.AUC(name='PR-AUC', curve='PR'),\n",
    "        keras.metrics.Precision(name='precision'),\n",
    "        keras.metrics.Recall(name='recall'),\n",
    "        keras.metrics.TruePositives(name='TP'),\n",
    "        keras.metrics.TrueNegatives(name='TN'),\n",
    "        keras.metrics.FalsePositives(name='FP'),\n",
    "        keras.metrics.FalseNegatives(name='FN'),]\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss=loss_fn,\n",
    "        metrics=metrics)\n",
    "    \n",
    "    model.summary(line_length=90, positions=[.60, .86, .96, 1.])\n",
    "    print()\n",
    "    print()\n",
    "    \n",
    "    \n",
    "    ###\n",
    "    params = {\n",
    "        'total_train' : total_train,\n",
    "        'total_val' : total_val,\n",
    "        'batch_size' : batch_size,\n",
    "        'steps_per_epoch' : total_train//batch_size,\n",
    "        'validation_steps' : total_val//batch_size,\n",
    "        'epochs' : epochs}\n",
    "\n",
    "    pprint(params)\n",
    "    print()\n",
    "    print()\n",
    "    \n",
    "    \n",
    "    ### model callbacks save dir\n",
    "    if am_I_using_colab:\n",
    "        epoch5_filepath = '%s/%s/%s.weights.{epoch:03d}.hdf5' % (CHECKPOINTS_DIR, model.name, model.name)\n",
    "        monitor_filepath = '%s/%s/%s.weights.{epoch:03d}_{val_categorical_accuracy:.4f}_{val_loss:.4f}.hdf5' % (CHECKPOINTS_DIR, model.name, model.name)\n",
    "        tensorboard_dir = '%s/%s/%s.tensorboard' % (CHECKPOINTS_DIR, model.name, model.name)\n",
    "    else:\n",
    "        epoch5_filepath = '%s/%s/%s.weights.{epoch:03d}.hdf5' % (CHECKPOINTS_DIR, model.name, model.name)\n",
    "        monitor_filepath = '%s/%s/%s.weights.{epoch:03d}_{val_categorical_accuracy:.4f}_{val_loss:.4f}.hdf5' % (CHECKPOINTS_DIR, model.name, model.name)\n",
    "        tensorboard_dir = '%s/%s/%s.tensorboard' % (CHECKPOINTS_DIR, model.name, model.name)\n",
    "\n",
    "    # Model's callback: ModelCheckpoint\n",
    "    epoch_checkpoint = keras.callbacks.ModelCheckpoint(\n",
    "        filepath=epoch5_filepath,\n",
    "        monitor='val_loss',\n",
    "        save_weights_only=True,\n",
    "        save_freq=int(5 * params['steps_per_epoch'])\n",
    "        )\n",
    "\n",
    "    acc_checkpoint = keras.callbacks.ModelCheckpoint(\n",
    "        filepath=monitor_filepath,\n",
    "        monitor='val_categorical_accuracy',\n",
    "        # verbose=1,\n",
    "        save_best_only=True,\n",
    "        mode='auto',\n",
    "        save_weights_only=True,\n",
    "        save_freq='epoch'\n",
    "        )\n",
    "\n",
    "    loss_checkpoint = keras.callbacks.ModelCheckpoint(\n",
    "        filepath=monitor_filepath,\n",
    "        monitor='val_loss',\n",
    "        # verbose=1,\n",
    "        save_best_only=True,\n",
    "        mode='auto',\n",
    "        save_weights_only=True,\n",
    "        save_freq='epoch'\n",
    "        )\n",
    "\n",
    "    # initialize tqdm callback with default parameters\n",
    "    # tqdm_callback = tfa.callbacks.TQDMProgressBar(\n",
    "    #     metrics_separator=', '\n",
    "    #     )\n",
    "\n",
    "    tensorboard_callback = keras.callbacks.TensorBoard(\n",
    "        log_dir=tensorboard_dir,\n",
    "        write_graph=True,\n",
    "        write_images=True\n",
    "        )\n",
    "\n",
    "    callbacks = [\n",
    "        epoch_checkpoint,\n",
    "        acc_checkpoint,\n",
    "        loss_checkpoint,\n",
    "        # tqdm_callback,\n",
    "        tensorboard_callback\n",
    "        ]\n",
    "    \n",
    "    \n",
    "    ### model.fit() verbosity\n",
    "    if am_I_using_colab:\n",
    "        verbosity = 2  # end of epoch\n",
    "    else:\n",
    "        verbosity = 1  # every step in epoch\n",
    "        \n",
    "        \n",
    "    ### model.fit()\n",
    "    print(model.name)\n",
    "    mulai_hitung_waktu()  ###\n",
    "\n",
    "    training_history = model.fit(\n",
    "        train_data_gen,\n",
    "        shuffle=True,\n",
    "        verbose=verbosity,  # 0 nope, 1 realtime, 2 epoch end\n",
    "        steps_per_epoch=total_train // batch_size,\n",
    "        epochs=epochs,\n",
    "        validation_data=val_data_gen,\n",
    "        validation_steps=total_val // batch_size,\n",
    "        callbacks=callbacks,\n",
    "        workers=0\n",
    "        )\n",
    "\n",
    "    cetak_lama_waktu()  ###\n",
    "    print()\n",
    "    print()\n",
    "    \n",
    "    \n",
    "    ### post-train save\n",
    "    history_df = pd.DataFrame(training_history.history)\n",
    "\n",
    "    history_savepath = f'{FINAL_EPOCH_DIR}/{model.name}/history.{model.name}.csv'\n",
    "    weights_savepath = f'{FINAL_EPOCH_DIR}/{model.name}/weights.{model.name}.h5'\n",
    "    model_savepath = f'{FINAL_EPOCH_DIR}/{model.name}/model.{model.name}.h5'\n",
    "\n",
    "\n",
    "    # save to csv: \n",
    "    with open(history_savepath, mode='w') as f:\n",
    "        history_df.to_csv(\n",
    "            f,\n",
    "            header=True,\n",
    "            index=False\n",
    "            )\n",
    "\n",
    "    model.save_weights(\n",
    "        filepath=weights_savepath,\n",
    "        overwrite=True,\n",
    "        save_format='h5'\n",
    "        )\n",
    "\n",
    "    model.save(\n",
    "        filepath=model_savepath,\n",
    "        overwrite=True,\n",
    "        include_optimizer=True,\n",
    "        save_format='h5'\n",
    "        )\n",
    "    \n",
    "    \n",
    "    ### rechecking local save dir\n",
    "    _checkpoints_listdir = os.listdir(f'{CHECKPOINTS_DIR}/{model.name}')\n",
    "    print(len(_checkpoints_listdir))\n",
    "    pprint(_checkpoints_listdir)\n",
    "    \n",
    "    _final_epoch_listdir = os.listdir(f'{FINAL_EPOCH_DIR}/{model.name}')\n",
    "    print(len(_final_epoch_listdir))\n",
    "    pprint(_final_epoch_listdir)\n",
    "    \n",
    "        \n",
    "    print()\n",
    "    if _checkpoints_listdir and _final_epoch_listdir:\n",
    "        print('all process done, please recheck before terminating runtime session')\n",
    "    else:\n",
    "        print('WARNING : there is empty directory')\n",
    "    print()\n",
    "    print('''don't forget to save the model.fit() verbose output''')\n",
    "    \n",
    "    \n",
    "    keyvalpairs = {\n",
    "        'name' : model.name,\n",
    "        'history_df' : history_df,\n",
    "        'model' : model,\n",
    "        'checkpoints_listdir' : _checkpoints_listdir,\n",
    "        'final_epoch_listdir' : _final_epoch_listdir\n",
    "    }\n",
    "    \n",
    "    print()\n",
    "    print()\n",
    "    print('Returned values using %d bytes of memory now' % (sys.getsizeof(keyvalpairs),))\n",
    "    return keyvalpairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "974937c4-085d-4542-8d3c-6b54b04a6ae3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:46:43.632390Z",
     "iopub.status.busy": "2021-06-19T13:46:43.632390Z",
     "iopub.status.idle": "2021-06-19T13:46:43.743398Z",
     "shell.execute_reply": "2021-06-19T13:46:43.742395Z",
     "shell.execute_reply.started": "2021-06-19T13:46:43.632390Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# global variable\n",
    "epochs = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c5de896b-a530-42c2-96f2-a2eae0809eb2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T11:33:53.480659Z",
     "iopub.status.busy": "2021-06-19T11:33:53.479659Z",
     "iopub.status.idle": "2021-06-19T11:33:53.541664Z",
     "shell.execute_reply": "2021-06-19T11:33:53.540663Z",
     "shell.execute_reply.started": "2021-06-19T11:33:53.480659Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# input_size,  # [32, 64, 96, 128] or divisible by 32\n",
    "# batch_size,  # [16, 32, 64] or power of 2\n",
    "# activation,  # ['tanh', 'relu']\n",
    "# pooling      # ['average', 'max']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "effcb99a-d923-4655-a4da-c254ef69f394",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1a420494-503e-40b4-bb41-c516abd69dcd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T11:33:53.579666Z",
     "iopub.status.busy": "2021-06-19T11:33:53.579666Z",
     "iopub.status.idle": "2021-06-19T13:35:00.551785Z",
     "shell.execute_reply": "2021-06-19T13:35:00.550782Z",
     "shell.execute_reply.started": "2021-06-19T11:33:53.579666Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 9072 images belonging to 12 classes.\n",
      "Found 6048 images belonging to 12 classes.\n",
      "\n",
      "\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_checkpoints/lenet5_actual_rm-64-32\n",
      "success\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_weights and models/lenet5_actual_rm-64-32\n",
      "success\n",
      "\n",
      "Model: \"lenet5_actual_rm-64-32\"\n",
      "__________________________________________________________________________________________\n",
      "Layer (type)                                          Output Shape           Param #  \n",
      "==========================================================================================\n",
      "input (InputLayer)                                    [(None, 64, 64, 3)]    0        \n",
      "__________________________________________________________________________________________\n",
      "C1 (Conv2D)                                           (None, 60, 60, 6)      456      \n",
      "__________________________________________________________________________________________\n",
      "S2 (MaxPooling2D)                                     (None, 30, 30, 6)      0        \n",
      "__________________________________________________________________________________________\n",
      "C3 (Conv2D)                                           (None, 26, 26, 16)     2416     \n",
      "__________________________________________________________________________________________\n",
      "S4 (MaxPooling2D)                                     (None, 13, 13, 16)     0        \n",
      "__________________________________________________________________________________________\n",
      "flatten (Flatten)                                     (None, 2704)           0        \n",
      "__________________________________________________________________________________________\n",
      "C5 (Dense)                                            (None, 120)            324600   \n",
      "__________________________________________________________________________________________\n",
      "F6 (Dense)                                            (None, 84)             10164    \n",
      "__________________________________________________________________________________________\n",
      "OUTPUT (Dense)                                        (None, 12)             1020     \n",
      "==========================================================================================\n",
      "Total params: 338,656\n",
      "Trainable params: 338,656\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________\n",
      "\n",
      "\n",
      "{'batch_size': 32,\n",
      " 'epochs': 50,\n",
      " 'steps_per_epoch': 283,\n",
      " 'total_train': 9072,\n",
      " 'total_val': 6048,\n",
      " 'validation_steps': 189}\n",
      "\n",
      "\n",
      "lenet5_actual_rm-64-32\n",
      "Epoch 1/50\n",
      "  1/283 [..............................] - ETA: 0s - loss: 2.5325 - categorical_accuracy: 0.0312 - top-3: 0.1562 - ROC-AUC: 0.4474 - PR-AUC: 0.0693 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 352.0000 - FP: 0.0000e+00 - FN: 32.0000WARNING:tensorflow:From X:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\summary_ops_v2.py:1277: stop (from tensorflow.python.eager.profiler) is deprecated and will be removed after 2020-07-01.\n",
      "Instructions for updating:\n",
      "use `tf.profiler.experimental.stop` instead.\n",
      "283/283 [==============================] - 474s 2s/step - loss: 2.4890 - categorical_accuracy: 0.0783 - top-3: 0.2429 - ROC-AUC: 0.4918 - PR-AUC: 0.0816 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99435.0000 - FP: 5.0000 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 2/50\n",
      "283/283 [==============================] - 141s 499ms/step - loss: 2.4864 - categorical_accuracy: 0.0875 - top-3: 0.2452 - ROC-AUC: 0.4989 - PR-AUC: 0.0834 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4864 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 3/50\n",
      "283/283 [==============================] - 139s 490ms/step - loss: 2.4873 - categorical_accuracy: 0.0743 - top-3: 0.2375 - ROC-AUC: 0.4898 - PR-AUC: 0.0808 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 4/50\n",
      "283/283 [==============================] - 137s 483ms/step - loss: 2.4868 - categorical_accuracy: 0.0773 - top-3: 0.2394 - ROC-AUC: 0.4933 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 5/50\n",
      "283/283 [==============================] - 138s 487ms/step - loss: 2.4868 - categorical_accuracy: 0.0824 - top-3: 0.2424 - ROC-AUC: 0.4947 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 6/50\n",
      "283/283 [==============================] - 138s 488ms/step - loss: 2.4872 - categorical_accuracy: 0.0801 - top-3: 0.2389 - ROC-AUC: 0.4927 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4851 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 7/50\n",
      "283/283 [==============================] - 134s 475ms/step - loss: 2.4869 - categorical_accuracy: 0.0778 - top-3: 0.2431 - ROC-AUC: 0.4912 - PR-AUC: 0.0816 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 8/50\n",
      "283/283 [==============================] - 134s 475ms/step - loss: 2.4869 - categorical_accuracy: 0.0806 - top-3: 0.2435 - ROC-AUC: 0.4932 - PR-AUC: 0.0817 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 9/50\n",
      "283/283 [==============================] - 135s 476ms/step - loss: 2.4865 - categorical_accuracy: 0.0796 - top-3: 0.2435 - ROC-AUC: 0.4956 - PR-AUC: 0.0826 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 10/50\n",
      "283/283 [==============================] - 136s 482ms/step - loss: 2.4868 - categorical_accuracy: 0.0817 - top-3: 0.2441 - ROC-AUC: 0.4957 - PR-AUC: 0.0824 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 11/50\n",
      "283/283 [==============================] - 139s 490ms/step - loss: 2.4870 - categorical_accuracy: 0.0821 - top-3: 0.2425 - ROC-AUC: 0.4943 - PR-AUC: 0.0820 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4851 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 12/50\n",
      "283/283 [==============================] - 137s 483ms/step - loss: 2.4866 - categorical_accuracy: 0.0783 - top-3: 0.2424 - ROC-AUC: 0.4955 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 13/50\n",
      "283/283 [==============================] - 136s 481ms/step - loss: 2.4869 - categorical_accuracy: 0.0812 - top-3: 0.2481 - ROC-AUC: 0.4944 - PR-AUC: 0.0820 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99616.0000 - FP: 0.0000e+00 - FN: 9056.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 14/50\n",
      "283/283 [==============================] - 137s 485ms/step - loss: 2.4867 - categorical_accuracy: 0.0836 - top-3: 0.2482 - ROC-AUC: 0.4943 - PR-AUC: 0.0822 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 15/50\n",
      "283/283 [==============================] - 140s 493ms/step - loss: 2.4866 - categorical_accuracy: 0.0821 - top-3: 0.2475 - ROC-AUC: 0.4974 - PR-AUC: 0.0826 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 16/50\n",
      "283/283 [==============================] - 135s 476ms/step - loss: 2.4866 - categorical_accuracy: 0.0794 - top-3: 0.2438 - ROC-AUC: 0.4945 - PR-AUC: 0.0822 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 17/50\n",
      "283/283 [==============================] - 135s 478ms/step - loss: 2.4866 - categorical_accuracy: 0.0741 - top-3: 0.2429 - ROC-AUC: 0.4948 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 18/50\n",
      "283/283 [==============================] - 136s 482ms/step - loss: 2.4869 - categorical_accuracy: 0.0782 - top-3: 0.2392 - ROC-AUC: 0.4922 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 19/50\n",
      "283/283 [==============================] - 136s 479ms/step - loss: 2.4863 - categorical_accuracy: 0.0806 - top-3: 0.2510 - ROC-AUC: 0.4995 - PR-AUC: 0.0831 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4865 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 20/50\n",
      "283/283 [==============================] - 138s 488ms/step - loss: 2.4869 - categorical_accuracy: 0.0785 - top-3: 0.2459 - ROC-AUC: 0.4925 - PR-AUC: 0.0817 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 21/50\n",
      "283/283 [==============================] - 137s 485ms/step - loss: 2.4868 - categorical_accuracy: 0.0823 - top-3: 0.2402 - ROC-AUC: 0.4931 - PR-AUC: 0.0811 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 22/50\n",
      "283/283 [==============================] - 137s 483ms/step - loss: 2.4870 - categorical_accuracy: 0.0793 - top-3: 0.2408 - ROC-AUC: 0.4910 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 23/50\n",
      "283/283 [==============================] - 136s 479ms/step - loss: 2.4868 - categorical_accuracy: 0.0785 - top-3: 0.2445 - ROC-AUC: 0.4952 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4852 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 24/50\n",
      "283/283 [==============================] - 144s 509ms/step - loss: 2.4861 - categorical_accuracy: 0.0823 - top-3: 0.2520 - ROC-AUC: 0.5004 - PR-AUC: 0.0831 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4869 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 25/50\n",
      "283/283 [==============================] - 139s 489ms/step - loss: 2.4867 - categorical_accuracy: 0.0799 - top-3: 0.2450 - ROC-AUC: 0.4934 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 26/50\n",
      "283/283 [==============================] - 138s 487ms/step - loss: 2.4868 - categorical_accuracy: 0.0782 - top-3: 0.2457 - ROC-AUC: 0.4924 - PR-AUC: 0.0816 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 27/50\n",
      "283/283 [==============================] - 137s 485ms/step - loss: 2.4864 - categorical_accuracy: 0.0830 - top-3: 0.2537 - ROC-AUC: 0.4970 - PR-AUC: 0.0830 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 28/50\n",
      "283/283 [==============================] - 140s 495ms/step - loss: 2.4866 - categorical_accuracy: 0.0808 - top-3: 0.2446 - ROC-AUC: 0.4974 - PR-AUC: 0.0824 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 29/50\n",
      "283/283 [==============================] - 137s 483ms/step - loss: 2.4866 - categorical_accuracy: 0.0841 - top-3: 0.2431 - ROC-AUC: 0.4959 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 30/50\n",
      "283/283 [==============================] - 138s 488ms/step - loss: 2.4866 - categorical_accuracy: 0.0842 - top-3: 0.2553 - ROC-AUC: 0.4979 - PR-AUC: 0.0832 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 31/50\n",
      "283/283 [==============================] - 140s 494ms/step - loss: 2.4867 - categorical_accuracy: 0.0843 - top-3: 0.2423 - ROC-AUC: 0.4956 - PR-AUC: 0.0823 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4852 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 32/50\n",
      "283/283 [==============================] - 142s 501ms/step - loss: 2.4869 - categorical_accuracy: 0.0809 - top-3: 0.2399 - ROC-AUC: 0.4927 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 33/50\n",
      "283/283 [==============================] - 140s 494ms/step - loss: 2.4869 - categorical_accuracy: 0.0799 - top-3: 0.2437 - ROC-AUC: 0.4947 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 34/50\n",
      "283/283 [==============================] - 138s 486ms/step - loss: 2.4863 - categorical_accuracy: 0.0815 - top-3: 0.2439 - ROC-AUC: 0.4963 - PR-AUC: 0.0825 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4864 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 35/50\n",
      "283/283 [==============================] - 139s 492ms/step - loss: 2.4866 - categorical_accuracy: 0.0813 - top-3: 0.2454 - ROC-AUC: 0.4956 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 36/50\n",
      "283/283 [==============================] - 137s 483ms/step - loss: 2.4867 - categorical_accuracy: 0.0814 - top-3: 0.2438 - ROC-AUC: 0.4921 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 37/50\n",
      "283/283 [==============================] - 141s 497ms/step - loss: 2.4866 - categorical_accuracy: 0.0851 - top-3: 0.2455 - ROC-AUC: 0.4949 - PR-AUC: 0.0825 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 38/50\n",
      "283/283 [==============================] - 136s 479ms/step - loss: 2.4866 - categorical_accuracy: 0.0769 - top-3: 0.2393 - ROC-AUC: 0.4955 - PR-AUC: 0.0820 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 39/50\n",
      "283/283 [==============================] - 137s 484ms/step - loss: 2.4868 - categorical_accuracy: 0.0831 - top-3: 0.2387 - ROC-AUC: 0.4911 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 40/50\n",
      "283/283 [==============================] - 137s 486ms/step - loss: 2.4868 - categorical_accuracy: 0.0784 - top-3: 0.2442 - ROC-AUC: 0.4924 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 41/50\n",
      "283/283 [==============================] - 138s 487ms/step - loss: 2.4868 - categorical_accuracy: 0.0749 - top-3: 0.2385 - ROC-AUC: 0.4946 - PR-AUC: 0.0818 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 42/50\n",
      "283/283 [==============================] - 136s 480ms/step - loss: 2.4870 - categorical_accuracy: 0.0764 - top-3: 0.2383 - ROC-AUC: 0.4915 - PR-AUC: 0.0812 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 43/50\n",
      "283/283 [==============================] - 136s 480ms/step - loss: 2.4865 - categorical_accuracy: 0.0812 - top-3: 0.2454 - ROC-AUC: 0.4951 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 44/50\n",
      "283/283 [==============================] - 136s 479ms/step - loss: 2.4868 - categorical_accuracy: 0.0764 - top-3: 0.2425 - ROC-AUC: 0.4957 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4852 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 45/50\n",
      "283/283 [==============================] - 137s 486ms/step - loss: 2.4865 - categorical_accuracy: 0.0789 - top-3: 0.2369 - ROC-AUC: 0.4915 - PR-AUC: 0.0808 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 46/50\n",
      "283/283 [==============================] - 137s 484ms/step - loss: 2.4869 - categorical_accuracy: 0.0774 - top-3: 0.2390 - ROC-AUC: 0.4917 - PR-AUC: 0.0813 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4852 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 47/50\n",
      "283/283 [==============================] - 136s 482ms/step - loss: 2.4866 - categorical_accuracy: 0.0785 - top-3: 0.2418 - ROC-AUC: 0.4925 - PR-AUC: 0.0813 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 48/50\n",
      "283/283 [==============================] - 138s 488ms/step - loss: 2.4867 - categorical_accuracy: 0.0780 - top-3: 0.2392 - ROC-AUC: 0.4940 - PR-AUC: 0.0816 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 49/50\n",
      "283/283 [==============================] - 142s 503ms/step - loss: 2.4866 - categorical_accuracy: 0.0830 - top-3: 0.2472 - ROC-AUC: 0.4960 - PR-AUC: 0.0823 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 50/50\n",
      "283/283 [==============================] - 161s 567ms/step - loss: 2.4867 - categorical_accuracy: 0.0769 - top-3: 0.2404 - ROC-AUC: 0.4935 - PR-AUC: 0.0818 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "-----\n",
      "(7263365.84 ms) == (121m:3s)\n",
      "-----\n",
      "\n",
      "\n",
      "14\n",
      "['lenet5_actual_rm-64-32.tensorboard',\n",
      " 'lenet5_actual_rm-64-32.weights.001_0.0833_2.4855.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.003_0.0833_2.4853.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.005.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.006_0.0833_2.4851.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.010.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.015.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.020.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.025.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.030.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.035.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.040.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.045.hdf5',\n",
      " 'lenet5_actual_rm-64-32.weights.050.hdf5']\n",
      "3\n",
      "['history.lenet5_actual_rm-64-32.csv',\n",
      " 'model.lenet5_actual_rm-64-32.h5',\n",
      " 'weights.lenet5_actual_rm-64-32.h5']\n",
      "\n",
      "all process done, please recheck before terminating runtime session\n",
      "\n",
      "don't forget to save the model.fit() verbose output\n",
      "\n",
      "\n",
      "Returned values using 232 bytes of memory now\n",
      "building file list ... done\n",
      "\n",
      "_checkpoints/\n",
      "_checkpoints/lenet5_actual_rm-64-32/\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.001_0.0833_2.4855.hdf5\n",
      "\n",
      "         32.77K   0%    0.00kB/s    0:00:00  \n",
      "          1.38M   7%   14.76MB/s    0:00:00 (xfr#1, to-chk=28/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.003_0.0833_2.4853.hdf5\n",
      "\n",
      "          2.76M  15%   14.77MB/s    0:00:00 (xfr#2, to-chk=27/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.005.hdf5\n",
      "\n",
      "          4.14M  22%   15.29MB/s    0:00:00 (xfr#3, to-chk=26/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.006_0.0833_2.4851.hdf5\n",
      "\n",
      "          5.52M  30%   15.75MB/s    0:00:00 (xfr#4, to-chk=25/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.010.hdf5\n",
      "\n",
      "          6.90M  38%   16.08MB/s    0:00:00 (xfr#5, to-chk=24/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.015.hdf5\n",
      "\n",
      "          8.27M  45%   15.98MB/s    0:00:00 (xfr#6, to-chk=23/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.020.hdf5\n",
      "\n",
      "          9.65M  53%   16.18MB/s    0:00:00 (xfr#7, to-chk=22/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.025.hdf5\n",
      "\n",
      "         11.03M  60%   16.34MB/s    0:00:00 (xfr#8, to-chk=21/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.030.hdf5\n",
      "\n",
      "         12.41M  68%   16.49MB/s    0:00:00 (xfr#9, to-chk=20/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.035.hdf5\n",
      "\n",
      "         13.79M  76%   16.63MB/s    0:00:00 (xfr#10, to-chk=19/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.040.hdf5\n",
      "\n",
      "         15.17M  83%   16.77MB/s    0:00:00 (xfr#11, to-chk=18/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.045.hdf5\n",
      "\n",
      "         16.55M  91%   16.77MB/s    0:00:00 (xfr#12, to-chk=17/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.weights.050.hdf5\n",
      "\n",
      "         17.70M  97%   16.85MB/s    0:00:00  \n",
      "         17.93M  98%   16.88MB/s    0:00:01 (xfr#13, to-chk=16/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/events.out.tfevents.1624102436.ELITERAIHAN.652.500.v2\n",
      "\n",
      "         17.97M  99%   16.89MB/s    0:00:01 (xfr#14, to-chk=13/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/events.out.tfevents.1624102444.ELITERAIHAN.profile-empty\n",
      "\n",
      "         17.97M  99%   16.87MB/s    0:00:01 (xfr#15, to-chk=12/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/profile/\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/profile/2021_06_19_11_34_04/\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/profile/2021_06_19_11_34_04/ELITERAIHAN.input_pipeline.pb\n",
      "\n",
      "         17.97M  99%   16.86MB/s    0:00:01 (xfr#16, to-chk=8/31) \n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/profile/2021_06_19_11_34_04/ELITERAIHAN.kernel_stats.pb\n",
      "\n",
      "         17.97M  99%   16.84MB/s    0:00:01 (xfr#17, to-chk=7/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/profile/2021_06_19_11_34_04/ELITERAIHAN.memory_profile.json.gz\n",
      "\n",
      "         17.98M  99%   16.81MB/s    0:00:01 (xfr#18, to-chk=6/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/profile/2021_06_19_11_34_04/ELITERAIHAN.overview_page.pb\n",
      "\n",
      "         17.99M  99%   16.80MB/s    0:00:01 (xfr#19, to-chk=5/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/profile/2021_06_19_11_34_04/ELITERAIHAN.tensorflow_stats.pb\n",
      "\n",
      "         18.02M  99%   16.80MB/s    0:00:01 (xfr#20, to-chk=4/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/profile/2021_06_19_11_34_04/ELITERAIHAN.trace.json.gz\n",
      "\n",
      "         18.02M  99%   16.77MB/s    0:00:01 (xfr#21, to-chk=3/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/train/plugins/profile/2021_06_19_11_34_04/ELITERAIHAN.xplane.pb\n",
      "\n",
      "         18.11M  99%   16.79MB/s    0:00:01 (xfr#22, to-chk=2/31)\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/validation/\n",
      "_checkpoints/lenet5_actual_rm-64-32/lenet5_actual_rm-64-32.tensorboard/validation/events.out.tfevents.1624102743.ELITERAIHAN.652.6469.v2\n",
      "\n",
      "         18.14M 100%   16.78MB/s    0:00:01 (xfr#23, to-chk=0/31)\n",
      "         18.14M 100%   16.78MB/s    0:00:01 (xfr#23, to-chk=0/31)\n",
      "         18.14M 100%   15.70MB/s    0:00:01 (xfr#23, to-chk=0/31)\n",
      "         18.14M 100%   15.70MB/s    0:00:01 (xfr#23, to-chk=0/31)\n",
      "\n",
      "sent 16.15M bytes  received 477 bytes  10.77M bytes/sec\n",
      "total size is 18.14M  speedup is 1.12\n",
      "building file list ... done\n",
      "_weights and models/\n",
      "_weights and models/lenet5_actual_rm-64-32/\n",
      "_weights and models/lenet5_actual_rm-64-32/history.lenet5_actual_rm-64-32.csv\n",
      "\n",
      "         11.81K   0%    0.00kB/s    0:00:00  \n",
      "         11.81K   0%    0.00kB/s    0:00:00 (xfr#1, to-chk=2/5)\n",
      "_weights and models/lenet5_actual_rm-64-32/model.lenet5_actual_rm-64-32.h5\n",
      "\n",
      "          4.14M  75%   24.29MB/s    0:00:00 (xfr#2, to-chk=1/5)\n",
      "_weights and models/lenet5_actual_rm-64-32/weights.lenet5_actual_rm-64-32.h5\n",
      "\n",
      "          5.52M 100%   21.88MB/s    0:00:00 (xfr#3, to-chk=0/5)\n",
      "          5.52M 100%   21.88MB/s    0:00:00 (xfr#3, to-chk=0/5)\n",
      "          5.52M 100%   17.62MB/s    0:00:00 (xfr#3, to-chk=0/5)\n",
      "          5.52M 100%   17.62MB/s    0:00:00 (xfr#3, to-chk=0/5)\n",
      "\n",
      "sent 3.71M bytes  received 79 bytes  7.43M bytes/sec\n",
      "total size is 5.52M  speedup is 1.49\n",
      "\n",
      "rsync operation completed\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rm_64_32 = consecutiveModelTraining(input_size=64, batch_size=32, activation='relu', pooling='max')\n",
    "do_rsync()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3c63c763-f263-419a-9681-8e80c160fa5b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:35:00.553788Z",
     "iopub.status.busy": "2021-06-19T13:35:00.553788Z",
     "iopub.status.idle": "2021-06-19T13:35:00.662791Z",
     "shell.execute_reply": "2021-06-19T13:35:00.661791Z",
     "shell.execute_reply.started": "2021-06-19T13:35:00.553788Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loss</th>\n",
       "      <th>categorical_accuracy</th>\n",
       "      <th>top-3</th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>PR-AUC</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>TP</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>...</th>\n",
       "      <th>val_categorical_accuracy</th>\n",
       "      <th>val_top-3</th>\n",
       "      <th>val_ROC-AUC</th>\n",
       "      <th>val_PR-AUC</th>\n",
       "      <th>val_precision</th>\n",
       "      <th>val_recall</th>\n",
       "      <th>val_TP</th>\n",
       "      <th>val_TN</th>\n",
       "      <th>val_FP</th>\n",
       "      <th>val_FN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.00</td>\n",
       "      <td>5.000000e+01</td>\n",
       "      <td>5.000000e+01</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.486759</td>\n",
       "      <td>0.080097</td>\n",
       "      <td>0.243454</td>\n",
       "      <td>0.494340</td>\n",
       "      <td>0.081931</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99443.420000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000394</td>\n",
       "      <td>0.002846</td>\n",
       "      <td>0.004021</td>\n",
       "      <td>0.002310</td>\n",
       "      <td>0.000598</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.914622</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.849920e-08</td>\n",
       "      <td>1.787379e-09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.486110</td>\n",
       "      <td>0.074115</td>\n",
       "      <td>0.236947</td>\n",
       "      <td>0.489837</td>\n",
       "      <td>0.080796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99435.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>4.999999e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.486589</td>\n",
       "      <td>0.078236</td>\n",
       "      <td>0.240210</td>\n",
       "      <td>0.492473</td>\n",
       "      <td>0.081480</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99440.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.486736</td>\n",
       "      <td>0.079978</td>\n",
       "      <td>0.243308</td>\n",
       "      <td>0.494457</td>\n",
       "      <td>0.081900</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99440.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.486852</td>\n",
       "      <td>0.082080</td>\n",
       "      <td>0.245326</td>\n",
       "      <td>0.495632</td>\n",
       "      <td>0.082258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99440.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.489034</td>\n",
       "      <td>0.087500</td>\n",
       "      <td>0.255310</td>\n",
       "      <td>0.500444</td>\n",
       "      <td>0.083412</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99616.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000001e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            loss  categorical_accuracy      top-3    ROC-AUC     PR-AUC  \\\n",
       "count  50.000000             50.000000  50.000000  50.000000  50.000000   \n",
       "mean    2.486759              0.080097   0.243454   0.494340   0.081931   \n",
       "std     0.000394              0.002846   0.004021   0.002310   0.000598   \n",
       "min     2.486110              0.074115   0.236947   0.489837   0.080796   \n",
       "25%     2.486589              0.078236   0.240210   0.492473   0.081480   \n",
       "50%     2.486736              0.079978   0.243308   0.494457   0.081900   \n",
       "75%     2.486852              0.082080   0.245326   0.495632   0.082258   \n",
       "max     2.489034              0.087500   0.255310   0.500444   0.083412   \n",
       "\n",
       "       precision  recall    TP            TN         FP  ...  \\\n",
       "count       50.0    50.0  50.0     50.000000  50.000000  ...   \n",
       "mean         0.0     0.0   0.0  99443.420000   0.100000  ...   \n",
       "std          0.0     0.0   0.0     24.914622   0.707107  ...   \n",
       "min          0.0     0.0   0.0  99435.000000   0.000000  ...   \n",
       "25%          0.0     0.0   0.0  99440.000000   0.000000  ...   \n",
       "50%          0.0     0.0   0.0  99440.000000   0.000000  ...   \n",
       "75%          0.0     0.0   0.0  99440.000000   0.000000  ...   \n",
       "max          0.0     0.0   0.0  99616.000000   5.000000  ...   \n",
       "\n",
       "       val_categorical_accuracy  val_top-3   val_ROC-AUC    val_PR-AUC  \\\n",
       "count                 50.000000      50.00  5.000000e+01  5.000000e+01   \n",
       "mean                   0.083333       0.25  5.000000e-01  8.333334e-02   \n",
       "std                    0.000000       0.00  1.849920e-08  1.787379e-09   \n",
       "min                    0.083333       0.25  4.999999e-01  8.333334e-02   \n",
       "25%                    0.083333       0.25  5.000000e-01  8.333334e-02   \n",
       "50%                    0.083333       0.25  5.000000e-01  8.333334e-02   \n",
       "75%                    0.083333       0.25  5.000000e-01  8.333334e-02   \n",
       "max                    0.083333       0.25  5.000001e-01  8.333334e-02   \n",
       "\n",
       "       val_precision  val_recall  val_TP   val_TN  val_FP  val_FN  \n",
       "count           50.0        50.0    50.0     50.0    50.0    50.0  \n",
       "mean             0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "std              0.0         0.0     0.0      0.0     0.0     0.0  \n",
       "min              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "25%              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "50%              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "75%              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "max              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "\n",
       "[8 rows x 22 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rm_64_32['history_df'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b112d67-0776-4e23-b576-a7cc225e3893",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "07f8c536-09d2-4ede-9c6f-6dc77e4d805e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T13:47:33.412565Z",
     "iopub.status.busy": "2021-06-19T13:47:33.412565Z",
     "iopub.status.idle": "2021-06-19T16:02:06.474479Z",
     "shell.execute_reply": "2021-06-19T16:02:06.473478Z",
     "shell.execute_reply.started": "2021-06-19T13:47:33.412565Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 9072 images belonging to 12 classes.\n",
      "Found 6048 images belonging to 12 classes.\n",
      "\n",
      "\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_checkpoints/lenet5_actual_rm-96-16\n",
      "success\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_weights and models/lenet5_actual_rm-96-16\n",
      "success\n",
      "\n",
      "Model: \"lenet5_actual_rm-96-16\"\n",
      "__________________________________________________________________________________________\n",
      "Layer (type)                                          Output Shape           Param #  \n",
      "==========================================================================================\n",
      "input (InputLayer)                                    [(None, 96, 96, 3)]    0        \n",
      "__________________________________________________________________________________________\n",
      "C1 (Conv2D)                                           (None, 92, 92, 6)      456      \n",
      "__________________________________________________________________________________________\n",
      "S2 (MaxPooling2D)                                     (None, 46, 46, 6)      0        \n",
      "__________________________________________________________________________________________\n",
      "C3 (Conv2D)                                           (None, 42, 42, 16)     2416     \n",
      "__________________________________________________________________________________________\n",
      "S4 (MaxPooling2D)                                     (None, 21, 21, 16)     0        \n",
      "__________________________________________________________________________________________\n",
      "flatten (Flatten)                                     (None, 7056)           0        \n",
      "__________________________________________________________________________________________\n",
      "C5 (Dense)                                            (None, 120)            846840   \n",
      "__________________________________________________________________________________________\n",
      "F6 (Dense)                                            (None, 84)             10164    \n",
      "__________________________________________________________________________________________\n",
      "OUTPUT (Dense)                                        (None, 12)             1020     \n",
      "==========================================================================================\n",
      "Total params: 860,896\n",
      "Trainable params: 860,896\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________\n",
      "\n",
      "\n",
      "{'batch_size': 16,\n",
      " 'epochs': 50,\n",
      " 'steps_per_epoch': 567,\n",
      " 'total_train': 9072,\n",
      " 'total_val': 6048,\n",
      " 'validation_steps': 378}\n",
      "\n",
      "\n",
      "lenet5_actual_rm-96-16\n",
      "Epoch 1/50\n",
      "  1/567 [..............................] - ETA: 0s - loss: 2.5309 - categorical_accuracy: 0.0625 - top-3: 0.3125 - ROC-AUC: 0.4466 - PR-AUC: 0.0762 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 176.0000 - FP: 0.0000e+00 - FN: 16.0000WARNING:tensorflow:From X:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\summary_ops_v2.py:1277: stop (from tensorflow.python.eager.profiler) is deprecated and will be removed after 2020-07-01.\n",
      "Instructions for updating:\n",
      "use `tf.profiler.experimental.stop` instead.\n",
      "567/567 [==============================] - 166s 292ms/step - loss: 2.4931 - categorical_accuracy: 0.0810 - top-3: 0.2472 - ROC-AUC: 0.4956 - PR-AUC: 0.0825 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99776.0000 - FP: 16.0000 - FN: 9072.0000 - val_loss: 2.4865 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 2/50\n",
      "567/567 [==============================] - 162s 285ms/step - loss: 2.4882 - categorical_accuracy: 0.0819 - top-3: 0.2362 - ROC-AUC: 0.4897 - PR-AUC: 0.0809 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 3/50\n",
      "567/567 [==============================] - 165s 291ms/step - loss: 2.4871 - categorical_accuracy: 0.0826 - top-3: 0.2511 - ROC-AUC: 0.4995 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4868 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 4/50\n",
      "567/567 [==============================] - 161s 284ms/step - loss: 2.4875 - categorical_accuracy: 0.0796 - top-3: 0.2433 - ROC-AUC: 0.4946 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 5/50\n",
      "567/567 [==============================] - 163s 287ms/step - loss: 2.4868 - categorical_accuracy: 0.0841 - top-3: 0.2511 - ROC-AUC: 0.4970 - PR-AUC: 0.0826 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4867 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 6/50\n",
      "567/567 [==============================] - 164s 289ms/step - loss: 2.4871 - categorical_accuracy: 0.0818 - top-3: 0.2447 - ROC-AUC: 0.4983 - PR-AUC: 0.0826 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4866 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 7/50\n",
      "567/567 [==============================] - 162s 286ms/step - loss: 2.4874 - categorical_accuracy: 0.0794 - top-3: 0.2423 - ROC-AUC: 0.4966 - PR-AUC: 0.0825 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4867 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 8/50\n",
      "567/567 [==============================] - 159s 281ms/step - loss: 2.4876 - categorical_accuracy: 0.0793 - top-3: 0.2412 - ROC-AUC: 0.4927 - PR-AUC: 0.0816 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 9/50\n",
      "567/567 [==============================] - 161s 284ms/step - loss: 2.4877 - categorical_accuracy: 0.0759 - top-3: 0.2367 - ROC-AUC: 0.4933 - PR-AUC: 0.0811 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 10/50\n",
      "567/567 [==============================] - 163s 287ms/step - loss: 2.4870 - categorical_accuracy: 0.0767 - top-3: 0.2454 - ROC-AUC: 0.4977 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4863 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 11/50\n",
      "567/567 [==============================] - 158s 279ms/step - loss: 2.4873 - categorical_accuracy: 0.0798 - top-3: 0.2447 - ROC-AUC: 0.4932 - PR-AUC: 0.0816 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4867 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 12/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4875 - categorical_accuracy: 0.0801 - top-3: 0.2454 - ROC-AUC: 0.4928 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4860 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 13/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4872 - categorical_accuracy: 0.0821 - top-3: 0.2466 - ROC-AUC: 0.4945 - PR-AUC: 0.0823 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4866 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 14/50\n",
      "567/567 [==============================] - 161s 284ms/step - loss: 2.4869 - categorical_accuracy: 0.0826 - top-3: 0.2464 - ROC-AUC: 0.5012 - PR-AUC: 0.0831 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4865 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 15/50\n",
      "567/567 [==============================] - 165s 290ms/step - loss: 2.4871 - categorical_accuracy: 0.0847 - top-3: 0.2488 - ROC-AUC: 0.4977 - PR-AUC: 0.0825 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4874 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 16/50\n",
      "567/567 [==============================] - 164s 289ms/step - loss: 2.4878 - categorical_accuracy: 0.0778 - top-3: 0.2386 - ROC-AUC: 0.4892 - PR-AUC: 0.0808 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 17/50\n",
      "567/567 [==============================] - 162s 285ms/step - loss: 2.4874 - categorical_accuracy: 0.0820 - top-3: 0.2407 - ROC-AUC: 0.4919 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 18/50\n",
      "567/567 [==============================] - 161s 285ms/step - loss: 2.4871 - categorical_accuracy: 0.0778 - top-3: 0.2468 - ROC-AUC: 0.4965 - PR-AUC: 0.0823 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4861 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 19/50\n",
      "567/567 [==============================] - 159s 281ms/step - loss: 2.4873 - categorical_accuracy: 0.0802 - top-3: 0.2386 - ROC-AUC: 0.4925 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 20/50\n",
      "567/567 [==============================] - 160s 283ms/step - loss: 2.4871 - categorical_accuracy: 0.0819 - top-3: 0.2433 - ROC-AUC: 0.4965 - PR-AUC: 0.0825 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4868 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 21/50\n",
      "567/567 [==============================] - 163s 287ms/step - loss: 2.4866 - categorical_accuracy: 0.0836 - top-3: 0.2498 - ROC-AUC: 0.5022 - PR-AUC: 0.0838 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 22/50\n",
      "567/567 [==============================] - 161s 284ms/step - loss: 2.4870 - categorical_accuracy: 0.0775 - top-3: 0.2467 - ROC-AUC: 0.4979 - PR-AUC: 0.0826 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4862 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 23/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4871 - categorical_accuracy: 0.0823 - top-3: 0.2455 - ROC-AUC: 0.4966 - PR-AUC: 0.0823 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4860 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 24/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4873 - categorical_accuracy: 0.0831 - top-3: 0.2388 - ROC-AUC: 0.4955 - PR-AUC: 0.0823 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4866 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 25/50\n",
      "567/567 [==============================] - 163s 287ms/step - loss: 2.4876 - categorical_accuracy: 0.0836 - top-3: 0.2384 - ROC-AUC: 0.4904 - PR-AUC: 0.0812 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 26/50\n",
      "567/567 [==============================] - 159s 280ms/step - loss: 2.4873 - categorical_accuracy: 0.0815 - top-3: 0.2381 - ROC-AUC: 0.4902 - PR-AUC: 0.0811 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 27/50\n",
      "567/567 [==============================] - 161s 283ms/step - loss: 2.4872 - categorical_accuracy: 0.0779 - top-3: 0.2413 - ROC-AUC: 0.4929 - PR-AUC: 0.0818 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 28/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4872 - categorical_accuracy: 0.0782 - top-3: 0.2468 - ROC-AUC: 0.4960 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 29/50\n",
      "567/567 [==============================] - 161s 283ms/step - loss: 2.4876 - categorical_accuracy: 0.0733 - top-3: 0.2390 - ROC-AUC: 0.4895 - PR-AUC: 0.0809 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 30/50\n",
      "567/567 [==============================] - 161s 283ms/step - loss: 2.4873 - categorical_accuracy: 0.0808 - top-3: 0.2417 - ROC-AUC: 0.4939 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4866 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 31/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4873 - categorical_accuracy: 0.0766 - top-3: 0.2423 - ROC-AUC: 0.4956 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4869 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 32/50\n",
      "567/567 [==============================] - 161s 283ms/step - loss: 2.4874 - categorical_accuracy: 0.0798 - top-3: 0.2411 - ROC-AUC: 0.4919 - PR-AUC: 0.0811 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 33/50\n",
      "567/567 [==============================] - 159s 280ms/step - loss: 2.4870 - categorical_accuracy: 0.0796 - top-3: 0.2405 - ROC-AUC: 0.4958 - PR-AUC: 0.0822 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4862 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 34/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4868 - categorical_accuracy: 0.0796 - top-3: 0.2434 - ROC-AUC: 0.5006 - PR-AUC: 0.0830 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4864 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 35/50\n",
      "567/567 [==============================] - 162s 285ms/step - loss: 2.4875 - categorical_accuracy: 0.0802 - top-3: 0.2415 - ROC-AUC: 0.4921 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 36/50\n",
      "567/567 [==============================] - 161s 285ms/step - loss: 2.4875 - categorical_accuracy: 0.0827 - top-3: 0.2378 - ROC-AUC: 0.4898 - PR-AUC: 0.0808 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 37/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4868 - categorical_accuracy: 0.0827 - top-3: 0.2459 - ROC-AUC: 0.4975 - PR-AUC: 0.0828 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4861 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 38/50\n",
      "567/567 [==============================] - 161s 284ms/step - loss: 2.4874 - categorical_accuracy: 0.0764 - top-3: 0.2371 - ROC-AUC: 0.4908 - PR-AUC: 0.0811 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 39/50\n",
      "567/567 [==============================] - 159s 281ms/step - loss: 2.4867 - categorical_accuracy: 0.0805 - top-3: 0.2406 - ROC-AUC: 0.4929 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4866 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 40/50\n",
      "567/567 [==============================] - 163s 288ms/step - loss: 2.4875 - categorical_accuracy: 0.0776 - top-3: 0.2420 - ROC-AUC: 0.4911 - PR-AUC: 0.0813 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 41/50\n",
      "567/567 [==============================] - 162s 287ms/step - loss: 2.4868 - categorical_accuracy: 0.0787 - top-3: 0.2478 - ROC-AUC: 0.4949 - PR-AUC: 0.0823 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4861 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 42/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4871 - categorical_accuracy: 0.0794 - top-3: 0.2420 - ROC-AUC: 0.4929 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 43/50\n",
      "567/567 [==============================] - 162s 285ms/step - loss: 2.4868 - categorical_accuracy: 0.0821 - top-3: 0.2451 - ROC-AUC: 0.4970 - PR-AUC: 0.0822 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4860 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 44/50\n",
      "567/567 [==============================] - 161s 284ms/step - loss: 2.4873 - categorical_accuracy: 0.0755 - top-3: 0.2394 - ROC-AUC: 0.4919 - PR-AUC: 0.0813 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 45/50\n",
      "567/567 [==============================] - 159s 281ms/step - loss: 2.4870 - categorical_accuracy: 0.0796 - top-3: 0.2399 - ROC-AUC: 0.4934 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 46/50\n",
      "567/567 [==============================] - 159s 281ms/step - loss: 2.4869 - categorical_accuracy: 0.0842 - top-3: 0.2458 - ROC-AUC: 0.4976 - PR-AUC: 0.0827 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 47/50\n",
      "567/567 [==============================] - 163s 287ms/step - loss: 2.4866 - categorical_accuracy: 0.0802 - top-3: 0.2461 - ROC-AUC: 0.4964 - PR-AUC: 0.0826 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4866 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 48/50\n",
      "567/567 [==============================] - 159s 280ms/step - loss: 2.4869 - categorical_accuracy: 0.0782 - top-3: 0.2427 - ROC-AUC: 0.4956 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4862 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 49/50\n",
      "567/567 [==============================] - 159s 281ms/step - loss: 2.4865 - categorical_accuracy: 0.0845 - top-3: 0.2460 - ROC-AUC: 0.4979 - PR-AUC: 0.0831 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4869 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 50/50\n",
      "567/567 [==============================] - 160s 282ms/step - loss: 2.4869 - categorical_accuracy: 0.0772 - top-3: 0.2446 - ROC-AUC: 0.4975 - PR-AUC: 0.0823 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99792.0000 - FP: 0.0000e+00 - FN: 9072.0000 - val_loss: 2.4862 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "-----\n",
      "(8067414.49 ms) == (134m:27s)\n",
      "-----\n",
      "\n",
      "\n",
      "16\n",
      "['lenet5_actual_rm-96-16.tensorboard',\n",
      " 'lenet5_actual_rm-96-16.weights.001_0.0833_2.4865.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.002_0.0833_2.4855.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.005.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.010.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.015.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.019_0.0833_2.4855.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.020.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.025.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.025_0.0833_2.4853.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.030.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.035.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.038_0.0833_2.4853.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.040.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.045.hdf5',\n",
      " 'lenet5_actual_rm-96-16.weights.050.hdf5']\n",
      "3\n",
      "['history.lenet5_actual_rm-96-16.csv',\n",
      " 'model.lenet5_actual_rm-96-16.h5',\n",
      " 'weights.lenet5_actual_rm-96-16.h5']\n",
      "\n",
      "all process done, please recheck before terminating runtime session\n",
      "\n",
      "don't forget to save the model.fit() verbose output\n",
      "\n",
      "\n",
      "Returned values using 232 bytes of memory now\n",
      "building file list ... done\n",
      "_checkpoints/\n",
      "_checkpoints/lenet5_actual_rm-96-16/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.001_0.0833_2.4865.hdf5\n",
      "\n",
      "         32.77K   0%    0.00kB/s    0:00:00  \n",
      "          3.47M   6%   18.61MB/s    0:00:00 (xfr#1, to-chk=30/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.002_0.0833_2.4855.hdf5\n",
      "\n",
      "          6.94M  13%   18.55MB/s    0:00:00 (xfr#2, to-chk=29/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.005.hdf5\n",
      "\n",
      "         10.40M  19%   18.56MB/s    0:00:00 (xfr#3, to-chk=28/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.010.hdf5\n",
      "\n",
      "         13.87M  26%   18.64MB/s    0:00:00 (xfr#4, to-chk=27/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.015.hdf5\n",
      "\n",
      "         17.34M  33%   18.50MB/s    0:00:00 (xfr#5, to-chk=26/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.019_0.0833_2.4855.hdf5\n",
      "\n",
      "         19.27M  36%   18.35MB/s    0:00:01  \n",
      "         20.81M  39%   18.19MB/s    0:00:01 (xfr#6, to-chk=25/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.020.hdf5\n",
      "\n",
      "         24.28M  46%   18.25MB/s    0:00:01 (xfr#7, to-chk=24/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.025.hdf5\n",
      "\n",
      "         27.74M  53%   18.30MB/s    0:00:01 (xfr#8, to-chk=23/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.025_0.0833_2.4853.hdf5\n",
      "\n",
      "         31.21M  59%   18.31MB/s    0:00:01 (xfr#9, to-chk=22/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.030.hdf5\n",
      "\n",
      "         34.68M  66%   18.35MB/s    0:00:01 (xfr#10, to-chk=21/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.035.hdf5\n",
      "\n",
      "         38.15M  73%   18.41MB/s    0:00:01 (xfr#11, to-chk=20/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.038_0.0833_2.4853.hdf5\n",
      "\n",
      "         38.64M  73%   18.41MB/s    0:00:00  \n",
      "         41.62M  79%   18.45MB/s    0:00:02 (xfr#12, to-chk=19/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.040.hdf5\n",
      "\n",
      "         45.09M  86%   18.46MB/s    0:00:02 (xfr#13, to-chk=18/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.045.hdf5\n",
      "\n",
      "         48.55M  92%   18.49MB/s    0:00:02 (xfr#14, to-chk=17/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.050.hdf5\n",
      "\n",
      "         52.02M  99%   18.49MB/s    0:00:02 (xfr#15, to-chk=16/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/events.out.tfevents.1624110454.ELITERAIHAN.18796.500.v2\n",
      "\n",
      "         52.06M  99%   18.49MB/s    0:00:02 (xfr#16, to-chk=13/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/events.out.tfevents.1624110460.ELITERAIHAN.profile-empty\n",
      "\n",
      "         52.06M  99%   18.49MB/s    0:00:02 (xfr#17, to-chk=12/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.input_pipeline.pb\n",
      "\n",
      "         52.07M  99%   18.48MB/s    0:00:02 (xfr#18, to-chk=8/33) \n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.kernel_stats.pb\n",
      "\n",
      "         52.07M  99%   18.47MB/s    0:00:02 (xfr#19, to-chk=7/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.memory_profile.json.gz\n",
      "\n",
      "         52.07M  99%   18.48MB/s    0:00:02 (xfr#20, to-chk=6/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.overview_page.pb\n",
      "\n",
      "         52.08M  99%   18.47MB/s    0:00:02 (xfr#21, to-chk=5/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.tensorflow_stats.pb\n",
      "\n",
      "         52.11M  99%   18.48MB/s    0:00:02 (xfr#22, to-chk=4/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.trace.json.gz\n",
      "\n",
      "         52.12M  99%   18.47MB/s    0:00:02 (xfr#23, to-chk=3/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.xplane.pb\n",
      "\n",
      "         52.20M  99%   18.49MB/s    0:00:02 (xfr#24, to-chk=2/33)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/validation/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/validation/events.out.tfevents.1624110560.ELITERAIHAN.18796.9593.v2\n",
      "\n",
      "         52.23M 100%   18.48MB/s    0:00:02 (xfr#25, to-chk=0/33)\n",
      "         52.23M 100%   18.48MB/s    0:00:02 (xfr#25, to-chk=0/33)\n",
      "         52.23M 100%   16.98MB/s    0:00:02 (xfr#25, to-chk=0/33)\n",
      "         52.23M 100%   16.98MB/s    0:00:02 (xfr#25, to-chk=0/33)\n",
      "\n",
      "sent 47.32M bytes  received 515 bytes  13.52M bytes/sec\n",
      "total size is 52.23M  speedup is 1.10\n",
      "\n",
      "building file list ... done\n",
      "_weights and models/\n",
      "_weights and models/lenet5_actual_rm-96-16/\n",
      "_weights and models/lenet5_actual_rm-96-16/history.lenet5_actual_rm-96-16.csv\n",
      "\n",
      "         11.68K   0%    0.00kB/s    0:00:00  \n",
      "         11.68K   0%    0.00kB/s    0:00:00 (xfr#1, to-chk=2/5)\n",
      "_weights and models/lenet5_actual_rm-96-16/model.lenet5_actual_rm-96-16.h5\n",
      "\n",
      "         10.41M  75%   26.86MB/s    0:00:00 (xfr#2, to-chk=1/5)\n",
      "_weights and models/lenet5_actual_rm-96-16/weights.lenet5_actual_rm-96-16.h5\n",
      "\n",
      "         13.87M 100%   18.16MB/s    0:00:00 (xfr#3, to-chk=0/5)\n",
      "         13.87M 100%   18.13MB/s    0:00:00 (xfr#3, to-chk=0/5)\n",
      "         13.87M 100%   17.26MB/s    0:00:00 (xfr#3, to-chk=0/5)\n",
      "         13.87M 100%   17.24MB/s    0:00:00 (xfr#3, to-chk=0/5)\n",
      "\n",
      "sent 9.15M bytes  received 79 bytes  6.10M bytes/sec\n",
      "total size is 13.87M  speedup is 1.52\n",
      "\n",
      "rsync operation completed\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rm_96_16 = consecutiveModelTraining(input_size=96, batch_size=16, activation='relu', pooling='max')\n",
    "do_rsync()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "582e7bf6-78ea-4ca8-835b-acdc1f2420f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T17:13:59.744777Z",
     "iopub.status.busy": "2021-06-19T17:13:59.744777Z",
     "iopub.status.idle": "2021-06-19T17:13:59.845787Z",
     "shell.execute_reply": "2021-06-19T17:13:59.844785Z",
     "shell.execute_reply.started": "2021-06-19T17:13:59.744777Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loss</th>\n",
       "      <th>categorical_accuracy</th>\n",
       "      <th>top-3</th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>PR-AUC</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>TP</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>...</th>\n",
       "      <th>val_categorical_accuracy</th>\n",
       "      <th>val_top-3</th>\n",
       "      <th>val_ROC-AUC</th>\n",
       "      <th>val_PR-AUC</th>\n",
       "      <th>val_precision</th>\n",
       "      <th>val_recall</th>\n",
       "      <th>val_TP</th>\n",
       "      <th>val_TN</th>\n",
       "      <th>val_FP</th>\n",
       "      <th>val_FN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.00</td>\n",
       "      <td>5.000000e+01</td>\n",
       "      <td>5.000000e+01</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.487300</td>\n",
       "      <td>0.080165</td>\n",
       "      <td>0.243135</td>\n",
       "      <td>0.494798</td>\n",
       "      <td>0.081997</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99791.680000</td>\n",
       "      <td>0.320000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000904</td>\n",
       "      <td>0.002604</td>\n",
       "      <td>0.003795</td>\n",
       "      <td>0.003186</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.262742</td>\n",
       "      <td>2.262742</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.346332e-08</td>\n",
       "      <td>1.787379e-09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.486509</td>\n",
       "      <td>0.073302</td>\n",
       "      <td>0.236221</td>\n",
       "      <td>0.489150</td>\n",
       "      <td>0.080848</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99776.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>4.999999e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.486937</td>\n",
       "      <td>0.078153</td>\n",
       "      <td>0.240548</td>\n",
       "      <td>0.492547</td>\n",
       "      <td>0.081416</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99792.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.487185</td>\n",
       "      <td>0.080192</td>\n",
       "      <td>0.243000</td>\n",
       "      <td>0.495224</td>\n",
       "      <td>0.082029</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99792.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.487417</td>\n",
       "      <td>0.082121</td>\n",
       "      <td>0.246004</td>\n",
       "      <td>0.497025</td>\n",
       "      <td>0.082497</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99792.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.493134</td>\n",
       "      <td>0.084656</td>\n",
       "      <td>0.251102</td>\n",
       "      <td>0.502243</td>\n",
       "      <td>0.083757</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99792.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            loss  categorical_accuracy      top-3    ROC-AUC     PR-AUC  \\\n",
       "count  50.000000             50.000000  50.000000  50.000000  50.000000   \n",
       "mean    2.487300              0.080165   0.243135   0.494798   0.081997   \n",
       "std     0.000904              0.002604   0.003795   0.003186   0.000716   \n",
       "min     2.486509              0.073302   0.236221   0.489150   0.080848   \n",
       "25%     2.486937              0.078153   0.240548   0.492547   0.081416   \n",
       "50%     2.487185              0.080192   0.243000   0.495224   0.082029   \n",
       "75%     2.487417              0.082121   0.246004   0.497025   0.082497   \n",
       "max     2.493134              0.084656   0.251102   0.502243   0.083757   \n",
       "\n",
       "       precision  recall    TP            TN         FP  ...  \\\n",
       "count       50.0    50.0  50.0     50.000000  50.000000  ...   \n",
       "mean         0.0     0.0   0.0  99791.680000   0.320000  ...   \n",
       "std          0.0     0.0   0.0      2.262742   2.262742  ...   \n",
       "min          0.0     0.0   0.0  99776.000000   0.000000  ...   \n",
       "25%          0.0     0.0   0.0  99792.000000   0.000000  ...   \n",
       "50%          0.0     0.0   0.0  99792.000000   0.000000  ...   \n",
       "75%          0.0     0.0   0.0  99792.000000   0.000000  ...   \n",
       "max          0.0     0.0   0.0  99792.000000  16.000000  ...   \n",
       "\n",
       "       val_categorical_accuracy  val_top-3   val_ROC-AUC    val_PR-AUC  \\\n",
       "count                 50.000000      50.00  5.000000e+01  5.000000e+01   \n",
       "mean                   0.083333       0.25  5.000000e-01  8.333334e-02   \n",
       "std                    0.000000       0.00  1.346332e-08  1.787379e-09   \n",
       "min                    0.083333       0.25  4.999999e-01  8.333334e-02   \n",
       "25%                    0.083333       0.25  5.000000e-01  8.333334e-02   \n",
       "50%                    0.083333       0.25  5.000000e-01  8.333334e-02   \n",
       "75%                    0.083333       0.25  5.000000e-01  8.333334e-02   \n",
       "max                    0.083333       0.25  5.000000e-01  8.333334e-02   \n",
       "\n",
       "       val_precision  val_recall  val_TP   val_TN  val_FP  val_FN  \n",
       "count           50.0        50.0    50.0     50.0    50.0    50.0  \n",
       "mean             0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "std              0.0         0.0     0.0      0.0     0.0     0.0  \n",
       "min              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "25%              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "50%              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "75%              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "max              0.0         0.0     0.0  66528.0     0.0  6048.0  \n",
       "\n",
       "[8 rows x 22 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rm_96_16['history_df'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf121ae6-ca00-4657-a681-9004739cb513",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2d33696a-bcf2-4615-9f68-2612d4791633",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T17:14:21.847270Z",
     "iopub.status.busy": "2021-06-19T17:14:21.847270Z",
     "iopub.status.idle": "2021-06-19T19:24:13.731229Z",
     "shell.execute_reply": "2021-06-19T19:24:13.730230Z",
     "shell.execute_reply.started": "2021-06-19T17:14:21.847270Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 9072 images belonging to 12 classes.\n",
      "Found 6048 images belonging to 12 classes.\n",
      "\n",
      "\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_checkpoints/lenet5_actual_rm-96-32\n",
      "success\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_weights and models/lenet5_actual_rm-96-32\n",
      "success\n",
      "\n",
      "Model: \"lenet5_actual_rm-96-32\"\n",
      "__________________________________________________________________________________________\n",
      "Layer (type)                                          Output Shape           Param #  \n",
      "==========================================================================================\n",
      "input (InputLayer)                                    [(None, 96, 96, 3)]    0        \n",
      "__________________________________________________________________________________________\n",
      "C1 (Conv2D)                                           (None, 92, 92, 6)      456      \n",
      "__________________________________________________________________________________________\n",
      "S2 (MaxPooling2D)                                     (None, 46, 46, 6)      0        \n",
      "__________________________________________________________________________________________\n",
      "C3 (Conv2D)                                           (None, 42, 42, 16)     2416     \n",
      "__________________________________________________________________________________________\n",
      "S4 (MaxPooling2D)                                     (None, 21, 21, 16)     0        \n",
      "__________________________________________________________________________________________\n",
      "flatten (Flatten)                                     (None, 7056)           0        \n",
      "__________________________________________________________________________________________\n",
      "C5 (Dense)                                            (None, 120)            846840   \n",
      "__________________________________________________________________________________________\n",
      "F6 (Dense)                                            (None, 84)             10164    \n",
      "__________________________________________________________________________________________\n",
      "OUTPUT (Dense)                                        (None, 12)             1020     \n",
      "==========================================================================================\n",
      "Total params: 860,896\n",
      "Trainable params: 860,896\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________\n",
      "\n",
      "\n",
      "{'batch_size': 32,\n",
      " 'epochs': 50,\n",
      " 'steps_per_epoch': 283,\n",
      " 'total_train': 9072,\n",
      " 'total_val': 6048,\n",
      " 'validation_steps': 189}\n",
      "\n",
      "\n",
      "lenet5_actual_rm-96-32\n",
      "Epoch 1/50\n",
      "  2/283 [..............................] - ETA: 11:24 - loss: 2.6273 - categorical_accuracy: 0.0156 - top-3: 0.2188 - ROC-AUC: 0.5222 - PR-AUC: 0.0879 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 704.0000 - FP: 0.0000e+00 - FN: 64.0000 WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.3420s vs `on_train_batch_end` time: 4.5273s). Check your callbacks.\n",
      "283/283 [==============================] - 168s 592ms/step - loss: 2.4888 - categorical_accuracy: 0.0835 - top-3: 0.2456 - ROC-AUC: 0.4968 - PR-AUC: 0.0829 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 2/50\n",
      "283/283 [==============================] - 161s 569ms/step - loss: 2.4871 - categorical_accuracy: 0.0815 - top-3: 0.2402 - ROC-AUC: 0.4913 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4860 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 3/50\n",
      "283/283 [==============================] - 157s 555ms/step - loss: 2.4869 - categorical_accuracy: 0.0771 - top-3: 0.2421 - ROC-AUC: 0.4930 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 4/50\n",
      "283/283 [==============================] - 157s 555ms/step - loss: 2.4867 - categorical_accuracy: 0.0795 - top-3: 0.2378 - ROC-AUC: 0.4909 - PR-AUC: 0.0810 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 5/50\n",
      "283/283 [==============================] - 155s 548ms/step - loss: 2.4869 - categorical_accuracy: 0.0789 - top-3: 0.2414 - ROC-AUC: 0.4943 - PR-AUC: 0.0817 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 6/50\n",
      "283/283 [==============================] - 153s 542ms/step - loss: 2.4865 - categorical_accuracy: 0.0861 - top-3: 0.2470 - ROC-AUC: 0.4983 - PR-AUC: 0.0828 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4863 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 7/50\n",
      "283/283 [==============================] - 155s 547ms/step - loss: 2.4868 - categorical_accuracy: 0.0782 - top-3: 0.2501 - ROC-AUC: 0.4984 - PR-AUC: 0.0822 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4865 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 8/50\n",
      "283/283 [==============================] - 157s 554ms/step - loss: 2.4872 - categorical_accuracy: 0.0744 - top-3: 0.2426 - ROC-AUC: 0.4940 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 9/50\n",
      "283/283 [==============================] - 155s 548ms/step - loss: 2.4867 - categorical_accuracy: 0.0858 - top-3: 0.2510 - ROC-AUC: 0.4975 - PR-AUC: 0.0829 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 10/50\n",
      "283/283 [==============================] - 155s 547ms/step - loss: 2.4860 - categorical_accuracy: 0.0862 - top-3: 0.2513 - ROC-AUC: 0.5012 - PR-AUC: 0.0835 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4863 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 11/50\n",
      "283/283 [==============================] - 156s 553ms/step - loss: 2.4865 - categorical_accuracy: 0.0816 - top-3: 0.2475 - ROC-AUC: 0.4996 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4864 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 12/50\n",
      "283/283 [==============================] - 156s 550ms/step - loss: 2.4872 - categorical_accuracy: 0.0813 - top-3: 0.2448 - ROC-AUC: 0.4932 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 13/50\n",
      "283/283 [==============================] - 152s 539ms/step - loss: 2.4866 - categorical_accuracy: 0.0792 - top-3: 0.2437 - ROC-AUC: 0.4963 - PR-AUC: 0.0823 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 14/50\n",
      "283/283 [==============================] - 153s 539ms/step - loss: 2.4865 - categorical_accuracy: 0.0783 - top-3: 0.2480 - ROC-AUC: 0.4978 - PR-AUC: 0.0832 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 15/50\n",
      "283/283 [==============================] - 155s 549ms/step - loss: 2.4871 - categorical_accuracy: 0.0810 - top-3: 0.2421 - ROC-AUC: 0.4915 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 16/50\n",
      "283/283 [==============================] - 154s 543ms/step - loss: 2.4871 - categorical_accuracy: 0.0795 - top-3: 0.2439 - ROC-AUC: 0.4949 - PR-AUC: 0.0817 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 17/50\n",
      "283/283 [==============================] - 152s 538ms/step - loss: 2.4869 - categorical_accuracy: 0.0785 - top-3: 0.2405 - ROC-AUC: 0.4882 - PR-AUC: 0.0803 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4852 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 18/50\n",
      "283/283 [==============================] - 155s 546ms/step - loss: 2.4868 - categorical_accuracy: 0.0829 - top-3: 0.2448 - ROC-AUC: 0.4929 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 19/50\n",
      "283/283 [==============================] - 157s 556ms/step - loss: 2.4865 - categorical_accuracy: 0.0795 - top-3: 0.2480 - ROC-AUC: 0.4980 - PR-AUC: 0.0825 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 20/50\n",
      "283/283 [==============================] - 156s 553ms/step - loss: 2.4868 - categorical_accuracy: 0.0811 - top-3: 0.2470 - ROC-AUC: 0.4955 - PR-AUC: 0.0826 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 21/50\n",
      "283/283 [==============================] - 153s 542ms/step - loss: 2.4870 - categorical_accuracy: 0.0796 - top-3: 0.2340 - ROC-AUC: 0.4901 - PR-AUC: 0.0812 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 22/50\n",
      "283/283 [==============================] - 156s 553ms/step - loss: 2.4866 - categorical_accuracy: 0.0869 - top-3: 0.2487 - ROC-AUC: 0.4933 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4865 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 23/50\n",
      "283/283 [==============================] - 157s 555ms/step - loss: 2.4871 - categorical_accuracy: 0.0782 - top-3: 0.2437 - ROC-AUC: 0.4932 - PR-AUC: 0.0813 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 24/50\n",
      "283/283 [==============================] - 153s 542ms/step - loss: 2.4866 - categorical_accuracy: 0.0761 - top-3: 0.2365 - ROC-AUC: 0.4932 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 25/50\n",
      "283/283 [==============================] - 155s 548ms/step - loss: 2.4864 - categorical_accuracy: 0.0840 - top-3: 0.2520 - ROC-AUC: 0.4992 - PR-AUC: 0.0833 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 26/50\n",
      "283/283 [==============================] - 155s 548ms/step - loss: 2.4864 - categorical_accuracy: 0.0801 - top-3: 0.2494 - ROC-AUC: 0.4992 - PR-AUC: 0.0829 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4862 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 27/50\n",
      "283/283 [==============================] - 157s 556ms/step - loss: 2.4867 - categorical_accuracy: 0.0821 - top-3: 0.2442 - ROC-AUC: 0.4929 - PR-AUC: 0.0817 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 28/50\n",
      "283/283 [==============================] - 154s 544ms/step - loss: 2.4868 - categorical_accuracy: 0.0819 - top-3: 0.2373 - ROC-AUC: 0.4913 - PR-AUC: 0.0812 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 29/50\n",
      "283/283 [==============================] - 154s 544ms/step - loss: 2.4868 - categorical_accuracy: 0.0774 - top-3: 0.2427 - ROC-AUC: 0.4946 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 30/50\n",
      "283/283 [==============================] - 152s 539ms/step - loss: 2.4871 - categorical_accuracy: 0.0781 - top-3: 0.2412 - ROC-AUC: 0.4910 - PR-AUC: 0.0812 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 31/50\n",
      "283/283 [==============================] - 155s 548ms/step - loss: 2.4865 - categorical_accuracy: 0.0757 - top-3: 0.2405 - ROC-AUC: 0.4945 - PR-AUC: 0.0815 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4852 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 32/50\n",
      "283/283 [==============================] - 153s 541ms/step - loss: 2.4870 - categorical_accuracy: 0.0802 - top-3: 0.2409 - ROC-AUC: 0.4891 - PR-AUC: 0.0808 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 33/50\n",
      "283/283 [==============================] - 156s 550ms/step - loss: 2.4867 - categorical_accuracy: 0.0781 - top-3: 0.2465 - ROC-AUC: 0.4938 - PR-AUC: 0.0822 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 34/50\n",
      "283/283 [==============================] - 154s 544ms/step - loss: 2.4867 - categorical_accuracy: 0.0789 - top-3: 0.2347 - ROC-AUC: 0.4915 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 35/50\n",
      "283/283 [==============================] - 158s 558ms/step - loss: 2.4867 - categorical_accuracy: 0.0761 - top-3: 0.2396 - ROC-AUC: 0.4921 - PR-AUC: 0.0816 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 36/50\n",
      "283/283 [==============================] - 154s 544ms/step - loss: 2.4865 - categorical_accuracy: 0.0823 - top-3: 0.2460 - ROC-AUC: 0.4942 - PR-AUC: 0.0817 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 37/50\n",
      "283/283 [==============================] - 154s 544ms/step - loss: 2.4865 - categorical_accuracy: 0.0772 - top-3: 0.2414 - ROC-AUC: 0.4941 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4859 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 38/50\n",
      "283/283 [==============================] - 155s 546ms/step - loss: 2.4868 - categorical_accuracy: 0.0803 - top-3: 0.2352 - ROC-AUC: 0.4918 - PR-AUC: 0.0810 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 39/50\n",
      "283/283 [==============================] - 155s 546ms/step - loss: 2.4869 - categorical_accuracy: 0.0821 - top-3: 0.2437 - ROC-AUC: 0.4911 - PR-AUC: 0.0812 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 40/50\n",
      "283/283 [==============================] - 154s 544ms/step - loss: 2.4867 - categorical_accuracy: 0.0823 - top-3: 0.2421 - ROC-AUC: 0.4931 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4852 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 41/50\n",
      "283/283 [==============================] - 154s 544ms/step - loss: 2.4865 - categorical_accuracy: 0.0801 - top-3: 0.2419 - ROC-AUC: 0.4951 - PR-AUC: 0.0820 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 42/50\n",
      "283/283 [==============================] - 156s 550ms/step - loss: 2.4864 - categorical_accuracy: 0.0789 - top-3: 0.2472 - ROC-AUC: 0.4970 - PR-AUC: 0.0825 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4858 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 43/50\n",
      "283/283 [==============================] - 154s 544ms/step - loss: 2.4865 - categorical_accuracy: 0.0795 - top-3: 0.2466 - ROC-AUC: 0.4963 - PR-AUC: 0.0821 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 44/50\n",
      "283/283 [==============================] - 152s 539ms/step - loss: 2.4869 - categorical_accuracy: 0.0778 - top-3: 0.2414 - ROC-AUC: 0.4939 - PR-AUC: 0.0816 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4860 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 45/50\n",
      "283/283 [==============================] - 156s 550ms/step - loss: 2.4867 - categorical_accuracy: 0.0793 - top-3: 0.2412 - ROC-AUC: 0.4948 - PR-AUC: 0.0819 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4856 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 46/50\n",
      "283/283 [==============================] - 155s 549ms/step - loss: 2.4869 - categorical_accuracy: 0.0757 - top-3: 0.2434 - ROC-AUC: 0.4915 - PR-AUC: 0.0814 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4854 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 47/50\n",
      "283/283 [==============================] - 154s 543ms/step - loss: 2.4867 - categorical_accuracy: 0.0813 - top-3: 0.2428 - ROC-AUC: 0.4935 - PR-AUC: 0.0818 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 48/50\n",
      "283/283 [==============================] - 154s 546ms/step - loss: 2.4867 - categorical_accuracy: 0.0811 - top-3: 0.2424 - ROC-AUC: 0.4953 - PR-AUC: 0.0822 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4857 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 49/50\n",
      "283/283 [==============================] - 155s 548ms/step - loss: 2.4871 - categorical_accuracy: 0.0762 - top-3: 0.2298 - ROC-AUC: 0.4881 - PR-AUC: 0.0805 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4855 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "Epoch 50/50\n",
      "283/283 [==============================] - 156s 550ms/step - loss: 2.4866 - categorical_accuracy: 0.0763 - top-3: 0.2356 - ROC-AUC: 0.4919 - PR-AUC: 0.0811 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 99440.0000 - FP: 0.0000e+00 - FN: 9040.0000 - val_loss: 2.4853 - val_categorical_accuracy: 0.0833 - val_top-3: 0.2500 - val_ROC-AUC: 0.5000 - val_PR-AUC: 0.0833 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 - val_TP: 0.0000e+00 - val_TN: 66528.0000 - val_FP: 0.0000e+00 - val_FN: 6048.0000\n",
      "-----\n",
      "(7783023.97 ms) == (129m:43s)\n",
      "-----\n",
      "\n",
      "\n",
      "15\n",
      "['lenet5_actual_rm-96-32.tensorboard',\n",
      " 'lenet5_actual_rm-96-32.weights.001_0.0833_2.4854.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.005.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.010.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.015.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.017_0.0833_2.4852.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.020.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.025.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.030.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.031_0.0833_2.4852.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.035.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.040.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.040_0.0833_2.4852.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.045.hdf5',\n",
      " 'lenet5_actual_rm-96-32.weights.050.hdf5']\n",
      "3\n",
      "['history.lenet5_actual_rm-96-32.csv',\n",
      " 'model.lenet5_actual_rm-96-32.h5',\n",
      " 'weights.lenet5_actual_rm-96-32.h5']\n",
      "\n",
      "all process done, please recheck before terminating runtime session\n",
      "\n",
      "don't forget to save the model.fit() verbose output\n",
      "\n",
      "\n",
      "Returned values using 232 bytes of memory now\n",
      "building file list ... done\n",
      "_checkpoints/\n",
      "_checkpoints/lenet5_actual_rm-96-16/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.001_0.0833_2.4865.hdf5\n",
      "\n",
      "         32.77K   0%    0.00kB/s    0:00:00  \n",
      "          3.47M   3%   18.94MB/s    0:00:00 (xfr#1, to-chk=61/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.002_0.0833_2.4855.hdf5\n",
      "\n",
      "          6.94M   6%   18.92MB/s    0:00:00 (xfr#2, to-chk=60/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.005.hdf5\n",
      "\n",
      "         10.40M  10%   18.84MB/s    0:00:00 (xfr#3, to-chk=59/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.010.hdf5\n",
      "\n",
      "         13.87M  13%   18.77MB/s    0:00:00 (xfr#4, to-chk=58/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.015.hdf5\n",
      "\n",
      "         17.34M  16%   18.84MB/s    0:00:00 (xfr#5, to-chk=57/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.019_0.0833_2.4855.hdf5\n",
      "\n",
      "         19.40M  18%   18.47MB/s    0:00:04  \n",
      "         20.81M  20%   18.04MB/s    0:00:01 (xfr#6, to-chk=56/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.020.hdf5\n",
      "\n",
      "         24.28M  23%   17.81MB/s    0:00:01 (xfr#7, to-chk=55/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.025.hdf5\n",
      "\n",
      "         27.74M  26%   17.93MB/s    0:00:01 (xfr#8, to-chk=54/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.025_0.0833_2.4853.hdf5\n",
      "\n",
      "         31.21M  30%   18.02MB/s    0:00:01 (xfr#9, to-chk=53/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.030.hdf5\n",
      "\n",
      "\n",
      "         34.68M  33%   18.09MB/s    0:00:01 (xfr#10, to-chk=52/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.035.hdf5\n",
      "\n",
      "         38.06M  36%   18.13MB/s    0:00:03  \n",
      "         38.15M  36%   18.14MB/s    0:00:02 (xfr#11, to-chk=51/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.038_0.0833_2.4853.hdf5\n",
      "\n",
      "         41.62M  40%   18.07MB/s    0:00:02 (xfr#12, to-chk=50/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.040.hdf5\n",
      "\n",
      "         45.09M  43%   18.11MB/s    0:00:02 (xfr#13, to-chk=49/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.045.hdf5\n",
      "\n",
      "         48.55M  47%   18.10MB/s    0:00:02 (xfr#14, to-chk=48/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.weights.050.hdf5\n",
      "\n",
      "         52.02M  50%   18.15MB/s    0:00:02 (xfr#15, to-chk=47/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/events.out.tfevents.1624110454.ELITERAIHAN.18796.500.v2\n",
      "\n",
      "         52.06M  50%   18.16MB/s    0:00:02 (xfr#16, to-chk=44/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/events.out.tfevents.1624110460.ELITERAIHAN.profile-empty\n",
      "\n",
      "         52.06M  50%   18.16MB/s    0:00:02 (xfr#17, to-chk=43/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.input_pipeline.pb\n",
      "\n",
      "         52.07M  50%   18.15MB/s    0:00:02 (xfr#18, to-chk=39/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.kernel_stats.pb\n",
      "\n",
      "         52.07M  50%   18.14MB/s    0:00:02 (xfr#19, to-chk=38/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.memory_profile.json.gz\n",
      "\n",
      "         52.07M  50%   18.14MB/s    0:00:02 (xfr#20, to-chk=37/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.overview_page.pb\n",
      "\n",
      "         52.08M  50%   18.14MB/s    0:00:02 (xfr#21, to-chk=36/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.tensorflow_stats.pb\n",
      "\n",
      "         52.11M  50%   18.15MB/s    0:00:02 (xfr#22, to-chk=35/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.trace.json.gz\n",
      "\n",
      "         52.12M  50%   18.14MB/s    0:00:02 (xfr#23, to-chk=34/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/train/plugins/profile/2021_06_19_13_47_40/ELITERAIHAN.xplane.pb\n",
      "\n",
      "         52.20M  50%   18.15MB/s    0:00:02 (xfr#24, to-chk=33/64)\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/validation/\n",
      "_checkpoints/lenet5_actual_rm-96-16/lenet5_actual_rm-96-16.tensorboard/validation/events.out.tfevents.1624110560.ELITERAIHAN.18796.9593.v2\n",
      "\n",
      "         52.23M  50%   18.15MB/s    0:00:02 (xfr#25, to-chk=31/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.001_0.0833_2.4854.hdf5\n",
      "\n",
      "         55.70M  53%   18.16MB/s    0:00:02 (xfr#26, to-chk=29/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.005.hdf5\n",
      "\n",
      "         57.20M  55%   18.17MB/s    0:00:02  \n",
      "         59.17M  57%   18.10MB/s    0:00:03 (xfr#27, to-chk=28/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.010.hdf5\n",
      "\n",
      "         62.63M  60%   18.12MB/s    0:00:03 (xfr#28, to-chk=27/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.015.hdf5\n",
      "\n",
      "         66.10M  64%   18.15MB/s    0:00:03 (xfr#29, to-chk=26/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.017_0.0833_2.4852.hdf5\n",
      "\n",
      "         69.57M  67%   18.17MB/s    0:00:03 (xfr#30, to-chk=25/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.020.hdf5\n",
      "\n",
      "         73.04M  70%   18.20MB/s    0:00:03 (xfr#31, to-chk=24/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.025.hdf5\n",
      "\n",
      "         76.51M  74%   18.23MB/s    0:00:01  \n",
      "         76.51M  74%   18.23MB/s    0:00:04 (xfr#32, to-chk=23/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.030.hdf5\n",
      "\n",
      "         79.97M  77%   18.25MB/s    0:00:04 (xfr#33, to-chk=22/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.031_0.0833_2.4852.hdf5\n",
      "\n",
      "         83.44M  80%   18.27MB/s    0:00:04 (xfr#34, to-chk=21/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.035.hdf5\n",
      "\n",
      "         86.91M  84%   18.28MB/s    0:00:04 (xfr#35, to-chk=20/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.040.hdf5\n",
      "\n",
      "         90.38M  87%   18.30MB/s    0:00:04 (xfr#36, to-chk=19/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.040_0.0833_2.4852.hdf5\n",
      "\n",
      "         93.85M  90%   18.31MB/s    0:00:04 (xfr#37, to-chk=18/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.045.hdf5\n",
      "\n",
      "         96.14M  93%   18.30MB/s    0:00:00  \n",
      "         97.31M  94%   18.34MB/s    0:00:05 (xfr#38, to-chk=17/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.weights.050.hdf5\n",
      "\n",
      "        100.78M  97%   18.35MB/s    0:00:05 (xfr#39, to-chk=16/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/events.out.tfevents.1624122863.ELITERAIHAN.18796.536254.v2\n",
      "\n",
      "        102.99M  99%   18.64MB/s    0:00:05 (xfr#40, to-chk=13/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/events.out.tfevents.1624122871.ELITERAIHAN.profile-empty\n",
      "\n",
      "        102.99M  99%   18.64MB/s    0:00:05 (xfr#41, to-chk=12/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/profile/\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/profile/2021_06_19_17_14_31/\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/profile/2021_06_19_17_14_31/ELITERAIHAN.input_pipeline.pb\n",
      "\n",
      "        102.99M  99%   18.64MB/s    0:00:05 (xfr#42, to-chk=8/64) \n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/profile/2021_06_19_17_14_31/ELITERAIHAN.kernel_stats.pb\n",
      "\n",
      "        102.99M  99%   18.64MB/s    0:00:05 (xfr#43, to-chk=7/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/profile/2021_06_19_17_14_31/ELITERAIHAN.memory_profile.json.gz\n",
      "\n",
      "        103.00M  99%   18.64MB/s    0:00:05 (xfr#44, to-chk=6/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/profile/2021_06_19_17_14_31/ELITERAIHAN.overview_page.pb\n",
      "\n",
      "        103.00M  99%   18.63MB/s    0:00:05 (xfr#45, to-chk=5/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/profile/2021_06_19_17_14_31/ELITERAIHAN.tensorflow_stats.pb\n",
      "\n",
      "        103.03M  99%   18.64MB/s    0:00:05 (xfr#46, to-chk=4/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/profile/2021_06_19_17_14_31/ELITERAIHAN.trace.json.gz\n",
      "\n",
      "        103.04M  99%   18.63MB/s    0:00:05 (xfr#47, to-chk=3/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/train/plugins/profile/2021_06_19_17_14_31/ELITERAIHAN.xplane.pb\n",
      "\n",
      "        103.13M  99%   18.64MB/s    0:00:05 (xfr#48, to-chk=2/64)\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/validation/\n",
      "_checkpoints/lenet5_actual_rm-96-32/lenet5_actual_rm-96-32.tensorboard/validation/events.out.tfevents.1624122967.ELITERAIHAN.18796.542223.v2\n",
      "\n",
      "        103.15M 100%   18.64MB/s    0:00:05 (xfr#49, to-chk=0/64)\n",
      "        103.15M 100%   18.64MB/s    0:00:05 (xfr#49, to-chk=0/64)\n",
      "        103.15M 100%   18.23MB/s    0:00:05 (xfr#49, to-chk=0/64)\n",
      "        103.15M 100%   18.23MB/s    0:00:05 (xfr#49, to-chk=0/64)\n",
      "\n",
      "sent 91.58M bytes  received 992 bytes  16.65M bytes/sec\n",
      "total size is 103.15M  speedup is 1.13\n",
      "building file list ... done\n",
      "_weights and models/\n",
      "_weights and models/lenet5_actual_rm-96-16/\n",
      "_weights and models/lenet5_actual_rm-96-16/history.lenet5_actual_rm-96-16.csv\n",
      "\n",
      "         11.68K   0%    0.00kB/s    0:00:00  \n",
      "         11.68K   0%    0.00kB/s    0:00:00 (xfr#1, to-chk=6/9)\n",
      "_weights and models/lenet5_actual_rm-96-16/model.lenet5_actual_rm-96-16.h5\n",
      "\n",
      "         10.41M  37%   26.43MB/s    0:00:00 (xfr#2, to-chk=5/9)\n",
      "_weights and models/lenet5_actual_rm-96-16/weights.lenet5_actual_rm-96-16.h5\n",
      "\n",
      "         13.87M  49%   14.43MB/s    0:00:00 (xfr#3, to-chk=4/9)\n",
      "_weights and models/lenet5_actual_rm-96-32/\n",
      "_weights and models/lenet5_actual_rm-96-32/history.lenet5_actual_rm-96-32.csv\n",
      "\n",
      "         13.88M  50%   14.43MB/s    0:00:00 (xfr#4, to-chk=2/9)\n",
      "_weights and models/lenet5_actual_rm-96-32/model.lenet5_actual_rm-96-32.h5\n",
      "\n",
      "         15.52M  55%   14.79MB/s    0:00:00  \n",
      "         24.28M  87%   17.79MB/s    0:00:01 (xfr#5, to-chk=1/9)\n",
      "_weights and models/lenet5_actual_rm-96-32/weights.lenet5_actual_rm-96-32.h5\n",
      "\n",
      "         27.75M 100%   17.87MB/s    0:00:01 (xfr#6, to-chk=0/9)\n",
      "         27.75M 100%   17.87MB/s    0:00:01 (xfr#6, to-chk=0/9)\n",
      "         27.75M 100%   17.66MB/s    0:00:01 (xfr#6, to-chk=0/9)\n",
      "         27.75M 100%   17.66MB/s    0:00:01 (xfr#6, to-chk=0/9)\n",
      "\n",
      "sent 18.39M bytes  received 139 bytes  12.26M bytes/sec\n",
      "total size is 27.75M  speedup is 1.51\n",
      "\n",
      "rsync operation completed\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rm_96_32 = consecutiveModelTraining(input_size=96, batch_size=32, activation='relu', pooling='max')\n",
    "do_rsync()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "654b2b95-11e2-45aa-8213-bcf1753ca877",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T19:24:13.733228Z",
     "iopub.status.busy": "2021-06-19T19:24:13.733228Z",
     "iopub.status.idle": "2021-06-19T19:24:13.827236Z",
     "shell.execute_reply": "2021-06-19T19:24:13.826233Z",
     "shell.execute_reply.started": "2021-06-19T19:24:13.733228Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loss</th>\n",
       "      <th>categorical_accuracy</th>\n",
       "      <th>top-3</th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>PR-AUC</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>TP</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>...</th>\n",
       "      <th>val_categorical_accuracy</th>\n",
       "      <th>val_top-3</th>\n",
       "      <th>val_ROC-AUC</th>\n",
       "      <th>val_PR-AUC</th>\n",
       "      <th>val_precision</th>\n",
       "      <th>val_recall</th>\n",
       "      <th>val_TP</th>\n",
       "      <th>val_TN</th>\n",
       "      <th>val_FP</th>\n",
       "      <th>val_FN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>...</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.00</td>\n",
       "      <td>5.000000e+01</td>\n",
       "      <td>5.000000e+01</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.486779</td>\n",
       "      <td>0.079960</td>\n",
       "      <td>0.243038</td>\n",
       "      <td>0.494089</td>\n",
       "      <td>0.081869</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.000388</td>\n",
       "      <td>0.002881</td>\n",
       "      <td>0.004754</td>\n",
       "      <td>0.002995</td>\n",
       "      <td>0.000741</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.593001e-08</td>\n",
       "      <td>2.445733e-09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.486009</td>\n",
       "      <td>0.074447</td>\n",
       "      <td>0.229757</td>\n",
       "      <td>0.488142</td>\n",
       "      <td>0.080342</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>4.999999e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.486527</td>\n",
       "      <td>0.078125</td>\n",
       "      <td>0.240985</td>\n",
       "      <td>0.491823</td>\n",
       "      <td>0.081382</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.486727</td>\n",
       "      <td>0.079535</td>\n",
       "      <td>0.242754</td>\n",
       "      <td>0.493816</td>\n",
       "      <td>0.081740</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.486914</td>\n",
       "      <td>0.081610</td>\n",
       "      <td>0.246543</td>\n",
       "      <td>0.496101</td>\n",
       "      <td>0.082212</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.488840</td>\n",
       "      <td>0.086947</td>\n",
       "      <td>0.251991</td>\n",
       "      <td>0.501170</td>\n",
       "      <td>0.083540</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>8.333334e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66528.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6048.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            loss  categorical_accuracy      top-3    ROC-AUC     PR-AUC  \\\n",
       "count  50.000000             50.000000  50.000000  50.000000  50.000000   \n",
       "mean    2.486779              0.079960   0.243038   0.494089   0.081869   \n",
       "std     0.000388              0.002881   0.004754   0.002995   0.000741   \n",
       "min     2.486009              0.074447   0.229757   0.488142   0.080342   \n",
       "25%     2.486527              0.078125   0.240985   0.491823   0.081382   \n",
       "50%     2.486727              0.079535   0.242754   0.493816   0.081740   \n",
       "75%     2.486914              0.081610   0.246543   0.496101   0.082212   \n",
       "max     2.488840              0.086947   0.251991   0.501170   0.083540   \n",
       "\n",
       "       precision  recall    TP       TN    FP  ...  val_categorical_accuracy  \\\n",
       "count       50.0    50.0  50.0     50.0  50.0  ...                 50.000000   \n",
       "mean         0.0     0.0   0.0  99440.0   0.0  ...                  0.083333   \n",
       "std          0.0     0.0   0.0      0.0   0.0  ...                  0.000000   \n",
       "min          0.0     0.0   0.0  99440.0   0.0  ...                  0.083333   \n",
       "25%          0.0     0.0   0.0  99440.0   0.0  ...                  0.083333   \n",
       "50%          0.0     0.0   0.0  99440.0   0.0  ...                  0.083333   \n",
       "75%          0.0     0.0   0.0  99440.0   0.0  ...                  0.083333   \n",
       "max          0.0     0.0   0.0  99440.0   0.0  ...                  0.083333   \n",
       "\n",
       "       val_top-3   val_ROC-AUC    val_PR-AUC  val_precision  val_recall  \\\n",
       "count      50.00  5.000000e+01  5.000000e+01           50.0        50.0   \n",
       "mean        0.25  5.000000e-01  8.333334e-02            0.0         0.0   \n",
       "std         0.00  1.593001e-08  2.445733e-09            0.0         0.0   \n",
       "min         0.25  4.999999e-01  8.333334e-02            0.0         0.0   \n",
       "25%         0.25  5.000000e-01  8.333334e-02            0.0         0.0   \n",
       "50%         0.25  5.000000e-01  8.333334e-02            0.0         0.0   \n",
       "75%         0.25  5.000000e-01  8.333334e-02            0.0         0.0   \n",
       "max         0.25  5.000000e-01  8.333334e-02            0.0         0.0   \n",
       "\n",
       "       val_TP   val_TN  val_FP  val_FN  \n",
       "count    50.0     50.0    50.0    50.0  \n",
       "mean      0.0  66528.0     0.0  6048.0  \n",
       "std       0.0      0.0     0.0     0.0  \n",
       "min       0.0  66528.0     0.0  6048.0  \n",
       "25%       0.0  66528.0     0.0  6048.0  \n",
       "50%       0.0  66528.0     0.0  6048.0  \n",
       "75%       0.0  66528.0     0.0  6048.0  \n",
       "max       0.0  66528.0     0.0  6048.0  \n",
       "\n",
       "[8 rows x 22 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rm_96_32['history_df'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f1cf964-0c4f-46d6-85ac-64d9c189a3e5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a4de6f44-a4d7-4f8e-bc3d-defcc2c0865a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T19:24:13.829236Z",
     "iopub.status.busy": "2021-06-19T19:24:13.829236Z",
     "iopub.status.idle": "2021-06-19T21:31:00.417789Z",
     "shell.execute_reply": "2021-06-19T21:31:00.416793Z",
     "shell.execute_reply.started": "2021-06-19T19:24:13.829236Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 9072 images belonging to 12 classes.\n",
      "Found 6048 images belonging to 12 classes.\n",
      "\n",
      "\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_checkpoints/lenet5_actual_rm-96-64\n",
      "success\n",
      "D:\\OneDrive - leverage proactive deliverables\\_OTHERS\\MyNotebook\\MySkripsi/_weights and models/lenet5_actual_rm-96-64\n",
      "success\n",
      "\n",
      "Model: \"lenet5_actual_rm-96-64\"\n",
      "__________________________________________________________________________________________\n",
      "Layer (type)                                          Output Shape           Param #  \n",
      "==========================================================================================\n",
      "input (InputLayer)                                    [(None, 96, 96, 3)]    0        \n",
      "__________________________________________________________________________________________\n",
      "C1 (Conv2D)                                           (None, 92, 92, 6)      456      \n",
      "__________________________________________________________________________________________\n",
      "S2 (MaxPooling2D)                                     (None, 46, 46, 6)      0        \n",
      "__________________________________________________________________________________________\n",
      "C3 (Conv2D)                                           (None, 42, 42, 16)     2416     \n",
      "__________________________________________________________________________________________\n",
      "S4 (MaxPooling2D)                                     (None, 21, 21, 16)     0        \n",
      "__________________________________________________________________________________________\n",
      "flatten (Flatten)                                     (None, 7056)           0        \n",
      "__________________________________________________________________________________________\n",
      "C5 (Dense)                                            (None, 120)            846840   \n",
      "__________________________________________________________________________________________\n",
      "F6 (Dense)                                            (None, 84)             10164    \n",
      "__________________________________________________________________________________________\n",
      "OUTPUT (Dense)                                        (None, 12)             1020     \n",
      "==========================================================================================\n",
      "Total params: 860,896\n",
      "Trainable params: 860,896\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________\n",
      "\n",
      "\n",
      "{'batch_size': 64,\n",
      " 'epochs': 50,\n",
      " 'steps_per_epoch': 141,\n",
      " 'total_train': 9072,\n",
      " 'total_val': 6048,\n",
      " 'validation_steps': 94}\n",
      "\n",
      "\n",
      "lenet5_actual_rm-96-64\n",
      "Epoch 1/50\n",
      "  2/141 [..............................] - ETA: 7:47 - loss: 2.9827 - categorical_accuracy: 0.0625 - top-3: 0.2031 - ROC-AUC: 0.4856 - PR-AUC: 0.0799 - precision: 0.0000e+00 - recall: 0.0000e+00 - TP: 0.0000e+00 - TN: 1406.0000 - FP: 2.0000 - FN: 128.0000WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.6431s vs `on_train_batch_end` time: 6.0746s). Check your callbacks.\n",
      "141/141 [==============================] - 159s 1s/step - loss: 0.7186 - categorical_accuracy: 0.7557 - top-3: 0.9019 - ROC-AUC: 0.9734 - PR-AUC: 0.8610 - precision: 0.8843 - recall: 0.6914 - TP: 6228.0000 - TN: 98273.0000 - FP: 815.0000 - FN: 2780.0000 - val_loss: 0.1515 - val_categorical_accuracy: 0.9515 - val_top-3: 0.9953 - val_ROC-AUC: 0.9976 - val_PR-AUC: 0.9881 - val_precision: 0.9560 - val_recall: 0.9466 - val_TP: 5695.0000 - val_TN: 65914.0000 - val_FP: 262.0000 - val_FN: 321.0000\n",
      "Epoch 2/50\n",
      "141/141 [==============================] - 152s 1s/step - loss: 0.1564 - categorical_accuracy: 0.9522 - top-3: 0.9957 - ROC-AUC: 0.9971 - PR-AUC: 0.9863 - precision: 0.9583 - recall: 0.9462 - TP: 8523.0000 - TN: 98717.0000 - FP: 371.0000 - FN: 485.0000 - val_loss: 0.0726 - val_categorical_accuracy: 0.9767 - val_top-3: 0.9980 - val_ROC-AUC: 0.9993 - val_PR-AUC: 0.9967 - val_precision: 0.9809 - val_recall: 0.9742 - val_TP: 5861.0000 - val_TN: 66062.0000 - val_FP: 114.0000 - val_FN: 155.0000\n",
      "Epoch 3/50\n",
      "141/141 [==============================] - 148s 1s/step - loss: 0.0925 - categorical_accuracy: 0.9715 - top-3: 0.9976 - ROC-AUC: 0.9983 - PR-AUC: 0.9937 - precision: 0.9743 - recall: 0.9694 - TP: 8732.0000 - TN: 98858.0000 - FP: 230.0000 - FN: 276.0000 - val_loss: 0.0853 - val_categorical_accuracy: 0.9749 - val_top-3: 0.9972 - val_ROC-AUC: 0.9981 - val_PR-AUC: 0.9929 - val_precision: 0.9784 - val_recall: 0.9721 - val_TP: 5848.0000 - val_TN: 66047.0000 - val_FP: 129.0000 - val_FN: 168.0000\n",
      "Epoch 4/50\n",
      "141/141 [==============================] - 153s 1s/step - loss: 0.0752 - categorical_accuracy: 0.9764 - top-3: 0.9976 - ROC-AUC: 0.9989 - PR-AUC: 0.9953 - precision: 0.9794 - recall: 0.9751 - TP: 8784.0000 - TN: 98903.0000 - FP: 185.0000 - FN: 224.0000 - val_loss: 0.0660 - val_categorical_accuracy: 0.9807 - val_top-3: 0.9985 - val_ROC-AUC: 0.9987 - val_PR-AUC: 0.9953 - val_precision: 0.9835 - val_recall: 0.9787 - val_TP: 5888.0000 - val_TN: 66077.0000 - val_FP: 99.0000 - val_FN: 128.0000\n",
      "Epoch 5/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0860 - categorical_accuracy: 0.9759 - top-3: 0.9974 - ROC-AUC: 0.9976 - PR-AUC: 0.9927 - precision: 0.9784 - recall: 0.9739 - TP: 8773.0000 - TN: 98894.0000 - FP: 194.0000 - FN: 235.0000 - val_loss: 0.0591 - val_categorical_accuracy: 0.9827 - val_top-3: 0.9980 - val_ROC-AUC: 0.9987 - val_PR-AUC: 0.9960 - val_precision: 0.9843 - val_recall: 0.9807 - val_TP: 5900.0000 - val_TN: 66082.0000 - val_FP: 94.0000 - val_FN: 116.0000\n",
      "Epoch 6/50\n",
      "141/141 [==============================] - 153s 1s/step - loss: 0.0572 - categorical_accuracy: 0.9849 - top-3: 0.9981 - ROC-AUC: 0.9987 - PR-AUC: 0.9961 - precision: 0.9870 - recall: 0.9838 - TP: 8862.0000 - TN: 98971.0000 - FP: 117.0000 - FN: 146.0000 - val_loss: 0.1789 - val_categorical_accuracy: 0.9531 - val_top-3: 0.9942 - val_ROC-AUC: 0.9946 - val_PR-AUC: 0.9814 - val_precision: 0.9570 - val_recall: 0.9506 - val_TP: 5719.0000 - val_TN: 65919.0000 - val_FP: 257.0000 - val_FN: 297.0000\n",
      "Epoch 7/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.1366 - categorical_accuracy: 0.9621 - top-3: 0.9956 - ROC-AUC: 0.9968 - PR-AUC: 0.9881 - precision: 0.9663 - recall: 0.9576 - TP: 8626.0000 - TN: 98787.0000 - FP: 301.0000 - FN: 382.0000 - val_loss: 0.0986 - val_categorical_accuracy: 0.9694 - val_top-3: 0.9967 - val_ROC-AUC: 0.9986 - val_PR-AUC: 0.9939 - val_precision: 0.9742 - val_recall: 0.9661 - val_TP: 5812.0000 - val_TN: 66022.0000 - val_FP: 154.0000 - val_FN: 204.0000\n",
      "Epoch 8/50\n",
      "141/141 [==============================] - 154s 1s/step - loss: 0.1057 - categorical_accuracy: 0.9684 - top-3: 0.9954 - ROC-AUC: 0.9977 - PR-AUC: 0.9917 - precision: 0.9723 - recall: 0.9653 - TP: 8695.0000 - TN: 98840.0000 - FP: 248.0000 - FN: 313.0000 - val_loss: 0.0375 - val_categorical_accuracy: 0.9887 - val_top-3: 0.9983 - val_ROC-AUC: 0.9998 - val_PR-AUC: 0.9989 - val_precision: 0.9928 - val_recall: 0.9857 - val_TP: 5930.0000 - val_TN: 66133.0000 - val_FP: 43.0000 - val_FN: 86.0000\n",
      "Epoch 9/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0511 - categorical_accuracy: 0.9866 - top-3: 0.9980 - ROC-AUC: 0.9988 - PR-AUC: 0.9961 - precision: 0.9896 - recall: 0.9856 - TP: 8878.0000 - TN: 98995.0000 - FP: 93.0000 - FN: 130.0000 - val_loss: 0.0348 - val_categorical_accuracy: 0.9904 - val_top-3: 0.9995 - val_ROC-AUC: 0.9993 - val_PR-AUC: 0.9976 - val_precision: 0.9915 - val_recall: 0.9890 - val_TP: 5950.0000 - val_TN: 66125.0000 - val_FP: 51.0000 - val_FN: 66.0000\n",
      "Epoch 10/50\n",
      "141/141 [==============================] - 152s 1s/step - loss: 0.0466 - categorical_accuracy: 0.9876 - top-3: 0.9980 - ROC-AUC: 0.9989 - PR-AUC: 0.9968 - precision: 0.9899 - recall: 0.9862 - TP: 8884.0000 - TN: 98997.0000 - FP: 91.0000 - FN: 124.0000 - val_loss: 0.0482 - val_categorical_accuracy: 0.9867 - val_top-3: 0.9982 - val_ROC-AUC: 0.9991 - val_PR-AUC: 0.9970 - val_precision: 0.9895 - val_recall: 0.9850 - val_TP: 5926.0000 - val_TN: 66113.0000 - val_FP: 63.0000 - val_FN: 90.0000\n",
      "Epoch 11/50\n",
      "141/141 [==============================] - 149s 1s/step - loss: 0.0832 - categorical_accuracy: 0.9779 - top-3: 0.9972 - ROC-AUC: 0.9977 - PR-AUC: 0.9925 - precision: 0.9804 - recall: 0.9757 - TP: 8789.0000 - TN: 98912.0000 - FP: 176.0000 - FN: 219.0000 - val_loss: 0.0320 - val_categorical_accuracy: 0.9897 - val_top-3: 0.9983 - val_ROC-AUC: 0.9995 - val_PR-AUC: 0.9986 - val_precision: 0.9913 - val_recall: 0.9889 - val_TP: 5949.0000 - val_TN: 66124.0000 - val_FP: 52.0000 - val_FN: 67.0000\n",
      "Epoch 12/50\n",
      "141/141 [==============================] - 153s 1s/step - loss: 0.0669 - categorical_accuracy: 0.9841 - top-3: 0.9973 - ROC-AUC: 0.9980 - PR-AUC: 0.9943 - precision: 0.9860 - recall: 0.9829 - TP: 8854.0000 - TN: 98962.0000 - FP: 126.0000 - FN: 154.0000 - val_loss: 0.2162 - val_categorical_accuracy: 0.9460 - val_top-3: 0.9943 - val_ROC-AUC: 0.9939 - val_PR-AUC: 0.9772 - val_precision: 0.9540 - val_recall: 0.9413 - val_TP: 5663.0000 - val_TN: 65903.0000 - val_FP: 273.0000 - val_FN: 353.0000\n",
      "Epoch 13/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0857 - categorical_accuracy: 0.9795 - top-3: 0.9966 - ROC-AUC: 0.9979 - PR-AUC: 0.9927 - precision: 0.9841 - recall: 0.9768 - TP: 8799.0000 - TN: 98946.0000 - FP: 142.0000 - FN: 209.0000 - val_loss: 0.0629 - val_categorical_accuracy: 0.9825 - val_top-3: 0.9948 - val_ROC-AUC: 0.9990 - val_PR-AUC: 0.9965 - val_precision: 0.9912 - val_recall: 0.9772 - val_TP: 5879.0000 - val_TN: 66124.0000 - val_FP: 52.0000 - val_FN: 137.0000\n",
      "Epoch 14/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0592 - categorical_accuracy: 0.9847 - top-3: 0.9973 - ROC-AUC: 0.9986 - PR-AUC: 0.9955 - precision: 0.9885 - recall: 0.9828 - TP: 8853.0000 - TN: 98985.0000 - FP: 103.0000 - FN: 155.0000 - val_loss: 0.0441 - val_categorical_accuracy: 0.9885 - val_top-3: 0.9987 - val_ROC-AUC: 0.9985 - val_PR-AUC: 0.9957 - val_precision: 0.9898 - val_recall: 0.9874 - val_TP: 5940.0000 - val_TN: 66115.0000 - val_FP: 61.0000 - val_FN: 76.0000\n",
      "Epoch 15/50\n",
      "141/141 [==============================] - 151s 1s/step - loss: 0.0747 - categorical_accuracy: 0.9806 - top-3: 0.9966 - ROC-AUC: 0.9981 - PR-AUC: 0.9938 - precision: 0.9837 - recall: 0.9781 - TP: 8811.0000 - TN: 98942.0000 - FP: 146.0000 - FN: 197.0000 - val_loss: 0.0338 - val_categorical_accuracy: 0.9915 - val_top-3: 0.9988 - val_ROC-AUC: 0.9990 - val_PR-AUC: 0.9974 - val_precision: 0.9938 - val_recall: 0.9909 - val_TP: 5961.0000 - val_TN: 66139.0000 - val_FP: 37.0000 - val_FN: 55.0000\n",
      "Epoch 16/50\n",
      "141/141 [==============================] - 154s 1s/step - loss: 0.0431 - categorical_accuracy: 0.9895 - top-3: 0.9976 - ROC-AUC: 0.9988 - PR-AUC: 0.9964 - precision: 0.9915 - recall: 0.9876 - TP: 8896.0000 - TN: 99012.0000 - FP: 76.0000 - FN: 112.0000 - val_loss: 0.0331 - val_categorical_accuracy: 0.9904 - val_top-3: 0.9967 - val_ROC-AUC: 0.9993 - val_PR-AUC: 0.9980 - val_precision: 0.9940 - val_recall: 0.9889 - val_TP: 5949.0000 - val_TN: 66140.0000 - val_FP: 36.0000 - val_FN: 67.0000\n",
      "Epoch 17/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0356 - categorical_accuracy: 0.9902 - top-3: 0.9981 - ROC-AUC: 0.9994 - PR-AUC: 0.9980 - precision: 0.9933 - recall: 0.9890 - TP: 8909.0000 - TN: 99028.0000 - FP: 60.0000 - FN: 99.0000 - val_loss: 0.0299 - val_categorical_accuracy: 0.9917 - val_top-3: 0.9980 - val_ROC-AUC: 0.9994 - val_PR-AUC: 0.9984 - val_precision: 0.9947 - val_recall: 0.9900 - val_TP: 5956.0000 - val_TN: 66144.0000 - val_FP: 32.0000 - val_FN: 60.0000\n",
      "Epoch 18/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0481 - categorical_accuracy: 0.9890 - top-3: 0.9978 - ROC-AUC: 0.9986 - PR-AUC: 0.9959 - precision: 0.9912 - recall: 0.9869 - TP: 8890.0000 - TN: 99009.0000 - FP: 79.0000 - FN: 118.0000 - val_loss: 0.1145 - val_categorical_accuracy: 0.9820 - val_top-3: 0.9993 - val_ROC-AUC: 0.9962 - val_PR-AUC: 0.9876 - val_precision: 0.9825 - val_recall: 0.9806 - val_TP: 5899.0000 - val_TN: 66071.0000 - val_FP: 105.0000 - val_FN: 117.0000\n",
      "Epoch 19/50\n",
      "141/141 [==============================] - 149s 1s/step - loss: 0.0793 - categorical_accuracy: 0.9796 - top-3: 0.9972 - ROC-AUC: 0.9980 - PR-AUC: 0.9933 - precision: 0.9839 - recall: 0.9782 - TP: 8812.0000 - TN: 98944.0000 - FP: 144.0000 - FN: 196.0000 - val_loss: 0.0700 - val_categorical_accuracy: 0.9860 - val_top-3: 0.9965 - val_ROC-AUC: 0.9978 - val_PR-AUC: 0.9941 - val_precision: 0.9896 - val_recall: 0.9852 - val_TP: 5927.0000 - val_TN: 66114.0000 - val_FP: 62.0000 - val_FN: 89.0000\n",
      "Epoch 20/50\n",
      "141/141 [==============================] - 155s 1s/step - loss: 0.0636 - categorical_accuracy: 0.9843 - top-3: 0.9973 - ROC-AUC: 0.9983 - PR-AUC: 0.9948 - precision: 0.9868 - recall: 0.9826 - TP: 8851.0000 - TN: 98970.0000 - FP: 118.0000 - FN: 157.0000 - val_loss: 0.0496 - val_categorical_accuracy: 0.9914 - val_top-3: 0.9975 - val_ROC-AUC: 0.9985 - val_PR-AUC: 0.9957 - val_precision: 0.9932 - val_recall: 0.9905 - val_TP: 5959.0000 - val_TN: 66135.0000 - val_FP: 41.0000 - val_FN: 57.0000\n",
      "Epoch 21/50\n",
      "141/141 [==============================] - 149s 1s/step - loss: 0.0804 - categorical_accuracy: 0.9813 - top-3: 0.9967 - ROC-AUC: 0.9980 - PR-AUC: 0.9934 - precision: 0.9853 - recall: 0.9794 - TP: 8822.0000 - TN: 98956.0000 - FP: 132.0000 - FN: 186.0000 - val_loss: 0.0598 - val_categorical_accuracy: 0.9811 - val_top-3: 0.9942 - val_ROC-AUC: 0.9989 - val_PR-AUC: 0.9967 - val_precision: 0.9886 - val_recall: 0.9776 - val_TP: 5881.0000 - val_TN: 66108.0000 - val_FP: 68.0000 - val_FN: 135.0000\n",
      "Epoch 22/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0572 - categorical_accuracy: 0.9863 - top-3: 0.9963 - ROC-AUC: 0.9988 - PR-AUC: 0.9970 - precision: 0.9902 - recall: 0.9848 - TP: 8871.0000 - TN: 99000.0000 - FP: 88.0000 - FN: 137.0000 - val_loss: 0.0426 - val_categorical_accuracy: 0.9917 - val_top-3: 0.9968 - val_ROC-AUC: 0.9988 - val_PR-AUC: 0.9971 - val_precision: 0.9943 - val_recall: 0.9900 - val_TP: 5956.0000 - val_TN: 66142.0000 - val_FP: 34.0000 - val_FN: 60.0000\n",
      "Epoch 23/50\n",
      "141/141 [==============================] - 149s 1s/step - loss: 0.1021 - categorical_accuracy: 0.9797 - top-3: 0.9944 - ROC-AUC: 0.9973 - PR-AUC: 0.9923 - precision: 0.9860 - recall: 0.9765 - TP: 8796.0000 - TN: 98963.0000 - FP: 125.0000 - FN: 212.0000 - val_loss: 0.0431 - val_categorical_accuracy: 0.9914 - val_top-3: 0.9972 - val_ROC-AUC: 0.9993 - val_PR-AUC: 0.9977 - val_precision: 0.9947 - val_recall: 0.9897 - val_TP: 5954.0000 - val_TN: 66144.0000 - val_FP: 32.0000 - val_FN: 62.0000\n",
      "Epoch 24/50\n",
      "141/141 [==============================] - 152s 1s/step - loss: 0.0620 - categorical_accuracy: 0.9860 - top-3: 0.9958 - ROC-AUC: 0.9986 - PR-AUC: 0.9955 - precision: 0.9907 - recall: 0.9838 - TP: 8862.0000 - TN: 99005.0000 - FP: 83.0000 - FN: 146.0000 - val_loss: 0.0731 - val_categorical_accuracy: 0.9824 - val_top-3: 0.9940 - val_ROC-AUC: 0.9981 - val_PR-AUC: 0.9938 - val_precision: 0.9888 - val_recall: 0.9796 - val_TP: 5893.0000 - val_TN: 66109.0000 - val_FP: 67.0000 - val_FN: 123.0000\n",
      "Epoch 25/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0829 - categorical_accuracy: 0.9835 - top-3: 0.9952 - ROC-AUC: 0.9983 - PR-AUC: 0.9945 - precision: 0.9891 - recall: 0.9787 - TP: 8816.0000 - TN: 98991.0000 - FP: 97.0000 - FN: 192.0000 - val_loss: 0.0413 - val_categorical_accuracy: 0.9885 - val_top-3: 0.9955 - val_ROC-AUC: 0.9987 - val_PR-AUC: 0.9967 - val_precision: 0.9936 - val_recall: 0.9867 - val_TP: 5936.0000 - val_TN: 66138.0000 - val_FP: 38.0000 - val_FN: 80.0000\n",
      "Epoch 26/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0542 - categorical_accuracy: 0.9871 - top-3: 0.9970 - ROC-AUC: 0.9985 - PR-AUC: 0.9958 - precision: 0.9907 - recall: 0.9857 - TP: 8879.0000 - TN: 99005.0000 - FP: 83.0000 - FN: 129.0000 - val_loss: 0.0344 - val_categorical_accuracy: 0.9910 - val_top-3: 0.9952 - val_ROC-AUC: 0.9995 - val_PR-AUC: 0.9987 - val_precision: 0.9963 - val_recall: 0.9880 - val_TP: 5944.0000 - val_TN: 66154.0000 - val_FP: 22.0000 - val_FN: 72.0000\n",
      "Epoch 27/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0726 - categorical_accuracy: 0.9863 - top-3: 0.9971 - ROC-AUC: 0.9981 - PR-AUC: 0.9941 - precision: 0.9890 - recall: 0.9851 - TP: 8874.0000 - TN: 98989.0000 - FP: 99.0000 - FN: 134.0000 - val_loss: 0.1010 - val_categorical_accuracy: 0.9787 - val_top-3: 0.9943 - val_ROC-AUC: 0.9968 - val_PR-AUC: 0.9909 - val_precision: 0.9839 - val_recall: 0.9759 - val_TP: 5871.0000 - val_TN: 66080.0000 - val_FP: 96.0000 - val_FN: 145.0000\n",
      "Epoch 28/50\n",
      "141/141 [==============================] - 154s 1s/step - loss: 0.0754 - categorical_accuracy: 0.9845 - top-3: 0.9961 - ROC-AUC: 0.9979 - PR-AUC: 0.9932 - precision: 0.9887 - recall: 0.9820 - TP: 8846.0000 - TN: 98987.0000 - FP: 101.0000 - FN: 162.0000 - val_loss: 0.0419 - val_categorical_accuracy: 0.9922 - val_top-3: 0.9975 - val_ROC-AUC: 0.9988 - val_PR-AUC: 0.9966 - val_precision: 0.9947 - val_recall: 0.9910 - val_TP: 5962.0000 - val_TN: 66144.0000 - val_FP: 32.0000 - val_FN: 54.0000\n",
      "Epoch 29/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0602 - categorical_accuracy: 0.9883 - top-3: 0.9967 - ROC-AUC: 0.9985 - PR-AUC: 0.9953 - precision: 0.9916 - recall: 0.9869 - TP: 8890.0000 - TN: 99013.0000 - FP: 75.0000 - FN: 118.0000 - val_loss: 0.0428 - val_categorical_accuracy: 0.9912 - val_top-3: 0.9965 - val_ROC-AUC: 0.9992 - val_PR-AUC: 0.9970 - val_precision: 0.9940 - val_recall: 0.9899 - val_TP: 5955.0000 - val_TN: 66140.0000 - val_FP: 36.0000 - val_FN: 61.0000\n",
      "Epoch 30/50\n",
      "141/141 [==============================] - 154s 1s/step - loss: 0.0685 - categorical_accuracy: 0.9863 - top-3: 0.9959 - ROC-AUC: 0.9984 - PR-AUC: 0.9950 - precision: 0.9914 - recall: 0.9843 - TP: 8867.0000 - TN: 99011.0000 - FP: 77.0000 - FN: 141.0000 - val_loss: 0.0680 - val_categorical_accuracy: 0.9865 - val_top-3: 0.9960 - val_ROC-AUC: 0.9984 - val_PR-AUC: 0.9956 - val_precision: 0.9911 - val_recall: 0.9832 - val_TP: 5915.0000 - val_TN: 66123.0000 - val_FP: 53.0000 - val_FN: 101.0000\n",
      "Epoch 31/50\n",
      "141/141 [==============================] - 149s 1s/step - loss: 0.0403 - categorical_accuracy: 0.9909 - top-3: 0.9967 - ROC-AUC: 0.9990 - PR-AUC: 0.9969 - precision: 0.9952 - recall: 0.9895 - TP: 8913.0000 - TN: 99045.0000 - FP: 43.0000 - FN: 95.0000 - val_loss: 0.0267 - val_categorical_accuracy: 0.9924 - val_top-3: 0.9960 - val_ROC-AUC: 0.9997 - val_PR-AUC: 0.9990 - val_precision: 0.9978 - val_recall: 0.9902 - val_TP: 5957.0000 - val_TN: 66163.0000 - val_FP: 13.0000 - val_FN: 59.0000\n",
      "Epoch 32/50\n",
      "141/141 [==============================] - 153s 1s/step - loss: 0.0533 - categorical_accuracy: 0.9901 - top-3: 0.9968 - ROC-AUC: 0.9985 - PR-AUC: 0.9958 - precision: 0.9935 - recall: 0.9881 - TP: 8901.0000 - TN: 99030.0000 - FP: 58.0000 - FN: 107.0000 - val_loss: 0.0616 - val_categorical_accuracy: 0.9874 - val_top-3: 0.9950 - val_ROC-AUC: 0.9986 - val_PR-AUC: 0.9962 - val_precision: 0.9930 - val_recall: 0.9855 - val_TP: 5929.0000 - val_TN: 66134.0000 - val_FP: 42.0000 - val_FN: 87.0000\n",
      "Epoch 33/50\n",
      "141/141 [==============================] - 152s 1s/step - loss: 0.1299 - categorical_accuracy: 0.9805 - top-3: 0.9929 - ROC-AUC: 0.9968 - PR-AUC: 0.9902 - precision: 0.9874 - recall: 0.9771 - TP: 8802.0000 - TN: 98976.0000 - FP: 112.0000 - FN: 206.0000 - val_loss: 0.0984 - val_categorical_accuracy: 0.9812 - val_top-3: 0.9880 - val_ROC-AUC: 0.9975 - val_PR-AUC: 0.9926 - val_precision: 0.9902 - val_recall: 0.9776 - val_TP: 5881.0000 - val_TN: 66118.0000 - val_FP: 58.0000 - val_FN: 135.0000\n",
      "Epoch 34/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0806 - categorical_accuracy: 0.9833 - top-3: 0.9938 - ROC-AUC: 0.9981 - PR-AUC: 0.9945 - precision: 0.9895 - recall: 0.9805 - TP: 8832.0000 - TN: 98994.0000 - FP: 94.0000 - FN: 176.0000 - val_loss: 0.0645 - val_categorical_accuracy: 0.9796 - val_top-3: 0.9947 - val_ROC-AUC: 0.9993 - val_PR-AUC: 0.9969 - val_precision: 0.9890 - val_recall: 0.9734 - val_TP: 5856.0000 - val_TN: 66111.0000 - val_FP: 65.0000 - val_FN: 160.0000\n",
      "Epoch 35/50\n",
      "141/141 [==============================] - 151s 1s/step - loss: 0.0527 - categorical_accuracy: 0.9898 - top-3: 0.9956 - ROC-AUC: 0.9990 - PR-AUC: 0.9970 - precision: 0.9948 - recall: 0.9855 - TP: 8877.0000 - TN: 99042.0000 - FP: 46.0000 - FN: 131.0000 - val_loss: 0.0189 - val_categorical_accuracy: 0.9943 - val_top-3: 0.9975 - val_ROC-AUC: 0.9996 - val_PR-AUC: 0.9990 - val_precision: 0.9977 - val_recall: 0.9925 - val_TP: 5971.0000 - val_TN: 66162.0000 - val_FP: 14.0000 - val_FN: 45.0000\n",
      "Epoch 36/50\n",
      "141/141 [==============================] - 154s 1s/step - loss: 0.0438 - categorical_accuracy: 0.9899 - top-3: 0.9961 - ROC-AUC: 0.9991 - PR-AUC: 0.9972 - precision: 0.9941 - recall: 0.9881 - TP: 8901.0000 - TN: 99035.0000 - FP: 53.0000 - FN: 107.0000 - val_loss: 0.0810 - val_categorical_accuracy: 0.9865 - val_top-3: 0.9929 - val_ROC-AUC: 0.9974 - val_PR-AUC: 0.9922 - val_precision: 0.9926 - val_recall: 0.9842 - val_TP: 5921.0000 - val_TN: 66132.0000 - val_FP: 44.0000 - val_FN: 95.0000\n",
      "Epoch 37/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.1132 - categorical_accuracy: 0.9797 - top-3: 0.9902 - ROC-AUC: 0.9974 - PR-AUC: 0.9918 - precision: 0.9893 - recall: 0.9741 - TP: 8775.0000 - TN: 98993.0000 - FP: 95.0000 - FN: 233.0000 - val_loss: 0.1287 - val_categorical_accuracy: 0.9799 - val_top-3: 0.9919 - val_ROC-AUC: 0.9968 - val_PR-AUC: 0.9897 - val_precision: 0.9879 - val_recall: 0.9762 - val_TP: 5873.0000 - val_TN: 66104.0000 - val_FP: 72.0000 - val_FN: 143.0000\n",
      "Epoch 38/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0891 - categorical_accuracy: 0.9829 - top-3: 0.9933 - ROC-AUC: 0.9977 - PR-AUC: 0.9929 - precision: 0.9911 - recall: 0.9801 - TP: 8829.0000 - TN: 99009.0000 - FP: 79.0000 - FN: 179.0000 - val_loss: 0.0495 - val_categorical_accuracy: 0.9892 - val_top-3: 0.9935 - val_ROC-AUC: 0.9990 - val_PR-AUC: 0.9968 - val_precision: 0.9960 - val_recall: 0.9870 - val_TP: 5938.0000 - val_TN: 66152.0000 - val_FP: 24.0000 - val_FN: 78.0000\n",
      "Epoch 39/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.1164 - categorical_accuracy: 0.9827 - top-3: 0.9917 - ROC-AUC: 0.9977 - PR-AUC: 0.9930 - precision: 0.9910 - recall: 0.9789 - TP: 8818.0000 - TN: 99008.0000 - FP: 80.0000 - FN: 190.0000 - val_loss: 0.0942 - val_categorical_accuracy: 0.9872 - val_top-3: 0.9937 - val_ROC-AUC: 0.9979 - val_PR-AUC: 0.9935 - val_precision: 0.9938 - val_recall: 0.9835 - val_TP: 5917.0000 - val_TN: 66139.0000 - val_FP: 37.0000 - val_FN: 99.0000\n",
      "Epoch 40/50\n",
      "141/141 [==============================] - 154s 1s/step - loss: 0.0946 - categorical_accuracy: 0.9827 - top-3: 0.9910 - ROC-AUC: 0.9978 - PR-AUC: 0.9932 - precision: 0.9909 - recall: 0.9794 - TP: 8822.0000 - TN: 99007.0000 - FP: 81.0000 - FN: 186.0000 - val_loss: 0.0475 - val_categorical_accuracy: 0.9892 - val_top-3: 0.9945 - val_ROC-AUC: 0.9993 - val_PR-AUC: 0.9978 - val_precision: 0.9963 - val_recall: 0.9865 - val_TP: 5935.0000 - val_TN: 66154.0000 - val_FP: 22.0000 - val_FN: 81.0000\n",
      "Epoch 41/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0598 - categorical_accuracy: 0.9879 - top-3: 0.9946 - ROC-AUC: 0.9986 - PR-AUC: 0.9961 - precision: 0.9940 - recall: 0.9851 - TP: 8874.0000 - TN: 99034.0000 - FP: 54.0000 - FN: 134.0000 - val_loss: 0.0706 - val_categorical_accuracy: 0.9905 - val_top-3: 0.9948 - val_ROC-AUC: 0.9983 - val_PR-AUC: 0.9949 - val_precision: 0.9950 - val_recall: 0.9899 - val_TP: 5955.0000 - val_TN: 66146.0000 - val_FP: 30.0000 - val_FN: 61.0000\n",
      "Epoch 42/50\n",
      "141/141 [==============================] - 152s 1s/step - loss: 0.0335 - categorical_accuracy: 0.9928 - top-3: 0.9960 - ROC-AUC: 0.9994 - PR-AUC: 0.9981 - precision: 0.9967 - recall: 0.9911 - TP: 8928.0000 - TN: 99058.0000 - FP: 30.0000 - FN: 80.0000 - val_loss: 0.0223 - val_categorical_accuracy: 0.9938 - val_top-3: 0.9960 - val_ROC-AUC: 0.9996 - val_PR-AUC: 0.9988 - val_precision: 0.9980 - val_recall: 0.9929 - val_TP: 5973.0000 - val_TN: 66164.0000 - val_FP: 12.0000 - val_FN: 43.0000\n",
      "Epoch 43/50\n",
      "141/141 [==============================] - 151s 1s/step - loss: 0.0954 - categorical_accuracy: 0.9806 - top-3: 0.9893 - ROC-AUC: 0.9983 - PR-AUC: 0.9941 - precision: 0.9943 - recall: 0.9760 - TP: 8792.0000 - TN: 99038.0000 - FP: 50.0000 - FN: 216.0000 - val_loss: 0.0448 - val_categorical_accuracy: 0.9892 - val_top-3: 0.9967 - val_ROC-AUC: 0.9989 - val_PR-AUC: 0.9968 - val_precision: 0.9970 - val_recall: 0.9875 - val_TP: 5941.0000 - val_TN: 66158.0000 - val_FP: 18.0000 - val_FN: 75.0000\n",
      "Epoch 44/50\n",
      "141/141 [==============================] - 152s 1s/step - loss: 0.0704 - categorical_accuracy: 0.9888 - top-3: 0.9943 - ROC-AUC: 0.9987 - PR-AUC: 0.9957 - precision: 0.9950 - recall: 0.9868 - TP: 8889.0000 - TN: 99043.0000 - FP: 45.0000 - FN: 119.0000 - val_loss: 0.1021 - val_categorical_accuracy: 0.9869 - val_top-3: 0.9935 - val_ROC-AUC: 0.9970 - val_PR-AUC: 0.9916 - val_precision: 0.9916 - val_recall: 0.9834 - val_TP: 5916.0000 - val_TN: 66126.0000 - val_FP: 50.0000 - val_FN: 100.0000\n",
      "Epoch 45/50\n",
      "141/141 [==============================] - 149s 1s/step - loss: 0.0590 - categorical_accuracy: 0.9897 - top-3: 0.9946 - ROC-AUC: 0.9991 - PR-AUC: 0.9974 - precision: 0.9955 - recall: 0.9876 - TP: 8896.0000 - TN: 99048.0000 - FP: 40.0000 - FN: 112.0000 - val_loss: 0.0304 - val_categorical_accuracy: 0.9920 - val_top-3: 0.9945 - val_ROC-AUC: 0.9994 - val_PR-AUC: 0.9983 - val_precision: 0.9980 - val_recall: 0.9909 - val_TP: 5961.0000 - val_TN: 66164.0000 - val_FP: 12.0000 - val_FN: 55.0000\n",
      "Epoch 46/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0694 - categorical_accuracy: 0.9879 - top-3: 0.9925 - ROC-AUC: 0.9989 - PR-AUC: 0.9967 - precision: 0.9956 - recall: 0.9856 - TP: 8878.0000 - TN: 99049.0000 - FP: 39.0000 - FN: 130.0000 - val_loss: 0.0529 - val_categorical_accuracy: 0.9889 - val_top-3: 0.9925 - val_ROC-AUC: 0.9990 - val_PR-AUC: 0.9967 - val_precision: 0.9963 - val_recall: 0.9855 - val_TP: 5929.0000 - val_TN: 66154.0000 - val_FP: 22.0000 - val_FN: 87.0000\n",
      "Epoch 47/50\n",
      "141/141 [==============================] - 152s 1s/step - loss: 0.0629 - categorical_accuracy: 0.9918 - top-3: 0.9956 - ROC-AUC: 0.9990 - PR-AUC: 0.9968 - precision: 0.9961 - recall: 0.9886 - TP: 8905.0000 - TN: 99053.0000 - FP: 35.0000 - FN: 103.0000 - val_loss: 0.0368 - val_categorical_accuracy: 0.9938 - val_top-3: 0.9978 - val_ROC-AUC: 0.9989 - val_PR-AUC: 0.9970 - val_precision: 0.9963 - val_recall: 0.9934 - val_TP: 5976.0000 - val_TN: 66154.0000 - val_FP: 22.0000 - val_FN: 40.0000\n",
      "Epoch 48/50\n",
      "141/141 [==============================] - 150s 1s/step - loss: 0.0486 - categorical_accuracy: 0.9910 - top-3: 0.9957 - ROC-AUC: 0.9986 - PR-AUC: 0.9964 - precision: 0.9956 - recall: 0.9891 - TP: 8910.0000 - TN: 99049.0000 - FP: 39.0000 - FN: 98.0000 - val_loss: 0.2164 - val_categorical_accuracy: 0.9814 - val_top-3: 0.9920 - val_ROC-AUC: 0.9956 - val_PR-AUC: 0.9861 - val_precision: 0.9859 - val_recall: 0.9772 - val_TP: 5879.0000 - val_TN: 66092.0000 - val_FP: 84.0000 - val_FN: 137.0000\n",
      "Epoch 49/50\n",
      "141/141 [==============================] - 149s 1s/step - loss: 0.0646 - categorical_accuracy: 0.9908 - top-3: 0.9947 - ROC-AUC: 0.9988 - PR-AUC: 0.9962 - precision: 0.9955 - recall: 0.9883 - TP: 8903.0000 - TN: 99048.0000 - FP: 40.0000 - FN: 105.0000 - val_loss: 0.0298 - val_categorical_accuracy: 0.9953 - val_top-3: 0.9967 - val_ROC-AUC: 0.9990 - val_PR-AUC: 0.9975 - val_precision: 0.9975 - val_recall: 0.9950 - val_TP: 5986.0000 - val_TN: 66161.0000 - val_FP: 15.0000 - val_FN: 30.0000\n",
      "Epoch 50/50\n",
      "141/141 [==============================] - 153s 1s/step - loss: 0.1144 - categorical_accuracy: 0.9817 - top-3: 0.9906 - ROC-AUC: 0.9974 - PR-AUC: 0.9918 - precision: 0.9913 - recall: 0.9784 - TP: 8813.0000 - TN: 99011.0000 - FP: 77.0000 - FN: 195.0000 - val_loss: 0.0362 - val_categorical_accuracy: 0.9924 - val_top-3: 0.9967 - val_ROC-AUC: 0.9992 - val_PR-AUC: 0.9978 - val_precision: 0.9967 - val_recall: 0.9910 - val_TP: 5962.0000 - val_TN: 66156.0000 - val_FP: 20.0000 - val_FN: 54.0000\n",
      "-----\n",
      "(7599574.03 ms) == (126m:39s)\n",
      "-----\n",
      "\n",
      "\n",
      "24\n",
      "['lenet5_actual_rm-96-64.tensorboard',\n",
      " 'lenet5_actual_rm-96-64.weights.001_0.9515_0.1515.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.002_0.9767_0.0726.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.004_0.9807_0.0660.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.005.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.005_0.9827_0.0591.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.008_0.9887_0.0375.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.009_0.9904_0.0348.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.010.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.011_0.9897_0.0320.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.015.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.015_0.9915_0.0338.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.017_0.9917_0.0299.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.020.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.025.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.028_0.9922_0.0419.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.030.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.031_0.9924_0.0267.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.035.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.035_0.9943_0.0189.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.040.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.045.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.049_0.9953_0.0298.hdf5',\n",
      " 'lenet5_actual_rm-96-64.weights.050.hdf5']\n",
      "3\n",
      "['history.lenet5_actual_rm-96-64.csv',\n",
      " 'model.lenet5_actual_rm-96-64.h5',\n",
      " 'weights.lenet5_actual_rm-96-64.h5']\n",
      "\n",
      "all process done, please recheck before terminating runtime session\n",
      "\n",
      "don't forget to save the model.fit() verbose output\n",
      "\n",
      "\n",
      "Returned values using 232 bytes of memory now\n",
      "building file list ... done\n",
      "_checkpoints/\n",
      "_checkpoints/lenet5_actual_rm-96-64/\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.001_0.9515_0.1515.hdf5\n",
      "\n",
      "         32.77K   0%    0.00kB/s    0:00:00  \n",
      "          3.47M   1%   17.81MB/s    0:00:00 (xfr#1, to-chk=38/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.002_0.9767_0.0726.hdf5\n",
      "\n",
      "          6.94M   3%   18.65MB/s    0:00:00 (xfr#2, to-chk=37/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.004_0.9807_0.0660.hdf5\n",
      "\n",
      "         10.40M   5%   18.56MB/s    0:00:00 (xfr#3, to-chk=36/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.005.hdf5\n",
      "\n",
      "         13.87M   7%   18.77MB/s    0:00:00 (xfr#4, to-chk=35/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.005_0.9827_0.0591.hdf5\n",
      "\n",
      "         17.34M   9%   18.82MB/s    0:00:00 (xfr#5, to-chk=34/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.008_0.9887_0.0375.hdf5\n",
      "\n",
      "         19.83M  10%   18.88MB/s    0:00:08  \n",
      "         20.81M  11%   18.94MB/s    0:00:01 (xfr#6, to-chk=33/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.009_0.9904_0.0348.hdf5\n",
      "\n",
      "         24.28M  13%   18.98MB/s    0:00:01 (xfr#7, to-chk=32/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.010.hdf5\n",
      "\n",
      "         27.74M  14%   19.10MB/s    0:00:01 (xfr#8, to-chk=31/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.011_0.9897_0.0320.hdf5\n",
      "\n",
      "         31.21M  16%   19.13MB/s    0:00:01 (xfr#9, to-chk=30/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.015.hdf5\n",
      "\n",
      "         34.68M  18%   19.19MB/s    0:00:01 (xfr#10, to-chk=29/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.015_0.9915_0.0338.hdf5\n",
      "\n",
      "         38.15M  20%   19.25MB/s    0:00:01 (xfr#11, to-chk=28/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.017_0.9917_0.0299.hdf5\n",
      "\n",
      "         40.48M  21%   19.29MB/s    0:00:07  \n",
      "         41.62M  22%   19.34MB/s    0:00:02 (xfr#12, to-chk=27/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.020.hdf5\n",
      "\n",
      "         45.09M  24%   19.36MB/s    0:00:02 (xfr#13, to-chk=26/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.025.hdf5\n",
      "\n",
      "         48.55M  26%   19.44MB/s    0:00:02 (xfr#14, to-chk=25/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.028_0.9922_0.0419.hdf5\n",
      "\n",
      "\n",
      "         52.02M  28%   19.45MB/s    0:00:02 (xfr#15, to-chk=24/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.030.hdf5\n",
      "\n",
      "         55.49M  29%   19.52MB/s    0:00:02 (xfr#16, to-chk=23/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.031_0.9924_0.0267.hdf5\n",
      "\n",
      "         58.96M  31%   19.55MB/s    0:00:02 (xfr#17, to-chk=22/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.035.hdf5\n",
      "\n",
      "         61.64M  33%   19.59MB/s    0:00:06  \n",
      "         62.43M  33%   19.60MB/s    0:00:03 (xfr#18, to-chk=21/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.035_0.9943_0.0189.hdf5\n",
      "\n",
      "         65.89M  35%   19.64MB/s    0:00:03 (xfr#19, to-chk=20/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.040.hdf5\n",
      "\n",
      "         69.36M  37%   19.65MB/s    0:00:03 (xfr#20, to-chk=19/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.045.hdf5\n",
      "\n",
      "         72.83M  39%   19.70MB/s    0:00:03 (xfr#21, to-chk=18/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.049_0.9953_0.0298.hdf5\n",
      "\n",
      "         76.30M  41%   19.73MB/s    0:00:03 (xfr#22, to-chk=17/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.weights.050.hdf5\n",
      "\n",
      "         79.77M  43%   19.74MB/s    0:00:03 (xfr#23, to-chk=16/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/events.out.tfevents.1624130656.ELITERAIHAN.18796.811838.v2\n",
      "\n",
      "         81.98M  44%   20.13MB/s    0:00:03 (xfr#24, to-chk=13/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/events.out.tfevents.1624130665.ELITERAIHAN.profile-empty\n",
      "\n",
      "         81.98M  44%   20.13MB/s    0:00:03 (xfr#25, to-chk=12/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/profile/\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/profile/2021_06_19_19_24_25/\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/profile/2021_06_19_19_24_25/ELITERAIHAN.input_pipeline.pb\n",
      "\n",
      "         81.98M  44%   20.13MB/s    0:00:03 (xfr#26, to-chk=8/104) \n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/profile/2021_06_19_19_24_25/ELITERAIHAN.kernel_stats.pb\n",
      "\n",
      "         81.98M  44%   20.12MB/s    0:00:03 (xfr#27, to-chk=7/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/profile/2021_06_19_19_24_25/ELITERAIHAN.memory_profile.json.gz\n",
      "\n",
      "         81.99M  44%   20.12MB/s    0:00:03 (xfr#28, to-chk=6/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/profile/2021_06_19_19_24_25/ELITERAIHAN.overview_page.pb\n",
      "\n",
      "         81.99M  44%   20.12MB/s    0:00:03 (xfr#29, to-chk=5/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/profile/2021_06_19_19_24_25/ELITERAIHAN.tensorflow_stats.pb\n",
      "\n",
      "         82.02M  44%   20.12MB/s    0:00:03 (xfr#30, to-chk=4/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/profile/2021_06_19_19_24_25/ELITERAIHAN.trace.json.gz\n",
      "\n",
      "         82.03M  44%   20.12MB/s    0:00:03 (xfr#31, to-chk=3/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/train/plugins/profile/2021_06_19_19_24_25/ELITERAIHAN.xplane.pb\n",
      "\n",
      "         82.12M  44%   20.12MB/s    0:00:03 (xfr#32, to-chk=2/104)\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/validation/\n",
      "_checkpoints/lenet5_actual_rm-96-64/lenet5_actual_rm-96-64.tensorboard/validation/events.out.tfevents.1624130757.ELITERAIHAN.18796.816245.v2\n",
      "\n",
      "         82.14M  44%   20.13MB/s    0:00:03 (xfr#33, to-chk=0/104)\n",
      "         82.14M  44%   20.13MB/s    0:00:03 (xfr#33, to-chk=0/104)\n",
      "         82.14M  44%   19.59MB/s    0:00:03 (xfr#33, to-chk=0/104)\n",
      "         82.14M  44%   19.59MB/s    0:00:03 (xfr#33, to-chk=0/104)\n",
      "\n",
      "sent 73.70M bytes  received 667 bytes  16.38M bytes/sec\n",
      "total size is 185.30M  speedup is 2.51\n",
      "building file list ... done\n",
      "_weights and models/\n",
      "_weights and models/lenet5_actual_rm-96-64/\n",
      "_weights and models/lenet5_actual_rm-96-64/history.lenet5_actual_rm-96-64.csv\n",
      "\n",
      "         16.19K   0%    0.00kB/s    0:00:00  \n",
      "         16.19K   0%    0.00kB/s    0:00:00 (xfr#1, to-chk=2/13)\n",
      "_weights and models/lenet5_actual_rm-96-64/model.lenet5_actual_rm-96-64.h5\n",
      "\n",
      "         10.41M  25%   24.12MB/s    0:00:00 (xfr#2, to-chk=1/13)\n",
      "_weights and models/lenet5_actual_rm-96-64/weights.lenet5_actual_rm-96-64.h5\n",
      "\n",
      "         13.88M  33%   22.99MB/s    0:00:00 (xfr#3, to-chk=0/13)\n",
      "         13.88M  33%   22.99MB/s    0:00:00 (xfr#3, to-chk=0/13)\n",
      "         13.88M  33%   22.33MB/s    0:00:00 (xfr#3, to-chk=0/13)\n",
      "         13.88M  33%   22.33MB/s    0:00:00 (xfr#3, to-chk=0/13)\n",
      "\n",
      "sent 10.85M bytes  received 79 bytes  7.24M bytes/sec\n",
      "total size is 41.62M  speedup is 3.84\n",
      "\n",
      "rsync operation completed\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rm_96_64 = consecutiveModelTraining(input_size=96, batch_size=64, activation='relu', pooling='max')\n",
    "do_rsync()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f2ea9fb8-0ede-4f84-868f-a1428d956e98",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-19T22:30:40.826362Z",
     "iopub.status.busy": "2021-06-19T22:30:40.826362Z",
     "iopub.status.idle": "2021-06-19T22:30:40.923369Z",
     "shell.execute_reply": "2021-06-19T22:30:40.922368Z",
     "shell.execute_reply.started": "2021-06-19T22:30:40.826362Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loss</th>\n",
       "      <th>categorical_accuracy</th>\n",
       "      <th>top-3</th>\n",
       "      <th>ROC-AUC</th>\n",
       "      <th>PR-AUC</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>TP</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>...</th>\n",
       "      <th>val_categorical_accuracy</th>\n",
       "      <th>val_top-3</th>\n",
       "      <th>val_ROC-AUC</th>\n",
       "      <th>val_PR-AUC</th>\n",
       "      <th>val_precision</th>\n",
       "      <th>val_recall</th>\n",
       "      <th>val_TP</th>\n",
       "      <th>val_TN</th>\n",
       "      <th>val_FP</th>\n",
       "      <th>val_FN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.087450</td>\n",
       "      <td>0.978905</td>\n",
       "      <td>0.993706</td>\n",
       "      <td>0.997798</td>\n",
       "      <td>0.991854</td>\n",
       "      <td>0.986167</td>\n",
       "      <td>0.975200</td>\n",
       "      <td>8784.600000</td>\n",
       "      <td>98968.700000</td>\n",
       "      <td>119.300000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.984807</td>\n",
       "      <td>0.995864</td>\n",
       "      <td>0.998432</td>\n",
       "      <td>0.994938</td>\n",
       "      <td>0.989376</td>\n",
       "      <td>0.982493</td>\n",
       "      <td>5910.680000</td>\n",
       "      <td>66112.580000</td>\n",
       "      <td>63.420000</td>\n",
       "      <td>105.320000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.094872</td>\n",
       "      <td>0.033093</td>\n",
       "      <td>0.013431</td>\n",
       "      <td>0.003581</td>\n",
       "      <td>0.019030</td>\n",
       "      <td>0.016586</td>\n",
       "      <td>0.041777</td>\n",
       "      <td>376.324111</td>\n",
       "      <td>121.551408</td>\n",
       "      <td>121.551408</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010444</td>\n",
       "      <td>0.002265</td>\n",
       "      <td>0.001265</td>\n",
       "      <td>0.004456</td>\n",
       "      <td>0.010100</td>\n",
       "      <td>0.011260</td>\n",
       "      <td>67.740031</td>\n",
       "      <td>60.167823</td>\n",
       "      <td>60.167823</td>\n",
       "      <td>67.740031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.033483</td>\n",
       "      <td>0.755662</td>\n",
       "      <td>0.901865</td>\n",
       "      <td>0.973366</td>\n",
       "      <td>0.861023</td>\n",
       "      <td>0.884282</td>\n",
       "      <td>0.691385</td>\n",
       "      <td>6228.000000</td>\n",
       "      <td>98273.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.945977</td>\n",
       "      <td>0.988032</td>\n",
       "      <td>0.993853</td>\n",
       "      <td>0.977162</td>\n",
       "      <td>0.954009</td>\n",
       "      <td>0.941323</td>\n",
       "      <td>5663.000000</td>\n",
       "      <td>65903.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>30.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.057192</td>\n",
       "      <td>0.980490</td>\n",
       "      <td>0.994560</td>\n",
       "      <td>0.997847</td>\n",
       "      <td>0.993040</td>\n",
       "      <td>0.985974</td>\n",
       "      <td>0.977381</td>\n",
       "      <td>8804.250000</td>\n",
       "      <td>98962.250000</td>\n",
       "      <td>58.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.981549</td>\n",
       "      <td>0.994390</td>\n",
       "      <td>0.998072</td>\n",
       "      <td>0.993786</td>\n",
       "      <td>0.988617</td>\n",
       "      <td>0.977851</td>\n",
       "      <td>5882.750000</td>\n",
       "      <td>66108.250000</td>\n",
       "      <td>30.500000</td>\n",
       "      <td>61.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.069915</td>\n",
       "      <td>0.984569</td>\n",
       "      <td>0.996059</td>\n",
       "      <td>0.998363</td>\n",
       "      <td>0.994893</td>\n",
       "      <td>0.990019</td>\n",
       "      <td>0.982849</td>\n",
       "      <td>8853.500000</td>\n",
       "      <td>98998.500000</td>\n",
       "      <td>89.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.988531</td>\n",
       "      <td>0.996260</td>\n",
       "      <td>0.998873</td>\n",
       "      <td>0.996686</td>\n",
       "      <td>0.992712</td>\n",
       "      <td>0.985622</td>\n",
       "      <td>5929.500000</td>\n",
       "      <td>66132.500000</td>\n",
       "      <td>43.500000</td>\n",
       "      <td>86.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.088307</td>\n",
       "      <td>0.988677</td>\n",
       "      <td>0.997225</td>\n",
       "      <td>0.998786</td>\n",
       "      <td>0.996175</td>\n",
       "      <td>0.993472</td>\n",
       "      <td>0.986651</td>\n",
       "      <td>8887.750000</td>\n",
       "      <td>99029.500000</td>\n",
       "      <td>125.750000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.991315</td>\n",
       "      <td>0.997507</td>\n",
       "      <td>0.999257</td>\n",
       "      <td>0.997553</td>\n",
       "      <td>0.994906</td>\n",
       "      <td>0.989860</td>\n",
       "      <td>5955.000000</td>\n",
       "      <td>66145.500000</td>\n",
       "      <td>67.750000</td>\n",
       "      <td>133.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.718602</td>\n",
       "      <td>0.992784</td>\n",
       "      <td>0.998113</td>\n",
       "      <td>0.999381</td>\n",
       "      <td>0.998093</td>\n",
       "      <td>0.996651</td>\n",
       "      <td>0.991119</td>\n",
       "      <td>8928.000000</td>\n",
       "      <td>99058.000000</td>\n",
       "      <td>815.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.995346</td>\n",
       "      <td>0.999501</td>\n",
       "      <td>0.999758</td>\n",
       "      <td>0.999012</td>\n",
       "      <td>0.997995</td>\n",
       "      <td>0.995013</td>\n",
       "      <td>5986.000000</td>\n",
       "      <td>66164.000000</td>\n",
       "      <td>273.000000</td>\n",
       "      <td>353.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            loss  categorical_accuracy      top-3    ROC-AUC     PR-AUC  \\\n",
       "count  50.000000             50.000000  50.000000  50.000000  50.000000   \n",
       "mean    0.087450              0.978905   0.993706   0.997798   0.991854   \n",
       "std     0.094872              0.033093   0.013431   0.003581   0.019030   \n",
       "min     0.033483              0.755662   0.901865   0.973366   0.861023   \n",
       "25%     0.057192              0.980490   0.994560   0.997847   0.993040   \n",
       "50%     0.069915              0.984569   0.996059   0.998363   0.994893   \n",
       "75%     0.088307              0.988677   0.997225   0.998786   0.996175   \n",
       "max     0.718602              0.992784   0.998113   0.999381   0.998093   \n",
       "\n",
       "       precision     recall           TP            TN          FP  ...  \\\n",
       "count  50.000000  50.000000    50.000000     50.000000   50.000000  ...   \n",
       "mean    0.986167   0.975200  8784.600000  98968.700000  119.300000  ...   \n",
       "std     0.016586   0.041777   376.324111    121.551408  121.551408  ...   \n",
       "min     0.884282   0.691385  6228.000000  98273.000000   30.000000  ...   \n",
       "25%     0.985974   0.977381  8804.250000  98962.250000   58.500000  ...   \n",
       "50%     0.990019   0.982849  8853.500000  98998.500000   89.500000  ...   \n",
       "75%     0.993472   0.986651  8887.750000  99029.500000  125.750000  ...   \n",
       "max     0.996651   0.991119  8928.000000  99058.000000  815.000000  ...   \n",
       "\n",
       "       val_categorical_accuracy  val_top-3  val_ROC-AUC  val_PR-AUC  \\\n",
       "count                 50.000000  50.000000    50.000000   50.000000   \n",
       "mean                   0.984807   0.995864     0.998432    0.994938   \n",
       "std                    0.010444   0.002265     0.001265    0.004456   \n",
       "min                    0.945977   0.988032     0.993853    0.977162   \n",
       "25%                    0.981549   0.994390     0.998072    0.993786   \n",
       "50%                    0.988531   0.996260     0.998873    0.996686   \n",
       "75%                    0.991315   0.997507     0.999257    0.997553   \n",
       "max                    0.995346   0.999501     0.999758    0.999012   \n",
       "\n",
       "       val_precision  val_recall       val_TP        val_TN      val_FP  \\\n",
       "count      50.000000   50.000000    50.000000     50.000000   50.000000   \n",
       "mean        0.989376    0.982493  5910.680000  66112.580000   63.420000   \n",
       "std         0.010100    0.011260    67.740031     60.167823   60.167823   \n",
       "min         0.954009    0.941323  5663.000000  65903.000000   12.000000   \n",
       "25%         0.988617    0.977851  5882.750000  66108.250000   30.500000   \n",
       "50%         0.992712    0.985622  5929.500000  66132.500000   43.500000   \n",
       "75%         0.994906    0.989860  5955.000000  66145.500000   67.750000   \n",
       "max         0.997995    0.995013  5986.000000  66164.000000  273.000000   \n",
       "\n",
       "           val_FN  \n",
       "count   50.000000  \n",
       "mean   105.320000  \n",
       "std     67.740031  \n",
       "min     30.000000  \n",
       "25%     61.000000  \n",
       "50%     86.500000  \n",
       "75%    133.250000  \n",
       "max    353.000000  \n",
       "\n",
       "[8 rows x 22 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rm_96_64['history_df'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb7e8e90-efa9-4e8e-b8a6-c1937f4ab3f9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d0d684f-d34d-41d8-9b7f-26dee03ae277",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rm_128_16 = consecutiveModelTraining(input_size=128, batch_size=16, activation='relu', pooling='max')\n",
    "do_rsync()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11f910f6-aebd-4d10-8adc-7f0a5ed3312b",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2021-06-19T22:31:07.113339Z",
     "iopub.status.idle": "2021-06-19T22:31:07.113339Z",
     "shell.execute_reply": "2021-06-19T22:31:07.113339Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "rm_128_16['history_df'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f13384c-e2a9-4ac6-b59b-df13b97278d6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3e60ac0-1c5a-4531-9398-2bba8b713124",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2021-06-19T21:31:00.458796Z",
     "iopub.status.idle": "2021-06-19T21:31:00.458796Z",
     "shell.execute_reply": "2021-06-19T21:31:00.458796Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "rm_128_32 = consecutiveModelTraining(input_size=128, batch_size=32, activation='relu', pooling='max')\n",
    "do_rsync()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b4f6f85-b219-404e-ac62-2bf3019dcc56",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2021-06-19T21:31:00.460795Z",
     "iopub.status.idle": "2021-06-19T21:31:00.460795Z",
     "shell.execute_reply": "2021-06-19T21:31:00.460795Z"
    }
   },
   "outputs": [],
   "source": [
    "rm_128_32['history_df'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd0b1cfe-88ca-4dbd-91a7-715a713408fb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}