train_inceptionres.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "37fde579-c955-4a16-a019-5ab53a8e56a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "# import PIL\n",
    "import tensorflow as tf\n",
    "import pickle5 as pickle\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.models import Sequential\n",
    "import glob\n",
    "import random\n",
    "\n",
    "import datetime\n",
    "off=0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "aa310424-8b15-4830-a20c-cb8d335cd735",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "ch1_mean, ch1_std = 970.4162, 1051.8454\n",
    "ch2_mean, ch2_std = 1176.249, 1031.262\n",
    "ch3_mean, ch3_std = 1273.2376, 1117.068\n",
    "ch4_mean, ch4_std = 2266.9050, 1241.5509\n",
    "means = [ch1_mean,ch2_mean,ch3_mean,ch4_mean]\n",
    "vari = [ch1_std, ch2_std, ch3_std, ch4_std]\n",
    "x_size, y_size = 230,230\n",
    "def load_features(name):\n",
    "    decoded = name.decode(\"UTF-8\")\n",
    "    if os.path.exists(decoded):\n",
    "        with open(decoded, 'rb') as f:\n",
    "            file = pickle.load(f)\n",
    "            label = tf.strings.split(tf.strings.split(name, '/')[-1], '\\\\')[-2]\n",
    "            if label == 'coal':\n",
    "                label = [1,0,0]\n",
    "            elif label == 'steel':\n",
    "                label = [0,1,0]\n",
    "            else: label = [0,0,1]\n",
    "            if (file[\"B02\"].shape[1]<230 or file[\"B02\"].shape[2]<230): \n",
    "                print(\"oh oh, downloaded patch too smol:\", decoded)\n",
    "            B02, B03, B04, B08 = file['B02'][0][0:x_size,0:x_size], file['B03'][0][0:x_size,0:y_size], file['B04'][0][0:x_size,0:y_size],file['B08'][0][0:x_size,0:y_size]\n",
    "            # B02, B03, B04, B08 = B02[off:-off,off:-off], B03[off:-off,off:-off], B04[off:-off,off:-off], B08[off:-off,off:-off] \n",
    "            B02 = (B02-ch1_mean)/ch1_std\n",
    "            B03 = (B03-ch2_mean)/ch2_std\n",
    "            B04 = (B04-ch3_mean)/ch3_std\n",
    "            B08 = (B08-ch4_mean)/ch4_std\n",
    "            features = np.array([ B02, B03, B04, B08]).transpose(1,2,0)\n",
    "            # features = np.expand_dims(features, axis=0)\n",
    "            return features, label\n",
    "            # I have commented the line below but this should return\n",
    "            # the features and the label in a one hot vector\n",
    "            # return file['features'], file['targets']\n",
    "    else:\n",
    "        print(\"Something went wrong!\")\n",
    "        exit(-1)\n",
    "        \n",
    "def data_loader(filename):\n",
    "    features, labels = tf.numpy_function(load_features, [filename], [tf.double, tf.int32])\n",
    "    # features.set_shape((None, 242,242,4))\n",
    "    # labels.set_shape(( 1))\n",
    "    return features, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "6f8d3b08-b09b-480d-8d8d-bf44131c2034",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'E:/Users/sentinel_industry/downloaded_aois/coal/'\n",
    "\n",
    "pkl_files = glob.glob((path+\"*.pickle\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "7def905d-f2c1-45f1-985c-aaf123860dd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2179\n"
     ]
    }
   ],
   "source": [
    "print(len(pkl_files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "f15f640f-7002-40d2-899a-21b4090c2128",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_steel = 'E:/Users/sentinel_industry/downloaded_aois/steel/'\n",
    "pkl_files_steel = glob.glob((path_steel+\"*.pickle\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "5e5a78e7-514d-4f65-a831-57faa0c5df59",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_steel_no_ind = 'E:/Users/sentinel_industry/downloaded_aois/steel/no_ind/'\n",
    "pkl_files_steel_no_ind = glob.glob((path_steel_no_ind+\"*.pickle\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "076b0a09-9866-4fee-a244-2bd75892bcc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_coal_no_ind = 'E:/Users/sentinel_industry/downloaded_aois/coal/no_ind/'\n",
    "pkl_files_coal_no_ind = glob.glob((path_coal_no_ind+\"*.pickle\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "da24ac7d-0f3f-4cfe-b518-fef29417d581",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1693\n"
     ]
    }
   ],
   "source": [
    "print(len(pkl_files_steel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "af872199-9192-48eb-8cea-9693c2291fa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkl_files = pkl_files + pkl_files_steel + pkl_files_steel_no_ind + pkl_files_coal_no_ind\n",
    "random.Random(42).shuffle(pkl_files)\n",
    "# pkl_files=pkl_files[0:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "cfe3ca4f-ca84-4d80-9218-95fc89e8fc1b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# from itertools import chain\n",
    "\n",
    "# pkl_files = list(chain.from_iterable(zip(pkl_files, pkl_files_steel)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "07182a88-11f4-4b7d-a461-4debb2944063",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7829\n"
     ]
    }
   ],
   "source": [
    "print(len(pkl_files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "dfc46d5e-854e-42c6-af9c-97123ce4b3be",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_len=len(pkl_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "037aad36-c73f-48bb-9c72-e4d8d4103687",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pkl_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "21e3f5d5-620d-4e9d-8abb-ecee1ba2dac3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# dataset = tf.data.Dataset.from_tensor_slices((pkl_files))\n",
    "\n",
    "# def data_loader(filename):\n",
    "#     features, labels = tf.numpy_function(load_features, [filename], [tf.double, tf.int32])\n",
    "#     # features.set_shape((None, 242,242,4))\n",
    "#     # labels.set_shape(( 1))\n",
    "#     return features, labels\n",
    "\n",
    "# dataset = dataset.map(data_loader)\n",
    "#                # load_features, [filename], [tf.uint16, tf.string])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "adaadc4a-c328-4c33-b9b7-3d2e140f92bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_len=len(pkl_files)\n",
    "\n",
    "train_size = int(np.floor(0.8 * dataset_len))\n",
    "valid_size = int(np.floor(0.1 * dataset_len))\n",
    "test_size = int(np.floor(0.1 * dataset_len))\n",
    "\n",
    "train = tf.data.Dataset.from_tensor_slices(pkl_files[0:train_size]).map(data_loader,num_parallel_calls=tf.data.AUTOTUNE)\n",
    "valid = tf.data.Dataset.from_tensor_slices(pkl_files[train_size:train_size+valid_size]).map(data_loader,num_parallel_calls=tf.data.AUTOTUNE)\n",
    "test = tf.data.Dataset.from_tensor_slices(pkl_files[train_size+valid_size:]).map(data_loader,num_parallel_calls=tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "d59cee74-4666-4d78-802d-ce0537fb4058",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tf.data.experimental.save(\n",
    "#     valid, 'savedata_valid')\n",
    "\n",
    "# tf.data.experimental.save(\n",
    "#     train, 'savedata_train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "80f35197-11ee-40af-bc9c-c5181c99583d",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid = tf.data.experimental.load('savedata_valid')\n",
    "train = tf.data.experimental.load('savedata_train')\n",
    "train = train.shuffle(100,  reshuffle_each_iteration=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "676ae2c1-03cd-42bf-b40d-fe2926f9f3f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_s = 230-2*off\n",
    "y_s = 230-2*off\n",
    "in_seed = 42\n",
    "resize_rescale_augment = tf.keras.Sequential([\n",
    "    layers.RandomRotation(0.2, seed=in_seed, input_shape=(x_s,y_s,4)),  # here, maybe try larger values\n",
    "    layers.CenterCrop(140,140, ),\n",
    "    layers.RandomCrop(120,120, seed=in_seed, ),  # to be changed with centercrop for data augmentation/ use centre first, then random?\n",
    "    \n",
    "    # layers.Rescaling(255),#,input_shape=(240,240,4)), # I think the data is scaled to 10000\n",
    "    layers.RandomFlip(\"horizontal_and_vertical\", seed=in_seed), \n",
    "    # norm_layer,\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bb171cdb-c5d8-4f7d-b778-18aa7efc08bf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " sequential (Sequential)     (None, 120, 120, 4)       0         \n",
      "                                                                 \n",
      " conv2d (Conv2D)             (None, 120, 120, 3)       303       \n",
      "                                                                 \n",
      " inception_resnet_v2 (Functi  (None, 2, 2, 1536)       54336736  \n",
      " onal)                                                           \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 6144)              0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 3)                 18435     \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 54,355,474\n",
      "Trainable params: 54,294,930\n",
      "Non-trainable params: 60,544\n",
      "_________________________________________________________________\n",
      "Epoch 1/200\n",
      "313/313 [==============================] - 149s 395ms/step - loss: 0.9758 - categorical_accuracy: 0.6823 - val_loss: 1.2240 - val_categorical_accuracy: 0.6233 - lr: 0.0010\n",
      "Epoch 2/200\n",
      "313/313 [==============================] - 115s 363ms/step - loss: 0.7263 - categorical_accuracy: 0.7265 - val_loss: 1.3880 - val_categorical_accuracy: 0.5767 - lr: 0.0010\n",
      "Epoch 3/200\n",
      "313/313 [==============================] - 115s 364ms/step - loss: 0.6732 - categorical_accuracy: 0.7427 - val_loss: 1.0597 - val_categorical_accuracy: 0.7433 - lr: 0.0010\n",
      "Epoch 4/200\n",
      "313/313 [==============================] - 116s 368ms/step - loss: 0.7026 - categorical_accuracy: 0.7353 - val_loss: 4.5358 - val_categorical_accuracy: 0.6333 - lr: 0.0010\n",
      "Epoch 5/200\n",
      "313/313 [==============================] - 114s 363ms/step - loss: 0.7088 - categorical_accuracy: 0.7278 - val_loss: 1.6286 - val_categorical_accuracy: 0.6500 - lr: 0.0010\n",
      "Epoch 6/200\n",
      "313/313 [==============================] - 112s 356ms/step - loss: 0.7065 - categorical_accuracy: 0.7208 - val_loss: 1.4408 - val_categorical_accuracy: 0.6167 - lr: 0.0010\n",
      "Epoch 7/200\n",
      "313/313 [==============================] - 113s 358ms/step - loss: 0.6606 - categorical_accuracy: 0.7299 - val_loss: 1.0873 - val_categorical_accuracy: 0.5600 - lr: 0.0010\n",
      "Epoch 8/200\n",
      "313/313 [==============================] - 122s 389ms/step - loss: 0.8027 - categorical_accuracy: 0.7019 - val_loss: 0.8572 - val_categorical_accuracy: 0.6533 - lr: 0.0010\n",
      "Epoch 9/200\n",
      "313/313 [==============================] - 114s 361ms/step - loss: 0.9473 - categorical_accuracy: 0.6272 - val_loss: 1.1652 - val_categorical_accuracy: 0.4333 - lr: 0.0010\n",
      "Epoch 10/200\n",
      "313/313 [==============================] - 116s 367ms/step - loss: 0.6884 - categorical_accuracy: 0.7187 - val_loss: 0.5774 - val_categorical_accuracy: 0.7400 - lr: 0.0010\n",
      "Epoch 11/200\n",
      "313/313 [==============================] - 117s 370ms/step - loss: 0.6114 - categorical_accuracy: 0.7458 - val_loss: 0.5398 - val_categorical_accuracy: 0.7733 - lr: 0.0010\n",
      "Epoch 12/200\n",
      "313/313 [==============================] - 116s 369ms/step - loss: 0.5691 - categorical_accuracy: 0.7695 - val_loss: 0.5327 - val_categorical_accuracy: 0.7933 - lr: 0.0010\n",
      "Epoch 13/200\n",
      "313/313 [==============================] - 115s 364ms/step - loss: 0.5527 - categorical_accuracy: 0.7759 - val_loss: 0.6204 - val_categorical_accuracy: 0.7067 - lr: 0.0010\n",
      "Epoch 14/200\n",
      "313/313 [==============================] - 115s 365ms/step - loss: 0.5772 - categorical_accuracy: 0.7692 - val_loss: 0.6317 - val_categorical_accuracy: 0.7100 - lr: 0.0010\n",
      "Epoch 15/200\n",
      "313/313 [==============================] - 116s 367ms/step - loss: 0.9624 - categorical_accuracy: 0.6847 - val_loss: 4.6904 - val_categorical_accuracy: 0.7067 - lr: 0.0010\n",
      "Epoch 16/200\n",
      "313/313 [==============================] - 116s 368ms/step - loss: 0.7422 - categorical_accuracy: 0.7184 - val_loss: 0.6505 - val_categorical_accuracy: 0.7033 - lr: 0.0010\n",
      "Epoch 17/200\n",
      "313/313 [==============================] - 117s 371ms/step - loss: 0.6752 - categorical_accuracy: 0.7283 - val_loss: 9.6351 - val_categorical_accuracy: 0.5633 - lr: 0.0010\n",
      "Epoch 18/200\n",
      "313/313 [==============================] - 120s 381ms/step - loss: 0.6672 - categorical_accuracy: 0.7304 - val_loss: 0.7633 - val_categorical_accuracy: 0.7667 - lr: 0.0010\n",
      "Epoch 19/200\n",
      "313/313 [==============================] - 119s 379ms/step - loss: 0.6163 - categorical_accuracy: 0.7481 - val_loss: 0.6142 - val_categorical_accuracy: 0.7667 - lr: 0.0010\n",
      "Epoch 20/200\n",
      "313/313 [==============================] - 120s 380ms/step - loss: 0.6506 - categorical_accuracy: 0.7450 - val_loss: 0.5840 - val_categorical_accuracy: 0.7633 - lr: 0.0010\n",
      "Epoch 21/200\n",
      "313/313 [==============================] - 119s 378ms/step - loss: 0.6334 - categorical_accuracy: 0.7457 - val_loss: 1.6610 - val_categorical_accuracy: 0.5767 - lr: 0.0010\n",
      "Epoch 22/200\n",
      "313/313 [==============================] - 120s 380ms/step - loss: 0.6608 - categorical_accuracy: 0.7272 - val_loss: 0.8638 - val_categorical_accuracy: 0.7067 - lr: 0.0010\n",
      "Epoch 23/200\n",
      "313/313 [==============================] - 118s 374ms/step - loss: 0.6851 - categorical_accuracy: 0.7327 - val_loss: 0.6554 - val_categorical_accuracy: 0.7333 - lr: 0.0010\n",
      "Epoch 24/200\n",
      "313/313 [==============================] - 117s 370ms/step - loss: 0.6273 - categorical_accuracy: 0.7489 - val_loss: 0.7434 - val_categorical_accuracy: 0.7467 - lr: 0.0010\n",
      "Epoch 25/200\n",
      "313/313 [==============================] - 116s 367ms/step - loss: 0.6055 - categorical_accuracy: 0.7580 - val_loss: 0.5706 - val_categorical_accuracy: 0.7600 - lr: 0.0010\n",
      "Epoch 26/200\n",
      "313/313 [==============================] - 114s 362ms/step - loss: 0.6118 - categorical_accuracy: 0.7583 - val_loss: 0.6074 - val_categorical_accuracy: 0.7200 - lr: 0.0010\n",
      "Epoch 27/200\n",
      "313/313 [==============================] - 117s 371ms/step - loss: 0.5857 - categorical_accuracy: 0.7604 - val_loss: 0.6354 - val_categorical_accuracy: 0.7100 - lr: 0.0010\n",
      "Epoch 28/200\n",
      "313/313 [==============================] - 130s 414ms/step - loss: 0.6330 - categorical_accuracy: 0.7423 - val_loss: 0.6014 - val_categorical_accuracy: 0.7600 - lr: 0.0010\n",
      "Epoch 29/200\n",
      "313/313 [==============================] - 120s 374ms/step - loss: 0.6171 - categorical_accuracy: 0.7510 - val_loss: 0.6038 - val_categorical_accuracy: 0.7533 - lr: 0.0010\n",
      "Epoch 30/200\n",
      "313/313 [==============================] - 119s 376ms/step - loss: 0.6081 - categorical_accuracy: 0.7554 - val_loss: 0.6836 - val_categorical_accuracy: 0.7400 - lr: 0.0010\n",
      "Epoch 31/200\n",
      "313/313 [==============================] - 114s 361ms/step - loss: 0.6162 - categorical_accuracy: 0.7481 - val_loss: 0.6232 - val_categorical_accuracy: 0.7333 - lr: 9.7045e-04\n",
      "Epoch 32/200\n",
      "313/313 [==============================] - 124s 395ms/step - loss: 0.6047 - categorical_accuracy: 0.7492 - val_loss: 0.5891 - val_categorical_accuracy: 0.7567 - lr: 9.4176e-04\n",
      "Epoch 33/200\n",
      "313/313 [==============================] - 117s 368ms/step - loss: 0.5774 - categorical_accuracy: 0.7644 - val_loss: 1.4511 - val_categorical_accuracy: 0.6867 - lr: 9.1393e-04\n",
      "Epoch 34/200\n",
      "313/313 [==============================] - 117s 369ms/step - loss: 0.6603 - categorical_accuracy: 0.7450 - val_loss: 49.3348 - val_categorical_accuracy: 0.5300 - lr: 8.8692e-04\n",
      "Epoch 35/200\n",
      "313/313 [==============================] - 118s 375ms/step - loss: 0.6500 - categorical_accuracy: 0.7457 - val_loss: 0.6718 - val_categorical_accuracy: 0.7233 - lr: 8.6071e-04\n",
      "Epoch 36/200\n",
      "313/313 [==============================] - 117s 369ms/step - loss: 0.5828 - categorical_accuracy: 0.7658 - val_loss: 0.5381 - val_categorical_accuracy: 0.7867 - lr: 8.3527e-04\n",
      "Epoch 37/200\n",
      "313/313 [==============================] - 117s 371ms/step - loss: 0.6176 - categorical_accuracy: 0.7466 - val_loss: 0.9741 - val_categorical_accuracy: 0.7333 - lr: 8.1058e-04\n",
      "Epoch 38/200\n",
      "313/313 [==============================] - 116s 369ms/step - loss: 0.5802 - categorical_accuracy: 0.7719 - val_loss: 0.7523 - val_categorical_accuracy: 0.6000 - lr: 7.8663e-04\n",
      "Epoch 39/200\n",
      "313/313 [==============================] - 118s 375ms/step - loss: 0.5799 - categorical_accuracy: 0.7658 - val_loss: 0.5123 - val_categorical_accuracy: 0.7833 - lr: 7.6338e-04\n",
      "Epoch 40/200\n",
      "313/313 [==============================] - 117s 371ms/step - loss: 0.5512 - categorical_accuracy: 0.7778 - val_loss: 0.5014 - val_categorical_accuracy: 0.7667 - lr: 7.4082e-04\n",
      "Epoch 41/200\n",
      "313/313 [==============================] - 117s 372ms/step - loss: 0.5471 - categorical_accuracy: 0.7786 - val_loss: 0.5230 - val_categorical_accuracy: 0.7533 - lr: 7.1892e-04\n",
      "Epoch 42/200\n",
      "313/313 [==============================] - 117s 370ms/step - loss: 0.5246 - categorical_accuracy: 0.7890 - val_loss: 0.4908 - val_categorical_accuracy: 0.7933 - lr: 6.9768e-04\n",
      "Epoch 43/200\n",
      "313/313 [==============================] - 118s 375ms/step - loss: 0.5261 - categorical_accuracy: 0.7960 - val_loss: 0.4694 - val_categorical_accuracy: 0.8100 - lr: 6.7706e-04\n",
      "Epoch 44/200\n",
      "313/313 [==============================] - 118s 375ms/step - loss: 0.5305 - categorical_accuracy: 0.7883 - val_loss: 0.4660 - val_categorical_accuracy: 0.7967 - lr: 6.5705e-04\n",
      "Epoch 45/200\n",
      "313/313 [==============================] - 116s 369ms/step - loss: 0.5361 - categorical_accuracy: 0.7845 - val_loss: 0.4807 - val_categorical_accuracy: 0.7800 - lr: 6.3763e-04\n",
      "Epoch 46/200\n",
      "313/313 [==============================] - 117s 371ms/step - loss: 0.5079 - categorical_accuracy: 0.7938 - val_loss: 0.4543 - val_categorical_accuracy: 0.8367 - lr: 6.1878e-04\n",
      "Epoch 47/200\n",
      "313/313 [==============================] - 115s 365ms/step - loss: 0.4982 - categorical_accuracy: 0.7990 - val_loss: 0.9776 - val_categorical_accuracy: 0.6700 - lr: 6.0050e-04\n",
      "Epoch 48/200\n",
      "313/313 [==============================] - 117s 372ms/step - loss: 0.5513 - categorical_accuracy: 0.7797 - val_loss: 0.5837 - val_categorical_accuracy: 0.7033 - lr: 5.8275e-04\n",
      "Epoch 49/200\n",
      "313/313 [==============================] - 116s 367ms/step - loss: 0.5307 - categorical_accuracy: 0.7895 - val_loss: 0.7276 - val_categorical_accuracy: 0.6633 - lr: 5.6553e-04\n",
      "Epoch 50/200\n",
      "313/313 [==============================] - 116s 368ms/step - loss: 0.5013 - categorical_accuracy: 0.8019 - val_loss: 0.4756 - val_categorical_accuracy: 0.8067 - lr: 5.4881e-04\n",
      "Epoch 51/200\n",
      "313/313 [==============================] - 116s 368ms/step - loss: 0.4672 - categorical_accuracy: 0.8118 - val_loss: 0.4692 - val_categorical_accuracy: 0.8100 - lr: 5.3259e-04\n",
      "Epoch 52/200\n",
      "313/313 [==============================] - 117s 371ms/step - loss: 0.4664 - categorical_accuracy: 0.8155 - val_loss: 0.4640 - val_categorical_accuracy: 0.8133 - lr: 5.1685e-04\n",
      "Epoch 53/200\n",
      "313/313 [==============================] - 117s 371ms/step - loss: 0.4544 - categorical_accuracy: 0.8120 - val_loss: 0.4288 - val_categorical_accuracy: 0.8367 - lr: 5.0158e-04\n",
      "Epoch 54/200\n",
      "313/313 [==============================] - 118s 375ms/step - loss: 0.4428 - categorical_accuracy: 0.8264 - val_loss: 0.4246 - val_categorical_accuracy: 0.8600 - lr: 4.8675e-04\n",
      "Epoch 55/200\n",
      "313/313 [==============================] - 119s 377ms/step - loss: 0.4441 - categorical_accuracy: 0.8243 - val_loss: 0.4327 - val_categorical_accuracy: 0.8100 - lr: 4.7237e-04\n",
      "Epoch 56/200\n",
      "313/313 [==============================] - 119s 379ms/step - loss: 0.4352 - categorical_accuracy: 0.8275 - val_loss: 0.4073 - val_categorical_accuracy: 0.8333 - lr: 4.5841e-04\n",
      "Epoch 57/200\n",
      "313/313 [==============================] - 122s 388ms/step - loss: 0.4183 - categorical_accuracy: 0.8350 - val_loss: 0.5696 - val_categorical_accuracy: 0.7567 - lr: 4.4486e-04\n",
      "Epoch 58/200\n",
      "313/313 [==============================] - 298s 938ms/step - loss: 0.4647 - categorical_accuracy: 0.8196 - val_loss: 0.4524 - val_categorical_accuracy: 0.8067 - lr: 4.3171e-04\n",
      "Epoch 59/200\n",
      "313/313 [==============================] - 150s 463ms/step - loss: 0.4486 - categorical_accuracy: 0.8230 - val_loss: 0.4363 - val_categorical_accuracy: 0.8400 - lr: 4.1895e-04\n",
      "Epoch 60/200\n",
      "313/313 [==============================] - 117s 373ms/step - loss: 0.4192 - categorical_accuracy: 0.8289 - val_loss: 0.4345 - val_categorical_accuracy: 0.8233 - lr: 4.0657e-04\n",
      "Epoch 61/200\n",
      "313/313 [==============================] - 116s 367ms/step - loss: 0.4138 - categorical_accuracy: 0.8387 - val_loss: 0.4877 - val_categorical_accuracy: 0.7833 - lr: 3.9455e-04\n",
      "Epoch 62/200\n",
      "313/313 [==============================] - 121s 383ms/step - loss: 0.4091 - categorical_accuracy: 0.8355 - val_loss: 0.3888 - val_categorical_accuracy: 0.8500 - lr: 3.8289e-04\n",
      "Epoch 63/200\n",
      "313/313 [==============================] - 116s 370ms/step - loss: 0.4029 - categorical_accuracy: 0.8431 - val_loss: 0.4085 - val_categorical_accuracy: 0.8233 - lr: 3.7158e-04\n",
      "Epoch 64/200\n",
      "313/313 [==============================] - 117s 372ms/step - loss: 0.4007 - categorical_accuracy: 0.8414 - val_loss: 0.3928 - val_categorical_accuracy: 0.8400 - lr: 3.6060e-04\n",
      "Epoch 65/200\n",
      "313/313 [==============================] - 118s 376ms/step - loss: 0.3966 - categorical_accuracy: 0.8436 - val_loss: 0.3748 - val_categorical_accuracy: 0.8467 - lr: 3.4994e-04\n",
      "Epoch 66/200\n",
      "313/313 [==============================] - 117s 372ms/step - loss: 0.3858 - categorical_accuracy: 0.8430 - val_loss: 0.4666 - val_categorical_accuracy: 0.8167 - lr: 3.3960e-04\n",
      "Epoch 67/200\n",
      "313/313 [==============================] - 116s 370ms/step - loss: 0.3839 - categorical_accuracy: 0.8466 - val_loss: 0.3844 - val_categorical_accuracy: 0.8433 - lr: 3.2956e-04\n",
      "Epoch 68/200\n",
      "313/313 [==============================] - 117s 373ms/step - loss: 0.3649 - categorical_accuracy: 0.8530 - val_loss: 0.3697 - val_categorical_accuracy: 0.8667 - lr: 3.1982e-04\n",
      "Epoch 69/200\n",
      "313/313 [==============================] - 118s 376ms/step - loss: 0.3654 - categorical_accuracy: 0.8530 - val_loss: 0.3614 - val_categorical_accuracy: 0.8600 - lr: 3.1037e-04\n",
      "Epoch 70/200\n",
      "313/313 [==============================] - 118s 376ms/step - loss: 0.3637 - categorical_accuracy: 0.8534 - val_loss: 0.3665 - val_categorical_accuracy: 0.8533 - lr: 3.0119e-04\n",
      "Epoch 71/200\n",
      "313/313 [==============================] - 118s 373ms/step - loss: 0.3695 - categorical_accuracy: 0.8514 - val_loss: 0.3979 - val_categorical_accuracy: 0.8333 - lr: 2.9229e-04\n",
      "Epoch 72/200\n",
      "313/313 [==============================] - 115s 365ms/step - loss: 0.3666 - categorical_accuracy: 0.8561 - val_loss: 0.3739 - val_categorical_accuracy: 0.8500 - lr: 2.8365e-04\n",
      "Epoch 73/200\n",
      "313/313 [==============================] - 120s 380ms/step - loss: 0.3684 - categorical_accuracy: 0.8542 - val_loss: 0.3621 - val_categorical_accuracy: 0.8600 - lr: 2.7527e-04\n",
      "Epoch 74/200\n",
      "313/313 [==============================] - 119s 377ms/step - loss: 0.3534 - categorical_accuracy: 0.8583 - val_loss: 0.3950 - val_categorical_accuracy: 0.8500 - lr: 2.6714e-04\n",
      "Epoch 75/200\n",
      "313/313 [==============================] - 116s 368ms/step - loss: 0.3547 - categorical_accuracy: 0.8585 - val_loss: 0.3713 - val_categorical_accuracy: 0.8400 - lr: 2.5924e-04\n",
      "Epoch 76/200\n",
      "313/313 [==============================] - 128s 408ms/step - loss: 0.3477 - categorical_accuracy: 0.8570 - val_loss: 0.3731 - val_categorical_accuracy: 0.8600 - lr: 2.5158e-04\n",
      "Epoch 77/200\n",
      "313/313 [==============================] - 120s 379ms/step - loss: 0.3461 - categorical_accuracy: 0.8594 - val_loss: 0.3425 - val_categorical_accuracy: 0.8567 - lr: 2.4414e-04\n",
      "Epoch 78/200\n",
      "313/313 [==============================] - 125s 395ms/step - loss: 0.3414 - categorical_accuracy: 0.8650 - val_loss: 0.3522 - val_categorical_accuracy: 0.8767 - lr: 2.3693e-04\n",
      "Epoch 79/200\n",
      "313/313 [==============================] - 254s 797ms/step - loss: 0.3476 - categorical_accuracy: 0.8559 - val_loss: 0.3618 - val_categorical_accuracy: 0.8667 - lr: 2.2993e-04\n",
      "Epoch 80/200\n",
      "313/313 [==============================] - 398s 1s/step - loss: 0.3392 - categorical_accuracy: 0.8650 - val_loss: 0.3576 - val_categorical_accuracy: 0.8433 - lr: 2.2313e-04\n",
      "Epoch 81/200\n",
      "313/313 [==============================] - 214s 680ms/step - loss: 0.3404 - categorical_accuracy: 0.8637 - val_loss: 0.3440 - val_categorical_accuracy: 0.8967 - lr: 2.1654e-04\n",
      "Epoch 82/200\n",
      "313/313 [==============================] - 261s 821ms/step - loss: 0.3389 - categorical_accuracy: 0.8658 - val_loss: 0.3397 - val_categorical_accuracy: 0.8500 - lr: 2.1014e-04\n",
      "Epoch 83/200\n",
      "313/313 [==============================] - 125s 394ms/step - loss: 0.3342 - categorical_accuracy: 0.8644 - val_loss: 0.3384 - val_categorical_accuracy: 0.8700 - lr: 2.0393e-04\n",
      "Epoch 84/200\n",
      "313/313 [==============================] - 122s 386ms/step - loss: 0.3277 - categorical_accuracy: 0.8733 - val_loss: 0.3398 - val_categorical_accuracy: 0.8633 - lr: 1.9790e-04\n",
      "Epoch 85/200\n",
      "313/313 [==============================] - 154s 485ms/step - loss: 0.3304 - categorical_accuracy: 0.8685 - val_loss: 0.3324 - val_categorical_accuracy: 0.8567 - lr: 1.9205e-04\n",
      "Epoch 86/200\n",
      "313/313 [==============================] - 275s 867ms/step - loss: 0.3226 - categorical_accuracy: 0.8727 - val_loss: 0.3606 - val_categorical_accuracy: 0.8667 - lr: 1.8637e-04\n",
      "Epoch 87/200\n",
      "313/313 [==============================] - 268s 851ms/step - loss: 0.3219 - categorical_accuracy: 0.8685 - val_loss: 0.3726 - val_categorical_accuracy: 0.8533 - lr: 1.8087e-04\n",
      "Epoch 88/200\n",
      "313/313 [==============================] - 269s 845ms/step - loss: 0.3222 - categorical_accuracy: 0.8717 - val_loss: 0.3411 - val_categorical_accuracy: 0.8700 - lr: 1.7552e-04\n",
      "Epoch 89/200\n",
      "313/313 [==============================] - 268s 847ms/step - loss: 0.3358 - categorical_accuracy: 0.8692 - val_loss: 0.3231 - val_categorical_accuracy: 0.8633 - lr: 1.7033e-04\n",
      "Epoch 90/200\n",
      "313/313 [==============================] - 268s 848ms/step - loss: 0.3244 - categorical_accuracy: 0.8743 - val_loss: 0.3244 - val_categorical_accuracy: 0.8733 - lr: 1.6530e-04\n",
      "Epoch 91/200\n",
      "313/313 [==============================] - 268s 851ms/step - loss: 0.3124 - categorical_accuracy: 0.8748 - val_loss: 0.3702 - val_categorical_accuracy: 0.8500 - lr: 1.6041e-04\n",
      "Epoch 92/200\n",
      "313/313 [==============================] - 266s 845ms/step - loss: 0.3141 - categorical_accuracy: 0.8770 - val_loss: 0.3196 - val_categorical_accuracy: 0.8633 - lr: 1.5567e-04\n",
      "Epoch 93/200\n",
      "313/313 [==============================] - 266s 840ms/step - loss: 0.3142 - categorical_accuracy: 0.8760 - val_loss: 0.6101 - val_categorical_accuracy: 0.8367 - lr: 1.5107e-04\n",
      "Epoch 94/200\n",
      "313/313 [==============================] - 268s 845ms/step - loss: 0.3094 - categorical_accuracy: 0.8772 - val_loss: 0.3256 - val_categorical_accuracy: 0.8667 - lr: 1.4661e-04\n",
      "Epoch 95/200\n",
      "313/313 [==============================] - 271s 855ms/step - loss: 0.3136 - categorical_accuracy: 0.8778 - val_loss: 0.3332 - val_categorical_accuracy: 0.8600 - lr: 1.4227e-04\n",
      "Epoch 96/200\n",
      "313/313 [==============================] - 269s 848ms/step - loss: 0.3164 - categorical_accuracy: 0.8709 - val_loss: 0.3705 - val_categorical_accuracy: 0.8533 - lr: 1.3807e-04\n",
      "Epoch 97/200\n",
      "313/313 [==============================] - 266s 839ms/step - loss: 0.3118 - categorical_accuracy: 0.8804 - val_loss: 0.6020 - val_categorical_accuracy: 0.8167 - lr: 1.3399e-04\n",
      "Epoch 98/200\n",
      "313/313 [==============================] - 266s 843ms/step - loss: 0.3274 - categorical_accuracy: 0.8724 - val_loss: 0.4544 - val_categorical_accuracy: 0.8633 - lr: 1.3003e-04\n",
      "Epoch 99/200\n",
      "313/313 [==============================] - 266s 838ms/step - loss: 0.3117 - categorical_accuracy: 0.8762 - val_loss: 0.3147 - val_categorical_accuracy: 0.8700 - lr: 1.2619e-04\n",
      "Epoch 100/200\n",
      "313/313 [==============================] - 150s 468ms/step - loss: 0.3044 - categorical_accuracy: 0.8802 - val_loss: 0.3536 - val_categorical_accuracy: 0.8567 - lr: 1.2246e-04\n",
      "Epoch 101/200\n",
      "313/313 [==============================] - 124s 391ms/step - loss: 0.3005 - categorical_accuracy: 0.8851 - val_loss: 0.3293 - val_categorical_accuracy: 0.8500 - lr: 1.1884e-04\n",
      "Epoch 102/200\n",
      "313/313 [==============================] - 122s 384ms/step - loss: 0.3037 - categorical_accuracy: 0.8810 - val_loss: 0.3199 - val_categorical_accuracy: 0.8700 - lr: 1.1533e-04\n",
      "Epoch 103/200\n",
      "313/313 [==============================] - 119s 376ms/step - loss: 0.3039 - categorical_accuracy: 0.8770 - val_loss: 0.3136 - val_categorical_accuracy: 0.8700 - lr: 1.1192e-04\n",
      "Epoch 104/200\n",
      "313/313 [==============================] - 119s 374ms/step - loss: 0.3032 - categorical_accuracy: 0.8780 - val_loss: 0.3235 - val_categorical_accuracy: 0.8567 - lr: 1.0861e-04\n",
      "Epoch 105/200\n",
      "313/313 [==============================] - 118s 372ms/step - loss: 0.3039 - categorical_accuracy: 0.8788 - val_loss: 0.3288 - val_categorical_accuracy: 0.8600 - lr: 1.0540e-04\n",
      "Epoch 106/200\n",
      "313/313 [==============================] - 120s 379ms/step - loss: 0.2966 - categorical_accuracy: 0.8867 - val_loss: 0.4635 - val_categorical_accuracy: 0.8700 - lr: 1.0228e-04\n",
      "Epoch 107/200\n",
      "313/313 [==============================] - 119s 374ms/step - loss: 0.2963 - categorical_accuracy: 0.8813 - val_loss: 0.4516 - val_categorical_accuracy: 0.8500 - lr: 9.9262e-05\n",
      "Epoch 108/200\n",
      "313/313 [==============================] - 119s 373ms/step - loss: 0.2929 - categorical_accuracy: 0.8858 - val_loss: 0.3237 - val_categorical_accuracy: 0.8600 - lr: 9.6328e-05\n",
      "Epoch 109/200\n",
      "313/313 [==============================] - 118s 373ms/step - loss: 0.2954 - categorical_accuracy: 0.8816 - val_loss: 0.3801 - val_categorical_accuracy: 0.8600 - lr: 9.3481e-05\n",
      "Epoch 110/200\n",
      "313/313 [==============================] - 120s 378ms/step - loss: 0.2951 - categorical_accuracy: 0.8816 - val_loss: 0.3717 - val_categorical_accuracy: 0.8433 - lr: 9.0718e-05\n",
      "Epoch 111/200\n",
      "313/313 [==============================] - 118s 371ms/step - loss: 0.2921 - categorical_accuracy: 0.8887 - val_loss: 0.3264 - val_categorical_accuracy: 0.8800 - lr: 8.8037e-05\n",
      "Epoch 112/200\n",
      "313/313 [==============================] - 120s 378ms/step - loss: 0.2925 - categorical_accuracy: 0.8901 - val_loss: 0.3160 - val_categorical_accuracy: 0.8800 - lr: 8.5435e-05\n",
      "Epoch 113/200\n",
      "313/313 [==============================] - 119s 375ms/step - loss: 0.2917 - categorical_accuracy: 0.8839 - val_loss: 0.3433 - val_categorical_accuracy: 0.8700 - lr: 8.2910e-05\n",
      "Epoch 114/200\n",
      "313/313 [==============================] - 119s 376ms/step - loss: 0.2913 - categorical_accuracy: 0.8869 - val_loss: 0.3897 - val_categorical_accuracy: 0.8667 - lr: 8.0460e-05\n",
      "Epoch 115/200\n",
      "313/313 [==============================] - 120s 377ms/step - loss: 0.2878 - categorical_accuracy: 0.8899 - val_loss: 0.3890 - val_categorical_accuracy: 0.8500 - lr: 7.8082e-05\n",
      "Epoch 116/200\n",
      "313/313 [==============================] - 120s 376ms/step - loss: 0.2875 - categorical_accuracy: 0.8896 - val_loss: 0.3952 - val_categorical_accuracy: 0.8400 - lr: 7.5774e-05\n",
      "Epoch 117/200\n",
      "313/313 [==============================] - 117s 369ms/step - loss: 0.2863 - categorical_accuracy: 0.8893 - val_loss: 0.9044 - val_categorical_accuracy: 0.8333 - lr: 7.3535e-05\n",
      "Epoch 118/200\n",
      "313/313 [==============================] - 120s 376ms/step - loss: 0.2897 - categorical_accuracy: 0.8882 - val_loss: 0.4365 - val_categorical_accuracy: 0.8367 - lr: 7.1362e-05\n",
      "Epoch 119/200\n",
      "313/313 [==============================] - 119s 375ms/step - loss: 0.2892 - categorical_accuracy: 0.8843 - val_loss: 0.4408 - val_categorical_accuracy: 0.8467 - lr: 6.9252e-05\n",
      "Epoch 120/200\n",
      "313/313 [==============================] - 119s 373ms/step - loss: 0.2891 - categorical_accuracy: 0.8871 - val_loss: 0.3858 - val_categorical_accuracy: 0.8467 - lr: 6.7206e-05\n",
      "Epoch 121/200\n",
      "313/313 [==============================] - 120s 376ms/step - loss: 0.2836 - categorical_accuracy: 0.8893 - val_loss: 0.3556 - val_categorical_accuracy: 0.8700 - lr: 6.5220e-05\n",
      "Epoch 122/200\n",
      "313/313 [==============================] - 118s 371ms/step - loss: 0.2905 - categorical_accuracy: 0.8859 - val_loss: 0.5701 - val_categorical_accuracy: 0.8600 - lr: 6.3292e-05\n",
      "Epoch 123/200\n",
      "313/313 [==============================] - 119s 375ms/step - loss: 0.2865 - categorical_accuracy: 0.8903 - val_loss: 0.4796 - val_categorical_accuracy: 0.8633 - lr: 6.1421e-05\n",
      "Epoch 124/200\n",
      "313/313 [==============================] - 126s 397ms/step - loss: 0.2877 - categorical_accuracy: 0.8888 - val_loss: 0.4884 - val_categorical_accuracy: 0.8567 - lr: 5.9606e-05\n",
      "Epoch 125/200\n",
      "313/313 [==============================] - 118s 373ms/step - loss: 0.2783 - categorical_accuracy: 0.8898 - val_loss: 0.4267 - val_categorical_accuracy: 0.8533 - lr: 5.7845e-05\n",
      "Epoch 126/200\n",
      "313/313 [==============================] - 120s 377ms/step - loss: 0.2823 - categorical_accuracy: 0.8930 - val_loss: 0.4063 - val_categorical_accuracy: 0.8600 - lr: 5.6135e-05\n",
      "Epoch 127/200\n",
      "313/313 [==============================] - 123s 385ms/step - loss: 0.2870 - categorical_accuracy: 0.8851 - val_loss: 0.3301 - val_categorical_accuracy: 0.8633 - lr: 5.4476e-05\n",
      "Epoch 128/200\n",
      "313/313 [==============================] - 127s 396ms/step - loss: 0.2778 - categorical_accuracy: 0.8949 - val_loss: 0.5818 - val_categorical_accuracy: 0.8500 - lr: 5.2866e-05\n",
      "Epoch 129/200\n",
      "313/313 [==============================] - 120s 376ms/step - loss: 0.2809 - categorical_accuracy: 0.8887 - val_loss: 0.5588 - val_categorical_accuracy: 0.8667 - lr: 5.1304e-05\n",
      "Epoch 130/200\n",
      "313/313 [==============================] - 121s 378ms/step - loss: 0.2750 - categorical_accuracy: 0.8942 - val_loss: 0.4276 - val_categorical_accuracy: 0.8700 - lr: 4.9787e-05\n",
      "Epoch 131/200\n",
      "313/313 [==============================] - 120s 378ms/step - loss: 0.2858 - categorical_accuracy: 0.8872 - val_loss: 0.3505 - val_categorical_accuracy: 0.8767 - lr: 4.8316e-05\n",
      "Epoch 132/200\n",
      "313/313 [==============================] - 120s 377ms/step - loss: 0.2775 - categorical_accuracy: 0.8893 - val_loss: 0.6309 - val_categorical_accuracy: 0.8733 - lr: 4.6888e-05\n",
      "Epoch 133/200\n",
      "313/313 [==============================] - 120s 378ms/step - loss: 0.2779 - categorical_accuracy: 0.8944 - val_loss: 0.5946 - val_categorical_accuracy: 0.8733 - lr: 4.5502e-05\n",
      "Epoch 134/200\n",
      "313/313 [==============================] - 119s 374ms/step - loss: 0.2768 - categorical_accuracy: 0.8920 - val_loss: 0.3823 - val_categorical_accuracy: 0.8733 - lr: 4.4157e-05\n",
      "Epoch 135/200\n",
      "313/313 [==============================] - 119s 373ms/step - loss: 0.2737 - categorical_accuracy: 0.8917 - val_loss: 0.6045 - val_categorical_accuracy: 0.8633 - lr: 4.2852e-05\n",
      "Epoch 136/200\n",
      "313/313 [==============================] - 121s 378ms/step - loss: 0.2715 - categorical_accuracy: 0.8911 - val_loss: 0.4074 - val_categorical_accuracy: 0.8800 - lr: 4.1586e-05\n",
      "Epoch 137/200\n",
      "313/313 [==============================] - 119s 375ms/step - loss: 0.2711 - categorical_accuracy: 0.8936 - val_loss: 0.3724 - val_categorical_accuracy: 0.8867 - lr: 4.0357e-05\n",
      "Epoch 138/200\n",
      "313/313 [==============================] - 122s 381ms/step - loss: 0.2723 - categorical_accuracy: 0.8941 - val_loss: 0.3406 - val_categorical_accuracy: 0.8733 - lr: 3.9164e-05\n",
      "Epoch 139/200\n",
      "313/313 [==============================] - 120s 376ms/step - loss: 0.2794 - categorical_accuracy: 0.8904 - val_loss: 0.3286 - val_categorical_accuracy: 0.8900 - lr: 3.8007e-05\n",
      "Epoch 140/200\n",
      "313/313 [==============================] - 121s 377ms/step - loss: 0.2673 - categorical_accuracy: 0.8965 - val_loss: 0.3502 - val_categorical_accuracy: 0.8833 - lr: 3.6883e-05\n",
      "Epoch 141/200\n",
      "313/313 [==============================] - 126s 379ms/step - loss: 0.2705 - categorical_accuracy: 0.8965 - val_loss: 0.4344 - val_categorical_accuracy: 0.8800 - lr: 3.5793e-05\n",
      "Epoch 142/200\n",
      "313/313 [==============================] - 124s 377ms/step - loss: 0.2756 - categorical_accuracy: 0.8925 - val_loss: 0.3196 - val_categorical_accuracy: 0.8867 - lr: 3.4735e-05\n",
      "Epoch 143/200\n",
      "313/313 [==============================] - 121s 381ms/step - loss: 0.2664 - categorical_accuracy: 0.8952 - val_loss: 0.3075 - val_categorical_accuracy: 0.8867 - lr: 3.3709e-05\n",
      "Epoch 144/200\n",
      "313/313 [==============================] - 123s 379ms/step - loss: 0.2748 - categorical_accuracy: 0.8927 - val_loss: 0.5600 - val_categorical_accuracy: 0.8867 - lr: 3.2713e-05\n",
      "Epoch 145/200\n",
      "313/313 [==============================] - 126s 378ms/step - loss: 0.2723 - categorical_accuracy: 0.8928 - val_loss: 0.3705 - val_categorical_accuracy: 0.8633 - lr: 3.1746e-05\n",
      "Epoch 146/200\n",
      "313/313 [==============================] - 125s 384ms/step - loss: 0.2656 - categorical_accuracy: 0.8965 - val_loss: 0.4426 - val_categorical_accuracy: 0.8800 - lr: 3.0808e-05\n",
      "Epoch 147/200\n",
      "313/313 [==============================] - 122s 383ms/step - loss: 0.2695 - categorical_accuracy: 0.8981 - val_loss: 0.4240 - val_categorical_accuracy: 0.8767 - lr: 2.9897e-05\n",
      "Epoch 148/200\n",
      "313/313 [==============================] - 121s 381ms/step - loss: 0.2595 - categorical_accuracy: 0.9021 - val_loss: 0.4884 - val_categorical_accuracy: 0.8800 - lr: 2.9013e-05\n",
      "Epoch 149/200\n",
      "313/313 [==============================] - 119s 376ms/step - loss: 0.2707 - categorical_accuracy: 0.8968 - val_loss: 0.4613 - val_categorical_accuracy: 0.8700 - lr: 2.8156e-05\n",
      "Epoch 150/200\n",
      "313/313 [==============================] - 127s 390ms/step - loss: 0.2617 - categorical_accuracy: 0.8970 - val_loss: 0.4751 - val_categorical_accuracy: 0.8733 - lr: 2.7324e-05\n",
      "Epoch 151/200\n",
      "313/313 [==============================] - 123s 389ms/step - loss: 0.2673 - categorical_accuracy: 0.8946 - val_loss: 0.7359 - val_categorical_accuracy: 0.8700 - lr: 2.6516e-05\n",
      "Epoch 152/200\n",
      "313/313 [==============================] - 123s 379ms/step - loss: 0.2716 - categorical_accuracy: 0.8898 - val_loss: 0.5187 - val_categorical_accuracy: 0.8767 - lr: 2.5733e-05\n",
      "Epoch 153/200\n",
      "313/313 [==============================] - 120s 377ms/step - loss: 0.2672 - categorical_accuracy: 0.8994 - val_loss: 0.4909 - val_categorical_accuracy: 0.8700 - lr: 2.4972e-05\n",
      "Epoch 154/200\n",
      "313/313 [==============================] - 121s 375ms/step - loss: 0.2725 - categorical_accuracy: 0.8955 - val_loss: 0.5587 - val_categorical_accuracy: 0.8800 - lr: 2.4234e-05\n",
      "Epoch 155/200\n",
      "313/313 [==============================] - 119s 371ms/step - loss: 0.2704 - categorical_accuracy: 0.8954 - val_loss: 0.5181 - val_categorical_accuracy: 0.8733 - lr: 2.3518e-05\n",
      "Epoch 156/200\n",
      "313/313 [==============================] - 122s 383ms/step - loss: 0.2716 - categorical_accuracy: 0.8960 - val_loss: 0.5337 - val_categorical_accuracy: 0.8767 - lr: 2.2823e-05\n",
      "Epoch 157/200\n",
      "313/313 [==============================] - 121s 377ms/step - loss: 0.2695 - categorical_accuracy: 0.8955 - val_loss: 0.5658 - val_categorical_accuracy: 0.8733 - lr: 2.2148e-05\n",
      "Epoch 158/200\n",
      "313/313 [==============================] - 127s 392ms/step - loss: 0.2646 - categorical_accuracy: 0.8955 - val_loss: 0.5194 - val_categorical_accuracy: 0.8733 - lr: 2.1494e-05\n",
      "Epoch 159/200\n",
      "313/313 [==============================] - 120s 380ms/step - loss: 0.2685 - categorical_accuracy: 0.8950 - val_loss: 0.4912 - val_categorical_accuracy: 0.8733 - lr: 2.0858e-05\n",
      "Epoch 160/200\n",
      "313/313 [==============================] - 120s 375ms/step - loss: 0.2675 - categorical_accuracy: 0.8994 - val_loss: 0.6194 - val_categorical_accuracy: 0.8733 - lr: 2.0242e-05\n",
      "Epoch 161/200\n",
      "313/313 [==============================] - 126s 374ms/step - loss: 0.2650 - categorical_accuracy: 0.8963 - val_loss: 0.6495 - val_categorical_accuracy: 0.8767 - lr: 1.9644e-05\n",
      "Epoch 162/200\n",
      "313/313 [==============================] - 128s 389ms/step - loss: 0.2699 - categorical_accuracy: 0.8944 - val_loss: 0.5915 - val_categorical_accuracy: 0.8700 - lr: 1.9063e-05\n",
      "Epoch 163/200\n",
      "313/313 [==============================] - 124s 391ms/step - loss: 0.2589 - categorical_accuracy: 0.9000 - val_loss: 0.5679 - val_categorical_accuracy: 0.8800 - lr: 1.8500e-05\n",
      "Epoch 164/200\n",
      "313/313 [==============================] - 125s 393ms/step - loss: 0.2607 - categorical_accuracy: 0.8971 - val_loss: 0.4867 - val_categorical_accuracy: 0.8767 - lr: 1.7953e-05\n",
      "Epoch 165/200\n",
      "313/313 [==============================] - 126s 398ms/step - loss: 0.2666 - categorical_accuracy: 0.8966 - val_loss: 0.4834 - val_categorical_accuracy: 0.8767 - lr: 1.7422e-05\n",
      "Epoch 166/200\n",
      "313/313 [==============================] - 127s 400ms/step - loss: 0.2659 - categorical_accuracy: 0.8952 - val_loss: 0.5351 - val_categorical_accuracy: 0.8800 - lr: 1.6908e-05\n",
      "Epoch 167/200\n",
      "313/313 [==============================] - 121s 382ms/step - loss: 0.2657 - categorical_accuracy: 0.8958 - val_loss: 0.5510 - val_categorical_accuracy: 0.8833 - lr: 1.6408e-05\n",
      "Epoch 168/200\n",
      "313/313 [==============================] - 126s 396ms/step - loss: 0.2601 - categorical_accuracy: 0.9013 - val_loss: 0.5646 - val_categorical_accuracy: 0.8700 - lr: 1.5923e-05\n",
      "Epoch 169/200\n",
      "313/313 [==============================] - 125s 396ms/step - loss: 0.2665 - categorical_accuracy: 0.8962 - val_loss: 0.6730 - val_categorical_accuracy: 0.8667 - lr: 1.5452e-05\n",
      "Epoch 170/200\n",
      "313/313 [==============================] - 126s 396ms/step - loss: 0.2669 - categorical_accuracy: 0.8942 - val_loss: 0.8734 - val_categorical_accuracy: 0.8700 - lr: 1.4996e-05\n",
      "Epoch 171/200\n",
      "313/313 [==============================] - 120s 380ms/step - loss: 0.2621 - categorical_accuracy: 0.8955 - val_loss: 0.6724 - val_categorical_accuracy: 0.8733 - lr: 1.4552e-05\n",
      "Epoch 172/200\n",
      "313/313 [==============================] - 126s 395ms/step - loss: 0.2680 - categorical_accuracy: 0.8901 - val_loss: 0.4557 - val_categorical_accuracy: 0.8833 - lr: 1.4122e-05\n",
      "Epoch 173/200\n",
      "313/313 [==============================] - 120s 377ms/step - loss: 0.2726 - categorical_accuracy: 0.8931 - val_loss: 0.5470 - val_categorical_accuracy: 0.8733 - lr: 1.3705e-05\n",
      "Epoch 174/200\n",
      "313/313 [==============================] - 125s 395ms/step - loss: 0.2610 - categorical_accuracy: 0.9005 - val_loss: 0.4614 - val_categorical_accuracy: 0.8733 - lr: 1.3300e-05\n",
      "Epoch 175/200\n",
      "313/313 [==============================] - 121s 382ms/step - loss: 0.2655 - categorical_accuracy: 0.8957 - val_loss: 0.4817 - val_categorical_accuracy: 0.8767 - lr: 1.2907e-05\n",
      "Epoch 176/200\n",
      "313/313 [==============================] - 122s 384ms/step - loss: 0.2680 - categorical_accuracy: 0.8958 - val_loss: 0.4963 - val_categorical_accuracy: 0.8733 - lr: 1.2525e-05\n",
      "Epoch 177/200\n",
      "313/313 [==============================] - 121s 380ms/step - loss: 0.2682 - categorical_accuracy: 0.8944 - val_loss: 0.4850 - val_categorical_accuracy: 0.8700 - lr: 1.2155e-05\n",
      "Epoch 178/200\n",
      "313/313 [==============================] - 122s 384ms/step - loss: 0.2635 - categorical_accuracy: 0.8955 - val_loss: 0.4541 - val_categorical_accuracy: 0.8800 - lr: 1.1796e-05\n",
      "Epoch 179/200\n",
      "313/313 [==============================] - 121s 381ms/step - loss: 0.2689 - categorical_accuracy: 0.8960 - val_loss: 0.4343 - val_categorical_accuracy: 0.8833 - lr: 1.1447e-05\n",
      "Epoch 180/200\n",
      "313/313 [==============================] - 120s 379ms/step - loss: 0.2585 - categorical_accuracy: 0.8963 - val_loss: 0.6065 - val_categorical_accuracy: 0.8767 - lr: 1.1109e-05\n",
      "Epoch 181/200\n",
      "313/313 [==============================] - 123s 387ms/step - loss: 0.2534 - categorical_accuracy: 0.9050 - val_loss: 0.5354 - val_categorical_accuracy: 0.8767 - lr: 1.0781e-05\n",
      "Epoch 182/200\n",
      "313/313 [==============================] - 120s 378ms/step - loss: 0.2644 - categorical_accuracy: 0.8935 - val_loss: 0.4911 - val_categorical_accuracy: 0.8733 - lr: 1.0462e-05\n",
      "Epoch 183/200\n",
      "313/313 [==============================] - 119s 376ms/step - loss: 0.2660 - categorical_accuracy: 0.8982 - val_loss: 0.5756 - val_categorical_accuracy: 0.8700 - lr: 1.0153e-05\n",
      "Epoch 184/200\n",
      "313/313 [==============================] - 119s 375ms/step - loss: 0.2594 - categorical_accuracy: 0.8979 - val_loss: 0.5790 - val_categorical_accuracy: 0.8733 - lr: 9.8529e-06\n",
      "Epoch 185/200\n",
      "313/313 [==============================] - 119s 374ms/step - loss: 0.2639 - categorical_accuracy: 0.8938 - val_loss: 0.4482 - val_categorical_accuracy: 0.8800 - lr: 9.5617e-06\n",
      "Epoch 186/200\n",
      "313/313 [==============================] - 120s 374ms/step - loss: 0.2635 - categorical_accuracy: 0.8970 - val_loss: 0.4748 - val_categorical_accuracy: 0.8800 - lr: 9.2791e-06\n",
      "Epoch 187/200\n",
      "313/313 [==============================] - 120s 378ms/step - loss: 0.2634 - categorical_accuracy: 0.8925 - val_loss: 0.5785 - val_categorical_accuracy: 0.8800 - lr: 9.0048e-06\n",
      "Epoch 188/200\n",
      "313/313 [==============================] - 121s 381ms/step - loss: 0.2609 - categorical_accuracy: 0.8973 - val_loss: 0.3828 - val_categorical_accuracy: 0.8833 - lr: 8.7387e-06\n",
      "Epoch 189/200\n",
      "313/313 [==============================] - 119s 373ms/step - loss: 0.2647 - categorical_accuracy: 0.8970 - val_loss: 0.4822 - val_categorical_accuracy: 0.8767 - lr: 8.4804e-06\n",
      "Epoch 190/200\n",
      "313/313 [==============================] - 122s 377ms/step - loss: 0.2654 - categorical_accuracy: 0.8938 - val_loss: 0.6101 - val_categorical_accuracy: 0.8800 - lr: 8.2298e-06\n",
      "Epoch 191/200\n",
      "313/313 [==============================] - 124s 385ms/step - loss: 0.2617 - categorical_accuracy: 0.8957 - val_loss: 0.6707 - val_categorical_accuracy: 0.8767 - lr: 7.9866e-06\n",
      "Epoch 192/200\n",
      "313/313 [==============================] - 121s 380ms/step - loss: 0.2579 - categorical_accuracy: 0.8986 - val_loss: 0.5560 - val_categorical_accuracy: 0.8800 - lr: 7.7505e-06\n",
      "Epoch 193/200\n",
      "313/313 [==============================] - ETA: 0s - loss: 0.2629 - categorical_accuracy: 0.8979Restoring model weights from the end of the best epoch: 143.\n",
      "313/313 [==============================] - 120s 375ms/step - loss: 0.2629 - categorical_accuracy: 0.8979 - val_loss: 0.5435 - val_categorical_accuracy: 0.8800 - lr: 7.5215e-06\n",
      "Epoch 193: early stopping\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model = tf.keras.Sequential([\n",
    "    resize_rescale_augment,\n",
    "    layers.Conv2D(3, 5, padding='same', activation='tanh'), # this seems like a bit of a brute force approach to handing a 3 channel image to resnet, \n",
    "                                                            # maybe try changing the source so it accepts 4 channels?\n",
    "    tf.keras.applications.InceptionResNetV2(\n",
    "    include_top=False,\n",
    "    weights=None,\n",
    "    input_shape=(120, 120, 3),\n",
    "    pooling=None,\n",
    "    classes=3),\n",
    "    layers.Flatten(), # does this make sense? or is there another way to get down to just three output dimensions?\n",
    "    layers.Dense(3)\n",
    "])\n",
    "\n",
    "log_dir = \"E:/Users/sentinel_industry/logs/train/threemod/inceptres\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, profile_batch = '120,140')\n",
    "\n",
    "def scheduler(epoch, lr):\n",
    "    if epoch < 30:\n",
    "        return lr\n",
    "    else:\n",
    "        return lr * tf.math.exp(-0.03)\n",
    "    \n",
    "lr_callback = tf.keras.callbacks.LearningRateScheduler(scheduler)\n",
    "es_callback = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=50,verbose=1, restore_best_weights=True)\n",
    "\n",
    "model.compile(optimizer=keras.optimizers.Adam(),\n",
    "              loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True), \n",
    "              metrics=[tf.keras.metrics.CategoricalAccuracy()])\n",
    "model.build()\n",
    "model.summary()\n",
    "\n",
    "# now with normalisation\n",
    "epochs=200\n",
    "batch_size = 20\n",
    "history = model.fit(\n",
    "    train.batch(batch_size, drop_remainder=True, num_parallel_calls=tf.data.AUTOTUNE).prefetch(tf.data.AUTOTUNE),\n",
    "    validation_data=valid.batch(batch_size, drop_remainder=True, num_parallel_calls=tf.data.AUTOTUNE).prefetch(tf.data.AUTOTUNE),\n",
    "    validation_steps=15,\n",
    "    epochs=epochs,\n",
    "    callbacks=[tensorboard_callback, lr_callback, es_callback]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "461561a4-bd73-4fd7-ad8b-84eaee27bfe0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "46647eff-f12c-4f55-bd21-09eb71452857",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 5 of 245). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: saved_models/inceptionres\\assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: saved_models/inceptionres\\assets\n"
     ]
    }
   ],
   "source": [
    "model.save('saved_models/inceptionres')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fb0b841-6684-4356-9323-38f6839cd194",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}