2_Base_Model_MobileNetV2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Base Model:    MobileNetV2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 403,
     "status": "ok",
     "timestamp": 1649441713507,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "0OxTZo0K1HzG"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "# from math import log\n",
    "import numpy as np\n",
    "# from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "# import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 399,
     "status": "ok",
     "timestamp": 1649441717589,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "_WN1jEJn1XBj"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PzzP4mUk2-JX"
   },
   "source": [
    "**Importing Generalized Tensor Image Data [Size: 224 X 224]**\n",
    "The dataset exist as Test and Train sets split in 7:3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 42766,
     "status": "ok",
     "timestamp": 1649441762114,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "4D6AScGQSv-G"
   },
   "outputs": [],
   "source": [
    "X_train = np.load('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/X_train_3.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1649441762115,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "13zTPQC0UEhg"
   },
   "outputs": [],
   "source": [
    "Y_train = np.load('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/Y_train_3.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gXJ9MgKkj6AU"
   },
   "source": [
    "**Creating validation and test sets. Final sets: Train: 70%, Validation: 14%, Test: 16%**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FK5ehUu3UEbC"
   },
   "outputs": [],
   "source": [
    "# X_val_and_test = np.load('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/X_test_3.npy') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jq_hBhRHUEX8"
   },
   "outputs": [],
   "source": [
    "# Y_val_and_test = np.load('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/Y_test_3.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wShZwJkfUEVR"
   },
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import train_test_split\n",
    "# X_test, X_val, Y_test, Y_val = train_test_split(X_val_and_test, Y_val_and_test, test_size=0.47)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 380,
     "status": "ok",
     "timestamp": 1649260523802,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "wPpGxnpgUESZ",
    "outputId": "84478f2b-7f33-4b9f-870b-a6628245f5a5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train Size:(7607, 224, 224, 3)\n",
      "Y_train Size:(7607, 9)\n"
     ]
    }
   ],
   "source": [
    "# print(f'X_train Size:{X_train.shape}\\nY_train Size:{Y_train.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 499,
     "status": "ok",
     "timestamp": 1649260526100,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "8lZGM5CqUEKR",
    "outputId": "ea6dad02-b837-46ef-de35-5dd33f02f25e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_val Size:(1533, 224, 224, 3)\n",
      "Y_val Size:(1533, 9)\n"
     ]
    }
   ],
   "source": [
    "# print(f'X_val Size:{X_val.shape}\\nY_val Size:{Y_val.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1649260527552,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "-RyYcEIPUD-v",
    "outputId": "2eb1ee5e-6ab8-4df4-b100-636d145171e4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_test Size:(1728, 224, 224, 3)\n",
      "Y_test Size:(1728, 9)\n"
     ]
    }
   ],
   "source": [
    "# print(f'X_test Size:{X_test.shape}\\nY_test Size:{Y_test.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0UCFZcDsilcg"
   },
   "outputs": [],
   "source": [
    "# np.save('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/X_val_3.npy', X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0ebU1l2yilRx"
   },
   "outputs": [],
   "source": [
    "# np.save('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/Y_val_3.npy', Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LvW7-dlqjED0"
   },
   "outputs": [],
   "source": [
    "# np.save('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/X_test_3_REAL.npy', X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Y_Q4jtlwjEAW"
   },
   "outputs": [],
   "source": [
    "# np.save('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/Y_test_3_REAL.npy', Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 7679,
     "status": "ok",
     "timestamp": 1649441769789,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "B-jX_m4_jXOr"
   },
   "outputs": [],
   "source": [
    "X_val = np.load('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/X_val_3.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1649441769789,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "RPYji1dmjW5u"
   },
   "outputs": [],
   "source": [
    "Y_val = np.load('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/Y_val_3.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 19509,
     "status": "ok",
     "timestamp": 1649441789289,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "MIDhqLTHjW2L"
   },
   "outputs": [],
   "source": [
    "X_test = np.load('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/X_test_3.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1649441789290,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "PK5ds02ljWzQ"
   },
   "outputs": [],
   "source": [
    "Y_test = np.load('/content/drive/MyDrive/Colab Notebooks/Dataset_Numpy_Array_3/Y_test_3.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PjaHM_xRdxWi"
   },
   "source": [
    "The training labels are one-hot encoded. The code below removes this to make a new array \"y_train_new\" which will be used to produce class weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1649441789290,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "e7F7ev2ueWoF",
    "outputId": "03c85219-469c-4a55-f975-d708d667c3d8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 7, 2, ..., 2, 1, 2])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train_new = np.argmax(Y_train, axis=1)\n",
    "Y_train_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "executionInfo": {
     "elapsed": 1405,
     "status": "ok",
     "timestamp": 1649441790691,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "sWYxm5KTehPZ"
   },
   "outputs": [],
   "source": [
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "class_weights = compute_class_weight('balanced', classes = np.unique(Y_train_new), y = Y_train_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1649441790691,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "0_dsGDBVfPq7",
    "outputId": "468f1a97-da53-480d-e5a6-12d796281f70"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(class_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "j6Qlmr6ifRnP"
   },
   "source": [
    "Note: The class weights is a numpy.ndarray. This does not work with the updated sklearn library and thus the class weights are converted to a dictionary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1649441790691,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "YK9MsSxwethD"
   },
   "outputs": [],
   "source": [
    "class_weight_dict = dict(enumerate(class_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1649441790692,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "G03heX5kfWLu"
   },
   "outputs": [],
   "source": [
    "nb_classes = 9\n",
    "img_size = 224\n",
    "channels = 3\n",
    "height = 224\n",
    "width = 224"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ru51u-rDgFPz"
   },
   "source": [
    "**Building The Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "executionInfo": {
     "elapsed": 413,
     "status": "ok",
     "timestamp": 1649441938674,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "htVFdFHHgQDQ"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "from keras.models import Sequential, Model\n",
    "from keras.applications.xception import Xception, preprocess_input\n",
    "from keras.applications.vgg19 import VGG19, preprocess_input\n",
    "from keras.applications.resnet import ResNet50 , preprocess_input\n",
    "from keras.applications.inception_v3 import InceptionV3 , preprocess_input\n",
    "from keras.applications.mobilenet import MobileNet , preprocess_input\n",
    "from keras.applications.mobilenet_v2 import MobileNetV2, preprocess_input\n",
    "from keras.applications.inception_resnet_v2 import InceptionResNetV2, preprocess_input\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n",
    "\n",
    "from keras.applications.nasnet import NASNetLarge , preprocess_input\n",
    "from keras.applications.xception import Xception , preprocess_input\n",
    "from keras.applications.densenet import DenseNet169 , preprocess_input\n",
    "from keras.applications.densenet import DenseNet121, preprocess_input\n",
    "\n",
    "from keras.layers import Dense, Flatten, Activation, Dropout, GlobalAveragePooling2D\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras import optimizers, applications\n",
    "from tensorflow.keras.optimizers import Adam, SGD, RMSprop, Nadam\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping\n",
    "from keras import backend as K \n",
    "from keras.losses import binary_crossentropy, categorical_crossentropy\n",
    "from keras.layers import BatchNormalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "executionInfo": {
     "elapsed": 357,
     "status": "ok",
     "timestamp": 1649441950517,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "tY23t0aGfo41"
   },
   "outputs": [],
   "source": [
    "def get_base_model(model_name):\n",
    "    if model_name == 'Xception' :\n",
    "        base_model = Xception(input_shape=(height, width, channels), weights='imagenet', include_top=False)\n",
    "        \n",
    "    elif model_name =='ResNet50':\n",
    "        base_model = ResNet50(input_shape=(height, width, channels), weights='imagenet', include_top=False)\n",
    "    \n",
    "    elif model_name =='inceptionv3':\n",
    "        base_model = InceptionV3(input_shape=(height, width, channels), weights='imagenet', include_top=False)\n",
    "  \n",
    "    elif model_name =='InceptionResNetV2':\n",
    "        base_model = InceptionResNetV2(input_shape=(height, width, channels), weights='imagenet', include_top=False)\n",
    "  \n",
    "    elif model_name =='MobileNet':\n",
    "        base_model = MobileNet(input_shape=(height, width, channels), weights='imagenet', include_top=False)\n",
    "    \n",
    "    elif model_name =='MobileNetV2':\n",
    "        base_model = MobileNetV2(input_shape=(height, width, channels), weights='imagenet', include_top=False)\n",
    "\n",
    "    elif model_name =='VGG19':\n",
    "        base_model = VGG19(input_shape=(height, width, channels), weights='imagenet', include_top=False)\n",
    "    \n",
    "    elif model_name =='NASNetLarge':\n",
    "        base_model = NASNetLarge(input_shape=(height, width, channels), weights='imagenet', include_top=False)\n",
    "  \n",
    "    elif model_name =='DenseNet169':\n",
    "        base_model = DenseNet169(input_shape=(height, width, channels), weights='imagenet', include_top=False)\n",
    "  \n",
    "    else :\n",
    "        base_model = Sequential()\n",
    "        base_model.add(Conv2D(filters = 128, kernel_size = (3, 3), activation = 'relu', input_shape = (height, width, channels)))\n",
    "        base_model.add(MaxPooling2D(pool_size = (2, 2)))\n",
    "\n",
    "    return base_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1649441953147,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "BIJGkCZBfpuV"
   },
   "outputs": [],
   "source": [
    "model_names = ['ResNet50', 'inceptionv3', 'InceptionResNetV2', 'VGG19', 'Xception', 'MobileNet', 'MobileNetV2', 'DenseNet169', 'NASNetLarge']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1649441954629,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "E4kiUZYbf1PQ",
    "outputId": "bb741fb6-5e36-4697-d018-ae2bec555167"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected model :  MobileNetV2\n"
     ]
    }
   ],
   "source": [
    "model_id = 6\n",
    "model_name = model_names[model_id]\n",
    "print('Selected model : ', model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2133,
     "status": "ok",
     "timestamp": 1649441958176,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "KSiOEK3ef3B5",
    "outputId": "d6e1a602-be90-47cd-bc29-666bd8a1d8ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/mobilenet_v2/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_224_no_top.h5\n",
      "9412608/9406464 [==============================] - 0s 0us/step\n",
      "9420800/9406464 [==============================] - 0s 0us/step\n"
     ]
    }
   ],
   "source": [
    "base_model = get_base_model(model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "G67q6rJenO40"
   },
   "source": [
    "Removing the last layer of Pre-Trained model and adding a fully connected dense layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2781,
     "status": "ok",
     "timestamp": 1649441965180,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "vt5qq2j0gqM5",
    "outputId": "308a2c13-edc6-487a-a1a2-4263f02c6e8a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)           [(None, 224, 224, 3  0           []                               \n",
      "                                )]                                                                \n",
      "                                                                                                  \n",
      " Conv1 (Conv2D)                 (None, 112, 112, 32  864         ['input_1[0][0]']                \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " bn_Conv1 (BatchNormalization)  (None, 112, 112, 32  128         ['Conv1[0][0]']                  \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " Conv1_relu (ReLU)              (None, 112, 112, 32  0           ['bn_Conv1[0][0]']               \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " expanded_conv_depthwise (Depth  (None, 112, 112, 32  288        ['Conv1_relu[0][0]']             \n",
      " wiseConv2D)                    )                                                                 \n",
      "                                                                                                  \n",
      " expanded_conv_depthwise_BN (Ba  (None, 112, 112, 32  128        ['expanded_conv_depthwise[0][0]']\n",
      " tchNormalization)              )                                                                 \n",
      "                                                                                                  \n",
      " expanded_conv_depthwise_relu (  (None, 112, 112, 32  0          ['expanded_conv_depthwise_BN[0][0\n",
      " ReLU)                          )                                ]']                              \n",
      "                                                                                                  \n",
      " expanded_conv_project (Conv2D)  (None, 112, 112, 16  512        ['expanded_conv_depthwise_relu[0]\n",
      "                                )                                [0]']                            \n",
      "                                                                                                  \n",
      " expanded_conv_project_BN (Batc  (None, 112, 112, 16  64         ['expanded_conv_project[0][0]']  \n",
      " hNormalization)                )                                                                 \n",
      "                                                                                                  \n",
      " block_1_expand (Conv2D)        (None, 112, 112, 96  1536        ['expanded_conv_project_BN[0][0]'\n",
      "                                )                                ]                                \n",
      "                                                                                                  \n",
      " block_1_expand_BN (BatchNormal  (None, 112, 112, 96  384        ['block_1_expand[0][0]']         \n",
      " ization)                       )                                                                 \n",
      "                                                                                                  \n",
      " block_1_expand_relu (ReLU)     (None, 112, 112, 96  0           ['block_1_expand_BN[0][0]']      \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " block_1_pad (ZeroPadding2D)    (None, 113, 113, 96  0           ['block_1_expand_relu[0][0]']    \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " block_1_depthwise (DepthwiseCo  (None, 56, 56, 96)  864         ['block_1_pad[0][0]']            \n",
      " nv2D)                                                                                            \n",
      "                                                                                                  \n",
      " block_1_depthwise_BN (BatchNor  (None, 56, 56, 96)  384         ['block_1_depthwise[0][0]']      \n",
      " malization)                                                                                      \n",
      "                                                                                                  \n",
      " block_1_depthwise_relu (ReLU)  (None, 56, 56, 96)   0           ['block_1_depthwise_BN[0][0]']   \n",
      "                                                                                                  \n",
      " block_1_project (Conv2D)       (None, 56, 56, 24)   2304        ['block_1_depthwise_relu[0][0]'] \n",
      "                                                                                                  \n",
      " block_1_project_BN (BatchNorma  (None, 56, 56, 24)  96          ['block_1_project[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_2_expand (Conv2D)        (None, 56, 56, 144)  3456        ['block_1_project_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_2_expand_BN (BatchNormal  (None, 56, 56, 144)  576        ['block_2_expand[0][0]']         \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " block_2_expand_relu (ReLU)     (None, 56, 56, 144)  0           ['block_2_expand_BN[0][0]']      \n",
      "                                                                                                  \n",
      " block_2_depthwise (DepthwiseCo  (None, 56, 56, 144)  1296       ['block_2_expand_relu[0][0]']    \n",
      " nv2D)                                                                                            \n",
      "                                                                                                  \n",
      " block_2_depthwise_BN (BatchNor  (None, 56, 56, 144)  576        ['block_2_depthwise[0][0]']      \n",
      " malization)                                                                                      \n",
      "                                                                                                  \n",
      " block_2_depthwise_relu (ReLU)  (None, 56, 56, 144)  0           ['block_2_depthwise_BN[0][0]']   \n",
      "                                                                                                  \n",
      " block_2_project (Conv2D)       (None, 56, 56, 24)   3456        ['block_2_depthwise_relu[0][0]'] \n",
      "                                                                                                  \n",
      " block_2_project_BN (BatchNorma  (None, 56, 56, 24)  96          ['block_2_project[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_2_add (Add)              (None, 56, 56, 24)   0           ['block_1_project_BN[0][0]',     \n",
      "                                                                  'block_2_project_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_3_expand (Conv2D)        (None, 56, 56, 144)  3456        ['block_2_add[0][0]']            \n",
      "                                                                                                  \n",
      " block_3_expand_BN (BatchNormal  (None, 56, 56, 144)  576        ['block_3_expand[0][0]']         \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " block_3_expand_relu (ReLU)     (None, 56, 56, 144)  0           ['block_3_expand_BN[0][0]']      \n",
      "                                                                                                  \n",
      " block_3_pad (ZeroPadding2D)    (None, 57, 57, 144)  0           ['block_3_expand_relu[0][0]']    \n",
      "                                                                                                  \n",
      " block_3_depthwise (DepthwiseCo  (None, 28, 28, 144)  1296       ['block_3_pad[0][0]']            \n",
      " nv2D)                                                                                            \n",
      "                                                                                                  \n",
      " block_3_depthwise_BN (BatchNor  (None, 28, 28, 144)  576        ['block_3_depthwise[0][0]']      \n",
      " malization)                                                                                      \n",
      "                                                                                                  \n",
      " block_3_depthwise_relu (ReLU)  (None, 28, 28, 144)  0           ['block_3_depthwise_BN[0][0]']   \n",
      "                                                                                                  \n",
      " block_3_project (Conv2D)       (None, 28, 28, 32)   4608        ['block_3_depthwise_relu[0][0]'] \n",
      "                                                                                                  \n",
      " block_3_project_BN (BatchNorma  (None, 28, 28, 32)  128         ['block_3_project[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_4_expand (Conv2D)        (None, 28, 28, 192)  6144        ['block_3_project_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_4_expand_BN (BatchNormal  (None, 28, 28, 192)  768        ['block_4_expand[0][0]']         \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " block_4_expand_relu (ReLU)     (None, 28, 28, 192)  0           ['block_4_expand_BN[0][0]']      \n",
      "                                                                                                  \n",
      " block_4_depthwise (DepthwiseCo  (None, 28, 28, 192)  1728       ['block_4_expand_relu[0][0]']    \n",
      " nv2D)                                                                                            \n",
      "                                                                                                  \n",
      " block_4_depthwise_BN (BatchNor  (None, 28, 28, 192)  768        ['block_4_depthwise[0][0]']      \n",
      " malization)                                                                                      \n",
      "                                                                                                  \n",
      " block_4_depthwise_relu (ReLU)  (None, 28, 28, 192)  0           ['block_4_depthwise_BN[0][0]']   \n",
      "                                                                                                  \n",
      " block_4_project (Conv2D)       (None, 28, 28, 32)   6144        ['block_4_depthwise_relu[0][0]'] \n",
      "                                                                                                  \n",
      " block_4_project_BN (BatchNorma  (None, 28, 28, 32)  128         ['block_4_project[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_4_add (Add)              (None, 28, 28, 32)   0           ['block_3_project_BN[0][0]',     \n",
      "                                                                  'block_4_project_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_5_expand (Conv2D)        (None, 28, 28, 192)  6144        ['block_4_add[0][0]']            \n",
      "                                                                                                  \n",
      " block_5_expand_BN (BatchNormal  (None, 28, 28, 192)  768        ['block_5_expand[0][0]']         \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " block_5_expand_relu (ReLU)     (None, 28, 28, 192)  0           ['block_5_expand_BN[0][0]']      \n",
      "                                                                                                  \n",
      " block_5_depthwise (DepthwiseCo  (None, 28, 28, 192)  1728       ['block_5_expand_relu[0][0]']    \n",
      " nv2D)                                                                                            \n",
      "                                                                                                  \n",
      " block_5_depthwise_BN (BatchNor  (None, 28, 28, 192)  768        ['block_5_depthwise[0][0]']      \n",
      " malization)                                                                                      \n",
      "                                                                                                  \n",
      " block_5_depthwise_relu (ReLU)  (None, 28, 28, 192)  0           ['block_5_depthwise_BN[0][0]']   \n",
      "                                                                                                  \n",
      " block_5_project (Conv2D)       (None, 28, 28, 32)   6144        ['block_5_depthwise_relu[0][0]'] \n",
      "                                                                                                  \n",
      " block_5_project_BN (BatchNorma  (None, 28, 28, 32)  128         ['block_5_project[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_5_add (Add)              (None, 28, 28, 32)   0           ['block_4_add[0][0]',            \n",
      "                                                                  'block_5_project_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_6_expand (Conv2D)        (None, 28, 28, 192)  6144        ['block_5_add[0][0]']            \n",
      "                                                                                                  \n",
      " block_6_expand_BN (BatchNormal  (None, 28, 28, 192)  768        ['block_6_expand[0][0]']         \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " block_6_expand_relu (ReLU)     (None, 28, 28, 192)  0           ['block_6_expand_BN[0][0]']      \n",
      "                                                                                                  \n",
      " block_6_pad (ZeroPadding2D)    (None, 29, 29, 192)  0           ['block_6_expand_relu[0][0]']    \n",
      "                                                                                                  \n",
      " block_6_depthwise (DepthwiseCo  (None, 14, 14, 192)  1728       ['block_6_pad[0][0]']            \n",
      " nv2D)                                                                                            \n",
      "                                                                                                  \n",
      " block_6_depthwise_BN (BatchNor  (None, 14, 14, 192)  768        ['block_6_depthwise[0][0]']      \n",
      " malization)                                                                                      \n",
      "                                                                                                  \n",
      " block_6_depthwise_relu (ReLU)  (None, 14, 14, 192)  0           ['block_6_depthwise_BN[0][0]']   \n",
      "                                                                                                  \n",
      " block_6_project (Conv2D)       (None, 14, 14, 64)   12288       ['block_6_depthwise_relu[0][0]'] \n",
      "                                                                                                  \n",
      " block_6_project_BN (BatchNorma  (None, 14, 14, 64)  256         ['block_6_project[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_7_expand (Conv2D)        (None, 14, 14, 384)  24576       ['block_6_project_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_7_expand_BN (BatchNormal  (None, 14, 14, 384)  1536       ['block_7_expand[0][0]']         \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " block_7_expand_relu (ReLU)     (None, 14, 14, 384)  0           ['block_7_expand_BN[0][0]']      \n",
      "                                                                                                  \n",
      " block_7_depthwise (DepthwiseCo  (None, 14, 14, 384)  3456       ['block_7_expand_relu[0][0]']    \n",
      " nv2D)                                                                                            \n",
      "                                                                                                  \n",
      " block_7_depthwise_BN (BatchNor  (None, 14, 14, 384)  1536       ['block_7_depthwise[0][0]']      \n",
      " malization)                                                                                      \n",
      "                                                                                                  \n",
      " block_7_depthwise_relu (ReLU)  (None, 14, 14, 384)  0           ['block_7_depthwise_BN[0][0]']   \n",
      "                                                                                                  \n",
      " block_7_project (Conv2D)       (None, 14, 14, 64)   24576       ['block_7_depthwise_relu[0][0]'] \n",
      "                                                                                                  \n",
      " block_7_project_BN (BatchNorma  (None, 14, 14, 64)  256         ['block_7_project[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_7_add (Add)              (None, 14, 14, 64)   0           ['block_6_project_BN[0][0]',     \n",
      "                                                                  'block_7_project_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_8_expand (Conv2D)        (None, 14, 14, 384)  24576       ['block_7_add[0][0]']            \n",
      "                                                                                                  \n",
      " block_8_expand_BN (BatchNormal  (None, 14, 14, 384)  1536       ['block_8_expand[0][0]']         \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " block_8_expand_relu (ReLU)     (None, 14, 14, 384)  0           ['block_8_expand_BN[0][0]']      \n",
      "                                                                                                  \n",
      " block_8_depthwise (DepthwiseCo  (None, 14, 14, 384)  3456       ['block_8_expand_relu[0][0]']    \n",
      " nv2D)                                                                                            \n",
      "                                                                                                  \n",
      " block_8_depthwise_BN (BatchNor  (None, 14, 14, 384)  1536       ['block_8_depthwise[0][0]']      \n",
      " malization)                                                                                      \n",
      "                                                                                                  \n",
      " block_8_depthwise_relu (ReLU)  (None, 14, 14, 384)  0           ['block_8_depthwise_BN[0][0]']   \n",
      "                                                                                                  \n",
      " block_8_project (Conv2D)       (None, 14, 14, 64)   24576       ['block_8_depthwise_relu[0][0]'] \n",
      "                                                                                                  \n",
      " block_8_project_BN (BatchNorma  (None, 14, 14, 64)  256         ['block_8_project[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_8_add (Add)              (None, 14, 14, 64)   0           ['block_7_add[0][0]',            \n",
      "                                                                  'block_8_project_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_9_expand (Conv2D)        (None, 14, 14, 384)  24576       ['block_8_add[0][0]']            \n",
      "                                                                                                  \n",
      " block_9_expand_BN (BatchNormal  (None, 14, 14, 384)  1536       ['block_9_expand[0][0]']         \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " block_9_expand_relu (ReLU)     (None, 14, 14, 384)  0           ['block_9_expand_BN[0][0]']      \n",
      "                                                                                                  \n",
      " block_9_depthwise (DepthwiseCo  (None, 14, 14, 384)  3456       ['block_9_expand_relu[0][0]']    \n",
      " nv2D)                                                                                            \n",
      "                                                                                                  \n",
      " block_9_depthwise_BN (BatchNor  (None, 14, 14, 384)  1536       ['block_9_depthwise[0][0]']      \n",
      " malization)                                                                                      \n",
      "                                                                                                  \n",
      " block_9_depthwise_relu (ReLU)  (None, 14, 14, 384)  0           ['block_9_depthwise_BN[0][0]']   \n",
      "                                                                                                  \n",
      " block_9_project (Conv2D)       (None, 14, 14, 64)   24576       ['block_9_depthwise_relu[0][0]'] \n",
      "                                                                                                  \n",
      " block_9_project_BN (BatchNorma  (None, 14, 14, 64)  256         ['block_9_project[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_9_add (Add)              (None, 14, 14, 64)   0           ['block_8_add[0][0]',            \n",
      "                                                                  'block_9_project_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_10_expand (Conv2D)       (None, 14, 14, 384)  24576       ['block_9_add[0][0]']            \n",
      "                                                                                                  \n",
      " block_10_expand_BN (BatchNorma  (None, 14, 14, 384)  1536       ['block_10_expand[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_10_expand_relu (ReLU)    (None, 14, 14, 384)  0           ['block_10_expand_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_10_depthwise (DepthwiseC  (None, 14, 14, 384)  3456       ['block_10_expand_relu[0][0]']   \n",
      " onv2D)                                                                                           \n",
      "                                                                                                  \n",
      " block_10_depthwise_BN (BatchNo  (None, 14, 14, 384)  1536       ['block_10_depthwise[0][0]']     \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " block_10_depthwise_relu (ReLU)  (None, 14, 14, 384)  0          ['block_10_depthwise_BN[0][0]']  \n",
      "                                                                                                  \n",
      " block_10_project (Conv2D)      (None, 14, 14, 96)   36864       ['block_10_depthwise_relu[0][0]']\n",
      "                                                                                                  \n",
      " block_10_project_BN (BatchNorm  (None, 14, 14, 96)  384         ['block_10_project[0][0]']       \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " block_11_expand (Conv2D)       (None, 14, 14, 576)  55296       ['block_10_project_BN[0][0]']    \n",
      "                                                                                                  \n",
      " block_11_expand_BN (BatchNorma  (None, 14, 14, 576)  2304       ['block_11_expand[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_11_expand_relu (ReLU)    (None, 14, 14, 576)  0           ['block_11_expand_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_11_depthwise (DepthwiseC  (None, 14, 14, 576)  5184       ['block_11_expand_relu[0][0]']   \n",
      " onv2D)                                                                                           \n",
      "                                                                                                  \n",
      " block_11_depthwise_BN (BatchNo  (None, 14, 14, 576)  2304       ['block_11_depthwise[0][0]']     \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " block_11_depthwise_relu (ReLU)  (None, 14, 14, 576)  0          ['block_11_depthwise_BN[0][0]']  \n",
      "                                                                                                  \n",
      " block_11_project (Conv2D)      (None, 14, 14, 96)   55296       ['block_11_depthwise_relu[0][0]']\n",
      "                                                                                                  \n",
      " block_11_project_BN (BatchNorm  (None, 14, 14, 96)  384         ['block_11_project[0][0]']       \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " block_11_add (Add)             (None, 14, 14, 96)   0           ['block_10_project_BN[0][0]',    \n",
      "                                                                  'block_11_project_BN[0][0]']    \n",
      "                                                                                                  \n",
      " block_12_expand (Conv2D)       (None, 14, 14, 576)  55296       ['block_11_add[0][0]']           \n",
      "                                                                                                  \n",
      " block_12_expand_BN (BatchNorma  (None, 14, 14, 576)  2304       ['block_12_expand[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_12_expand_relu (ReLU)    (None, 14, 14, 576)  0           ['block_12_expand_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_12_depthwise (DepthwiseC  (None, 14, 14, 576)  5184       ['block_12_expand_relu[0][0]']   \n",
      " onv2D)                                                                                           \n",
      "                                                                                                  \n",
      " block_12_depthwise_BN (BatchNo  (None, 14, 14, 576)  2304       ['block_12_depthwise[0][0]']     \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " block_12_depthwise_relu (ReLU)  (None, 14, 14, 576)  0          ['block_12_depthwise_BN[0][0]']  \n",
      "                                                                                                  \n",
      " block_12_project (Conv2D)      (None, 14, 14, 96)   55296       ['block_12_depthwise_relu[0][0]']\n",
      "                                                                                                  \n",
      " block_12_project_BN (BatchNorm  (None, 14, 14, 96)  384         ['block_12_project[0][0]']       \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " block_12_add (Add)             (None, 14, 14, 96)   0           ['block_11_add[0][0]',           \n",
      "                                                                  'block_12_project_BN[0][0]']    \n",
      "                                                                                                  \n",
      " block_13_expand (Conv2D)       (None, 14, 14, 576)  55296       ['block_12_add[0][0]']           \n",
      "                                                                                                  \n",
      " block_13_expand_BN (BatchNorma  (None, 14, 14, 576)  2304       ['block_13_expand[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_13_expand_relu (ReLU)    (None, 14, 14, 576)  0           ['block_13_expand_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_13_pad (ZeroPadding2D)   (None, 15, 15, 576)  0           ['block_13_expand_relu[0][0]']   \n",
      "                                                                                                  \n",
      " block_13_depthwise (DepthwiseC  (None, 7, 7, 576)   5184        ['block_13_pad[0][0]']           \n",
      " onv2D)                                                                                           \n",
      "                                                                                                  \n",
      " block_13_depthwise_BN (BatchNo  (None, 7, 7, 576)   2304        ['block_13_depthwise[0][0]']     \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " block_13_depthwise_relu (ReLU)  (None, 7, 7, 576)   0           ['block_13_depthwise_BN[0][0]']  \n",
      "                                                                                                  \n",
      " block_13_project (Conv2D)      (None, 7, 7, 160)    92160       ['block_13_depthwise_relu[0][0]']\n",
      "                                                                                                  \n",
      " block_13_project_BN (BatchNorm  (None, 7, 7, 160)   640         ['block_13_project[0][0]']       \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " block_14_expand (Conv2D)       (None, 7, 7, 960)    153600      ['block_13_project_BN[0][0]']    \n",
      "                                                                                                  \n",
      " block_14_expand_BN (BatchNorma  (None, 7, 7, 960)   3840        ['block_14_expand[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_14_expand_relu (ReLU)    (None, 7, 7, 960)    0           ['block_14_expand_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_14_depthwise (DepthwiseC  (None, 7, 7, 960)   8640        ['block_14_expand_relu[0][0]']   \n",
      " onv2D)                                                                                           \n",
      "                                                                                                  \n",
      " block_14_depthwise_BN (BatchNo  (None, 7, 7, 960)   3840        ['block_14_depthwise[0][0]']     \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " block_14_depthwise_relu (ReLU)  (None, 7, 7, 960)   0           ['block_14_depthwise_BN[0][0]']  \n",
      "                                                                                                  \n",
      " block_14_project (Conv2D)      (None, 7, 7, 160)    153600      ['block_14_depthwise_relu[0][0]']\n",
      "                                                                                                  \n",
      " block_14_project_BN (BatchNorm  (None, 7, 7, 160)   640         ['block_14_project[0][0]']       \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " block_14_add (Add)             (None, 7, 7, 160)    0           ['block_13_project_BN[0][0]',    \n",
      "                                                                  'block_14_project_BN[0][0]']    \n",
      "                                                                                                  \n",
      " block_15_expand (Conv2D)       (None, 7, 7, 960)    153600      ['block_14_add[0][0]']           \n",
      "                                                                                                  \n",
      " block_15_expand_BN (BatchNorma  (None, 7, 7, 960)   3840        ['block_15_expand[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_15_expand_relu (ReLU)    (None, 7, 7, 960)    0           ['block_15_expand_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_15_depthwise (DepthwiseC  (None, 7, 7, 960)   8640        ['block_15_expand_relu[0][0]']   \n",
      " onv2D)                                                                                           \n",
      "                                                                                                  \n",
      " block_15_depthwise_BN (BatchNo  (None, 7, 7, 960)   3840        ['block_15_depthwise[0][0]']     \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " block_15_depthwise_relu (ReLU)  (None, 7, 7, 960)   0           ['block_15_depthwise_BN[0][0]']  \n",
      "                                                                                                  \n",
      " block_15_project (Conv2D)      (None, 7, 7, 160)    153600      ['block_15_depthwise_relu[0][0]']\n",
      "                                                                                                  \n",
      " block_15_project_BN (BatchNorm  (None, 7, 7, 160)   640         ['block_15_project[0][0]']       \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " block_15_add (Add)             (None, 7, 7, 160)    0           ['block_14_add[0][0]',           \n",
      "                                                                  'block_15_project_BN[0][0]']    \n",
      "                                                                                                  \n",
      " block_16_expand (Conv2D)       (None, 7, 7, 960)    153600      ['block_15_add[0][0]']           \n",
      "                                                                                                  \n",
      " block_16_expand_BN (BatchNorma  (None, 7, 7, 960)   3840        ['block_16_expand[0][0]']        \n",
      " lization)                                                                                        \n",
      "                                                                                                  \n",
      " block_16_expand_relu (ReLU)    (None, 7, 7, 960)    0           ['block_16_expand_BN[0][0]']     \n",
      "                                                                                                  \n",
      " block_16_depthwise (DepthwiseC  (None, 7, 7, 960)   8640        ['block_16_expand_relu[0][0]']   \n",
      " onv2D)                                                                                           \n",
      "                                                                                                  \n",
      " block_16_depthwise_BN (BatchNo  (None, 7, 7, 960)   3840        ['block_16_depthwise[0][0]']     \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " block_16_depthwise_relu (ReLU)  (None, 7, 7, 960)   0           ['block_16_depthwise_BN[0][0]']  \n",
      "                                                                                                  \n",
      " block_16_project (Conv2D)      (None, 7, 7, 320)    307200      ['block_16_depthwise_relu[0][0]']\n",
      "                                                                                                  \n",
      " block_16_project_BN (BatchNorm  (None, 7, 7, 320)   1280        ['block_16_project[0][0]']       \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " Conv_1 (Conv2D)                (None, 7, 7, 1280)   409600      ['block_16_project_BN[0][0]']    \n",
      "                                                                                                  \n",
      " Conv_1_bn (BatchNormalization)  (None, 7, 7, 1280)  5120        ['Conv_1[0][0]']                 \n",
      "                                                                                                  \n",
      " out_relu (ReLU)                (None, 7, 7, 1280)   0           ['Conv_1_bn[0][0]']              \n",
      "                                                                                                  \n",
      " flate1 (Flatten)               (None, 62720)        0           ['out_relu[0][0]']               \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 512)          32113152    ['flate1[0][0]']                 \n",
      "                                                                                                  \n",
      " dropout (Dropout)              (None, 512)          0           ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      " batch_normalization (BatchNorm  (None, 512)         2048        ['dropout[0][0]']                \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 128)          65664       ['batch_normalization[0][0]']    \n",
      "                                                                                                  \n",
      " drop3 (Dropout)                (None, 128)          0           ['dense_1[0][0]']                \n",
      "                                                                                                  \n",
      " dense_2 (Dense)                (None, 64)           8256        ['drop3[0][0]']                  \n",
      "                                                                                                  \n",
      " dropout_1 (Dropout)            (None, 64)           0           ['dense_2[0][0]']                \n",
      "                                                                                                  \n",
      " fc3 (Dense)                    (None, 9)            585         ['dropout_1[0][0]']              \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 34,447,689\n",
      "Trainable params: 32,188,681\n",
      "Non-trainable params: 2,259,008\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#  get the output of the second last dense layer \n",
    "base_model_output = base_model.layers[-1].output\n",
    "x = Flatten(name = \"flate1\")(base_model_output)\n",
    "\n",
    "# add new layers \n",
    "x = Dense(512, activation='relu')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "x = BatchNormalization()(x)\n",
    "# x = Dense(256, activation='relu')(x)\n",
    "# x = Dropout(0.4,name='drop2')(x)\n",
    "x = Dense(128, activation='relu')(x)\n",
    "x = Dropout(0.3,name='drop3')(x)\n",
    "x = Dense(64, activation='relu')(x)\n",
    "x = Dropout(0.2)(x)\n",
    "output = Dense(nb_classes, activation='sigmoid', name='fc3')(x)\n",
    "\n",
    "# define a new model \n",
    "model = Model(base_model.input, output)\n",
    "\n",
    "# Freeze all the base model layers \n",
    "for layer in base_model.layers:\n",
    "    layer.trainable=False\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)\n",
    "#optimizer = Adam(lr=0.001, decay=1e-6, beta_1=0.9, epsilon=None, amsgrad=True)\n",
    "#optimizer = Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)\n",
    "model.compile(loss='binary_crossentropy', optimizer = optimizer, metrics=['accuracy'])\n",
    "#model.compile(optimizers.rmsprop(lr=0.001, decay=1e-6),loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "executionInfo": {
     "elapsed": 377,
     "status": "ok",
     "timestamp": 1649442004463,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "Hj93Dti3pW6M"
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping, ReduceLROnPlateau, CSVLogger\n",
    "\n",
    "checkpoint = ModelCheckpoint(filepath='/content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.{epoch:02d}-{val_loss:.2f}.h5', \n",
    "                             monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', save_freq='epoch')\n",
    "\n",
    "earlyStop = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')\n",
    "learningRateReduce = ReduceLROnPlateau(monitor='val_loss', factor=0.95, patience=2, verbose=1, mode='max', min_lr=0.00001)\n",
    "csvLogger = CSVLogger(filename='/content/drive/MyDrive/Colab Notebooks/Model_4_Files/model_4.csv', separator=',', append=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "executionInfo": {
     "elapsed": 354,
     "status": "ok",
     "timestamp": 1649442007342,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "pHac5z68lskB"
   },
   "outputs": [],
   "source": [
    "my_callbacks = [checkpoint, earlyStop, learningRateReduce, csvLogger]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 7047472,
     "status": "ok",
     "timestamp": 1649449056328,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "JfpmkASPzs3u",
    "outputId": "1ca7ed39-3a5d-42df-b0ef-ddad5f4e0f4f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.1893 - accuracy: 0.7519\n",
      "Epoch 1: val_loss improved from inf to 0.04657, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.01-0.05.h5\n",
      "238/238 [==============================] - 344s 1s/step - loss: 0.1893 - accuracy: 0.7519 - val_loss: 0.0466 - val_accuracy: 0.9341 - lr: 0.0010\n",
      "Epoch 2/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0864 - accuracy: 0.9094\n",
      "Epoch 2: val_loss improved from 0.04657 to 0.03642, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.02-0.04.h5\n",
      "238/238 [==============================] - 339s 1s/step - loss: 0.0864 - accuracy: 0.9094 - val_loss: 0.0364 - val_accuracy: 0.9485 - lr: 0.0010\n",
      "Epoch 3/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0581 - accuracy: 0.9290\n",
      "Epoch 3: val_loss improved from 0.03642 to 0.03012, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.03-0.03.h5\n",
      "\n",
      "Epoch 3: ReduceLROnPlateau reducing learning rate to 0.0009500000451225787.\n",
      "238/238 [==============================] - 350s 1s/step - loss: 0.0581 - accuracy: 0.9290 - val_loss: 0.0301 - val_accuracy: 0.9563 - lr: 0.0010\n",
      "Epoch 4/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0431 - accuracy: 0.9441\n",
      "Epoch 4: val_loss did not improve from 0.03012\n",
      "238/238 [==============================] - 351s 1s/step - loss: 0.0431 - accuracy: 0.9441 - val_loss: 0.0305 - val_accuracy: 0.9498 - lr: 9.5000e-04\n",
      "Epoch 5/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0394 - accuracy: 0.9520\n",
      "Epoch 5: val_loss improved from 0.03012 to 0.02290, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.05-0.02.h5\n",
      "\n",
      "Epoch 5: ReduceLROnPlateau reducing learning rate to 0.0009025000152178108.\n",
      "238/238 [==============================] - 348s 1s/step - loss: 0.0394 - accuracy: 0.9520 - val_loss: 0.0229 - val_accuracy: 0.9609 - lr: 9.5000e-04\n",
      "Epoch 6/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0317 - accuracy: 0.9568\n",
      "Epoch 6: val_loss did not improve from 0.02290\n",
      "238/238 [==============================] - 361s 2s/step - loss: 0.0317 - accuracy: 0.9568 - val_loss: 0.0250 - val_accuracy: 0.9628 - lr: 9.0250e-04\n",
      "Epoch 7/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0346 - accuracy: 0.9604\n",
      "Epoch 7: val_loss did not improve from 0.02290\n",
      "\n",
      "Epoch 7: ReduceLROnPlateau reducing learning rate to 0.0008573750033974647.\n",
      "238/238 [==============================] - 381s 2s/step - loss: 0.0346 - accuracy: 0.9604 - val_loss: 0.0266 - val_accuracy: 0.9609 - lr: 9.0250e-04\n",
      "Epoch 8/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0333 - accuracy: 0.9575\n",
      "Epoch 8: val_loss did not improve from 0.02290\n",
      "238/238 [==============================] - 363s 2s/step - loss: 0.0333 - accuracy: 0.9575 - val_loss: 0.0230 - val_accuracy: 0.9635 - lr: 8.5737e-04\n",
      "Epoch 9/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0275 - accuracy: 0.9632\n",
      "Epoch 9: val_loss improved from 0.02290 to 0.02218, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.09-0.02.h5\n",
      "\n",
      "Epoch 9: ReduceLROnPlateau reducing learning rate to 0.0008145062311086804.\n",
      "238/238 [==============================] - 357s 2s/step - loss: 0.0275 - accuracy: 0.9632 - val_loss: 0.0222 - val_accuracy: 0.9680 - lr: 8.5737e-04\n",
      "Epoch 10/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0262 - accuracy: 0.9654\n",
      "Epoch 10: val_loss improved from 0.02218 to 0.02201, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.10-0.02.h5\n",
      "238/238 [==============================] - 364s 2s/step - loss: 0.0262 - accuracy: 0.9654 - val_loss: 0.0220 - val_accuracy: 0.9700 - lr: 8.1451e-04\n",
      "Epoch 11/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0231 - accuracy: 0.9673\n",
      "Epoch 11: val_loss improved from 0.02201 to 0.02023, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.11-0.02.h5\n",
      "\n",
      "Epoch 11: ReduceLROnPlateau reducing learning rate to 0.0007737808919046074.\n",
      "238/238 [==============================] - 371s 2s/step - loss: 0.0231 - accuracy: 0.9673 - val_loss: 0.0202 - val_accuracy: 0.9746 - lr: 8.1451e-04\n",
      "Epoch 12/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0213 - accuracy: 0.9675\n",
      "Epoch 12: val_loss improved from 0.02023 to 0.01979, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.12-0.02.h5\n",
      "238/238 [==============================] - 367s 2s/step - loss: 0.0213 - accuracy: 0.9675 - val_loss: 0.0198 - val_accuracy: 0.9746 - lr: 7.7378e-04\n",
      "Epoch 13/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0162 - accuracy: 0.9765\n",
      "Epoch 13: val_loss improved from 0.01979 to 0.01970, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.13-0.02.h5\n",
      "\n",
      "Epoch 13: ReduceLROnPlateau reducing learning rate to 0.000735091819660738.\n",
      "238/238 [==============================] - 372s 2s/step - loss: 0.0162 - accuracy: 0.9765 - val_loss: 0.0197 - val_accuracy: 0.9739 - lr: 7.7378e-04\n",
      "Epoch 14/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0154 - accuracy: 0.9786\n",
      "Epoch 14: val_loss improved from 0.01970 to 0.01958, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.14-0.02.h5\n",
      "238/238 [==============================] - 356s 1s/step - loss: 0.0154 - accuracy: 0.9786 - val_loss: 0.0196 - val_accuracy: 0.9739 - lr: 7.3509e-04\n",
      "Epoch 15/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0130 - accuracy: 0.9796\n",
      "Epoch 15: val_loss improved from 0.01958 to 0.01774, saving model to /content/drive/MyDrive/Colab Notebooks/Model_4_Files/model4.15-0.02.h5\n",
      "\n",
      "Epoch 15: ReduceLROnPlateau reducing learning rate to 0.0006983372120885178.\n",
      "238/238 [==============================] - 341s 1s/step - loss: 0.0130 - accuracy: 0.9796 - val_loss: 0.0177 - val_accuracy: 0.9804 - lr: 7.3509e-04\n",
      "Epoch 16/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0184 - accuracy: 0.9771\n",
      "Epoch 16: val_loss did not improve from 0.01774\n",
      "238/238 [==============================] - 331s 1s/step - loss: 0.0184 - accuracy: 0.9771 - val_loss: 0.0226 - val_accuracy: 0.9720 - lr: 6.9834e-04\n",
      "Epoch 17/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0155 - accuracy: 0.9770\n",
      "Epoch 17: val_loss did not improve from 0.01774\n",
      "\n",
      "Epoch 17: ReduceLROnPlateau reducing learning rate to 0.0006634203542489559.\n",
      "238/238 [==============================] - 325s 1s/step - loss: 0.0155 - accuracy: 0.9770 - val_loss: 0.0194 - val_accuracy: 0.9778 - lr: 6.9834e-04\n",
      "Epoch 18/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0185 - accuracy: 0.9767\n",
      "Epoch 18: val_loss did not improve from 0.01774\n",
      "238/238 [==============================] - 328s 1s/step - loss: 0.0185 - accuracy: 0.9767 - val_loss: 0.0194 - val_accuracy: 0.9759 - lr: 6.6342e-04\n",
      "Epoch 19/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0207 - accuracy: 0.9687\n",
      "Epoch 19: val_loss did not improve from 0.01774\n",
      "\n",
      "Epoch 19: ReduceLROnPlateau reducing learning rate to 0.0006302493420662358.\n",
      "238/238 [==============================] - 321s 1s/step - loss: 0.0207 - accuracy: 0.9687 - val_loss: 0.0209 - val_accuracy: 0.9778 - lr: 6.6342e-04\n",
      "Epoch 20/20\n",
      "238/238 [==============================] - ETA: 0s - loss: 0.0133 - accuracy: 0.9807\n",
      "Epoch 20: val_loss did not improve from 0.01774\n",
      "238/238 [==============================] - 320s 1s/step - loss: 0.0133 - accuracy: 0.9807 - val_loss: 0.0183 - val_accuracy: 0.9785 - lr: 6.3025e-04\n"
     ]
    }
   ],
   "source": [
    "probability = model.fit(X_train, Y_train, validation_data = (X_val, Y_val), callbacks= my_callbacks, class_weight = class_weight_dict, epochs = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 23541,
     "status": "ok",
     "timestamp": 1649449079866,
     "user": {
      "displayName": "Anagh Sharma",
      "userId": "05484678154619773185"
     },
     "user_tz": -330
    },
    "id": "qB_FKqnRBwbG",
    "outputId": "4111e593-465b-491a-8f1f-1bbb2736a96b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /content/drive/MyDrive/Colab Notebooks/Model_4_Files/FULL_model_4/assets\n"
     ]
    }
   ],
   "source": [
    "model.save('/content/drive/MyDrive/Colab Notebooks/Model_4_Files/FULL_model_4') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "H_2v8DeXIOXL"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "authorship_tag": "ABX9TyPipZnzWD/LR2D6R7bxbQ+s",
   "collapsed_sections": [],
   "mount_file_id": "1m14sjjeHHulXtK_tEcOl7p7pien-bWyD",
   "name": "Model_4.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}