histopathology-classification-with-resnet-50-v2 (2).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lung Histopathology Classification using ResNet-50 V2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Setup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from contextlib import ExitStack\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from datetime import datetime\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from keras import (\n",
    "    Sequential,\n",
    "    layers,\n",
    "    optimizers,\n",
    "    regularizers,\n",
    "    callbacks,\n",
    "    metrics,\n",
    "    losses,\n",
    "    activations,\n",
    ")\n",
    "import tensorflow_hub as hub\n",
    "from tensorflow.train import Example, Feature, Features, BytesList, Int64List, FloatList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow_hub in c:\\users\\pc-lenovo\\anaconda3\\lib\\site-packages (0.12.0)\n",
      "Requirement already satisfied: numpy>=1.12.0 in c:\\users\\pc-lenovo\\anaconda3\\lib\\site-packages (from tensorflow_hub) (1.21.5)\n",
      "Requirement already satisfied: protobuf>=3.8.0 in c:\\users\\pc-lenovo\\anaconda3\\lib\\site-packages (from tensorflow_hub) (3.19.6)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install tensorflow_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keras in c:\\users\\pc-lenovo\\anaconda3\\lib\\site-packages (2.11.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.applications.resnet50 import ResNet50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 256\n",
    "AUTOTUNE = tf.data.AUTOTUNE\n",
    "SHUFFLE_BUFFER = 1000\n",
    "DATASET_PATH = tf.io.gfile.join(os.environ[\"HOMEPATH\"],\"Downloads\\\\LUNG_DATASET\\\\lung_colon_image_set\")\n",
    "RAW_DATASET_PATH = tf.io.gfile.join(DATASET_PATH, \"colon_image_sets\")\n",
    "TFRECORDS_PATH = tf.io.gfile.join(DATASET_PATH, \"tfrecord_data\")\n",
    "MODEL_PATH = os.path.join(\"..\", \"..\", \"..\", \"models\", \"chapter_14\", \"lung_colon_histopathology\")\n",
    "SEED = 1992\n",
    "\n",
    "# List of all the diagnostics\n",
    "TYPE_TISSUE = {\n",
    "    0: \"Benign\",\n",
    "    1: \"Adenocarcinoma\",\n",
    "    2: \"Squamous Cell Carcinoma\",\n",
    "}\n",
    "\n",
    "NAME_CLASSES = [\"Benign\", \"Adenocarcinoma\", \"Squamous Cell Carcinoma\"]\n",
    "\n",
    "IMG_SIZE = (768, 768)\n",
    "IMG_CHANNELS = 3\n",
    "NEW_IMG_SIZE = [448, 448, 3]\n",
    "\n",
    "# Model Url\n",
    "MODEL_URL = \"https://tfhub.dev/google/imagenet/resnet_v2_50/feature_vector/5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User's Environment variable:\n",
      "{'ALLUSERSPROFILE': 'C:\\\\ProgramData', 'APPDATA': 'C:\\\\Users\\\\PC-LENOVO\\\\AppData\\\\Roaming', 'CHOCOLATEYINSTALL': 'C:\\\\ProgramData\\\\chocolatey', 'CHOCOLATEYLASTPATHUPDATE': '132895484049443084', 'COMMONPROGRAMFILES': 'C:\\\\Program Files\\\\Common Files', 'COMMONPROGRAMFILES(X86)': 'C:\\\\Program Files (x86)\\\\Common Files', 'COMMONPROGRAMW6432': 'C:\\\\Program Files\\\\Common Files', 'COMPUTERNAME': 'LAPTOP-V0CNSTBE', 'COMSPEC': 'C:\\\\WINDOWS\\\\system32\\\\cmd.exe', 'DRIVERDATA': 'C:\\\\Windows\\\\System32\\\\Drivers\\\\DriverData', 'EFC_8420': '1', 'ERLANG_HOME': 'C:\\\\Program Files\\\\erl9.2', 'HOMEDRIVE': 'C:', 'HOMEPATH': '\\\\Users\\\\PC-LENOVO', 'LOCALAPPDATA': 'C:\\\\Users\\\\PC-LENOVO\\\\AppData\\\\Local', 'LOGONSERVER': '\\\\\\\\LAPTOP-V0CNSTBE', 'NIEXTCCOMPILERSUPP': 'C:\\\\Program Files (x86)\\\\National Instruments\\\\Shared\\\\ExternalCompilerSupport\\\\C\\\\', 'NUMBER_OF_PROCESSORS': '8', 'ONEDRIVE': 'C:\\\\Users\\\\PC-LENOVO\\\\OneDrive', 'ONEDRIVECONSUMER': 'C:\\\\Users\\\\PC-LENOVO\\\\OneDrive', 'OS': 'Windows_NT', 'PATH': 'C:\\\\Users\\\\PC-LENOVO\\\\anaconda3;C:\\\\Users\\\\PC-LENOVO\\\\anaconda3\\\\Library\\\\mingw-w64\\\\bin;C:\\\\Users\\\\PC-LENOVO\\\\anaconda3\\\\Library\\\\usr\\\\bin;C:\\\\Users\\\\PC-LENOVO\\\\anaconda3\\\\Library\\\\bin;C:\\\\Users\\\\PC-LENOVO\\\\anaconda3\\\\Scripts;C:\\\\Program Files\\\\Common Files\\\\Oracle\\\\Java\\\\javapath;C:\\\\WINDOWS\\\\system32;C:\\\\WINDOWS;C:\\\\WINDOWS\\\\System32\\\\Wbem;C:\\\\WINDOWS\\\\System32\\\\WindowsPowerShell\\\\v1.0\\\\;C:\\\\WINDOWS\\\\System32\\\\OpenSSH\\\\;C:\\\\Program Files\\\\Git\\\\cmd;C:\\\\ProgramData\\\\chocolatey\\\\bin;C:\\\\Program Files\\\\nodejs\\\\;C:\\\\Program Files (x86)\\\\IVI Foundation\\\\VISA\\\\WinNT\\\\Bin\\\\;C:\\\\Program Files\\\\IVI Foundation\\\\VISA\\\\Win64\\\\Bin\\\\;C:\\\\Program Files (x86)\\\\IVI Foundation\\\\VISA\\\\WinNT\\\\Bin;C:\\\\Users\\\\PC-LENOVO\\\\AppData\\\\Local\\\\Microsoft\\\\WindowsApps;C:\\\\Microsoft VS Code\\\\bin;C:\\\\Users\\\\PC-LENOVO\\\\AppData\\\\Roaming\\\\npm', 'PATHEXT': '.COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC', 'PROCESSOR_ARCHITECTURE': 'AMD64', 'PROCESSOR_IDENTIFIER': 'Intel64 Family 6 Model 140 Stepping 1, GenuineIntel', 'PROCESSOR_LEVEL': '6', 'PROCESSOR_REVISION': '8c01', 'PROGRAMDATA': 'C:\\\\ProgramData', 'PROGRAMFILES': 'C:\\\\Program Files', 'PROGRAMFILES(X86)': 'C:\\\\Program Files (x86)', 'PROGRAMW6432': 'C:\\\\Program Files', 'PSMODULEPATH': 'C:\\\\Program Files\\\\WindowsPowerShell\\\\Modules;C:\\\\WINDOWS\\\\system32\\\\WindowsPowerShell\\\\v1.0\\\\Modules', 'PUBLIC': 'C:\\\\Users\\\\Public', 'RABBITMQ_NODENAME': 'rabbit@localhost', 'SESSIONNAME': 'Console', 'SYSTEMDRIVE': 'C:', 'SYSTEMROOT': 'C:\\\\WINDOWS', 'TEMP': 'C:\\\\Users\\\\PC-LEN~1\\\\AppData\\\\Local\\\\Temp', 'TMP': 'C:\\\\Users\\\\PC-LEN~1\\\\AppData\\\\Local\\\\Temp', 'USERDOMAIN': 'LAPTOP-V0CNSTBE', 'USERDOMAIN_ROAMINGPROFILE': 'LAPTOP-V0CNSTBE', 'USERNAME': 'PC-LENOVO', 'USERPROFILE': 'C:\\\\Users\\\\PC-LENOVO', 'VBOX_MSI_INSTALL_PATH': 'C:\\\\Program Files\\\\Oracle\\\\VirtualBox\\\\', 'VXIPNPPATH': 'C:\\\\Program Files (x86)\\\\IVI Foundation\\\\VISA\\\\', 'VXIPNPPATH64': 'C:\\\\Program Files\\\\IVI Foundation\\\\VISA\\\\', 'WINDIR': 'C:\\\\WINDOWS', 'ZES_ENABLE_SYSMAN': '1', 'CONDA_PREFIX': 'C:\\\\Users\\\\PC-LENOVO\\\\anaconda3', 'PYDEVD_USE_FRAME_EVAL': 'NO', 'JPY_INTERRUPT_EVENT': '2524', 'IPY_INTERRUPT_EVENT': '2524', 'JPY_PARENT_PID': '2544', 'TERM': 'xterm-color', 'CLICOLOR': '1', 'PAGER': 'cat', 'GIT_PAGER': 'cat', 'MPLBACKEND': 'module://matplotlib_inline.backend_inline', 'TPU_ML_PLATFORM': 'Tensorflow', 'TF2_BEHAVIOR': '1'}\n"
     ]
    }
   ],
   "source": [
    "env_var = os.environ\n",
    "  \n",
    "# Print the list of user's\n",
    "# environment variables\n",
    "print(\"User's Environment variable:\")\n",
    "print(dict(env_var))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exponential_decay_with_warmup(\n",
    "    lr_start=1e-4,\n",
    "    lr_max=1e-3,\n",
    "    lr_min=1e-5,\n",
    "    lr_rampup_epochs=4,\n",
    "    lr_sustain_epochs=1,\n",
    "    lr_exp_decay=0.8,\n",
    "):\n",
    "\n",
    "    def exponential_decay_fn(epoch):\n",
    "        if epoch < lr_rampup_epochs:\n",
    "            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n",
    "        elif epoch < (lr_rampup_epochs + lr_sustain_epochs):\n",
    "            lr = lr_max\n",
    "        else:\n",
    "            lr = (lr_max - lr_min) * lr_exp_decay ** (\n",
    "                epoch - lr_rampup_epochs - lr_sustain_epochs\n",
    "            ) + lr_min\n",
    "\n",
    "        return lr\n",
    "\n",
    "    return exponential_decay_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "def balanced_split(dataset, percentages=[0.80, 0.10, 0.10], verbose=False):\n",
    "    \n",
    "    # Obtain the different classes in the datasets and sort the list\n",
    "    list_classes = dataset.map(lambda x, y: y, num_parallel_calls=_AUTOTUNE).unique()\n",
    "    list_classes = [class_.numpy() for class_ in list_classes]\n",
    "\n",
    "    # Initialize the sets to False. This is just to avoid creating\n",
    "    # a dataset without knowing the dimensions. This will not affect\n",
    "    # the final dataset since the variable will be totally overwritten\n",
    "    train_set = False\n",
    "    valid_set = False\n",
    "    test_set = False\n",
    "    \n",
    "    # Keep track of the total samples per set\n",
    "    samples_train = 0\n",
    "    samples_valid = 0\n",
    "    samples_test = 0\n",
    "\n",
    "    for class_ in list_classes:\n",
    "        # Get the samples that match every class and samples per set\n",
    "        tmp_dataset = dataset.filter(lambda x, y: y == class_)\n",
    "        n_samples = len(list(tmp_dataset.as_numpy_iterator()))\n",
    "\n",
    "        n_valid = int(percentages[1] * n_samples)\n",
    "        n_test = int(percentages[1] * n_samples)\n",
    "        n_train = n_samples - n_valid - n_test\n",
    "\n",
    "        samples_train += n_train\n",
    "        samples_valid += n_valid\n",
    "        samples_test += n_test\n",
    "\n",
    "        # Separate the sets and concatenate to the other classes sets\n",
    "        tmp_train_set = tmp_dataset.take(n_train)\n",
    "        tmp_valid_set = tmp_dataset.skip(n_train).take(n_valid)\n",
    "        tmp_test_set = tmp_dataset.skip(n_train).skip(n_valid)\n",
    "\n",
    "        train_set = (\n",
    "            tmp_train_set\n",
    "            if train_set == False\n",
    "            else train_set.concatenate(tmp_train_set)\n",
    "        )\n",
    "        valid_set = (\n",
    "            tmp_valid_set\n",
    "            if valid_set == False\n",
    "            else valid_set.concatenate(tmp_valid_set)\n",
    "        )\n",
    "        test_set = (\n",
    "            tmp_test_set if test_set == False else test_set.concatenate(tmp_test_set)\n",
    "        )\n",
    "\n",
    "        if verbose == True:\n",
    "            print(f\"\\tSplit for class {class_} is [{n_train}, {n_valid}, {n_test}]\")\n",
    "\n",
    "    if verbose == True:\n",
    "        print(\"\\nThe Split has been completed. The final split is the following: \")\n",
    "        print(\n",
    "            f\"\\tTraining Set: {samples_train}\\n\\tValidation Set:{samples_valid}\\n\\tTesting Set:{samples_valid}\"\n",
    "        )\n",
    "    return train_set, valid_set, test_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_logdir(date_type=\"date\", path_folder=None, id_value=None):\n",
    "   \n",
    "    log_dir = datetime.now().strftime(\"%Y%m%d_%H%M%S\") if date_type == \"datetype\" else datetime.now().strftime(\"%Y%m%d\")\n",
    "    log_dir = f\"run_{log_dir}\" if not path_folder else os.path.join(path_folder, f\"run_{log_dir}\")\n",
    "    \n",
    "    log_id = \"\"\n",
    "    if date_type == \"date\":\n",
    "        log_id = datetime.now().strftime(\"%Y%m%d\")\n",
    "    elif date_type == \"datetime\":\n",
    "        log_id = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
    "    elif date_type == \"id\":\n",
    "        log_id = id_value\n",
    "    \n",
    "    log_folder = f\"run_{log_id}\"\n",
    "    \n",
    "    if path_folder:\n",
    "        log_dir = os.path.join(path_folder, log_folder)\n",
    "    else:\n",
    "        log_dir = log_folder\n",
    "\n",
    "    return log_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Data Pipeline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The images are provided in two separate folders as follows:\n",
    "<pre>\n",
    "lung_images\n",
    "    ├── lung_aca\n",
    "    ├── lung_n\n",
    "    └── lung_scc</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_folders_images(filepath):\n",
    "    \n",
    "    name_folders = tf.io.gfile.listdir(filepath)\n",
    "    path_folders = [tf.io.gfile.join(filepath, folder) for folder in name_folders]\n",
    "\n",
    "    list_folders = {folder: path for folder, path in zip(name_folders, path_folders)}\n",
    "    return list_folders\n",
    "\n",
    "\n",
    "def create_example(image, tissue_label):\n",
    "   \n",
    "    image_data = tf.io.serialize_tensor(image)\n",
    "    feature = {\n",
    "        \"image\": Feature(bytes_list=BytesList(value=[image_data.numpy()])),\n",
    "        \"tissue_label\": Feature(int64_list=Int64List(value=[tissue_label.numpy()])),\n",
    "    }\n",
    "\n",
    "    return Example(features=Features(feature=feature))\n",
    "\n",
    "\n",
    "def save_protobufs(dataset, type_set=\"train\", n_shards=10):\n",
    "   \n",
    "    set_folder = tf.io.gfile.join(TFRECORDS_PATH, type_set)\n",
    "\n",
    "    tf.io.gfile.makedirs(set_folder)\n",
    "    file_paths = [tf.io.gfile.join(set_folder, filepath) for filepath in files]\n",
    "\n",
    "    if type_set == \"train\":\n",
    "        dataset.shuffle(SHUFFLE_BUFFER)\n",
    "    files = [\n",
    "        f\"{type_set}.tfrecord-{shard.numpy() + 1:02d}-of-{n_shards:02d}\"\n",
    "        for shard in tf.range(n_shards)\n",
    "    ]\n",
    "\n",
    "    with ExitStack() as stack:\n",
    "        writers = [\n",
    "            stack.enter_context(tf.io.TFRecordWriter(file)) for file in file_paths\n",
    "        ]\n",
    "        for index, (image, organ_label, tissue_label) in dataset.enumerate():\n",
    "            shard = index % n_shards\n",
    "            example = create_example(image, organ_label, tissue_label)\n",
    "            writers[shard].write(example.SerializeToString())\n",
    "    return file_paths\n",
    "\n",
    "\n",
    "def get_folders_tfrecords(type_set=\"train\"):\n",
    "    \n",
    "    folder = tf.io.gfile.join(TFRECORDS_PATH, type_set)\n",
    "    files = tf.io.gfile.listdir(folder)\n",
    "    list_files = [tf.io.gfile.join(folder, filepath) for filepath in files]\n",
    "    return list_files\n",
    "\n",
    "\n",
    "def save_images_protobufs():\n",
    "   \n",
    "    # Verify if the folder for the TFRecords exist to process the data\n",
    "    if not tf.io.gfile.exists(TFRECORDS_PATH):\n",
    "\n",
    "        tf.io.gfile.makedirs(TFRECORDS_PATH)\n",
    "\n",
    "        list_folders = get_folders_images(RAW_DATASET_PATH)\n",
    "\n",
    "        # Create the dataset per folder, split the dataset into train, valid and test sets\n",
    "#         colon_dataset = tf.keras.preprocessing.image_dataset_from_directory(\n",
    "#             list_folders[\"colon_images\"], batch_size=None, image_size=IMG_SIZE\n",
    "#         )\n",
    "#         train_colon, valid_colon, test_colon = balanced_split(\n",
    "#             colon_dataset\n",
    "#         )\n",
    "\n",
    "        lung_dataset = tf.keras.preprocessing.image_dataset_from_directory(\n",
    "            list_folders[\"lung_images_sets\"], batch_size=None, image_size=IMG_SIZE\n",
    "        )\n",
    "        train_lung, valid_lung, test_lung = balanced_split(lung_dataset)\n",
    "\n",
    "        # Combines the datasets into a single one\n",
    "        train_set = train_lung\n",
    "        valid_set = valid_lung\n",
    "        test_set = test_lung\n",
    "\n",
    "        train_paths = save_protobufs(train_set, \"train\")\n",
    "        valid_paths = save_protobufs(valid_set, \"valid\")\n",
    "        test_paths = save_protobufs(test_set, \"test\")\n",
    "    else:\n",
    "         #If the folders exist, only return the list of filepaths\n",
    "         train_paths = get_folders_tfrecords(\"train\")\n",
    "         valid_paths = get_folders_tfrecords(\"valid\")\n",
    "         test_paths = get_folders_tfrecords(\"test\")\n",
    "\n",
    "    return train_paths, valid_paths, test_paths\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'lung_images_sets'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_18356\\1415073758.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtrain_files\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_files\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_files\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msave_images_protobufs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_18356\\854540791.py\u001b[0m in \u001b[0;36msave_images_protobufs\u001b[1;34m()\u001b[0m\n\u001b[0;32m     70\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m         lung_dataset = tf.keras.preprocessing.image_dataset_from_directory(\n\u001b[1;32m---> 72\u001b[1;33m             \u001b[0mlist_folders\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"lung_images_sets\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mimage_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mIMG_SIZE\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     73\u001b[0m         )\n\u001b[0;32m     74\u001b[0m         \u001b[0mtrain_lung\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_lung\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_lung\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbalanced_split\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlung_dataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'lung_images_sets'"
     ]
    }
   ],
   "source": [
    "train_files, valid_files, test_files = save_images_protobufs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_record(tfrecord):\n",
    "   \n",
    "    feature_descriptions = {\n",
    "        \"image\": tf.io.FixedLenFeature([], tf.string, default_value=\"\"),\n",
    "        \"tissue_label\": tf.io.FixedLenFeature([], tf.int64, default_value=-1),\n",
    "    }\n",
    "\n",
    "    example = tf.io.parse_single_example(tfrecord, feature_descriptions)\n",
    "    image = tf.io.parse_tensor(example[\"image\"], out_type=tf.float32)\n",
    "    image = tf.reshape(image, shape=[IMG_SIZE[0], IMG_SIZE[1], IMG_CHANNELS])\n",
    "    image = tf.image.resize(image, size=[NEW_IMG_SIZE[0], NEW_IMG_SIZE[1]])\n",
    "    return image, example[\"tissue_label\"]\n",
    "\n",
    "\n",
    "def get_dataset(file_paths, cache=False, shuffle_buffer=None):\n",
    "    \n",
    "    dataset = tf.data.TFRecordDataset(file_paths, num_parallel_reads=AUTOTUNE)\n",
    "\n",
    "    if cache:\n",
    "        dataset = dataset.cache()\n",
    "    if shuffle_buffer:\n",
    "        dataset = dataset.shuffle(shuffle_buffer)\n",
    "\n",
    "    dataset = (\n",
    "        dataset.map(get_record, num_parallel_calls=AUTOTUNE)\n",
    "        .batch(BATCH_SIZE)\n",
    "        .prefetch(AUTOTUNE)\n",
    "    )\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_files' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_18356\\4247284404.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtrain_set\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_dataset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_files\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshuffle_buffer\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mSHUFFLE_BUFFER\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mvalid_set\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_dataset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalid_files\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mtest_set\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_dataset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtest_files\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'train_files' is not defined"
     ]
    }
   ],
   "source": [
    "train_set = get_dataset(train_files, shuffle_buffer=SHUFFLE_BUFFER)\n",
    "valid_set = get_dataset(valid_files)\n",
    "test_set = get_dataset(test_files)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Training\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clear the TensorFlow session\n",
    "tf.keras.backend.clear_session()\n",
    "tf.random.set_seed(SEED)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model is trained taking into consideration the following:\n",
    "\n",
    "- The images are normalized being divided by `255.0`\n",
    "- The first layer of the model is a max pooling layer to reduce the spatial dimensionality, and match the suggested size of the images.\n",
    "- The ResNet model is not set as trainable to reduce the training time. \n",
    "- In the article mentioned above, they used the learning rate with decay. In this case, I selected learning rate with warmup and exponential decay. \n",
    "- The optimizer selected is Adam.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(\n",
    "    train_set,\n",
    "    valid_set,\n",
    "    learning_rate,\n",
    "    epochs=100,\n",
    "    trainable=False,\n",
    "):\n",
    "    # Normalization of the data\n",
    "    def normalize(image, label):\n",
    "        norm_image = image / 255.0\n",
    "        return (norm_image, label)\n",
    "\n",
    "    train_set_normalized = train_set.map(normalize, num_parallel_calls=AUTOTUNE)\n",
    "    valid_set_normalized = valid_set.map(normalize, num_parallel_calls=AUTOTUNE)\n",
    "\n",
    "    # Load the ResNet model\n",
    "    base_model = hub.KerasLayer(\n",
    "        MODEL_URL,\n",
    "        trainable=trainable,\n",
    "    )\n",
    "\n",
    "    model = Sequential(\n",
    "        [layers.MaxPool2D(), base_model, layers.Dense(3, activation=\"softmax\")]\n",
    "    )\n",
    "\n",
    "    # Compilation of the model\n",
    "    optimizer_ = optimizers.Adam(learning_rate=learning_rate)\n",
    "    metrics_ = [\n",
    "        \"accuracy\",\n",
    "    ]\n",
    "    \n",
    "    loss_ = \"sparse_categorical_crossentropy\"\n",
    "\n",
    "    model.compile(optimizer=optimizer_, metrics=metrics_, loss=loss_)\n",
    "\n",
    "    # Callbacks\n",
    "    exponential_decay_fn = ml_learning_rate.exponential_decay_with_warmup(\n",
    "        lr_start=learning_rate,\n",
    "        lr_max=learning_rate * 10,\n",
    "        lr_min=learning_rate / 10,\n",
    "    )\n",
    "    lr_scheduler_cb = callbacks.LearningRateScheduler(exponential_decay_fn)\n",
    "\n",
    "    folder_logs = tf.io.gfile.join(\n",
    "        \"..\", \"..\", \"..\", \"reports\", \"logs\", \"chapter_14\", \"lung_colon_histopathology\"\n",
    "    )\n",
    "    logdir = get_logdir(path_folder=folder_logs)\n",
    "    tensorboard_cb = callbacks.TensorBoard(log_dir=logdir, histogram_freq=1)\n",
    "\n",
    "    model_path = tf.io.gfile.join(MODEL_PATH)\n",
    "    model_checkpoint_cb = callbacks.ModelCheckpoint(\n",
    "        filepath=model_path, save_best_only=True\n",
    "    )\n",
    "\n",
    "    early_stopping_cb = callbacks.EarlyStopping(patience=5)\n",
    "\n",
    "    callbacks_list = [\n",
    "        lr_scheduler_cb,\n",
    "        tensorboard_cb,\n",
    "        model_checkpoint_cb,\n",
    "        early_stopping_cb,\n",
    "    ]\n",
    "\n",
    "    # Train the model\n",
    "    model.fit(\n",
    "        train_set_normalized,\n",
    "        validation_data=valid_set_normalized,\n",
    "        epochs=epochs,\n",
    "        callbacks=callbacks_list,\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train_model(\n",
    "    train_set,\n",
    "    valid_set,\n",
    "    test_set,\n",
    "    epochs=100,\n",
    "    learning_rate=5e-4,\n",
    "    trainable=False,\n",
    "    lr_function=None\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "<h4 align=\"center\">Epochs vs Accuracy</h4>\n",
    "\n",
    "<img src=\"https://storage.googleapis.com/mmenendezg-ml-bucket/models/lung_colon_histopathological/training_images/lung_colon_epoch_accuracy.png\" alt=\"Epochs Vs Accuracy\" style=\"width: 70%; margin-left: 15%; margin-right: 15%\"/>\n",
    "\n",
    "\n",
    "<h4 align=\"center\">Epochs vs Loss</h4>\n",
    "\n",
    "<img src=\"https://storage.googleapis.com/mmenendezg-ml-bucket/models/lung_colon_histopathological/training_images/lung_colon_epoch_loss.png\" alt=\"Epochs Vs Accuracy\" style=\"width: 70%; margin-left: 15%; margin-right: 15%\"/>\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(image, label):\n",
    "    \"\"\"\n",
    "    Normalize image by dividing each pixel by 255.0\n",
    "    \n",
    "    Args:\n",
    "        image (ndarray): Image data in numpy array format\n",
    "        label (int): Image label\n",
    "    \n",
    "    Returns:\n",
    "        tuple: Normaliz\n",
    "        ed image in numpy array format and label\n",
    "    \"\"\"\n",
    "    norm_image = image / 255.0\n",
    "    return (norm_image, label)\n",
    "\n",
    "\n",
    "def plot_confusion_matrix(confusion_matrix, precision, recall, labels=NAME_CLASSES):\n",
    "    \"\"\"\n",
    "    Plot confusion matrix using seaborn heatmap.\n",
    "    \n",
    "    Args:\n",
    "        confusion_matrix (ndarray): Confusion matrix in numpy array format\n",
    "        precision (float): Precision score\n",
    "        recall (float): Recall score\n",
    "        labels (list, optional): List of class labels, defaults to NAME_CLASSES\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(5, 5))\n",
    "    sns.heatmap(\n",
    "        confusion_matrix,\n",
    "        annot=True,\n",
    "        cmap=\"flare\",\n",
    "        cbar=False,\n",
    "        fmt=\".2f\",\n",
    "        xticklabels=labels,\n",
    "        yticklabels=labels,\n",
    "    )\n",
    "    title_plot = f\"Confusion Matrix\\nPrecision: {precision * 100:.2f}%\\nRecall: {recall * 100:.2f}%\"\n",
    "    plt.xticks(rotation=60)\n",
    "    plt.yticks(rotation=60)\n",
    "    plt.ylabel(\"Labels\", weight=\"bold\")\n",
    "    plt.xlabel(\"Predictions\", weight=\"bold\")\n",
    "    plt.title(title_plot)\n",
    "\n",
    "\n",
    "def create_confusion_matrix(model_path, dataset):\n",
    "    \"\"\"\n",
    "    Create a confusion matrix for a model given a dataset.\n",
    "    Args:\n",
    "        model_path (str): File path of the model.\n",
    "        dataset (tf.data.Dataset): A dataset containing the images and labels\n",
    "    \"\"\"\n",
    "    # Load the model\n",
    "    model = tf.keras.models.load_model(model_path)\n",
    "    # Preprocess the images to predict\n",
    "    dataset_normalized = dataset.map(normalize, num_parallel_calls=AUTOTUNE)\n",
    "    # Separate the images and the labels\n",
    "    images = dataset_normalized.map(\n",
    "        lambda image, label: image, num_parallel_calls=AUTOTUNE\n",
    "    )\n",
    "    labels = dataset_normalized.map(\n",
    "        lambda image, label: label, num_parallel_calls=AUTOTUNE\n",
    "    )\n",
    "    labels = [label.numpy() for label in labels.unbatch()]\n",
    "    # Create the confusion matrix with the predicted labels\n",
    "    predictions = model.predict(images, verbose=0)\n",
    "    predicted_classes = tf.argmax(predictions, axis=1).numpy()\n",
    "    # Calculate the precision and recall of the model\n",
    "    precision_metric = metrics.Precision()\n",
    "    precision_metric.update_state(labels, predicted_classes)\n",
    "    precision = precision_metric.result().numpy()\n",
    "    \n",
    "    recall_metric = metrics.Recall()\n",
    "    recall_metric.update_state(labels, predicted_classes)\n",
    "    recall = recall_metric.result().numpy()\n",
    "    \n",
    "    # Create the confusion matrix\n",
    "    conf_matrix = tf.math.confusion_matrix(labels, predicted_classes)\n",
    "    conf_matrix = conf_matrix.numpy() / conf_matrix.numpy().sum(axis=1)[:, np.newaxis]\n",
    "    \n",
    "    # Plot the results\n",
    "    plot_confusion_matrix(conf_matrix, precision, recall)\n",
    "\n",
    "\n",
    "def evaluate_model(model_path, test_set):\n",
    "    \"\"\"\n",
    "    Evaluate a model given a test set.\n",
    "    \n",
    "    Args:\n",
    "        model_path (str): File path of the model.\n",
    "        test_set (tf.data.Dataset): A dataset containing the test images and labels\n",
    "    \"\"\"\n",
    "    # Load the model\n",
    "    model = tf.keras.models.load_model(model_path)\n",
    "    evaluations = model.evaluate(test_set, verbose=0)\n",
    "    print(\n",
    "        f\"The evaluation of the model is the following:\\n\\tLoss: {evaluations[0]:.2f}\\n\\tAccuracy: {evaluations[1] * 100:.2f}%\"\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set_normalized = test_set.map(normalize, num_parallel_calls=AUTOTUNE)\n",
    "evaluate_model(MODEL_PATH, test_set_normalized)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_confusion_matrix(MODEL_PATH, test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Making Predictions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_image(image):\n",
    "    # Normalize the image\n",
    "    norm_image = image / 255.0\n",
    "\n",
    "    # Add dimension at the beginning\n",
    "    norm_image = tf.expand_dims(norm_image, axis=0)\n",
    "    return norm_image\n",
    "\n",
    "\n",
    "def make_prediction(dataset, model_path=MODEL_PATH, n_cols=4):\n",
    "    # Load the model\n",
    "    model = tf.keras.models.load_model(model_path)\n",
    "    # Preprocess the images to predict\n",
    "    dataset_normalized = dataset.map(preprocess_image, num_parallel_calls=AUTOTUNE)\n",
    "    predictions = model.predict(dataset_normalized, verbose=0)\n",
    "    predicted_classes = tf.argmax(predictions, axis=1).numpy()\n",
    "\n",
    "    # Determine the number of images to show\n",
    "    n_images = len(predicted_classes)\n",
    "    n_rows = (\n",
    "        (n_images // n_cols) if (n_images % n_cols) == 0 else (n_images // n_cols) + 1\n",
    "    )\n",
    "    idx = 1\n",
    "    plt.figure(figsize=(5 * n_cols, 5 * n_rows))\n",
    "    for image, label in zip(dataset_normalized, predicted_classes):\n",
    "        prob_prediction = predictions[idx - 1][label] * 100\n",
    "        plt.subplot(n_rows, n_cols, idx)\n",
    "        plt.imshow(image[0].numpy())\n",
    "        plt.title(f\"Type Tissue: {TYPE_TISSUE[label]}\\nProbability: {prob_prediction:.2f}%\")\n",
    "        plt.axis(\"off\")\n",
    "        idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = (\n",
    "    test_set.map(lambda image, label: image, num_parallel_calls=AUTOTUNE)\n",
    "    .unbatch()\n",
    "    .shuffle(2500)\n",
    "    .take(15)\n",
    ")\n",
    "make_prediction(preds)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "b20fb8559386147dc77cba9f7518a8c175fc982b8c10bebe842520e16b3ac5d9"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}