ProstateCancerPipeline2_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import cv2\n",
    "import time\n",
    "import copy\n",
    "import pickle  # Log dictionary data\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F # stateless functions\n",
    "import torchvision.transforms as T\n",
    "import torchvision.models as models\n",
    "\n",
    "import multiprocessing\n",
    "# We must import this explicitly, it is not imported by the top-level\n",
    "# multiprocessing module.\n",
    "import multiprocessing.pool\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import cohen_kappa_score,confusion_matrix\n",
    "from tqdm.auto import tqdm\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from datetime import datetime\n",
    "from multiprocessing import Manager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CFG:\n",
    "    batch_size = 16\n",
    "    debug = False\n",
    "    device = torch.device('cuda')\n",
    "    dtype = torch.float32\n",
    "    epochs = 30\n",
    "    lr = 1e-4\n",
    "    model_name = 'alexnet'\n",
    "    num_classes = 6\n",
    "    nworkers = 3\n",
    "    nfolds = 4\n",
    "    seed = 524\n",
    "    TRAIN = '../yi_data/panda-16x128x128-tiles-data/train/'\n",
    "    LABELS = '../data/train.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(CFG.LABELS).set_index('image_id')\n",
    "files = sorted(set([p[:32] for p in os.listdir(CFG.TRAIN)]))\n",
    "train = train.loc[files].reset_index()\n",
    "\n",
    "if CFG.debug:\n",
    "    df = train.sample(n=50, random_state=CFG.seed).copy()\n",
    "else:\n",
    "    df = train.copy()\n",
    "\n",
    "# Generate train/validation sets containing the same distribution of isup_grade\n",
    "splits = StratifiedKFold(n_splits=CFG.nfolds, random_state=CFG.seed, shuffle=True)\n",
    "splits = list(splits.split(df,df.isup_grade))\n",
    "# Assign split index to training samples\n",
    "folds_splits = np.zeros(len(df)).astype(np.int)\n",
    "for i in range(CFG.nfolds):\n",
    "    folds_splits[splits[i][1]] = i\n",
    "df['split'] = folds_splits\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/yasufuminakama/panda-se-resnext50-regression-baseline\n",
    "class TrainDataset(Dataset):\n",
    "    \"\"\"Prostate Cancer Biopsy Dataset\"\"\"\n",
    "    \n",
    "    def __init__(self, df, labels, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            csv_file (string): Path to the csv file\n",
    "            root_dir (string): Path to the directory with all images\n",
    "            transform (callable, optional): Optional transform to be applied on an image sample\n",
    "        \"\"\"\n",
    "        # Shuffle dataframes with fixed seed; otherwise, validation set only get cancerous samples\n",
    "        self.df = df\n",
    "        self.labels = labels\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        #worker = torch.utils.data.get_worker_info()\n",
    "        #worker_id = worker.id if worker is not None else -1\n",
    "        #start = time.time()\n",
    "        # https://stackoverflow.com/questions/33369832/read-multiple-images-on-a-folder-in-opencv-python\n",
    "        img_fns = [fn for fn in glob.glob(f\"{CFG.TRAIN}/{self.df['image_id'][idx]}_*.png\")]\n",
    "        imgs = [cv2.imread(fn) for fn in img_fns]\n",
    "        # (D,W,H)\n",
    "        img = cv2.hconcat([cv2.vconcat([imgs[0], imgs[1], imgs[2], imgs[3]]),\n",
    "                           cv2.vconcat([imgs[4], imgs[5], imgs[6], imgs[7]]),\n",
    "                           cv2.vconcat([imgs[8], imgs[9], imgs[10], imgs[11]]),\n",
    "                           cv2.vconcat([imgs[12], imgs[13], imgs[14], imgs[15]])])\n",
    "        \n",
    "        if self.transform:\n",
    "            img = self.transform(img)\n",
    "            \n",
    "        label = torch.tensor(self.labels[idx])\n",
    "        #end = time.time()\n",
    "        return img, label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_transforms(phase):\n",
    "    assert phase in {'train', 'val'}\n",
    "    \n",
    "    if phase == 'train':\n",
    "        return T.Compose([\n",
    "            T.ToTensor(),\n",
    "            T.Normalize(\n",
    "                mean=[0.8776, 0.8186, 0.9090],\n",
    "                std=[0.1659, 0.2507, 0.1357],\n",
    "            ),\n",
    "        ])\n",
    "    else:\n",
    "        return T.Compose([\n",
    "            T.ToTensor(),\n",
    "            T.Normalize(\n",
    "                mean=[0.8776, 0.8186, 0.9090],\n",
    "                std=[0.1659, 0.2507, 0.1357],\n",
    "            ),\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use fold idx as validation set\n",
    "def data_loader(fold_idx):\n",
    "    train_idx = df[df['split'] != fold_idx].index\n",
    "    val_idx = df[df['split'] == fold_idx].index\n",
    "\n",
    "    train_dataset = TrainDataset(df.loc[train_idx].reset_index(drop=True),\n",
    "                                 df.loc[train_idx].reset_index(drop=True)['isup_grade'],\n",
    "                                 transform = get_transforms(phase='train'))\n",
    "    val_dataset = TrainDataset(df.loc[val_idx].reset_index(drop=True),\n",
    "                               df.loc[val_idx].reset_index(drop=True)['isup_grade'],\n",
    "                               transform = get_transforms(phase='train'))\n",
    "    \n",
    "    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, num_workers=CFG.nworkers)\n",
    "    val_loader = DataLoader(val_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.nworkers)\n",
    "    return train_loader, val_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, fold, dataloaders, criterion, optimizer, scheduler, num_epochs=25):\n",
    "    since = time.time()\n",
    "    \n",
    "    # Send the model to GPU/CPU\n",
    "    model = model.to(device=CFG.device)\n",
    "    \n",
    "    train_acc_history = []\n",
    "    val_acc_history = []\n",
    "    loss_history = []\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "    \n",
    "    preds, targets = [], []\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        \n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                model.train()   # Set model to training phase\n",
    "            else:\n",
    "                model.eval()    # Set model to evaluate phase\n",
    "            \n",
    "            avg_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            \n",
    "            print(' ', end='', flush=True)  # To workaround tqdm issue in multiprocess\n",
    "            for inputs, labels in tqdm(dataloaders[phase],\n",
    "                                       desc='[{}] {}/{}({:5s})'.format(fold, epoch+1,num_epochs,phase)):\n",
    "                inputs = inputs.to(device=CFG.device, dtype=CFG.dtype)\n",
    "                labels = labels.to(device=CFG.device, dtype=torch.long)\n",
    "                \n",
    "                # Zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                # Forward, track history if only in training\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    outputs = model(inputs)\n",
    "                    loss = criterion(outputs, labels)\n",
    "                    \n",
    "                    pred = torch.argmax(outputs, 1)\n",
    "\n",
    "                    # Backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                    \n",
    "                # Statistics\n",
    "                avg_loss += loss.item()*(inputs.size(0)/len(dataloaders[phase].dataset))  # len(dataloaders[phase].dataset) not len(dataloaders[phase])\n",
    "                running_corrects += torch.sum(pred == labels)\n",
    "                preds.append(pred)\n",
    "                targets.append(labels)\n",
    "            \n",
    "            # End of epoch\n",
    "            with torch.no_grad():\n",
    "                epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n",
    "\n",
    "                if phase == 'val':\n",
    "                    val_acc_history.append(epoch_acc)\n",
    "                    p = torch.cat(preds).cpu()\n",
    "                    t = torch.cat(targets).cpu()\n",
    "                    kappa = cohen_kappa_score(t, p, weights='quadratic')\n",
    "                    #print(confusion_matrix(t,p)) https://www.dataschool.io/simple-guide-to-confusion-matrix-terminology/\n",
    "                    # deep copy the model\n",
    "                    if epoch_acc > best_acc:\n",
    "                        best_acc = epoch_acc\n",
    "                        best_model_wts = copy.deepcopy(model.state_dict())\n",
    "                else:\n",
    "                    train_acc_history.append(epoch_acc)\n",
    "                    loss_history.append(avg_loss)\n",
    "                print('[{}] {} Loss: {:4f} Acc: {:4f} Kaap: {:4f}'.format(\n",
    "                          fold, phase, avg_loss, epoch_acc, kappa if phase=='val' else 1))\n",
    "\n",
    "                if scheduler is not None and phase == 'train':\n",
    "                    scheduler.step()       \n",
    "        #print()\n",
    "    \n",
    "    time_elapsed = time.time() - since\n",
    "    print('[{}] Training complete in {:.0f}m {:0f}s'.format(fold, time_elapsed//60, time_elapsed%60))\n",
    "    print('[{}] Best val Acc: {:4f}'.format(fold, best_acc))\n",
    "    print()\n",
    "    \n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model, loss_history, train_acc_history, val_acc_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Multiprocessing\n",
    "\"\"\"\n",
    "class NoDaemonProcess(multiprocessing.Process):\n",
    "    # make 'daemon' attribute always return False\n",
    "    def _get_daemon(self):\n",
    "        return False\n",
    "    def _set_daemon(self, value):\n",
    "        pass\n",
    "    daemon = property(_get_daemon, _set_daemon)\n",
    "\n",
    "# We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool\n",
    "# because the latter is only a wrapper function, not a proper class.\n",
    "class MyPool(multiprocessing.pool.Pool):\n",
    "    Process = NoDaemonProcess\n",
    "\n",
    "def train_fn(fold):\n",
    "    model_ft = initialize_model(CFG.model_name, CFG.num_classes, use_pretrained=True)\n",
    "\n",
    "    optimizer = optim.SGD(model_ft.parameters(),\n",
    "                          lr=CFG.lr,\n",
    "                          momentum=.9,\n",
    "                          nesterov=True)\n",
    "\n",
    "    #print(f'### FOLD: {fold} ###', flush=True)\n",
    "    loader_train, loader_val = data_loader(fold)\n",
    "    best_model, loss_history, train_acc_history, val_acc_history = train_model(model_ft, fold, {'train': loader_train, 'val': loader_val}, F.cross_entropy, optimizer, None, CFG.epochs)\n",
    "\n",
    "    return best_model, loss_history, train_acc_history, val_acc_history\n",
    "\n",
    "def progressor(fold):\n",
    "    best_model, loss_history, train_acc_history, val_acc_history = train_fn(fold)\n",
    "    return {f'best_mode_{fold}': best_model,\n",
    "            f'loss_history_{fold}': loss_history,\n",
    "            f'train_acc_history_{fold}': train_acc_history,\n",
    "            f'val_acc_history_{fold}': val_acc_history}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_parameter_requires_grad(model, feature_extracting):\n",
    "    if feature_extracting:\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False\n",
    "            \n",
    "\n",
    "def initialize_model(model_name, num_classes, feature_extract=False, use_pretrained=False):\n",
    "    \"\"\"\n",
    "    Params:\n",
    "        feature_extract\n",
    "            True - fine tunning\n",
    "            False - fix the model\n",
    "    \"\"\"\n",
    "    model_ft = None\n",
    "    \n",
    "    if model_name == 'alexnet':\n",
    "        \"\"\"AlexNet\n",
    "        \"\"\"\n",
    "        model_ft = models.alexnet(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.classifier[6].in_features\n",
    "        model_ft.classifier[6] = nn.Linear(num_ftrs, num_classes)\n",
    "    elif model_name == 'resnet':\n",
    "        \"\"\"Resnet\n",
    "        \"\"\"\n",
    "        model_ft = models.resnet18(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.fc.in_features\n",
    "        model_ft.fc = nn.Linear(num_ftrs, num_classes)\n",
    "    \n",
    "    return model_ft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dict = {'batch_size': CFG.batch_size,\n",
    "                'epochs': CFG.epochs,\n",
    "                'learning_rate': CFG.lr,\n",
    "                'model': CFG.model_name,\n",
    "                'nworkers': CFG.nworkers,\n",
    "                'nfolds': CFG.nfolds,\n",
    "                'random_seed': CFG.seed}\n",
    "\n",
    "nfold = range(CFG.nfolds)\n",
    "result_list = list(MyPool(CFG.nfolds).map(progressor, nfold))\n",
    "\n",
    "# Accumulate result from each process\n",
    "for result in result_list:\n",
    "    log_dict.update(result)\n",
    "\n",
    "# Log results\n",
    "log_file = f'{CFG.model_name}_{datetime.now().strftime(\"%m_%d_%Y_%H_%M\")}.pkl'\n",
    "with open(log_file, 'wb') as pkl_file:\n",
    "    pickle.dump(log_dict, pkl_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "for fold in range(CFG.nfolds):\n",
    "    best_model, loss_history, train_acc_history, val_acc_history = train_fn(fold)\n",
    "    log_dict[f'best_mode_{fold}'] = best_model\n",
    "    log_dict[f'loss_history_{fold}'] = loss_history\n",
    "    log_dict[f'train_acc_history_{fold}'] = train_acc_history\n",
    "    log_dict[f'val_acc_history_{fold}'] = val_acc_history\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\"\"\"\n",
    "Graphs\n",
    "1. loss vs. iterations\n",
    "2. Train/Validation accuracy along epoch\n",
    "\"\"\"\n",
    "plt.subplot(2,1,1)\n",
    "plt.plot(log_dict['loss_history_0'], 'o')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('loss')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(log_dict['train_acc_history_0'], '-o')\n",
    "plt.plot(log_dict['val_acc_history_0'], '-o')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('accuracy')\n",
    "\n",
    "plt.tight_layout(pad=3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read python dict back from the file\n",
    "with open(log_file, 'rb') as pfile:\n",
    "    test_dict = pickle.load(pfile)\n",
    "\n",
    "\"\"\"\n",
    "Graphs\n",
    "1. loss vs. iterations\n",
    "2. Train/Validation accuracy along epoch\n",
    "\"\"\"\n",
    "plt.subplot(2,1,1)\n",
    "plt.plot(test_dict['loss_history_0'], 'o')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('loss')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(test_dict['train_acc_history_0'], '-o')\n",
    "plt.plot(test_dict['val_acc_history_0'], '-o')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('accuracy')\n",
    "\n",
    "plt.tight_layout(pad=3)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}