5 Interpretability.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "58a867a1",
   "metadata": {},
   "source": [
    "torch-cam: https://github.com/frgfm/torch-cam\n",
    "        \n",
    "Copyright (C) 2023, Zhao Bingqiang, All Rights Reserved\n",
    "\n",
    "Email: zbqherb@163.com\n",
    "\n",
    "2023-07-10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5aeaa27b",
   "metadata": {},
   "source": [
    "# Load Libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a344d4d0-da96-44de-ac73-4e5fa539e8c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "from torchvision import transforms\n",
    "\n",
    "# CAM GradCAM GradCAMpp ISCAM LayerCAM SSCAM ScoreCAM SmoothGradCAMpp XGradCAM\n",
    "from torchcam.methods import GradCAM\n",
    "\n",
    "from torchcam.utils import overlay_mask\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import json\n",
    "import shap\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "print('device', device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecf63be8-4dbd-4487-b76b-fa350b6abe9b",
   "metadata": {},
   "source": [
    "## image path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7bb5eed-138c-46b9-96db-18508427146b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "img_path = 'data_split/val/NonCOVID/6_Rahimzadeh_normal3_patient344_SR_4_IM00022.png'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45466f74-d3ae-43e1-a544-12a1abb7280b",
   "metadata": {},
   "source": [
    "# Grad-CAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0faf6280",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Image data preprocessing\n",
    "# COVID-19 CT\n",
    "train_transform = transforms.Compose([transforms.RandomResizedCrop(512),\n",
    "                                      transforms.RandomHorizontalFlip(),\n",
    "                                      transforms.ToTensor(),\n",
    "                                      transforms.Normalize((0.5,), (0.5,))])\n",
    "\n",
    "test_transform = transforms.Compose([transforms.Resize(1000),\n",
    "                                     transforms.CenterCrop(512),\n",
    "                                     transforms.ToTensor(),\n",
    "                                     transforms.Normalize((0.5,), (0.5,))])\n",
    "\n",
    "\n",
    "# Load `trained model`, `idx_to_labels.npy`, `labels_to_idx.npy` and `interpretability analysis method`\n",
    "model = torch.load('checkpoint/best-0.953.pth')\n",
    "model = model.eval().to(device)\n",
    "idx_to_labels = np.load('table/idx_to_labels.npy', allow_pickle=True).item()\n",
    "labels_to_idx = np.load('table/labels_to_idx.npy', allow_pickle=True).item()\n",
    "print(idx_to_labels)\n",
    "cam_extractor = GradCAM(model)\n",
    "\n",
    "# Grad-CAM heat map\n",
    "# Class for visualization, if not specify, it would be the highest confidence class\n",
    "show_class = 'CAP'\n",
    "\n",
    "# fowward prediction\n",
    "img_pil = Image.open(img_path).convert('RGB')\n",
    "input_tensor = test_transform(img_pil).unsqueeze(0).to(device) \n",
    "pred_logits = model(input_tensor)\n",
    "pred_id = torch.topk(pred_logits, 1)[1].detach().cpu().numpy().squeeze().item()\n",
    "\n",
    "if show_class:\n",
    "    class_id = labels_to_idx[show_class]\n",
    "    show_id = class_id\n",
    "else:\n",
    "    show_id = pred_id\n",
    "\n",
    "# Heat map\n",
    "activation_map = cam_extractor(show_id, pred_logits)\n",
    "activation_map = activation_map[0][0].detach().cpu().numpy()\n",
    "result = overlay_mask(img_pil, Image.fromarray(activation_map), alpha=0.4)\n",
    "\n",
    "os.environ[\"KMP_DUPLICATE_LIB_OK\"] = \"TRUE\"\n",
    "plt.imshow(result)\n",
    "plt.axis('off')\n",
    "# plt.title('{}\\nPred:{} Show:{}'.format(img_path, idx_to_labels[pred_id], show_class))\n",
    "plt.savefig('figure/GradCAM_6_Rahimzadeh_normal3_patient344_SR_4_IM00022_CAP.tif', dpi = 300, bbox_inches = 'tight') \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "526e979a-66f6-4217-b119-de3c78bf6d43",
   "metadata": {},
   "source": [
    "# SHAP"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c8d2458-55d1-407f-abc5-1ba3634885e8",
   "metadata": {},
   "source": [
    "shap: https://github.com/slundberg/shap\n",
    "\n",
    "https://proceedings.neurips.cc/paper/2017/file/8a20a8621978632d76c43dfd28b67767-Paper.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f76854a-b438-46ec-bd45-69d379851dce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Image data preprocessing\n",
    "mean = (0.5,)\n",
    "std = (0.5,)\n",
    "\n",
    "def nhwc_to_nchw(x: torch.Tensor) -> torch.Tensor:\n",
    "    if x.dim() == 4:\n",
    "        x = x if x.shape[1] == 3 else x.permute(0, 3, 1, 2)\n",
    "    elif x.dim() == 3:\n",
    "        x = x if x.shape[0] == 3 else x.permute(2, 0, 1)\n",
    "    return x\n",
    "\n",
    "def nchw_to_nhwc(x: torch.Tensor) -> torch.Tensor:\n",
    "    if x.dim() == 4:\n",
    "        x = x if x.shape[3] == 3 else x.permute(0, 2, 3, 1)\n",
    "    elif x.dim() == 3:\n",
    "        x = x if x.shape[2] == 3 else x.permute(1, 2, 0)\n",
    "    return x      \n",
    "    \n",
    "transform= [\n",
    "    transforms.Lambda(nhwc_to_nchw),\n",
    "    transforms.Resize(512),\n",
    "    transforms.Lambda(lambda x: x*(1/255)),\n",
    "    transforms.Normalize(mean=mean, std=std),\n",
    "    transforms.Lambda(nchw_to_nhwc),\n",
    "]\n",
    "\n",
    "inv_transform= [\n",
    "    transforms.Lambda(nhwc_to_nchw),\n",
    "    transforms.Normalize(\n",
    "        mean = (-1 * np.array(mean) / np.array(std)).tolist(),\n",
    "        std = (1 / np.array(std)).tolist()\n",
    "    ),\n",
    "    transforms.Lambda(nchw_to_nhwc),\n",
    "]\n",
    "\n",
    "transform = torchvision.transforms.Compose(transform)\n",
    "inv_transform = torchvision.transforms.Compose(inv_transform)\n",
    "X = torch.Tensor(np.array(img_pil)).unsqueeze(0)\n",
    "\n",
    "# Predict function\n",
    "def predict(img: np.ndarray) -> torch.Tensor:\n",
    "    img = nhwc_to_nchw(torch.Tensor(img)).to(device)\n",
    "    output = model(img)\n",
    "    return output\n",
    "def predict(img):\n",
    "    img = nhwc_to_nchw(torch.Tensor(img)).to(device)\n",
    "    output = model(img)\n",
    "    return output\n",
    "\n",
    "\n",
    "Xtr = transform(X)\n",
    "class_names = list(idx_to_labels.values())\n",
    "out = predict(Xtr[0:1])\n",
    "print(out.shape)\n",
    "classes = torch.argmax(out, axis = 1).detach().cpu().numpy()\n",
    "print(f'Classes: {classes}: {np.array(class_names)[classes]}')\n",
    "\n",
    "\n",
    "# SHAP setting\n",
    "## Create input image\n",
    "input_img = Xtr[0].unsqueeze(0)\n",
    "print(input_img.shape)\n",
    "batch_size = 8\n",
    "## The larger the iterations, the finer the granularity of significance analysis \n",
    "## and the longer the calculation time\n",
    "n_evals = 5000 \n",
    "## Define a mask to mask a local area on the input image\n",
    "masker_blur = shap.maskers.Image(\"blur(64, 64)\", Xtr[0].shape)\n",
    "## Create interpretability algorithm\n",
    "explainer = shap.Explainer(predict, masker_blur, output_names = class_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7443503-82f6-42f7-a7a4-9aaa9fe57e09",
   "metadata": {},
   "source": [
    "## 1. The first *k* prediction categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07a7ea46-b00c-4271-8526-938800a1935b",
   "metadata": {},
   "outputs": [],
   "source": [
    "topk = 3\n",
    "shap_values = explainer(input_img, max_evals=n_evals, batch_size=batch_size, outputs=shap.Explanation.argsort.flip[:topk])\n",
    "print(shap_values.shape)\n",
    "\n",
    "# original image\n",
    "shap_values.data = inv_transform(shap_values.data).cpu().numpy()[0]\n",
    "# shap heat map of each class\n",
    "shap_values.values = [val for val in np.moveaxis(shap_values.values[0],-1, 0)]\n",
    "\n",
    "# Visualization\n",
    "shap.image_plot(shap_values = shap_values.values,\n",
    "                pixel_values = shap_values.data,\n",
    "                labels = shap_values.output_names,\n",
    "                show = False)\n",
    "\n",
    "plt.savefig('figure/shap+_6_Rahimzadeh_normal3_patient344_SR_4_IM00022.tif', dpi = 300, bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58ed7302-47b5-42f8-8286-6ced6a90ad72",
   "metadata": {},
   "source": [
    "## 2. Specify a single prediction category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7f7108d-2f80-4908-bcc3-9dc2f9a22003",
   "metadata": {},
   "outputs": [],
   "source": [
    "shap_values = explainer(input_img, max_evals = n_evals, batch_size = batch_size, outputs = [0])\n",
    "\n",
    "# original image\n",
    "shap_values.data = inv_transform(shap_values.data).cpu().numpy()[0]\n",
    "# shap heat map of each class\n",
    "shap_values.values = [val for val in np.moveaxis(shap_values.values[0],-1, 0)]\n",
    "\n",
    "print(shap_values.data.shape)\n",
    "print(shap_values.values[0].shape)\n",
    "\n",
    "# Visualization\n",
    "shap.image_plot(shap_values = shap_values.values,\n",
    "                         pixel_values = shap_values.data,\n",
    "                         labels = shap_values.output_names,\n",
    "                         show = False)\n",
    "\n",
    "plt.savefig('figure/shap_17_Zhao_PIIS0140673620302117_1.tif', dpi = 300, bbox_inches = 'tight')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ImageClassification",
   "language": "python",
   "name": "imageclassification"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}