3 Test_dataset prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "68aa2c39-1a4b-4d3c-ab35-3d44896f2f67",
   "metadata": {},
   "source": [
    "# Making predictions on the `test_dataset`\n",
    "\n",
    "This notebook demonstrates how a fine-tuned CNN can be evaluated using randomly selected validation splits. Here, we'll demonstrate using a trained and fine-tuned CNN to make predictions on an independent test set.\n",
    "\n",
    "Copyright (C) 2023, Zhao Bingqiang, All Rights Reserved\n",
    "\n",
    "Email: zbqherb@163.com\n",
    "\n",
    "2023-07-02"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0423c89",
   "metadata": {},
   "source": [
    "# Import Libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1119b007",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import math\n",
    "import cv2\n",
    "import itertools\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import colors as mcolors\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from tqdm import tqdm\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets\n",
    "from torchvision import transforms\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
    "\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "print('device', device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bf6ade0",
   "metadata": {},
   "source": [
    "## `test_dataset`  prediction results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9e2a237",
   "metadata": {},
   "source": [
    "### `test_dataset` image preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08871cf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# COVID-19 CT\n",
    "train_transform = transforms.Compose([transforms.RandomResizedCrop(512),\n",
    "                                      transforms.RandomHorizontalFlip(),\n",
    "                                      transforms.ToTensor(),\n",
    "                                      transforms.Normalize((0.5,), (0.5,))])\n",
    "\n",
    "test_transform = transforms.Compose([transforms.Resize(1000),\n",
    "                                     transforms.CenterCrop(512),\n",
    "                                     transforms.ToTensor(),\n",
    "                                     transforms.Normalize((0.5,), (0.5,))])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6c61a0a",
   "metadata": {},
   "source": [
    "### Load `trained model` and `test_dataset`\n",
    "\n",
    "Now we load weights that we previously trained using the `ImageNet` and fine-tuned using the `train_dataset`. We will use this trained model to make predictions on the `test_dataset`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2c10114",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.load('checkpoint/best-0.971.pth')\n",
    "model = model.eval().to(device)\n",
    "idx_to_labels = np.load('table/idx_to_labels.npy', allow_pickle = True).item()\n",
    "classes = list(idx_to_labels.values())\n",
    "\n",
    "dataset_dir = 'data_split'\n",
    "test_path = os.path.join(dataset_dir, 'val')\n",
    "test_dataset = datasets.ImageFolder(test_path, test_transform)\n",
    "print('test_dataset number', len(test_dataset))\n",
    "print('test_dataset class number', len(test_dataset.classes))\n",
    "print('test_dataset class name', test_dataset.classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d58158f",
   "metadata": {},
   "source": [
    "### Table A - test_dataset image path and annotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2cdab48",
   "metadata": {},
   "outputs": [],
   "source": [
    "img_paths = [each[0] for each in test_dataset.imgs]\n",
    "df = pd.DataFrame()\n",
    "df['Data Path'] = img_paths\n",
    "df['Label ID'] = test_dataset.targets\n",
    "df['Label Name'] = [idx_to_labels[ID] for ID in test_dataset.targets]\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0587491",
   "metadata": {},
   "source": [
    "### Table B - Prediction result of each image in `test_dataset` and their confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a0e5cdc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# top-n Prediction result\n",
    "n = 3\n",
    "\n",
    "df_pred = pd.DataFrame()\n",
    "\n",
    "for idx, row in tqdm(df.iterrows()):\n",
    "    img_path = row['Data Path']\n",
    "    img_pil = Image.open(img_path).convert('RGB')\n",
    "    input_img = test_transform(img_pil).unsqueeze(0).to(device) # Preprocess\n",
    "    pred_logits = model(input_img) # feed-forward prediction\n",
    "    pred_softmax = F.softmax(pred_logits, dim=1) # softmax on logit score\n",
    "    pred_dict = {}\n",
    "\n",
    "    top_n = torch.topk(pred_softmax, n) #  n highest confidence level\n",
    "    pred_ids = top_n[1].cpu().detach().numpy().squeeze() # \n",
    "    \n",
    "    # top-n Prediction results\n",
    "    for i in range(1, n+1):\n",
    "        pred_dict['top-{} Predicted ID'.format(i)] = pred_ids[i-1]\n",
    "        pred_dict['top-{} Predicted Name'.format(i)] = idx_to_labels[pred_ids[i-1]]\n",
    "    pred_dict['top-n Predicted True'] = row['Label ID'] in pred_ids\n",
    "    \n",
    "    # Prediction confidence for each class\n",
    "    for idx, each in enumerate(classes):\n",
    "        pred_dict['{} Predicted Confidence'.format(each)] = pred_softmax[0][idx].cpu().detach().numpy()\n",
    "        \n",
    "    df_pred = df_pred._append(pred_dict, ignore_index = True)\n",
    "    \n",
    "df_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b7e01c1",
   "metadata": {},
   "source": [
    "### Concatenate `Table A` and `Table B`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12e021ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df, df_pred], axis = 1)\n",
    "df.to_csv('table/test_dataset prediction results.csv', index = False)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b58bd4d",
   "metadata": {},
   "source": [
    "# `test_dataset` overal performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97d13a84",
   "metadata": {},
   "source": [
    "## Load `idx_to_labels.npy`, `test_dataset prediction results.csv`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0283eb72",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('table/test_dataset prediction results.csv')\n",
    "\n",
    "# Accuracy\n",
    "sum(df['Label Name'] == df['top-1 Predicted Name']) / len(df)\n",
    "# top-n Accuracy\n",
    "sum(df['top-n Predicted True']) / len(df)\n",
    "# Other Metrics\n",
    "print(classification_report(df['Label Name'], df['top-1 Predicted Name'], target_names=classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a749b6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "report = classification_report(df['Label Name'], df['top-1 Predicted Name'], target_names=classes, output_dict=True)\n",
    "del report['accuracy']\n",
    "\n",
    "df_report = pd.DataFrame(report).transpose()\n",
    "df_report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "293446c5",
   "metadata": {},
   "source": [
    "## all classes accuracy（recall）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5258515b",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_list = []\n",
    "for image in tqdm(classes):\n",
    "    df_temp = df[df['Label Name'] == image]\n",
    "    accuracy = sum(df_temp['Label Name'] == df_temp['top-1 Predicted Name']) / len(df_temp)\n",
    "    accuracy_list.append(accuracy)\n",
    "    \n",
    "# macro avg accuracy and weighted avg accuracy\n",
    "acc_macro = np.mean(accuracy_list)\n",
    "acc_weighted = sum(accuracy_list * df_report.iloc[:-2]['support'] / len(df))\n",
    "\n",
    "accuracy_list.append(acc_macro)\n",
    "accuracy_list.append(acc_weighted)\n",
    "\n",
    "df_report['accuracy'] = accuracy_list\n",
    "df_report.to_csv('table/all classes accuracy.csv', index_label = 'Label')\n",
    "\n",
    "df_report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b61baeb",
   "metadata": {},
   "source": [
    "# Confusion Matrix\n",
    "\n",
    "We use the predictions to plot a version of the confusion matrix. Each row represents the true class and each columen represents the predicted class. The accuracy for each class can be seen in the diagonal entries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ce175d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "confusion_matrix_model = confusion_matrix(df['Label Name'], df['top-1 Predicted Name'])\n",
    "confusion_matrix_model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f93520d",
   "metadata": {},
   "source": [
    "## Visualize Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aaab23c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def CM_plotter(cm, classes, cmap = plt.cm.Blues):\n",
    "   \n",
    "    plt.figure()\n",
    "    plt.imshow(cm, interpolation = 'nearest', cmap = cmap, aspect = 'equal') \n",
    "    tick_marks = np.arange(len(classes))\n",
    "    \n",
    "    plt.xlabel('Predicted', fontsize = 10, c = 'k')\n",
    "    plt.ylabel('Ground Truth', fontsize = 10, c = 'k')\n",
    "    plt.tick_params(labelsize = 10) \n",
    "    plt.xticks(tick_marks, classes, rotation = 45) \n",
    "    plt.yticks(tick_marks, classes, rotation = 45)\n",
    "    plt.colorbar()\n",
    "    \n",
    "    threshold = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment = \"center\",\n",
    "                 color = \"white\" if cm[i, j] > threshold else \"black\",\n",
    "                 fontsize = 10)\n",
    "    plt.tight_layout()\n",
    "\n",
    "    plt.savefig('figure/Confusion Matrix.tif', dpi = 300, bbox_inches = 'tight') \n",
    "    plt.show()\n",
    "    \n",
    "CM_plotter(confusion_matrix_model, classes, cmap = 'Blues') # RdPu"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cdb6b99",
   "metadata": {},
   "source": [
    "# PR curves"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "827c1fef",
   "metadata": {},
   "source": [
    "## PR curves for a class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "953e0114",
   "metadata": {},
   "outputs": [],
   "source": [
    "specific_class = 'COVID'\n",
    "\n",
    "# Binary classification labeling\n",
    "y_test = (df['Label Name'] == specific_class)\n",
    "# Binary classification prediction confidence\n",
    "y_score = df['COVID Predicted Confidence']\n",
    "\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, y_score)\n",
    "AP = average_precision_score(y_test, y_score, average='weighted')\n",
    "\n",
    "label = 'COVID (AP = %0.4f)' % AP\n",
    "\n",
    "\n",
    "plt.figure(figsize = (10, 10))\n",
    "plt.plot(recall, precision, linewidth = 2, label = label)\n",
    "\n",
    "\n",
    "plt.plot([0, 0], [0, 1], ls = \"--\", c = '.3', linewidth = 2, label = 'Random guess')\n",
    "plt.plot([0, 1], [0.5, sum(y_test == 1)/len(df)], ls = \"--\", c = '.3', linewidth = 2)\n",
    "\n",
    "plt.xlim([-0.01, 1.01])\n",
    "plt.ylim([0.0, 1.01])\n",
    "plt.title('{}'.format(specific_class),fontsize = 20)\n",
    "plt.xlabel('Recall', fontsize = 20, c = 'k')\n",
    "plt.ylabel('Precision', fontsize = 20, c = 'k')\n",
    "plt.grid(True)\n",
    "plt.legend(loc = 3, fontsize = 15, frameon = False)\n",
    "plt.savefig('figure/{}-PR Curve.tif'.format(specific_class), dpi = 300, bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a2dacfb",
   "metadata": {},
   "source": [
    "## PR curve for all classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca2f8c48",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(124)\n",
    "colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan', 'black', 'indianred', 'brown', 'firebrick', 'maroon', 'darkred', 'red', 'sienna', 'chocolate', 'yellow', 'olivedrab', 'yellowgreen', 'darkolivegreen', 'forestgreen', 'limegreen', 'darkgreen', 'green', 'lime', 'seagreen', 'mediumseagreen', 'darkslategray', 'darkslategrey', 'teal', 'darkcyan', 'dodgerblue', 'navy', 'darkblue', 'mediumblue', 'blue', 'slateblue', 'darkslateblue', 'mediumslateblue', 'mediumpurple', 'rebeccapurple', 'blueviolet', 'indigo', 'darkorchid', 'darkviolet', 'mediumorchid', 'purple', 'darkmagenta', 'fuchsia', 'magenta', 'orchid', 'mediumvioletred', 'deeppink', 'hotpink']\n",
    "markers = [\".\",\",\",\"o\",\"v\",\"^\",\"<\",\">\",\"1\",\"2\",\"3\",\"4\",\"8\",\"s\",\"p\",\"P\",\"*\",\"h\",\"H\",\"+\",\"x\",\"X\",\"D\",\"d\",\"|\",\"_\",0,1,2,3,4,5,6,7,8,9,10,11]\n",
    "linestyle = ['--', '-.', '-']\n",
    "\n",
    "def get_line_arg():\n",
    "    '''\n",
    "    Randomly generate a drawing line type\n",
    "    '''\n",
    "    line_arg = {}\n",
    "    line_arg['color'] = random.choice(colors)\n",
    "    # line_arg['marker'] = random.choice(markers)\n",
    "    line_arg['linestyle'] = random.choice(linestyle)\n",
    "#     line_arg['linewidth'] = random.randint(1, 4)\n",
    "    line_arg['linewidth'] = 2\n",
    "    # line_arg['markersize'] = random.randint(3, 5)\n",
    "    return line_arg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "277876da",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize = (10, 10))\n",
    "plt.xlim([-0.01, 1.01])\n",
    "plt.ylim([0.0, 1.01])\n",
    "plt.plot([0, 0], [0, 1],ls = \"--\", c = '.3', linewidth = 2, label = 'Random guess')\n",
    "\n",
    "plt.xlabel('Recall', fontsize = 20, c = 'k')\n",
    "plt.ylabel('Precision', fontsize = 20, c = 'k')\n",
    "plt.rcParams['font.size'] = 22\n",
    "plt.grid(True)\n",
    "\n",
    "ap_list = []\n",
    "for each_class in classes:\n",
    "    y_test = list((df['Label Name'] == each_class))\n",
    "    y_score = list(df['{} Predicted Confidence'.format(each_class)])\n",
    "    precision, recall, thresholds = precision_recall_curve(y_test, y_score)\n",
    "    AP = average_precision_score(y_test, y_score, average = 'weighted')\n",
    "    \n",
    "    label = '{} (AP = %0.4f)'.format(each_class) % AP\n",
    "    \n",
    "    plt.plot(recall, precision, **get_line_arg(), label = label)\n",
    "    ap_list.append(AP)\n",
    "\n",
    "plt.legend(loc = 3, fontsize = 15, frameon = False)\n",
    "plt.savefig('figure/Total PR Curve.tif'.format(specific_class), dpi = 300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25859c81",
   "metadata": {},
   "source": [
    "## Add AP value to `all classes accuracy.csv`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01eab49f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_report = pd.read_csv('table/all classes accuracy.csv')\n",
    "\n",
    "macro_avg_auc = np.mean(ap_list)\n",
    "weighted_avg_auc = sum(ap_list * df_report.iloc[:-2]['support'] / len(df))\n",
    "\n",
    "ap_list.append(macro_avg_auc)\n",
    "ap_list.append(weighted_avg_auc)\n",
    "\n",
    "df_report['AP'] = ap_list\n",
    "df_report.to_csv('table/all classes accuracy.csv', index = False)\n",
    "\n",
    "df_report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d253713",
   "metadata": {},
   "source": [
    "# ROC curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e3a6ced",
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, threshold = roc_curve(y_test, y_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07545660",
   "metadata": {},
   "source": [
    "## ROC curves for a class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "908f12b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 'COVID (AUC = %0.4f)' % auc(fpr, tpr)\n",
    "\n",
    "plt.figure(figsize = (10, 10))\n",
    "plt.plot(fpr, tpr, linewidth = 2, label = label)\n",
    "plt.plot([0, 1], [0, 1],ls = \"--\", c = '.3', linewidth = 2, label = 'Random guess')\n",
    "plt.xlim([-0.01, 1.0])\n",
    "plt.ylim([0.0, 1.01])\n",
    "plt.title('{}'.format(specific_class), fontsize = 20)\n",
    "plt.xlabel('1-Specificity', fontsize = 20, c = 'k')\n",
    "plt.ylabel('Recall', fontsize = 20, c = 'k')\n",
    "plt.legend(loc = 4, fontsize = 15, frameon = False)\n",
    "plt.grid(True)\n",
    "\n",
    "plt.savefig('figure/{}-ROC Curve.tif'.format(specific_class), dpi = 300, bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f067d28b",
   "metadata": {},
   "source": [
    "## ROC curve for all classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4756eb20",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize = (10, 10))\n",
    "plt.xlim([-0.01, 1.0])\n",
    "plt.ylim([0.0, 1.01])\n",
    "plt.plot([0, 1], [0, 1],ls = \"--\", c = '.3', linewidth = 2, label = 'Random guess')\n",
    "plt.xlabel('1-Specificity', fontsize = 20, c = 'k')\n",
    "plt.ylabel('Recall', fontsize = 20, c = 'k')\n",
    "plt.rcParams['font.size'] = 22\n",
    "plt.grid(True)\n",
    "\n",
    "auc_list = []\n",
    "for each_class in classes:\n",
    "    y_test = list((df['Label Name'] == each_class))\n",
    "    y_score = list(df['{} Predicted Confidence'.format(each_class)])\n",
    "    fpr, tpr, threshold = roc_curve(y_test, y_score) \n",
    "    label = '{} (AUC = %0.4f)'.format(each_class) % auc(fpr, tpr)\n",
    "    plt.plot(fpr, tpr, **get_line_arg(), label = label)\n",
    "    auc_list.append(auc(fpr, tpr))\n",
    "\n",
    "plt.legend(loc = 4, fontsize = 15, frameon = False)\n",
    "plt.savefig('figure/all classes ROC curve.tif'.format(specific_class), dpi = 300, bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e9d2efe",
   "metadata": {},
   "source": [
    "## Add AUC value to `all classes accuracy.csv`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82b062d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_report = pd.read_csv('table/all classes accuracy.csv')\n",
    "\n",
    "macro_avg_auc = np.mean(auc_list)\n",
    "weighted_avg_auc = sum(auc_list * df_report.iloc[:-2]['support'] / len(df))\n",
    "\n",
    "auc_list.append(macro_avg_auc)\n",
    "auc_list.append(weighted_avg_auc)\n",
    "\n",
    "df_report['AUC'] = auc_list\n",
    "\n",
    "df_report.to_csv('table/all classes accuracy.csv', index = False)\n",
    "\n",
    "df_report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c17a392",
   "metadata": {},
   "source": [
    "# Hist plot of each class accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74cd5652",
   "metadata": {},
   "source": [
    "## Hist plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8a3c5ee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc91de0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('table/all classes accuracy.csv')\n",
    "\n",
    "# Evaluation Index\n",
    "# feature = 'precision'\n",
    "# feature = 'f1-score'\n",
    "# feature = 'accuracy'\n",
    "# feature = 'AP'\n",
    "feature = 'AUC'\n",
    "\n",
    "df_plot = df.sort_values(by = feature, ascending = False)\n",
    "plt.figure(figsize = (15, 5))\n",
    "x = df_plot['Label']\n",
    "y = df_plot[feature]\n",
    "ax = plt.bar(x, y, width = 0.6, facecolor = '#1f77b4', edgecolor = 'k')\n",
    "plt.bar_label(ax, fmt = '%.3f', fontsize = 20) # Confidence\n",
    "plt.xticks(rotation = 60)\n",
    "plt.tick_params(labelsize = 20)\n",
    "plt.ylim(0, 1.1)\n",
    "plt.ylabel(feature, fontsize = 25)\n",
    "plt.title('Evaluation metrics: {}'.format(feature), fontsize=25)\n",
    "plt.savefig('figure/all classes accuracy-{}.tif'.format(feature), dpi = 300, bbox_inches = 'tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "804d1e48",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ImageClassification",
   "language": "python",
   "name": "imageclassification"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}