Evaluation_breast_cancer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LFHsu2dxAWlU",
    "outputId": "bce2894a-d4b1-48d6-fc4e-611e04f0ec07",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: scikeras in /opt/conda/lib/python3.10/site-packages (0.11.0)\n",
      "Requirement already satisfied: packaging>=0.21 in /opt/conda/lib/python3.10/site-packages (from scikeras) (23.1)\n",
      "Requirement already satisfied: scikit-learn>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from scikeras) (1.3.0)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /opt/conda/lib/python3.10/site-packages (from scikit-learn>=1.0.0->scikeras) (1.24.3)\n",
      "Requirement already satisfied: scipy>=1.5.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn>=1.0.0->scikeras) (1.11.1)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from scikit-learn>=1.0.0->scikeras) (1.3.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn>=1.0.0->scikeras) (3.2.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install scikeras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_MhvAaMkPw3L",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"-1\"\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4ksxWsvZ857k",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "from os.path import join, getctime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from string import ascii_lowercase\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils.multiclass import type_of_target\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import metrics\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from scikeras.wrappers import KerasClassifier\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "G2tFdhO6poIv",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Machine Learning Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kCI69YZ1HDJ-",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def knn(X_train, y_train):\n",
    "    knn_model = KNeighborsClassifier()\n",
    "    # Define the parameter grid for grid search\n",
    "    param_grid = {'n_neighbors': [3, 5, 7, 10],\n",
    "                  'weights': ['uniform', 'distance'],\n",
    "                  'metric' : ['euclidean', 'manhattan', 'minkowski']}\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(knn_model, param_grid, cv=3, n_jobs=-1)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_knn_model = grid_search.best_estimator_\n",
    "    return best_knn_model\n",
    "\n",
    "def logistic_regression(X_train, y_train):\n",
    "    lr_model = LogisticRegression()\n",
    "\n",
    "    # Define the parameter grid for grid search\n",
    "    param_grid = {'penalty':['l2'],\n",
    "                  'C': [0.01, 0.1, 1.0, 10]}\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(lr_model, param_grid, cv=3, n_jobs=-1)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_lr_model = grid_search.best_estimator_\n",
    "    return best_lr_model\n",
    "\n",
    "def svm(X_train, y_train):\n",
    "    svm_model = SVC(probability=True)\n",
    "\n",
    "    # Define the parameter grid for grid search\n",
    "    param_grid = {'C': [0.01, 0.1, 1.0,10],\n",
    "                  'kernel': ['linear', 'rbf','sigmoid','poly']}\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(svm_model, param_grid, cv=3, n_jobs=-1)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_svm_model = grid_search.best_estimator_\n",
    "    return best_svm_model\n",
    "\n",
    "def naive_bayes(X_train, y_train):\n",
    "    nb_model = GaussianNB()\n",
    "    # No hyperparameters to tune for Naive Bayes\n",
    "    nb_model.fit(X_train, y_train)\n",
    "    return nb_model\n",
    "\n",
    "\n",
    "def random_forest(X_train, y_train):\n",
    "    rf_model = RandomForestClassifier()\n",
    "    # Define the parameter grid for grid search\n",
    "    param_grid = {'n_estimators': [10, 100, 200],\n",
    "                  'max_depth': [None, 10, 50]}\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(rf_model, param_grid, cv=3, n_jobs=-1)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_rf_model = grid_search.best_estimator_\n",
    "    return best_rf_model\n",
    "\n",
    "\n",
    "def binary_neural_network(X_train, y_train,net_type):\n",
    "  if net_type == 'snn':\n",
    "    def create_model():\n",
    "      model = keras.Sequential()\n",
    "      model.add(keras.layers.Dense(64, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "      model.add(keras.layers.Dense(32, activation='relu'))\n",
    "      model.add(keras.layers.Dense(1, activation='sigmoid'))\n",
    "      model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "      return model\n",
    "    model = KerasClassifier(model=create_model, verbose=0)\n",
    "    param_grid = {'batch_size': [32, 64, 128],\n",
    "                'epochs': [10, 30, 50, 100]}\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=3)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_snn_model = grid_search.best_estimator_\n",
    "    return best_snn_model\n",
    "\n",
    "  elif net_type == 'dnn':\n",
    "    def create_model():\n",
    "      model = keras.Sequential()\n",
    "      model.add(keras.layers.Dense(128, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "      model.add(keras.layers.Dropout(0.2))\n",
    "      model.add(keras.layers.Dense(64, activation='relu'))\n",
    "      model.add(keras.layers.Dropout(0.2))\n",
    "      model.add(keras.layers.Dense(32, activation='relu'))\n",
    "      model.add(keras.layers.Dense(1, activation='sigmoid'))\n",
    "      model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "      return model\n",
    "    model = KerasClassifier(model=create_model, verbose=0)\n",
    "    param_grid = {'batch_size': [32, 64, 128],\n",
    "                  'epochs': [10, 30, 50, 100]}\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=3)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_dnn_model = grid_search.best_estimator_\n",
    "    return best_dnn_model\n",
    "\n",
    "\n",
    "def multiclass_neural_network(X_train, y_train,net_type):\n",
    "  y_train = to_categorical(y_train, 3)\n",
    "  if net_type == 'snn':\n",
    "    def create_model():\n",
    "      model = keras.Sequential()\n",
    "      model.add(keras.layers.Dense(64, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "      model.add(keras.layers.Dense(32, activation='relu'))\n",
    "      model.add(keras.layers.Dense(3, activation='softmax'))\n",
    "      model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "      return model\n",
    "    model = KerasClassifier(model=create_model, verbose=0)\n",
    "    param_grid = {'batch_size': [32, 64, 128],\n",
    "                'epochs': [10, 30, 50, 100]}\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=3)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_snn_model = grid_search.best_estimator_\n",
    "    return best_snn_model\n",
    "\n",
    "  elif net_type == 'dnn':\n",
    "    def create_model():\n",
    "      model = keras.Sequential()\n",
    "      model.add(keras.layers.Dense(128, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "      model.add(keras.layers.Dropout(0.2))\n",
    "      model.add(keras.layers.Dense(64, activation='relu'))\n",
    "      model.add(keras.layers.Dropout(0.2))\n",
    "      model.add(keras.layers.Dense(32, activation='relu'))\n",
    "      model.add(keras.layers.Dense(3, activation='softmax'))\n",
    "      model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "      return model\n",
    "    model = KerasClassifier(model=create_model, verbose=0)\n",
    "    param_grid = {'batch_size': [32, 64, 128],\n",
    "                  'epochs': [10, 30, 50, 100]}\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=3)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_dnn_model = grid_search.best_estimator_\n",
    "    return best_dnn_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gChfIDlspshO",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Evaluation metrics\n",
    "- accuracy, sensitivity, specificity, ppv, npv, roc_auc\n",
    "- d_accuracy, d_sensitivity, d_specificity, d_ppv, d_npv, d_roc_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YuPLHO8arVjU",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def binary_metrics(TP, FP, TN, FN):\n",
    "    accuracy = (TP + TN) / (TP + FP + TN + FN)\n",
    "    sensitivity = TP / (TP + FN) if (TP + FN) > 0 else 0.0\n",
    "    specificity = TN / (TN + FP) if (TN + FP) > 0 else 0.0\n",
    "    ppv = TP / (TP + FP) if (TP + FP) > 0 else 0.0\n",
    "    npv = TN / (TN + FN) if (TN + FN) > 0 else 0.0\n",
    "    return accuracy, sensitivity, specificity, ppv, npv\n",
    "\n",
    "def binary_confusion_matrix(true_labels, y_pred, class_weights):\n",
    "    TP = sum((a == 1) and (p == 1) for a, p in zip(true_labels, y_pred))\n",
    "    FP = sum((a != 1) and (p == 1) for a, p in zip(true_labels, y_pred))\n",
    "    TN = sum((a != 1) and (p != 1) for a, p in zip(true_labels, y_pred))\n",
    "    FN = sum((a == 1) and (p != 1) for a, p in zip(true_labels, y_pred))\n",
    "\n",
    "    d_TP = sum(w * ((a == 1) and (p == 1)) for a, p, w in zip(true_labels, y_pred, class_weights))\n",
    "    d_FP = sum((1-w) * ((a != 1) and (p == 1)) for a, p, w in zip(true_labels, y_pred, class_weights))\n",
    "    d_TN = sum(w * ((a != 1) and (p != 1)) for a, p, w in zip(true_labels, y_pred, class_weights))\n",
    "    d_FN = sum((1-w) * ((a == 1) and (p != 1)) for a, p, w in zip(true_labels, y_pred, class_weights))\n",
    "\n",
    "    if sum(class_weights) == 0:  # If all weights are 0, accuracy is 1\n",
    "        print(\"Sum of class weights is 0\")\n",
    "        return TP, FP, TN, FN, TP, FP, TN, FN\n",
    "    else:\n",
    "        return TP, FP, TN, FN, d_TP, d_FP, d_TN, d_FN\n",
    "\n",
    "\n",
    "def binary_evaluation(true_labels, predicted_labels, class_weights):\n",
    "    predicted_labels =  np.array(predicted_labels)[:,1]\n",
    "    y_pred_binary = np.where(predicted_labels >= 0.5, 1, 0)\n",
    "    TP, FP, TN, FN, d_TP, d_FP, d_TN, d_FN = binary_confusion_matrix(true_labels,y_pred_binary,class_weights)\n",
    "    accuracy, sensitivity, specificity, ppv, npv = binary_metrics(TP, FP, TN, FN)\n",
    "    d_accuracy, d_sensitivity, d_specificity, d_ppv, d_npv = binary_metrics(d_TP, d_FP, d_TN, d_FN)\n",
    "\n",
    "    #AUC\n",
    "    thresholds = np.linspace(0, 1, 100)\n",
    "    tpr_list = []\n",
    "    fpr_list = []\n",
    "    d_tpr_list = []\n",
    "    d_fpr_list = []\n",
    "    for threshold in thresholds:    # Calculate TPR and FPR for each threshold\n",
    "        y_pred_binary = np.where(predicted_labels >= threshold, 1, 0)\n",
    "        TP, FP, TN, FN, d_TP, d_FP, d_TN, d_FN = binary_confusion_matrix(true_labels,y_pred_binary,class_weights)\n",
    "\n",
    "        tpr = TP / (TP + FN)\n",
    "        fpr = FP / (FP + TN)\n",
    "        d_tpr = d_TP / (d_TP + d_FN)\n",
    "        d_fpr = d_FP / (d_FP + d_TN)\n",
    "\n",
    "        tpr_list.append(tpr)\n",
    "        fpr_list.append(fpr)\n",
    "        d_tpr_list.append(d_tpr)\n",
    "        d_fpr_list.append(d_fpr)\n",
    "\n",
    "    roc_auc = metrics.auc(fpr_list,tpr_list)\n",
    "    d_roc_auc = metrics.auc(d_fpr_list,d_tpr_list)\n",
    "\n",
    "    return [accuracy, sensitivity, specificity, ppv, npv, roc_auc , d_accuracy, d_sensitivity, d_specificity, d_ppv, d_npv, d_roc_auc]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uoCQZxJfp213",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "62oMeIpOpT8e",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## CDmc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 455
    },
    "id": "8dZ-UBSeG9RG",
    "outputId": "94fb6623-248c-4816-f5b5-08edf5fc4a91",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CT</th>\n",
       "      <th>UCSize</th>\n",
       "      <th>UCShape</th>\n",
       "      <th>MA</th>\n",
       "      <th>SECSize</th>\n",
       "      <th>BN</th>\n",
       "      <th>BC</th>\n",
       "      <th>NN</th>\n",
       "      <th>Mitoses</th>\n",
       "      <th>y</th>\n",
       "      <th>difficulty</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>case_index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>694</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>695</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>696</th>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>697</th>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>699 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            CT  UCSize  UCShape  MA  SECSize    BN  BC  NN  Mitoses  y  \\\n",
       "case_index                                                               \n",
       "0            5       1        1   1        2   1.0   3   1        1  0   \n",
       "1            5       4        4   5        7  10.0   3   2        1  0   \n",
       "2            3       1        1   1        2   2.0   3   1        1  0   \n",
       "3            6       8        8   1        3   4.0   3   7        1  0   \n",
       "4            4       1        1   3        2   1.0   3   1        1  0   \n",
       "...         ..     ...      ...  ..      ...   ...  ..  ..      ... ..   \n",
       "694          3       1        1   1        3   2.0   1   1        1  0   \n",
       "695          2       1        1   1        2   1.0   1   1        1  0   \n",
       "696          5      10       10   3        7   3.0   8  10        2  1   \n",
       "697          4       8        6   4        3   4.0  10   6        1  1   \n",
       "698          4       8        8   5        4   5.0  10   4        1  1   \n",
       "\n",
       "            difficulty  \n",
       "case_index              \n",
       "0             0.142857  \n",
       "1             1.000000  \n",
       "2             0.142857  \n",
       "3             1.000000  \n",
       "4             0.142857  \n",
       "...                ...  \n",
       "694           0.142857  \n",
       "695           0.142857  \n",
       "696           0.142857  \n",
       "697           0.142857  \n",
       "698           0.142857  \n",
       "\n",
       "[699 rows x 11 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app1_uci = pd.read_excel('../Evaluation/real_world/app1_UCI_breast_cancer.xlsx',index_col=[0])\n",
    "app1_uci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "a8K3LCqQslYc",
    "outputId": "50f5b440-c7c4-4fd6-8ab1-bbbec920cb8c",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.965235 using {'metric': 'euclidean', 'n_neighbors': 10, 'weights': 'distance'}\n",
      "Best: 0.965235 using {'C': 1.0, 'penalty': 'l2'}\n",
      "Best: 0.969325 using {'C': 0.1, 'kernel': 'rbf'}\n",
      "Best: 0.959100 using {'max_depth': None, 'n_estimators': 100}\n",
      "binary\n",
      "Best: 0.957055 using {'batch_size': 32, 'epochs': 100}\n",
      "Best: 0.961145 using {'batch_size': 32, 'epochs': 50}\n",
      "KNN\n",
      "Logistic Regression\n",
      "SVM\n",
      "Naive Bayes\n",
      "Random Forest\n",
      "Simple Neural Network\n",
      "Deep Neural Network\n",
      "     accuracy  sensitivity  specificity       ppv       npv   roc_auc  \\\n",
      "knn  0.966667     0.925373     0.986014  0.968750  0.965753  0.997234   \n",
      "lr   0.952381     0.880597     0.986014  0.967213  0.946309  0.997704   \n",
      "svm  0.961905     0.955224     0.965035  0.927536  0.978723  0.996712   \n",
      "nb   0.961905     0.970149     0.958042  0.915493  0.985612  0.983718   \n",
      "rf   0.966667     0.940299     0.979021  0.954545  0.972222  0.995460   \n",
      "snn  0.971429     0.955224     0.979021  0.955224  0.979021  0.996399   \n",
      "dnn  0.980952     0.985075     0.979021  0.956522  0.992908  0.996295   \n",
      "\n",
      "     d_accuracy  d_sensitivity  d_specificity     d_ppv     d_npv  d_roc_auc  \n",
      "knn    0.973684       0.920000       1.000000  1.000000  0.962264   1.000000  \n",
      "lr     0.922078       0.769231       1.000000  1.000000  0.894737   1.000000  \n",
      "svm    0.972851       1.000000       0.958333  0.927711  1.000000   1.000000  \n",
      "nb     0.948498       1.000000       0.919463  0.875000  1.000000   0.957864  \n",
      "rf     0.973684       0.926829       1.000000  1.000000  0.960526   1.000000  \n",
      "snn    0.933852       0.888889       0.962025  0.936170  0.932515   0.972409  \n",
      "dnn    0.954023       0.941748       0.962025  0.941748  0.962025   0.982406  \n"
     ]
    }
   ],
   "source": [
    "breast_cancer_evaluation = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                          'UCI_breast_cancer_evaluation_App1.xlsx',engine='openpyxl')\n",
    "breast_cancer_detail = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                          'UCI_breast_cancer_evaluation_App1_detailed.xlsx',engine='openpyxl')\n",
    "\n",
    "X = app1_uci.iloc[:,:-2]\n",
    "X['difficulty'] = app1_uci['difficulty']\n",
    "y = app1_uci['y']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "X_test_difficulty = X_test['difficulty']\n",
    "X_train = X_train.iloc[:,:-1]\n",
    "X_test = X_test.iloc[:,:-1]\n",
    "\n",
    "best_knn_model = knn(X_train,y_train)\n",
    "best_lr_model = logistic_regression(X_train,y_train)\n",
    "best_svm_model = svm(X_train, y_train)\n",
    "nb_model = naive_bayes(X_train, y_train)\n",
    "best_rf_model = random_forest(X_train, y_train)\n",
    "\n",
    "if len(np.unique(y_train)) == 2:\n",
    "  print('binary')\n",
    "  best_snn_model = binary_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = binary_neural_network(X_train, y_train,'dnn')\n",
    "elif len(np.unique(y_train)) > 2:\n",
    "  print('multi')\n",
    "  best_snn_model = multiclass_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = multiclass_neural_network(X_train, y_train,'dnn')\n",
    "\n",
    "y_val_pred_knn = best_knn_model.predict_proba(X_test)\n",
    "y_val_pred_lr = best_lr_model.predict_proba(X_test)\n",
    "y_val_pred_svm = best_svm_model.predict_proba(X_test)\n",
    "y_val_pred_nb = nb_model.predict_proba(X_test)\n",
    "y_val_pred_rf = best_rf_model.predict_proba(X_test)\n",
    "y_val_pred_snn = best_snn_model.predict_proba(X_test)\n",
    "y_val_pred_dnn = best_dnn_model.predict_proba(X_test)\n",
    "\n",
    "y_type = type_of_target(y_test)\n",
    "if y_type == 'binary':\n",
    "  true_labels = y_test\n",
    "  weights = X_test_difficulty\n",
    "\n",
    "  print(\"KNN\")\n",
    "  knn_result =  binary_evaluation(true_labels, y_val_pred_knn,weights)\n",
    "\n",
    "  print(\"Logistic Regression\")\n",
    "  lr_result =   binary_evaluation(true_labels, y_val_pred_lr,weights)\n",
    "\n",
    "  print(\"SVM\")\n",
    "  svm_result =   binary_evaluation(true_labels, y_val_pred_svm,weights)\n",
    "\n",
    "  print(\"Naive Bayes\")\n",
    "  nb_result =  binary_evaluation(true_labels, y_val_pred_nb,weights)\n",
    "\n",
    "  print(\"Random Forest\")\n",
    "  rf_result =  binary_evaluation(true_labels, y_val_pred_rf,weights)\n",
    "\n",
    "  print(\"Simple Neural Network\")\n",
    "  snn_result =  binary_evaluation(true_labels, y_val_pred_snn,weights)\n",
    "\n",
    "  print(\"Deep Neural Network\")\n",
    "  dnn_result =  binary_evaluation(true_labels, y_val_pred_dnn,weights)\n",
    "\n",
    "  result_list = [knn_result, lr_result, svm_result, nb_result, rf_result,snn_result,dnn_result]\n",
    "  result = pd.DataFrame(result_list, index=['knn','lr','svm','nb','rf','snn','dnn'],\n",
    "                        columns=['accuracy', 'sensitivity', 'specificity', 'ppv', 'npv', 'roc_auc',\n",
    "                                 'd_accuracy', 'd_sensitivity', 'd_specificity', 'd_ppv', 'd_npv', 'd_roc_auc'])\n",
    "  print(result)\n",
    "\n",
    "sheet_name = 'App1'\n",
    "result.to_excel(breast_cancer_evaluation, sheet_name=sheet_name)\n",
    "\n",
    "predictions = pd.DataFrame({'label': y_test,'difficulty':X_test_difficulty,\n",
    "         'knn': np.argmax(y_val_pred_knn, axis=1),'lr': np.argmax(y_val_pred_lr, axis=1),'svm': np.argmax(y_val_pred_svm, axis=1),\n",
    "         'nb': np.argmax(y_val_pred_nb, axis=1),'rf': np.argmax(y_val_pred_rf, axis=1),'snn':np.argmax(y_val_pred_snn, axis=1),\n",
    "                          'dnn':np.argmax(y_val_pred_dnn, axis=1)})\n",
    "\n",
    "new_df = pd.concat([X_test, predictions], axis=1)\n",
    "new_df.to_excel(breast_cancer_detail, sheet_name=sheet_name)\n",
    "\n",
    "breast_cancer_evaluation.close()\n",
    "breast_cancer_detail.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uakslNSHp743",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## CDdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "id": "g0ysCYVKJ2R0",
    "outputId": "e46cdd75-7995-42a6-8ddb-b7ed33169cdc",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "      <th>x8</th>\n",
       "      <th>x9</th>\n",
       "      <th>label</th>\n",
       "      <th>difficulty</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.004322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3.113636</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.028894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>694</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>695</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>696</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.005097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>697</th>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.224570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.001418</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>699 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     x1  x2  x3  x4  x5        x6  x7  x8  x9  label  difficulty\n",
       "0     4   2   1   1   2  1.000000   2   1   1      0    0.000590\n",
       "1     2   1   1   1   2  1.000000   2   1   1      0    0.000560\n",
       "2     6   1   1   1   2  1.000000   3   1   1      0    0.004322\n",
       "3     8   4   5   1   2  3.113636   7   3   1      1    0.028894\n",
       "4     2   1   1   1   2  1.000000   2   1   1      0    0.000560\n",
       "..   ..  ..  ..  ..  ..       ...  ..  ..  ..    ...         ...\n",
       "694   2   1   1   1   2  1.000000   3   1   1      0    0.000987\n",
       "695   1   1   1   1   2  1.000000   1   1   1      0    0.000815\n",
       "696   3   1   1   1   2  1.000000   2   3   1      0    0.005097\n",
       "697   5  10  10   5   4  5.000000   4   4   1      1    0.224570\n",
       "698   3   1   1   1   3  2.000000   1   1   1      0    0.001418\n",
       "\n",
       "[699 rows x 11 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app2_uci = pd.read_excel('../Evaluation/real_world/app2_UCI_breast_cancer.xlsx')\n",
    "app2_uci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8Mmka2hRKCh6",
    "outputId": "b0a8e84d-4c40-41c4-89f7-52b771b751df",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.975460 using {'metric': 'euclidean', 'n_neighbors': 5, 'weights': 'uniform'}\n",
      "Best: 0.963190 using {'C': 0.1, 'penalty': 'l2'}\n",
      "Best: 0.967280 using {'C': 0.01, 'kernel': 'linear'}\n",
      "Best: 0.965235 using {'max_depth': None, 'n_estimators': 100}\n",
      "binary\n",
      "Best: 0.957055 using {'batch_size': 32, 'epochs': 30}\n",
      "Best: 0.963190 using {'batch_size': 64, 'epochs': 30}\n",
      "KNN\n",
      "Logistic Regression\n",
      "SVM\n",
      "Naive Bayes\n",
      "Random Forest\n",
      "Simple Neural Network\n",
      "Deep Neural Network\n",
      "     accuracy  sensitivity  specificity       ppv       npv   roc_auc  \\\n",
      "knn  0.971429     0.958904     0.978102  0.958904  0.978102  0.983302   \n",
      "lr   0.966667     0.945205     0.978102  0.958333  0.971014  0.994501   \n",
      "svm  0.961905     0.945205     0.970803  0.945205  0.970803  0.994251   \n",
      "nb   0.957143     0.972603     0.948905  0.910256  0.984848  0.976202   \n",
      "rf   0.971429     0.986301     0.963504  0.935065  0.992481  0.987401   \n",
      "snn  0.928571     0.863014     0.963504  0.926471  0.929577  0.987451   \n",
      "dnn  0.938095     0.972603     0.919708  0.865854  0.984375  0.990701   \n",
      "\n",
      "     d_accuracy  d_sensitivity  d_specificity     d_ppv     d_npv  d_roc_auc  \n",
      "knn    0.643672       0.757519       0.327450  0.757783  0.327133   0.417616  \n",
      "lr     0.611969       0.711998       0.327450  0.750699  0.285574   0.515215  \n",
      "svm    0.578197       0.711998       0.267370  0.693028  0.285526   0.511643  \n",
      "nb     0.547415       0.824114       0.152441  0.581233  0.377788   0.319216  \n",
      "rf     0.647212       0.903584       0.207568  0.661637  0.556618   0.496432  \n",
      "snn    0.402500       0.480809       0.143391  0.650008  0.077039   0.343233  \n",
      "dnn    0.449958       0.824134       0.036024  0.486065  0.156235   0.394539  \n"
     ]
    }
   ],
   "source": [
    "breast_cancer_evaluation = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                          'UCI_breast_cancer_evaluation_App2.xlsx',engine='openpyxl')\n",
    "breast_cancer_detail = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                          'UCI_breast_cancer_evaluation_App2_detailed.xlsx',engine='openpyxl')\n",
    "\n",
    "X = app2_uci.iloc[:,:-2]\n",
    "X['difficulty'] = app2_uci['difficulty']\n",
    "y = app2_uci['label']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "X_test_difficulty = X_test['difficulty']\n",
    "X_train = X_train.iloc[:,:-1]\n",
    "X_test = X_test.iloc[:,:-1]\n",
    "\n",
    "best_knn_model = knn(X_train,y_train)\n",
    "best_lr_model = logistic_regression(X_train,y_train)\n",
    "best_svm_model = svm(X_train, y_train)\n",
    "nb_model = naive_bayes(X_train, y_train)\n",
    "best_rf_model = random_forest(X_train, y_train)\n",
    "\n",
    "if len(np.unique(y_train)) == 2:\n",
    "  print('binary')\n",
    "  best_snn_model = binary_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = binary_neural_network(X_train, y_train,'dnn')\n",
    "elif len(np.unique(y_train)) > 2:\n",
    "  print('multi')\n",
    "  best_snn_model = multiclass_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = multiclass_neural_network(X_train, y_train,'dnn')\n",
    "\n",
    "y_val_pred_knn = best_knn_model.predict_proba(X_test)\n",
    "y_val_pred_lr = best_lr_model.predict_proba(X_test)\n",
    "y_val_pred_svm = best_svm_model.predict_proba(X_test)\n",
    "y_val_pred_nb = nb_model.predict_proba(X_test)\n",
    "y_val_pred_rf = best_rf_model.predict_proba(X_test)\n",
    "y_val_pred_snn = best_snn_model.predict_proba(X_test)\n",
    "y_val_pred_dnn = best_dnn_model.predict_proba(X_test)\n",
    "\n",
    "y_type = type_of_target(y_test)\n",
    "if y_type == 'binary':\n",
    "  true_labels = y_test\n",
    "  weights = X_test_difficulty\n",
    "\n",
    "  print(\"KNN\")\n",
    "  knn_result =  binary_evaluation(true_labels, y_val_pred_knn,weights)\n",
    "\n",
    "  print(\"Logistic Regression\")\n",
    "  lr_result =   binary_evaluation(true_labels, y_val_pred_lr,weights)\n",
    "\n",
    "  print(\"SVM\")\n",
    "  svm_result =   binary_evaluation(true_labels, y_val_pred_svm,weights)\n",
    "\n",
    "  print(\"Naive Bayes\")\n",
    "  nb_result =  binary_evaluation(true_labels, y_val_pred_nb,weights)\n",
    "\n",
    "  print(\"Random Forest\")\n",
    "  rf_result =  binary_evaluation(true_labels, y_val_pred_rf,weights)\n",
    "\n",
    "  print(\"Simple Neural Network\")\n",
    "  snn_result =  binary_evaluation(true_labels, y_val_pred_snn,weights)\n",
    "\n",
    "  print(\"Deep Neural Network\")\n",
    "  dnn_result =  binary_evaluation(true_labels, y_val_pred_dnn,weights)\n",
    "\n",
    "  result_list = [knn_result, lr_result, svm_result, nb_result, rf_result,snn_result,dnn_result]\n",
    "  result = pd.DataFrame(result_list, index=['knn','lr','svm','nb','rf','snn','dnn'],\n",
    "                        columns=['accuracy', 'sensitivity', 'specificity', 'ppv', 'npv', 'roc_auc',\n",
    "                                 'd_accuracy', 'd_sensitivity', 'd_specificity', 'd_ppv', 'd_npv', 'd_roc_auc'])\n",
    "  print(result)\n",
    "\n",
    "sheet_name = 'App2'\n",
    "result.to_excel(breast_cancer_evaluation, sheet_name=sheet_name)\n",
    "\n",
    "predictions = pd.DataFrame({'label': y_test,'difficulty':X_test_difficulty,\n",
    "         'knn': np.argmax(y_val_pred_knn, axis=1),'lr': np.argmax(y_val_pred_lr, axis=1),'svm': np.argmax(y_val_pred_svm, axis=1),\n",
    "         'nb': np.argmax(y_val_pred_nb, axis=1),'rf': np.argmax(y_val_pred_rf, axis=1),'snn':np.argmax(y_val_pred_snn, axis=1),\n",
    "                          'dnn':np.argmax(y_val_pred_dnn, axis=1)})\n",
    "\n",
    "new_df = pd.concat([X_test, predictions], axis=1)\n",
    "new_df.to_excel(breast_cancer_detail, sheet_name=sheet_name)\n",
    "\n",
    "breast_cancer_evaluation.close()\n",
    "breast_cancer_detail.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PT8J9g86LUSS",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## CDpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "id": "yCl3CA-XLXXh",
    "outputId": "2b11f292-e0bc-4e8f-a17f-b7130b748b6e",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "      <th>x8</th>\n",
       "      <th>x9</th>\n",
       "      <th>label</th>\n",
       "      <th>difficulty</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.016598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.611221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.007619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.545318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.005387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>694</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.067376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>695</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.005687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>696</th>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.054388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>697</th>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.538430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.213233</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>699 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     x1  x2  x3  x4  x5    x6  x7  x8  x9  label  difficulty\n",
       "0     5   1   1   1   2   1.0   3   1   1      0    0.016598\n",
       "1     5   4   4   5   7  10.0   3   2   1      0    0.611221\n",
       "2     3   1   1   1   2   2.0   3   1   1      0    0.007619\n",
       "3     6   8   8   1   3   4.0   3   7   1      0    0.545318\n",
       "4     4   1   1   3   2   1.0   3   1   1      0    0.005387\n",
       "..   ..  ..  ..  ..  ..   ...  ..  ..  ..    ...         ...\n",
       "694   3   1   1   1   3   2.0   1   1   1      0    0.067376\n",
       "695   2   1   1   1   2   1.0   1   1   1      0    0.005687\n",
       "696   5  10  10   3   7   3.0   8  10   2      1    0.054388\n",
       "697   4   8   6   4   3   4.0  10   6   1      1    0.538430\n",
       "698   4   8   8   5   4   5.0  10   4   1      1    0.213233\n",
       "\n",
       "[699 rows x 11 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app3_uci = pd.read_excel('../Evaluation/real_world/app3_UCI_breast_cancer.xlsx',index_col=[0])\n",
    "app3_uci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GNpWBcifLrnz",
    "outputId": "5afbe562-3d7a-439d-c245-91b538534b0f",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.965235 using {'metric': 'euclidean', 'n_neighbors': 10, 'weights': 'distance'}\n",
      "Best: 0.965235 using {'C': 1.0, 'penalty': 'l2'}\n",
      "Best: 0.969325 using {'C': 0.1, 'kernel': 'rbf'}\n",
      "Best: 0.959100 using {'max_depth': None, 'n_estimators': 100}\n",
      "binary\n",
      "Best: 0.959100 using {'batch_size': 32, 'epochs': 50}\n",
      "Best: 0.961145 using {'batch_size': 32, 'epochs': 50}\n",
      "KNN\n",
      "Logistic Regression\n",
      "SVM\n",
      "Naive Bayes\n",
      "Random Forest\n",
      "Simple Neural Network\n",
      "Deep Neural Network\n",
      "     accuracy  sensitivity  specificity       ppv       npv   roc_auc  \\\n",
      "knn  0.966667     0.925373     0.986014  0.968750  0.965753  0.997234   \n",
      "lr   0.952381     0.880597     0.986014  0.967213  0.946309  0.997704   \n",
      "svm  0.961905     0.955224     0.965035  0.927536  0.978723  0.996608   \n",
      "nb   0.961905     0.970149     0.958042  0.915493  0.985612  0.983718   \n",
      "rf   0.961905     0.925373     0.979021  0.953846  0.965517  0.996973   \n",
      "snn  0.961905     0.910448     0.986014  0.968254  0.959184  0.995669   \n",
      "dnn  0.980952     0.985075     0.979021  0.956522  0.992908  0.997130   \n",
      "\n",
      "     d_accuracy  d_sensitivity  d_specificity     d_ppv     d_npv  d_roc_auc  \n",
      "knn    0.784926       0.728580       0.861798  0.877935  0.699457   0.905705  \n",
      "lr     0.677977       0.555198       0.861798  0.857441  0.564097   0.919897  \n",
      "svm    0.759556       0.819511       0.656584  0.803865  0.679292   0.879343  \n",
      "nb     0.750348       0.860759       0.563423  0.769474  0.705021   0.755633  \n",
      "rf     0.751271       0.721576       0.794380  0.835918  0.662770   0.889422  \n",
      "snn    0.752052       0.671918       0.861798  0.869426  0.657300   0.863567  \n",
      "dnn    0.894291       0.949776       0.811226  0.882798  0.915175   0.919952  \n"
     ]
    }
   ],
   "source": [
    "breast_cancer_evaluation = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                          'UCI_breast_cancer_evaluation_App3.xlsx',engine='openpyxl')\n",
    "breast_cancer_detail = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                          'UCI_breast_cancer_evaluation_App3_detailed.xlsx',engine='openpyxl')\n",
    "\n",
    "X = app3_uci.iloc[:,:-2]\n",
    "X['difficulty'] = app3_uci['difficulty']\n",
    "y = app3_uci['label']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "X_test_difficulty = X_test['difficulty']\n",
    "X_train = X_train.iloc[:,:-1]\n",
    "X_test = X_test.iloc[:,:-1]\n",
    "\n",
    "best_knn_model = knn(X_train,y_train)\n",
    "best_lr_model = logistic_regression(X_train,y_train)\n",
    "best_svm_model = svm(X_train, y_train)\n",
    "nb_model = naive_bayes(X_train, y_train)\n",
    "best_rf_model = random_forest(X_train, y_train)\n",
    "\n",
    "if len(np.unique(y_train)) == 2:\n",
    "  print('binary')\n",
    "  best_snn_model = binary_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = binary_neural_network(X_train, y_train,'dnn')\n",
    "elif len(np.unique(y_train)) > 2:\n",
    "  print('multi')\n",
    "  best_snn_model = multiclass_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = multiclass_neural_network(X_train, y_train,'dnn')\n",
    "\n",
    "y_val_pred_knn = best_knn_model.predict_proba(X_test)\n",
    "y_val_pred_lr = best_lr_model.predict_proba(X_test)\n",
    "y_val_pred_svm = best_svm_model.predict_proba(X_test)\n",
    "y_val_pred_nb = nb_model.predict_proba(X_test)\n",
    "y_val_pred_rf = best_rf_model.predict_proba(X_test)\n",
    "y_val_pred_snn = best_snn_model.predict_proba(X_test)\n",
    "y_val_pred_dnn = best_dnn_model.predict_proba(X_test)\n",
    "\n",
    "y_type = type_of_target(y_test)\n",
    "if y_type == 'binary':\n",
    "  true_labels = y_test\n",
    "  weights = X_test_difficulty\n",
    "\n",
    "  print(\"KNN\")\n",
    "  knn_result =  binary_evaluation(true_labels, y_val_pred_knn,weights)\n",
    "\n",
    "  print(\"Logistic Regression\")\n",
    "  lr_result =   binary_evaluation(true_labels, y_val_pred_lr,weights)\n",
    "\n",
    "  print(\"SVM\")\n",
    "  svm_result =   binary_evaluation(true_labels, y_val_pred_svm,weights)\n",
    "\n",
    "  print(\"Naive Bayes\")\n",
    "  nb_result =  binary_evaluation(true_labels, y_val_pred_nb,weights)\n",
    "\n",
    "  print(\"Random Forest\")\n",
    "  rf_result =  binary_evaluation(true_labels, y_val_pred_rf,weights)\n",
    "\n",
    "  print(\"Simple Neural Network\")\n",
    "  snn_result =  binary_evaluation(true_labels, y_val_pred_snn,weights)\n",
    "\n",
    "  print(\"Deep Neural Network\")\n",
    "  dnn_result =  binary_evaluation(true_labels, y_val_pred_dnn,weights)\n",
    "\n",
    "  result_list = [knn_result, lr_result, svm_result, nb_result, rf_result,snn_result,dnn_result]\n",
    "  result = pd.DataFrame(result_list, index=['knn','lr','svm','nb','rf','snn','dnn'],\n",
    "                        columns=['accuracy', 'sensitivity', 'specificity', 'ppv', 'npv', 'roc_auc',\n",
    "                                 'd_accuracy', 'd_sensitivity', 'd_specificity', 'd_ppv', 'd_npv', 'd_roc_auc'])\n",
    "  print(result)\n",
    "\n",
    "sheet_name = 'App3'\n",
    "result.to_excel(breast_cancer_evaluation, sheet_name=sheet_name)\n",
    "\n",
    "predictions = pd.DataFrame({'label': y_test,'difficulty':X_test_difficulty,\n",
    "         'knn': np.argmax(y_val_pred_knn, axis=1),'lr': np.argmax(y_val_pred_lr, axis=1),'svm': np.argmax(y_val_pred_svm, axis=1),\n",
    "         'nb': np.argmax(y_val_pred_nb, axis=1),'rf': np.argmax(y_val_pred_rf, axis=1),'snn':np.argmax(y_val_pred_snn, axis=1),\n",
    "                          'dnn':np.argmax(y_val_pred_dnn, axis=1)})\n",
    "\n",
    "new_df = pd.concat([X_test, predictions], axis=1)\n",
    "new_df.to_excel(breast_cancer_detail, sheet_name=sheet_name)\n",
    "\n",
    "breast_cancer_evaluation.close()\n",
    "breast_cancer_detail.close()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}