Evaluation_customer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pjdPv_w8VNPG",
    "outputId": "33ca12d4-fff8-4aa2-a80f-438aff40c006",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: scikeras in /opt/conda/lib/python3.10/site-packages (0.11.0)\n",
      "Requirement already satisfied: packaging>=0.21 in /opt/conda/lib/python3.10/site-packages (from scikeras) (23.1)\n",
      "Requirement already satisfied: scikit-learn>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from scikeras) (1.2.2)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /opt/conda/lib/python3.10/site-packages (from scikit-learn>=1.0.0->scikeras) (1.24.3)\n",
      "Requirement already satisfied: scipy>=1.3.2 in /opt/conda/lib/python3.10/site-packages (from scikit-learn>=1.0.0->scikeras) (1.11.1)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from scikit-learn>=1.0.0->scikeras) (1.3.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn>=1.0.0->scikeras) (3.2.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install scikeras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kJbOl41jWU4S",
    "outputId": "5fdccea4-e1ea-4ba5-e8bb-dbb7855c99cc",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: scikit-learn==1.2.2 in /opt/conda/lib/python3.10/site-packages (1.2.2)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /opt/conda/lib/python3.10/site-packages (from scikit-learn==1.2.2) (1.24.3)\n",
      "Requirement already satisfied: scipy>=1.3.2 in /opt/conda/lib/python3.10/site-packages (from scikit-learn==1.2.2) (1.11.1)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from scikit-learn==1.2.2) (1.3.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn==1.2.2) (3.2.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install scikit-learn==1.2.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9cgOJvr7WU4T",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"-1\"\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IY1UrUdUG8jq",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "from os.path import join, getctime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from string import ascii_lowercase\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils.multiclass import type_of_target\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import metrics\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from scikeras.wrappers import KerasClassifier\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iMtQr8A_Axh4",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "numeric_columns = ['Age', 'Work_Experience', 'Family_Size']\n",
    "numeric_transformer = Pipeline([\n",
    "    ('imputer', SimpleImputer(strategy='most_frequent')),\n",
    "    ('scaler', StandardScaler())\n",
    "])\n",
    "\n",
    "def standard(original_data):\n",
    "  # Customer Data preprocessing\n",
    "  categorical_transformed = original_data.drop(columns=numeric_columns)\n",
    "  numeric_transformed = pd.DataFrame(numeric_transformer.fit_transform(original_data[numeric_columns]), columns=numeric_columns)\n",
    "  # Concatenate the transformed columns back into a new DataFrame\n",
    "  standard_data = pd.concat([numeric_transformed, categorical_transformed.set_index(numeric_transformed.index)], axis=1)\n",
    "  # Set the original index back\n",
    "  standard_data = standard_data.set_index(original_data.index)\n",
    "  return(standard_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "G2tFdhO6poIv",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Machine Learning Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kCI69YZ1HDJ-",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def knn(X_train, y_train):\n",
    "    knn_model = KNeighborsClassifier()\n",
    "    # Define the parameter grid for grid search\n",
    "    param_grid = {'n_neighbors': [3, 5, 7, 10],\n",
    "                  'weights': ['uniform', 'distance'],\n",
    "                  'metric' : ['euclidean', 'manhattan', 'minkowski']}\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(knn_model, param_grid, cv=3, n_jobs=-1)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_knn_model = grid_search.best_estimator_\n",
    "    return best_knn_model\n",
    "\n",
    "\n",
    "def logistic_regression(X_train, y_train):\n",
    "    lr_model = LogisticRegression(max_iter=1000)\n",
    "\n",
    "    # Define the parameter grid for grid search\n",
    "    param_grid = {'penalty':['l2'],\n",
    "                  'C': [0.01, 0.1, 1.0, 10]}\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(lr_model, param_grid, cv=3, n_jobs=-1)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_lr_model = grid_search.best_estimator_\n",
    "    return best_lr_model\n",
    "\n",
    "\n",
    "def svm(X_train, y_train):\n",
    "    svm_model = SVC(probability=True)\n",
    "\n",
    "    # Define the parameter grid for grid search\n",
    "    param_grid = {'C': [0.01, 0.1, 1.0,10],\n",
    "                  'kernel': ['linear', 'rbf','sigmoid','poly']}\n",
    "\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(svm_model, param_grid, cv=3, n_jobs=-1)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_svm_model = grid_search.best_estimator_\n",
    "    return best_svm_model\n",
    "\n",
    "\n",
    "def naive_bayes(X_train, y_train):\n",
    "    nb_model = GaussianNB()\n",
    "    # No hyperparameters to tune for Naive Bayes\n",
    "    nb_model.fit(X_train, y_train)\n",
    "    return nb_model\n",
    "\n",
    "\n",
    "def random_forest(X_train, y_train):\n",
    "    rf_model = RandomForestClassifier()\n",
    "    # Define the parameter grid for grid search\n",
    "    param_grid = {'n_estimators': [10, 100, 200],\n",
    "                  'max_depth': [None, 10, 50]}\n",
    "    # Perform grid search to find the best hyperparameters\n",
    "    grid_search = GridSearchCV(rf_model, param_grid, cv=3, n_jobs=-1)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_rf_model = grid_search.best_estimator_\n",
    "    return best_rf_model\n",
    "\n",
    "\n",
    "def binary_neural_network(X_train, y_train,net_type):\n",
    "  if net_type == 'snn':\n",
    "    def create_model():\n",
    "      model = keras.Sequential()\n",
    "      model.add(keras.layers.Dense(64, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "      model.add(keras.layers.Dense(32, activation='relu'))\n",
    "      model.add(keras.layers.Dense(1, activation='sigmoid'))\n",
    "      model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "      return model\n",
    "    model = KerasClassifier(model=create_model, verbose=0)\n",
    "    param_grid = {'batch_size': [32, 64, 128],\n",
    "                'epochs': [10, 30, 50, 100]}\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=3)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_snn_model = grid_search.best_estimator_\n",
    "    return best_snn_model\n",
    "\n",
    "  elif net_type == 'dnn':\n",
    "    def create_model():\n",
    "      model = keras.Sequential()\n",
    "      model.add(keras.layers.Dense(128, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "      model.add(keras.layers.Dropout(0.2))\n",
    "      model.add(keras.layers.Dense(64, activation='relu'))\n",
    "      model.add(keras.layers.Dropout(0.2))\n",
    "      model.add(keras.layers.Dense(32, activation='relu'))\n",
    "      model.add(keras.layers.Dense(1, activation='sigmoid'))\n",
    "      model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "      return model\n",
    "    model = KerasClassifier(model=create_model, verbose=0)\n",
    "    param_grid = {'batch_size': [32, 64, 128],\n",
    "                  'epochs': [10, 30, 50, 100]}\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=3)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_dnn_model = grid_search.best_estimator_\n",
    "    return best_dnn_model\n",
    "\n",
    "\n",
    "def multiclass_neural_network(X_train, y_train,net_type):\n",
    "  y_train = to_categorical(y_train, 4)\n",
    "  if net_type == 'snn':\n",
    "    def create_model():\n",
    "      model = keras.Sequential()\n",
    "      model.add(keras.layers.Dense(64, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "      model.add(keras.layers.Dense(32, activation='relu'))\n",
    "      model.add(keras.layers.Dense(4, activation='softmax'))\n",
    "      model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "      return model\n",
    "    model = KerasClassifier(model=create_model, verbose=0)\n",
    "    param_grid = {'batch_size': [32, 64, 128],\n",
    "                'epochs': [10, 30, 50, 100]}\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=3)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_snn_model = grid_search.best_estimator_\n",
    "    return best_snn_model\n",
    "\n",
    "  elif net_type == 'dnn':\n",
    "    def create_model():\n",
    "      model = keras.Sequential()\n",
    "      model.add(keras.layers.Dense(128, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "      model.add(keras.layers.Dropout(0.2))\n",
    "      model.add(keras.layers.Dense(64, activation='relu'))\n",
    "      model.add(keras.layers.Dropout(0.2))\n",
    "      model.add(keras.layers.Dense(32, activation='relu'))\n",
    "      model.add(keras.layers.Dense(4, activation='softmax'))\n",
    "      model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "      return model\n",
    "    model = KerasClassifier(model=create_model, verbose=0)\n",
    "    param_grid = {'batch_size': [32, 64, 128],\n",
    "                  'epochs': [10, 30, 50, 100]}\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=3)\n",
    "    grid_result = grid_search.fit(X_train, y_train)\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    best_dnn_model = grid_search.best_estimator_\n",
    "    return best_dnn_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DtmwfnCepwbG",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Evaluation metrics\n",
    "- micro_accuracy, micro_sensitivity, micro_specificity, micro_ppv, micro_npv, micro_auc\n",
    "- macro_accuracy, macro_sensitivity, macro_specificity, macro_ppv, macro_npv, macro_auc\n",
    "- d_micro_accuracy, d_micro_sensitivity, d_micro_specificity, d_micro_ppv, d_micro_npv, d_micro_auc\n",
    "- d_macro_accuracy, d_macro_sensitivity, d_macro_specificity, d_macro_ppv, macro_npv, d_macro_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Gu6X8a31-Z01",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def multiclass_metrics(confusion_matrix):\n",
    "    num_classes = len(confusion_matrix)\n",
    "    macro_accuracy = 0.0\n",
    "    macro_sensitivity = 0.0\n",
    "    macro_specificity = 0.0\n",
    "    macro_ppv = 0.0\n",
    "    macro_npv = 0.0\n",
    "\n",
    "    for cls in confusion_matrix:\n",
    "        TP = confusion_matrix[cls]['TP']\n",
    "        FP = confusion_matrix[cls]['FP']\n",
    "        TN = confusion_matrix[cls]['TN']\n",
    "        FN = confusion_matrix[cls]['FN']\n",
    "\n",
    "        # Calculate metrics for the current class\n",
    "        accuracy = (TP + TN) / (TP + FP + TN + FN)\n",
    "        sensitivity = TP / (TP + FN) if (TP + FN) > 0 else 0.0\n",
    "        specificity = TN / (TN + FP) if (TN + FP) > 0 else 0.0\n",
    "        ppv = TP / (TP + FP) if (TP + FP) > 0 else 0.0\n",
    "        npv = TN / (TN + FN) if (TN + FN) > 0 else 0.0\n",
    "\n",
    "        macro_accuracy += accuracy\n",
    "        macro_sensitivity += sensitivity\n",
    "        macro_specificity += specificity\n",
    "        macro_ppv += ppv\n",
    "        macro_npv += npv\n",
    "\n",
    "    # Average the metrics across all classes\n",
    "    macro_accuracy /= num_classes\n",
    "    macro_sensitivity /= num_classes\n",
    "    macro_specificity /= num_classes\n",
    "    macro_ppv /= num_classes\n",
    "    macro_npv /= num_classes\n",
    "\n",
    "    ### calculate_micro_averaged_metrics\n",
    "    total_TP = sum(confusion_matrix[cls]['TP'] for cls in confusion_matrix)\n",
    "    total_FP = sum(confusion_matrix[cls]['FP'] for cls in confusion_matrix)\n",
    "    total_TN = sum(confusion_matrix[cls]['TN'] for cls in confusion_matrix)\n",
    "    total_FN = sum(confusion_matrix[cls]['FN'] for cls in confusion_matrix)\n",
    "\n",
    "    # Calculate metrics using aggregated values\n",
    "    micro_accuracy = (total_TP + total_TN) / (total_TP + total_FP + total_TN + total_FN)\n",
    "    micro_sensitivity = total_TP / (total_TP + total_FN) if (total_TP + total_FN) > 0 else 0.0\n",
    "    micro_specificity = total_TN / (total_TN + total_FP) if (total_TN + total_FP) > 0 else 0.0\n",
    "    micro_ppv = total_TP / (total_TP + total_FP) if (total_TP + total_FP) > 0 else 0.0\n",
    "    micro_npv = total_TN / (total_TN + total_FN) if (total_TN + total_FN) > 0 else 0.0\n",
    "\n",
    "    metrics = [micro_accuracy, micro_sensitivity, micro_specificity, micro_ppv, micro_npv,\n",
    "              macro_accuracy, macro_sensitivity, macro_specificity, macro_ppv,macro_npv]\n",
    "    return metrics\n",
    "\n",
    "def multiclass_confusion_matrix(true_labels, predicted_labels, classes, case_weights):\n",
    "    confusion_matrix = {cls: {'TP': 0, 'FP': 0, 'TN': 0, 'FN': 0} for cls in classes}\n",
    "    for a, p in zip(true_labels, predicted_labels):\n",
    "        for cls in classes:\n",
    "            TP = ((a == cls) and (p == cls))\n",
    "            FP = ((a != cls) and (p == cls))\n",
    "            TN = ((a != cls) and (p != cls))\n",
    "            FN = ((a == cls) and (p != cls))\n",
    "            confusion_matrix[cls]['TP'] += TP\n",
    "            confusion_matrix[cls]['FP'] += FP\n",
    "            confusion_matrix[cls]['TN'] += TN\n",
    "            confusion_matrix[cls]['FN'] += FN\n",
    "\n",
    "    d_confusion_matrix = {cls: {'TP': 0, 'FP': 0, 'TN': 0, 'FN': 0} for cls in classes}\n",
    "    for a, p, w in zip(true_labels, predicted_labels, case_weights):\n",
    "        for cls in classes:\n",
    "            TP = w * ((a == cls) and (p == cls))\n",
    "            FP = (1-w) * ((a != cls) and (p == cls))\n",
    "            TN = w * ((a != cls) and (p != cls))\n",
    "            FN = (1-w) * ((a == cls) and (p != cls))\n",
    "            d_confusion_matrix[cls]['TP'] += TP\n",
    "            d_confusion_matrix[cls]['FP'] += FP\n",
    "            d_confusion_matrix[cls]['TN'] += TN\n",
    "            d_confusion_matrix[cls]['FN'] += FN\n",
    "\n",
    "    if sum(case_weights) == 0:  # If all weights are 0, accuracy is 1\n",
    "        print(\"Sum of class weights is 0\")\n",
    "        return confusion_matrix, confusion_matrix\n",
    "    else:\n",
    "        return confusion_matrix, d_confusion_matrix\n",
    "\n",
    "\n",
    "def multi_evaluation(true_labels, prediction_probabilities, classes, case_weights):\n",
    "    confusion_matrix, d_confusion_matrix = multiclass_confusion_matrix(true_labels,np.argmax(prediction_probabilities, axis=1),classes,case_weights)\n",
    "    conventional_result = multiclass_metrics(confusion_matrix)\n",
    "    new_result = multiclass_metrics(d_confusion_matrix)\n",
    "\n",
    "    #AUC\n",
    "    y_true = true_labels\n",
    "    y_true = to_categorical(y_true)\n",
    "    y_pred = prediction_probabilities\n",
    "    expanded_weight = np.repeat(case_weights, len(classes))\n",
    "\n",
    "    # Flatten the true labels and predicted probabilities for a binary classification approach\n",
    "    y_true_flat = y_true.ravel()\n",
    "    y_pred_flat = y_pred.ravel()\n",
    "\n",
    "    # Compute true positive rate (TPR) and false positive rate (FPR) for binary classification\n",
    "    def calculate_tpr_fpr_binary(y_true, y_pred, threshold, expanded_weight):\n",
    "        y_pred_binary = (y_pred >= threshold).astype(int)\n",
    "        tp = np.sum((y_true == 1) & (y_pred_binary == 1))\n",
    "        fp = np.sum((y_true == 0) & (y_pred_binary == 1))\n",
    "        tn = np.sum((y_true == 0) & (y_pred_binary == 0))\n",
    "        fn = np.sum((y_true == 1) & (y_pred_binary == 0))\n",
    "        tpr = tp / (tp + fn)\n",
    "        fpr = fp / (fp + tn)\n",
    "\n",
    "        d_tp = np.sum(expanded_weight * ((y_true == 1) & (y_pred_binary == 1)))\n",
    "        d_fp = np.sum((1.0 - np.array(expanded_weight)) * ((y_true == 0) & (y_pred_binary == 1)))\n",
    "        d_tn = np.sum(expanded_weight * ((y_true == 0) & (y_pred_binary == 0)))\n",
    "        d_fn = np.sum((1.0 - np.array(expanded_weight)) * ((y_true == 1) & (y_pred_binary == 0)))\n",
    "        d_tpr = d_tp / (d_tp + d_fn)\n",
    "        d_fpr = d_fp / (d_fp + d_tn)\n",
    "\n",
    "        if sum(expanded_weight) == 0:  # If all weights are 0, accuracy is 1\n",
    "            print(\"Sum of class weights is 0\")\n",
    "            return tpr, fpr, tpr, fpr\n",
    "        else:\n",
    "            return tpr, fpr, d_tpr, d_fpr\n",
    "\n",
    "    # Calculate TPR and FPR for different thresholds for binary classification\n",
    "    thresholds = np.linspace(0, 1, 100)\n",
    "    tprs = []\n",
    "    fprs = []\n",
    "    d_tprs = []\n",
    "    d_fprs = []\n",
    "    for threshold in thresholds:\n",
    "        tpr, fpr, d_tpr, d_fpr = calculate_tpr_fpr_binary(y_true_flat, y_pred_flat, threshold,expanded_weight)\n",
    "        tprs.append(tpr)\n",
    "        fprs.append(fpr)\n",
    "        d_tprs.append(d_tpr)\n",
    "        d_fprs.append(d_fpr)\n",
    "\n",
    "    # Calculate the micro AUC by integrating the area under the micro-average ROC curve\n",
    "    micro_auc = np.trapz(tprs, fprs)\n",
    "    d_micro_auc = np.trapz(d_tprs, d_fprs)\n",
    "\n",
    "    # Calculate the macro AUC by averaging the AUC for each class\n",
    "    macro_auc = 0.0\n",
    "    d_macro_auc = 0.0\n",
    "    for i in range(len(classes)):\n",
    "        class_tprs = []\n",
    "        class_fprs = []\n",
    "        d_class_tprs = []\n",
    "        d_class_fprs = []\n",
    "        for threshold in thresholds:\n",
    "            tpr, fpr, d_tpr, d_fpr = calculate_tpr_fpr_binary(y_true[:, i], y_pred[:, i], threshold,case_weights)\n",
    "            class_tprs.append(tpr)\n",
    "            class_fprs.append(fpr)\n",
    "            d_class_tprs.append(d_tpr)\n",
    "            d_class_fprs.append(d_fpr)\n",
    "\n",
    "        auc_i = np.trapz(class_tprs, class_fprs)\n",
    "        d_auc_i = np.trapz(d_class_tprs, d_class_fprs)\n",
    "        macro_auc += auc_i\n",
    "        d_macro_auc += d_auc_i\n",
    "\n",
    "    macro_auc /= len(classes)\n",
    "    d_macro_auc /= len(classes)\n",
    "\n",
    "    conventional_result.insert(5,abs(micro_auc))\n",
    "    conventional_result.insert(11,abs(macro_auc))\n",
    "    new_result.insert(5,abs(d_micro_auc))\n",
    "    new_result.insert(11,abs(d_macro_auc))\n",
    "\n",
    "    conventional_result = [np.round(x,3) for x in conventional_result]\n",
    "    new_result = [np.round(x,3) for x in new_result]\n",
    "    return conventional_result + new_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uoCQZxJfp213",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "62oMeIpOpT8e",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## CDmc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 519
    },
    "id": "8dZ-UBSeG9RG",
    "outputId": "205bbe90-c3e1-4bca-b3e7-34a7699d15d8",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender_Female</th>\n",
       "      <th>Gender_Male</th>\n",
       "      <th>Ever_Married_No</th>\n",
       "      <th>Ever_Married_Yes</th>\n",
       "      <th>Graduated_No</th>\n",
       "      <th>Graduated_Yes</th>\n",
       "      <th>Profession_Artist</th>\n",
       "      <th>Profession_Doctor</th>\n",
       "      <th>Profession_Engineer</th>\n",
       "      <th>Profession_Entertainment</th>\n",
       "      <th>...</th>\n",
       "      <th>Var_1_Cat_3</th>\n",
       "      <th>Var_1_Cat_4</th>\n",
       "      <th>Var_1_Cat_5</th>\n",
       "      <th>Var_1_Cat_6</th>\n",
       "      <th>Var_1_Cat_7</th>\n",
       "      <th>Age</th>\n",
       "      <th>Work_Experience</th>\n",
       "      <th>Family_Size</th>\n",
       "      <th>y</th>\n",
       "      <th>difficulty</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>case_index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.012346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.395062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.024691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8063</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.024691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8064</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.012346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8065</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.012346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8066</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8067</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>37</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.407407</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8068 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Gender_Female  Gender_Male  Ever_Married_No  Ever_Married_Yes  \\\n",
       "case_index                                                                  \n",
       "0                       0            1                1                 0   \n",
       "1                       1            0                0                 1   \n",
       "2                       1            0                0                 1   \n",
       "3                       0            1                0                 1   \n",
       "4                       1            0                0                 1   \n",
       "...                   ...          ...              ...               ...   \n",
       "8063                    0            1                1                 0   \n",
       "8064                    0            1                1                 0   \n",
       "8065                    1            0                1                 0   \n",
       "8066                    1            0                1                 0   \n",
       "8067                    0            1                0                 1   \n",
       "\n",
       "            Graduated_No  Graduated_Yes  Profession_Artist  Profession_Doctor  \\\n",
       "case_index                                                                      \n",
       "0                      1              0                  0                  0   \n",
       "1                      0              1                  0                  0   \n",
       "2                      0              1                  0                  0   \n",
       "3                      0              1                  0                  0   \n",
       "4                      0              1                  0                  0   \n",
       "...                  ...            ...                ...                ...   \n",
       "8063                   1              0                  0                  0   \n",
       "8064                   1              0                  0                  0   \n",
       "8065                   0              1                  0                  0   \n",
       "8066                   0              1                  0                  0   \n",
       "8067                   0              1                  0                  0   \n",
       "\n",
       "            Profession_Engineer  Profession_Entertainment  ...  Var_1_Cat_3  \\\n",
       "case_index                                                 ...                \n",
       "0                             0                         0  ...            0   \n",
       "1                             1                         0  ...            0   \n",
       "2                             1                         0  ...            0   \n",
       "3                             0                         0  ...            0   \n",
       "4                             0                         1  ...            0   \n",
       "...                         ...                       ...  ...          ...   \n",
       "8063                          0                         0  ...            0   \n",
       "8064                          0                         0  ...            0   \n",
       "8065                          0                         0  ...            0   \n",
       "8066                          0                         0  ...            0   \n",
       "8067                          0                         0  ...            0   \n",
       "\n",
       "            Var_1_Cat_4  Var_1_Cat_5  Var_1_Cat_6  Var_1_Cat_7  Age  \\\n",
       "case_index                                                            \n",
       "0                     1            0            0            0   22   \n",
       "1                     1            0            0            0   38   \n",
       "2                     0            0            1            0   67   \n",
       "3                     0            0            1            0   67   \n",
       "4                     0            0            1            0   40   \n",
       "...                 ...          ...          ...          ...  ...   \n",
       "8063                  0            0            0            0   22   \n",
       "8064                  1            0            0            0   35   \n",
       "8065                  0            0            1            0   33   \n",
       "8066                  0            0            1            0   27   \n",
       "8067                  1            0            0            0   37   \n",
       "\n",
       "            Work_Experience  Family_Size  y  difficulty  \n",
       "case_index                                               \n",
       "0                       1.0          4.0  3    0.012346  \n",
       "1                       NaN          3.0  0    1.000000  \n",
       "2                       1.0          1.0  1    0.395062  \n",
       "3                       0.0          2.0  1    0.024691  \n",
       "4                       NaN          6.0  0    1.000000  \n",
       "...                     ...          ... ..         ...  \n",
       "8063                    0.0          7.0  3    0.024691  \n",
       "8064                    3.0          4.0  3    0.012346  \n",
       "8065                    1.0          1.0  3    0.012346  \n",
       "8066                    1.0          4.0  1    1.000000  \n",
       "8067                    0.0          3.0  1    0.407407  \n",
       "\n",
       "[8068 rows x 30 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app1_customer = pd.read_excel('../Evaluation/real_world/app1_customer_one_hot.xlsx',\n",
    "                              index_col=[0])\n",
    "app1_customer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "a8K3LCqQslYc",
    "outputId": "a5b35eae-a5ae-4d80-990f-f1760645dbae",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.494775 using {'metric': 'manhattan', 'n_neighbors': 10, 'weights': 'uniform'}\n",
      "Best: 0.508058 using {'C': 0.1, 'penalty': 'l2'}\n",
      "Best: 0.522579 using {'C': 1.0, 'kernel': 'rbf'}\n",
      "Best: 0.528069 using {'max_depth': 10, 'n_estimators': 200}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-07 00:08:35.272088: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:268] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.529663 using {'batch_size': 128, 'epochs': 30}\n",
      "Best: 0.526830 using {'batch_size': 128, 'epochs': 30}\n",
      "KNN\n",
      "Logistic Regression\n",
      "SVM\n",
      "Naive Bayes\n",
      "Random Forest\n",
      "Simpe Neural Network\n",
      "Deep Neural Network\n",
      "     micro_accuracy  micro_sensitivity  micro_specificity  micro_ppv  \\\n",
      "knn           0.743              0.487              0.829      0.487   \n",
      "lr            0.752              0.505              0.835      0.505   \n",
      "svm           0.763              0.526              0.842      0.526   \n",
      "nb            0.747              0.494              0.831      0.494   \n",
      "rf            0.768              0.537              0.846      0.537   \n",
      "snn           0.761              0.523              0.841      0.523   \n",
      "dnn           0.766              0.532              0.844      0.532   \n",
      "\n",
      "     micro_npv  micro_auc  macro_accuracy  macro_sensitivity  \\\n",
      "knn      0.829      0.757           0.743              0.477   \n",
      "lr       0.835      0.776           0.752              0.491   \n",
      "svm      0.842      0.790           0.763              0.513   \n",
      "nb       0.831      0.741           0.747              0.481   \n",
      "rf       0.846      0.800           0.768              0.525   \n",
      "snn      0.841      0.791           0.761              0.511   \n",
      "dnn      0.844      0.797           0.766              0.520   \n",
      "\n",
      "     macro_specificity  macro_ppv  ...  d_micro_specificity  d_micro_ppv  \\\n",
      "knn              0.829      0.474  ...                0.859        0.357   \n",
      "lr               0.835      0.483  ...                0.901        0.326   \n",
      "svm              0.842      0.508  ...                0.911        0.397   \n",
      "nb               0.831      0.471  ...                0.875        0.349   \n",
      "rf               0.845      0.521  ...                0.910        0.438   \n",
      "snn              0.841      0.509  ...                0.902        0.397   \n",
      "dnn              0.844      0.516  ...                0.903        0.428   \n",
      "\n",
      "     d_micro_npv  d_micro_auc  d_macro_accuracy  d_macro_sensitivity  \\\n",
      "knn        0.859        0.760             0.768                0.338   \n",
      "lr         0.901        0.823             0.826                0.315   \n",
      "svm        0.911        0.853             0.843                0.412   \n",
      "nb         0.875        0.768             0.790                0.377   \n",
      "rf         0.910        0.853             0.843                0.418   \n",
      "snn        0.902        0.831             0.829                0.412   \n",
      "dnn        0.903        0.837             0.833                0.427   \n",
      "\n",
      "     d_macro_specificity  d_macro_ppv  d_macro_npv  d_macro_auc  \n",
      "knn                0.855        0.355        0.858        0.764  \n",
      "lr                 0.898        0.343        0.900        0.823  \n",
      "svm                0.909        0.399        0.912        0.856  \n",
      "nb                 0.873        0.351        0.878        0.754  \n",
      "rf                 0.907        0.421        0.910        0.852  \n",
      "snn                0.898        0.406        0.904        0.836  \n",
      "dnn                0.901        0.413        0.905        0.844  \n",
      "\n",
      "[7 rows x 24 columns]\n"
     ]
    }
   ],
   "source": [
    "Customer_evaluation = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                   'Customer_evaluation_App1.xlsx',engine='openpyxl')\n",
    "Customer_detail = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                   'Customer_evaluation_App1_detailed.xlsx',engine='openpyxl')\n",
    "\n",
    "X = app1_customer.iloc[:,:-2]\n",
    "X['difficulty'] = app1_customer['difficulty']\n",
    "y = app1_customer['y']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "X_test_difficulty = X_test['difficulty']\n",
    "X_train = X_train.iloc[:,:-1]\n",
    "X_test = X_test.iloc[:,:-1]\n",
    "\n",
    "X_train = standard(X_train)\n",
    "X_test = standard(X_test)\n",
    "\n",
    "best_knn_model = knn(X_train,y_train)\n",
    "best_lr_model = logistic_regression(X_train,y_train)\n",
    "best_svm_model = svm(X_train, y_train)\n",
    "nb_model = naive_bayes(X_train, y_train)\n",
    "best_rf_model = random_forest(X_train, y_train)\n",
    "\n",
    "if len(np.unique(y_train)) == 2:\n",
    "  best_snn_model = binary_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = binary_neural_network(X_train, y_train,'dnn')\n",
    "elif len(np.unique(y_train)) > 2:\n",
    "  best_snn_model = multiclass_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = multiclass_neural_network(X_train, y_train,'dnn')\n",
    "\n",
    "y_val_pred_knn = best_knn_model.predict_proba(X_test)\n",
    "y_val_pred_lr = best_lr_model.predict_proba(X_test)\n",
    "y_val_pred_svm = best_svm_model.predict_proba(X_test)\n",
    "y_val_pred_nb = nb_model.predict_proba(X_test)\n",
    "y_val_pred_rf = best_rf_model.predict_proba(X_test)\n",
    "y_val_pred_snn = best_snn_model.predict_proba(X_test)\n",
    "y_val_pred_dnn = best_dnn_model.predict_proba(X_test)\n",
    "\n",
    "y_type = type_of_target(y_test)\n",
    "if y_type == 'multiclass':\n",
    "      classes = [0, 1, 2, 3]\n",
    "      true_labels = y_test\n",
    "      weights = X_test_difficulty\n",
    "\n",
    "      print(\"KNN\")\n",
    "      knn_result = multi_evaluation(true_labels, y_val_pred_knn, classes, weights)\n",
    "\n",
    "      print(\"Logistic Regression\")\n",
    "      lr_result = multi_evaluation(true_labels, y_val_pred_lr, classes, weights)\n",
    "\n",
    "      print(\"SVM\")\n",
    "      svm_result = multi_evaluation(true_labels, y_val_pred_svm, classes, weights)\n",
    "\n",
    "      print(\"Naive Bayes\")\n",
    "      nb_result = multi_evaluation(true_labels, y_val_pred_nb, classes, weights)\n",
    "\n",
    "      print(\"Random Forest\")\n",
    "      rf_result = multi_evaluation(true_labels, y_val_pred_rf, classes, weights)\n",
    "\n",
    "      print(\"Simpe Neural Network\")\n",
    "      snn_result = multi_evaluation(true_labels, y_val_pred_snn, classes, weights)\n",
    "\n",
    "      print(\"Deep Neural Network\")\n",
    "      dnn_result = multi_evaluation(true_labels, y_val_pred_dnn, classes, weights)\n",
    "\n",
    "      result_list = [knn_result, lr_result, svm_result, nb_result, rf_result,snn_result,dnn_result]\n",
    "      result = pd.DataFrame(result_list, index=['knn','lr','svm','nb','rf','snn','dnn'],\n",
    "                            columns=['micro_accuracy', 'micro_sensitivity', 'micro_specificity', 'micro_ppv', 'micro_npv', 'micro_auc',\n",
    "                                    'macro_accuracy', 'macro_sensitivity', 'macro_specificity', 'macro_ppv', 'macro_npv', 'macro_auc',\n",
    "                                    'd_micro_accuracy', 'd_micro_sensitivity', 'd_micro_specificity', 'd_micro_ppv', 'd_micro_npv', 'd_micro_auc',\n",
    "                                    'd_macro_accuracy', 'd_macro_sensitivity', 'd_macro_specificity', 'd_macro_ppv', 'd_macro_npv', 'd_macro_auc'])\n",
    "      print(result)\n",
    "\n",
    "sheet_name = 'App1'\n",
    "result.to_excel(Customer_evaluation, sheet_name=sheet_name)\n",
    "\n",
    "predictions = pd.DataFrame({'label': y_test,'difficulty':X_test_difficulty,\n",
    "         'knn': np.argmax(y_val_pred_knn, axis=1),'lr': np.argmax(y_val_pred_lr, axis=1),'svm': np.argmax(y_val_pred_svm, axis=1),\n",
    "         'nb': np.argmax(y_val_pred_nb, axis=1),'rf': np.argmax(y_val_pred_rf, axis=1),'snn':np.argmax(y_val_pred_snn, axis=1),\n",
    "                          'dnn':np.argmax(y_val_pred_dnn, axis=1)})\n",
    "new_df = pd.concat([X_test, predictions], axis=1)\n",
    "new_df.to_excel(Customer_detail, sheet_name=sheet_name)\n",
    "\n",
    "Customer_evaluation.close()\n",
    "Customer_detail.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uakslNSHp743",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## CDdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 661
    },
    "id": "g0ysCYVKJ2R0",
    "outputId": "cda60eb9-e86c-4940-f067-a5e492c18f96",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender_Female</th>\n",
       "      <th>Gender_Male</th>\n",
       "      <th>Ever_Married_No</th>\n",
       "      <th>Ever_Married_Yes</th>\n",
       "      <th>Graduated_No</th>\n",
       "      <th>Graduated_Yes</th>\n",
       "      <th>Profession_Artist</th>\n",
       "      <th>Profession_Doctor</th>\n",
       "      <th>Profession_Engineer</th>\n",
       "      <th>Profession_Entertainment</th>\n",
       "      <th>...</th>\n",
       "      <th>Var_1_Cat_3</th>\n",
       "      <th>Var_1_Cat_4</th>\n",
       "      <th>Var_1_Cat_5</th>\n",
       "      <th>Var_1_Cat_6</th>\n",
       "      <th>Var_1_Cat_7</th>\n",
       "      <th>Age</th>\n",
       "      <th>Work_Experience</th>\n",
       "      <th>Family_Size</th>\n",
       "      <th>label</th>\n",
       "      <th>difficulty</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3.576279e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3</td>\n",
       "      <td>5.083613e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>9.298325e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>7.616209e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>56</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3.303696e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8063</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.890299e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8064</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.637352e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8065</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>85</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>6.728786e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8066</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>65</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5.811445e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8067</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>41</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.122628e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8068 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Gender_Female  Gender_Male  Ever_Married_No  Ever_Married_Yes  \\\n",
       "0                 0            1                1                 0   \n",
       "1                 1            0                1                 0   \n",
       "2                 1            0                1                 0   \n",
       "3                 1            0                0                 0   \n",
       "4                 0            1                0                 1   \n",
       "...             ...          ...              ...               ...   \n",
       "8063              0            1                0                 1   \n",
       "8064              0            1                1                 0   \n",
       "8065              0            1                0                 1   \n",
       "8066              0            1                0                 1   \n",
       "8067              0            1                0                 1   \n",
       "\n",
       "      Graduated_No  Graduated_Yes  Profession_Artist  Profession_Doctor  \\\n",
       "0                1              0                  0                  0   \n",
       "1                0              1                  0                  0   \n",
       "2                1              0                  0                  0   \n",
       "3                1              0                  0                  0   \n",
       "4                1              0                  1                  0   \n",
       "...            ...            ...                ...                ...   \n",
       "8063             1              0                  0                  0   \n",
       "8064             1              0                  0                  0   \n",
       "8065             1              0                  0                  0   \n",
       "8066             1              0                  1                  0   \n",
       "8067             0              1                  1                  0   \n",
       "\n",
       "      Profession_Engineer  Profession_Entertainment  ...  Var_1_Cat_3  \\\n",
       "0                       0                         0  ...            0   \n",
       "1                       0                         0  ...            0   \n",
       "2                       0                         0  ...            0   \n",
       "3                       0                         0  ...            1   \n",
       "4                       0                         0  ...            0   \n",
       "...                   ...                       ...  ...          ...   \n",
       "8063                    0                         0  ...            0   \n",
       "8064                    0                         0  ...            0   \n",
       "8065                    0                         0  ...            0   \n",
       "8066                    0                         0  ...            0   \n",
       "8067                    0                         0  ...            0   \n",
       "\n",
       "      Var_1_Cat_4  Var_1_Cat_5  Var_1_Cat_6  Var_1_Cat_7  Age  \\\n",
       "0               1            0            0            0   22   \n",
       "1               0            0            1            0   33   \n",
       "2               0            0            1            0   18   \n",
       "3               0            0            0            0   58   \n",
       "4               0            0            1            0   56   \n",
       "...           ...          ...          ...          ...  ...   \n",
       "8063            1            0            0            0   39   \n",
       "8064            0            0            0            0   23   \n",
       "8065            0            0            1            0   85   \n",
       "8066            0            0            1            0   65   \n",
       "8067            0            0            1            0   41   \n",
       "\n",
       "      Work_Experience  Family_Size  label    difficulty  \n",
       "0                 1.0          4.0      3  3.576279e-07  \n",
       "1                 1.0          3.0      3  5.083613e-01  \n",
       "2                 3.0          4.0      3  9.298325e-06  \n",
       "3                 1.0          3.0      1  7.616209e-01  \n",
       "4                 1.0          3.0      2  3.303696e-01  \n",
       "...               ...          ...    ...           ...  \n",
       "8063              8.0          4.0      0  6.890299e-01  \n",
       "8064              1.0          3.0      3  1.637352e-02  \n",
       "8065              1.0          1.0      3  6.728786e-01  \n",
       "8066              0.0          2.0      2  5.811445e-01  \n",
       "8067              0.0          5.0      1  5.122628e-01  \n",
       "\n",
       "[8068 rows x 30 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app2_customer = pd.read_excel('../Evaluation/real_world/app2_customer_one_hot.xlsx',\n",
    "                              index_col=[0])\n",
    "app2_customer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8Mmka2hRKCh6",
    "outputId": "a45e1a0f-38ef-40bf-cb72-ed2ae7575cce",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.491058 using {'metric': 'euclidean', 'n_neighbors': 10, 'weights': 'uniform'}\n",
      "Best: 0.508235 using {'C': 10, 'penalty': 'l2'}\n",
      "Best: 0.525413 using {'C': 1.0, 'kernel': 'rbf'}\n",
      "Best: 0.525059 using {'max_depth': 10, 'n_estimators': 200}\n",
      "Best: 0.523465 using {'batch_size': 128, 'epochs': 30}\n",
      "Best: 0.528778 using {'batch_size': 64, 'epochs': 30}\n",
      "KNN\n",
      "Logistic Regression\n",
      "SVM\n",
      "Naive Bayes\n",
      "Random Forest\n",
      "Simpe Neural Network\n",
      "Deep Neural Network\n",
      "     micro_accuracy  micro_sensitivity  micro_specificity  micro_ppv  \\\n",
      "knn           0.746              0.493              0.831      0.493   \n",
      "lr            0.752              0.504              0.835      0.504   \n",
      "svm           0.762              0.525              0.842      0.525   \n",
      "nb            0.738              0.475              0.825      0.475   \n",
      "rf            0.767              0.534              0.845      0.534   \n",
      "snn           0.759              0.519              0.840      0.519   \n",
      "dnn           0.761              0.523              0.841      0.523   \n",
      "\n",
      "     micro_npv  micro_auc  macro_accuracy  macro_sensitivity  \\\n",
      "knn      0.831      0.754           0.746              0.484   \n",
      "lr       0.835      0.769           0.752              0.491   \n",
      "svm      0.842      0.785           0.762              0.512   \n",
      "nb       0.825      0.728           0.738              0.462   \n",
      "rf       0.845      0.792           0.767              0.522   \n",
      "snn      0.840      0.790           0.759              0.506   \n",
      "dnn      0.841      0.791           0.761              0.511   \n",
      "\n",
      "     macro_specificity  macro_ppv  ...  d_micro_specificity  d_micro_ppv  \\\n",
      "knn              0.832      0.487  ...                0.864        0.529   \n",
      "lr               0.835      0.483  ...                0.866        0.541   \n",
      "svm              0.842      0.509  ...                0.873        0.565   \n",
      "nb               0.825      0.453  ...                0.858        0.509   \n",
      "rf               0.845      0.519  ...                0.876        0.575   \n",
      "snn              0.840      0.499  ...                0.872        0.558   \n",
      "dnn              0.841      0.505  ...                0.873        0.563   \n",
      "\n",
      "     d_micro_npv  d_micro_auc  d_macro_accuracy  d_macro_sensitivity  \\\n",
      "knn        0.864        0.793             0.788                0.531   \n",
      "lr         0.866        0.805             0.792                0.541   \n",
      "svm        0.873        0.823             0.803                0.566   \n",
      "nb         0.858        0.771             0.779                0.506   \n",
      "rf         0.876        0.827             0.807                0.577   \n",
      "snn        0.872        0.824             0.801                0.561   \n",
      "dnn        0.873        0.825             0.803                0.566   \n",
      "\n",
      "     d_macro_specificity  d_macro_ppv  d_macro_npv  d_macro_auc  \n",
      "knn                0.863        0.533        0.863        0.787  \n",
      "lr                 0.866        0.540        0.867        0.798  \n",
      "svm                0.873        0.564        0.873        0.816  \n",
      "nb                 0.857        0.508        0.861        0.762  \n",
      "rf                 0.876        0.575        0.876        0.820  \n",
      "snn                0.872        0.559        0.873        0.820  \n",
      "dnn                0.873        0.562        0.873        0.819  \n",
      "\n",
      "[7 rows x 24 columns]\n"
     ]
    }
   ],
   "source": [
    "Customer_evaluation = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                   'Customer_evaluation_App2.xlsx',engine='openpyxl')\n",
    "Customer_detail = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                   'Customer_evaluation_App2_detailed.xlsx',engine='openpyxl')\n",
    "\n",
    "X = app2_customer.iloc[:,:-2]\n",
    "X['difficulty'] = app2_customer['difficulty']\n",
    "y = app2_customer['label']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "X_test_difficulty = X_test['difficulty']\n",
    "X_train = X_train.iloc[:,:-1]\n",
    "X_test = X_test.iloc[:,:-1]\n",
    "\n",
    "X_train = standard(X_train)\n",
    "X_test = standard(X_test)\n",
    "\n",
    "best_knn_model = knn(X_train,y_train)\n",
    "best_lr_model = logistic_regression(X_train,y_train)\n",
    "best_svm_model = svm(X_train, y_train)\n",
    "nb_model = naive_bayes(X_train, y_train)\n",
    "best_rf_model = random_forest(X_train, y_train)\n",
    "\n",
    "if len(np.unique(y_train)) == 2:\n",
    "  best_snn_model = binary_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = binary_neural_network(X_train, y_train,'dnn')\n",
    "elif len(np.unique(y_train)) > 2:\n",
    "  best_snn_model = multiclass_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = multiclass_neural_network(X_train, y_train,'dnn')\n",
    "\n",
    "y_val_pred_knn = best_knn_model.predict_proba(X_test)\n",
    "y_val_pred_lr = best_lr_model.predict_proba(X_test)\n",
    "y_val_pred_svm = best_svm_model.predict_proba(X_test)\n",
    "y_val_pred_nb = nb_model.predict_proba(X_test)\n",
    "y_val_pred_rf = best_rf_model.predict_proba(X_test)\n",
    "y_val_pred_snn = best_snn_model.predict_proba(X_test)\n",
    "y_val_pred_dnn = best_dnn_model.predict_proba(X_test)\n",
    "\n",
    "y_type = type_of_target(y_test)\n",
    "if y_type == 'multiclass':\n",
    "      classes = [0, 1, 2, 3]\n",
    "      true_labels = y_test\n",
    "      weights = X_test_difficulty\n",
    "\n",
    "      print(\"KNN\")\n",
    "      knn_result = multi_evaluation(true_labels, y_val_pred_knn, classes, weights)\n",
    "\n",
    "      print(\"Logistic Regression\")\n",
    "      lr_result = multi_evaluation(true_labels, y_val_pred_lr, classes, weights)\n",
    "\n",
    "      print(\"SVM\")\n",
    "      svm_result = multi_evaluation(true_labels, y_val_pred_svm, classes, weights)\n",
    "\n",
    "      print(\"Naive Bayes\")\n",
    "      nb_result = multi_evaluation(true_labels, y_val_pred_nb, classes, weights)\n",
    "\n",
    "      print(\"Random Forest\")\n",
    "      rf_result = multi_evaluation(true_labels, y_val_pred_rf, classes, weights)\n",
    "\n",
    "      print(\"Simpe Neural Network\")\n",
    "      snn_result = multi_evaluation(true_labels, y_val_pred_snn, classes, weights)\n",
    "\n",
    "      print(\"Deep Neural Network\")\n",
    "      dnn_result = multi_evaluation(true_labels, y_val_pred_dnn, classes, weights)\n",
    "\n",
    "      result_list = [knn_result, lr_result, svm_result, nb_result, rf_result,snn_result,dnn_result]\n",
    "      result = pd.DataFrame(result_list, index=['knn','lr','svm','nb','rf','snn','dnn'],\n",
    "                            columns=['micro_accuracy', 'micro_sensitivity', 'micro_specificity', 'micro_ppv', 'micro_npv', 'micro_auc',\n",
    "                                    'macro_accuracy', 'macro_sensitivity', 'macro_specificity', 'macro_ppv', 'macro_npv', 'macro_auc',\n",
    "                                    'd_micro_accuracy', 'd_micro_sensitivity', 'd_micro_specificity', 'd_micro_ppv', 'd_micro_npv', 'd_micro_auc',\n",
    "                                    'd_macro_accuracy', 'd_macro_sensitivity', 'd_macro_specificity', 'd_macro_ppv', 'd_macro_npv', 'd_macro_auc'])\n",
    "      print(result)\n",
    "\n",
    "sheet_name = 'App2'\n",
    "result.to_excel(Customer_evaluation, sheet_name=sheet_name)\n",
    "\n",
    "predictions = pd.DataFrame({'label': y_test,'difficulty':X_test_difficulty,\n",
    "         'knn': np.argmax(y_val_pred_knn, axis=1),'lr': np.argmax(y_val_pred_lr, axis=1),'svm': np.argmax(y_val_pred_svm, axis=1),\n",
    "         'nb': np.argmax(y_val_pred_nb, axis=1),'rf': np.argmax(y_val_pred_rf, axis=1),'snn':np.argmax(y_val_pred_snn, axis=1),\n",
    "                          'dnn':np.argmax(y_val_pred_dnn, axis=1)})\n",
    "new_df = pd.concat([X_test, predictions], axis=1)\n",
    "new_df.to_excel(Customer_detail, sheet_name=sheet_name)\n",
    "\n",
    "Customer_evaluation.close()\n",
    "Customer_detail.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PT8J9g86LUSS",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## CDpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 487
    },
    "id": "yCl3CA-XLXXh",
    "outputId": "ed2a3c67-ec4e-4ee2-c8ba-5b916d17ca56",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender_Female</th>\n",
       "      <th>Gender_Male</th>\n",
       "      <th>Ever_Married_No</th>\n",
       "      <th>Ever_Married_Yes</th>\n",
       "      <th>Graduated_No</th>\n",
       "      <th>Graduated_Yes</th>\n",
       "      <th>Profession_Artist</th>\n",
       "      <th>Profession_Doctor</th>\n",
       "      <th>Profession_Engineer</th>\n",
       "      <th>Profession_Entertainment</th>\n",
       "      <th>...</th>\n",
       "      <th>Var_1_Cat_3</th>\n",
       "      <th>Var_1_Cat_4</th>\n",
       "      <th>Var_1_Cat_5</th>\n",
       "      <th>Var_1_Cat_6</th>\n",
       "      <th>Var_1_Cat_7</th>\n",
       "      <th>Age</th>\n",
       "      <th>Work_Experience</th>\n",
       "      <th>Family_Size</th>\n",
       "      <th>label</th>\n",
       "      <th>difficulty</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.063331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.327182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.287890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.273518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.358977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8063</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.306748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8064</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.255205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8065</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.229414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8066</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.308741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8067</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>37</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.310926</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8068 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Gender_Female  Gender_Male  Ever_Married_No  Ever_Married_Yes  \\\n",
       "0                 0            1                1                 0   \n",
       "1                 1            0                0                 1   \n",
       "2                 1            0                0                 1   \n",
       "3                 0            1                0                 1   \n",
       "4                 1            0                0                 1   \n",
       "...             ...          ...              ...               ...   \n",
       "8063              0            1                1                 0   \n",
       "8064              0            1                1                 0   \n",
       "8065              1            0                1                 0   \n",
       "8066              1            0                1                 0   \n",
       "8067              0            1                0                 1   \n",
       "\n",
       "      Graduated_No  Graduated_Yes  Profession_Artist  Profession_Doctor  \\\n",
       "0                1              0                  0                  0   \n",
       "1                0              1                  0                  0   \n",
       "2                0              1                  0                  0   \n",
       "3                0              1                  0                  0   \n",
       "4                0              1                  0                  0   \n",
       "...            ...            ...                ...                ...   \n",
       "8063             1              0                  0                  0   \n",
       "8064             1              0                  0                  0   \n",
       "8065             0              1                  0                  0   \n",
       "8066             0              1                  0                  0   \n",
       "8067             0              1                  0                  0   \n",
       "\n",
       "      Profession_Engineer  Profession_Entertainment  ...  Var_1_Cat_3  \\\n",
       "0                       0                         0  ...            0   \n",
       "1                       1                         0  ...            0   \n",
       "2                       1                         0  ...            0   \n",
       "3                       0                         0  ...            0   \n",
       "4                       0                         1  ...            0   \n",
       "...                   ...                       ...  ...          ...   \n",
       "8063                    0                         0  ...            0   \n",
       "8064                    0                         0  ...            0   \n",
       "8065                    0                         0  ...            0   \n",
       "8066                    0                         0  ...            0   \n",
       "8067                    0                         0  ...            0   \n",
       "\n",
       "      Var_1_Cat_4  Var_1_Cat_5  Var_1_Cat_6  Var_1_Cat_7  Age  \\\n",
       "0               1            0            0            0   22   \n",
       "1               1            0            0            0   38   \n",
       "2               0            0            1            0   67   \n",
       "3               0            0            1            0   67   \n",
       "4               0            0            1            0   40   \n",
       "...           ...          ...          ...          ...  ...   \n",
       "8063            0            0            0            0   22   \n",
       "8064            1            0            0            0   35   \n",
       "8065            0            0            1            0   33   \n",
       "8066            0            0            1            0   27   \n",
       "8067            1            0            0            0   37   \n",
       "\n",
       "      Work_Experience  Family_Size  label  difficulty  \n",
       "0                 1.0          4.0      3    0.063331  \n",
       "1                 NaN          3.0      0    0.327182  \n",
       "2                 1.0          1.0      1    0.287890  \n",
       "3                 0.0          2.0      1    0.273518  \n",
       "4                 NaN          6.0      0    0.358977  \n",
       "...               ...          ...    ...         ...  \n",
       "8063              0.0          7.0      3    0.306748  \n",
       "8064              3.0          4.0      3    0.255205  \n",
       "8065              1.0          1.0      3    0.229414  \n",
       "8066              1.0          4.0      1    0.308741  \n",
       "8067              0.0          3.0      1    0.310926  \n",
       "\n",
       "[8068 rows x 30 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app3_customer = pd.read_excel('../Evaluation/real_world/app3_customer_one_hot.xlsx',\n",
    "                              index_col=[0])\n",
    "app3_customer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GNpWBcifLrnz",
    "outputId": "a39b6292-d1ab-4d91-b114-6630b175bee3",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.494775 using {'metric': 'manhattan', 'n_neighbors': 10, 'weights': 'uniform'}\n",
      "Best: 0.508058 using {'C': 0.1, 'penalty': 'l2'}\n",
      "Best: 0.522579 using {'C': 1.0, 'kernel': 'rbf'}\n",
      "Best: 0.528069 using {'max_depth': 10, 'n_estimators': 200}\n",
      "Best: 0.527185 using {'batch_size': 128, 'epochs': 30}\n",
      "Best: 0.525059 using {'batch_size': 64, 'epochs': 10}\n",
      "KNN\n",
      "Logistic Regression\n",
      "SVM\n",
      "Naive Bayes\n",
      "Random Forest\n",
      "Simpe Neural Network\n",
      "Deep Neural Network\n",
      "     micro_accuracy  micro_sensitivity  micro_specificity  micro_ppv  \\\n",
      "knn           0.743              0.487              0.829      0.487   \n",
      "lr            0.752              0.505              0.835      0.505   \n",
      "svm           0.764              0.527              0.842      0.527   \n",
      "nb            0.747              0.494              0.831      0.494   \n",
      "rf            0.768              0.535              0.845      0.535   \n",
      "snn           0.768              0.535              0.845      0.535   \n",
      "dnn           0.770              0.539              0.846      0.539   \n",
      "\n",
      "     micro_npv  micro_auc  macro_accuracy  macro_sensitivity  \\\n",
      "knn      0.829      0.757           0.743              0.477   \n",
      "lr       0.835      0.776           0.752              0.491   \n",
      "svm      0.842      0.790           0.764              0.514   \n",
      "nb       0.831      0.741           0.747              0.481   \n",
      "rf       0.845      0.799           0.768              0.523   \n",
      "snn      0.845      0.796           0.768              0.523   \n",
      "dnn      0.846      0.797           0.770              0.527   \n",
      "\n",
      "     macro_specificity  macro_ppv  ...  d_micro_specificity  d_micro_ppv  \\\n",
      "knn              0.829      0.474  ...                0.628        0.218   \n",
      "lr               0.835      0.483  ...                0.639        0.230   \n",
      "svm              0.842      0.509  ...                0.652        0.250   \n",
      "nb               0.831      0.471  ...                0.632        0.224   \n",
      "rf               0.845      0.518  ...                0.657        0.257   \n",
      "snn              0.845      0.517  ...                0.657        0.257   \n",
      "dnn              0.846      0.520  ...                0.660        0.260   \n",
      "\n",
      "     d_micro_npv  d_micro_auc  d_macro_accuracy  d_macro_sensitivity  \\\n",
      "knn        0.628        0.410             0.499                0.224   \n",
      "lr         0.639        0.428             0.511                0.248   \n",
      "svm        0.652        0.452             0.527                0.267   \n",
      "nb         0.632        0.399             0.503                0.254   \n",
      "rf         0.657        0.459             0.533                0.272   \n",
      "snn        0.657        0.456             0.533                0.276   \n",
      "dnn        0.660        0.457             0.535                0.283   \n",
      "\n",
      "     d_macro_specificity  d_macro_ppv  d_macro_npv  d_macro_auc  \n",
      "knn                0.630        0.221        0.630        0.413  \n",
      "lr                 0.643        0.229        0.646        0.428  \n",
      "svm                0.652        0.247        0.659        0.449  \n",
      "nb                 0.642        0.226        0.646        0.403  \n",
      "rf                 0.657        0.254        0.663        0.460  \n",
      "snn                0.658        0.255        0.665        0.457  \n",
      "dnn                0.660        0.256        0.669        0.460  \n",
      "\n",
      "[7 rows x 24 columns]\n"
     ]
    }
   ],
   "source": [
    "Customer_evaluation = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                   'Customer_evaluation_App3.xlsx',engine='openpyxl')\n",
    "Customer_detail = pd.ExcelWriter('../Evaluation/outcome/'+\n",
    "                                   'Customer_evaluation_App3_detailed.xlsx',engine='openpyxl')\n",
    "\n",
    "X = app3_customer.iloc[:,:-2]\n",
    "X['difficulty'] = app3_customer['difficulty']\n",
    "y = app3_customer['label']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "X_test_difficulty = X_test['difficulty']\n",
    "X_train = X_train.iloc[:,:-1]\n",
    "X_test = X_test.iloc[:,:-1]\n",
    "\n",
    "X_train = standard(X_train)\n",
    "X_test = standard(X_test)\n",
    "\n",
    "best_knn_model = knn(X_train,y_train)\n",
    "best_lr_model = logistic_regression(X_train,y_train)\n",
    "best_svm_model = svm(X_train, y_train)\n",
    "nb_model = naive_bayes(X_train, y_train)\n",
    "best_rf_model = random_forest(X_train, y_train)\n",
    "\n",
    "if len(np.unique(y_train)) == 2:\n",
    "  best_snn_model = binary_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = binary_neural_network(X_train, y_train,'dnn')\n",
    "elif len(np.unique(y_train)) > 2:\n",
    "  best_snn_model = multiclass_neural_network(X_train, y_train,'snn')\n",
    "  best_dnn_model = multiclass_neural_network(X_train, y_train,'dnn')\n",
    "\n",
    "y_val_pred_knn = best_knn_model.predict_proba(X_test)\n",
    "y_val_pred_lr = best_lr_model.predict_proba(X_test)\n",
    "y_val_pred_svm = best_svm_model.predict_proba(X_test)\n",
    "y_val_pred_nb = nb_model.predict_proba(X_test)\n",
    "y_val_pred_rf = best_rf_model.predict_proba(X_test)\n",
    "y_val_pred_snn = best_snn_model.predict_proba(X_test)\n",
    "y_val_pred_dnn = best_dnn_model.predict_proba(X_test)\n",
    "\n",
    "y_type = type_of_target(y_test)\n",
    "if y_type == 'multiclass':\n",
    "      classes = [0, 1, 2, 3]\n",
    "      true_labels = y_test\n",
    "      weights = X_test_difficulty\n",
    "\n",
    "      print(\"KNN\")\n",
    "      knn_result = multi_evaluation(true_labels, y_val_pred_knn, classes, weights)\n",
    "\n",
    "      print(\"Logistic Regression\")\n",
    "      lr_result = multi_evaluation(true_labels, y_val_pred_lr, classes, weights)\n",
    "\n",
    "      print(\"SVM\")\n",
    "      svm_result = multi_evaluation(true_labels, y_val_pred_svm, classes, weights)\n",
    "\n",
    "      print(\"Naive Bayes\")\n",
    "      nb_result = multi_evaluation(true_labels, y_val_pred_nb, classes, weights)\n",
    "\n",
    "      print(\"Random Forest\")\n",
    "      rf_result = multi_evaluation(true_labels, y_val_pred_rf, classes, weights)\n",
    "\n",
    "      print(\"Simpe Neural Network\")\n",
    "      snn_result = multi_evaluation(true_labels, y_val_pred_snn, classes, weights)\n",
    "\n",
    "      print(\"Deep Neural Network\")\n",
    "      dnn_result = multi_evaluation(true_labels, y_val_pred_dnn, classes, weights)\n",
    "\n",
    "      result_list = [knn_result, lr_result, svm_result, nb_result, rf_result,snn_result,dnn_result]\n",
    "      result = pd.DataFrame(result_list, index=['knn','lr','svm','nb','rf','snn','dnn'],\n",
    "                            columns=['micro_accuracy', 'micro_sensitivity', 'micro_specificity', 'micro_ppv', 'micro_npv', 'micro_auc',\n",
    "                                    'macro_accuracy', 'macro_sensitivity', 'macro_specificity', 'macro_ppv', 'macro_npv', 'macro_auc',\n",
    "                                    'd_micro_accuracy', 'd_micro_sensitivity', 'd_micro_specificity', 'd_micro_ppv', 'd_micro_npv', 'd_micro_auc',\n",
    "                                    'd_macro_accuracy', 'd_macro_sensitivity', 'd_macro_specificity', 'd_macro_ppv', 'd_macro_npv', 'd_macro_auc'])\n",
    "      print(result)\n",
    "\n",
    "sheet_name = 'App3'\n",
    "result.to_excel(Customer_evaluation, sheet_name=sheet_name)\n",
    "\n",
    "predictions = pd.DataFrame({'label': y_test,'difficulty':X_test_difficulty,\n",
    "         'knn': np.argmax(y_val_pred_knn, axis=1),'lr': np.argmax(y_val_pred_lr, axis=1),'svm': np.argmax(y_val_pred_svm, axis=1),\n",
    "         'nb': np.argmax(y_val_pred_nb, axis=1),'rf': np.argmax(y_val_pred_rf, axis=1),'snn':np.argmax(y_val_pred_snn, axis=1),\n",
    "                          'dnn':np.argmax(y_val_pred_dnn, axis=1)})\n",
    "new_df = pd.concat([X_test, predictions], axis=1)\n",
    "new_df.to_excel(Customer_detail, sheet_name=sheet_name)\n",
    "\n",
    "Customer_evaluation.close()\n",
    "Customer_detail.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XsOrCos4WU4c",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}