CCC_Default.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CCC-Default.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "x6A3vMVvZDrr"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import statsmodels.api as sm\n",
        "from joblib import dump, load\n",
        "from imblearn.over_sampling import SMOTE\n",
        "from sklearn import preprocessing\n",
        "from sklearn.metrics import (confusion_matrix,accuracy_score,jaccard_score,f1_score,log_loss,classification_report)\n",
        "from sklearn.model_selection import train_test_split\n",
        "BaseDat = pd.read_csv(\"/content/default-1.csv\")\n",
        "dest = '/content/'\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0oOJZ_dvaZaG"
      },
      "source": [
        "print(BaseDat.head)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bO_iaaXXI_60"
      },
      "source": [
        "drop irrelevant vars and create dummy variables from categorical variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zg5nxRWFJAFv"
      },
      "source": [
        "print(BaseDat.columns)\n",
        "#- categorical Vars change -#\n",
        "print(BaseDat['SEX'].value_counts())\n",
        "print(BaseDat['EDUCATION'].value_counts())\n",
        "print(BaseDat['MARRIAGE'].value_counts())\n",
        "BaseDat['SEX'] = BaseDat['SEX'].replace(1,'Male')\n",
        "BaseDat['SEX'] = BaseDat['SEX'].replace(2,'FeMale')\n",
        "BaseDat['EDUCATION'] = BaseDat['EDUCATION'].replace(1,'Grad_School')\n",
        "BaseDat['EDUCATION'] = BaseDat['EDUCATION'].replace(2,'University')\n",
        "BaseDat['EDUCATION'] = BaseDat['EDUCATION'].replace(3,'High_School')\n",
        "BaseDat['EDUCATION'] = BaseDat['EDUCATION'].replace([0,4,5,6],'Educ_Others')\n",
        "BaseDat['MARRIAGE'] = BaseDat['MARRIAGE'].replace(1,'Married')\n",
        "BaseDat['MARRIAGE'] = BaseDat['MARRIAGE'].replace(2,'Single')\n",
        "BaseDat['MARRIAGE'] = BaseDat['MARRIAGE'].replace([0,3],'Marriage_Others')\n",
        "BaseDat = pd.concat((BaseDat,pd.get_dummies(BaseDat['SEX'])), axis=1)\n",
        "BaseDat = pd.concat((BaseDat,pd.get_dummies(BaseDat['EDUCATION'])), axis=1)\n",
        "BaseDat = pd.concat((BaseDat,pd.get_dummies(BaseDat['MARRIAGE'])), axis=1)\n",
        "#- add age Square -#\n",
        "BaseDat['Age_Sqr'] = BaseDat['AGE'] ** 2\n",
        "#- drop irrelevant -#\n",
        "BaseDat = BaseDat.drop(['ID','SEX','EDUCATION','MARRIAGE'],axis=1)\n",
        "print(BaseDat.columns)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6QVjv0xNXmzf"
      },
      "source": [
        "## Step 1 – descriptive statistics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JQh0YluxXm7F"
      },
      "source": [
        "#- means of features by default -#\n",
        "Tab = pd.DataFrame(BaseDat.groupby(['default']).mean())\n",
        "Tab.to_excel(dest + 'Desc-Stat.xlsx')\n",
        "#- correlation matrix -#\n",
        "pd.DataFrame(BaseDat.corr()).to_excel(dest + 'CorrMatrix.xlsx')\n",
        "#- simple ols regression -#\n",
        "X = BaseDat.drop(['default','Male','Educ_Others','Marriage_Others'],axis=1)\n",
        "Ols = sm.OLS(BaseDat['default'], X).fit()\n",
        "print(Ols.summary())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swlWvlIDJ1Sz"
      },
      "source": [
        "Credit Card Defualt is relativly rare so we will use SMOTE to resample it equaly"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dGZHVD9Hb2vH"
      },
      "source": [
        "y = BaseDat.loc[:, BaseDat.columns == 'default']\n",
        "X = BaseDat.drop(['default'],axis=1)\n",
        "#- convert to arrays -#\n",
        "X = np.asarray(X)\n",
        "y = np.asarray(y)\n",
        "#- standardizing the features and spliting for train/test -#\n",
        "X = preprocessing.StandardScaler().fit(X).transform(X)\n",
        "print(X.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ygKCa7-m3No"
      },
      "source": [
        "spliting for train/test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cit73Q5km6eG"
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.7, random_state=0)\n",
        "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aojdIfdspe2u"
      },
      "source": [
        "## Step 2 – fitting 7 different ML algorithms"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XlE7kYk94v90"
      },
      "source": [
        "**1. K Nearest Neighbor (KNN)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SIlNIZcZcpQ1"
      },
      "source": [
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "\n",
        "for n in range(1, 6):\n",
        "    # Train Model\n",
        "    KNN_Model = KNeighborsClassifier(n_neighbors=n, n_jobs=-1).fit(X_train, y_train.ravel())\n",
        "   # save the model for later exploration\n",
        "    FName = str(dest + 'KNN_' + str(n) + '.joblib')\n",
        "    dump(KNN_Model, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cJTUo014-dNc"
      },
      "source": [
        "**2. Random Forest Classifier (RFC)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-l_Ds6C-ca-"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "RFC_Model = RandomForestClassifier(random_state=0, n_jobs=-1).fit(X_train, y_train.ravel())\n",
        "FName = str(dest + 'RFC.joblib')\n",
        "dump(RFC_Model, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "48sb_OPe9M34"
      },
      "source": [
        "**3. AdaBoost Classifier**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f-orkVFI9eA9"
      },
      "source": [
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "Adaboost_Model = AdaBoostClassifier(random_state=0).fit(X_train, y_train.ravel())\n",
        "FName = str(dest + 'Adaboost.joblib')\n",
        "dump(Adaboost_Model, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XPV6diS4q21R"
      },
      "source": [
        "**4. Suport Vector Machine (SVM)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sHqdmKXipZ8O"
      },
      "source": [
        "from sklearn import svm\n",
        "SVM_Model = svm.SVC(kernel='rbf').fit(X_train, y_train.ravel())\n",
        "FName = str(dest + 'SVM_rbf.joblib')\n",
        "dump(SVM_Model, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "30wX5nym_gTA"
      },
      "source": [
        "**5. Gaussian naive Bayes**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TTK1Y4Vj_jeO"
      },
      "source": [
        "from sklearn.naive_bayes import GaussianNB\n",
        "Gauss_Model = GaussianNB().fit(X_train, y_train.ravel())\n",
        "FName = str(dest + 'GaussianNB.joblib')\n",
        "dump(Gauss_Model, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VUtykS1C2TP0"
      },
      "source": [
        "**6. Multi-layer Perceptron classifier (MLPC)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iHywQxj92TZZ"
      },
      "source": [
        "from sklearn.neural_network import MLPClassifier\n",
        "MLPC_Model = MLPClassifier(random_state=0).fit(X_train, y_train.ravel())\n",
        "FName = str(dest + 'MLPC.joblib')\n",
        "dump(MLPC_Model, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wTcY7HFw32mo"
      },
      "source": [
        "**7. Logistic Regression CV (LR)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WfRGJWj232tk"
      },
      "source": [
        "from sklearn.linear_model import LogisticRegressionCV\n",
        "LR_Model = LogisticRegressionCV(random_state=0).fit(X_train, y_train.ravel())\n",
        "FName = str(dest + 'LR.joblib')\n",
        "dump(LR_Model, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sX_nKQ6UNiiu"
      },
      "source": [
        "## Step 3 – Reuse of the fitted Models to predict the test data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gwKwK24E7Rjy"
      },
      "source": [
        "# import sys\n",
        "# sys.stdout = open(\"/content/drive/MyDrive/Python/Credit-Card-Default/Models_Comp.txt\", \"w\")\n",
        "Mod_List=['LR','MLPC','GaussianNB','SVM_rbf','Adaboost','RFC','KNN_1','KNN_2','KNN_3','KNN_4','KNN_5']\n",
        "Compare_Table = pd.DataFrame(columns=['Model','Accuracy','TP','FP','FN','TN'])\n",
        "for mod in Mod_List:\n",
        "  Test_Mod = load(dest + mod + '.joblib')\n",
        "  y_pred = Test_Mod.predict(X_test)\n",
        "  Accu = accuracy_score(y_test, y_pred)\n",
        "  Conf_Mat = np.ndarray.tolist(confusion_matrix(y_test, y_pred))\n",
        "  Compare_Table = Compare_Table.append({'Model': mod, 'Accuracy': Accu,\n",
        "                                    'TN':Conf_Mat[0][0],'FP':Conf_Mat[0][1],\n",
        "                                    'FN':Conf_Mat[1][0],'TP':Conf_Mat[1][1]}, ignore_index=True)\n",
        "\n",
        "Compare_Table = Compare_Table.sort_values(by='Accuracy', ascending=False)\n",
        "print(Compare_Table)\n",
        "# sys.stdout.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lz6BzFe01WOr"
      },
      "source": [
        "the Best 3 models are: SVM (82.12%), Adaboost (81.98%), RFC (81.5%)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BJm6kHGFcI_t"
      },
      "source": [
        "## Step 4 – oversample the data With SMOTE and fit the best 3 Alg. from previous step"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vq-jqOFTcziS"
      },
      "source": [
        "y = BaseDat.loc[:, BaseDat.columns == 'default']\n",
        "X = BaseDat.drop(['default'],axis=1)\n",
        "os = SMOTE(random_state=0)\n",
        "X_os,y_os=os.fit_resample(X, y)\n",
        "#- Check the changes of our data distribution and design -#\n",
        "print(\"Original data: Length=\",len(X), \"Percent default=\",len(y[y['default']==1])/len(X))\n",
        "print('OverSampled data: Length=',len(X_os), 'Percent default=',len(y_os[y_os['default']==1])/len(X_os))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fR3EedXCnlFY"
      },
      "source": [
        "#- convert to arrays -#\n",
        "X_os = np.asarray(X_os)\n",
        "y_os = np.asarray(y_os)\n",
        "#- standardizing the features and spliting for train/test -#\n",
        "X_os = preprocessing.StandardScaler().fit(X_os).transform(X_os)\n",
        "#- slpit to train/test -#\n",
        "X_train_os, X_test_os, y_train_os, y_test_os = train_test_split(X_os, y_os, test_size=0.7, random_state=0)\n",
        "print(X_train_os.shape, X_test_os.shape, y_train_os.shape, y_test_os.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u7Zgte27dOKx"
      },
      "source": [
        "from sklearn import svm\n",
        "SVM_Model_os = svm.SVC(kernel='rbf').fit(X_train_os, y_train_os.ravel())\n",
        "FName = str(dest + 'SVM_rbf_os.joblib')\n",
        "dump(SVM_Model_os, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "my-hR-_umvUH"
      },
      "source": [
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "Adaboost_Model_os = AdaBoostClassifier(random_state=0).fit(X_train_os, y_train_os.ravel())\n",
        "FName = str(dest + 'Adaboost_os.joblib')\n",
        "dump(Adaboost_Model_os, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cr--x-4wdOBq"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "RFC_Model_os = RandomForestClassifier(random_state=0, n_jobs=-1).fit(X_train_os, y_train_os.ravel())\n",
        "FName = str(dest + 'RFC_os.joblib')\n",
        "dump(RFC_Model_os, FName)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GyyNVW7mf3nO"
      },
      "source": [
        "## Step 5 – test prediction accuracy after resampling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IS7irFDRf3xC"
      },
      "source": [
        "# import sys\n",
        "# sys.stdout = open(dest + \"Models_Comp_os.txt\", \"w\")\n",
        "Mod_List=['SVM_rbf_os','Adaboost_os','RFC_os']\n",
        "Compare_Table = pd.DataFrame(columns=['Model','Accuracy','TP','FP','FN','TN'])\n",
        "for mod in Mod_List:\n",
        "  Test_Mod = load(dest + mod + '.joblib')\n",
        "  y_pred = Test_Mod.predict(X_test_os)\n",
        "  Accu = accuracy_score(y_test_os, y_pred)\n",
        "  Conf_Mat = np.ndarray.tolist(confusion_matrix(y_test_os, y_pred))\n",
        "  Compare_Table = Compare_Table.append({'Model': mod, 'Accuracy': Accu,\n",
        "                                    'TN':Conf_Mat[0][0],'FP':Conf_Mat[0][1],\n",
        "                                    'FN':Conf_Mat[1][0],'TP':Conf_Mat[1][1]}, ignore_index=True)\n",
        "\n",
        "Compare_Table = Compare_Table.sort_values(by='Accuracy', ascending=False)\n",
        "print(Compare_Table)\n",
        "# sys.stdout.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nUc6xISdoX--"
      },
      "source": [
        "the rate of accuracy for all 3 models improved with resampled data. the best Alg. is RFC (86.13%)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "haQaEeERg45A"
      },
      "source": [
        "## Step 6 – Find best hyper parameters for most accurate alg. from prevoius step"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a337q3QJg5AL"
      },
      "source": [
        "from pprint import pprint\n",
        "#- this will show the default parameters of the algorithm -#\n",
        "pprint(RandomForestClassifier(random_state=0, n_jobs=-1).get_params())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "12rZ2mNbh15g"
      },
      "source": [
        "create a dictionary with lists of hyper parameters to test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pf1MgdNXh2Aa"
      },
      "source": [
        "Param_grid = {'bootstrap': [False,True],\n",
        " 'max_features': ['auto','log2'],              \n",
        " 'criterion': ['gini','entropy'],\n",
        " 'n_estimators': [500,1000],\n",
        " 'warm_start': [False,True]}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f9u7n1zfh_qk"
      },
      "source": [
        "run the grid search algorithm to find the best hyper parameters from the given options in step 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2_Yjvtegh_z9"
      },
      "source": [
        "from sklearn.model_selection import GridSearchCV\n",
        "grid_search = GridSearchCV(estimator = RandomForestClassifier(random_state=0, n_jobs=-1), param_grid = Param_grid, n_jobs = -1)\n",
        "grid_search.fit(X_train_os, y_train_os.ravel())\n",
        "grid_search.best_params_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kMkrOIFbjMMq"
      },
      "source": [
        "test the best hyper parameters on the data\n",
        "\n",
        "note: only the n_estimators and bootstrap changed in compare with the default"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y5Gd8hUIjMTc"
      },
      "source": [
        "from sklearn.metrics import plot_confusion_matrix\n",
        "RFC_Model_BestTest = RandomForestClassifier(bootstrap=False,\n",
        "                                            criterion='gini',\n",
        "                                            max_features='auto',\n",
        "                                            n_estimators=1000,\n",
        "                                            warm_start=False,\n",
        "                                            random_state=0, n_jobs=-1,).fit(X_train_os, y_train_os.ravel())\n",
        "FName = str(dest + 'RFC_BestTest.joblib')\n",
        "dump(RFC_Model_BestTest, FName)\n",
        "y_pred = RFC_Model_BestTest.predict(X_test_os)\n",
        "print(accuracy_score(y_test_os, y_pred))\n",
        "plot_confusion_matrix(RFC_Model_BestTest, X_test_os, y_test_os) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_NBwruqepsF3"
      },
      "source": [
        "after changing the bootstrap and n_estimators hyper parameters the the accuracy of prdection changed slightly to 86.52%"
      ]
    }
  ]
}