Machine_Learning_Classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.5"
    },
    "colab": {
      "name": "Machine_Learning_Classification.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Elgendi/Machine-Learning-and-Data-Viz-Workshop/blob/master/Machine_Learning_Classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x_h1nWAWjjY7"
      },
      "source": [
        "# Copyright (c) Mohamed Elgendi 2020. All rights reserved\n",
        "# Author: Mohamed Elgendi \n",
        "# Version: 1.0 (2020.11.29)\n",
        "# Email: moe.elgendi@gmail.com\n",
        "# Platform: Tested on Windows 10 with Python 3.6"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "21_0sXzZjjZD"
      },
      "source": [
        "from sklearn.datasets import load_breast_cancer"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Wte_8R3jjZD"
      },
      "source": [
        "data = load_breast_cancer()"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_bLRamSFjjZE"
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "id": "WhSFHNIzjjZE",
        "outputId": "22796651-3503-4eba-c578-240a42ae2b11"
      },
      "source": [
        "X = pd.DataFrame(data.data, columns=data.feature_names)\n",
        "X.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean radius</th>\n",
              "      <th>mean texture</th>\n",
              "      <th>mean perimeter</th>\n",
              "      <th>mean area</th>\n",
              "      <th>mean smoothness</th>\n",
              "      <th>mean compactness</th>\n",
              "      <th>mean concavity</th>\n",
              "      <th>mean concave points</th>\n",
              "      <th>mean symmetry</th>\n",
              "      <th>mean fractal dimension</th>\n",
              "      <th>radius error</th>\n",
              "      <th>texture error</th>\n",
              "      <th>perimeter error</th>\n",
              "      <th>area error</th>\n",
              "      <th>smoothness error</th>\n",
              "      <th>compactness error</th>\n",
              "      <th>concavity error</th>\n",
              "      <th>concave points error</th>\n",
              "      <th>symmetry error</th>\n",
              "      <th>fractal dimension error</th>\n",
              "      <th>worst radius</th>\n",
              "      <th>worst texture</th>\n",
              "      <th>worst perimeter</th>\n",
              "      <th>worst area</th>\n",
              "      <th>worst smoothness</th>\n",
              "      <th>worst compactness</th>\n",
              "      <th>worst concavity</th>\n",
              "      <th>worst concave points</th>\n",
              "      <th>worst symmetry</th>\n",
              "      <th>worst fractal dimension</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>17.99</td>\n",
              "      <td>10.38</td>\n",
              "      <td>122.80</td>\n",
              "      <td>1001.0</td>\n",
              "      <td>0.11840</td>\n",
              "      <td>0.27760</td>\n",
              "      <td>0.3001</td>\n",
              "      <td>0.14710</td>\n",
              "      <td>0.2419</td>\n",
              "      <td>0.07871</td>\n",
              "      <td>1.0950</td>\n",
              "      <td>0.9053</td>\n",
              "      <td>8.589</td>\n",
              "      <td>153.40</td>\n",
              "      <td>0.006399</td>\n",
              "      <td>0.04904</td>\n",
              "      <td>0.05373</td>\n",
              "      <td>0.01587</td>\n",
              "      <td>0.03003</td>\n",
              "      <td>0.006193</td>\n",
              "      <td>25.38</td>\n",
              "      <td>17.33</td>\n",
              "      <td>184.60</td>\n",
              "      <td>2019.0</td>\n",
              "      <td>0.1622</td>\n",
              "      <td>0.6656</td>\n",
              "      <td>0.7119</td>\n",
              "      <td>0.2654</td>\n",
              "      <td>0.4601</td>\n",
              "      <td>0.11890</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>20.57</td>\n",
              "      <td>17.77</td>\n",
              "      <td>132.90</td>\n",
              "      <td>1326.0</td>\n",
              "      <td>0.08474</td>\n",
              "      <td>0.07864</td>\n",
              "      <td>0.0869</td>\n",
              "      <td>0.07017</td>\n",
              "      <td>0.1812</td>\n",
              "      <td>0.05667</td>\n",
              "      <td>0.5435</td>\n",
              "      <td>0.7339</td>\n",
              "      <td>3.398</td>\n",
              "      <td>74.08</td>\n",
              "      <td>0.005225</td>\n",
              "      <td>0.01308</td>\n",
              "      <td>0.01860</td>\n",
              "      <td>0.01340</td>\n",
              "      <td>0.01389</td>\n",
              "      <td>0.003532</td>\n",
              "      <td>24.99</td>\n",
              "      <td>23.41</td>\n",
              "      <td>158.80</td>\n",
              "      <td>1956.0</td>\n",
              "      <td>0.1238</td>\n",
              "      <td>0.1866</td>\n",
              "      <td>0.2416</td>\n",
              "      <td>0.1860</td>\n",
              "      <td>0.2750</td>\n",
              "      <td>0.08902</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>19.69</td>\n",
              "      <td>21.25</td>\n",
              "      <td>130.00</td>\n",
              "      <td>1203.0</td>\n",
              "      <td>0.10960</td>\n",
              "      <td>0.15990</td>\n",
              "      <td>0.1974</td>\n",
              "      <td>0.12790</td>\n",
              "      <td>0.2069</td>\n",
              "      <td>0.05999</td>\n",
              "      <td>0.7456</td>\n",
              "      <td>0.7869</td>\n",
              "      <td>4.585</td>\n",
              "      <td>94.03</td>\n",
              "      <td>0.006150</td>\n",
              "      <td>0.04006</td>\n",
              "      <td>0.03832</td>\n",
              "      <td>0.02058</td>\n",
              "      <td>0.02250</td>\n",
              "      <td>0.004571</td>\n",
              "      <td>23.57</td>\n",
              "      <td>25.53</td>\n",
              "      <td>152.50</td>\n",
              "      <td>1709.0</td>\n",
              "      <td>0.1444</td>\n",
              "      <td>0.4245</td>\n",
              "      <td>0.4504</td>\n",
              "      <td>0.2430</td>\n",
              "      <td>0.3613</td>\n",
              "      <td>0.08758</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>11.42</td>\n",
              "      <td>20.38</td>\n",
              "      <td>77.58</td>\n",
              "      <td>386.1</td>\n",
              "      <td>0.14250</td>\n",
              "      <td>0.28390</td>\n",
              "      <td>0.2414</td>\n",
              "      <td>0.10520</td>\n",
              "      <td>0.2597</td>\n",
              "      <td>0.09744</td>\n",
              "      <td>0.4956</td>\n",
              "      <td>1.1560</td>\n",
              "      <td>3.445</td>\n",
              "      <td>27.23</td>\n",
              "      <td>0.009110</td>\n",
              "      <td>0.07458</td>\n",
              "      <td>0.05661</td>\n",
              "      <td>0.01867</td>\n",
              "      <td>0.05963</td>\n",
              "      <td>0.009208</td>\n",
              "      <td>14.91</td>\n",
              "      <td>26.50</td>\n",
              "      <td>98.87</td>\n",
              "      <td>567.7</td>\n",
              "      <td>0.2098</td>\n",
              "      <td>0.8663</td>\n",
              "      <td>0.6869</td>\n",
              "      <td>0.2575</td>\n",
              "      <td>0.6638</td>\n",
              "      <td>0.17300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>20.29</td>\n",
              "      <td>14.34</td>\n",
              "      <td>135.10</td>\n",
              "      <td>1297.0</td>\n",
              "      <td>0.10030</td>\n",
              "      <td>0.13280</td>\n",
              "      <td>0.1980</td>\n",
              "      <td>0.10430</td>\n",
              "      <td>0.1809</td>\n",
              "      <td>0.05883</td>\n",
              "      <td>0.7572</td>\n",
              "      <td>0.7813</td>\n",
              "      <td>5.438</td>\n",
              "      <td>94.44</td>\n",
              "      <td>0.011490</td>\n",
              "      <td>0.02461</td>\n",
              "      <td>0.05688</td>\n",
              "      <td>0.01885</td>\n",
              "      <td>0.01756</td>\n",
              "      <td>0.005115</td>\n",
              "      <td>22.54</td>\n",
              "      <td>16.67</td>\n",
              "      <td>152.20</td>\n",
              "      <td>1575.0</td>\n",
              "      <td>0.1374</td>\n",
              "      <td>0.2050</td>\n",
              "      <td>0.4000</td>\n",
              "      <td>0.1625</td>\n",
              "      <td>0.2364</td>\n",
              "      <td>0.07678</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   mean radius  mean texture  ...  worst symmetry  worst fractal dimension\n",
              "0        17.99         10.38  ...          0.4601                  0.11890\n",
              "1        20.57         17.77  ...          0.2750                  0.08902\n",
              "2        19.69         21.25  ...          0.3613                  0.08758\n",
              "3        11.42         20.38  ...          0.6638                  0.17300\n",
              "4        20.29         14.34  ...          0.2364                  0.07678\n",
              "\n",
              "[5 rows x 30 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "doEjuishjjZF",
        "outputId": "85757a8e-6e5c-4506-aafd-f0d7e44bfb2a"
      },
      "source": [
        "print(data.DESCR)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ".. _breast_cancer_dataset:\n",
            "\n",
            "Breast cancer wisconsin (diagnostic) dataset\n",
            "--------------------------------------------\n",
            "\n",
            "**Data Set Characteristics:**\n",
            "\n",
            "    :Number of Instances: 569\n",
            "\n",
            "    :Number of Attributes: 30 numeric, predictive attributes and the class\n",
            "\n",
            "    :Attribute Information:\n",
            "        - radius (mean of distances from center to points on the perimeter)\n",
            "        - texture (standard deviation of gray-scale values)\n",
            "        - perimeter\n",
            "        - area\n",
            "        - smoothness (local variation in radius lengths)\n",
            "        - compactness (perimeter^2 / area - 1.0)\n",
            "        - concavity (severity of concave portions of the contour)\n",
            "        - concave points (number of concave portions of the contour)\n",
            "        - symmetry \n",
            "        - fractal dimension (\"coastline approximation\" - 1)\n",
            "\n",
            "        The mean, standard error, and \"worst\" or largest (mean of the three\n",
            "        largest values) of these features were computed for each image,\n",
            "        resulting in 30 features.  For instance, field 3 is Mean Radius, field\n",
            "        13 is Radius SE, field 23 is Worst Radius.\n",
            "\n",
            "        - class:\n",
            "                - WDBC-Malignant\n",
            "                - WDBC-Benign\n",
            "\n",
            "    :Summary Statistics:\n",
            "\n",
            "    ===================================== ====== ======\n",
            "                                           Min    Max\n",
            "    ===================================== ====== ======\n",
            "    radius (mean):                        6.981  28.11\n",
            "    texture (mean):                       9.71   39.28\n",
            "    perimeter (mean):                     43.79  188.5\n",
            "    area (mean):                          143.5  2501.0\n",
            "    smoothness (mean):                    0.053  0.163\n",
            "    compactness (mean):                   0.019  0.345\n",
            "    concavity (mean):                     0.0    0.427\n",
            "    concave points (mean):                0.0    0.201\n",
            "    symmetry (mean):                      0.106  0.304\n",
            "    fractal dimension (mean):             0.05   0.097\n",
            "    radius (standard error):              0.112  2.873\n",
            "    texture (standard error):             0.36   4.885\n",
            "    perimeter (standard error):           0.757  21.98\n",
            "    area (standard error):                6.802  542.2\n",
            "    smoothness (standard error):          0.002  0.031\n",
            "    compactness (standard error):         0.002  0.135\n",
            "    concavity (standard error):           0.0    0.396\n",
            "    concave points (standard error):      0.0    0.053\n",
            "    symmetry (standard error):            0.008  0.079\n",
            "    fractal dimension (standard error):   0.001  0.03\n",
            "    radius (worst):                       7.93   36.04\n",
            "    texture (worst):                      12.02  49.54\n",
            "    perimeter (worst):                    50.41  251.2\n",
            "    area (worst):                         185.2  4254.0\n",
            "    smoothness (worst):                   0.071  0.223\n",
            "    compactness (worst):                  0.027  1.058\n",
            "    concavity (worst):                    0.0    1.252\n",
            "    concave points (worst):               0.0    0.291\n",
            "    symmetry (worst):                     0.156  0.664\n",
            "    fractal dimension (worst):            0.055  0.208\n",
            "    ===================================== ====== ======\n",
            "\n",
            "    :Missing Attribute Values: None\n",
            "\n",
            "    :Class Distribution: 212 - Malignant, 357 - Benign\n",
            "\n",
            "    :Creator:  Dr. William H. Wolberg, W. Nick Street, Olvi L. Mangasarian\n",
            "\n",
            "    :Donor: Nick Street\n",
            "\n",
            "    :Date: November, 1995\n",
            "\n",
            "This is a copy of UCI ML Breast Cancer Wisconsin (Diagnostic) datasets.\n",
            "https://goo.gl/U2Uwz2\n",
            "\n",
            "Features are computed from a digitized image of a fine needle\n",
            "aspirate (FNA) of a breast mass.  They describe\n",
            "characteristics of the cell nuclei present in the image.\n",
            "\n",
            "Separating plane described above was obtained using\n",
            "Multisurface Method-Tree (MSM-T) [K. P. Bennett, \"Decision Tree\n",
            "Construction Via Linear Programming.\" Proceedings of the 4th\n",
            "Midwest Artificial Intelligence and Cognitive Science Society,\n",
            "pp. 97-101, 1992], a classification method which uses linear\n",
            "programming to construct a decision tree.  Relevant features\n",
            "were selected using an exhaustive search in the space of 1-4\n",
            "features and 1-3 separating planes.\n",
            "\n",
            "The actual linear program used to obtain the separating plane\n",
            "in the 3-dimensional space is that described in:\n",
            "[K. P. Bennett and O. L. Mangasarian: \"Robust Linear\n",
            "Programming Discrimination of Two Linearly Inseparable Sets\",\n",
            "Optimization Methods and Software 1, 1992, 23-34].\n",
            "\n",
            "This database is also available through the UW CS ftp server:\n",
            "\n",
            "ftp ftp.cs.wisc.edu\n",
            "cd math-prog/cpo-dataset/machine-learn/WDBC/\n",
            "\n",
            ".. topic:: References\n",
            "\n",
            "   - W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction \n",
            "     for breast tumor diagnosis. IS&T/SPIE 1993 International Symposium on \n",
            "     Electronic Imaging: Science and Technology, volume 1905, pages 861-870,\n",
            "     San Jose, CA, 1993.\n",
            "   - O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and \n",
            "     prognosis via linear programming. Operations Research, 43(4), pages 570-577, \n",
            "     July-August 1995.\n",
            "   - W.H. Wolberg, W.N. Street, and O.L. Mangasarian. Machine learning techniques\n",
            "     to diagnose breast cancer from fine-needle aspirates. Cancer Letters 77 (1994) \n",
            "     163-171.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hmrRlY--jjZG",
        "outputId": "f0c2350a-7120-4276-8373-74b28b6dd8c7"
      },
      "source": [
        "y = data.target\n",
        "y"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0,\n",
              "       1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0,\n",
              "       1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0,\n",
              "       0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1,\n",
              "       1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0,\n",
              "       0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0,\n",
              "       1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1,\n",
              "       1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0,\n",
              "       0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0,\n",
              "       0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0,\n",
              "       1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1,\n",
              "       1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1,\n",
              "       1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0,\n",
              "       1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1,\n",
              "       1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1,\n",
              "       1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W60X0QBTjjZH"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y) # default 25 testing 75 training"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iyS7vFWvjjZH"
      },
      "source": [
        "from sklearn import model_selection\n",
        "from sklearn.dummy import DummyClassifier\n",
        "from sklearn.linear_model import (\n",
        "    LogisticRegression,\n",
        ")\n",
        "from sklearn.neighbors import (\n",
        "    KNeighborsClassifier,\n",
        ")\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.ensemble import (\n",
        "    RandomForestClassifier,\n",
        ")\n",
        "import xgboost"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8GqSrO7NjjZH",
        "outputId": "36b32a04-5812-4a7f-97d8-a96311d5e11b"
      },
      "source": [
        "# Create first classifier\n",
        "classifier1 = LogisticRegression()\n",
        "classifier1.fit(X_train, y_train)\n",
        "y_hat1 = classifier1.predict(X_test)\n",
        "y_prob1 = classifier1.predict_proba(X_test)\n",
        "y_prob1=y_prob1[:,1] \n",
        "# Create second classifier\n",
        "classifier2 = RandomForestClassifier()\n",
        "classifier2.fit(X_train, y_train)\n",
        "y_hat2 = classifier2.predict(X_test)\n",
        "y_prob2 = classifier2.predict_proba(X_test)\n",
        "y_prob2=y_prob2[:,1] \n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZf5nzWMjjZI"
      },
      "source": [
        "%matplotlib inline\n",
        "# roc curve and auc on an imbalanced dataset\n",
        "from sklearn.metrics import matthews_corrcoef\n",
        "from sklearn.datasets import make_classification\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import roc_curve\n",
        "from sklearn.metrics import precision_recall_curve\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from matplotlib import pyplot\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import auc\n",
        "from matplotlib import pyplot"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "id": "hCAWkSUIjjZI",
        "outputId": "d83d6ef9-370c-4b2c-84d3-6acbe2d77488"
      },
      "source": [
        "ns_auc = roc_auc_score(y_test, y_prob1)\n",
        "lr_auc = roc_auc_score(y_test, y_prob2)\n",
        "Mcc1=matthews_corrcoef(y_test, y_hat1)\n",
        "Mcc2=matthews_corrcoef(y_test, y_hat2)\n",
        "\n",
        "# summarize scores\n",
        "print('Logistic: ROC AUC=%.3f MCC=%.3f  Randforest: ROC AUC=%.3f MCC=%.3f  ' % (ns_auc, Mcc1,lr_auc,Mcc2))\n",
        "\n",
        "# calculate roc curves\n",
        "ns_fpr, ns_tpr, _ = roc_curve(y_test, y_prob1)\n",
        "lr_fpr, lr_tpr, _ = roc_curve(y_test, y_prob2)\n",
        "# plot the roc curve for the model\n",
        "pyplot.plot(ns_fpr, ns_tpr, linestyle='--', label='Dummy')\n",
        "pyplot.plot(lr_fpr, lr_tpr, marker='.', label='Logistic')\n",
        "# axis labels\n",
        "pyplot.xlabel('False Positive Rate')\n",
        "pyplot.ylabel('True Positive Rate')\n",
        "# show the legend\n",
        "pyplot.legend()\n",
        "# show the plot\n",
        "pyplot.show()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Logistic: ROC AUC=0.991 MCC=0.869  Randforest: ROC AUC=0.999 MCC=0.956  \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV9Z3/8dcnISxCiAho0YBBi1ZkUyLWUiruaBVbN9xadZw62mo7dZkyow9rdcbRcWnHlplKXaBWq9ZRhyLoT+sCY4UQNLKKIiIJICAgIouE5PP745zQm/2G3HOXnPfz8bgPzvI9536+NyGf+z3fc75fc3dERCS+8jIdgIiIZJYSgYhIzCkRiIjEnBKBiEjMKRGIiMRcp0wH0FZ9+vTxkpKSTIchIpJT5s+f/6m7921qX84lgpKSEsrLyzMdhohITjGzj5vbp0tDIiIxp0QgIhJzSgQiIjGnRCAiEnNKBCIiMRdZIjCzR8xsvZktama/mdkDZrbczBaY2dFRxSIiIs2LskUwBRjXwv7TgUHh6yrgvyOMJTUqy2D2fcG/uSyqerT3vG09vqnyqahbrv2ccy1e2TsR/pwje47A3WeZWUkLRc4Gfu/BONhzzGxfM+vn7mujiqldPnwNHj8fandDXj4MmwA9D4r0LReu3sKydVs5e/iBFOTnsaDqM95f/0WjcucdXQzA/FWb+ejTbfX2dcozvjMiiHPuR5vY9unHfGvHq+RRQy35zC08hdFHDQPgzQ8/Ze2WnfWOL+zSidOO/AoAb3ywgQ1bv6y3v9c+BZz0tQPg89XUVDyJheed1e1ENuX3pW9hF44fFDzD8tLiT9j65e56x/cr6sroQ/s0e3z/Xvtw7MD9AHi+YjW7a4Nh0/er2cC3drxKPrVY+PN4ddn6enWb1e1E+vQbyLDifamuqeV/313T6LMb3K8ng/v1ZEd1DS8sXLvnvHXnWDNgPANKBrH1y2peWryu0fFHD+jFIX26s2n7Ll59b32j/aNK9mPAfvuw/osvmfX+hkb7v3Fobw4s6saaLTv464cbG+3/1mF92b9HF1Zt2k7Zyk2N9p9avJue7z+L19ZQQ96ez63OaUceQGGXAt5ft5UFq7c0Ov7bQ/vRrSCfJWs/Z8nazxvtT+XvXuXm7fX2dyvI59tD+wHt/N0D/vLeOjZvr663P+nfPeCFhWvZUV1Tb39zv3t1BvbpzsgBvQB45u0qGjps/x5t+t1raNhBRRx2QCFbv6xm3oJFfGvHq3Qyh/wucNk06D+q0TF7y6KcjyBMBNPdfUgT+6YDd7n7/4XrfwF+5u6NnhYzs6sIWg0MGDBg5McfN/tcRGpUlsHK2VAyBjp1gfJH4J3Hoba6QUGLNIxaAAez4J08fDVU16xLbr9j1D9fXliP9p3fcRrGaXvea099Gvjb/rYe7/X21e1vWDcLz9Fc7A3L0+DzqYsi+ePTvd/3fAaJn1u63r9tv3vR7W/5dyuz+1Pz2Sf8vls+nHgzjLmhiaOaZ2bz3b20qX058WSxu08GJgOUlpZGO5NOZRlMHQ+7d4Y/BYdO3eDQE2HF60GLIL9zuzLyn99dwx/mNE5m/33pSPbr3pk/lVfyzPwqlqz9nMEH9uSpfzgOqP+L15Sk9tfVr2YXlt8ZS6hHu85fWYY1c946LV6HbOvxCfWo+3nkQfvr1uDzqfs5p+Szj2J/Cz/PrIgvTftbu8adyf2p/n9LfufgS2oKZTIRrAb6J6wXh9sya+XsIAngQSoeNA7O+S1061W/pbAXSWDNZzvaVH5wv56cPSLFl5/6jwr+uLWjHpGct63HN1e+vXWL6vOJSq7FK3sn4p9zJi8NfRu4FjgDOBZ4wN1brV1paamndKyhVXPho1lQ8k0oPgaq5sHUM//2zf/yF1L2oU948C2APd/wRUTSJSOXhszsj8BYoI+ZVQE/BwoA3P23wAyCJLAc2A5cEVUszaosg0fHgTd1hS81Xlmyjt/NXgEQXOrp1zOy9xIR2RtR3jV0USv7HfhRVO/fSFOXdVbOTkgCBoeMDRZXvA441NYEZfaiRfDhhsZ3WERyqUdEpJ1yorO43SrLYMqZUPMlWB70PQK6FMIXibcDOgz+DhwwGFbNaXenzL88uxAILgOdPPiAFFRCRCQa8UgEK2cHf9ghaAFUb4fufRoUyoMdG9X5JiKxE49EUDImeAisdjd06grnTA7+wDd3S1b/UUoAIhIb8UgE/UfByMth3kNw8dN/+yOvb/8iIjFJBAD7Dgj+LW5w91Qbvv0/MXcVh3+lByMP3o/5H2/iP15c1qjMrWcN5sgDi+iUb+yuifbZNxGRVIhPIkiB/61YTWHXTjx02X6tlr3m+K+yatP2VsuJiGSaEkEbbd0ZDFw18uD9Wnww7JuDGnZGi4hkJ01M04Qn5q5iwoNv8X8ffArA4jVbmPDgW02OzigikuuUCJrwvxWrm/yjrwfCRKQj0qWhZgzu13PP5Z0jDyzS+EAi0mHFp0Xw2arg36oUDlgnItIBxCMRVJbB/CnB8hMXaEo/EZEE8bg0tHJ2MIAcQE11qwPJ3XnO0DQFJiKSefFIBIlDTOQXtDqQ3KF9e6QpMBGRzIvHpaG6ISag/hATzXhlyTpeWdJ4onIRkY4oHi0CaH6IiSbUTSSj4aNFJA7i0SIQEZFmKRGIiMRcfC4NhTZt28U1jy5otP3Srx/MWcMPZM1nOzS3sIjESmxaBO9WfZZ0WQ0lISJxEpsWwcKqLQwH9uveucXhIg7ct5uGkxCRWIlNi0BERJqmRCAiEnNKBCIiMRebRNCnJnxSWKOPiojUE49EUFnGaTtfDJY1+qiISD3xSAQrZ2MNRx8VEREgLomgZAy1lh8sJzH6qIhInMQjEfQfRVnnY4PlU25vdfRREZE4iUciqCyj9Ms5wfLLt6qPQEQkQTwSwcrZ5FHXR7BLfQQiIgkiTQRmNs7MlpnZcjOb2MT+AWb2mpm9Y2YLzOyMSALp1hurW/Za6NY7krcREclFkSUCM8sHJgGnA4OBi8xscINitwBPu/tRwIXAf0USzI6N+J6VPNixMZK3ERHJRVG2CEYBy919hbvvAp4Ezm5QxoG68Z6LgDWRRFIyhry8cHy9Tp1115CISIIoE8FBQGXCelW4LdFtwKVmVgXMAK5r6kRmdpWZlZtZ+YYNG9oeSRvnLBYRiZNMdxZfBExx92LgDOAxM2sUk7tPdvdSdy/t27fvXr3RnE3dg4Uk5iwWEYmTKBPBaqB/wnpxuC3RlcDTAO7+FtAV6BNFMMvXfxHFaUVEcl6UiWAeMMjMBppZZ4LO4GkNyqwCTgIwsyMIEsFeXPtpnQadExFpWmSJwN13A9cCLwFLCe4OWmxmt5vZ+LDYDcAPzOxd4I/A5e7uTZ+xHSrLOHn7zGBZg86JiNQT6VSV7j6DoBM4cdutCctLgNFRxgCED5TVBst1g86pw1hEBMh8Z3F6lIyhtq6qGnRORKSeeCSC/qPodMwVwbJuHxURqSceiUBERJoVj0RQWUZN+ZRgWZ3FIiL1xCMRrJyNuWYoExFpSjwSgTqLRUSaFY9E0H8Ur+xzerCszmIRkXrikQiAT/MPCBY01pCISD2xSQTdCvIzHYKISFaKTSI4d2RxpkMQEclKSScCM9snykBERCQzWk0EZvYNM1sCvBeuDzezaKaUjNBry9ZnOgQRkayUTIvgl8BpwEYAd38X+FaUQUVh9eYdmQ5BRCQrJXVpyN0rG2yqiSAWERHJgGSGoa40s28AbmYFwE8I5hcQEZEOIJkWwdXAjwgmnl8NjAB+GGVQIiKSPsm0CA5390sSN5jZaODNaEKKRmHXAtia6ShERLJPMi2CXye5LaudPeLATIcgIpKVmm0RmNlxwDeAvmZ2fcKunoAe0xUR6SBaahF0BnoQJIvChNfnwHnRh5ZaLy9Zl+kQRESyUrMtAnd/A3jDzKa4+8dpjCkS6z7fmekQRESyUjKdxdvN7B7gSKBr3UZ3PzGyqEREJG2S6Sx+nGB4iYHAL4CVwLwIY4pEn5rw0lBVeWYDERHJMskkgt7u/jBQ7e5vuPvfAbnVGqgs4+TtM4NlzVksIlJPMomgOvx3rZl928yOAvaLMKbUWzmbPGqDZc1ZLCJSTzJ9BP9qZkXADQTPD/QE/jHSqFKtZAx5eflQu1tzFouINNBqi8Ddp7v7Fndf5O4nuPtIYFMaYkud/qNg5OXBsuYsFhGpp9lEYGb5ZnaRmd1oZkPCbWea2V+B36QtwhSZUVkQLGjOYhGRelq6NPQw0B8oAx4wszVAKTDR3Z9PR3CptGnbrkyHICKSlVpKBKXAMHevNbOuwCfAoe6+MT2hiYhIOrTUR7DL3WsB3H0nsKKtScDMxpnZMjNbbmYTmylzgZktMbPFZvZEW84vIiLt11KL4GtmtiBcNuDQcN0Ad/dhLZ3YzPKBScApQBUwz8ymufuShDKDgH8GRrv7ZjPbvx11aVG9B8oOOT6qtxERyTktJYIj2nnuUcByd18BYGZPAmcDSxLK/ACY5O6bAdw9mhnmK8s4ZceLwfITF8Blf9adQyIioZYGnWvvQHMHAYlzHVcBxzYocxiAmb1JMLT1be7+YsMTmdlVwFUAAwYMaHskK2eT7+E0y3UPlCkRiIgASU5eH6FOwCBgLHAR8Dsz27dhIXef7O6l7l7at2/ftr9LyRjIC6dQ0ANlIiL1RJkIVhPcflqnONyWqAqY5u7V7v4R8D5BYkit/qOY1fPMYFkPlImI1JNUIjCzbmZ2eBvPPQ8YZGYDzawzcCEwrUGZ5wlaA5hZH4JLRSva+D5JWVXTO1jQA2UiIvW0mgjM7CygAngxXB9hZg3/oDfi7ruBa4GXgKXA0+6+2MxuN7PxYbGXgI1mtgR4DbhJzymIiKRXMoPO3UZwB9DrAO5eYWYDkzm5u88AZjTYdmvCsgPXhy8REcmApIahdvctDbZ5FMGIiEj6JZMIFpvZxUC+mQ0ys18Df404rpQ7qFe3TIcgIpKVkkkE1xHMV/wl8ASwhVybjwA44fDIHloWEclpyfQRfM3dbwZujjoYERFJv2RaBPeZ2VIzu6NuXoJc9D/zqzIdgohIVkpmhrITgBOADcCDZrbQzG6JPLIU21Fdk+kQRESyUlIPlLn7J+7+AHA1wTMFt7ZyiIiI5IhkHig7wsxuM7OFBJPX/5VguAgREekAkuksfgR4CjjN3ddEHI+IiKRZq4nA3Y9LRyBRK+nTHbZmOgoRkezTbCIws6fd/YLwklDik8RJzVCWbb751T7wUaajEBHJPi21CH4S/ntmOgIREZHMaLaz2N3Xhos/dPePE1/AD9MTXuo8Oa+y9UIiIjGUzO2jpzSx7fRUBxK13TW1mQ5BRCQrNZsIzOyasH/gcDNbkPD6CFiQvhBTo0/NumChqjyzgYiIZJmW+gieAGYC/w5MTNi+1d03RRpVqlWWcfL2mcHyExfAZX/WdJUiIqGWLg25u68EfkRw42XdCzPbL/rQUmjlbPIILw3VVMPK2ZmNR0Qki7TWIjgTmE9w+6gl7HPgkAjjSq2SMbjlg++G/AIoGZPpiEREskazicDdzwz/TWpayqzWfxT5pZfDvIfg4qd1WUhEJEEyYw2NNrPu4fKlZna/mQ2IPrQU2zcMubg0s3GIiGSZZG4f/W9gu5kNB24APgQeizSqCPxhzseZDkFEJCslkwh2u7sDZwO/cfdJQGG0YYmISLokM/roVjP7Z+B7wBgzywMKog1LRETSJZkWwQSCiev/zt0/IZiL4J5IoxIRkbRJZqrKT4DHgSIzOxPY6e6/jzwyERFJi2TuGroAKAPOBy4A5prZeVEHlmpH9OuZ6RBERLJSMn0ENwPHuPt6ADPrC7wCPBNlYKk28uBe8H6moxARyT7J9BHk1SWB0MYkj8sq1Rp9VESkScm0CF40s5eAP4brE4AZ0YUUjafmVXJppoMQEclCycxZfJOZnQN8M9w02d2fizYsERFJl5bmLB4E3AscCiwEbnT31ekKTERE0qOla/2PANOBcwlGIP11W09uZuPMbJmZLTeziS2UO9fM3Mw0EJCISJq1dGmo0N1/Fy4vM7O323JiM8sHJhFMdVkFzDOzae6+pEG5QuAnwNy2nF9ERFKjpRZBVzM7ysyONrOjgW4N1lszClju7ivcfRfwJMF4RQ3dAdwN7Gxz9G0wtLgoytOLiOSslloEa4H7E9Y/SVh34MRWzn0QUJmwXgUcm1ggTCj93f0FM7upuROZ2VXAVQADBuzdCNjDi/eFpXt1qIhIh9bSxDQnRPnG4eB19wOXt1bW3ScDkwFKS0t9b95v+64a9tmbA0VEOrgoHwxbDfRPWC8Ot9UpBIYAr5vZSuDrwLSoOoyffbsqitOKiOS8KBPBPGCQmQ00s87AhcC0up3uvsXd+7h7ibuXAHOA8e5eHmFMIiLSQGSJwN13A9cCLxFcnX/a3Reb2e1mNj6q9xURkbZp9cliMzPgEuAQd789nK/4K+5e1tqx7j6DBsNRuPutzZQdm1TEIiKSUsm0CP4LOA64KFzfSvB8gIiIdADJJIJj3f1HhPf5u/tmoHOkUUXg6AG9Mh2CiEhWSiYRVIdPCTvsmY8g58Z0HnygJqYREWlKMongAeA5YH8z+zfg/4A7I40qAlt2VGc6BBGRrJTMMNSPm9l84CTAgO+4e849o/vnd9doPgIRkSYkc9fQAGA78OfEbe6+KsrAREQkPZKZoewFgv4BA7oCA4FlwJERxiUiImmSzKWhoYnr4UBxP4wsIhERSas2P1ns7m/TYBRRERHJXcn0EVyfsJoHHA2siSyiiBw7sDcsyHQUIiLZJ5k+gsKE5d0EfQb/E0040Rl0QI9MhyAikpVaTAThg2SF7n5jmuKJzMYvdtE700GIiGShZvsIzKyTu9cAo9MYT2RmLlqb6RBERLJSSy2CMoL+gAozmwb8CdhWt9Pdn404NhERSYNk+gi6AhsJ5iiue57AASUCEZEOoKVEsH94x9Ai/pYA6uzVvMEiIpJ9WkoE+UAP6ieAOkoEIiIdREuJYK273562SCI2+qt94J1MRyEikn1aerK4qZZAzhrYp3umQxARyUotJYKT0hZFGnzy+c5MhyAikpWaTQTuvimdgUTtlSXrMh2CiEhWavOgcyIi0rEoEYiIxJwSgYhIzCkRiIjEXGwSwdjD+2Y6BBGRrBSbRFDca59MhyAikpVikwiqNm/PdAgiIlkpNong9WUbMh2CiEhWik0iEBGRpkWaCMxsnJktM7PlZjaxif3Xm9kSM1tgZn8xs4OjjEdERBqLLBGE8x1PAk4HBgMXmdngBsXeAUrdfRjwDPAfUcUjIiJNi7JFMApY7u4r3H0X8CRwdmIBd3/N3et6cecAxRHGIyIiTYgyERwEVCasV4XbmnMlMLOpHWZ2lZmVm1n5hg171+l78uAD9uo4EZGOLis6i83sUqAUuKep/e4+2d1L3b20b9+9ezDsKz27tiNCEZGOK8pEsBron7BeHG6rx8xOBm4Gxrv7l1EF89Gn26I6tYhITosyEcwDBpnZQDPrDFwITEssYGZHAQ8SJIH1EcbCm8s/jfL0IiI5K7JE4O67gWuBl4ClwNPuvtjMbjez8WGxe4AewJ/MrMLMpjVzOhERiUhLk9e3m7vPAGY02HZrwvLJUb6/iIi0Lis6i0VEJHOUCEREYi42ieD0If0yHYKISFaKTSLo3aNzpkMQEclKsUkEH6z7ItMhiIhkpdgkgrkfbcx0CCIiWSnS20dFRFKlurqaqqoqdu7cmelQslrXrl0pLi6moKAg6WOUCEQkJ1RVVVFYWEhJSQlmlulwspK7s3HjRqqqqhg4cGDSx8Xm0pCI5LadO3fSu3dvJYEWmBm9e/duc6tJiUBEcoaSQOv25jOKTSI4a/iBmQ5BRCQrxSYRFHVLvuNERKQp+fn5jBgxgiOPPJLhw4dz3333UVtbm+mw2i02ncVL1nxOwwmTRUTaolu3blRUVACwfv16Lr74Yj7//HN+8YtfZDiy9olNInh71WYlApEOZMKDbzXaduawfnzvuBJ27Krh8kfLGu0/b2Qx55f2Z9O2XVzzh/n19j31D8e16f33339/Jk+ezDHHHMNtt93G1KlTKS8v5ze/+U0Qy5lncuONNzJ27Fh69OjBNddcw4wZM+jXrx933nkn//RP/8SqVav41a9+xfjx45kyZQrPP/8827Zt44MPPuDGG29k165dPPbYY3Tp0oUZM2awefNmzj//fN5++20APvjgAyZMmLBnfW/F5tKQiEiqHXLIIdTU1LB+fcvzam3bto0TTzyRxYsXU1hYyC233MLLL7/Mc889x6237hmZn0WLFvHss88yb948br75ZvbZZx/eeecdjjvuOH7/+99z6KGHUlRUtKdV8uijj3LFFVe0ux6xaRGISMfS0jf4bp3zW9y/X/fObW4BtEfnzp0ZN24cAEOHDqVLly4UFBQwdOhQVq5cuafcCSecQGFhIYWFhRQVFXHWWWftOWbBggUA/P3f/z2PPvoo999/P0899RRlZY1bPm2lFoGIyF5asWIF+fn57L///nTq1Klex3HivfwFBQV7buvMy8ujS5cue5Z37969p1zd9pbKnXvuucycOZPp06czcuRIevfu3e56KBGIiOyFDRs2cPXVV3PttddiZpSUlFBRUUFtbS2VlZUp+abelK5du3LaaadxzTXXpOSyEMTo0tA5RxfDG5mOQkRy2Y4dOxgxYgTV1dV06tSJ733ve1x//fUAjB49moEDBzJ48GCOOOIIjj766MjiuOSSS3juuec49dRTU3I+c/eUnChdSktLvby8vO0Hvvmf8PKt8C9roHP31AcmIpFaunQpRxxxRKbDyAr33nsvW7Zs4Y477mhyf1OflZnNd/fSpsrHpkXwbtVnDM90ECIi7fTd736XDz/8kFdffTVl54xNIlhYtUWJQERy3nPPPZfyc6qzWEQk5pQIRERiTolARCTmlAhERGIuNolgwjH9Mx2CiOS4Hj16tPsc5eXl/PjHP252/8qVK3niiSeSLp8KsblrqCA/NjlPROpUlsHK2VAyBvqPynQ0AJSWllJa2uTt/MDfEsHFF1+cVPlUiE0imP/xZkZmOggRSY2ZE+GThS2X+fJzWLcIvBYsDw4YAl16Nl/+K0Ph9LvaHEpFRQVXX30127dv59BDD+WRRx6hV69ezJs3jyuvvJK8vDxOOeUUZs6cyaJFi3j99de59957mT59Om+88QY/+clPgGCKyVmzZjFx4kSWLl3KiBEjuOyyyzjqqKP2lP/iiy+47rrrKC8vx8z4+c9/zrnnntvmmBuKzdfkpWs/z3QIIpJOO7cESQCCf3duieRtvv/973P33XezYMEChg4dumeSmiuuuIIHH3yQiooK8vPzmzz23nvvZdKkSVRUVDB79my6devGXXfdxZgxY6ioqOCnP/1pvfJ33HEHRUVFLFy4kAULFnDiiSempA6xaRGISAeSzDf3yjKYOh5qdkF+Zzj3oZRfHtqyZQufffYZxx9/PACXXXYZ559/Pp999hlbt27luOOCoa4vvvhipk+f3uj40aNHc/3113PJJZdwzjnnUFxc3OL7vfLKKzz55JN71nv16pWSekTaIjCzcWa2zMyWm9nEJvZ3MbOnwv1zzawkynhEJEb6j4LLpsGJNwf/ZkkfQaKJEyfy0EMPsWPHDkaPHs17772XkTgiSwRmlg9MAk4HBgMXmVnD2SKvBDa7+1eBXwJ3RxVPn5p1wULVXgxYJyK5qf8oGHNDZEmgqKiIXr16MXv2bAAee+wxjj/+ePbdd18KCwuZO3cuQL1v8Yk+/PBDhg4dys9+9jOOOeYY3nvvPQoLC9m6dWuT5U855RQmTZq0Z33z5s0pqUeULYJRwHJ3X+Huu4AngbMblDkbmBouPwOcZHWzN6RSZRknb58ZLD9xQdBkFBFpo+3bt1NcXLzndf/99zN16lRuuukmhg0bRkVFxZ6pJx9++GF+8IMfMGLECLZt20ZRUVGj8/3qV79iyJAhDBs2jIKCAk4//XSGDRtGfn4+w4cP55e//GW98rfccgubN29myJAhDB8+nNdeey0l9Yqyj+AgoDJhvQo4trky7r7bzLYAvYFPEwuZ2VXAVQADBgxoeyQrZ9OJcLjtmurgdrIsbCaKSHZLnIEs0Zw5cxptO/LII/dML3nXXXftuQV07NixjB07FoBf//rXTZ6v4ciideV79OjB1KlTmziifXLiriF3n+zupe5e2rdv37afoGQMdOoClh90GpWMSX2QIiIJXnjhBUaMGMGQIUOYPXs2t9xyS6ZDalaULYLVQOLjvMXhtqbKVJlZJ6AI2JjySOo6jbLswRIR6bgmTJjAhAkTMh1GUqJMBPOAQWY2kOAP/oXAxQ3KTAMuA94CzgNe9aimTOs/SglAJMe5O1F0I3Yke/MnNLJLQ+6+G7gWeAlYCjzt7ovN7HYzGx8WexjobWbLgeuBRreYiohAMGn7xo0b9+oPXVy4Oxs3bqRr165tOi4+cxaLSE6rrq6mqqqKnTt3ZjqUrNa1a1eKi4spKCiot11zFotIzisoKGDgwIGZDqNDyom7hkREJDpKBCIiMadEICIScznXWWxmG4CP9/LwPjR4ajkGVOd4UJ3joT11Ptjdm3wiN+cSQXuYWXlzveYdleocD6pzPERVZ10aEhGJOSUCEZGYi1simJzpADJAdY4H1TkeIqlzrPoIRESksbi1CEREpAElAhGRmOuQicDMxpnZMjNbbmaNRjQ1sy5m9lS4f66ZlaQ/ytRKos7Xm9kSM1tgZn8xs4MzEWcqtVbnhHLnmpmbWc7faphMnc3sgvBnvdjMnkh3jKmWxO/2ADN7zczeCX+/z8hEnKliZo+Y2XozW9TMfjOzB8LPY4GZHd3uN3X3DvUC8oEPgUOAzsC7wOAGZX4I/DZcvhB4KtNxp6HOJwD7hMvXxKHOYblCYBYwByjNdNxp+DkPAt4BeoXr+2c67jTUeTJwTbg8GFiZ6bjbWedvAUcDi5rZfwYwEzDg68Dc9r5nR2wRjAKWu/sKd98FPAmc3aDM2UDdxJ/PACdZbs920Wqd3f01d98erh7b9wUAAAZYSURBVM4hmDEulyXzcwa4A7gb6AhjFydT5x8Ak9x9M4C7r09zjKmWTJ0d6BkuFwFr0hhfyrn7LGBTC0XOBn7vgTnAvmbWrz3v2RETwUFAZcJ6VbityTIeTKCzBeidluiikUydE11J8I0il7Va57DJ3N/dX0hnYBFK5ud8GHCYmb1pZnPMbFzaootGMnW+DbjUzKqAGcB16QktY9r6/71Vmo8gZszsUqAUOD7TsUTJzPKA+4HLMxxKunUiuDw0lqDVN8vMhrr7ZxmNKloXAVPc/T4zOw54zMyGuHttpgPLFR2xRbAa6J+wXhxua7KMmXUiaE5uTEt00UimzpjZycDNwHh3/zJNsUWltToXAkOA181sJcG11Gk53mGczM+5Cpjm7tXu/hHwPkFiyFXJ1PlK4GkAd38L6EowOFtHldT/97boiIlgHjDIzAaaWWeCzuBpDcpMAy4Ll88DXvWwFyZHtVpnMzsKeJAgCeT6dWNopc7uvsXd+7h7ibuXEPSLjHf3XJ7nNJnf7ecJWgOYWR+CS0Ur0hlkiiVT51XASQBmdgRBItiQ1ijTaxrw/fDuoa8DW9x9bXtO2OEuDbn7bjO7FniJ4I6DR9x9sZndDpS7+zTgYYLm43KCTpkLMxdx+yVZ53uAHsCfwn7xVe4+PmNBt1OSde5QkqzzS8CpZrYEqAFucvecbe0mWecbgN+Z2U8JOo4vz+Uvdmb2R4Jk3ifs9/g5UADg7r8l6Ac5A1gObAeuaPd75vDnJSIiKdARLw2JiEgbKBGIiMScEoGISMwpEYiIxJwSgYhIzCkRSFYysxozq0h4lbRQ9osUvN8UM/sofK+3wydU23qOh8xscLj8Lw32/bW9MYbnqftcFpnZn81s31bKj8j10Tglerp9VLKSmX3h7j1SXbaFc0wBprv7M2Z2KnCvuw9rx/naHVNr5zWzqcD77v5vLZS/nGDU1WtTHYt0HGoRSE4wsx7hPApvm9lCM2s00qiZ9TOzWQnfmMeE2081s7fCY/9kZq39gZ4FfDU89vrwXIvM7B/Dbd3N7AUzezfcPiHc/rqZlZrZXUC3MI7Hw31fhP8+aWbfToh5ipmdZ2b5ZnaPmc0Lx5j/hyQ+lrcIBxszs1FhHd8xs7+a2eHhk7i3AxPCWCaEsT9iZmVh2aZGbJW4yfTY23rp1dSL4KnYivD1HMFT8D3DfX0Inqqsa9F+Ef57A3BzuJxPMN5QH4I/7N3D7T8Dbm3i/aYA54XL5wNzgZHAQqA7wVPZi4GjgHOB3yUcWxT++zrhnAd1MSWUqYvxu8DUcLkzwSiS3YCrgFvC7V2AcmBgE3F+kVC/PwHjwvWeQKdw+WTgf8Lly4HfJBx/J3BpuLwvwVhE3TP989Yrs68ON8SEdBg73H1E3YqZFQB3mtm3gFqCb8IHAJ8kHDMPeCQs+7y7V5jZ8QSTlbwZDq3RmeCbdFPuMbNbCMapuZJg/Jrn3H1bGMOzwBjgReA+M7ub4HLS7DbUaybwn2bWBRgHzHL3HeHlqGFmdl5YrohgsLiPGhzfzcwqwvovBV5OKD/VzAYRDLNQ0Mz7nwqMN7Mbw/WuwIDwXBJTSgSSKy4B+gIj3b3aghFFuyYWcPdZYaL4NjDFzO4HNgMvu/tFSbzHTe7+TN2KmZ3UVCF3f9+CuQ7OAP7VzP7i7rcnUwl332lmrwOnARMIJlqBYLap69z9pVZOscPdR5jZPgTj7/wIeIBgAp7X3P27Ycf6680cb8C57r4smXglHtRHILmiCFgfJoETgEZzLlswD/M6d/8d8BDBdH9zgNFmVnfNv7uZHZbke84GvmNm+5hZd4LLOrPN7EBgu7v/gWAwv6bmjK0OWyZNeYpgoLC61gUEf9SvqTvGzA4L37NJHsw292PgBvvbUOp1QxFfnlB0K8ElsjovAddZ2DyyYFRaiTklAskVjwOlZrYQ+D7wXhNlxgLvmtk7BN+2/9PdNxD8YfyjmS0guCz0tWTe0N3fJug7KCPoM3jI3d8BhgJl4SWanwP/2sThk4EFdZ3FDfw/gomBXvFg+kUIEtcS4G0LJi1/kFZa7GEsCwgmZvkP4N/Duice9xowuK6zmKDlUBDGtjhcl5jT7aMiIjGnFoGISMwpEYiIxJwSgYhIzCkRiIjEnBKBiEjMKRGIiMScEoGISMz9fwJYGwguS7aQAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "id": "iQeNDoXLjjZJ",
        "outputId": "f052c02f-96b3-49ec-f448-ed78e1dd3105"
      },
      "source": [
        "precision1, recall1, _ = precision_recall_curve(y_test, y_prob1)\n",
        "precision2, recall2, _ = precision_recall_curve(y_test, y_prob2)\n",
        "\n",
        "f1_1, auc_1 = f1_score(y_test, y_hat1), auc(recall1, precision1)\n",
        "f1_2, auc_2 = f1_score(y_test, y_hat2), auc(recall2, precision2)\n",
        "\n",
        "Mcc_1=matthews_corrcoef(y_test, y_hat1)\n",
        "Mcc_2=matthews_corrcoef(y_test, y_hat2)\n",
        "\n",
        "# summarize scores\n",
        "print('Logistic: f1=%.3f  auc=%.3f  MCC=%.3f RandForest: f1=%.3f  auc=%.3f  MCC=%.3f' % (f1_1, auc_1, Mcc_1,f1_2, auc_2, Mcc_2))\n",
        "# plot the precision-recall curves\n",
        "\n",
        "pyplot.plot(recall1, precision1, marker='.', label='Logistic')\n",
        "pyplot.plot(recall2, precision2, marker='.', label='RandForest')\n",
        "\n",
        "# axis labels\n",
        "pyplot.xlabel('Recall')\n",
        "pyplot.ylabel('Precision')\n",
        "# show the legend\n",
        "pyplot.legend()\n",
        "# show the plot\n",
        "pyplot.show()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Logistic: f1=0.951  auc=0.995  MCC=0.869 RandForest: f1=0.983  auc=0.999  MCC=0.956\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU9d338fc3k7BYthAQlSWAWhURAoQYyo0gLtVatejT4vaIVqSLWh9725bWDbStaF1aLbeVKoIVl2p7e6Fi3RHaGoFIQBAERCLBpRDCDpLl+/xxJjGESTLAnEyWz+u6cs1ZfnPO92SZT35nNXdHRESkppRkFyAiIo2TAkJERGJSQIiISEwKCBERiUkBISIiMaUmu4BE6dKli/fu3TvZZYiINCn5+fmb3L1rrHnNJiB69+7NokWLkl2GiEiTYmaFtc3TLiYREYlJASEiIjEpIEREJCYFhIiIxKSAEBGRmEILCDObbmb/MbNltcw3M3vAzNaY2VIzG1xt3jgzWx39GhdWjSIiUrvIpEmTQlnw5MmTS4DpwJhJkyb9T4z53wLOBnKBxcAfJ02a9IiZdQaeBHKAPwNPTp48+fFJkybtqWt906ZNmzRhwoSDqnXlwtdZ/foj7Cx1CsvS+d/FG4ikGEd1akt+YUnV+Gdb98Q1L952DT2vsdbVFGuuq9221f+q+n3q0r1vUmuuXsuB/G4n+m/iQL6Xda073rpq/gzCWGat1i+ApU9DSip07L7v+LYNBzevY/eD+myLx+TJkz+bNGnStFjzLMzbfZtZb+BFd+8fY97DwFx3fyo6/iEwqvLL3X8Qq11tsrOz/WCug1j2z9kc99oVRCinnAjPlw/nMzIwMzI7t6Vw827cHcMAx4Oaap8Xb7uGntdY62qKNdfR7kg2853IP6t+n/LanUbBtnZJqblmLfH+bmd12EHujjcS9zdxAN/LutYdb101t3t52yGcuDs/ocvccuwFdD2qz/4fKNs2wNJnoKIcUiJw9Gj46M1g3KI7bLwi/nleAZHWMG429Mw5kI+2uJlZvrtnx5yXxIB4EZji7v+Mjr8B/IIgINq4+6+j028Bdrv7PTGWMQGYANCrV68hhYW1Xu9Rq3f+fD25RTMwA3cIvht2wMsRCTgGjeT36WBrSeY21LXueOvav12il4lVhtv+7RLOIjD6Jhjx34lfNnUHRJO+ktrdpwHTIOhBHMwy0rPOZU/Rk6R5GaWkMq78JvLLjyUtNYVbv30it7+4nNKyCiKRFHCnvMLrnBdvu4ae11jraoo119VuSGQNMyO/rvp9mj/sUX7yr7Sk1Fyzlnh/tx8YXsqId65K2N/EgXwv61p3vHXV3O6l/ScyYNmUhC6z8JynOH7o6ft/oKxfADPPg/K9EGkFZ02Bf0wMxlNSAQ96DPXNO/VX8NotgAXjvUcc/AflIUhmQGwAelYb7xGdtoGgF1F9+tywijh+6Oms5ClKPniT9H6j+cXhQ8hbW0xu3wyGZKZz3BHtq8aBuObF266h5zXWuppizbW3+waF/zmx6vfpzKGnM+uEkiTVvG8tB/K7vTIjsX8TB/K9rGvd8dW173afPPR0VvYeeMjL/NWzbTn+yyWMPHNM7HCAYDfQuNmwbn7wod4zB7r1+2oc4pvXKTMIiOPOhv+6IbTdS/VJ5i6mc4BrgW8BJwMPuHtO9CB1PlB5VtN7wBB331zXug72GISISDy++6d/kxZJ4cmrc8Nf2fYv4N6vwzn3wdCrQl1VUnYxmdlTBD2BLmZWBNwGpAG4+5+AOQThsAbYBVwZnbfZzO4AFkYXdXt94SAiIokXWkC4+8X1zHfgmlrmTSc4RVZERJJEV1KLiEhMCggREYlJASEiIjEpIEREJCYFhIiIxKSAEBGRmBQQIiISkwJCRERiUkCIiEhMCggREYlJASEiIjEpIEREJCYFhIiIxKSAEBGRmBQQIiISkwJCRERiUkCIiEhMCggREYlJASEiIjEpIEREGrv1C2D+vcFrA0pt0LWJiEj8CmYFobDsOaiogNTWMG429MxpkNWrByEi0th8sSx43ZAPS5+GijKgAsr3wrr5DVaGAkJEJA479pRRVLKb/MKS8Ff2WQFY9OPZUgALhiOtoPeI8NcfpYAQEalHfmEJK7/Yziebd3HpI3nhh0TvERBpDRYJXrsPgTadGnT3EiggRETqlbe2GPdguLSsgry1xeGusGdOEAajbwpeM46Gtp0aNBxAB6lFROqV2zcDM3CHtNQUcvtmhL/SnjlfBcLCR8JfXwwKCBGRegzJTOf4bu3Zubec+8dmMSQzPdklNQgFhIhIHNq1SSX9a61aTDiAjkGIiEgtFBAiIhKTAkJERGJSQIiISEwKCBERiUkBISIiMSkgREQkJgWEiIjEpIAQEZGYFBAiIhJTqAFhZmeZ2YdmtsbMJsaYn2lmb5jZUjOba2Y9qs2728yWm9kKM3vAzCzMWkVEZF+hBYSZRYCpwNlAP+BiM+tXo9k9wOPuPgC4Hbgz+t5vAMOBAUB/YCgwMqxaRURkf2H2IHKANe6+1t33Ak8D59do0w94Mzr8VrX5DrQBWgGtgTTgixBrFRGRGsIMiO7A+mrjRdFp1S0BLogOjwHam1mGu79DEBifRb9ecfcVNVdgZhPMbJGZLdq4cWPCN0BEpCVL9kHqG4GRZraYYBfSBqDczI4BTgB6EITKaDPb70Gs7j7N3bPdPbtr164NWbeISLMX5vMgNgA9q433iE6r4u6fEu1BmFk74EJ332JmVwN57r4jOu9lYBgwP8R6RUSkmjB7EAuBY82sj5m1Ai4CZldvYGZdzKyyhl8C06PDnxD0LFLNLI2gd7HfLiYREQlPaAHh7mXAtcArBB/uf3X35WZ2u5mdF202CvjQzFYB3YDfRKc/B3wEvE9wnGKJu78QVq0iIk3K+gUw/97gNUShPnLU3ecAc2pMu7Xa8HMEYVDzfeXAD8KsTUSkydi5CXZshFdvhs0fw4fRj9VIaxg3G3rmhLJaPZNaRKQxW78A1s4FL4d/PwgpqeAVwbzyvbBufmgBkeyzmEREWoz8whKmvrWG/MKS+N+0bj7BpWGARWDQZdEZBpFW0Hu/EzwTRj0IEZEEyi8sIW9tMbl9MxiSmU5+YQlvrPiCnXvLmJX3CeUVTuu0FGaNz2VIZnr9C+w9ItiVVL43CISsS2HNm9DqMDjvwdB6D6CAEBE5ZJWhkH5YKya/sJy9ZRVEUoxjDm/Hys+379e+tKyCvLXF8QVEz5zgOMO6+UFY9MyBbifClk9CDQdQQIiIHLDKQDjpqI6s2bid385ZSVmF79OmrMIpLN5VNW6AGVQ4pEVSyO2bEf8Ke+bsGwad+0aPS3iw0JAoIERE4rBjTxmbdnzJtbPymbPsc2rkQZWU6Od1q0gKt3y7H7e/uJzSsgrSUlM45diuvPrBFzx25dD4eg+16dwHynbD9s+hw5EHv5x6KCBEROqRX1jCh19sp8Lhxfc/r5puwOjjD+efazZRVh6EwK3fPpGSXXurjkEcd0T7qmMS735czKsffMGgXocQDhD0IAA2r1VAiIgkU97a4qrhFIMUM9ydtNQUfnzqMfz41GP2OTBd3ZDM9Kpp735cTEJUD4jewxOzzBgUECIi9cjtm0Gr1JSqXUU1ewnAoe0yOlAdewbXQ2xeG+pqFBAiIvUYkpnOrPG5tfYSGlwkFTplKiBERBqD6ruKGoXOfUMPCF1JLSLSFHXuG9yXyWs5nSoBFBAiIk1R576wd3twI7+QKCBERJqi6mcyhUQBISLSFCkgREQkpk69wFKg5OPQVqGAEBFpilJbBddDqAchIiL7CflUVwWEiEhTpYAQEZGYOveF3SWwa3Moi1dAiIg0VZVnMoV0oFoBISLSVHXuE7xuVkCIiEh16b2D15COQyggRESaqrS2cFgXWPECrF+Q8MUrIEREmqr1C2D3Zvh8Kcw8L+EhoYAQEWmq1s2Hypu5lu8NxhNIASEi0lT1HgGprcEiEGkVjCdQXA8MMrPhwCQgM/oeA9zd+ya0GhGRFiS/sOTQnlLXMwfGzQ56Dr1HBOMJFO8T5R4FbgDygfKEViAi0kJsKNkNwDsfbWLl59u559VVuDutUlOYNT734EMiwcFQKd6A2OruL4dSgYhIC5BfWMIzC9cDcOWMRfvMKy2rIG9tceN6pCnxH4N4y8x+Z2bDzGxw5VeolYmINCN5a4upiD4e1IDRxx9eNZyWmkJu34zkFVeLeHsQJ0dfs6tNc2B0YssREWmecvtm0Co1hdKyCtJSU/jRqKN5c+V/GH5MF2444+uNrvcAcQaEu58adiEiIs3ZkMx0Zo3PrToondWzEwA5fTo3ynCA+M9i6gjcBpwSnfQ2cLu7bw2rMBGR5mZIZnpVGJRXeD2tky/eYxDTge3A96Jf24DHwipKRESSL95jEEe7+4XVxiebWUEYBYmISOMQbw9it5n9V+VI9MK53eGUJCIijUG8PYgfATOjxyIM2AxcEVZRIiKSfHH1INy9wN0HAgOAk9x9kLsvqe99ZnaWmX1oZmvMbGKM+Zlm9oaZLTWzuWbWo9q8Xmb2qpmtMLMPzKx3/JslIiKHqs4ehJld5u5PmNlPa0wHwN3vq+O9EWAqcAZQBCw0s9nu/kG1ZvcAj7v7TDMbDdwJ/N/ovMeB37j7a2bWDqg4sE0TEZFDUV8P4mvR1/a1fNUlB1jj7mvdfS/wNHB+jTb9gDejw29VzjezfkCqu78G4O473H1X/ZsjIiKJUmcPwt0fjr5OPohldwfWVxsv4qsrsistAS4A/gCMAdqbWQbwdWCLmf0d6AO8Dkx0931uFGhmE4AJAL169TqIEkVEpDZxHYMws7vNrIOZpUWPGWw0s8sSsP4bgZFmthgYCWwguFtsKjAiOn8o0JcYB8XdfZq7Z7t7dteuXRNQjoiIVIr3NNcz3X0b8G1gHXAM8LN63rMB6FltvEd0WhV3/9TdL3D3QcBN0WlbCHobBdHdU2XA84BuDigi0oDiDYjKXVHnAM/GeYuNhcCxZtbHzFoBFwGzqzcwsy5mVlnDLwmu2K58byczq+wWjAaqH9wWEZGQxRsQL5rZSmAI8Eb0g3tPXW+I/ud/LfAKsAL4q7svN7Pbzey8aLNRwIdmtgroBvwm+t5ygt1Lb5jZ+wTXXvz5gLZMREQOSbx3c51oZncTPDio3Mx2sv8ZSbHeNweYU2PardWGnwOeq+W9rxFcdyEiIklQ33UQo939TTO7oNq06k3+HlZhIiKSXPX1IEYSXKdwbox5jgJCRKTZqu86iNuir1c2TDkiItJYxHsdxG/NrFO18XQz+3V4ZYmISLLFexbT2dHrEwBw9xLgW+GUJCIijUG8ARExs9aVI2bWFmhdR3sREWni4g2IWQTXJFxlZlcBrwEzwytLRKRlWPDxZvILS5JdRkzxXgdxl5ktAU6PTrrD3V8JrywRkebtvU+CUPjXmk0sKtzMrPG5DMlMT3JV+4r3iXIQXA1d5u6vm9lhZtbe3beHVZiISHP27tpiILheoLSsgry1xY0uIOI9i+lqgiueH45O6k5wAz0RETkIJ/fNAIL7CKWlppAbHW9M4j0GcQ0wHNgG4O6rgcPDKkpEpLkb3CvoLQw/pkuj3L0E8QfEl9GnwgFgZqkEPSMRETkEOX06N8pwgPgD4m0z+xXQ1szOAJ4FXgivLBERSbZ4A+IXwEbgfeAHBHdovTmsokREJPnqPYvJzCLAcnc/Hj2TQUSkxai3BxF9eM+HZtarAeoREZFGIt7rINKB5Wa2ANhZOdHdz6v9LSIi0pTFGxC3hFqFiIg0OvU9Ua4N8EPgGIID1I9GnzUtIiLNXH3HIGYC2QThcDZwb+gViYhIo1DfLqZ+7n4SgJk9CiwIvyQREWkM6utBlFYOaNeSiEjLUl8PYqCZbYsOG8GV1Nuiw+7uHUKtTkREkqbOgHD3SEMVIiIijUu8t9oQEZEWRgEhIiIxKSBERCQmBYSISCOXX1jC1LfWkF9Y0qDrPZBnUouISAN7d20xlz36LuUVTqvUlAZ9+pwCQkSkEXpl+efM+NfHvPfJFkrLgwd4lpZVkLe2WAEhItKS5BeWkLd2E+1apzHn/c949+PNAJh91SYtNYXcvhkNVpMCQkQkyf69ZhPjHltQ1VNoFbHgamSCA8U9uxxGyc5Spl8xtEGfX62D1CIiSfSXd9btEw4GjBncndZpKUQs6DX06nwYHdumNWg4gHoQIiJJsfiT4IykjTv2kmKQmmK4O2mpKXwvuxffy+5F3tpicvtm8EReIes27WrwGhUQIiJJ8O7Hm0kxqPCg1zB2aE+O6tSW3L4ZVT2Fytcn8gqTUqMCQkQkCXL7ZtAqNYXSsgrSUlO4YHCPBt+FVB8FhIhIEgzJTGfW+Nyq3UiNLRxAASEikjRDMtMbZTBUCvUsJjM7y8w+NLM1ZjYxxvxMM3vDzJaa2Vwz61FjfgczKzKzP4ZZp4iI7C+0gDCzCDCV4FnW/YCLzaxfjWb3AI+7+wDgduDOGvPvAOaFVaOIiNQuzB5EDrDG3de6+17gaeD8Gm36AW9Gh9+qPt/MhgDdgFdDrFFERGoRZkB0B9ZXGy+KTqtuCXBBdHgM0N7MMswsBbgXuLGuFZjZBDNbZGaLNm7cmKCyRUQEkn8l9Y3ASDNbDIwENgDlwI+BOe5eVNeb3X2au2e7e3bXrl3Dr1ZEpAUJ8yymDUDPauM9otOquPunRHsQZtYOuNDdt5jZMGCEmf0YaAe0MrMd7r7fgW4REQlHmAGxEDjWzPoQBMNFwCXVG5hZF2Czu1cAvwSmA7j7pdXaXAFkKxxERBpWaLuY3L0MuBZ4BVgB/NXdl5vZ7WZ2XrTZKOBDM1tFcED6N2HVIyIiBybUC+XcfQ4wp8a0W6sNPwc8V88yZgAzQihPRETqkOyD1CIi0kgpIEREJCYFhIiIxKSAEBFpYvILS5j61hryC0tCXY/u5ioi0sgV7/ySrbtLeX7xBv6+uIh5qzaRYtAqNYVZ43NDuyOsAkJEpBHLLyzhX6uLKXfn/z1TUDW9wqG0rIK8tcWhBYR2MYmINGJ5a4upcAeCR5OedsLhAKQYpKWmkNs3I7R1KyBERBqx3L4ZtE5LIWLQOi2Fc046EoBzBxwV6u4l0C4mEZFGreajSb/WOgLAWf2PCP1pdAoIEZFGrvqjSVd+vq3B1qtdTCIiEpMCQkREYlJAiIhITAoIERGJSQEhIiIxKSBERCQmBYSIiMSkgBARkZgUECIiEpMCQkREYlJAiIhITAoIERGJSQEhIiIxNeu7uZaWllJUVMSePXuSXUqz16ZNG3r06EFaWlqySxGRBGnWAVFUVET79u3p3bs3Zpbscpotd6e4uJiioiL69OmT7HJEJEGa9S6mPXv2kJGRoXAImZmRkZGhnppIM9OsAwJQODQQfZ9FGtY/ln1OfmFJqOto9gEhItKcfPBp8ES52Us+5dJH8kINCQVEyNq1a3fIy1i0aBE/+clPap2/bt06nnzyybjbi0jTtbRoKwAOlJZVkLe2OLR1NeuD1Acjv7Ck6uHgYT8QPF7Z2dlkZ2fXOr8yIC655JK42otI0zWgR0cAUgzSUlPI7ZsR2rpaTEBMfmF5VdesNtv3lLLy8+1UePDNP/6I9rRvU/tpm/2O6sBt5554wLUUFBTwwx/+kF27dnH00Uczffp00tPTWbhwIVdddRUpKSmcccYZvPzyyyxbtoy5c+dyzz338OKLL/L2229z/fXXA8F+/3nz5jFx4kRWrFhBVlYW48aNY9CgQVXtd+zYwXXXXceiRYswM2677TYuvPDCA65ZRBqHfkd1AODcAUdx+Td6h/qPrHYxVbNtTxkVHgxXeDAehssvv5y77rqLpUuXctJJJzF58mQArrzySh5++GEKCgqIRCIx33vPPfcwdepUCgoKmD9/Pm3btmXKlCmMGDGCgoICbrjhhn3a33HHHXTs2JH333+fpUuXMnr06FC2SUQa1ln9jwh9L0eL6UHE859+fmEJlz6SR2lZBWmpKfzhokEJ/wFs3bqVLVu2MHLkSADGjRvHd7/7XbZs2cL27dsZNmwYAJdccgkvvvjifu8fPnw4P/3pT7n00ku54IIL6NGjR53re/3113n66aerxtPTG8duMxFp/FpMQMRjSGY6s8bnNrpjENVNnDiRc845hzlz5jB8+HBeeeWVZJckIs2UdjHVMCQznWtOPSa0cOjYsSPp6enMnz8fgL/85S+MHDmSTp060b59e959912Aff7rr+6jjz7ipJNO4he/+AVDhw5l5cqVtG/fnu3bt8dsf8YZZzB16tSq8ZKScM+bFpHmQwERsl27dtGjR4+qr/vuu4+ZM2fys5/9jAEDBlBQUMCtt94KwKOPPsrVV19NVlYWO3fupGPHjvst7/e//z39+/dnwIABpKWlcfbZZzNgwAAikQgDBw7k/vvv36f9zTffTElJCf3792fgwIG89dZbDbLdItL0aRdTyCoqKmJOz8vL22/aiSeeyNKlSwGYMmVK1amqo0aNYtSoUQA8+OCDMZf35ptv7jNe2b5du3bMnDnzYEoXkRZOAdGIvPTSS9x5552UlZWRmZnJjBkzkl2SiLRgoQaEmZ0F/AGIAI+4+5Qa8zOB6UBXYDNwmbsXmVkW8BDQASgHfuPuz4RZa2MwduxYxo4dm+wyRESAEI9BmFkEmAqcDfQDLjazfjWa3QM87u4DgNuBO6PTdwGXu/uJwFnA782sU1i1iojI/sI8SJ0DrHH3te6+F3gaOL9Gm35A5c7ztyrnu/sqd18dHf4U+A9BL0NERBpImAHRHVhfbbwoOq26JcAF0eExQHsz2+fGImaWA7QCPqq5AjObYGaLzGzRxo0bE1a4iIgk/zTXG4GRZrYYGAlsIDjmAICZHQn8BbjS3fc7Hcjdp7l7trtnd+2qDoaISCKFGRAbgJ7VxntEp1Vx90/d/QJ3HwTcFJ22BcDMOgAvATe5+/7nhDYRkUiErKws+vfvz7nnnsuWLVsSstwZM2Zw7bXXAjBp0iS6d+9OVlYWWVlZTJw4MSHriOW3v/1taMsWkcYlzIBYCBxrZn3MrBVwETC7egMz62JmlTX8kuCMJqLt/5fgAPZzIda4v/ULYP69wWsCtG3bloKCApYtW0bnzp33uao5kW644QYKCgooKChgypQp9b8BKCs78JsRKiBEWo7QTnN19zIzuxZ4heA01+nuvtzMbgcWuftsYBRwp5k5MA+4Jvr27wGnABlmdkV02hXuXnDQBb08ET5/v+42X26DL5aBV4ClQLf+0LpD7e2POAnOju/DGGDYsGFVF8ItWLCA66+/nj179tC2bVsee+wxjjvuOGbMmMHs2bPZtWsXH330EWPGjOHuu+8G4LHHHuPOO++kU6dODBw4kNatW9e6Lnfn5z//OS+//DJmxs0338zYsWOZO3cut9xyC+np6axcuZIVK1YwceJE5s6dy5dffsk111zDD37wAz777DPGjh3Ltm3bKCsr46GHHuKll15i9+7dZGVlceKJJzJr1qy4t11Emp5Qr4Nw9znAnBrTbq02/BywXw/B3Z8Angiztpj2bA3CAYLXPVvrDogDUF5ezhtvvMFVV10FwPHHH8/8+fNJTU3l9ddf51e/+hV/+9vfgOB5EYsXL6Z169Ycd9xxXHfddaSmpnLbbbeRn59Px44dOfXUUxk0aFDV8u+//36eeCL4lt11113s2LGDgoIClixZwqZNmxg6dCinnHIKAO+99x7Lli2jT58+TJs2jY4dO7Jw4UK+/PJLhg8fzplnnsnf//53vvnNb3LTTTdRXl7Orl27GDFiBH/84x8pKDj4nBaRpqPlXEkdz3/66xfAzPOgfC9EWsGFj0DPnENabeV/3Bs2bOCEE07gjDPOAILbfo8bN47Vq1djZpSWlla957TTTqu6D1O/fv0oLCxk06ZNjBo1isqD8WPHjmXVqlVV77nhhhu48cYb9xm/+OKLiUQidOvWjZEjR7Jw4UI6dOhATk4Offr0AeDVV19l6dKlPPfcc1V1rV69mqFDh/L973+f0tJSvvOd75CVlXVI3wcRCUd+YQn/XL2R/zq2a8JvMprss5gal545MG42jL4peD3EcICvjkEUFhbi7lXHIG655RZOPfVUli1bxgsvvMCePXuq3lN911EkEjmoYwV1+drXvlY17O48+OCDVccvPv74Y84880xOOeUU5s2bR/fu3bniiit4/PHHE1qDiBy6l5Z+ynf/9G/uf301lz6SR35hYu/WrICoqWcOjPjvhIRDdYcddhgPPPAA9957L2VlZWzdupXu3YPLQuK559LJJ5/M22+/TXFxMaWlpTz77LN1th8xYgTPPPMM5eXlbNy4kXnz5pGTs/82ffOb3+Shhx6q6sGsWrWKnTt3UlhYSLdu3bj66qsZP3487733HgBpaWn79HZEJDmmvLyS655cXPUUzNKyCvLWFid0HS1nF1MjMGjQIAYMGMBTTz3Fz3/+c8aNG8evf/1rzjnnnHrfe+SRRzJp0iSGDRtGp06d6t3lM2bMGN555x0GDhyImXH33XdzxBFHsHLlyn3ajR8/nnXr1jF48GDcna5du/L8888zd+5cfve735GWlka7du2qehATJkxgwIABDB48WAepRZLgg0+3AVC4eRcpBmkpRkWFk5aaQm7fjHrefWDM3RO6wGTJzs72RYsW7TNtxYoVnHDCCUmqqOXR91skfA++uZr7Xl2FAxGDi3J6cVSntgf9FEwzy3f37Fjz1IMQEWlCvnF0F6amraG0rIK01BQuGNwjtCdgKiBERJqQIZnpzBqfS97a4oPuNcSr2QeEu2NmyS6j2WsuuypFmoIhmemhBkOlZn0WU5s2bSguLtaHV8jcneLiYtq0aZPsUkQkgZp1D6JHjx4UFRWhW4GHr02bNvTo0SPZZYhIAjXrgEhLS6u6YlhERA5Ms97FJCIiB08BISIiMSkgREQkpmZzJbWZbQQKD2ERXYBNCSqnqWhp29zSthe0zS3FoWxzprvHfGZzswmIQ2Vmi2q73Ly5amnb3NK2F7TNLUVY26xdTCIiEpMCQkREYlJAfGVasgtIgpa2zSrbMBAAAAUYSURBVC1te0Hb3FKEss06BiEiIjGpByEiIjEpIEREJKYWFRBmdpaZfWhma8xsYoz5rc3smej8d82sd8NXmVhxbPNPzewDM1tqZm+YWWYy6kyk+ra5WrsLzczNrMmfEhnPNpvZ96I/6+Vm9mRD15hocfxu9zKzt8xscfT3+1vJqDNRzGy6mf3HzJbVMt/M7IHo92OpmQ0+5JW6e4v4AiLAR0BfoBWwBOhXo82PgT9Fhy8Cnkl23Q2wzacCh0WHf9QStjnarj0wD8gDspNddwP8nI8FFgPp0fHDk113A2zzNOBH0eF+wLpk132I23wKMBhYVsv8bwEvAwbkAu8e6jpbUg8iB1jj7mvdfS/wNHB+jTbnAzOjw88Bp1nTftpQvdvs7m+5+67oaB7Q1O/ZHc/PGeAO4C5gT0MWF5J4tvlqYKq7lwC4+38auMZEi2ebHegQHe4IfNqA9SWcu88DNtfR5HzgcQ/kAZ3M7MhDWWdLCojuwPpq40XRaTHbuHsZsBXIaJDqwhHPNld3FcF/IE1Zvdsc7Xr3dPeXGrKwEMXzc/468HUz+5eZ5ZnZWQ1WXTji2eZJwGVmVgTMAa5rmNKS5kD/3uvVrJ8HIfEzs8uAbGBksmsJk5mlAPcBVyS5lIaWSrCbaRRBL3GemZ3k7luSWlW4LgZmuPu9ZjYM+IuZ9Xf3imQX1lS0pB7EBqBntfEe0Wkx25hZKkG3tLhBqgtHPNuMmZ0O3ASc5+5fNlBtYalvm9sD/YG5ZraOYF/t7CZ+oDqen3MRMNvdS939Y2AVQWA0VfFs81XAXwHc/R2gDcFN7ZqruP7eD0RLCoiFwLFm1sfMWhEchJ5do81sYFx0+P8Ab3r06E8TVe82m9kg4GGCcGjq+6Whnm12963u3sXde7t7b4LjLue5+6LklJsQ8fxuP0/Qe8DMuhDsclrbkEUmWDzb/AlwGoCZnUAQEM35+cOzgcujZzPlAlvd/bNDWWCL2cXk7mVmdi3wCsEZENPdfbmZ3Q4scvfZwKME3dA1BAeDLkpexYcuzm3+HdAOeDZ6PP4Tdz8vaUUfoji3uVmJc5tfAc40sw+AcuBn7t5ke8dxbvN/A382sxsIDlhf0ZT/4TOzpwhCvkv0uMptQBqAu/+J4DjLt4A1wC7gykNeZxP+fomISIha0i4mERE5AAoIERGJSQEhIiIxKSBERCQmBYSIiMSkgBA5AGZWbmYFZrbMzF4ws04JXv666HUKmNmORC5b5EApIEQOzG53z3L3/gTXylyT7IJEwqKAEDl47xC9GZqZHW1m/zCzfDObb2bHR6d3M7P/NbMl0a9vRKc/H2273MwmJHEbRGrVYq6kFkkkM4sQ3Mbh0eikacAP3X21mZ0M/A8wGngAeNvdx0Tf0y7a/vvuvtnM2gILzexvTfnKZmmeFBAiB6atmRUQ9BxWAK+ZWTvgG3x1uxKA1tHX0cDlAO5eTnALeYCfmNmY6HBPghvnKSCkUVFAiByY3e6eZWaHEdwH6BpgBrDF3bPiWYCZjQJOB4a5+y4zm0twIzmRRkXHIEQOQvQpfD8huCHcLuBjM/suVD0beGC06RsEj3LFzCJm1pHgNvIl0XA4nuCW4yKNjgJC5CC5+2JgKcGDaS4FrjKzJcByvnr85fXAqWb2PpBP8GzkfwCpZrYCmEJwy3GRRkd3cxURkZjUgxARkZgUECIiEpMCQkREYlJAiIhITAoIERGJSQEhIiIxKSBERCSm/w/hO0G3SSFf5wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J_nT0dhjms4C"
      },
      "source": [
        "**A quick example for how to cross validate.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N36iM3RcjjZJ",
        "outputId": "17cd68d9-1791-4776-ed55-d5268919b507"
      },
      "source": [
        "from sklearn.model_selection import cross_val_score\r\n",
        "\r\n",
        "scores = cross_val_score(classifier1, X, y, cv=5, scoring='f1')\r\n",
        "scores"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.94594595, 0.95104895, 0.97260274, 0.95172414, 0.96453901])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SY3Uixl8jjZJ",
        "outputId": "919b7d3a-09c5-448b-9c5e-626d3c5cab95"
      },
      "source": [
        "print(\"F1: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "F1: 0.96 (+/- 0.02)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "un8_hxi4mXTQ"
      },
      "source": [
        ""
      ],
      "execution_count": 15,
      "outputs": []
    }
  ]
}