ML7up.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMXKcPZLr3CNUh4fkwKJGwY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/francescopatane96/Bioactivity-prediction-with-ML/blob/main/ML7up.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install rdkit"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sTiafVkFRvo8",
        "outputId": "82a3fade-699f-4cd8-b2a5-42d05438bad9"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting rdkit\n",
            "  Downloading rdkit-2022.3.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (36.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 36.8 MB 263 kB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from rdkit) (1.21.6)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from rdkit) (7.1.2)\n",
            "Installing collected packages: rdkit\n",
            "Successfully installed rdkit-2022.3.5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/volkamerlab/teachopencadd.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L10Sz7LoR5f0",
        "outputId": "3714c864-292b-4254-80af-7c50d7723f2e"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/volkamerlab/teachopencadd.git\n",
            "  Cloning https://github.com/volkamerlab/teachopencadd.git to /tmp/pip-req-build-lvf6odzi\n",
            "  Running command git clone -q https://github.com/volkamerlab/teachopencadd.git /tmp/pip-req-build-lvf6odzi\n",
            "Building wheels for collected packages: teachopencadd\n",
            "  Building wheel for teachopencadd (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for teachopencadd: filename=teachopencadd-2022.9.0+2.g8f2c87a-py3-none-any.whl size=108231919 sha256=50725aed4ce8dee79b4c289998ec6b1c5bd81a15b44e7e5c5295bda05bec02bc\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-stam_fbk/wheels/ee/da/82/b2c07c8ddbf7e09a56a283a394ebff26d956480b85a9ef011f\n",
            "Successfully built teachopencadd\n",
            "Installing collected packages: teachopencadd\n",
            "Successfully installed teachopencadd-2022.9.0+2.g8f2c87a\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "VqMlkn50RdJD"
      },
      "outputs": [],
      "source": [
        "from pathlib import Path\n",
        "from warnings import filterwarnings\n",
        "import time\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn import svm, metrics, clone\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.model_selection import KFold, train_test_split\n",
        "from sklearn.metrics import auc, accuracy_score, recall_score\n",
        "from sklearn.metrics import roc_curve, roc_auc_score\n",
        "import matplotlib.pyplot as plt\n",
        "from rdkit import Chem\n",
        "from rdkit.Chem import MACCSkeys\n",
        "from rdkit.Chem.AllChem import GetMorganFingerprintAsBitVect\n",
        "\n",
        "from teachopencadd.utils import seed_everything\n",
        "\n",
        "# Silence some expected warnings\n",
        "filterwarnings(\"ignore\")\n",
        "# Fix seed for reproducible results\n",
        "SEED = 22\n",
        "seed_everything(SEED)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Read data from previous talktorials\n",
        "chembl_df = pd.read_csv(\n",
        "    \"TNFB_compounds_lipinski.csv\",\n",
        "    index_col=0,\n",
        ")\n",
        "\n",
        "# Look at head\n",
        "print(\"Shape of dataframe : \", chembl_df.shape)\n",
        "chembl_df.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 462
        },
        "id": "QWPd9haHSfne",
        "outputId": "240a5659-5853-4690-ff2d-f4902b3cdf1e"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of dataframe :  (49, 11)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id   IC50 units  \\\n",
              "0       CHEMBL336546   50.0    nM   \n",
              "1       CHEMBL131636  170.0    nM   \n",
              "2       CHEMBL134490  280.0    nM   \n",
              "3       CHEMBL135133  380.0    nM   \n",
              "4       CHEMBL334485  500.0    nM   \n",
              "\n",
              "                                              smiles     pIC50  \\\n",
              "0  O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...  7.301030   \n",
              "1  CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...  6.769551   \n",
              "2  CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...  6.552842   \n",
              "3       O=C(Nc1ccc(C(F)(F)F)cc1)c1cnc(Cl)nc1C(F)(F)F  6.420216   \n",
              "4         O=C(Nc1cc(Cl)cc(Cl)c1)c1cnc(Cl)nc1C(F)(F)F  6.301030   \n",
              "\n",
              "                                               ROMol  molecular_weight  n_hba  \\\n",
              "0  <img data-content=\"rdkit/molecule\" src=\"data:i...        436.997743      3   \n",
              "1  <img data-content=\"rdkit/molecule\" src=\"data:i...        469.062788      5   \n",
              "2  <img data-content=\"rdkit/molecule\" src=\"data:i...        441.031488      5   \n",
              "3  <img data-content=\"rdkit/molecule\" src=\"data:i...        369.010359      3   \n",
              "4  <img data-content=\"rdkit/molecule\" src=\"data:i...        368.945029      3   \n",
              "\n",
              "   n_hbd    logp  ro5_fulfilled  \n",
              "0      1  5.4387           True  \n",
              "1      1  5.3768           True  \n",
              "2      1  4.5966           True  \n",
              "3      1  4.4199           True  \n",
              "4      1  4.7079           True  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-709da86c-c16d-45dc-852b-c1c96e6e21e9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>IC50</th>\n",
              "      <th>units</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "      <th>ROMol</th>\n",
              "      <th>molecular_weight</th>\n",
              "      <th>n_hba</th>\n",
              "      <th>n_hbd</th>\n",
              "      <th>logp</th>\n",
              "      <th>ro5_fulfilled</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL336546</td>\n",
              "      <td>50.0</td>\n",
              "      <td>nM</td>\n",
              "      <td>O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...</td>\n",
              "      <td>7.301030</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>436.997743</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>5.4387</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL131636</td>\n",
              "      <td>170.0</td>\n",
              "      <td>nM</td>\n",
              "      <td>CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...</td>\n",
              "      <td>6.769551</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>469.062788</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>5.3768</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL134490</td>\n",
              "      <td>280.0</td>\n",
              "      <td>nM</td>\n",
              "      <td>CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...</td>\n",
              "      <td>6.552842</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>441.031488</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>4.5966</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CHEMBL135133</td>\n",
              "      <td>380.0</td>\n",
              "      <td>nM</td>\n",
              "      <td>O=C(Nc1ccc(C(F)(F)F)cc1)c1cnc(Cl)nc1C(F)(F)F</td>\n",
              "      <td>6.420216</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>369.010359</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>4.4199</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CHEMBL334485</td>\n",
              "      <td>500.0</td>\n",
              "      <td>nM</td>\n",
              "      <td>O=C(Nc1cc(Cl)cc(Cl)c1)c1cnc(Cl)nc1C(F)(F)F</td>\n",
              "      <td>6.301030</td>\n",
              "      <td>&lt;img data-content=\"rdkit/molecule\" src=\"data:i...</td>\n",
              "      <td>368.945029</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>4.7079</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-709da86c-c16d-45dc-852b-c1c96e6e21e9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-709da86c-c16d-45dc-852b-c1c96e6e21e9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-709da86c-c16d-45dc-852b-c1c96e6e21e9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Keep only the columns we want\n",
        "chembl_df = chembl_df[[\"molecule_chembl_id\", \"smiles\", \"pIC50\"]]\n",
        "chembl_df.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ui_bJaycSn6u",
        "outputId": "cb928146-903b-4301-e997-90626acbd3a1"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id                                             smiles  \\\n",
              "0       CHEMBL336546  O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...   \n",
              "1       CHEMBL131636  CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...   \n",
              "2       CHEMBL134490  CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...   \n",
              "3       CHEMBL135133       O=C(Nc1ccc(C(F)(F)F)cc1)c1cnc(Cl)nc1C(F)(F)F   \n",
              "4       CHEMBL334485         O=C(Nc1cc(Cl)cc(Cl)c1)c1cnc(Cl)nc1C(F)(F)F   \n",
              "\n",
              "      pIC50  \n",
              "0  7.301030  \n",
              "1  6.769551  \n",
              "2  6.552842  \n",
              "3  6.420216  \n",
              "4  6.301030  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4c8d92f2-baad-4fea-a1ec-36bbf767a821\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL336546</td>\n",
              "      <td>O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...</td>\n",
              "      <td>7.301030</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL131636</td>\n",
              "      <td>CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...</td>\n",
              "      <td>6.769551</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL134490</td>\n",
              "      <td>CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...</td>\n",
              "      <td>6.552842</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CHEMBL135133</td>\n",
              "      <td>O=C(Nc1ccc(C(F)(F)F)cc1)c1cnc(Cl)nc1C(F)(F)F</td>\n",
              "      <td>6.420216</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CHEMBL334485</td>\n",
              "      <td>O=C(Nc1cc(Cl)cc(Cl)c1)c1cnc(Cl)nc1C(F)(F)F</td>\n",
              "      <td>6.301030</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4c8d92f2-baad-4fea-a1ec-36bbf767a821')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4c8d92f2-baad-4fea-a1ec-36bbf767a821 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4c8d92f2-baad-4fea-a1ec-36bbf767a821');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Add column for activity\n",
        "chembl_df[\"active\"] = np.zeros(len(chembl_df))\n",
        "\n",
        "# Mark every molecule as active with an pIC50 of >= 6.3, 0 otherwise\n",
        "chembl_df.loc[chembl_df[chembl_df.pIC50 >= 6.3].index, \"active\"] = 1.0\n",
        "\n",
        "# NBVAL_CHECK_OUTPUT\n",
        "print(\"Number of active compounds:\", int(chembl_df.active.sum()))\n",
        "print(\"Number of inactive compounds:\", len(chembl_df) - int(chembl_df.active.sum()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8NqHNrxjS1Zj",
        "outputId": "6380e55c-1ff6-4b9b-cfa4-142c7dc22d29"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of active compounds: 5\n",
            "Number of inactive compounds: 44\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "chembl_df.head()\n",
        "# NBVAL_CHECK_OUTPUT"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "oW3bY18rS8cD",
        "outputId": "87e03060-1f52-4441-fc5e-8f397b6060c6"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id                                             smiles  \\\n",
              "0       CHEMBL336546  O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...   \n",
              "1       CHEMBL131636  CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...   \n",
              "2       CHEMBL134490  CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...   \n",
              "3       CHEMBL135133       O=C(Nc1ccc(C(F)(F)F)cc1)c1cnc(Cl)nc1C(F)(F)F   \n",
              "4       CHEMBL334485         O=C(Nc1cc(Cl)cc(Cl)c1)c1cnc(Cl)nc1C(F)(F)F   \n",
              "\n",
              "      pIC50  active  \n",
              "0  7.301030     1.0  \n",
              "1  6.769551     1.0  \n",
              "2  6.552842     1.0  \n",
              "3  6.420216     1.0  \n",
              "4  6.301030     1.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1ae19278-c94a-43dd-9785-1e9771c037fb\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "      <th>active</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL336546</td>\n",
              "      <td>O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...</td>\n",
              "      <td>7.301030</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL131636</td>\n",
              "      <td>CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...</td>\n",
              "      <td>6.769551</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL134490</td>\n",
              "      <td>CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...</td>\n",
              "      <td>6.552842</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CHEMBL135133</td>\n",
              "      <td>O=C(Nc1ccc(C(F)(F)F)cc1)c1cnc(Cl)nc1C(F)(F)F</td>\n",
              "      <td>6.420216</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CHEMBL334485</td>\n",
              "      <td>O=C(Nc1cc(Cl)cc(Cl)c1)c1cnc(Cl)nc1C(F)(F)F</td>\n",
              "      <td>6.301030</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1ae19278-c94a-43dd-9785-1e9771c037fb')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1ae19278-c94a-43dd-9785-1e9771c037fb button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1ae19278-c94a-43dd-9785-1e9771c037fb');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def smiles_to_fp(smiles, method=\"maccs\", n_bits=2048):\n",
        "    \"\"\"\n",
        "    Encode a molecule from a SMILES string into a fingerprint.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    smiles : str\n",
        "        The SMILES string defining the molecule.\n",
        "\n",
        "    method : str\n",
        "        The type of fingerprint to use. Default is MACCS keys.\n",
        "\n",
        "    n_bits : int\n",
        "        The length of the fingerprint.\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    array\n",
        "        The fingerprint array.\n",
        "\n",
        "    \"\"\"\n",
        "\n",
        "    # convert smiles to RDKit mol object\n",
        "    mol = Chem.MolFromSmiles(smiles)\n",
        "\n",
        "    if method == \"maccs\":\n",
        "        return np.array(MACCSkeys.GenMACCSKeys(mol))\n",
        "    if method == \"morgan2\":\n",
        "        return np.array(GetMorganFingerprintAsBitVect(mol, 2, nBits=n_bits))\n",
        "    if method == \"morgan3\":\n",
        "        return np.array(GetMorganFingerprintAsBitVect(mol, 3, nBits=n_bits))\n",
        "    else:\n",
        "        # NBVAL_CHECK_OUTPUT\n",
        "        print(f\"Warning: Wrong method specified: {method}. Default will be used instead.\")\n",
        "        return np.array(MACCSkeys.GenMACCSKeys(mol))"
      ],
      "metadata": {
        "id": "O4iESr2UTAAz"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Add column for fingerprint\n",
        "chembl_df[\"fp\"] = chembl_df[\"smiles\"].apply(smiles_to_fp)\n",
        "chembl_df.head(3)\n",
        "# NBVAL_CHECK_OUTPUT"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "id": "XBm4Na25TCcd",
        "outputId": "ba5f1e35-d121-451f-8c0e-fcb2969b43c2"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id                                             smiles  \\\n",
              "0       CHEMBL336546  O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...   \n",
              "1       CHEMBL131636  CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...   \n",
              "2       CHEMBL134490  CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...   \n",
              "\n",
              "      pIC50  active                                                 fp  \n",
              "0  7.301030     1.0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
              "1  6.769551     1.0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
              "2  6.552842     1.0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7cce5da9-8d63-4753-985d-f5efdc348639\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "      <th>active</th>\n",
              "      <th>fp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL336546</td>\n",
              "      <td>O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...</td>\n",
              "      <td>7.301030</td>\n",
              "      <td>1.0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL131636</td>\n",
              "      <td>CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...</td>\n",
              "      <td>6.769551</td>\n",
              "      <td>1.0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL134490</td>\n",
              "      <td>CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...</td>\n",
              "      <td>6.552842</td>\n",
              "      <td>1.0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7cce5da9-8d63-4753-985d-f5efdc348639')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7cce5da9-8d63-4753-985d-f5efdc348639 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7cce5da9-8d63-4753-985d-f5efdc348639');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_roc_curves_for_models(models, test_x, test_y, save_png=False):\n",
        "    \"\"\"\n",
        "    Helper function to plot customized roc curve.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    models: dict\n",
        "        Dictionary of pretrained machine learning models.\n",
        "    test_x: list\n",
        "        Molecular fingerprints for test set.\n",
        "    test_y: list\n",
        "        Associated activity labels for test set.\n",
        "    save_png: bool\n",
        "        Save image to disk (default = False)\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    fig:\n",
        "        Figure.\n",
        "    \"\"\"\n",
        "\n",
        "    fig, ax = plt.subplots()\n",
        "\n",
        "    # Below for loop iterates through your models list\n",
        "    for model in models:\n",
        "        # Select the model\n",
        "        ml_model = model[\"model\"]\n",
        "        # Prediction probability on test set\n",
        "        test_prob = ml_model.predict_proba(test_x)[:, 1]\n",
        "        # Prediction class on test set\n",
        "        test_pred = ml_model.predict(test_x)\n",
        "        # Compute False postive rate and True positive rate\n",
        "        fpr, tpr, thresholds = metrics.roc_curve(test_y, test_prob)\n",
        "        # Calculate Area under the curve to display on the plot\n",
        "        auc = roc_auc_score(test_y, test_prob)\n",
        "        # Plot the computed values\n",
        "        ax.plot(fpr, tpr, label=(f\"{model['label']} AUC area = {auc:.2f}\"))\n",
        "\n",
        "    # Custom settings for the plot\n",
        "    ax.plot([0, 1], [0, 1], \"r--\")\n",
        "    ax.set_xlabel(\"False Positive Rate\")\n",
        "    ax.set_ylabel(\"True Positive Rate\")\n",
        "    ax.set_title(\"Receiver Operating Characteristic\")\n",
        "    ax.legend(loc=\"lower right\")\n",
        "    # Save plot\n",
        "    if save_png:\n",
        "        fig.savefig(f\"{DATA}/roc_auc\", dpi=300, bbox_inches=\"tight\", transparent=True)\n",
        "    return fig"
      ],
      "metadata": {
        "id": "COhODkg0TiVk"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def model_performance(ml_model, test_x, test_y, verbose=True):\n",
        "    \"\"\"\n",
        "    Helper function to calculate model performance\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    ml_model: sklearn model object\n",
        "        The machine learning model to train.\n",
        "    test_x: list\n",
        "        Molecular fingerprints for test set.\n",
        "    test_y: list\n",
        "        Associated activity labels for test set.\n",
        "    verbose: bool\n",
        "        Print performance measure (default = True)\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    tuple:\n",
        "        Accuracy, sensitivity, specificity, auc on test set.\n",
        "    \"\"\"\n",
        "\n",
        "    # Prediction probability on test set\n",
        "    test_prob = ml_model.predict_proba(test_x)[:, 1]\n",
        "\n",
        "    # Prediction class on test set\n",
        "    test_pred = ml_model.predict(test_x)\n",
        "\n",
        "    # Performance of model on test set\n",
        "    accuracy = accuracy_score(test_y, test_pred)\n",
        "    sens = recall_score(test_y, test_pred)\n",
        "    spec = recall_score(test_y, test_pred, pos_label=0)\n",
        "    auc = roc_auc_score(test_y, test_prob)\n",
        "\n",
        "    if verbose:\n",
        "        # Print performance results\n",
        "        # NBVAL_CHECK_OUTPUT        print(f\"Accuracy: {accuracy:.2}\")\n",
        "        print(f\"Sensitivity: {sens:.2f}\")\n",
        "        print(f\"Specificity: {spec:.2f}\")\n",
        "        print(f\"AUC: {auc:.2f}\")\n",
        "\n",
        "    return accuracy, sens, spec, auc"
      ],
      "metadata": {
        "id": "lyGpuP7BTlyF"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def model_training_and_validation(ml_model, name, splits, verbose=True):\n",
        "    \"\"\"\n",
        "    Fit a machine learning model on a random train-test split of the data\n",
        "    and return the performance measures.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    ml_model: sklearn model object\n",
        "        The machine learning model to train.\n",
        "    name: str\n",
        "        Name of machine learning algorithm: RF, SVM, ANN\n",
        "    splits: list\n",
        "        List of desciptor and label data: train_x, test_x, train_y, test_y.\n",
        "    verbose: bool\n",
        "        Print performance info (default = True)\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    tuple:\n",
        "        Accuracy, sensitivity, specificity, auc on test set.\n",
        "\n",
        "    \"\"\"\n",
        "    train_x, test_x, train_y, test_y = splits\n",
        "\n",
        "    # Fit the model\n",
        "    ml_model.fit(train_x, train_y)\n",
        "\n",
        "    # Calculate model performance results\n",
        "    accuracy, sens, spec, auc = model_performance(ml_model, test_x, test_y, verbose)\n",
        "\n",
        "    return accuracy, sens, spec, auc"
      ],
      "metadata": {
        "id": "nGak5nNLTpOh"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fingerprint_to_model = chembl_df.fp.tolist()\n",
        "label_to_model = chembl_df.active.tolist()\n",
        "\n",
        "# Split data randomly in train and test set\n",
        "# note that we use test/train_x for the respective fingerprint splits\n",
        "# and test/train_y for the respective label splits\n",
        "(\n",
        "    static_train_x,\n",
        "    static_test_x,\n",
        "    static_train_y,\n",
        "    static_test_y,\n",
        ") = train_test_split(fingerprint_to_model, label_to_model, test_size=0.2, random_state=SEED)\n",
        "splits = [static_train_x, static_test_x, static_train_y, static_test_y]\n",
        "# NBVAL_CHECK_OUTPUT\n",
        "print(\"Training data size:\", len(static_train_x))\n",
        "print(\"Test data size:\", len(static_test_x))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m9wCL7BPTrqU",
        "outputId": "1f51748b-7671-4d72-8243-c4655488d1a9"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training data size: 39\n",
            "Test data size: 10\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set model parameter for random forest\n",
        "param = {\n",
        "    \"n_estimators\": 100,  # number of trees to grows\n",
        "    \"criterion\": \"entropy\",  # cost function to be optimized for a split\n",
        "}\n",
        "model_RF = RandomForestClassifier(**param)"
      ],
      "metadata": {
        "id": "tPyPvcrpT1wc"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit model on single split\n",
        "performance_measures = model_training_and_validation(model_RF, \"RF\", splits)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5k-n0yqqT4P_",
        "outputId": "d8e74796-95f7-4b65-915a-6c65ab0b9552"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sensitivity: 0.50\n",
            "Specificity: 0.88\n",
            "AUC: 0.88\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize the list that stores all models. First one is RF.\n",
        "models = [{\"label\": \"Model_RF\", \"model\": model_RF}]\n",
        "# Plot roc curve\n",
        "plot_roc_curves_for_models(models, static_test_x, static_test_y);"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "pNiArRiUT72r",
        "outputId": "0b142118-8cc3-4d2b-e16d-7af918523026"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gU5fbA8e+RYgBBEdArTYogJEAQIoheiiC9yUW9driCiijqT8Vru4JY7lWsKIgoAjaKBUVFURHEggUVkA7SkRqpUgPn98c7gSVskg3JZLK75/M8+2Rn5t2ZM5tkz868M+cVVcUYY0z8OiHoAIwxxgTLEoExxsQ5SwTGGBPnLBEYY0ycs0RgjDFxzhKBMcbEOUsEJkdEZL6ItAg6joJCRO4TkVcC2vZoEXkkiG3nNRG5SkQ+O87X2t9kLlkiiGIislJE9ojILhHZ4H0wnOTnNlU1SVWn+7mNdCJyooj8V0RWe/u5VET6i4jkx/bDxNNCRNaGzlPVx1S1t0/bExG5VUTmichfIrJWRN4Wkbp+bO94ichAEXkjN+tQ1TdVtU0E2zom+eXn32SsskQQ/Tqr6klAfeAc4N6A48kxESmcyaK3gVZAB6AkcA1wA/CcDzGIiBS0/4fngNuAW4FTgZrA+0DHvN5QFr8D3wW5beNRVXtE6QNYCVwUMv0E8HHI9HnAd8A2YA7QImTZqcAo4A9gK/B+yLJOwGzvdd8B9TJuEygP7AFODVl2DrAFKOJNXwcs9NY/BTgzpK0CNwNLgRVh9q0VsBeolGF+Y+AgcJY3PR34L/AjsAP4IENMWb0H04FHgW+9fTkL+JcX805gOXCj17aE1+YQsMt7lAcGAm94bap4+9UDWO29F/eHbK8YMMZ7PxYCdwNrM/nd1vD2s1EWv//RwFDgYy/eH4DqIcufA9Z478vPQNOQZQOBd4A3vOW9gUbATO+9Wg+8ABQNeU0S8DnwJ7ARuA9oB+wHDnjvyRyv7cnASG8964BHgELesp7ee/4MkOot6wl84y0Xb9kmL7bfgDq4LwEHvO3tAj7M+H8AFPLi+t17T34mw9+QPcL8LQUdgD1y8cs7+h+govcP85w3XcH7J+uAO/Jr7U2X85Z/DIwHSgNFgObe/HO8f8DG3j9VD287J4bZ5pfA9SHxDAaGe8+7AsuA2kBh4AHgu5C26n2onAoUC7Nv/wO+ymS/V3HkA3q690FTB/dh/S5HPpizew+m4z6wk7wYi+C+bVf3PoyaA7uBBl77FmT44CZ8IngZ96GfDOwDaofuk/eeVwTmZlxfyHr7AKuy+f2P9vankRf/m8C4kOVXA2W8ZXcCG4CEkLgPABd7700xoCEucRb29mUhcLvXviTuQ/1OIMGbbpzxPQjZ9kTgJe93chouUaf/znoCaUA/b1vFODoRtMV9gJ/i/R5qA2eE7PMjWfwf9Mf9H5ztvTYZKBP0/2pBfwQegD1y8ctz/wC7cN98FJgKnOIt+zfweob2U3Af7GfgvtmWDrPOF4GHM8xbzJFEEfpP1xv40nsuuG+fzbzpT4BeIes4AfeheqY3rUDLLPbtldAPtQzLvsf7po37MP9fyLJE3DfGQlm9ByGvHZTNe/w+cJv3vAWRJYKKIct/BC73ni8H2oYs651xfSHL7ge+zya20cArIdMdgEVZtN8KJIfEPSOb9d8OTPSeXwH8mkm7w++BN306LgEWC5l3BTDNe94TWJ1hHT05kghaAktwSemEMPucVSJYDHT14/8tlh8F7ZyoybmLVbUk7kOqFlDWm38mcKmIbEt/AH/HJYFKwJ+qujXM+s4E7szwukq40yAZvQs0EZEzgGa45PJ1yHqeC1nHn7hkUSHk9Wuy2K8tXqzhnOEtD7eeVbhv9mXJ+j0IG4OItBeR70XkT699B468p5HaEPJ8N5DegV8+w/ay2v9UMt//SLaFiNwlIgtFZLu3Lydz9L5k3PeaIvKRd+HBDuCxkPaVcKdbInEm7newPuR9fwl3ZBB226FU9UvcaamhwCYRGSEipSLcdk7iNB5LBDFCVb/CfVt60pu1Bvdt+JSQRwlV/Z+37FQROSXMqtYAj2Z4XXFVHRtmm1uBz4B/AlfivsFryHpuzLCeYqr6XegqstilL4DGIlIpdKaINMb9s38ZMju0TWXcKY8t2bwHx8QgIifiktuTwOmqegowGZfAsos3Eutxp4TCxZ3RVKCiiKQcz4ZEpCmuD+Iy3JHfKcB2juwLHLs/LwKLgBqqWgp3rj29/RqgWiaby7ieNbgjgrIh73spVU3K4jVHr1B1iKo2xB3h1cSd8sn2dd62q2fTxmRgiSC2PAu0FpFkXCdgZxFpKyKFRCTBu/yxoqqux526GSYipUWkiIg089bxMtBHRBp7V9KUEJGOIlIyk22+BVwLXOI9TzccuFdEkgBE5GQRuTTSHVHVL3Afhu+KSJK3D+d5+/Wiqi4NaX61iCSKSHFgEPCOqh7M6j3IZLNFgROBzUCaiLQHQi9p3AiUEZGTI92PDCbg3pPSIlIBuCWzht7+DQPGejEX9eK/XETuiWBbJXHn4TcDhUXkQSC7b9UlcZ2zu0SkFnBTyLKPgDNE5Hbvst6SXlIG975USb/qyvv7+gx4SkRKicgJIlJdRJpHEDcicq7391cE+At30cChkG1llpDAnVJ8WERqeH+/9USkTCTbjWeWCGKIqm4GXgMeVNU1uA7b+3AfBmtw36rSf+fX4L45L8J1Dt/urWMWcD3u0HwrrsO3ZxabnYS7wmWDqs4JiWUi8DgwzjvNMA9on8Nd6g5MAz7F9YW8gbsSpV+Gdq/jjoY24Doyb/ViyO49OIqq7vReOwG371d6+5e+fBEwFljunfIId7osK4OAtcAK3BHPO7hvzpm5lSOnSLbhTnl0Az6MYFtTcO/bEtzpsr1kfSoK4C7cPu/EfSEYn77Ae29aA51x7/NS4EJv8dvez1QR+cV7fi0usS7AvZfvENmpLnAJ62Xvdatwp8kGe8tGAone+/9+mNc+jfv9fYZLaiNxndEmC3LkSN6Y6CMi03EdlYHc3ZsbInITriM5om/KxvjFjgiMyScicoaIXOCdKjkbdynmxKDjMsbu6DMm/xTFXT1TFXeqZxyuH8CYQNmpIWOMiXN2asgYY+Jc1J0aKlu2rFapUiXoMIwxJqr8/PPPW1S1XLhlUZcIqlSpwqxZs4IOwxhjooqIrMpsmZ0aMsaYOGeJwBhj4pwlAmOMiXOWCIwxJs5ZIjDGmDjnWyIQkVdFZJOIzMtkuYjIEBFZJiJzRaSBX7EYY4zJnJ9HBKNx45lmpj2uamUN3FikL/oYizHGmEz4dh+Bqs4QkSpZNOkKvOYNZPK9iJwiImd4tcxNwN76YTUfzF4XdBjGGODEfXsotXMb5eqdzYDOSdm/IIeC7COowNH10ddy9DCGh4nIDSIyS0Rmbd68OV+Ci3cfzF7HgvU7gg7DmLiXtGgWTzx8DXe+dC8cOpT9C45DVNxZrKojgBEAKSkpViUvnySeUYrxNzYJOgxj4tO2bdC/P7zyCpx1FrwyggHN6/qyqSATwTqOHrO1ojfPGGPi28GDcP75sHgx3H03DBwIxfwbaC3IRDAJuEVExgGNge3WP2CMiWupqXDqqVCoEDz6KFSqBCkpvm/Wz8tHxwIzgbNFZK2I9BKRPiLSx2syGViOGxP3ZaCvX7EYY0yBpgpvvAE1a7pTQQDduuVLEgB/rxq6IpvlCtzs1/aNMSYqrFkDffrA5Mlw3nlwwQX5HoLdWWyMMUEZOxaSkmD6dHj2WfjmG0hMzPcwouKqIWOMiUmlS0PjxjBiBFStGlgYlgiMMSa/pKXBM8/A/v1w//3Qrh20bQsigYZlp4aMMSY/zJnj+gDuvhvmznUdxBB4EgBLBMYY4699++A//3FXAK1ZA2+/DePGFYgEkM4SgTHG+GnpUnj8cbjySliwAC65pEAlAbA+AmOMyXu7dsEHH8BVV0GdOrBoEVSrFnRUmbIjAmOMyUuffw5168I118DChW5eAU4CYInAGGPyxtat0KsXtGkDRYvCV19B7dpBRxUROzVkjDG5dfCguyN4yRK491548EFISAg6qohZIjDGmOO1ZcuRInGPPQaVK0OD6Bt1104NGWNMTqnCa68dXSTu4oujMgmAJQJjjMmZVaugfXvo0cP1ATRrFnREuWaJwBhjIvXGG+5y0G++geefh6+/hlq1go4q16yPwBhjIlWunOsUfuklOPPMoKPJM5YIjDEmMwcOwFNPuZ//+Y8rENemTYG7Mzi37NSQMcaE8+uvrkT0vfe60hAFqEhcXrNEYIwxofbuhfvug3PPhT/+gHffdQPIxGACSGeJwBhjQi1bBk8+Cdde60pE/OMfQUfkO+sjMMaYXbtg4kRXH6hOHVi8ONARw/KbHREYY+LblClu3OAePY4UiYujJACWCIwx8So11X34t2sHxYu7ewKipEhcXrNTQ8aY+JNeJG7ZMjd28AMPRFWRuLxmicAYEz82b4YyZVyRuMcfdzeF1a8fdFSBs1NDxpjYpwqjRrkicS+/7OZ17WpJwGOJwBgT21audHcEX3edGznswguDjqjAsURgjIldr7/uLgedOROGDYPp091RgTmK9REYY2LX6ae7MtHDh7tBY0xYlgiMMbHjwAF44gl3VdCDD7oCcW3aBB1VgWenhowxseGXX1x9oAcecHcGpxeJM9myRGCMiW579sA990CjRrBxoysV8eabMV0kLq/5mghEpJ2ILBaRZSJyT5jllUVkmoj8KiJzRaSDn/EYY2LQ8uXw9NPQs6crF33xxUFHFHV8SwQiUggYCrQHEoErRCQxQ7MHgAmqeg5wOTDMr3iMMTFkxw4YPdo9T0qCpUvdIPKlSwcaVrTy84igEbBMVZer6n5gHNA1QxsFSnnPTwb+8DEeY0wsmDzZXRLaq9eRInExNGxkEPxMBBWANSHTa715oQYCV4vIWmAy0C/cikTkBhGZJSKzNm/e7EesxpiCbssWVya6Y0coWRK+/TZui8TltaA7i68ARqtqRaAD8LqIHBOTqo5Q1RRVTSlXrly+B2mMCVh6kbhx49xlob/8AuedF3RUMcPP+wjWAZVCpit680L1AtoBqOpMEUkAygKbfIzLGBMtNm6EcuVckbgnn3SngOrVCzqqmOPnEcFPQA0RqSoiRXGdwZMytFkNtAIQkdpAAmDnfoyJd6owciScfTaMGOHmde5sScAnviUCVU0DbgGmAAtxVwfNF5FBItLFa3YncL2IzAHGAj1V7S4QY+La8uVw0UXQu7erDnrRRUFHFPN8LTGhqpNxncCh8x4Meb4AuMDPGIwxUWTMGOjb150KGj4crr8eTgi6KzP2Wa0hY0zBUb48tGwJL74IFSsGHU3csERgjAnO/v3wv//BoUMwcCC0bu0eJl/ZMZcxJhg//QQNG8KAAa5fwLoHA2OJwBiTv3bvhrvucvcBbN0KkybBa69ZkbgAWSIwxuSvFSvg+eddR/D8+e6yUBMo6yMwxvhv+3Z47z34179ckbhly6BSpexfZ/KFHREYY/z18cfuw793b1i0yM2zJFCgWCIwxvhj82a46iro1MmVh545E2rVCjoqE4adGjLG5L2DB+Hvf3f9AQ895EYQK1o06KhMJiwRGGPyzoYNcNpp7s7gp56CKlXc2AGmQIv41JCIFPczEGNMFDt0CF56CWrWdD/BnRKyJBAVsk0EInK+iCwAFnnTySJiQ0oaY5xly6BVK+jTB849F9q2DToik0ORHBE8A7QFUgFUdQ7QzM+gjDFRYtQoqFvXDRTz8svwxRdQrVrQUZkciqiPQFXXyNF3/R30JxxjTFSpXNkdAQwdChUyjkRrokUkiWCNiJwPqIgUAW7DjS9gjIk3+/bBf//r+gQGDXKnhFq1Cjoqk0uRnBrqA9yMG3h+HVAf6OtnUMaYAuiHH1yRuIcegtWrrUhcDIkkEZytqlep6umqepqqXg3U9jswY0wB8ddfcMcd0KSJKxXx0UcwerQViYshkSSC5yOcZ4yJRatWwbBh7qqg+fOhY8egIzJ5LNM+AhFpApwPlBORO0IWlQIK+R2YMSZA27bBO++4+kCJie4SURsxLGZldURQFDgJlyxKhjx2AJf4H5oxJhAffOA+/Pv0OVIkzpJATMv0iEBVvwK+EpHRqroqH2MyxgRh0ya49VYYPx7q1XMDxliRuLgQyeWju0VkMJAEJKTPVNWWvkVljMlfBw/CBRe4q4EeeQTuvhuKFAk6KpNPIkkEbwLjgU64S0l7AJv9DMoYk0/++AP+9jdXJO6551yRuMTEoKMy+SySq4bKqOpI4ICqfqWq1wF2NGBMNDt0CF580Z36GT7czevQwZJAnIrkiOCA93O9iHQE/gBO9S8kY4yvlixx4wXPmAEXXQTt2wcdkQlYJIngERE5GbgTd/9AKeB2X6Myxvhj5Ei45RZISIBXX4WePe3GMJN9IlDVj7yn24ELAUTkAj+DMsb4pEoVdwQwdCiccUbQ0ZgCIqsbygoBl+FqDH2qqvNEpBNwH1AMOCd/QjTGHLd9++Dhh93zRx6xInEmrKyOCEYClYAfgSEi8geQAtyjqu/nR3DGmFz47jvo1cvdFHbdda5InJ0GMmFklQhSgHqqekhEEoANQHVVTc2f0Iwxx2XXLrj/fnj+eahUCT791EYNM1nK6vLR/ap6CEBV9wLLc5oERKSdiCwWkWUick8mbS4TkQUiMl9E3srJ+o0xYaxe7cYNvvlmmDfPkoDJVlZHBLVEZK73XIDq3rQAqqr1slqx18cwFGgNrAV+EpFJqrogpE0N4F7gAlXdKiKn5WJfjIlfW7fC22/DDTe4ewGWL4fy5YOOykSJrBJBbsccaAQsU9XlACIyDugKLAhpcz0wVFW3Aqjqplxu05j4M3Ei9O0LmzdD8+Zw9tmWBEyOZFV0LreF5ioAa0Km1wKNM7SpCSAi3+JKWw9U1U8zrkhEbgBuAKhcuXIuwzImRmzYAP36uXLR9evDxx+7JGBMDkU0eL3P268BtAAqAjNEpK6qbgttpKojgBEAKSkpNj6eMQcPQtOmsGYNPPYY3HWXFYkzx83PRLAOd/lpuorevFBrgR9U9QCwQkSW4BLDTz7GZUz0WrvWnfYpVAiGDIGqVa1UtMm1SIrOISLFRCSnx5w/ATVEpKqIFAUuByZlaPM+7mgAESmLO1W0PIfbMSb2HTrkLgetVcsViwN3h7AlAZMHsk0EItIZmA186k3XF5GMH+jHUNU04BZgCrAQmKCq80VkkIh08ZpNAVJFZAEwDehv9ykYk8GiRdCsmRs05u9/h06dgo7IxJhITg0NxF0BNB1AVWeLSNVIVq6qk4HJGeY9GPJcgTu8hzEmo1decUXiiheHMWPgmmvs7mCT5yIqQ62q2+XoPz7rsDUmP1SvDp07wwsvwOmnBx2NiVGRJIL5InIlUMi7AexW4Dt/wzImTu3dC4MGueePPQYXXugexvgoks7ifrjxivcBb+HKUdt4BMbktW+/dfcD/Pe/7uYwtQNvkz8iOSKopar3A/f7HYwxcWnnTrjvPjdGwJlnwpQp0KZN0FGZOBLJEcFTIrJQRB4WkTq+R2RMvFm71nUK9+sHv/1mScDku2wTgapeiBuZbDPwkoj8JiIP+B6ZMbEsNfXI/QC1a7sicc89ByedFGxcJi5FdEOZqm5Q1SFAH9w9BQ9m8xJjTDiqrjZQYqK7L2DxYjffho00AYrkhrLaIjJQRH7DDV7/Ha5chDEmJ9avh+7d4dJL3YAxs2ZZkThTIETSWfwqMB5oq6p/+ByPMbEpvUjcunXwxBPwf/8HhYOu+WiMk+1foqo2yY9AjIlJa9ZAhQquSNzQoa5IXM2aQUdlzFEyPTUkIhO8n7+JyNyQx28hI5cZY8I5eNBVBw0tEte2rSUBUyBldURwm/fTKlwZkxMLF0KvXjBzpqsQ2rlz0BEZk6VMjwhUdb33tK+qrgp9AH3zJzxjosyIEe7u4CVL4PXX3ahhNqqeKeAi6a1qDfw7w7z2YeYVaG/9sJoPZmccF8dkZsH6HSSeUSroMKJPjRrQrZs7LXTaaUFHY0xEMk0EInIT7pt/tQx9AiWBb/0OLK99MHudfbjlQOIZpehav0LQYRR8e/bAwIGuNPT//mdF4kxUyuqI4C3gE+C/wD0h83eq6p++RuWTxDNKMf5GuwjK5JEZM6B3b1i6FPr0cTeL2VgBJgpldUOZqupK4GZgZ8gDETnV/9CMKaB27IC+faF5c3d10NSp7sogSwImSmV3RNAJ+Bk3EE3oX7kC1XyMy5iC648/YPRouOMON3ZAiRJBR2RMrmSaCFS1k/czomEpjYlpW7bAhAnuSKBWLVixwkYMMzEjklpDF4hICe/51SLytIjY9XAmPqjC+PGuSNztt7vLQsGSgIkpkVQffRHYLSLJwJ3A78DrvkZlTEHwxx9w8cVw+eVuwJiff7Y7g01MiiQRpKmqAl2BF1R1KO4SUmNi18GD0KwZfPYZPPmku0u4bt2gozLGF5HcULZTRO4FrgGaisgJQBF/wzImIKtWQcWKrkjcsGFQrRqcdVbQURnjq0iOCP6JG7j+OlXdgBuLYLCvURmT3w4ehKefdqOFpReJa9PGkoCJC5EMVbkBeBM4WUQ6AXtV9TXfIzMmv8ybB+efD3feCa1auX4BY+JIJFcNXQb8CFwKXAb8ICKX+B2YMfli+HBo0MCNGfzWWzBpkjs1ZEwciaSP4H7gXFXdBCAi5YAvgHf8DMwYX6WXg6hd2w0d+eyzUK5c0FEZE4hIEsEJ6UnAk0qEg94bU+Ds3g0PPug6gx9/3JWJaN486KiMCVQkH+ifisgUEekpIj2Bj4HJ/oZljA+mT4d69eCpp2DXLndUYIyJqLO4P/ASUM97jFDVqBqLwMS57dvhxhuPlIf+8ks3frAViTMGyHo8ghrAk0B14DfgLlW1kV1M9Fm/Ht54A+66Cx56CIoXDzoiYwqUrI4IXgU+ArrjKpA+n9OVi0g7EVksIstE5J4s2nUXERWRlJxuw5iwNm+G570/2Vq1YOVKGDzYkoAxYWSVCEqq6suqulhVnwSq5GTFIlIIGIob1jIRuEJEEsO0KwncBvyQk/UbE5aquwy0dm13X0B6kTi7IsiYTGWVCBJE5BwRaSAiDYBiGaaz0whYpqrLVXU/MA5Xryijh4HHgb05jt6YUGvWQOfOcNVV7o7gX3+1InHGRCCry0fXA0+HTG8ImVagZTbrrgCsCZleCzQObeAllEqq+rGI9M9sRSJyA3ADQOXKVgHbhJGWBi1awIYN8Mwz0K+fu0TUGJOtrAam8XUEbq943dNAz+zaquoIYARASkqKXfNnjli5EipVgsKF4aWXXJG4ajZ4njE54eeNYeuASiHTFb156UoCdYDpIrISOA+YZB3GJiJpaa48dO3arkoowEUXWRIw5jhEcmfx8foJqCEiVXEJ4HLgyvSFqrodKJs+LSLTcZeozvIxJhML5s6FXr1g1izo2hW6dw86ImOimm9HBKqaBtwCTAEWAhNUdb6IDBKRLn5t18S4YcOgYUM3bsD48TBxIpQvH3RUxkS1bI8IRESAq4BqqjrIG6/4b6r6Y3avVdXJZChHoaoPZtK2RUQRm/iUXiSuTh03dOQzz0DZstm/zhiTrUiOCIYBTYArvOmduPsDjPHfX3/B//0f3H23m27WDF5/3ZKAMXkokkTQWFVvxrvOX1W3AkV9jcoYgKlT3TjBzz4L+/ZZkThjfBJJIjjg3SWscHg8gkO+RmXi27Zt0Lu3uwqocGGYMQOGDLEiccb4JJJEMASYCJwmIo8C3wCP+RqViW8bN8K4cfDvf8OcOdC0adARGRPTsu0sVtU3ReRnoBUgwMWqutD3yEx8Sf/wv+02OPtsd6OY9QMYky8iGbO4MrAb+BCYBPzlzTMm91RdiejERNchvHSpm29JwJh8E8kNZR/j+gcESACqAouBJB/jMvFg9Wro0wc++QSaNIGRI6FGjaCjMibuRHJqqG7otFcorq9vEZn4kF4kbtMm1xHct68ViTMmIDkuMaGqv4hI4+xbGhPG8uVw5pnuaqCXX4bq1aFKlaCjMiauRXJn8R0hkycADYA/fIvIxKa0NDdo/IAB8MQTcOut0KpV0FEZY4jsiKBkyPM0XJ/Bu/6EY2LS7NmuSNwvv0C3bnDppUFHZIwJkWUi8G4kK6mqd+VTPCbWvPCCKxFRpgy8845VCjWmAMr08lERKayqB4EL8jEeEyvSy0HUq+eGjlywwJKAMQVUVkcEP+L6A2aLyCTgbeCv9IWq+p7PsZlotGsX3H8/FCniBo5p1sw9jDEFViQlJhKAVNwYxZ2Azt5PY4722WeuTPTzz8OBA1YkzpgokdURwWneFUPzOHJDWTr7DzdHbN0Kd9wBo0e78hAzZsDf/x50VMaYCGWVCAoBJ3F0AkhnicAcsWmT6wi+91548EFISAg6ImNMDmSVCNar6qB8i8RElw0bYOxYd0VQepG4MmWCjsoYcxyy6iOw4u/mWKowZowrEnfvvUeKxFkSMCZqZZUI7LZPc7SVK6FdO+jZ0yWC2bOtSJwxMSDTU0Oq+md+BmIKuLQ0uPBC2LIFhg51VUNPiOSiM2NMQZfjonMmzixbBlWruiJxr74K1aq5onHGmJhhX+lMeAcOwGOPQVKSOwIAd0RgScCYmGNHBOZYv/ziisTNnu0KxP3zn0FHZIzxkR0RmKMNGQKNGrnLQ997DyZMgNNPDzoqY4yPLBEYJ70cxDnnwLXXuiJx3boFG5MxJl/YqaF4t3Onux/gxBPdwDFNm7qHMSZu2BFBPPv0U1ckbtgwd0RgReKMiUuWCOJRair06AHt20OJEvDtt/D00yB2M7kx8cgSQTxKTYWJE+E//4Fff4UmTYKOyBgTIF8TgYi0E5HFIrJMRO4Js/wOEVkgInNFZKqI2EXqflm/3g0Uowo1a8KqVTBokOsbMMbENd8SgTfe8VCgPZAIXCEiiRma/QqkqGo94B3gCb/iiVuq7o7g2rXdEcCyZW5+6dLBxmWMKTD8PCJoBCxT1eWquh8YB3QNbaCq01R1tzf5PVDRx3jiz4oV0CMad8wAABWqSURBVKaNuzksORnmzLEiccaYY/h5+WgFYE3I9FqgcRbtewGfhFsgIjcANwBUrlw5r+KLbWlp0LKl6w948UW44QYrEmeMCatA3EcgIlcDKUDzcMtVdQQwAiAlJcWucczK0qWuMFzhwjBqFFSvDpUqBR2VMaYA8/Mr4jog9BOoojfvKCJyEXA/0EVV9/kYT2w7cAAeecTdF/DCC25eixaWBIwx2fLziOAnoIaIVMUlgMuBK0MbiMg5wEtAO1Xd5GMssW3WLNcPMHcuXH45XHFF0BEZY6KIb0cEqpoG3AJMARYCE1R1vogMEpEuXrPBwEnA2yIyW0Qm+RVPzHruOWjc2A0Y88EHbhzh004LOipjTBTxtY9AVScDkzPMezDk+UV+bj+mqbo7gVNS3NHAE0/AKacEHZUxJgoViM5ikwM7dsC//w0JCfDMM3DBBe5hjDHHya4njCaTJ7sRw0aMcFcFWZE4Y0wesEQQDbZsgauvho4d4eST4bvvYPBgKxJnjMkTlgiiwdat8OGHMGCAG0aycVb35RljTM5YH0FBtW4dvPkm9O/vykKsWmWdwcYYX9gRQUGjCi+/DImJMHAg/P67m29JwBjjE0sEBcnvv0OrVq4uUIMG7gaxs84KOipjTIyzU0MFRVqaSwJ//gkvvQS9e1uROGNMvrBEELTFi11huMKFYcwY97yiVeM2xuQf+8oZlP374aGHoG5dGDrUzWve3JKAMSbf2RFBEH780ZWFmDcPrrwSrroq6IiMMXHMjgjy27PPusHi0+8NePNNKFs26KiMMXHMEkF+SS8H0agRXH89zJ8PnToFG5MxxmCnhvy3fTvcfTcUK+aOBs4/3z2MMaaAsCMCP334obsx7JVX4MQTrUicMaZAskTgh82bXSdwly5Qpgx8/z08/rgViTPGFEiWCPywfbsrGf3QQ24YyXPPDToiY4zJlPUR5JU1a+CNN+Cee1xZiFWrXMloY4wp4OyIILcOHYLhw92AMY88cqRInCUBY0yUsESQG0uXQsuWcNNN7rLQ336zInHGmKhjp4aOV1oatG4N27bByJHwr39ZZ3AcOHDgAGvXrmXv3r1Bh2JMWAkJCVSsWJEiRYpE/BpLBDm1cKEbKKZwYXj9dVckrnz5oKMy+WTt2rWULFmSKlWqIJb4TQGjqqSmprJ27VqqVq0a8evs1FCk9u1zQ0XWqwcvvODmNW1qSSDO7N27lzJlylgSMAWSiFCmTJkcH7HaEUEkvv/eFYlbsACuucY9TNyyJGAKsuP5+7Qjguw89ZQrCbFzp7s34LXX3E1ixhgTIywRZObQIfezSRPo08eVjG7fPtiYjDHGB5YIMtq2zZ0Guu02N33++TBsGJQqFWxcxnhEhKuvvvrwdFpaGuXKlaNTDqvZVqlShS1btuSqTaFChahfvz516tShc+fObNu2DYCVK1dSrFgx6tevf/ixf//+sOu4/fbbqVChAofSv3wBAwcO5Mknn8w0lg0bNnD55ZdTvXp1GjZsSIcOHViyZElE+x20MWPGUKNGDWrUqMGYMWPCtpk9ezbnnXce9evXJyUlhR9//BGA7du307lzZ5KTk0lKSmLUqFF5EpP1EYR6/33o2xc2bXIVQ1XtklCTqYc+nM+CP3bk6ToTy5diQOekLNuUKFGCefPmsWfPHooVK8bnn39OhQoV8jSOSBUrVozZs2cD0KNHD4YOHcr9998PQPXq1Q8vy8yhQ4eYOHEilSpV4quvvuLCCy/MdpuqSrdu3ejRowfjxo0DYM6cOWzcuJGaNWvmeB8OHjxIoUKFcvy64/Hnn3/y0EMPMWvWLESEhg0b0qVLF0qXLn1Uu7vvvpsBAwbQvn17Jk+ezN1338306dMZOnQoiYmJfPjhh2zevJmzzz6bq666iqJFi+YqLjsiAPfBf9ll0K0bnH66G0HssccsCZgCq0OHDnz88ccAjB07liuuuOLwsj///JOLL76YevXqcd555zF37lwAUlNTadOmDUlJSfTu3RsNqYb7xhtv0KhRI+rXr8+NN97IwYMHcxxTkyZNWLduXY5eM336dJKSkrjpppsYO3ZsRK+ZNm0aRYoUoU+fPofnJScn07Rp02PaXnzxxTRs2JCkpCRGjBhxeP5JJ53EnXfeSXJyMjNnzsx0/2+66SZSUlJISkpiwIABOdq3cKZMmULr1q059dRTKV26NK1bt+bTTz89pp2IsGOH+5Kxfft2yntXJ4oIO3fuRFXZtWsXp556KoUL58H3eVWNqkfDhg31eFw2/Du9bPh34RcuXap6yimqjz6qun//ca3fxIcFCxYEHYKWKFFC58yZo927d9c9e/ZocnKyTps2TTt27KiqqrfccosOHDhQVVWnTp2qycnJqqrar18/feihh1RV9aOPPlJAN2/erAsWLNBOnTrpfu9v/6abbtIxY8aoquqZZ56pmzdvzjIWVdW0tDS95JJL9JNPPlFV1RUrVmhCQoImJydrcnKy9u3bN+zre/fura+99ppu375dy5cvfziGAQMG6ODBg49qmx7Lc889p7fffntE71Vqaqqqqu7evVuTkpJ0y5YtqqoK6Pjx41VVs9z/9NenpaVp8+bNdc6cOcds44knnji8n6GPfv36HdN28ODB+vDDDx+eHjRo0DH7mR5TpUqVtGLFilq+fHlduXKlqqru2LFDW7RooX/729+0RIkS+tFHH4Xd73B/p8AszeRzNX5PDa1e7W4Iu+8+VxZi9WooWTLoqIyJSL169Vi5ciVjx46lQ4cORy375ptvePfddwFo2bIlqamp7NixgxkzZvDee+8B0LFjx8OnI6ZOncrPP//MuV6V3D179nDaaadFFMeePXuoX78+69ato3bt2rRu3frwsuxODe3fv5/Jkyfz9NNPU7JkSRo3bsyUKVPo1KlTppdA5vTSyCFDhjBx4kQA1qxZw9KlSylTpgyFChWie/fuQNb7P2HCBEaMGEFaWhrr169nwYIF1KtX76ht9O/fn/79++coruy8+OKLPPPMM3Tv3p0JEybQq1cvvvjiC6ZMmUL9+vX58ssv+f3332ndujVNmzalVC77MH09NSQi7URksYgsE5F7wiw/UUTGe8t/EJEqfsYDuKuBhg1zReIee+xIkThLAibKdOnShbvuuuuo00LHQ1Xp0aMHs2fPZvbs2SxevJiBAwdG9Nr0PoJVq1ahqgwdOjTi7U6ZMoVt27ZRt25dqlSpwjfffHP49FCZMmXYunXrUe137tzJKaecQlJSEj///HO2658+fTpffPEFM2fOZM6cOZxzzjmHb7RKSEg43C+Q2f6vWLGCJ598kqlTpzJ37lw6duwY9katwYMHH9Upnv649dZbj2lboUIF1qxZc3h67dq1Yft3xowZwz/+8Q8ALr300sOdxaNGjeIf//gHIsJZZ51F1apVWbRoUbbvRXZ8SwQiUggYCrQHEoErRCQxQ7NewFZVPQt4Bnjcr3gAztiwClq0gJtvdpeFzp9vReJM1LruuusYMGAAdevWPWp+06ZNefPNNwH3YVi2bFlKlSpFs2bNeOuttwD45JNPDn/QtmrVinfeeYdNmzYBro9h1apVOYqlePHiDBkyhKeeeoq0tLSIXjN27FheeeUVVq5cycqVK1mxYgWff/45u3fvplmzZkyaNImdO3cC8N5775GcnEyhQoVo2bIl+/btO+qc/9y5c/n666+PWv/27dspXbo0xYsXZ9GiRXz//fdh48hs/3fs2EGJEiU4+eST2bhxI5988knY1/fv3/9wEgl9DBky5Ji2bdu25bPPPmPr1q1s3bqVzz77jLZt2x7Trnz58nz11VcAfPnll9SoUQOAypUrM3XqVAA2btzI4sWLqVatWpbvc0QyO2eU2wfQBJgSMn0vcG+GNlOAJt7zwsAWQLJa7/H2EVw+dIZuPPVvri9g1CjVQ4eOaz0mvhWUPoKMQvsIUlNTtWvXrlq3bl1t3Ljx4fPaW7Zs0datW2tiYqL27t1bK1eufPj8/7hx4zQ5OVnr1q2rDRo00JkzZ6pq5H0E6Tp16qSvvfaarlixQpOSkjJ93V9//aWlS5fW7du3HzW/W7duOm7cOFVVHT58uNarV0+Tk5O1devW+vvvvx9ut27dOr300ku1WrVqmpiYqB06dNAlS5Ycta69e/dqu3bttFatWtq1a1dt3ry5Tps2LWzcme1/jx49tEaNGtqyZUvt1q2bjho1KtN9itTIkSO1evXqWr16dX311VcPz+/Vq5f+9NNPqqr69ddfa4MGDbRevXraqFEjnTVr1uH9bt26tdapU0eTkpL09ddfD7uNnPYRiPo0jq6IXAK0U9Xe3vQ1QGNVvSWkzTyvzVpv+nevzZYM67oBuAGgcuXKDXP6bQXcpX6VFvzCdddeBGeccby7ZeLcwoULqV27dtBhGJOlcH+nIvKzqqaEax8VncWqOgIYAZCSknJcmWtA5yTI5vpsY4yJR34mgnVApZDpit68cG3Wikhh4GQg1ceYjDE5lJqaSqtWrY6ZP3XqVMpY3a2Y4Gci+AmoISJVcR/4lwNXZmgzCegBzAQuAb5Uv85VGZNHVDWuKpCWKVMm2zuETcFxPB+hvl01pKppwC24DuGFwARVnS8ig0Ski9dsJFBGRJYBdwDHXGJqTEGSkJBAamrqcf2zGeM39QamSUhIyNHrfOss9ktKSorOmjUr6DBMnLKhKk1Bl9lQlVHfWWxMQVGkSJEcDQFoTDSwonPGGBPnLBEYY0ycs0RgjDFxLuo6i0VkM5DzW4udsrgyFvHE9jk+2D7Hh9zs85mqWi7cgqhLBLkhIrMy6zWPVbbP8cH2OT74tc92asgYY+KcJQJjjIlz8ZYIRmTfJObYPscH2+f44Ms+x1UfgTHGmGPF2xGBMcaYDCwRGGNMnIvJRCAi7URksYgsE5FjKpqKyIkiMt5b/oOIVMn/KPNWBPt8h4gsEJG5IjJVRM4MIs68lN0+h7TrLiIqIlF/qWEk+ywil3m/6/ki8lZ+x5jXIvjbriwi00TkV+/vu0MQceYVEXlVRDZ5IziGWy4iMsR7P+aKSINcbzSzMSyj9QEUAn4HqgFFgTlAYoY2fYHh3vPLgfFBx50P+3whUNx7flM87LPXriQwA/geSAk67nz4PdcAfgVKe9OnBR13PuzzCOAm73kisDLouHO5z82ABsC8TJZ3AD4BBDgP+CG324zFI4JGwDJVXa6q+4FxQNcMbboCY7zn7wCtJLpHGsl2n1V1mqru9ia/x40YF80i+T0DPAw8DsRC3ehI9vl6YKiqbgVQ1U35HGNei2SfFSjlPT8Z+CMf48tzqjoD+DOLJl2B19T5HjhFRHI1EHssJoIKwJqQ6bXevLBt1A2gsx2I5jH3ItnnUL1w3yiiWbb77B0yV1LVj/MzMB9F8nuuCdQUkW9F5HsRaZdv0fkjkn0eCFwtImuByUC//AktMDn9f8+WjUcQZ0TkaiAFaB50LH4SkROAp4GeAYeS3wrjTg+1wB31zRCRuqq6LdCo/HUFMFpVnxKRJsDrIlJHVQ8FHVi0iMUjgnVApZDpit68sG1EpDDucDI1X6LzRyT7jIhcBNwPdFHVffkUm1+y2+eSQB1guoisxJ1LnRTlHcaR/J7XApNU9YCqrgCW4BJDtIpkn3sBEwBUdSaQgCvOFqsi+n/PiVhMBD8BNUSkqogUxXUGT8rQZhLQw3t+CfCler0wUSrbfRaRc4CXcEkg2s8bQzb7rKrbVbWsqlZR1Sq4fpEuqhrN45xG8rf9Pu5oABEpiztVtDw/g8xjkezzaqAVgIjUxiWCzfkaZf6aBFzrXT10HrBdVdfnZoUxd2pIVdNE5BZgCu6Kg1dVdb6IDAJmqeokYCTu8HEZrlPm8uAizr0I93kwcBLwttcvvlpVuwQWdC5FuM8xJcJ9ngK0EZEFwEGgv6pG7dFuhPt8J/CyiPwfruO4ZzR/sRORsbhkXtbr9xgAFAFQ1eG4fpAOwDJgN/CvXG8zit8vY4wxeSAWTw0ZY4zJAUsExhgT5ywRGGNMnLNEYIwxcc4SgTHGxDlLBKZAEpGDIjI75FEli7a78mB7o0VkhbetX7w7VHO6jldEJNF7fl+GZd/lNkZvPenvyzwR+VBETsmmff1or8Zp/GeXj5oCSUR2qepJed02i3WMBj5S1XdEpA3wpKrWy8X6ch1TdusVkTHAElV9NIv2PXFVV2/J61hM7LAjAhMVROQkbxyFX0TkNxE5ptKoiJwhIjNCvjE39ea3EZGZ3mvfFpHsPqBnAGd5r73DW9c8Ebndm1dCRD4WkTne/H9686eLSIqI/A8o5sXxprdsl/dznIh0DIl5tIhcIiKFRGSwiPzk1Zi/MYK3ZSZesTERaeTt468i8p2InO3diTsI+KcXyz+92F8VkR+9tuEqtpp4E3TtbXvYI9wDd1fsbO8xEXcXfClvWVncXZXpR7S7vJ93Avd7zwvh6g2VxX2wl/Dm/xt4MMz2RgOXeM8vBX4AGgK/ASVwd2XPB84BugMvh7z2ZO/ndLwxD9JjCmmTHmM3YIz3vCiuimQx4AbgAW/+icAsoGqYOHeF7N/bQDtvuhRQ2Ht+EfCu97wn8ELI6x8Drvaen4KrRVQi6N+3PYJ9xFyJCRMz9qhq/fQJESkCPCYizYBDuG/CpwMbQl7zE/Cq1/Z9VZ0tIs1xg5V865XWKIr7Jh3OYBF5AFenpheufs1EVf3Li+E9oCnwKfCUiDyOO530dQ726xPgORE5EWgHzFDVPd7pqHoiconX7mRcsbgVGV5fTERme/u/EPg8pP0YEamBK7NQJJPttwG6iMhd3nQCUNlbl4lTlghMtLgKKAc0VNUD4iqKJoQ2UNUZXqLoCIwWkaeBrcDnqnpFBNvor6rvpE+ISKtwjVR1ibixDjoAj4jIVFUdFMlOqOpeEZkOtAX+iRtoBdxoU/1UdUo2q9ijqvVFpDiu/s7NwBDcADzTVLWb17E+PZPXC9BdVRdHEq+JD9ZHYKLFycAmLwlcCBwz5rK4cZg3qurLwCu44f6+By4QkfRz/iVEpGaE2/wauFhEiotICdxpna9FpDywW1XfwBXzCzdm7AHvyCSc8bhCYelHF+A+1G9Kf42I1PS2GZa60eZuBe6UI6XU00sR9wxpuhN3iizdFKCfeIdH4qrSmjhnicBEizeBFBH5DbgWWBSmTQtgjoj8ivu2/ZyqbsZ9MI4Vkbm400K1Itmgqv6C6zv4Eddn8Iqq/grUBX70TtEMAB4J8/IRwNz0zuIMPsMNDPSFuuEXwSWuBcAv4gYtf4lsjti9WObiBmZ5Avivt++hr5sGJKZ3FuOOHIp4sc33pk2cs8tHjTEmztkRgTHGxDlLBMYYE+csERhjTJyzRGCMMXHOEoExxsQ5SwTGGBPnLBEYY0yc+3+oncvgDsmf2AAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Specify model\n",
        "model_SVM = svm.SVC(kernel=\"rbf\", C=1, gamma=0.1, probability=True)\n",
        "\n",
        "# Fit model on single split\n",
        "performance_measures = model_training_and_validation(model_SVM, \"SVM\", splits)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EfQ5UuKQUCj9",
        "outputId": "e83339e1-ab3d-4aa4-8d68-665c0cba1b5f"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sensitivity: 0.00\n",
            "Specificity: 1.00\n",
            "AUC: 0.06\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Append SVM model\n",
        "models.append({\"label\": \"Model_SVM\", \"model\": model_SVM})\n",
        "# Plot roc curve\n",
        "plot_roc_curves_for_models(models, static_test_x, static_test_y);"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "yF9PJI5NUAQU",
        "outputId": "a3e38493-19ea-4569-92b4-c149f6766b62"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVhUZfvA8e/jlnshWqmgomgKCqSoaamkqWUu+WqZr/VmaotptqvVm1u226KlmWVq6QsuZXtZmVqZmcvPfQlNFNcQV3JB8P798QzjAAMMyjAs9+e65mLmnGfOuc8A55mzPPdtRASllFLFVwlfB6CUUsq3tCNQSqliTjsCpZQq5rQjUEqpYk47AqWUKua0I1BKqWJOOwKVK8aYzcaYKF/HUVAYY54xxnzgo3XPNMaM98W685oxpp8x5vuLfK/+TV4i7QgKMWNMnDHmtDEmyRhz0LFjqOjNdYpIqIgs9eY60hhjLjPGvGSM2ePYzlhjzFPGGJMf63cTT5QxZq/rNBF5UUQGeWl9xhgzzBizyRjzjzFmrzFmvjGmiTfWd7GMMWOMMbMvZRkiMkdEOnmwrkydX37+TRZV2hEUft1EpCIQAVwLPO3jeHLNGFMqi1nzgQ5AF6AScDdwPzDRCzEYY0xB+3+YCDwCDAOqAA2Az4Bb83pF2fwOvM6X61YOIqKPQvoA4oCbXF6/Cnzt8vo64DfgGLAeiHKZVwWYAewHjgKfuczrCqxzvO83ICzjOoEawGmgisu8a4HDQGnH6wHAVsfyFwG1XdoKMASIBXa52bYOwBkgMMP0lkAqEOx4vRR4CfgDOAF8niGm7D6DpcALwHLHtgQD9zpiPgn8BTzgaFvB0eY8kOR41ADGALMdbeo4tuseYI/js3jWZX3lgFmOz2MrMBzYm8Xvtr5jO1tk8/ufCUwGvnbEuxKo5zJ/IhDv+FzWAG1c5o0BFgCzHfMHAS2AFY7P6gDwDlDG5T2hwA/AEeAQ8AxwM5AMnHN8JusdbS8HpjuWsw8YD5R0zOvv+MzfBBId8/oDvzrmG8e8vx2xbQQaY78EnHOsLwn4MuP/AVDSEddOx2eyhgx/Q/pw87fk6wD0cQm/vPT/AAGOf5iJjtc1Hf9kXbBHfh0dr6s55n8NzAX8gNJAO8f0ax3/gC0d/1T3ONZzmZt1/gTc5xLPa8BUx/MewA6gEVAK+C/wm0tbcexUqgDl3Gzby8CyLLZ7Nxd20EsdO5rG2J31J1zYMef0GSzF7rBDHTGWxn7brufYGbUDTgFNHe2jyLDjxn1H8D52px8OnAUauW6T4zMPADZkXJ7Lch8Edufw+5/p2J4WjvjnADEu8+8C/B3zngAOAmVd4j4H3Ob4bMoBzbAdZynHtmwFHnW0r4TdqT8BlHW8bpnxM3BZ90LgPcfv5EpsR532O+sPpAAPO9ZVjvQdQWfsDvwKx++hEVDdZZvHZ/N/8BT2/+Aax3vDAX9f/68W9IfPA9DHJfzy7D9AEvabjwCLgSsc80YAH2dovwi7Y6+O/Wbr52aZ7wLPZ5i2nQsdhes/3SDgJ8dzg/322dbx+ltgoMsySmB3qrUdrwVon822feC6U8sw73cc37SxO/OXXeaFYL8xlszuM3B577gcPuPPgEccz6PwrCMIcJn/B3Cn4/lfQGeXeYMyLs9l3rPA7znENhP4wOV1F2BbNu2PAuEucf+cw/IfBRY6nvcF/i+Lds7PwPH6KmwHWM5lWl9gieN5f2BPhmX050JH0B74E9splXCzzdl1BNuBHt74fyvKj4J2TlTl3m0iUgm7k2oIVHVMrw3cbow5lvYAbsB2AoHAERE56mZ5tYEnMrwvEHsaJKNPgFbGmOpAW2zn8ovLcia6LOMItrOo6fL++Gy267AjVneqO+a7W85u7Df7qmT/GbiNwRhzizHmd2PMEUf7Llz4TD110OX5KSDtAn6NDOvLbvsTyXr7PVkXxpgnjTFbjTHHHdtyOem3JeO2NzDGfOW48eAE8KJL+0Ds6RZP1Mb+Dg64fO7vYY8M3K7blYj8hD0tNRn42xgzzRhT2cN15yZO5aAdQREhIsuw35YmOCbFY78NX+HyqCAiLzvmVTHGXOFmUfHACxneV15Eot2s8yjwPdAH+Df2G7y4LOeBDMspJyK/uS4im036EWhpjAl0nWiMaYn9Z//JZbJrm1rYUx6Hc/gMMsVgjLkM27lNAK4SkSuAb7AdWE7xeuIA9pSQu7gzWgwEGGMiL2ZFxpg22GsQd2CP/K4AjnNhWyDz9rwLbAPqi0hl7Ln2tPbxQN0sVpdxOfHYI4KqLp97ZREJzeY96RcoMklEmmGP8BpgT/nk+D7Huuvl0EZloB1B0fIW0NEYE469CNjNGNPZGFPSGFPWcftjgIgcwJ66mWKM8TPGlDbGtHUs433gQWNMS8edNBWMMbcaYyplsc7/Af8Bejuep5kKPG2MCQUwxlxujLnd0w0RkR+xO8NPjDGhjm24zrFd74pIrEvzu4wxIcaY8sA4YIGIpGb3GWSx2jLAZUACkGKMuQVwvaXxEOBvjLnc0+3IYB72M/EzxtQEhmbV0LF9U4BoR8xlHPHfaYwZ6cG6KmHPwycApYwxo4CcvlVXwl6cTTLGNAQGu8z7CqhujHnUcVtvJUenDPZzqZN215Xj7+t74HVjTGVjTAljTD1jTDsP4sYY09zx91ca+Ad708B5l3Vl1SGBPaX4vDGmvuPvN8wY4+/Jeosz7QiKEBFJAD4CRolIPPaC7TPYnUE89ltV2u/8buw3523Yi8OPOpaxGrgPe2h+FHvBt382q/0Ce4fLQRFZ7xLLQuAVIMZxmmETcEsuN6kXsAT4DnstZDb2TpSHM7T7GHs0dBB7IXOYI4acPoN0ROSk473zsNv+b8f2pc3fBkQDfzlOebg7XZadccBeYBf2iGcB9ptzVoZx4RTJMewpj57Alx6saxH2c/sTe7rsDNmfigJ4ErvNJ7FfCOamzXB8Nh2BbtjPORa40TF7vuNnojFmreP5f7Ad6xbsZ7kAz051ge2w3ne8bzf2NNlrjnnTgRDH5/+Zm/e+gf39fY/t1KZjL0arbJgLR/JKFT7GmKXYC5U+Gd17KYwxg7EXkj36pqyUt+gRgVL5xBhT3RhzveNUyTXYWzEX+joupXREn1L5pwz27pkg7KmeGOx1AKV8Sk8NKaVUMaenhpRSqpgrdKeGqlatKnXq1PF1GEopVaisWbPmsIhUczev0HUEderUYfXq1b4OQymlChVjzO6s5umpIaWUKua0I1BKqWJOOwKllCrmtCNQSqliTjsCpZQq5rzWERhjPjTG/G2M2ZTFfGOMmWSM2WGM2WCMaeqtWJRSSmXNm0cEM7H1TLNyCzZrZX1sLdJ3vRiLUkqpLHhtHIGI/GyMqZNNkx7AR45CJr8bY64wxlR35DJXPvbB6KH4/7bC12EopYCSqamcq9CW41WrMmDGqDxfvi+vEdQkfX70vaQvY+hkjLnfGLPaGLM6ISEhX4Ir7vx/W0Hg36d8HYZSxV7VIydp//tW2v32Gaklrsz5DRehUIwsFpFpwDSAyMhIzZKXT+KvLE/PH9b4Ogyliqdjx+Cpp+DHDyA4mLXNelO1indq7PjyiGAf6Wu2BjimKaVU8ZaaCq1bw4cfwvDhsGEDiYENvbY6Xx4RfAEMNcbEAC2B43p9QClVrCUmQpUqULIkvPACBAZCZKTXV+vN20ejgRXANcaYvcaYgcaYB40xDzqafAP8ha2J+z7wkLdiUUqpAk0EZs+GBg3gA0fV1Z4986UTAO/eNdQ3h/kCDPHW+pVSqlCIj4cHH4RvvoHrroPrr8/3EHRksVJK+Up0NISGwtKl8NZb8OuvEBKS72EUiruGlFKqSPLzg5YtYdo0CAryWRjaESilVH5JSYE334TkZHj2Wbj5ZujcGYzxaVh6akgppfLD+vX2GoDjdlDEMSTKx50AaEeglFLedfYsPPecvQMoPh7mz4eYmALRAaTRjkAppbwpNhZeeQX+/W/YsgV69y5QnQDoNQKllMp7SUnw+efQrx80bgzbtkHdur6OKkt6RKCUUnnphx+gSRO4+27YutVOK8CdAGhHoJRSeePoURg4EDp1gjJlYNkyaNTI11F5RE8NKaXUpUpNtSOC//wTnn4aRo2CsmV9HZXHtCNQSqmLdfjwhSRxL74ItWpB08JXdVdPDSmlVG6JwEcfpU8Sd9tthbITAO0IlFIqd3bvhltugXvusdcA2rb1dUSXTDsCpZTy1OzZ9nbQX3+Ft9+GX36Bht4rGJNf9BqBUkp5qlo1e1H4vfegdm1fR5NntCNQSqmsnDsHr79ufz73nE0Q16lTgRsZfKn01JBSSrnzf/9nU0Q//bRNDVGAksTlNe0IlFLK1Zkz8Mwz0Lw57N8Pn3xiC8gUwQ4gjXYESinlascOmDAB/vMfmyLiX//ydURep9cIlFIqKQkWLrT5gRo3hu3bfVoxLL/pEYFSqnhbtMjWDb7nngtJ4opRJwDaESiliqvERLvzv/lmKF/ejgkoJEni8pqeGlJKFT9pSeJ27LC1g//730KVJC6vaUeglCo+EhLA398miXvlFTsoLCLC11H5nJ4aUkoVfSIwY4ZNEvf++3Zajx7aCThoR6CUKtri4uyI4AEDbOWwG2/0dUQFjnYESqmi6+OP7e2gK1bAlCmwdKk9KlDp6DUCpVTRddVVNk301Km2aIxySzsCpVTRce4cvPqqvSto1CibIK5TJ19HVeDpqSGlVNGwdq3ND/Tf/9qRwWlJ4lSOtCNQShVup0/DyJHQogUcOmRTRcyZU6STxOU1r3YExpibjTHbjTE7jDEj3cyvZYxZYoz5P2PMBmNMF2/Go5Qqgv76C954A/r3t+mib7vN1xEVOl7rCIwxJYHJwC1ACNDXGBOSodl/gXkici1wJzDFW/EopYqQEydg5kz7PDQUYmNtEXk/P5+GVVh584igBbBDRP4SkWQgBuiRoY0AlR3PLwf2ezEepVRR8M039pbQgQMvJIkrQmUjfcGbHUFNIN7l9V7HNFdjgLuMMXuBb4CH3S3IGHO/MWa1MWZ1QkKCN2JVShV0hw/bNNG33gqVKsHy5cU2SVxe8/XF4r7ATBEJALoAHxtjMsUkItNEJFJEIqtVq5bvQSqlfCwtSVxMjL0tdO1auO46X0dVZHhzHME+INDldYBjmquBwM0AIrLCGFMWqAr87cW4lFKFxaFDUK2aTRI3YYI9BRQW5uuoihxvHhGsAuobY4KMMWWwF4O/yNBmD9ABwBjTCCgL6LkfpYo7EZg+Ha65BqZNs9O6ddNOwEu81hGISAowFFgEbMXeHbTZGDPOGNPd0ewJ4D5jzHogGugvoqNAlCrW/voLbroJBg2y2UFvusnXERV5Xk0xISLfYC8Cu04b5fJ8C3C9N2NQShUis2bBQw/ZU0FTp8J990EJX1/KLPo015BSquCoUQPat4d334WAAF9HU2xoR6CU8p3kZHj5ZTh/HsaMgY4d7UPlKz3mUkr5xqpV0KwZjB5trwvo5UGf0Y5AKZW/Tp2CJ5+04wCOHoUvvoCPPtIkcT6kHYFSKn/t2gVvv20vBG/ebG8LVT6l1wiUUt53/Dh8+ince69NErdjBwQG5vw+lS/0iEAp5V1ff213/oMGwbZtdpp2AgWKdgRKKe9ISIB+/aBrV5seesUKaNjQ11EpN/TUkFIq76Wmwg032OsBY8faCmJlyvg6KpUF7QiUUnnn4EG48ko7Mvj116FOHVs7QBVoHp8aMsaU92YgSqlC7Px5eO89aNDA/gR7Skg7gUIhx47AGNPaGLMF2OZ4HW6M0ZKSSilrxw7o0AEefBCaN4fOnX0dkcolT44I3gQ6A4kAIrIeaOvNoJRShcSMGdCkiS0U8/778OOPULeur6NSueTRNQIRiTfpR/2leiccpVShUquWPQKYPBlqZqxEqwoLTzqCeGNMa0CMMaWBR7D1BZRSxc3Zs/DSS/aawLhx9pRQhw6+jkpdIk9ODT0IDMEWnt8HRAAPeTMopVQBtHKlTRI3dizs2aNJ4ooQTzqCa0Skn4hcJSJXishdQCNvB6aUKiD++QcefxxatbKpIr76CmbO1CRxRYgnHcHbHk5TShVFu3fDlCn2rqDNm+HWW30dkcpjWV4jMMa0AloD1Ywxj7vMqgyU9HZgSikfOnYMFiyw+YFCQuwtoloxrMjK7oigDFAR21lUcnmcAHp7PzSllE98/rnd+T/44IUkcdoJFGlZHhGIyDJgmTFmpojszseYlFK+8PffMGwYzJ0LYWG2YIwmiSsWPLl99JQx5jUgFCibNlFE2nstKqVU/kpNheuvt3cDjR8Pw4dD6dK+jkrlE086gjnAXKAr9lbSe4AEbwallMon+/fD1VfbJHETJ9okcSEhvo5K5TNP7hryF5HpwDkRWSYiAwA9GlCqMDt/Ht591576mTrVTuvSRTuBYsqTI4Jzjp8HjDG3AvuBKt4LSSnlVX/+aesF//wz3HQT3HKLryNSPuZJRzDeGHM58AR2/EBl4FGvRqWU8o7p02HoUChbFj78EPr314FhKueOQES+cjw9DtwIYIy53ptBKaW8pE4dewQweTJUr+7raFQBkd2AspLAHdgcQ9+JyCZjTFfgGaAccG3+hKiUumhnz8Lzz9vn48drkjjlVnZHBNOBQOAPYJIxZj8QCYwUkc/yIzil1CX47TcYONAOChswwCaJ09NAyo3sOoJIIExEzhtjygIHgXoikpg/oSmlLkpSEjz7LLz9NgQGwnffadUwla3sbh9NFpHzACJyBvgrt52AMeZmY8x2Y8wOY8zILNrcYYzZYozZbIz5X26Wr5RyY88eWzd4yBDYtEk7AZWj7I4IGhpjNjieG6Ce47UBRETCsluw4xrDZKAjsBdYZYz5QkS2uLSpDzwNXC8iR40xV17CtihVfB09CvPnw/3327EAf/0FNWr4OipVSGTXEVxqzYEWwA4R+QvAGBMD9AC2uLS5D5gsIkcBROTvS1ynUsXPwoXw0EOQkADt2sE112gnoHIlu6Rzl5poriYQ7/J6L9AyQ5sGAMaY5djU1mNE5LuMCzLG3A/cD1CrVq1LDEupIuLgQXj4YZsuOiICvv7adgJK5ZJHxeu9vP76QBQQAPxsjGkiIsdcG4nINGAaQGRkpNbHUyo1Fdq0gfh4ePFFePJJTRKnLpo3O4J92NtP0wQ4prnaC6wUkXPALmPMn9iOYZUX41Kq8Nq71572KVkSJk2CoCBNFa0umSdJ5zDGlDPG5PaYcxVQ3xgTZIwpA9wJfJGhzWfYowGMMVWxp4r+yuV6lCr6zp+3t4M2bGiTxYEdIaydgMoDOXYExphuwDrgO8frCGNMxh16JiKSAgwFFgFbgXkistkYM84Y093RbBGQaIzZAiwBntJxCkplsG0btG1ri8bccAN07erriFQR48mpoTHYO4CWAojIOmNMkCcLF5FvgG8yTBvl8lyAxx0PpVRGH3xgk8SVLw+zZsHdd+voYJXnPEpDLSLHTfo/Pr1gq1R+qFcPunWDd96Bq67ydTSqiPKkI9hsjPk3UNIxAGwY8Jt3w1KqmDpzBsaNs89ffBFuvNE+lPIiTy4WP4ytV3wW+B82HbXWI1Aqry1fbscDvPSSHRwmeuCt8ocnRwQNReRZ4FlvB6NUsXTyJDzzjK0RULs2LFoEnTr5OipVjHhyRPC6MWarMeZ5Y0xjr0ekVHGzd6+9KPzww7Bxo3YCKt/l2BGIyI3YymQJwHvGmI3GmP96PTKlirLExAvjARo1skniJk6EihV9G5cqljwaUCYiB0VkEvAgdkzBqBzeopRyR8TmBgoJseMCtm+307VspPIhTwaUNTLGjDHGbMQWr/8Nmy5CKZUbBw5Ar15w++22YMzq1ZokThUInlws/hCYC3QWkf1ejkepoiktSdy+ffDqq/DYY1DK1zkflbJy/EsUkVb5EYhSRVJ8PNSsaZPETZ5sk8Q1aODrqJRKJ8tTQ8aYeY6fG40xG1weG10qlyml3ElNtdlBXZPEde6snYAqkLI7InjE8VMzXCmVG1u3wsCBsGKFzRDarZuvI1IqW1keEYjIAcfTh0Rkt+sDeCh/wlOqkJk2zY4O/vNP+PhjWzVMq+qpAs6Tq1UdgREZpt3iZlqB9sHoofj/tsLXYRQagX+fIv7K8r4Oo/CpXx969rSnha680tfRqIuwefZ8/lx/ytdhZHI4yZ+qFb2TpT/LjsAYMxj7zb9uhmsClYDlXonGi/x/W6E7t1yIv7I8ia31PoEcnT4NY8bY1NAvv6xJ4oqAP9ef8upO92JVrZhIg3Dv7L+yOyL4H/At8BIw0mX6SRE54pVovCz+yvL0/GGNr8NQRcXPP8OgQRAbCw8+aAeLaa2AIqFqxUR6vnaPr8PIN9kNKBMRiQOGACddHhhjqng/NKUKqBMn4KGHoF07e3fQ4sX2ziDtBFQhldMRQVdgDbYQjetfuQB1vRiXUgXX/v0wcyY8/ritHVChgq8jUuqSZNkRiEhXx0+PylIqVaQdPgzz5tkjgYYNYdcurRimigxPcg1db4yp4Hh+lzHmDWOM3g+nigcRmDvXJol79FF7WyhoJ6CKFE+yj74LnDLGhANPADuBj70alVIFwf79cNttcOedtmDMmjU6MlgVSZ50BCkiIkAP4B0RmYy9hVSpois1Fdq2he+/hwkT7CjhJk18HZVSXuHJgLKTxpingbuBNsaYEkBp74allI/s3g0BATZJ3JQpULcuBAf7OiqlvMqTI4I+2ML1A0TkILYWwWtejUqp/JaaCm+8YauFpSWJ69RJOwFVLHhSqvIgMAe43BjTFTgjIh95PTKl8sumTdC6NTzxBHToYK8LKFWMeHLX0B3AH8DtwB3ASmNMb28HplS+mDoVmja1NYP/9z/44gt7akipYsSTawTPAs1F5G8AY0w14EdggTcDU8qr0tJBNGpkS0e+9RZUq+brqJTyCU86ghJpnYBDIh4WvVeqwDl1CkaNsheDX3nFpolo187XUSnlU57s0L8zxiwyxvQ3xvQHvga+8W5YSnnB0qUQFgavvw5JSfaoQCnl0cXip4D3gDDHY5qIFKpaBKqYO34cHnjgQnron36y9YM1SZxSQPb1COoDE4B6wEbgSRHZl1+BKZVnDhyA2bPhySdh7FgorzUplHKV3RHBh8BXQC9sBtK3c7twY8zNxpjtxpgdxpiR2bTrZYwRY0xkbtehlFsJCfC240+2YUOIi4PXXtNOQCk3susIKonI+yKyXUQmAHVys2BjTElgMrasZQjQ1xgT4qZdJeARYGVulq+UWyL2NtBGjey4gLQkcXpHkFJZyq4jKGuMudYY09QY0xQol+F1TloAO0TkLxFJBmKw+Yoyeh54BTiT6+iVchUfD926Qb9+dkTw//2fJolTygPZ3T56AHjD5fVBl9cCtM9h2TWBeJfXe4GWrg0cHUqgiHxtjHkqqwUZY+4H7geoVUszYCs3UlIgKgoOHoQ334SHH7a3iCqlcpRdYRqvVuB2JK97A+ifU1sRmQZMA4iMjNR7/tQFcXEQGAilSsF779kkcXUvrnjeuXPn2Lt3L2fO6MFpcVarpz2DvXXrVh9HcnHKli1LQEAApUt7nhvUkwFlF2sfEOjyOsAxLU0loDGw1Njb+K4GvjDGdBeR1V6MSxUFKSl2NPBzz8Grr9ojgJtuuqRF7t27l0qVKlGnTh2M3lpabB2NPwyAX2BVH0eSeyJCYmIie/fuJSjI8+KS3hwhvAqob4wJMsaUAe4EvkibKSLHRaSqiNQRkTrA74B2AipnGzZAq1bw1FPQuTP06pUniz1z5gz+/v7aCahCyxiDv79/ro9qvdYRiEgKMBRYBGwF5onIZmPMOGNMd2+tVxVxU6ZAs2a2bsDcubBwIdSokWeL105AFXYX8zec46khY5faD6grIuMc9YqvFpE/cnqviHxDhnQUIjIqi7ZRHkWsiqe0JHGNG9vSkW++CVUL36G7UgWRJ0cEU4BWQF/H65PY8QFKed8//8Bjj8Hw4fZ127bw8cfaCSiVhzzpCFqKyBAc9/mLyFGgjFejUgpg8WJbJ/itt+Ds2WKRJM4Yw1133eV8nZKSQrVq1ejatWuullOnTh0OHz58SW1KlixJREQEjRs3plu3bhw7dgyAuLg4ypUrR0REhPORnJzsdhmPPvooNWvW5Pz5885pY8aMYcKECVnGcvDgQe68807q1atHs2bN6NKlC3+mDQws4GbNmkX9+vWpX78+s2bNcttm3bp1XHfddURERBAZGckff9iTK8ePH6dbt26Eh4cTGhrKjBkz8i1uTzqCc45RwgLOegTns3+LUpfg2DEYNMjeBVSqFPz8M0yaVCySxFWoUIFNmzZx+vRpAH744Qdq1qzpk1jKlSvHunXr2LRpE1WqVGHy5AsnAurVq8e6deucjzJlMn83PH/+PAsXLiQwMJBly5Z5tE4RoWfPnkRFRbFz507WrFnDSy+9xKFDhy5qG1JTUy/qfRfjyJEjjB07lpUrV/LHH38wduxYjh49mqnd8OHDGT16NOvWrWPcuHEMdxztTp48mZCQENavX8/SpUt54oknsuxg85ont49OAhYCVxpjXgB6A//1alSqeDt0CGJiYMQIGD0aypXL9xDGfrmZLftP5OkyQ2pUZnS30BzbdenSha+//prevXsTHR1N3759+eWXXwC7sxkwYAB//fUX5cuXZ9q0aYSFhZGYmEjfvn3Zt28frVq1QlyOnmbPns2kSZNITk6mZcuWTJkyhZK5HGzXqlUrNmzYkKv3LF26lNDQUPr06UN0dDQ33pjz0KQlS5ZQunRpHnzwQee08PBwt21vu+024uPjOXPmDI888gj3338/ABUrVuSBBx7gxx9/ZPLkycTFxbnd/sGDB7Nq1SpOnz5N7969GTt2bK62L6NFixbRsWNHqlSpAkDHjh357rvv6Nu3b7p2xhhOnLB/W8ePH6eG42YHYwwnT55EREhKSqJKlSqUKuXNO/wv8CQN9RxgOPASdrTxbSIy39uBqWLm0CGYONE+v+YaO1Ds5Zd90gn42p133klMTAxnzpxhw4YNtGx5YUD+6NGjufbaa3sx6z4AACAASURBVNmwYQMvvvgi//nPfwAYO3YsN9xwA5s3b6Znz57s2bMHsIOi5s6dy/Lly1m3bh0lS5Zkzpw5uYonNTWVxYsX0737hZv9du7c6TwtNGTIELfvS+vEevbsyddff825c+dyXNemTZto1qyZR3F9+OGHrFmzhtWrVzNp0iQSExMB+Oeff2jZsiXr16/H398/y+1/4YUXWL16NRs2bGDZsmVuO7rXXnst3SmwtMewYcMytd23bx+BgReGTgUEBLBvX+aEzW+99RZPPfUUgYGBPPnkk7z00ksADB06lK1bt1KjRg2aNGnCxIkTKVEif2qAeXLXUC3gFPCl6zQR2ePNwFQxIQJz5sAjj9hiMV26QP36Pr8Y7Mk3d28JCwsjLi6O6OhounTpkm7er7/+yieffAJA+/btSUxM5MSJE/z88898+umnANx66634+fkBsHjxYtasWUPz5s0BOH36NFdeeaVHcZw+fZqIiAj27dtHo0aN6Nixo3Ne2qmhrCQnJ/PNN9/wxhtvUKlSJVq2bMmiRYvo2rVrlrc35va2x0mTJrFw4UIA4uPjiY2Nxd/fn5IlS9LLMbYku+2fN28e06ZNIyUlhQMHDrBlyxbCwsLSreOpp57iqaeyzH5zUd59913efPNNevXqxbx58xg4cCA//vgjixYtIiIigp9++omdO3fSsWNH2rRpQ+XKlfN0/e54ctzxNfb6gAHKAkHAdsB3/ymqaNizBx58EL791g4Qmz7ddgKK7t278+STT7J06VLnN92LISLcc889zm+duZF2jeDUqVN07tyZyZMnu/0m7M6iRYs4duwYTZo0AeDUqVOUK1eOrl274u/vz4EDB9K1P3nyJFdccQWhoaEsWJBzOfSlS5fy448/smLFCsqXL09UVJRzEFXZsmWdp76y2v5du3YxYcIEVq1ahZ+fH/3793c7COu1115zewTVtm1bJk2alG5azZo1Wbp0qfP13r17iYqKyvTeWbNmMdFx9Hv77bczaNAgAGbMmMHIkSMxxhAcHExQUBDbtm2jRYsWOX4el8qTU0NNRCTM8bM+NqvoCq9Hpoq2tCRxaReCf/nFpo5WAAwYMIDRo0c7d6Rp2rRp49wxLV26lKpVq1K5cmXatm3L//73PwC+/fZb50XKDh06sGDBAv7+25YdP3LkCLt3785VLOXLl2fSpEm8/vrrpKSkePSe6OhoPvjgA+Li4oiLi2PXrl388MMPnDp1irZt2/LFF19w8uRJAD799FPCw8MpWbIk7du35+zZs0ybNs25rA0bNjivkaQ5fvw4fn5+lC9fnm3btvH777+7jSOr7T9x4gQVKlTg8ssv59ChQ3z77bdu3//UU0+luyie9sjYCQB07tyZ77//nqNHj3L06FG+//57OnfunKldjRo1nBfPf/rpJ+o7vvzUqlWLxYsXA3Do0CG2b99O3YvMm5VrIpLrB7DxYt6XF49mzZrJxfj0pqby6U1NL+q9Kg/t3CmSkmKf//ijyK5dPg3H1ZYtW3wdglSoUCHTtCVLlsitt94qIiKJiYnSo0cPadKkibRs2VLWr18vIiKHDx+Wjh07SkhIiAwaNEhq1aolCQkJIiISExMj4eHh0qRJE2natKmsWLFCRERq167tbONJLF27dpWPPvpIdu3aJaGhoVm+759//hE/Pz85fvx4uuk9e/aUmJgYERGZOnWqhIWFSXh4uHTs2FF27tzpbLdv3z65/fbbpW7duhISEiJdunSRP//8M92yzpw5IzfffLM0bNhQevToIe3atZMlS5a4jTur7b/nnnukfv360r59e+nZs6fMmDFDRESO7EmQI3uy/lyyM336dKlXr57Uq1dPPvzwQ+f0gQMHyqpVq0RE5JdffpGmTZtKWFiYtGjRQlavXu3c7o4dO0rjxo0lNDRUPv7444uKQcT93zKwWrLYrxrJ4d5sY8zjLi9LAE0BfxHJ3NXlg8jISFm9OvfpiBZ2tBegev6wJq9DUp5ISbFF40ePtkniPDzFkJ+2bt1KIz0qKfYKc9K5NO7+lo0xa0TEbRVIT64RVHJ5noK9ZvDJRUeoip9162DgQFi7Fnr2hNtv93VESikX2XYEjoFklUTkyXyKRxU177xjU0T4+8OCBXmWKVTljcTERDp06JBp+uLFi/H39/dBRMoXsuwIjDGlRCTFGHN9fgakioi0JHFhYbZ05BtvgGOgjSo4/P39s70NVBUP2R0R/IG9HrDOGPMFMB/4J22miHzq5dhUYZSUBM8+C6VLw4QJNklc27a+jkoplQ1Phq2VBRKxNYq7At0cP5VK7/vvbZrot9+Gc+eKRZI4pYqC7I4IrnTcMbSJCwPK0uh/uLrg6FF4/HGYOdOmh/j5Z7jhBl9HpZTyUHYdQUmgIuk7gDTaEagL/v7bXgh++mkYNQrKlvV1REqpXMju1NABERknImPdPMblW4SqYDp40FYJgwtJ4l58UTuBS1SQ6hG88MILhIaGEhYWRkREBCtXrmTs2LE8/fTT6dqtW7fOec96nTp1aNOmTbr5aTUNsvLWW29RtmxZjh8/7pw2c+ZMhg4dmq5dVFQUaWOIkpKSeOCBB5w1C6Kioli5cmW221tQfPfdd1xzzTUEBwfz8ssvu21z9uxZ+vTpQ3BwMC1btiQuLs45b8OGDbRq1YrQ0FCaNGmS6/rE7mTXERT95O8q90Rg1iwICbFHALGxdrreapgnCko9ghUrVvDVV1+xdu1aNmzYwI8//khgYCB9+/Zl7ty56drGxMSkS7V88uRJ4uPjATuwKSfR0dE0b97cmTTPE4MGDaJKlSrExsayZs0aZsyYkWPHl5X8rFmQmprKkCFD+Pbbb9myZQvR0dFs2bIlU7vp06fj5+fHjh07eOyxxxgxYgRgvxjcddddTJ06lc2bN7N06VJKly59yXFld2oo883FqniLi4MHHrAXha+/Hj74oOgmift2JBzcmLfLvLoJ3OL+G6CrglCP4MCBA1StWpXLLrsMgKou2WD9/PxYuXKlMz32vHnzWLRokXP+HXfcwdy5c3nyySed8X/88cdu17Nz506SkpKYMmUKL7zwAvfee2+On8/OnTtZuXIlc+bMcaZpDgoKIigoKFPbrGoO1KlThz59+vDDDz8wfPhwqlSpwujRozl79iyB1QN45/VJ+FGVcePG8eWXX3L69Glat27Ne++9d1HF4dP88ccfBAcHO3MI3XnnnXz++eeEhISka/f5558zZswYAHr37s3QoUMREb7//nvCwsKcNRryaqxHlkcEInIkT9agioaUFLjxRvjtN5g82V4QbtjQ11EVSQWhHkGnTp2Ij4+nQYMGPPTQQ+kqjPXt25eYmBgAfv/9d6pUqeJMnAbQq1cv57f7L7/8km7dumW5npiYGO68807atGnD9u3bPapEtnnzZiIiIjwqrpNdzQF/f3/Wrl3LTTfdxPjx4/nxxx9Zu3Yt14ZFMOX9qYCtEbBq1SrnUdpXX32VaR1z5sxxW7Ogd+/emdp6WrPAtV2pUqW4/PLLSUxM5M8//8QYQ+fOnWnatCmvvvpqjp+BJ/Kn/I0qvHbsgKAgWzLyww+hbl2oXdvXUXmfB9/cvaUg1COoWLEia9as4ZdffmHJkiX06dOHl19+mf79+9OnTx9at27N66+/num0ENgdrJ+fHzExMTRq1Ijy5ctnuZ7o6GgWLlxIiRIl6NWrF/Pnz2fo0KF5VrMgu5oDffr0AWxntmXLFq6/3o6dPf3PaZo3syl5lixZwquvvsqpU6c4cuQIoaGhmTq2fv360a9fv1zFdbFSUlL49ddfWbVqFeXLl6dDhw40a9bM7ejw3NCOQLl37hy89hqMHWt/DhtmjwhUvigI9QhKlixJVFQUUVFRNGnShFmzZtG/f38CAwMJCgpi2bJlfPLJJ6xYkTkrfZ8+fRgyZAgzZ87McvkbN24kNjbWWfAmOTmZoKAghg4dir+/f6Z6v0eOHKFq1apcccUVrF+/ntTU1GyPCnKqOVChQgXAfkYdO3YkOjoauJB07syZMzz00EOsXr2awMBAxowZ4/bC7Jw5c3jttdcyTQ8ODs5UW6FmzZrO6ydgaxa4uwaU1i4gIICUlBSOHz+Ov78/AQEBtG3b1nmqrkuXLqxdu/aSO4L8qYOmCpe1a6FFCztCuEcPcHxzUvnH1/UItm/fTmzajQDYO4NquxwJ9u3bl8cee4y6desSEBCQ6f09e/Zk+PDhbvPxp4mOjmbMmDHOmgX79+9n//797N69m+bNm7N8+XIOHjwIwOrVq+35+8BA6tWrR2RkJKNHj3ZeC4mLi+Prr79Ot3xPaw5cd911LF++nB07dgDwz6l/2PHXTudOv2rVqiQlJWVZMKdfv35uaxa4a9+8eXNiY2PZtWsXycnJxMTEpCsBmqZ79+7MmjULgAULFtC+fXvnKaGNGzdy6tQpUlJSWLZsWabrCxdDjwhUepMm2cFh1arBp5/abKEq3wUEBLitBjZmzBgGDBhAWFgY5cuXd+4sRo8eTd++fQkNDaV169bUqlULgJCQEMaPH0+nTp04f/48pUuXZvLkyel26u4kJSXx8MMPc+zYMUqVKkVwcHC6YjG33347w4YN4+2333b7/kqVKjnvdMlKTEwM33zzTbppPXv2JCYmhhEjRjBx4kS6dOnC+fPnqVixItHR0c6Lwx988AFPPPEEwcHBlCtXjqpVq2b6Vh4eHs61115Lw4YNCQwMdJ76yahatWrMnDmTvn37cvbsWVLPpfLsk0/TvF1L7rvvPho3bszVV1/tPL12KUqVKsU777xD586dSU1NZcCAAYSG2mKPo0aNIjIyku7duzNw4EDuvvtugoODqVKlivOajJ+fH48//jjNmzfHGEOXLl249dZbLzmuHOsRFDRaj8BL0pLE/fILzJhhawc4zjMXF1qPQIHWI1DF0cmTdjzAZZfZnX+bNvahlCo2tCMozr77zo4LiI+HRx+9cFSgig2tR6BAO4LiKTHRXgf46CNbMH75cmjVytdRKR/QegQK9K6h4ikxERYuhOeeg//7P+0ElCrmvNoRGGNuNsZsN8bsMMaMdDP/cWPMFmPMBmPMYmNMMRip5CMHDthCMSLQoAHs3g3jxtlrA0qpYs1rHYGj3vFk4BYgBOhrjMl4w+v/AZEiEgYsAPJmvLS6QMSOCG7UyB4BOO6VLm53BCmlsubNI4IWwA4R+UtEkoEYoIdrAxFZIiKnHC9/BzKPTFEXb9cu6NQJBg6E8HBYv77oJolTSl00b3YENYF4l9d7HdOyMhBwO/TPGHO/MWa1MWZ1QkJCHoZYhKWkQPv2sHIlvPsuLFliTwmpAq0g1SMA+OyzzzDGsG3bNue0uLg4jDHpBpMNHTrUmU6if//+1KxZk7NnzwJw+PBh6tSpk6t1LF26NNM29+/f3zla99y5c4wcOZL69evTtGlTWrVqleXI4YJmzZo1NGnShODgYIYNG4a7sVzHjx+nW7duhIeHExoayowZM5zzhg8fTmhoKI0aNcry/blVIC4WG2PuAiKBzAk7ABGZJiKRIhJZrVq1/A2usImNhdRUmyRuxgzYvBkefBBKFIhftcpBQalHkCY6OpobbrjBmYcnzZVXXsnEiRNJTk52+76SJUvy4YcfXtI6svPcc89x4MABNm3axNq1a/nss884efKkx+93lZKSclHvu1iDBw/m/fffJzY2ltjYWL777rtMbSZPnkxISAjr169n6dKlPPHEEyQnJ/Pbb7+xfPlyNmzYwKZNm1i1alW6zLAXy5u3j+4DAl1eBzimpWOMuQl4FmgnIme9GE/Rdu4cvPIKPP88vPoqPPIIREX5OqpC65U/XmHbkW05N8yFhlUaMqJF9mkXoGDUIwCbZuLXX39lyZIldOvWzZnLH2xahuuvv55Zs2Zx3333ZXrvo48+yptvvul2nqfryMqpU6d4//332bVrl7NewlVXXcUdd9yRqW1W9QSioqKIiIjg119/pW/fvkRFRfH444+TlJTE5RUrM/n1t/ELrMr777/PtGnTSE5OJjg4mI8//jjbbKo5OXDgACdOnOC6664D4D//+Q+fffYZt9xyS7p2xhhOnjyJiJCUlESVKlUoVaoUxhjOnDlDcnIyIsK5c+e46qqrLjqeNN78mrgKqG+MCTLGlAHuBL5wbWCMuRZ4D+guIn97MZaibfVqiIy0F4P/9S/IkBZYFS4FoR4B2OIoN998Mw0aNMDf3581a9KnZxkxYgQTJkxwW+GrVq1a3HDDDVkWpPF0He7s2LGDWrVqUbly5RzbZldPIDk5mdWrVzNs2DAefvhhFixYwJo1a+h3x78Z/9qLAPzrX/9i1apVrF+/nkaNGjF9+vRM61iyZInbegStW7fO1Hbfvn3pkvRlVY9g6NChbN26lRo1atCkSRMmTpxIiRIlaNWqFTfeeCPVq1enevXqdO7cOU/SonjtiEBEUowxQ4FFQEngQxHZbIwZB6wWkS+wp4IqAvMdecb3iEjmVHwqaxMn2sFhV18Nn38ObjIZqtzz5Ju7txSEegRgT9k88sgjgO2coqOjadasmXN+3bp1admypTPraUZPP/00PXr0yDYpWlbryKt6BNnVE0irR7B9+3Y2bdp0IR32mWSuutJ+y960aRP//e9/OXbsGElJSW6zqd544415Pihv0aJFRERE8NNPP7Fz5046duxImzZt+Pvvv9m6dSt79+4FoGPHjvzyyy+Z6kTnlldHFovIN8A3GaaNcnl+kzfXX6SlpYOIjLR3Bb36Klxxha+jUnnE1/UIjhw5wk8//cTGjRsxxpCamooxJlOGz2eeeYbevXvTrl27TMuoX78+ERERzJs3L9fryK4eQXBwMHv27OHEiRPZHhXkVE/AtR5BaGios65CWtI5sBeoP/vsM8LDw5k5cyZLly7NtJ4lS5bw2GOPZZpevnx5fvvtt3TTatas6dyJQ9b1CGbMmMHIkSMxxhAcHExQUBDbtm1j2bJlXHfddVSsWBGAW265hRUrVlxyR6BXEAubEydg8GB7FAC2dvC0adoJFDG+rkewYMEC7r77bnbv3k1cXBzx8fEEBQU5r1WkadiwISEhIXz55Zdul/Pss88yYcKEXK+jfv367N+/n61btwKwe/du1q9fT0REBOXLl2fgwIE88sgjzovVCQkJzJ8/P93yPa0ncM0115CQkODsCM6dO8fW7fb60MmTJ6levTrnzp3L8pRa2hFBxkfGTgCgevXqVK5cmd9//x0R4aOPPqJHjx6Z2tWqVYvFixcDcOjQIbZv307dunWpVasWy5YtIyUlhXPnzrFs2bI8OTWkHUFh8s03EBpqd/ylStmjAlUkZVePYM2aNYSFhTFy5Mh09Qh+/vlnQkND+fTTT93WIwgLC6Njx44cOHAgx/VHR0fTM0Mtil69erm9s+fZZ59N9y3XVWhoKE2bNs31Oi677DJmz57Nvffe66z/+8EHH3D55ZcDMH78eKpVq0ZISAiNGzema9eumY4OrrjiCmc9gc6dO2dZT6BMmTIsWLCAESNGEB4eTtubb+SPNasAeP7552nZsiXXX389DfOoRveUKVMYNGgQwcHB1KtXz3mheOrUqUydamslP/fcc/z22280adKEDh068Morr1C1alV69+5NvXr1aNKkCeHh4YSHh2dbE9pTWo+gMDh82GYHnTPHdgTTp4PLBUSVN7QegYLiWY9AjwgKg6NH4csvYfRoW0ZSOwGlVB7SNNQF1b599gjgqadsWojdu/U6gMpzWo9AgXYEBY8IfPABPPmkHST2r39BcLB2AsortB6BAj01VLDs3AkdOsD990PTprBhg+0ElFLKi/SIoKBISbGdwJEj8N57MGiQ5gdSSuUL7Qh8bft2qFfP3g46a5Z9HqDZuJVS+Ue/cvpKcjKMHQtNmsDkyXZau3baCSil8p12BL7wxx/QrBmMGQO33w79+vk6IlVAaD0CS+sR5G89Aj01lN/eegueeAKqV7djA3L5D67yx8EXX+Ts1rxNQ31Zo4Zc/cwz2bZxrUdQrly5AlWPwDVFdFo9ggceeIAyZcpkel9aPYLBgwdf9Dqy41qP4LLLLuPQoUMXnZc/JSWFUqXyb1eYVo+gZcuWdOnShe+++y5TGuq0egRffvklCQkJXHPNNfTr14/Vq1c76xEA3HDDDSxbtoyoS0w5r0cE+SWt127RAu67zxaM0U5AuZFWjwBw1iNIc+TIEW677TbCwsK47rrrnDuExMREOnXqRGhoKIMGDcpUj6BFixZERETwwAMPuE0b7U5arYDp06cTExOTbl61atXo0KGDM8VFRmn1CHIq+pLdOrKSVo/g7bff9qgeQfPmzWncuDH333+/83OJiori0UcfJTIykokTJ7JmzRratWtHs2bN6HXX7Rw8dBCA999/n+bNmxMeHk6vXr04depUpnXkhms9AmOMsx5BRp7UIzh79mye1SNARArVo1mzZnIxPr2pqXx6U9OLeu8lOXZM5P77RR55JP/XrXJly5Ytvg5BKlSoIOvXr5devXrJ6dOnJTw8XJYsWSK33nqriIgMHTpUxowZIyIiixcvlvDwcBERefjhh2Xs2LEiIvLVV18JIAkJCbJlyxbp2rWrJCcni4jI4MGDZdasWSIiUrt2bUlISMgyltmzZ8uAAQNERKRVq1ayevVqERHZtWuXhIaGys6dO6VBgwaSkpIiQ4YMkRkzZoiIyD333CPz58+Xe++9Vz788ENJSEiQ2rVr52odrtucJm2569evl4iICI8+z8TEROfzu+66S7744gsREWnXrp0MHjxYRESSk5OlVatW8vfff4uIyAfvTJN/39FXREQOHz7sfP+zzz4rkyZNyrSOn376ScLDwzM9WrVqlantqlWrpEOHDs7XP//8c6btFBE5ceKEREVFydVXXy0VKlSQr776yjnviSeekMsvv1wqV64szzzzjNvtdve3jE3/73a/qqeGvOnLL22ZyIMH7QCxtNTRSmVD6xFoPYIiVY+g2EpIsKUio6PtXUGffQZZZD5Uyh2tR6D1CLQeQWF3/LhNGT12rC0jqZ2AyiWtR6D1CLQeQWEUHw8vvWRP/wQH2yRxo0aBmzsqlMqJ1iPQegRajyAbBa4ewfnztlDM8OGQmgrr12t+oEJK6xEo0HoEKrdiY6F9e1s6skUL2LhROwGlVKGjF4svVkoKdOwIx47ZimH33qt3BKlCR+sRKNCOIPe2brWFYkqVgo8/tkniatTwdVRKXRStR6BATw157uxZWyoyLAzeecdOa9NGOwGlVKGnRwSe+P13GDgQtmyBu++2D6WUKiL0iCAnr78OrVvDyZN2bMBHH4GeO1VKFSHaEWTl/Hn7s1UrmyZi0ybIkCFQKaWKAu0IMjp2zJ4GcuQ/oXVrmDIFshnKrlReKUj1CF544QVCQ0MJCwsjIiKClStXMnbsWJ5++ul07datW+e8Z71OnTqZ0h1ERETQuHHjLNfz1ltvUbZsWY4fP+6cNnPmTIYOHZquXVRUFGljiJKSknjggQeoV68ezZo1IyoqipUrV2a7vQXFd999xzXXXENwcDAvv/yy2zZnz56lT58+BAcH07JlS+Li4pzzNmzYQKtWrQgNDaVJkybp0mZcLL1G4Oqzz+Chh+Dvv+0AMU0SV2z9Mu9PDscn5ekyqwZWpM0dDbJtU1DqEaxYsYKvvvqKtWvXctlll3H48GGSk5Pp27cvN998c7rcRTExMelSZZ88eZL4+HgCAwOdKSKyEx0dTfPmzfn000+59957PYpv0KBBBAUFERsbS4kSJdi1axdbtmzJ/YYCqamplCxZ8qLeezHrGjJkCD/88AMBAQE0b96c7t27ExISkq7d9OnT8fPzY8eOHcTExDBixAjmzp1LSkoKd911Fx9//DHh4eEkJiZSunTpS45LjwjA7vjvuAN69oSrrrIVxF58UTsB5RMFoR7BgQMHqFq1qjPff9WqValRowYNGjTAz88v3bfvefPmpYvxjjvuYO7cuW7jz2jnzp0kJSUxfvx4t+krsnrPypUrGT9+PCVK2F1YUFCQ2yyngwcPJjIyktDQUEaPHu2cXqdOHUaMGEHTpk2ZP38+33//Pa1ataJp06b0f3AASf/YLwFZ1TO4WH/88QfBwcHUrVuXMmXKcOedd/L5559navf5559zzz33ANC7d28WL16MiPD9998TFhZGeHg4YG//zZNOLKv81AX14ZV6BLGxIldcIfLCCyKOvO2q+NF6BBecPHlSwsPDpX79+jJ48GBZunSpc95rr70mjz76qIiIrFixQlz/J2vXri3btm1z5uKPiIiQzZs3S2hoqNv1jB8/XsaNGyepqalSq1YtOXjwoIiIzJgxQ4YMGZKubbt27WTVqlXy+eefy2233ebR55lWjyAlJUXatWsn69evd8b5yiuviIhIQkKCtGnTRpKSkkREZPTI52Tk4yPSvV8kfT0DV7Nnz3Zbj6BXr16Z2s6fP18GDhzofP3RRx9l2k4RkdDQUImPj3e+rlu3riQkJMibb74pd911l3Tq1EmuvfZa5zZkpPUIPLVnjx0Q9swzNi3Enj1QqZKvo1KqQNQjqFixImvWrOGXX35hyZIl9OnTh5dffpn+/fvTp08fWrduzeuvv57ptBDYb6l+fn7ExMTQqFEjypcvn+V6oqOjWbhwISVKlKBXr17Mnz+foUOH5lk9gnnz5jFt2jRSUlI4cOAAW7ZsISwsDLhQj+D3339ny5YtXH/99QCc/uc0zZvZlDzZ1TNI069fP/rlU93xlJQUfv31V1atWkX58uXp0KEDzZo1czs6PDe82hEYY24GJgIlgQ9E5OUM8y8DPgKaAYlAHxGJ82ZMnD8PU6fCiBH2eZ8+tiPQTkAVIL6uRwC27nBUVBRRUVE0adKEWbNm0b9/fwIDAwkKCmLZsmV88sknzvTNrvr06cOQIUOcBe3d2bhxK/9YmQAADJRJREFUI7GxsRcKwiQnExQUxNChQ7OtR3DFFVewfv36HM/t79q1iwkTJrBq1Sr8/Pzo379/lvUIOnbs6Dw1lZZ0Lqd6BmnmzJmTqU4DQHBwcKbU1zVr1iQ+Pt75Oqt6BGntAgICSElJ4fjx4/j7+xMQEEDbtm2pWtUmxOvSpQtr16695I7Aa9cIjDElgcnALUAI0NcYE5Kh2UDgqIgEA28Cr3grHoCK/5yBqCgYMsTeFrp5syaJUwWSr+sRbN++ndjYWOfrdevWUbt2befrvn378thjj1G3bl0CAgIyvb9nz54MHz7cbUWvNNHR0YwZM4a4uDji4uLYv38/+/fvZ/fu3TRv3pzly5dz8KCtHbx69WrOnj1LYGAg9erVIzIyktGjRzvP2cfFxTmvq6Q5ceIEFSpU4PLLL+fQoUN8++23buO47rrrWL58OTt27ADgn1P/sOOvnR7XM+jXr5/begTu2jdv3pzY2Fh27dpFcnIyMTExdO/ePVO77t27O1OML1iwgPbt22OMoXPnzmzcuJFTp06RkpLCsmXLMl1ovhjePCJoAewQkb8AjDExQA/A9dJ+D2CM4/kC4B1jjJG0324eSvK7hZtWvEtyShwbbx7IntAbYPIyYFler0oVUrV6hqSrTuUTIhyNP0wFynJ3z39zNP4wJ/8+zrnTyRyNP8yj9z3Mw08+QmijUMqVK8fbr7zF0fjDPDJoKIOGPsDsj2bTollzAmoGcGxfItWrXMnIR4fT4cb2nD8vlC5VilfHv0LlEhU4n3qeY/sSKXk6cxj7d+xlxOinOX78OKVKlaJunSDefPl15+fTsXV7hg0bxitjX0r3maUt07+KP/f/eyD/HDrB8f1HSD2Xmumz/d/s/zF3VnS66bfcdDMzpk7nkYeG8cJzz9O5YyfOnz9PhfIVeO/NKRzfdwSA18a8wnPPj6JunbqULVsW/ypVGPvsmHTLqlWlJqENGtEguD41q9ekRdPm/HPkJEfjD6fb9lIY3n51Inf0up2zyckI8N8nn6Z5u5bOegZXX311lvUMcqNUqVK88847dO7cmdTUVAYMGEBoaCgAo0aNIjIyku7duzNw4EDuvvtugoODqVKlCjExMQD4+fnx+OOP07x5c4wxdOnSJdtSoJ7yWj0CY0xv4GYRGeR4fTfQUkSGurTZ5Giz1/F6p6PN4QzLuh+4H+D/27v7GKmuMo7j319hYWApoN1qamHLNoJK2oYW0tSYvoUGCU1AAgpV0lKJGhSMgo3GNq3BWq3YJm00aYHSRa0tBW2zWivWFrKk5TW8v1iCpalbXyBrJa5g3baPf5wzcbIMzF1mZ4Y79/kkN3vvnXPvfZ6d3Tlz7pk5p7m5eXySdzQ9rbx9CU3Huug/8mLeHjL8bNNydax5+lguHdlS6zDcOSA3+DwGXfD+Wodx1no7H0EqOovNbBmwDMLENGdzjs8/fnefxuTqz8GDB1M9GYlzZ6uSFcGbwMiC7RFxX7EyHZL6A8MIncbOuSrw+QgcVLYi2AaMltRCeMGfDXy2R5k24DZgEzATeKkS/QPOJWVmvf6IYpr5fAT152xeQiv2qSEzewdYAKwDDgJPm9l+SUsk5bvJHwMukHQYWAR8q1LxOFdKLpejs7Oz7G+POlcrZkZnZye5XK5Xx2Vm8nrnSunu7qajo6NPBvFyrlZyuRwjRow4ZQyi1HcWO1cNDQ0NtLT4p4Zc9vigc845l3FeETjnXMZ5ReCccxmXus5iSceA3n+1OGgCajyGQNV5ztngOWdDOTlfYmYXFnsgdRVBOSRtP12veb3ynLPBc86GSuXst4accy7jvCJwzrmMy1pFsKzWAdSA55wNnnM2VCTnTPUROOecO1XWWgTOOed68IrAOecyri4rAkmTJb0q6bCkU0Y0lTRQ0ur4+BZJo6ofZd9KkPMiSQck7ZH0oqRLip0nTUrlXFBuhiSTlPqPGibJWdJn4nO9X9Ivqh1jX0vwt90sab2knfHve0ot4uwrklZKOhpncCz2uCQ9HH8feyRdVfZFzayuFqAf8CfgUmAAsBsY26PMl4FH4vpsYHWt465CzjcCg+P6/CzkHMudD7QDm4EJtY67Cs/zaGAn8L64/YFax12FnJcB8+P6WOD1WsddZs7XAVcB+07z+BTgeUDANcCWcq9Zjy2Cq4HDZvaamf0XeAqY1qPMNGBVXF8LTFS6ZyMpmbOZrTezE3FzM2HGuDRL8jwDfBe4H6iHsaWT5PwF4Cdm9haAmR2tcox9LUnOBgyN68OAv1Qxvj5nZu3AP85QZBrwUws2A8MlXVTONeuxIrgY+HPBdkfcV7SMhQl0jgNpnpcvSc6F5hHeUaRZyZxjk3mkmT1XzcAqKMnzPAYYI+llSZslTa5adJWRJOfvAHMkdQC/BRZWJ7Sa6e3/e0k+H0HGSJoDTACur3UslSTpPOBBYG6NQ6m2/oTbQzcQWn3tki43s3/WNKrKugVoNbMHJH0c+Jmky8zsvVoHlhb12CJ4ExhZsD0i7itaRlJ/QnOysyrRVUaSnJF0E3AnMNXM3q5SbJVSKufzgcuADZJeJ9xLbUt5h3GS57kDaDOzbjM7AhwiVAxplSTnecDTAGa2CcgRBmerV4n+33ujHiuCbcBoSS2SBhA6g9t6lGkDbovrM4GXLPbCpFTJnCVdCTxKqATSft8YSuRsZsfNrMnMRpnZKEK/yFQzS/M8p0n+tp8ltAaQ1ES4VfRaNYPsY0lyfgOYCCDpY4SK4FhVo6yuNuDW+Omha4DjZvbXck5Yd7eGzOwdSQuAdYRPHKw0s/2SlgDbzawNeIzQfDxM6JSZXbuIy5cw56XAEGBN7Bd/w8ym1izoMiXMua4kzHkdMEnSAeBd4A4zS21rN2HOi4Hlkr5O6Diem+Y3dpKeJFTmTbHf4x6gAcDMHiH0g0wBDgMngNvLvmaKf1/OOef6QD3eGnLOOdcLXhE451zGeUXgnHMZ5xWBc85lnFcEzjmXcV4RuHOSpHcl7SpYRp2hbFcfXK9V0pF4rR3xG6q9PccKSWPj+rd7PPZKuTHG8+R/L/sk/VrS8BLlx6V9NE5Xef7xUXdOktRlZkP6uuwZztEK/MbM1kqaBPzIzK4o43xlx1TqvJJWAYfM7HtnKD+XMOrqgr6OxdUPbxG4VJA0JM6jsEPSXkmnjDQq6SJJ7QXvmK+N+ydJ2hSPXSOp1At0O/DheOyieK59kr4W9zVKek7S7rh/Vty/QdIEST8ABsU4noiPdcWfT0m6uSDmVkkzJfWTtFTStjjG/JcS/Fo2EQcbk3R1zHGnpFckfSR+E3cJMCvGMivGvlLS1li22IitLmtqPfa2L74UWwjfit0Vl2cI34IfGh9rInyrMt+i7Yo/FwN3xvV+hPGGmggv7I1x/zeBu4tcrxWYGdc/DWwBxgN7gUbCt7L3A1cCM4DlBccOiz83EOc8yMdUUCYf43RgVVwfQBhFchDwReCuuH8gsB1oKRJnV0F+a4DJcXso0D+u3wT8Mq7PBX5ccPx9wJy4PpwwFlFjrZ9vX2q71N0QE65unDSzcfkNSQ3AfZKuA94jvBP+IPC3gmO2AStj2WfNbJek6wmTlbwch9YYQHgnXcxSSXcRxqmZRxi/5hkz+3eM4VfAtcDvgAck3U+4nbSxF3k9DzwkaSAwGWg3s5PxdtQVkmbGcsMIg8Ud6XH8IEm7Yv4HgRcKyq+SNJowzELDaa4/CZgq6RtxOwc0x3O5jPKKwKXF54ALgfFm1q0womiusICZtceK4magVdKDwFvAC2Z2S4Jr3GFma/MbkiYWK2RmhxTmOpgC3CvpRTNbkiQJM/uPpA3AJ4FZhIlWIMw2tdDM1pU4xUkzGydpMGH8na8ADxMm4FlvZtNjx/qG0xwvYIaZvZokXpcN3kfg0mIYcDRWAjcCp8y5rDAP89/NbDmwgjDd32bgE5Ly9/wbJY1JeM2NwKckDZbUSLits1HSh4ATZvZzwmB+xeaM7Y4tk2JWEwYKy7cuILyoz88fI2lMvGZRFmab+yqwWP8fSj0/FPHcgqL/Itwiy1sHLFRsHimMSusyzisClxZPABMk7QVuBf5YpMwNwG5JOwnvth8ys2OEF8YnJe0h3Bb6aJILmtkOQt/BVkKfwQoz2wlcDmyNt2juAe4tcvgyYE++s7iH3xMmBvqDhekXIVRcB4AdCpOWP0qJFnuMZQ9hYpYfAt+PuRcetx4Ym+8sJrQcGmJs++O2yzj/+KhzzmWctwiccy7jvCJwzrmM84rAOecyzisC55zLOK8InHMu47wicM65jPOKwDnnMu5/w30cnNci0VgAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Specify model\n",
        "model_ANN = MLPClassifier(hidden_layer_sizes=(5, 3), random_state=SEED)\n",
        "\n",
        "# Fit model on single split\n",
        "performance_measures = model_training_and_validation(model_ANN, \"ANN\", splits)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jeWNf9DcULvk",
        "outputId": "7c003dc6-c0b6-4082-d335-2b6d1859f339"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sensitivity: 0.00\n",
            "Specificity: 1.00\n",
            "AUC: 0.88\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Append ANN model\n",
        "models.append({\"label\": \"Model_ANN\", \"model\": model_ANN})\n",
        "# Plot roc curve\n",
        "plot_roc_curves_for_models(models, static_test_x, static_test_y, True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 593
        },
        "id": "-L9IChGeUQ87",
        "outputId": "2c1cb3fd-527d-4640-d240-62c1d76a0ac6"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-38-bfcf35ae90f4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"label\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"Model_ANN\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"model\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmodel_ANN\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# Plot roc curve\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mplot_roc_curves_for_models\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatic_test_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatic_test_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-17-5a65b02e7941>\u001b[0m in \u001b[0;36mplot_roc_curves_for_models\u001b[0;34m(models, test_x, test_y, save_png)\u001b[0m\n\u001b[1;32m     45\u001b[0m     \u001b[0;31m# Save plot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msave_png\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0mfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msavefig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{DATA}/roc_auc\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdpi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbbox_inches\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"tight\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtransparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'DATA' is not defined"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydeVhV1f7/X0twQkWZVGQQ5ODAkSGVUEtFzVBzyNSUb1mkVtdEbVTLckq7DTZo165pDpVeSK00s7JSIOcQf4oompoojiE4QMq8fn/sw+kABzggCOh6Pc9+OHvNex/O/uw1fd5CSolCoVAo7l7qVHcDFAqFQlG9KEOgUCgUdznKECgUCsVdjjIECoVCcZejDIFCoVDc5ShDoFAoFHc5yhAoyoUQ4rAQIri621FTEEK8JoT4rJrqXiWEmFcddVc2QojHhBA/VzCv+p+8RZQhqMUIIZKEEDeFEBlCiIuGB0PjqqxTSqmXUkZXZR0FCCHqCyH+LYQ4Y7jO40KIV4QQ4nbUb6Y9wUKIs6ZhUsq3pJTjq6g+IYSYLIRIEEL8LYQ4K4RYJ4TwrYr6KooQYrYQYvWtlCGlXCOlfNCCuooZv9v5P3mnogxB7WewlLIxEADcA7xaze0pN0II6xKi1gF9gYFAE2AM8AywsAraIIQQNe33sBCYAkwG7IG2wAbgocquqJTvoMqpzroVBqSU6qilB5AEPGBy/i6w2eS8K7ALuAocBIJN4uyBlcB54AqwwSRuEHDAkG8X4Fe0TqAVcBOwN4m7B7gM1DWcjwUSDeVvAVqbpJXAROA4cMrMtfUFMgG3IuFBQB6gM5xHA/8GfgeuAxuLtKm0exANzAd2Gq5FBzxlaHM68CfwrCFtI0OafCDDcLQCZgOrDWk8DNf1JHDGcC9mmNTXEPjccD8SganA2RK+W2/Ddd5byve/ClgMbDa0dy/gZRK/EEg23Jc4oIdJ3GxgPbDaED8euBfYbbhXF4D/APVM8uiBX4A04BLwGtAfyAZyDPfkoCFtU2C5oZxzwDzAyhAXZrjnHwKphrgwYIchXhji/jK07RDQEe0lIMdQXwawqejvALAytOuk4Z7EUeR/SB1m/pequwHquIUvr/APwNXwg1loOHcx/MgGovX8+hnOnQzxm4GvADugLtDLEH6P4QcYZPhRPWmop76ZOrcBT5u05z1gieHzUOAE0AGwBl4HdpmklYaHij3Q0My1vQ3ElHDdp/nnAR1teNB0RHtYf80/D+ay7kE02gNbb2hjXbS3bS/Dw6gXcAPoZEgfTJEHN+YNwTK0h74/kAV0ML0mwz13BeKLlmdS7r+A02V8/6sM13Ovof1rgEiT+McBB0PcS8BFoIFJu3OAhw33piHQGc1wWhuuJRF43pC+CdpD/SWggeE8qOg9MKn7W+BTw3fSHM1QF3xnYUAuMMlQV0MKG4IQtAd4M8P30AFwNrnmeaX8Dl5B+x20M+T1Bxyq+7da049qb4A6buHL034AGWhvPhLYCjQzxE0DviySfgvag90Z7c3WzkyZ/wXeLBJ2jH8MhemPbjywzfBZoL199jSc/wiMMymjDtpDtbXhXAJ9Srm2z0wfakXi9mB400Z7mL9tEueD9sZoVdo9MMk7t4x7vAGYYvgcjGWGwNUk/ndgtOHzn0CISdz4ouWZxM0A9pTRtlXAZybnA4GjpaS/AvibtPu3Msp/HvjW8DkU+H8lpDPeA8N5CzQD2NAkLBSIMnwOA84UKSOMfwxBH+APNKNUx8w1l2YIjgFDq+L3dicfNW1MVFF+HpZSNkF7SLUHHA3hrYGRQoirBQdwP5oRcAPSpJRXzJTXGnipSD43tGGQonwNdBNCOAM90YzLdpNyFpqUkYZmLFxM8ieXcl2XDW01h7Mh3lw5p9He7B0p/R6YbYMQYoAQYo8QIs2QfiD/3FNLuWjy+QZQMIHfqkh9pV1/KiVfvyV1IYR4WQiRKIS4ZriWphS+lqLX3lYI8b1h4cF14C2T9G5owy2W0BrtO7hgct8/ResZmK3bFCnlNrRhqcXAX0KIpUIIWwvrLk87FQaUIbhDkFLGoL0tLTAEJaO9DTczORpJKd82xNkLIZqZKSoZmF8kn42UMsJMnVeAn4FRwP+hvcFLk3KeLVJOQynlLtMiSrmkX4EgIYSbaaAQIgjtx77NJNg0jTvakMflMu5BsTYIIeqjGbcFQAspZTPgBzQDVlZ7LeEC2pCQuXYXZSvgKoToUpGKhBA90OYgHkXr+TUDrvHPtUDx6/kvcBTwllLaoo21F6RPBtqUUF3RcpLRegSOJvfdVkqpLyVP4QKlXCSl7IzWw2uLNuRTZj5D3V5lpFEUQRmCO4uPgH5CCH+0ScDBQogQIYSVEKKBYfmjq5TyAtrQzSdCCDshRF0hRE9DGcuAfwkhggwraRoJIR4SQjQpoc7/AU8AIwyfC1gCvCqE0AMIIZoKIUZaeiFSyl/RHoZfCyH0hmvoariu/0opj5skf1wI4SOEsAHmAuullHml3YMSqq0H1AdSgFwhxADAdEnjJcBBCNHU0usowlq0e2InhHABwktKaLi+T4AIQ5vrGdo/Wggx3YK6mqCNw6cA1kKImUBZb9VN0CZnM4QQ7YEJJnHfA85CiOcNy3qbGIwyaPfFo2DVleH/62fgfSGErRCijhDCSwjRy4J2I4QINPz/1QX+Rls0kG9SV0kGCbQhxTeFEN6G/18/IYSDJfXezShDcAchpUwBvgBmSimT0SZsX0N7GCSjvVUVfOdj0N6cj6JNDj9vKGMf8DRa1/wK2oRvWCnVfoe2wuWilPKgSVu+Bd4BIg3DDAnAgHJe0nAgCvgJbS5kNdpKlElF0n2J1hu6iDaROdnQhrLuQSGklOmGvGvRrv3/DNdXEH8UiAD+NAx5mBsuK425wFngFFqPZz3am3NJTOafIZKraEMew4BNFtS1Be2+/YE2XJZJ6UNRAC+jXXM62gvBVwURhnvTDxiMdp+PA70N0esMf1OFEPsNn59AM6xH0O7leiwb6gLNYC0z5DuNNkz2niFuOeBjuP8bzOT9AO37+xnNqC1Hm4xWlIL4pyevUNQ+hBDRaBOV1bK791YQQkxAm0i26E1ZoagqVI9AobhNCCGchRD3GYZK2qEtxfy2utulUKgdfQrF7aMe2uoZT7Shnki0eQCFolpRQ0MKhUJxl6OGhhQKheIup9YNDTk6OkoPD4/qboZCoVDUKuLi4i5LKZ3MxdU6Q+Dh4cG+ffuquxkKhUJRqxBCnC4pTg0NKRQKxV1OresRKBRVRU5ODmfPniUzM7O6m6JQVJgGDRrg6upK3bp1Lc6jDIFCYeDs2bM0adIEDw8PRPWIoCkUt4SUktTUVM6ePYunp6fF+dTQkEJhIDMzEwcHB2UEFLUWIQQODg7l7tUqQ6BQmKCMgKK2U5H/4SozBEKIFUKIv4QQCSXECyHEIiHECSFEvBCiU1W1RaFQKBQlU5U9glVoeqYlMQDNa6U3mhbpf6uwLQqFQqEogSqbLJZS/iaE8CglyVDgC4OQyR4hRDMhhLPBl7mimpn48WAu1jtT3c24rUy55wPqpuSXnbAK8W7ekSHDH+L9/74DQG5uLt19e+PfyZdlayx3SxTc+UG++fkr7B3sKpymXUs/2nbwJi8vD1d3FxYs/je2TW05e+Yc/e8fgqeXhzHt11siqVev+CqVea+/zU/f/cxvB36lTh3tvXPRu4uxaWTD+IlPmW1LyqXLzH/jbeL/XwK2TW1xdHJgxrxpheqrqXwTuZFPPvwUgOdeeJZHRg8tlubIoaPMnDqXrMwsrK2tmP3OG/h38iX9ejovPTedC2cvkJuXx7jnwhgROgyAOvlglS+RVtZ4NG9f6e2uzlVDLhT2j37WEFbMEAghnkHrNeDu7n5bGne3c7HeGc7XzaVVjlpYdjuxsWnIH0dPkHkzkwYNG7AzZjctWjYvO2MV0KBBfTZFfQ3A1PDXWL0igudeeBYAdw83Y1xJ5Ofn88sPW2np0pLfd+2j6/33llmnlJLnwqYwbNQQPlqqie0lJhzlckpqhQxBXl4eVlZW5c5XEa5eucbHC/7Lt798hRDw8AOj6Ns/mKbNCusYvTv3fSa9PIFefXsQ/etvvDv3fdZsWMXqFRHo2nqxdPViUi+nEdJ9EEOGD6JpvhXNU7PJrwMXWlTN77FW/MqllEuBpQBdunRRXvJuE61yrPn62YNlJ7xDSExMROfUAYA5mw5z5Pz1Si3fp5UtswbrS00jRB2GDXmEo7+fYsSIEcz94R3CxjzF9u3b0TnpSUtLY+zYsfz555/Y2NiwdOlS/Pz8SE1NJTQ0lHPnztGtWzeshDVtHNvj6OjI6tWrWbRoEdnZ2QQFBfHJJ59gZWWFdZ26xjQltUXnpLX3wd4DiI+PR+ekx/rvRtSzqm+MK4lt27bh7xvAqFGj+O3HnTw+TOsB2DdqTuPGjQvlL2hLfHw8TWxsmfnKXGOcrrf5eh5++GGSk5PJzMxkypQpPPPMMwA0btyYZ599ll9//ZXFixeTlJRk9vonTJhAbGwsN2/eZMSIEcyZM6fU6ymLiF8jGNh/IF3adgdgYP+B/LHvNKGhoYXSNarfhMbCDp2Tnlji8Wrtjc5Jj2Pjlty8koyXow9WGUk4OjjRPsuWOmlpUL8+6TZ2NM1ucEttLInqXDV0jsKara6GMIXirmb06NFERkaSmZlJfHw8QUFBxrhZs2Zxzz33EB8fz1tvvcUTTzwBwJw5c7j//vs5fPgww4YN48wZbVgvMTGRr776ip07d3LgwAGsrKxYs2ZNudqTl5fH1q1bGTJkiDHs5MmTBAQEEBAQwMSJE83mi4iIIDQ0lGHDhrF582ZycnLKrCshIYHOnTtb1K4VK1YQFxfHvn37WLRoEampqQD8/fffBAUFcfDgQRwcHEq8/vnz57Nv3z7i4+OJiYkhPj6+WB3vvfee8TpNj8mTJxdLe+7cOdzc/nmkubq6cu5c8UfaRx99xCuvvIKbmxsvv/wy//73vwEIDw8nMTGRVq1a4evry8IXX9SMQMuWoNeTW69qjABUb4/gOyBcCBEJBAHX1PyAoqZQ1pt7VeLn50dSUhIREREMHDiwUNyOHTv4+mttSKZPnz6kpqZy/fp1fvvtN7755hsAHnroIezstHH/rVu3EhcXR2BgIAA3b96keXPLhppu3rxJQEAA586do0OHDvTr188Y5+XlxYEDB0rMm52dzQ8//MAHH3xAkyZNCAoKYsuWLQwaNKjE5Y3lXfa4aNEivv1W0/VJTk7m+PHjODg4YGVlxfDhw4HSr3/t2rUsXbqU3NxcLly4wJEjR/Dz8ytUxyuvvMIrr7xSrnaVxX//+18+/PBDhg8fztq1axk3bhy//vorW7ZsIcDPj21bt3Lyzz/p17cvPfbswdbZUoXPilNlhkAIEQEEA45CiLPALKAugJRyCfADMBBNE/cG8JT5khSKu48hQ4bw8ssvEx0dbXzTrQhSSp588knjW2d5aNiwIQcOHODGjRuEhISwePFis2/C5tiyZQtXr17F19cXgBs3btCwYUMGDRqEg4MDFy4UfudLT0+nWbNm6PV61q9fX2b50dHR/Prrr+zevRsbGxuCg4ONm6gaNGhgnBco6fpPnTrFggULiI2Nxc7OjrCwMLObsN577z2zPaiePXuyaNGiQmEuLi5ER0cbz8+ePUtwcHCxvJ9//jkLFy4EYOTIkYwfPx6kZOWnnzI9NBSRmopOp8PTy4ujycncexsMQZUNDUkpQ6WUzlLKulJKVynlcinlEoMRQGpMlFJ6SSl9DaLpCoUCGDt2LLNmzTI+SAvo0aOH8cEUHR2No6Mjtra29OzZk//9738A/Pjjj1y5cgWAvn37sn79ev766y8A0tLSOH26RCeUZrGxsWHRokW8//775ObmWpQnIiKCzz77jKSkJJKSkjh16hS//PILN27coGfPnnz33Xekp6cD8M033+Dv74+VlRV9+vQhKyuLpUuXGsuKj49n+/bthcq/du0adnZ22NjYcPToUfbs2WO2HSVd//Xr12nUqBFNmzbl0qVL/Pjjj2bzv/LKKxw4cKDYUdQIAISEhPDzzz9z5coVrly5ws8//0xISEixdK1atSImJgbQ5lG8dTo4cQL3pk3ZGhcHjRtz6dIljh07Rps2bSy425WAlLJWHZ07d5aKqueRJX7ykSV+1d2M28qRI0equwmyUaNGxcKioqLkQw89JKWUMjU1VQ4dOlT6+vrKoKAgefDgQSmllJcvX5b9+vWTPj4+cvz48dLd3V2mpKRIKaWMjIyU/v7+0tfXV3bq1Enu3r1bSill69atjWksacugQYPkF198IU+dOiX1en2J+f7++29pZ2cnr127Vih82LBhMjIyUkop5ZIlS6Sfn5/09/eX/fr1kydPnjSmO3funBw5cqRs06aN9PHxkQMHDpR//PFHobIyMzNl//79Zfv27eXQoUNlr169ZFRUlNl2l3T9Tz75pPT29pZ9+vSRw4YNkytXrizxmixl+fLl0svLS3p5eckVK1YYw8eNGydjY2OllFJu375ddurUSfr5+cl7O3WS+1avljIuTp47cED269dPduzYUer1evnll18WKjvtTIpMO1Py92WKuf9lYJ8s4bla66Qqu3TpIpUeQdUz/FN/gLtu1VCHDh2quxmKu4lr1+DSJWjdGurXLzXpleTLANi5mV/lZYq5/2UhRJyUsou59LVi+ahCoVDcEUipPfilBGdnaNoUbG2hmn1cKUOgUNzFpKam0rdv32LhW7duxcHBoRpadAdz4wYkJWl/7ew0YyBEtRsBUIZAobircXBwKHUZqKISyM+HCxfg4kWwsoI2bTRDUAMMQAHKECgUCkVVkpWlGQF7e3BzA+ua99iteS1SKBSK2k5eHly9Cg4O0LAhdOxY5mRwdaIMgUKhUFQm169rcwHZ2WBjoxmCGmwEQBkChUKhqBxyc+HsWbh8GRo0gHbtNCNQC1BSlQpFDUIIweOPP248z83NxcnJiUGDBpWrHA8PDy5fvnxLaebPn49er8fPz4+AgAD27t3LnDlzePXVVwulO3DggHHNuoeHBz169CgUHxAQQMeOHUus56OPPqJBgwZcu3bNGLZq1SrCw8MLpQsODqZgD1FGRgbPPvssXl5edO7cmeDgYPbu3Vvq9VYpUsLRo5oRaNkSfHygSROzSX/66SfatWuHTqfj7bffNpsmKyuLUaNGodPpCAoKIikpyRgXHx9Pt27d0Ov1+Pr6lluf2ByqR6BQmOPH6XDxUOWW2dIXBpj/4RfQqFEjEhISuHnzJg0bNuSXX37BxcWlctthAbt37+b7779n//791K9fn8uXL5OdnU1oaCj9+/cv5LsnMjKykKvl9PR0kpOTcXNzIzExscy6IiIiCAwM5JtvvuGppyxzOTZ+/Hg8PT05fvw4derU4dSpUxw5cqT8F8otahbk5GiTv0KAiwvUqweNGpVa18SJE/nll19wdXUlMDCQIUOG4OPjUyjd8uXLsbOz48SJE0RGRjJt2jSWLFhMbm4ujz/+OF9++SX+/v6kpqZSt25xQaDyonoECkUNY+DAgWzevBn4x5VzAWlpaTz88MP4+fnRtWtXo+vk1NRUHnzwQfR6PePHj8fUY8Dq1au59957CQgI4NlnnyUvL6/MNly4cAFHR0fqG8a2HR0dadWqFW3btsXOzq7Q2/fatWsLtfHRRx/lq6++Mtv+opw8eZKMjAzmzZtHRESEJbeHkydPsnfvXubNm2dUPfP09OShhx4qlnbChAl06dIFvV7PrFmzjOEeHh5MmzaNTp06sW7dOn7++We6detGp06dGDlyJBkZGQDMnTuXwMBAOnbsyDPPPPPPfZVSe/tPSND+grYktBQjAPD777+j0+lo06YN9erVY/To0WzcuLFYuo0bN/Lkk08CMGLECLZu3YqUkqjfovDz88PfX9v5X+Bt9VZRPQKFwhxlvLlXJaNHj2bu3LkMGjSI+Ph4xo4da3S6VqBHsGHDBrZt28YTTzzBgQMHjHoEM2fOZPPmzSxfvhworEdQt25dnnvuOdasWWPUMSiJBx98kLlz59K2bVseeOABRo0aRa9evQAIDQ0lMjKSoKAg9uzZg729Pd7e3sa8w4cP56mnnuLll19m06ZNrFmzhi+//NJsPZGRkYwePZoePXpw7NgxLl26RIsWLUpt2+HDhwkICLDoATh//nzs7e3Jy8ujb9++xMfHG11NOzg4sH//fi5fvswjjzzCr7/+SqNGjXjnnXf44IMPmDlzJuHh4cycOROAMWPG8P333zP4wQfh9GltUrhxY9Z8/z3vGbyJmqLT6Yp5UjWnWWBuSMs0nbW1NU2bNiXtShon/jyJEIKQkBBSUlIYPXo0U6dOLfM+lIUyBApFDaMm6BE0btyYuLg4tm/fTlRUFKNGjeLtt98mLCyMUaNG0b17d95///1iw0KgPWDt7OyIjIykQ4cO2NjYlFhPREQE3377LXXq1GH48OGsW7eO8PDwStMsKE1zYNSoUQDs2bOHI0eOcN999wGalkK3bt0AiIqK4t133+XGjRukpaWh9/RkcMFQnbs7ODnxWPv2PGbhkNatkpuXx44dO4iNjcXGxoa+ffvSuXNns7vDy4MyBApFDaQm6BFYWVkRHBxMcHAwvr6+fP7554SFheHm5oanpycxMTF8/fXX7N69u1jeUaNGMXHiRFatWlVi+YcOHeL48eNGwZvs7Gw8PT0JDw/HwcHB6Eq7gLS0NBwdHWnWrBkHDx4sc2y/LM2BRoZhHCkl/fr1KzY0lZmZyXPPPce+fftwc3Nj9uzZWv7GjQs5iVuzZg3vvfdesfrN9QhcXFxITv5Hqv3s2bNm54AK0rm6upKbm8u1a9ewt7OnlXMrevbsaZQXHThwIPv3779lQ6DmCBSKGkh16xEcO3aM48ePG88PHDhA69atjeehoaG88MILtGnTBldX12L5hw0bxtSpU8364y8gIiKC2bNnGzULzp8/z/nz5zl9+jSBgYHs3LmTixcvArBv3z6ysrJwc3PDy8uLLl26MGvWLOOYfVJSknFepQBLNQe6du3Kzp07OXHiBKBJXf7xxx9Go+GYk0PG8ePaQ71BA/D2LrQv4LHHHjOrWWBOYCcwMJDjx49z6tQpsrOziYyMLCQBWsCQIUP4/PPPAVi/fj19+vRBCEHfnr05dOgQN27cIDc3l5iYmGITzRVB9QgUihqIq6urWTWw2bNnM3bsWPz8/LCxsTE+LGbNmkVoaCh6vZ7u3bvj7u4OgI+PD/PmzePBBx8kPz+funXrsnjx4kIPdXNkZGQwadIkrl69irW1NTqdrpBYzMiRI5k8eTIff/yx2fxNmjRh2rRppdYRGRnJDz/8UChs2LBhxlUyCxcuZODAgeTn59O4cWMiIiKMk8OfffYZL730EjqdjoYNG+Lo6Fjsrdzf35977rmH9u3b4+bmZhz6KYqTkxOrVq0iNDSUrKwsAObNm0dbV1eefvhhOvbsScsWLYzDa7fiI8ja2pr//Oc/hISEkJeXx9ixY9HrNVnUmTNn0qVLF4YMGcK4ceMYM2YMOp0Oe3t7IiMjAWjWrBkvvvgigYGBCCEYOHCg2Uny8qL0CBRmUXoEiruW/Hw4f17zD1S3rjYXYJhzqU6UHoFCoVDcLrKyNM0AR0dwda2RTuIqmzv/ChUKRYkoPQIDtcxJXGWjDIFCcRej9AjQ5CJPn65VTuIqG2UIFArF3UluLiQnQ2qqthqoffta4ySuslGGQKFQ3H0UOInLzNS0g52doc7du5peGQKFQnH3YOokztVVcxJXys7nu4W71wQqFIq7B3NO4po1U0bAgDIECkUNoibpEQBs2LABIQRHjx41hiUlJSGEKLSZLDw83OhOIiwsDBcXF+PmrMuXL+Ph4VGuOqKjo4tdc1hYmHG3bk5ODtOnT8fb25tOnTrRrVu3EncOk5UFx49rqmENG5aoE3C7iIuLw9fXF51Ox+TJkzG3l+vatWsMHjwYf39/9Ho9K1euNMZNnToVvV5Phw4dSsxfXtTQkEJhhnd+f4ejaUfLTlgO2tu3Z9q9pe+2rSl6BAVERERw//33ExERwZw5c4zhzZs3Z+HChTz77LPUq1evWD4rKytWrFjBhAkTKlxHabzxxhtcuHCBhIQE6tevz6VLl4iJiSmeMDVVWxEERidxRXcG5+bmYn0b9wpMmDCBZcuWERQUxMCBA/npp58YMGBAoTSLFy/Gx8eHTZs2kZKSQrt27RgYG8L/iz/Azp07je7H77//fmJiYggODr6lNqkegUJRw6gJegSguZnYsWMHy5cvN7o4KMDJyYm+ffsaXVwU5fnnn+fDDz8kNze3wnWUxI0bN1i2bBkff/yxUS+hRYsWPProo8XSzn3/fQKffJKOY8bwzOuvU3BXgoODef755+nSpQsLFy4kLi6OXr160blzZ0JCQrhw4QIAy5YtIzAwEH9/f4YPH86NGzcsamNJXLhwgevXr9O1a1eEEDzxxBNs2LChWDohBOnp6UgpycjIwN7eHmtra4QQZGZmkp2dTVZWFjk5OWW67bYE1SNQKMxQ1pt7VVIT9AhAE0fp378/bdu2xcHBgbi4ODp37myMnzZtGgMGDGDs2LHF8rq7u3P//ffz5ZdfMnjw4ArXYY4TJ07g7u6Ora1t8cj8fG1XsJTQqhXhL7/MzLfeAkz0BAztyc7OZt++feTk5NCrVy82btyIk5MTX331FTNmzGDFihU88sgjPP300wC8/vrrLF++nEmTJhWqMioqihdeeKFYU2xsbNi1a1ehsHPnzhVy0ufq6sq5c+eK5Q0PD2fIkCG0atWK9PR0vvrqK+rUqcO9nQPp3bs3zs7OSCkJDw+vFLcoyhAoFDWMmqBHAFpvZMqUKYBmnCIiIgo9pNu0aUNQUJDR62lRXn31VYYOHVqqU7SS6qiQHsHff2vzADdvgr09SFlcT0CvNxqCAj2CY8eOkZCQYHSHnZeXh7OzMwAJCQm8/vrrXL16lYyMDLPeVHv37l3pm/K2bNlCQEAA27Zt4+TJk/Tr14/ozdu4nJpCYmIiZ8+eBaBfv35s3769mE50eVGGQPhZ79oAACAASURBVKGogVS3HkFaWhrbtm3j0KFDCCHIy8tDCFHMw+drr73GiBEjjOplpnh7exMQEMDatWvLXUdpegQ6nY4zZ85w/fp1rVdQ1EmclxfY2ZWsJ2DAVI9Ar9eb1VUICwtjw4YN+Pv7s2rVKqKjo4ulKU+PwMXFxfgQh5L1CFauXMn06dMRQqDT6TR95pPH2blnF127dqVx48YADBgwgN27d9+yIajSOQIhRH8hxDEhxAkhxHQz8e5CiCghxP8TQsQLIQaaK0ehuNuobj2C9evXM2bMGE6fPk1SUhLJycl4enoah6gKaN++vXFS0xwzZsxgwYIF5a7D29ub8+fPk5iYCMDp06c5ePAgAQEB2NjYMG7cOKZMmUJ2djZkZZFy9Cjr9u4Fvd7oKdSoJ+DoSEZGhll9AIB27dqRkpJiNAQ5OTkcPnwYgPT0dJydncnJyTHe96IU9AiKHkWNAICzszO2trbs2bMHKSVffPEFQ4cOLZbO3d2drVu3AnDp0iWOHTuGh3trXFu5EBMTQ25uLjk5OcTExFTK0FCVGQIhhBWwGBgA+AChQoiiCgqvA2ullPcAo4FPqqo9CkVtojQ9gri4OPz8/Jg+fXohPYLffvsNvV7PN998Y1aPwM/Pj379+hknQksjIiKCYcOGFQobPny4WYH5GTNmFHrLNUWv19OpU6dy11G/fn1Wr17NU089RUBAACNGjOCzzz6jadOmAMybMwenxo3x8fGhY2Agg157DVudrpCn0GbNmvH000/TsWNHQkJC/tETKEK9evVYv34906ZNw9/fn4CAAOND/M033yQoKIj77ruP9u3bl3C3yscnn3zC+PHj0el0eHl5GVcMLVmyhCVLlgDaqqhdu3bh6+tL3759eeedd3Cwd2DoQ0Pw8vLC19cXf39//P39S52DsZQq0yMQQnQDZkspQwznrwJIKf9tkuZT4E8p5TuG9O9LKbuXVq7SI7g9KD0CRY3F1EmcXn/X+AeqrXoELkCyyflZIKhImtnAz0KISUAj4AFzBQkhngGeAYxvOgqF4i4jJ0dzEpeWdtc7iatsqnuyOBRYJaV839Aj+FII0VFKmW+aSEq5FFgKWo+gGtqpUNyR1Bo9Ainh2DFtl7ByElfpVKUhOAe4mZy7GsJMGQf0B5BS7hZCNAAcgb+qsF0KhcJAjdcjUE7ibgtVaVJjAW8hhKcQoh7aZPB3RdKcAfoCCCE6AA2AlCpsk0KhqA1ICSkpmpO4FMMjQTmJqzKqrEcgpcwVQoQDWwArYIWU8rAQYi6wT0r5HfASsEwI8QIggTBZVbPXCoWidpCVpW0MS0/XHMSZ20GsqFSqdI5ASvkD8EORsJkmn48A91VlGxQKRS3i8mU4c0b73Lq1JiBf2m5iRaWgZlsUihrEXe+Gul49aNKE6NRUBj31VCEjUGE31DWMmuiGWhkChaIGYeqGGqhRbqhNKXBDnZ2dbTZfgRvqMsnPJ2LFCu6/916tDltb8PbWXEWUgqkb6v3797NhwwbS09Mtvi5TyvKQWtkUuKE+fvw4x48f56effiqWpsAN9cGDB4mOjuall14iOzubvft+N7qhTkhIIDY21rz77XJS3ctHFYoaycW33iIrsXL1COp3aE/L114rM12BG+oRI0YY3VAXuHZIS0tj7Nix/Pnnn9jY2LB06VL8/PxITU0lNDSUc+fO0a1bt2JuqBctWkR2djZBQUF88sknWFlZldmOAhfRUVFRDB48uJBWgJOTE/fddx+ff/650TunKQVuqM3FGfn7bzKOHGHHnj1ERUQw+LnnLNIjKHBDferUqbLdUM+dy6ZNm7h58ybdu3fn008/RQhBcHAwAQEB7Nixg9DQUIKDg3nxxRfJyMjA0dGRVatW4ezszLJly1i6dCnZ2dnodDq+/PJLbG5hwtrUDTVgdENdVI/AEjfUUspKc0OtegQKRQ1j9OjRREZGkpmZSXx8PEFB/+zDLHBDHR8fz1tvvWV0J13ghvrw4cMMGzaMM4ZxdlM31AcOHMDKyqpEnzlFMeci2pRp06axYMECs/oGpm6oi5GXp20MS0xk4y+/0D8khLZ9+5qtwxyluqEuQnh4OLGxscZe1vfff2+MK3BDPXnyZCZNmsT69euJi4tj7NixzJgxA4BHHnmE2NhYDh48SIcOHYzuvU2JiooiICCg2NG9e3EnCeVxQ52YmEirVq3w9fVl4cKFxdxQOzs7ExISotxQKxRVhSVv7lXFHe+GOjsb/voLnJyI2LmTKc8/X6yOCrmhNoNyQ20ZyhAoFDWQO9IN9eXL2iqghg3B15e0jAy2RUVxKCHh1txQl4ByQ205amhIoaiB3FFuqN99V+sFFIjGABg8flaKG2ogJSWFdevWFapbuaG2HGUIFIoayB3hhjonB33DhnTy8tKWgXboUMhJ3C25oZ43DycnJ80NdceODBo0qFjvQLmhtpwqc0NdVSg31LcH5YZacUtICYcP/+MkrmVL5STuFqmtbqgVCsXdRlEncfXrK1fRtQCLDYEQwkZKeaMqG6NQKG4vleaGWkptMjg5WTMAzZtrTuIUtYIyDYEQojvwGdAYcBdC+APPSimfq+rGKRSKqqVS3FBnZmqKYQVO4gzj+IragyU9gg+BEAwupKWUB4UQPau0VQqFonZw+bJmBOrUUU7iajEWDQ1JKZOLbOQovpVQoVDcfdSrp/UA3N21z4paiSWGINkwPCSFEHWBKUBi1TZLoVDUSPLz4eJFbU7AxUVzEqf0Amo9lqzn+hcwEU2M/hwQAKj5AYXibiMjAxIT4fx5bYNYLVt6rigZSwxBOynlY1LKFlLK5lLKxwG12FqhqAJqkh7B/Pnz0ev1+Pn5EaDXs3f9euYsXsyra9aAp6dxLuDAgQPGNeseHh7F3B0EBATQsWPHEuv56KOPaNCgAdeuXTOGrVq1ivDw8ELpgoODKdhDlJGRwbPPPouXlxedO3cmODiYvXv3lnq9NYWffvqJdu3aodPpePvtt82mycrKYtSoUeh0OoKCgkhKSjLGxcfH061bN/R6Pb6+voXcZlQUSwzBxxaGKRSKW6Sm6BHs3r2b77//nv379xO/dy+/LlqEm48PoZMm8dXGjYXSRkZGEhoaajxPT08nOTkZwOgiojQiIiIIDAw0Os2zhPHjx2Nvb8/x48eJi4tj5cqVZRq+kjDnPbWqyMvLY+LEifz4448cOXKEiIgIjhw5Uizd8uXLsbOz48SJE7zwwgtMmzYN0F4MHn/8cZYsWcLhw4eJjo6mbhnaDZZQ4hyBEKIb0B1wEkK8aBJli6ZBrFDcsWxf+weXkzMqtUxHt8b0eLRtmelqgh7BhbNncbS1Nfr7d+zRwzgZbGdnx969e43usdeuXcuWLVuMeR999FG++uorXn75ZWP7zbqjBk6ePElGRgaffPIJ8+fP56mnnirz/pw8eZK9e/eyZs0a6hh2K3t6euLp6Vks7YQJE4iNjeXmzZuMGDHCqHfg4eHBqFGj+OWXX5g6dSr29vbMmjWLrKwsvLy8WLlyJY0bNy5Rz6Ci/P777+h0Otq0aQNoHlc3btyIj49PoXQbN25k9uzZAIwYMYLw8HCklET9FoWfnx/+/trO/3Lt9SiF0noE9dD2DlgDTUyO68CISqldoVAUo9r1CK5e5UFXV5JPnaKttzfPPfccMSaeOUNDQ4mMjARgz5492Nvb4+3tbYwfPny48e1+06ZNpfrCiYyMZPTo0fTo0YNjx45x6dKlMu/P4cOHCQgIsEhcZ/78+ezbt4/4+HhiYmKIj483xjk4OLB//34eeOAB5s2bx6+//sr+/fvp0qULH3zwAVC6nkEBa9asMatHMGJE8cfkuXPncHNzM56XpEdgms7a2pqmTZuSdiWNE3+eRAhBSEgInTp14t133y3zHlhCiT0CKWUMECOEWCWlLNtdoUJxB2HJm3tVUW16BDk5mnD8lSs0trUlbt8+tu/fT1RUFKNGjeLtt98mLCyMUaNG0b17d95///1iw0KgPWDt7OyIjIykQ4cOpSp6RURE8O2331KnTh2GDx/OunXrCA8PrzQ9grVr17J06VJyc3O5cOECR44cwc/PD/hHj2DPnj0cOXKE++67D9AEa7p16waUrmdQwGOPPcZjjz1WrnZVlNy8PHbs2EFsbCw2Njb07duXzp07m90dXh4sWT56QwjxHqAHGhQESin73FLNCoWiRG67HoGUcPSothrIxQVatMCqTh2Cg4MJDg7G19eXzz//nLCwMNzc3PD09CQmJoavv/7arB//UaNGMXHiRKOgvTkOHTrE8ePHjYIw2dnZeHp6Eh4eXqoeQbNmzTh48CB5eXml9gpOnTrFggULiI2Nxc7OjrCwsBL1CPr161fMs2pZegYFrFmzpphOA4BOpyvm+trFxcU4fwIl6xEUpHN1dSU3N5dr165hb2dPK+dW9OzZE0dHzfHcwIED2b9//y0bAksmi9cARwFPYA6QBMTeUq0KhaJUbpseQcEyUCG0TWE+PuDszDGDsHoBBw4coHXr1sbz0NBQXnjhBdq0aVNIerGAYcOGMXXqVLOKXgVEREQwe/ZskpKSSEpK4vz585w/f57Tp08TGBjIzp07uXjxIgD79u0jKysLNzc3vLy86NKlC7NmzTLOhSQlJbF58+ZC5V+/fp1GjRrRtGlTLl26xI8//mi2HV27dmXnzp2cOHECgL///ps//vjDYj2Dxx57zKwegbn0gYGBHD9+nFOnTpGdnU1kZCRDhgwplm7IkCFGF+Pr16+nT58+CCHo27M3hw4d4saNG+Tm5hITE1NsfqEiWNIjcJBSLhdCTDEZLlKGQKGoQkrTIxg7dix+fn7Y2NgU0iMIDQ1Fr9fTvXt3s3oE+fn51K1bl8WLF9Pa3V3bHJaYqHkLbd68kI+gjIwMJk2axNWrV7G2tkan07F06VJj/MiRI5k8eTIff2x+AWGTJk2MK11KIjIykh9++KFQ2LBhw4iMjGTatGksXLiQgQMHkp+fT+PGjYmIiDBODn/22We89NJL6HQ6GjZsiKOjY7G3cn9/f+655x7at2+Pm5ubceinKE5OTqxatYrQ0FCysrIATe+gbdu2Rj2Dli1blqhnUB6sra35z3/+Q0hICHl5eYwdOxa9Xg/AzJkz6dKlC0OGDGHcuHGMGTMGnU6Hvb29cU6mWbNmvPjiiwQGBiKEYODAgcWlQCtAmXoEQog9UsquQogtwCLgPLBeSul1y7VXAKVHcHtQegR3MJmZmlpYRoa2K7h1a81dtKJGU916BPOEEE2Bl9D2D9gCz1uQT6FQ1DRSUrQJ4Tp1wMMDHByUkzhF2YZASlmwXuoa0BtACGG+j6VQKGo29esXchJXaXoEilpNaRvKrIBH0XwM/SSlTBBCDAJeAxoC99yeJioUigqTnw8FGsVmnMRVih6BotZTWo9gOeAG/A4sEkKcB7oA06WUG25H4xQKxS2QkaHNBWRmajoBBauDFIoilGYIugB+Usp8IUQD4CLgJaWs+KJmhUJR9eTlwblz8NdfmlsIb2+lGqYoldL2EWRLKfMBpJSZwJ/lNQJCiP5CiGNCiBNCiOklpHlUCHFECHFYCPG/8pSvUCjMkJ2tTQo3bw56vTICijIprUfQXghR4JhDAF6GcwFIKaVfaQUb5hgWA/2As0CsEOI7KeURkzTewKvAfVLKK0KIEva+KxSKUsnNhStXwMkJGjYEX1+lGKawmNJ6BB2AwYZjkMn5IMPfsrgXOCGl/FNKmQ1EAkOLpHkaWCylvAIgpfyrfM1XKO4sKqRHcOUKHD6saQcbdsN6tG17y3oEABs2bEAIwdGjR41hSUlJCCEKbSYLDw83upMICwvDxcXFuDnr8uXLeHh4lKuO6OjoYtccFhZm3K2bk5PD9OnT8fb2plOnTnTr1q3EncM1jbi4OHx9fdHpdEyePBlze7muXbvG4MGD8ff3R6/Xs3LlSmPc1KlT0ev1dOjQocT85aVEQyClPF3aYUHZLkCyyflZQ5gpbYG2QoidQog9Qoj+5goSQjwjhNgnhNiXkpJiQdUKRe2kXHoEOTlw8qR21K0LHTpAgwbm01aQiIgI7r///mJ+eJo3b87ChQvJzs42m8/KyooVK1bcUh2l8cYbb3DhwgUSEhLYv38/GzZsID093eL8puTm5lYoX0WZMGECy5Yt47jBjcdPP/1ULM3ixYvx8fHh4MGDREdH89JLL5Gdnc3efb+zc+dO4uPjSUhIIDY2lpiYmFtuk0Xi9VWINeANBAOuwG9CCF8p5VXTRFLKpcBS0HYW3+5GKu4+olYt5a/Tf1Zqmc1bt6F32DNlprNYjwBY+tpr+PXqRaq1NaHDhlWaHgFobiZ27NhBVFQUgwcPNvryB80tw3333cfnn3/O008/XSzv888/z4cffmg2ztI6SuLGjRssW7aMU6dOGfUSWrRowaOPPlosbUl6AsHBwQQEBLBjxw5CQ0MJDg7mxRdfJCMjA0dHR1atWoWzszPLli1j6dKlZGdno9Pp+PLLL0v1ploWFy5c4Pr163Tt2hWAJ554gg0bNjBgwIBC6YQQpKenI6UkIyMDe3t7rK2tEUKQmZlJdnY2UkpycnJo0aJFhdtTgCVO5yrKObTlpwW4GsJMOQt8J6XMkVKeAv5AMwwKxV1LqXoEr7/OPQEBmh7BvHk88dZb4OzMnDffrDw9AgMbN26kf//+tG3bFgcHB+Li4grFT5s2jQULFphV+HJ3d+f+++8vUZDG0jrMceLECdzd3bE12Q9REqXpCWRnZ7Nv3z4mT57MpEmTWL9+PXFxcYwdO5YZM2YA8MgjjxAbG8vBgwfp0KEDy5cvL1ZHVFSUWT2C7t27F0t77ty5Qk76StIjCA8PJzExkVatWuHr68vChQupU6cO93YOpHfv3jg7O+Ps7ExISEiluEWxqEcghGgIuEspj5Wj7FjAWwjhiWYARgP/VyTNBiAUWCmEcEQbKqrc1zCFogJY8uZeVZSoR3DpEju2bePrL74AoM+QIaROnFh5egRFiIiIYMqUKYBmnCIiIujcubMxvk2bNgQFBRm9nhbl1VdfZejQoaU6RSupjsrSIyhNT6BAj+DYsWMkJCQY3WHn5eXh7OwMQEJCAq+//jpXr14lIyPDrDfV3r17V/qmvC1bthAQEMC2bds4efIk/fr1I3rzNi6nppCYmMjZs2cB6NevH9u3by+mE11eyjQEQojBwAI0xTJPIUQAMFdKWdx3qglSylwhRDiwBU3acoWU8rAQYi6wT0r5nSHuQSHEESAPeEXtU1AoiugRnD+vbQ5LTtZ8BDVpYnE5FusRFCEtLY1t27Zx6NAhhBDk5eUhhCjm4fO1115jxIgR9OrVq1gZ3t7eBAQEsHbt2nLXUZoegU6n48yZM1y/fr3UXkFZegKmegR6vd6srkJYWBgbNmzA39+fVatWER0dXSxNVFQUL7zwQrFwGxsbdu3aVSjMxcXF+BCHkvUIVq5cyfTp0xFCoNPp8PT05PjJ4+zcs4uuXbvSuHFjAAYMGMDu3btv2RBYMjQ0G20F0FUAKeUBNG2CMpFS/iClbCul9JJSzjeEzTQYAaTGi1JKHymlr5QyskJXoVDcYRj1CFq21HYH5+WBhwc9+vZljWHlTKXoEZTA+vXrGTNmDKdPnyYpKYnk5GQ8PT2NcxUFtG/fHh8fHzZt2mS2nBkzZrBgwYJy1+Ht7c358+dJTEwE4PTp0xw8eJCAgABsbGwYN24cU6ZMMU5Wp6SksG7dukLlW6on0K5dO1JSUoyGICcnh8OHDwOQnp6Os7MzOTk5JQ6pFfQIih5FjQCAs7Mztra27NmzByklX3zxBUOHFl1MqQ2tbd26FYBLly5x7NgxPNxb49rKhZiYGHJzc8nJySEmJqZShoYsMQQ5UsprRcLUhK1CUYUY9Qjq14fGjTX/QI6OzJ4zh7i4OPz8/Jg+fXohPYLffvsNvV7PN998Y1aPwM/Pj379+nGhwPdQKURERDBs2LBCYcOHDze7smfGjBmF3nJN0ev1dOrUqdx11K9fn9WrV/PUU08Z9X8/++wzmho2x82bNw8nJyd8fHzo2LEjgwYNKtY7aNasmVFPICQkpEQ9gXr16rF+/XqmTZuGv78/AQEBxof4m2++SVBQEPfddx/t27c3m7+8fPLJJ4wfPx6dToeXl5dxonjJkiUsWbIE0FZF7dq1C19fX/r27cs777yDg70DQx8agpeXF76+vvj7++Pv71+qJrSlWKJHsBzYCkwHhgOTgbpSyn/dcu0VQOkR3B6UHkE1kZ8P589rn80ofynuXqpSj8CSHsEkNL3iLOB/aO6olR6BQlHZZGTAkSNw8aK2U7gSNgopFJZgyaqh9lLKGcCMqm6MQnFXUo1O4pQegQIsMwTvCyFaAuuBr6SUCVXcJoXi7sLUSZyLC1iw2auyUHoECrBgaEhK2RtNmSwF+FQIcUgI8XqVt0yhuJPJzdV6APCPkzh399tqBBSKAizaWSylvCilXAT8CzgAzKzSVikUdypSQloaJCRo+wIK1rUrT6GKasSSDWUdgFFoK4ZSga/QhOwVCkV5yM7WhOOvXgUbG008vpKdxCkUFcGSOYIVaA//ECnl+Spuj0JxZyIlHDumGQNXV2jRQslGKmoMlswRdJNSfqSMgEJRAbKz/9EKdnfXFMNatizRCFRIj8AMlmgNKD2C6qFW6REIIdYa/h4SQsSbHIdMlMsUCoU5pIRLl7S5gAINjaZNyxwKKpcewW1A6RFUPrVNj2CK4W/5XkUUijuAq5tOkn3+74plzs/TJoHz8sHaChqcB3GReq0a0WywV5nZLdYjsLFh6dKl+Pn5kZqaSmhoqNIjKILSI7CM0hTKChySPGdGney5W65ZobgTycmGGzcgX0LDBtrSUFE+2Y9S9QhmzeKee+7R9AjeeosnnngCgDlz5ig9AjMoPQLLsGSyuB8wrUjYADNhNZqJHw/mYr0z1d2MWsP5urm0yqluAbvqw5I3d7Ncv64NBbm7a/KRFaBEPQJgx44dfP311wD06dOH1NRUpUdQChXRI8jNzsbJsQVXki+ze89O5r/3b65dv87fN/6mT8/eXHm48LxKgM6XqE2/mq2/wD9QAdcvXiU3M8cYnn7pKjk3s4ul27j5O9p7tePrz9dy6vQpHnlsJFE/bOfKlYu3V49ACDEB7c2/TZE5gSbAzluqtRq4WO/MXf9wKw+tcqxpme1e3c2o+RR1Emdrqx23SCE9gtSKS3QoPYLy6xFcSb5Mbr41kMvElyazetnndPTpyP/WRbBzd/FH3/ZdO5gxt/ge24YNbdjy7Q+FwpxbtuT8xX/W3Zy/eAHnls7F8v5vXQTPT5iMEII2Hm1o7ebOn6eO8nvc7irRI0BKafYAmgIeQATQ2uSwLynP7Tg6d+4sK8IjS/zkI0v8KpRXcXdw5MiR8mW4fl3K+HgpY2OlTEqSMj//ltvQqFEjKaWUycnJcuHChVJKKaOiouRDDz0kpZRy0qRJcu7cucbwgIAAY/ibb74ppZTyhx9+kIBMSUmRhw8fljqdTl66dElKKWVqaqpMSkqSUkrZunVrmZKSYrYdn376qXzmmWcKhfXs2VPGxMTIU6dOSb1ebwwfOXKkdHNzkytXrpRSSvnkk0/KdevWSSmlTEhIkK1bt5atW7cuVx2ZmZnSw8PD+J0kJSVJd3d3efXqVSmllK+88ooMCwuTWVlZUkop//rrL7l27dpCZV25ckU2b95c3rhxQ6anp0u9Xi9nzZolpZSyV69eMjY2VkopZVZWlvTy8pK7du2SUkp56eR5ufOX7VJKKR0cHOSlS5dkdna2fOCBB+STTz5p9n6Vh8DAQLl7926Zn58v+/fvLzdv3lwszb/+9S9jWy9evChbtWolU1JSZGRkpOzbt6/MycmR2dnZsk+fPvK7774rlt/c/zKaIJjZ52ppg5dSSpkETATSTQ6EEPa3Zn4UilpMXh6cPq3tCwBo2xZat67UfQFGPYIizJ49W+kRULV6BD379+b3uFigZukRODo6MmLEiNurRyCE+F5KOUgIcQpNiMb0v1xKKdvccu0VoKJ6BHejf31F+bBYj+DmTUhMBCcnaNVK+Qe6wyiP3/+aSnn1CEocMJdSDjL8tUiWUqG4o8nJgStXNA+hBU7iKjgZrFDUNCzxNXQfcEBK+bcQ4nGgE/CRlFItwVHc+UipGYAzZ7QhIVtbbVPYHWIElB6BAixbPvpfwF8I4Y/mbO4z4Eug+DIBheJO4i5wEqf0CBRgmRvqXMOM81DgP1LKxWhLSBWKO5cCJ3HXrmnLQjt00IyBQnEHYkmPIF0I8SowBughhKgD3Bn9YoWiKFlZmjZAgZO4+vXvuF6AQlEUS3oEo9CE68dKKS8CrsB7pWdRKGoZeXnaruDDh8vlJE6huBOwxA31RWAN0FQIMQjIlFJ+UeUtUyhuFwkJ0L27NincpAk0a1bdLVIobitlGgIhxKPA78BI4FFgrxBiRFU3TKG4LSxZAp06wZ9/gqMj6HTVKhup9Ag0lB5BDdEjMGEGECilfFJK+QRwL/DGLdesUFQnBT+eDh1g5Eg4cgQaNap21TClR2AZd6sewa5du267HkEBdaSUf5mcp2Kh6L1CUeO4cQNmztR2A7/zDvTqpR0AJm/HP/74IxcvXqzUqlu2bFnM77w5lB5B6VS1HkHTxrYsfv9j7NwclR6BCT8JIbYIIcKEEGHAZuCHMvIoFDWP6Gjw84P334eMjH96BTUMpUdQOlWtR/DYo//HvPfeAmqeHkG3bt2qR49ASvmKEOIR4H5D0FIp5be3XLNCcbu4dg2mToWlS8HLC7Ztg969S81iyZt7VaH0CKpXjyA7M5sWzbW37ISEBF5//XWuXr1KRkYGISEhxero3bt3pW/K27JlCwEBAWzbto2TJ0/Sr18/evTowV9//XXb9Qi8gQWAF3AIP0aZ9wAAHANJREFUeFlKWdx0KRQ1nQsXYPVqePllmDOnVmwMU3oE1atHUEBYWBgbNmzA39+fVatWER0dXayeqKgoXnjhhWLhNjY27Nq1q1CYi4tLIU+tZ8+eNTsHtHLlSqZPn44QAp1Oh6enJ0ePHiUmJqZK9AhKGxpaAXwPDAfigI9LSWsWIUR/IcQxIcQJIcT0UtINF0JIIYRZz3gKRblJSYGCVS3t20NSErz3Xq0wAgBjx45l1qxZ+Pr6Fgrv0aOHcWgnOjoaR0dHbG1t6dmzp/HN/McffzQ+RPv27cv69ev56y9tmi8tLY3Tp0+XWf/69esZM2YMp0+fJikpieTkZDw9PY1zFQW0b98eHx8fNm3aZLacGTNmsGDBgnLX4e3tzfnz50lMTATg9OnTHDx4kICAAGxsbBg3bhxTpkwxTlanpKSwbt26QuUXPPQdHR3JyMgwrjgqSrt27UhJSTEagpycHBKPaSuY0tPTcXZ2Jicnp8QhtYIeQdGjqBEAcHZ2xtbWlj179iCl5IsvvmDo0KHF0rm7u7N161YALl26xLFjx2jTpg3u7u7ExMSQm5tLTk4OMTExlTI0VJohaCKlXCalPCalXIAmUmMxQggrYDGarKUPECqE8DGTrgkwBdhbnvIVCrNICf/7n7Ya6KWX4I8/tHAnp+ptVzlRegRKj6Cm6BEcBUL5R4dgDfB/BedSyv2lFixEN2C2lDLEcP6qId+/i6T7CPgFeAVt+KlUsQGlR6AokeRkmDABNm+GoCBYvhz0eouzW6xHoLijUXoEhbkAfGByftHkXAJ9ymiLC5Bscn4WCDJNIIToBLhJKTcLIV4pqSAhxDPAM4DxTUehKERuLgQHw8WL8OGHMGmSEoxRKCykNGGa0pdV3CIG53UfAGFlpZVSLgWWgtYjqMp2KWoZSUng5gbW1vDpp9CmjXYoLELpESjAsg1lFeUc4GZy7moIK6AJ0BGINiwJawl8J4QYUtbwkEJBbi589BG88Qa8+67WA3jggepuVa1D6REooGoNQSzgLYTwRDMAo9HmGACQUl4DjINwQohoLJgjUCiIj4f/397dx9lc5n8cf30yk5HCr5purJI2qmGwEU0pSmtRS6hoUUKYUDaVNkV0q7Z2CVvuKRuxJXS3+0PpV0mzucnNFqXQhjHuRsQMn98f1/dMx8wxc8bMuf88H4/zmO855zrne10znOt87z7vXr0gKwvat4dOnSLdI2NiWshKRahqPjAAeB9YD7yuqmtFZKSItAvVek2cGz8eGjWC77+H2bPhzTddgLwx5oQFk1ksQFfgQlUdKSLnA+eo6vKSXquq71CoHIWqDjtO2xZB9dgkJlVXEK5ePejSxR0QPjN2z+owJpoEs0UwHsjAnUoKkIu7PsCY0PvpJ/jjH12JCIBrroFXXrFJwJhyFMxE0FRV+wM/A6jqbiByBdtN4li0CNLT3UHhQ4eitkhcebI8AsfyCKIvjyDPu0pYAUQkFTha5jUbczx79kDv3u4soKQkWLoUxoyJeFZAOFgeQXAsjyD8eQRjgDeBs0TkSeBm4JEyr9mY49m+HWbNgiFDYPhwqFQp7F34+uvHyd2/vlzf87RTL6VOnZIznSyPoHiWRxCBPAJVnQk8CDyNu9r4JlWdU/yrjCml7dth9Gi3fPHF7kKxZ56JyCQQaZZHUDzLI4hAHoF3ltABYIH/Y6q6ucxrN0YVZs6Ee+91YTFt20Lt2hE/GBzMN/dQsTwCyyOImjwCP2/jjg8IkALUAr4Cgq/mZUwgmzdDv37w7ruQkeGKxNWuHeleRQXLI7A8gmjJIwBAVdNVtb73szYuvP7TMq3VGF+RON+B4I8+cqWjDWB5BJZHED15BAF55aebltjQmEC+/RaOHHFnA02cCGvWWKXQACyPwPIIoiKPoKCByH1+d08CLgPO8OUMhJvlEcSo/HwXGj98uCsSF+BDLtIsj8CA5REcz2l+y/m4Ywb/OOEemsSzcqUrEvfFF9ChA9xyS6R7ZIzxU+xE4F1Idpqq3h+m/ph4M3asKxFxxhkwd65VCo0ylkdgoJiJQESSVDVfRK4KZ4dMnPAViatfH7p2hRdegNNPj3SvTCGWR2Cg+C2C5bjjAStFZD4wB/jJ96SqvhHivplYtH8/DB0Kycnw5z+7InHXXBPpXhljihHMWUMpQA4uo/hG4PfeT2OO9c9/ujLRL74IeXkJUSTOmHhQ3BbBWd4ZQ2v45YIyH/sfbn6xezfcdx9Mm+bKQyxdCs2aRbpXxpggFTcRVABO5dgJwMcmAvOLHTvcgeA//QmGDYOUlEj3yBhTCsXtGvpRVUeq6ogAt5Fh66GJTtu2uZQw+KVI3FNP2SRQRpZH4FgeQfTkEcR/8XdTeqowfTqkpbktgA0b3ON2qmG5sDyC4FgeQfjyCIqeXGwS23ffQd++7qDwVVfBpElxWyTu0Q1bWbP/YLm+Z71TK/F47RoltrM8guJZHkEY8whUdVeZ393Ej/x8uPZa+OQTGDfOHRAup9or5liWR1A8yyOIQB6BSXAbN0KtWq5I3JQpcOGFULNmpHsVcsF8cw8VyyOwPIJozCMwiSgvD557DkaMcD/vucdtEZiwsDwCyyOIqjwCk4C++AKaNHFXCLdvD943JxM+lkdgeQRRnUdg4tyYMW4S2LYN3ngDXn8dyuFglCkdyyOwPIKoyiOINpZHECK+InEffQRTp7rsAG8/c6KwPAIDlkdgElFurrseoGJF9+F/9dXuZoxJGDYRJLL33nPXBWzZAoMG/bJVYBKG5REYsIkgMeXkuCJxM2a4wPiPP4aMjEj3ykSA5REYsIPFiSknB958Ex59FFassEnAmAQX0olARFqLyFcislFEHgrw/H0isk5EVovIIhGJ/yuVIuXHH11QjCrUqQPffw8jR7pjA8aYhBayicDLOx4HtAHSgNtEJK1QsxVAY1WtD8wFng1VfxKWqrsi+NJL3RbAxo3u8QQ7I8gYc3yh3CJoAmxU1W9V9TAwCzjmyglVXaKqB7y7y4DIXdcfjzZtglatoFcvaNAAVq2K2yJxxpgTF8qJ4FfAFr/7W73HjqcXELCguIj0EZEsEcnKzs4uxy7Gsfx8uO46+Owz+NvfYMkSt0vIRDXLI3AsjyB68gjCRkS6AY2B5wI9r6oTVLWxqjZOTU0Nb+dizYYNcOSIKxI3dSqsXQv9+sFJUfGnNiWwPILgWB5B+PIIyuoH4Dy/+zW8x44hItcDQ4HmqnoohP2Jb3l5MGoUPP44PPss3HsvtGgR6V7FrBEL1rLuv/vK9T3Tqldh+O/rltjO8giKZ3kEYcwjKAefA7VFpJaInAx0Aeb7NxCR3wAvA+1UdUcI+xLfsrKgcWN3MLhjR7jttkj3yJSB5REUz/IIYiiPQFXzRWQA8D5QAZiiqmtFZCSQparzcbuCTgXmeHXGN6tqu1D1KS6NHu0uDjvnHHjrLWhnv77yEMw391CxPALLI4irPAJVfQd4p9Bjw/yWrw/l+uOarxxE48burKBnn4Vq1SLdK1NOLI/A8ggsj8Ac3759kJnptgLAZQdPmGCTQJyxPALLI7A8AhPYO+9A3brugz8pyW0VmLhkeQSWR2B5BMVIyDyCnTtdddCZM91EMHky+B1ANOXD8ggMJGYegW0RxILdu2HBAhg+3MVI2iRgjClHVoY6Wv3wg9sCeOABVxbi++/tOIApd5ZHYMAmguijCpMmwf33u4vEOnaEiy6yScCEhOURGLBdQ9Hlm2+gZUvo0wcuuwxWr3aTgDHGhJBtEUSL/Hw3CezaBS+/DL17W30gY0xY2EQQaV99Bb/+tTsddPp0t1zDqnEbY8LHvnJGyuHDMGIEpKfDuHHusebNbRIwxoSdTQSRsHw5NGoEjz0Gt9wCXbtGukcmSlgegWN5BOHNI7BdQ+H217/C4MFw7rnu2oBS/gc3YfLuQ7Dty/J9z3PSoc0zxTbxzyOoVKlSVOUR+JeI9uUR9O3bl5NPPrnI63x5BJmZmSe8juL45xFUrFiR7du3n3Bd/vz8fJKSwvdR6MsjaNq0KW3btuW9994rUobal0ewYMECsrOzufjii+natStZWVkFeQQAzZo148MPP6RFGUvO2xZBuPhm7SZN4K67XGCMTQImAF8eAVCQR+Cza9cubrrpJurXr88VV1xR8IGQk5NDq1atqFu3Lr179y6SR9CkSRMaNmxI3759A5aNDsSXFTB58mRmzZp1zHOpqam0bNmyoMRFYb48gpJCX4pbx/H48ghefPHFoPIILr/8curVq0efPn0Kfi8tWrRg0KBBNG7cmNGjR/Pvf/+b5s2b06hRIzp1u4Vt27cBMHHiRC6//HIaNGhAp06dOHDgQJF1lIZ/HoGIFOQRFBZMHsGhQ4fKLY/AtghCbe9eePBBqFTJbQ1ceaW7mehWwjf3UOrSpQsjR47kxhtvZPXq1fTs2bOg2Jsvj2DevHksXryY22+/nZUrVxbkEQwbNoy33367oG6+fx5BcnIyd999NzNnzizIMShOoKwA/zLUQ4YMoU2bNvTs2bPIa/3zCIqrhVPSOgIpbR7BsGGu4HH37t1ZuHBhQX98eQR5eXk0b96ct956i9TUVCaPm8gTzz3FzNl/p2PHjgXhOo888giTJ09m4MCBx6yjNNVHS5NH0K5dO6pXr05ubi6zZ88ukkegqgwYMCC68wgMbtdPv36wbZu7QMxXOtqYYlgegeURxFUeQcLKznZRka+95s4KmjcPjlP50JhALI/A8ggsjyDW7d3rSkaPGOFiJG0SMKVkeQSWR2B5BLFoyxZ4+mm3++eii1yRuGHDIMAZFcaUxPIILI/A8giKEXV5BEePuqCYBx+EI0dg1SqrDxSjLI/AgOURmNLasAGuu85FRzZpAl9+aZOAMSbm2MHiE5WfD7/9LezZ4xLD7rzTzggyMcfyCAzYRFB669e7oJikJHjlFVckrnr1SPfKmBNieQQGbNdQ8A4dclGR9evD2LHusauvtknAGBPzbIsgGMuWQa9esG4ddO/ubsYYEydsi6Akzz/vSkLk5rprA2bMANt3aoyJIzYRHM/Ro+5nRoYrE7FmDRSqEGiMMfHAJoLC9uxxu4G8+idceSWMHw9BFLgypqwsj8CxPALLI4icefPg7rthxw53gZgViUtYo5aP4j+7/lNyw1K45PRLGNJkSLFtLI/A8gjA8ggiY8cOuPVW6NABzj7bJYg99ZRNAiYiLI+geJZHUP55BKhqTN0aNWqkJ6LjS/W140v1Az+5YYNqtWqqTz6pevjwCb2/iX3r1q2LdBe0cuXKumrVKu3UqZMePHhQGzRooEuWLNEbbrhBVVUHDBigjz32mKqqLlq0SBs0aKCqqgMHDtQRI0aoqurChQsV0OzsbF23bp3eeOONetj7d52ZmanTp09XVdWaNWtqdnb2cfvy6quvas+ePVVVNSMjQ7OyslRVddOmTVq3bl395ptvtE6dOpqfn6/9+/fXqVOnqqrqHXfcoXPmzNE777xTp0yZotnZ2VqzZs1SrcN/zD6+9121apU2bNgwqN9nTk5OwXK3bt10/vz5qqravHlzzczMVFXVw4cPa0ZGhu7YsUNVVSeNnaB/uPU2VVXduXNnweuHDh2qY8aMKbKOxYsXa4MGDYrcMjIyirT9/PPPtWXLlgX3ly5dWmScqqr79u3TFi1a6DnnnKOVK1fWhQsXFjw3ePBgrVq1qlapUkUffvjhgOMO9G8ZyNLjfK4m7q6hzZvdBWEPP+zKQmzeDKedFuleGWN5BJZHEF95BCLSGhgNVAAmqeozhZ6vCMwAGgE5QGdV/S6UfeLoUXjpJRgyxC137uwmApsETBSxPALLI4iLPAIRqQCMA9oAacBtIpJWqFkvYLeqXgT8BRgVqv4AVN/2M7RoAf37u9NC1661InEmKlkegeURhDOPIJRbBE2Ajar6LYCIzALaA+v82rQHHvOW5wJjRUS8/Vnl6qQjyqNjNnH4p6182boXm+s2g3EfAid2poGJP+d3SDvm22BEqLJ7y04qk0L3Dn9g95ad5O7YS97Bw+zespNBdw1k4P33UvfSulSqVIkXR/2V3Vt2cm/vAfQe0JdXZ7xKk0aXU+NXNdjzQw7nnn4WDw16kJbXXsfRo0pyUhLPPjGKKidV5uiRo+z5IYcKB4t245VpM7gnc+Axv4821/2OaROncm/mQI7kHSl4bmCvu5k7dy4/7cpl95adHP7pZ/bv3MfuLTupXuVs0i+tx6o1q4v8botbR3qtNMY/P5buXbtz6NDPJCcl85ennufovjx279vJ4L6DePK5p7mkzsVUrJjCKaecwp8GDymyju6du5J2SRpnpaZSPy2dg3sPsHvLTvIP5bFv256C9lPGTmTwoPvYl5tLfv4R+vXqw5XXNyvII0hNTaVp06bk5uaW+U88fvx4evTowcGDB2nTps0xeQQA/fr149FHH6VHjx6kp6ejqsfkESxevJj09HREhNatW0d3HoGI3Ay0VtXe3v3uQFNVHeDXZo3XZqt3/xuvzc5C79UH6ANw/vnnNwrmG01hPcc1o/Y3+2hwsBeHTq12osMycez8DmlceF6tSHfDRIGUU06i0hmnR7obJ6y0eQQxcbBYVScAE8AF05zIe0zp/3/l2icTf9avXx/TYSTGnKhQTgQ/AOf53a/hPRaozVYRSQKq4g4aG2PCwPIIDIR2IvgcqC0itXAf+F2APxRqMx+4A/gUuBlYHIrjA8YES1VLfYpiLLM8gvhzIh+hITtrSFXzgQHA+8B64HVVXSsiI0WknddsMnCGiGwE7gMeClV/jClJSkoKOTk55VK7xZhIUFVycnJISUkp1esSJrzemJLk5eWxdevWY841NybWpKSkUKNGDZKTk495POYPFhsTDsnJydSqZWcNmcRjReeMMSbB2URgjDEJziYCY4xJcDF3sFhEsoHSX1rsnAlEuIZA2NmYE4ONOTGUZcw1VTU10BMxNxGUhYhkHe+oebyyMScGG3NiCNWYbdeQMcYkOJsIjDEmwSXaRDAh0h2IABtzYrAxJ4aQjDmhjhEYY4wpKtG2CIwxxhRiE4ExxiS4uJwIRKS1iHwlIhtFpEhFUxGpKCKzvec/E5ELwt/L8hXEmO8TkXUislpEFolIzUj0szyVNGa/dp1EREUk5k81DGbMInKr97deKyJ/D3cfy1sQ/7bPF5ElIrLC+/fdNhL9LC8iMkVEdngJjoGeFxEZ4/0+VovIZWVeqarG1Q2oAHwDXAicDKwC0gq1uRt4yVvuAsyOdL/DMOZrgVO85cxEGLPX7jRgKbAMaBzpfofh71wbWAH8j3f/rEj3OwxjngBkestpwHeR7ncZx3wNcBmw5jjPtwXeBQS4AvisrOuMxy2CJsBGVf1WVQ8Ds4D2hdq0B6Z7y3OBlhLbaSQljllVl6jqAe/uMlxiXCwL5u8M8DgwCoiH2tLBjPkuYJyq7gZQ1R1h7mN5C2bMClTxlqsC/w1j/8qdqi4FdhXTpD0wQ51lQDURObcs64zHieBXwBa/+1u9xwK2URegsxeI5Vy+YMbsrxfuG0UsK3HM3ibzear6djg7FkLB/J3rAHVE5GMRWSYircPWu9AIZsyPAd1EZCvwDjAwPF2LmNL+fy+R5REkGBHpBjQGmke6L6EkIicBLwA9ItyVcEvC7R5qgdvqWyoi6aq6J6K9Cq3bgGmq+ryIZACviEg9VT0a6Y7FinjcIvgBOM/vfg3vsYBtRCQJtzmZE5behUYwY0ZErgeGAu1U9VCY+hYqJY35NKAe8IGIfIfblzo/xg8YB/N33grMV9U8Vd0EfI2bGGJVMGPuBbwOoKqfAim44mzxKqj/76URjxPB50BtEaklIifjDgbPL9RmPnCHt3wzsFi9ozAxqsQxi8hvgJdxk0Cs7zeGEsasqntV9UxVvUBVL8AdF2mnqrGccxrMv+15uK0BRORM3K6ib8PZyXIWzJg3Ay0BRORS3ESQHdZehtd84Hbv7KErgL2q+mNZ3jDudg2par6IDADex51xMEVV14rISCBLVecDk3GbjxtxB2W6RK7HZRfkmJ8DTgXmeMfFN6tqu4h1uoyCHHNcCXLM7wOtRGQdcAR4QFVjdms3yDEPBiaKyB9xB457xPIXOxF5DTeZn+kd9xgOJAOo6ku44yBtgY3AAeDOMq8zhn9fxhhjykE87hoyxhhTCjYRGGNMgrOJwBhjEpxNBMYYk+BsIjDGmARnE4GJSiJyRERW+t0uKKbt/nJY3zQR2eSt6wvvCtXSvsckEUnzlh8u9NwnZe2j9z6+38saEVkgItVKaN8w1qtxmtCz00dNVBKR/ap6anm3LeY9pgELVXWuiLQC/qyq9cvwfmXuU0nvKyLTga9V9cli2vfAVV0dUN59MfHDtghMTBCRU70chS9E5EsRKVJpVETOFZGlft+Yr/YebyUin3qvnSMiJX1ALwUu8l57n/dea0RkkPdYZRF5W0RWeY939h7/QEQai8gzQCWvHzO95/Z7P2eJyA1+fZ4mIjeLSAUReU5EPvdqzPcN4tfyKV6xMRFp4o1xhYh8IiIXe1fijgQ6e33p7PV9iogs99oGqthqEk2ka2/bzW6BbrirYld6tzdxV8FX8Z47E3dVpW+Ldr/3czAw1FuugKs3dCbug72y9/gQYFiA9U0DbvaWbwE+AxoBXwKVcVdlrwV+A3QCJvq9tqr38wO8zANfn/za+PrYAZjuLZ+MqyJZCegDPOI9XhHIAmoF6Od+v/HNAVp796sASd7y9cA/vOUewFi/1z8FdPOWq+FqEVWO9N/bbpG9xV2JCRM3DqpqQ98dEUkGnhKRa4CjuG/CZwPb/F7zOTDFaztPVVeKSHNcWMnHXmmNk3HfpAN5TkQewdWp6YWrX/Omqv7k9eEN4GrgPeB5ERmF2530USnG9S4wWkQqAq2Bpap60NsdVV9EbvbaVcUVi9tU6PWVRGSlN/71wL/82k8Xkdq4MgvJx1l/K6CdiNzv3U8BzvfeyyQomwhMrOgKpAKNVDVPXEXRFP8GqrrUmyhuAKaJyAvAbuBfqnpbEOt4QFXn+u6ISMtAjVT1a3FZB22BJ0RkkaqODGYQqvqziHwA/A7ojAtaAZc2NVBV3y/hLQ6qakMROQVXf6c/MAYXwLNEVTt4B9Y/OM7rBeikql8F01+TGOwYgYkVVYEd3iRwLVAkc1lcDvN2VZ0ITMLF/S0DrhIR3z7/yiJSJ8h1fgTcJCKniEhl3G6dj0SkOnBAVV/FFfMLlBmb522ZBDIbVyjMt3UB7kM90/caEanjrTMgdWlz9wCD5ZdS6r5SxD38mubidpH5vA8MFG/zSFxVWpPgbCIwsWIm0FhEvgRuB/4ToE0LYJWIrMB92x6tqtm4D8bXRGQ1brfQJcGsUFW/wB07WI47ZjBJVVcA6cBybxfNcOCJAC+fAKz2HSwu5J+4YKD/VRe/CG7iWgd8IS60/GVK2GL3+rIaF8zyLPC0N3b/1y0B0nwHi3FbDsle39Z6902Cs9NHjTEmwdkWgTHGJDibCIwxJsHZRGCMMQnOJgJjjElwNhEYY0yCs4nAGGMSnE0ExhiT4P4f/NfGfgMFiNAAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def crossvalidation(ml_model, df, n_folds=5, verbose=False):\n",
        "    \"\"\"\n",
        "    Machine learning model training and validation in a cross-validation loop.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    ml_model: sklearn model object\n",
        "        The machine learning model to train.\n",
        "    df: pd.DataFrame\n",
        "        Data set with SMILES and their associated activity labels.\n",
        "    n_folds: int, optional\n",
        "        Number of folds for cross-validation.\n",
        "    verbose: bool, optional\n",
        "        Performance measures are printed.\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    None\n",
        "\n",
        "    \"\"\"\n",
        "    t0 = time.time()\n",
        "    # Shuffle the indices for the k-fold cross-validation\n",
        "    kf = KFold(n_splits=n_folds, shuffle=True, random_state=SEED)\n",
        "\n",
        "    # Results for each of the cross-validation folds\n",
        "    acc_per_fold = []\n",
        "    sens_per_fold = []\n",
        "    spec_per_fold = []\n",
        "    auc_per_fold = []\n",
        "\n",
        "    # Loop over the folds\n",
        "    for train_index, test_index in kf.split(df):\n",
        "        # clone model -- we want a fresh copy per fold!\n",
        "        fold_model = clone(ml_model)\n",
        "        # Training\n",
        "\n",
        "        # Convert the fingerprint and the label to a list\n",
        "        train_x = df.iloc[train_index].fp.tolist()\n",
        "        train_y = df.iloc[train_index].active.tolist()\n",
        "\n",
        "        # Fit the model\n",
        "        fold_model.fit(train_x, train_y)\n",
        "\n",
        "        # Testing\n",
        "\n",
        "        # Convert the fingerprint and the label to a list\n",
        "        test_x = df.iloc[test_index].fp.tolist()\n",
        "        test_y = df.iloc[test_index].active.tolist()\n",
        "\n",
        "        # Performance for each fold\n",
        "        accuracy, sens, spec, auc = model_performance(fold_model, test_x, test_y, verbose)\n",
        "\n",
        "        # Save results\n",
        "        acc_per_fold.append(accuracy)\n",
        "        sens_per_fold.append(sens)\n",
        "        spec_per_fold.append(spec)\n",
        "        auc_per_fold.append(auc)\n",
        "\n",
        "    # Print statistics of results\n",
        "    print(\n",
        "        f\"Mean accuracy: {np.mean(acc_per_fold):.2f} \\t\"\n",
        "        f\"and std : {np.std(acc_per_fold):.2f} \\n\"\n",
        "        f\"Mean sensitivity: {np.mean(sens_per_fold):.2f} \\t\"\n",
        "        f\"and std : {np.std(sens_per_fold):.2f} \\n\"\n",
        "        f\"Mean specificity: {np.mean(spec_per_fold):.2f} \\t\"\n",
        "        f\"and std : {np.std(spec_per_fold):.2f} \\n\"\n",
        "        f\"Mean AUC: {np.mean(auc_per_fold):.2f} \\t\"\n",
        "        f\"and std : {np.std(auc_per_fold):.2f} \\n\"\n",
        "        f\"Time taken : {time.time() - t0:.2f}s\\n\"\n",
        "    )\n",
        "\n",
        "    return acc_per_fold, sens_per_fold, spec_per_fold, auc_per_fold"
      ],
      "metadata": {
        "id": "1zuuMlr8VXR5"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "N_FOLDS = 3"
      ],
      "metadata": {
        "id": "Xs_3cFU8VeIv"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for model in models:\n",
        "    print(\"\\n======= \")\n",
        "    print(f\"{model['label']}\")\n",
        "    crossvalidation(model[\"model\"], chembl_df, n_folds=N_FOLDS)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sLOCCUQEURDT",
        "outputId": "a7d40dd8-b000-40bb-d4e6-9cbd47dbf9e6"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======= \n",
            "Model_RF\n",
            "Mean accuracy: 0.84 \tand std : 0.03 \n",
            "Mean sensitivity: 0.50 \tand std : 0.41 \n",
            "Mean specificity: 0.89 \tand std : 0.08 \n",
            "Mean AUC: 0.91 \tand std : 0.08 \n",
            "Time taken : 0.54s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_SVM\n",
            "Mean accuracy: 0.86 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 0.96 \tand std : 0.06 \n",
            "Mean AUC: 0.54 \tand std : 0.36 \n",
            "Time taken : 0.03s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.46s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.45s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_SVM\n",
            "Mean accuracy: 0.86 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 0.96 \tand std : 0.06 \n",
            "Mean AUC: 0.85 \tand std : 0.08 \n",
            "Time taken : 0.02s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.41s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.41s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.41s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.45s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.47s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.43s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.43s\n",
            "\n",
            "\n",
            "======= \n",
            "Model_ANN\n",
            "Mean accuracy: 0.90 \tand std : 0.03 \n",
            "Mean sensitivity: 0.00 \tand std : 0.00 \n",
            "Mean specificity: 1.00 \tand std : 0.00 \n",
            "Mean AUC: 0.90 \tand std : 0.08 \n",
            "Time taken : 0.43s\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Reset data frame\n",
        "compound_df = chembl_df.copy()"
      ],
      "metadata": {
        "id": "Pl17-_yWVpRU"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Use Morgan fingerprint with radius 3\n",
        "compound_df[\"fp\"] = compound_df[\"smiles\"].apply(smiles_to_fp, args=(\"morgan3\",))\n",
        "compound_df.head(3)\n",
        "# NBVAL_CHECK_OUTPUT"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "id": "UmUOy83KVr6m",
        "outputId": "1a229d7c-601b-4531-d576-f7602d6dfbe3"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  molecule_chembl_id                                             smiles  \\\n",
              "0       CHEMBL336546  O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...   \n",
              "1       CHEMBL131636  CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...   \n",
              "2       CHEMBL134490  CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...   \n",
              "\n",
              "      pIC50  active                                                 fp  \n",
              "0  7.301030     1.0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
              "1  6.769551     1.0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, ...  \n",
              "2  6.552842     1.0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9928a47f-4ed6-41e0-b9e4-9dc1d221d373\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>molecule_chembl_id</th>\n",
              "      <th>smiles</th>\n",
              "      <th>pIC50</th>\n",
              "      <th>active</th>\n",
              "      <th>fp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CHEMBL336546</td>\n",
              "      <td>O=C(Nc1cc(C(F)(F)F)cc(C(F)(F)F)c1)c1cnc(Cl)nc1...</td>\n",
              "      <td>7.301030</td>\n",
              "      <td>1.0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CHEMBL131636</td>\n",
              "      <td>CCCCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C...</td>\n",
              "      <td>6.769551</td>\n",
              "      <td>1.0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CHEMBL134490</td>\n",
              "      <td>CCOC(=O)c1cc(NC(=O)c2cnc(Cl)nc2C(F)(F)F)cc(C(F...</td>\n",
              "      <td>6.552842</td>\n",
              "      <td>1.0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9928a47f-4ed6-41e0-b9e4-9dc1d221d373')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9928a47f-4ed6-41e0-b9e4-9dc1d221d373 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9928a47f-4ed6-41e0-b9e4-9dc1d221d373');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for model in models:\n",
        "    if model[\"label\"] == \"Model_SVM\":\n",
        "        # SVM is super slow with long fingerprints\n",
        "        # and will have a performance similar to RF\n",
        "        # We can skip it in this test, but if you want\n",
        "        # to run it, feel free to replace `continue` with `pass`\n",
        "        continue\n",
        "    print(\"\\n=======\")\n",
        "    print(model[\"label\"])\n",
        "    crossvalidation(model[\"model\"], compound_df, n_folds=N_FOLDS)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5MqfKsELVu0D",
        "outputId": "4d03ecd7-1df3-4fd3-81eb-d8961fe5fe84"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "=======\n",
            "Model_RF\n",
            "Mean accuracy: 0.86 \tand std : 0.03 \n",
            "Mean sensitivity: 0.33 \tand std : 0.47 \n",
            "Mean specificity: 0.93 \tand std : 0.09 \n",
            "Mean AUC: 0.96 \tand std : 0.03 \n",
            "Time taken : 0.52s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.37s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.09s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.15s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.29s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.09s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.06s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.15s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.30s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.03s\n",
            "\n",
            "\n",
            "=======\n",
            "Model_ANN\n",
            "Mean accuracy: 0.80 \tand std : 0.13 \n",
            "Mean sensitivity: 1.00 \tand std : 0.00 \n",
            "Mean specificity: 0.78 \tand std : 0.14 \n",
            "Mean AUC: 0.94 \tand std : 0.04 \n",
            "Time taken : 1.25s\n",
            "\n"
          ]
        }
      ]
    }
  ]
}