ore-psych-ml-ibs-loocv.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e84c8a0b",
   "metadata": {},
   "source": [
    "`ore-psych-ml-ibs-loocv.ipynb` notebook (ver 2022-12-16) for the Leave-One-Out Cross-Validtaion (LOOCV) procedure accompanying the manuscript \n",
    "\n",
    "### A psychological symptom based machine learning model for clinical evaluation of irritable bowel syndrome\n",
    " \n",
    "by Noman Haleem, ..., Arvid Lundervold to be submitted to Open Research Europe.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04fde26a",
   "metadata": {},
   "source": [
    "Runs under the following conda environment, denoted `oreibs0`:\n",
    "\n",
    "```\n",
    "conda create --name oreibs0 -c conda-forge scikit-learn python=3.8\n",
    "conda activate oreibs0\n",
    "conda install ipython\n",
    "conda install jupyter\n",
    "conda install pandas\n",
    "conda install matplotlib\n",
    "conda install statsmodels\n",
    "conda install -c conda-forge  pingouin\n",
    "conda install -c plotly plotly=5.10.0\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "632a63a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cedaa016",
   "metadata": {},
   "source": [
    "### Import libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "27f119c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "import matplotlib.pyplot as plt\n",
    "#import seaborn as sns\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.model_selection import LeaveOneOut\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.inspection import permutation_importance\n",
    "from sklearn.model_selection import StratifiedShuffleSplit"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ee31d9f",
   "metadata": {},
   "source": [
    "### Read the `testing`, `training` and `validation` .csv files generatet previously"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e26e2f2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv('data/testing.csv')\n",
    "df_train = pd.read_csv('data/training.csv')\n",
    "df_validation = pd.read_csv('data/validation.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e662bda2",
   "metadata": {},
   "source": [
    "### Contatenate these dataframes to prepare the full dataset  for LOOCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5bc1f15c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df_test, df_train, df_validation])  #.drop_duplicates(keep=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94e8749a",
   "metadata": {},
   "source": [
    "#### Inspect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "62db9a0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(84, 39)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "77a1c191",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['pt_anon_ids', 'Category', 'Sub_category', 'Gender', 'Age', 'BIS_Q1_BL',\n",
      "       'BIS_Q2_BL', 'BIS_Q3_BL', 'BIS_Q4_BL', 'BIS_Q5_BL', 'BIS_Q6_BL',\n",
      "       'FSS_Q1_BL', 'FSS_Q10_BL', 'FSS_Q11_BL', 'FSS_Q12_BL', 'FSS_Q13_BL',\n",
      "       'FSS_Q2_BL', 'FSS_Q3_BL', 'FSS_Q4_BL', 'FSS_Q5_BL', 'FSS_Q6_BL',\n",
      "       'FSS_Q7_BL', 'FSS_Q8_BL', 'FSS_Q9_BL', 'HADS_q1_BL', 'HADS_q2_BL',\n",
      "       'HADS_q3_BL', 'HADS_q4_BL', 'HADS_q5_BL', 'HADS_q6_BL', 'HADS_q7_BL',\n",
      "       'HADS_q8_BL', 'HADS_q9_BL', 'HADS_q10_BL', 'HADS_q11_BL', 'HADS_q12_BL',\n",
      "       'HADS_q13_BL', 'HADS_q14_BL', 'stratification'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1bcf1ed6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pt_anon_ids</th>\n",
       "      <th>Category</th>\n",
       "      <th>Sub_category</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>BIS_Q1_BL</th>\n",
       "      <th>BIS_Q2_BL</th>\n",
       "      <th>BIS_Q3_BL</th>\n",
       "      <th>BIS_Q4_BL</th>\n",
       "      <th>BIS_Q5_BL</th>\n",
       "      <th>...</th>\n",
       "      <th>HADS_q6_BL</th>\n",
       "      <th>HADS_q7_BL</th>\n",
       "      <th>HADS_q8_BL</th>\n",
       "      <th>HADS_q9_BL</th>\n",
       "      <th>HADS_q10_BL</th>\n",
       "      <th>HADS_q11_BL</th>\n",
       "      <th>HADS_q12_BL</th>\n",
       "      <th>HADS_q13_BL</th>\n",
       "      <th>HADS_q14_BL</th>\n",
       "      <th>stratification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BGA_082</td>\n",
       "      <td>HC</td>\n",
       "      <td>HC</td>\n",
       "      <td>F</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HC_F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BGA_136</td>\n",
       "      <td>IBS</td>\n",
       "      <td>Diarrhea</td>\n",
       "      <td>F</td>\n",
       "      <td>30.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Diarrhea_F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BGA_095</td>\n",
       "      <td>IBS</td>\n",
       "      <td>Mixed</td>\n",
       "      <td>M</td>\n",
       "      <td>28.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Mixed_M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BGA_153</td>\n",
       "      <td>IBS</td>\n",
       "      <td>Mixed</td>\n",
       "      <td>F</td>\n",
       "      <td>32.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Mixed_F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BGA_154</td>\n",
       "      <td>IBS</td>\n",
       "      <td>Mixed</td>\n",
       "      <td>F</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Mixed_F</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  pt_anon_ids Category Sub_category Gender   Age  BIS_Q1_BL  BIS_Q2_BL  \\\n",
       "0     BGA_082       HC           HC      F  46.0        0.0        0.0   \n",
       "1     BGA_136      IBS     Diarrhea      F  30.0        7.0        3.0   \n",
       "2     BGA_095      IBS        Mixed      M  28.0        3.0        0.0   \n",
       "3     BGA_153      IBS        Mixed      F  32.0        5.0        2.0   \n",
       "4     BGA_154      IBS        Mixed      F  37.0        0.0        0.0   \n",
       "\n",
       "   BIS_Q3_BL  BIS_Q4_BL  BIS_Q5_BL  ...  HADS_q6_BL  HADS_q7_BL  HADS_q8_BL  \\\n",
       "0        0.0        1.0        0.0  ...         0.0         0.0         0.0   \n",
       "1        3.0        5.0        3.0  ...         1.0         2.0         1.0   \n",
       "2        0.0        6.0        3.0  ...         1.0         2.0         1.0   \n",
       "3        2.0        5.0        3.0  ...         0.0         2.0         1.0   \n",
       "4        0.0        7.0        5.0  ...         1.0         0.0         3.0   \n",
       "\n",
       "   HADS_q9_BL  HADS_q10_BL  HADS_q11_BL  HADS_q12_BL  HADS_q13_BL  \\\n",
       "0         0.0          0.0          0.0          0.0          0.0   \n",
       "1         2.0          1.0          3.0          2.0          3.0   \n",
       "2         2.0          0.0          1.0          2.0          2.0   \n",
       "3         1.0          0.0          1.0          0.0          0.0   \n",
       "4         2.0          0.0          0.0          0.0          0.0   \n",
       "\n",
       "   HADS_q14_BL  stratification  \n",
       "0          0.0            HC_F  \n",
       "1          0.0      Diarrhea_F  \n",
       "2          0.0         Mixed_M  \n",
       "3          0.0         Mixed_F  \n",
       "4          0.0         Mixed_F  \n",
       "\n",
       "[5 rows x 39 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "74c25c1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pt_anon_ids</th>\n",
       "      <th>Category</th>\n",
       "      <th>Sub_category</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>BIS_Q1_BL</th>\n",
       "      <th>BIS_Q2_BL</th>\n",
       "      <th>BIS_Q3_BL</th>\n",
       "      <th>BIS_Q4_BL</th>\n",
       "      <th>BIS_Q5_BL</th>\n",
       "      <th>...</th>\n",
       "      <th>HADS_q6_BL</th>\n",
       "      <th>HADS_q7_BL</th>\n",
       "      <th>HADS_q8_BL</th>\n",
       "      <th>HADS_q9_BL</th>\n",
       "      <th>HADS_q10_BL</th>\n",
       "      <th>HADS_q11_BL</th>\n",
       "      <th>HADS_q12_BL</th>\n",
       "      <th>HADS_q13_BL</th>\n",
       "      <th>HADS_q14_BL</th>\n",
       "      <th>stratification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>BGA_157</td>\n",
       "      <td>HC</td>\n",
       "      <td>HC</td>\n",
       "      <td>F</td>\n",
       "      <td>57.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HC_F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>BGA_119</td>\n",
       "      <td>HC</td>\n",
       "      <td>HC</td>\n",
       "      <td>F</td>\n",
       "      <td>36.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HC_F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>BGA_104</td>\n",
       "      <td>IBS</td>\n",
       "      <td>Diarrhea</td>\n",
       "      <td>M</td>\n",
       "      <td>30.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Diarrhea_M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>BGA_099</td>\n",
       "      <td>HC</td>\n",
       "      <td>HC</td>\n",
       "      <td>M</td>\n",
       "      <td>39.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>HC_M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>BGA_061</td>\n",
       "      <td>IBS</td>\n",
       "      <td>Constipation</td>\n",
       "      <td>F</td>\n",
       "      <td>42.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Constipation_F</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   pt_anon_ids Category  Sub_category Gender   Age  BIS_Q1_BL  BIS_Q2_BL  \\\n",
       "12     BGA_157       HC            HC      F  57.0        7.0        7.0   \n",
       "13     BGA_119       HC            HC      F  36.0        2.0        0.0   \n",
       "14     BGA_104      IBS      Diarrhea      M  30.0        6.0        0.0   \n",
       "15     BGA_099       HC            HC      M  39.0        1.0        0.0   \n",
       "16     BGA_061      IBS  Constipation      F  42.0        5.0        2.0   \n",
       "\n",
       "    BIS_Q3_BL  BIS_Q4_BL  BIS_Q5_BL  ...  HADS_q6_BL  HADS_q7_BL  HADS_q8_BL  \\\n",
       "12        2.0        4.0        0.0  ...         0.0         0.0         0.0   \n",
       "13        1.0        4.0        0.0  ...         0.0         1.0         0.0   \n",
       "14        0.0        5.0        1.0  ...         2.0         2.0         1.0   \n",
       "15        0.0        1.0        0.0  ...         0.0         1.0         1.0   \n",
       "16        4.0        4.0        3.0  ...         1.0         2.0         2.0   \n",
       "\n",
       "    HADS_q9_BL  HADS_q10_BL  HADS_q11_BL  HADS_q12_BL  HADS_q13_BL  \\\n",
       "12         0.0          0.0          0.0          0.0          0.0   \n",
       "13         1.0          0.0          1.0          0.0          0.0   \n",
       "14         2.0          1.0          1.0          0.0          1.0   \n",
       "15         0.0          1.0          2.0          1.0          0.0   \n",
       "16         2.0          0.0          1.0          2.0          2.0   \n",
       "\n",
       "    HADS_q14_BL  stratification  \n",
       "12          0.0            HC_F  \n",
       "13          0.0            HC_F  \n",
       "14          0.0      Diarrhea_M  \n",
       "15          0.0            HC_M  \n",
       "16          1.0  Constipation_F  \n",
       "\n",
       "[5 rows x 39 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b291f8e",
   "metadata": {},
   "source": [
    "### Make the X and the y in the framework $$y \\sim f(X, \\theta)$$\n",
    "\n",
    "where $f$ is the model (LR, RF, SVM, etc.) and $\\theta$ is parameters of the model being learned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d15e84bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>HADS_q1_BL</th>\n",
       "      <th>HADS_q3_BL</th>\n",
       "      <th>HADS_q5_BL</th>\n",
       "      <th>FSS_Q1_BL</th>\n",
       "      <th>FSS_Q5_BL</th>\n",
       "      <th>FSS_Q6_BL</th>\n",
       "      <th>FSS_Q8_BL</th>\n",
       "      <th>FSS_Q12_BL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>F</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>F</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>F</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>F</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>F</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>M</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>M</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>F</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>84 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Gender  HADS_q1_BL  HADS_q3_BL  HADS_q5_BL  FSS_Q1_BL  FSS_Q5_BL  \\\n",
       "0       F         0.0         0.0         0.0        3.0        3.0   \n",
       "1       F         3.0         2.0         2.0        4.0        4.0   \n",
       "2       M         2.0         2.0         2.0        5.0        5.0   \n",
       "3       F         2.0         0.0         1.0        4.0        4.0   \n",
       "4       F         1.0         1.0         0.0        5.0        5.0   \n",
       "..    ...         ...         ...         ...        ...        ...   \n",
       "12      F         1.0         0.0         0.0        3.0        3.0   \n",
       "13      F         1.0         0.0         0.0        3.0        3.0   \n",
       "14      M         2.0         1.0         2.0        4.0        3.0   \n",
       "15      M         1.0         0.0         1.0        3.0        3.0   \n",
       "16      F         2.0         2.0         1.0        3.0        3.0   \n",
       "\n",
       "    FSS_Q6_BL  FSS_Q8_BL  FSS_Q12_BL  \n",
       "0         1.0        3.0         2.0  \n",
       "1         1.0        3.0         3.0  \n",
       "2         1.0        3.0         5.0  \n",
       "3         4.0        4.0         3.0  \n",
       "4         5.0        4.0         5.0  \n",
       "..        ...        ...         ...  \n",
       "12        1.0        3.0         1.0  \n",
       "13        3.0        3.0         1.0  \n",
       "14        3.0        3.0         3.0  \n",
       "15        3.0        3.0         1.0  \n",
       "16        3.0        3.0         5.0  \n",
       "\n",
       "[84 rows x 9 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The selected variables by Noman\n",
    "Xlist = ['Gender', 'HADS_q1_BL', 'HADS_q3_BL', 'HADS_q5_BL',\n",
    "         'FSS_Q1_BL', 'FSS_Q5_BL', 'FSS_Q6_BL', 'FSS_Q8_BL', 'FSS_Q12_BL']\n",
    "\n",
    "dfX1 = df[Xlist]\n",
    "dfX1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e801b010",
   "metadata": {},
   "outputs": [],
   "source": [
    "def numberfygender(str):\n",
    "    if str == 'F':\n",
    "        return 0\n",
    "    elif str == 'M':\n",
    "        return 1\n",
    "    else:\n",
    "        return 99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b9a34843",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfX = dfX1.copy()\n",
    "gendernum_list = list(map(numberfygender, dfX1['Gender']))\n",
    "dfX['Gender'] = gendernum_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "08e16a04",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make numpy ndarray\n",
    "X = dfX.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "54e71404",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfy = df['Category']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eb5dd40c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfXy = pd.concat([dfX, dfy], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "26d2c655",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>HADS_q1_BL</th>\n",
       "      <th>HADS_q3_BL</th>\n",
       "      <th>HADS_q5_BL</th>\n",
       "      <th>FSS_Q1_BL</th>\n",
       "      <th>FSS_Q5_BL</th>\n",
       "      <th>FSS_Q6_BL</th>\n",
       "      <th>FSS_Q8_BL</th>\n",
       "      <th>FSS_Q12_BL</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>HC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>IBS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>IBS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>IBS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>IBS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>HC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>HC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>IBS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>HC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>IBS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>84 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Gender  HADS_q1_BL  HADS_q3_BL  HADS_q5_BL  FSS_Q1_BL  FSS_Q5_BL  \\\n",
       "0        0         0.0         0.0         0.0        3.0        3.0   \n",
       "1        0         3.0         2.0         2.0        4.0        4.0   \n",
       "2        1         2.0         2.0         2.0        5.0        5.0   \n",
       "3        0         2.0         0.0         1.0        4.0        4.0   \n",
       "4        0         1.0         1.0         0.0        5.0        5.0   \n",
       "..     ...         ...         ...         ...        ...        ...   \n",
       "79       0         1.0         0.0         0.0        3.0        3.0   \n",
       "80       0         1.0         0.0         0.0        3.0        3.0   \n",
       "81       1         2.0         1.0         2.0        4.0        3.0   \n",
       "82       1         1.0         0.0         1.0        3.0        3.0   \n",
       "83       0         2.0         2.0         1.0        3.0        3.0   \n",
       "\n",
       "    FSS_Q6_BL  FSS_Q8_BL  FSS_Q12_BL Category  \n",
       "0         1.0        3.0         2.0       HC  \n",
       "1         1.0        3.0         3.0      IBS  \n",
       "2         1.0        3.0         5.0      IBS  \n",
       "3         4.0        4.0         3.0      IBS  \n",
       "4         5.0        4.0         5.0      IBS  \n",
       "..        ...        ...         ...      ...  \n",
       "79        1.0        3.0         1.0       HC  \n",
       "80        3.0        3.0         1.0       HC  \n",
       "81        3.0        3.0         3.0      IBS  \n",
       "82        3.0        3.0         1.0       HC  \n",
       "83        3.0        3.0         5.0      IBS  \n",
       "\n",
       "[84 rows x 10 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset index\n",
    "dfXy = dfXy.reset_index(drop=True)\n",
    "dfXy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c180ebd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['HC', 'IBS', 'IBS', 'IBS', 'IBS', 'HC', 'HC', 'IBS', 'IBS', 'HC',\n",
       "       'IBS', 'IBS', 'HC', 'IBS', 'IBS', 'HC', 'HC', 'IBS', 'IBS', 'IBS',\n",
       "       'IBS', 'IBS', 'IBS', 'IBS', 'IBS', 'IBS', 'IBS', 'IBS', 'HC', 'HC',\n",
       "       'IBS', 'HC', 'IBS', 'HC', 'HC', 'IBS', 'HC', 'IBS', 'HC', 'IBS',\n",
       "       'IBS', 'HC', 'HC', 'HC', 'HC', 'HC', 'HC', 'IBS', 'IBS', 'HC',\n",
       "       'HC', 'IBS', 'IBS', 'HC', 'HC', 'HC', 'HC', 'IBS', 'IBS', 'IBS',\n",
       "       'IBS', 'IBS', 'IBS', 'HC', 'IBS', 'HC', 'IBS', 'HC', 'IBS', 'IBS',\n",
       "       'HC', 'IBS', 'IBS', 'IBS', 'HC', 'IBS', 'IBS', 'HC', 'IBS', 'HC',\n",
       "       'HC', 'IBS', 'HC', 'IBS'], dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = dfy.to_numpy()\n",
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "751ae375",
   "metadata": {},
   "source": [
    "###  Perform the LOOCV with different classfiers\n",
    "\n",
    "https://machinelearningmastery.com/loocv-for-evaluating-machine-learning-algorithms/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d172ab3",
   "metadata": {},
   "source": [
    "The **Leave-One-Out cross-validator** (`scikit-learn`)\n",
    "\n",
    "Provides train/test indices to split data in train/test sets. Each sample is used once as a test set (singleton) while the remaining samples form the training set.\n",
    "\n",
    "When compared with $k$-fold cross validation, one builds $n$  models from  $n$ samples instead of $k$ models, \n",
    "where  $n > k$. Moreover, each is trained on $n-1$ samples rather than $(k-1)n/k$. In both ways, assuming $k$ is not too large and $k < n$ , LOO is more computationally expensive than $k$-fold cross validation.\n",
    "\n",
    "In terms of accuracy, LOO often results in high variance as an estimator for the test error. Intuitively, since $n-1$ of the $n$ samples are used to build each model, models constructed from folds are virtually identical to each other and to the model built from the entire training set.\n",
    "\n",
    "Note: `LeaveOneOut()` is equivalent to `KFold(n_splits=n)` and `LeavePOut(p=1)` where `n` is the number of samples.\n",
    "\n",
    "Due to the high number of test sets (which is the same as the number of samples) this cross-validation method can be very costly. For large datasets one should favor `KFold`, `ShuffleSplit` or `StratifiedKFold`.\n",
    "\n",
    "\n",
    "**References:**\n",
    "\n",
    "- http://www.faqs.org/faqs/ai-faq/neural-nets/part3/section-12.html;\n",
    "\n",
    "- T. Hastie, R. Tibshirani, J. Friedman, The Elements of Statistical Learning, Springer 2009\n",
    "\n",
    "- L. Breiman, P. Spector Submodel selection and evaluation in regression: The X-random case, International Statistical Review 1992;\n",
    "\n",
    "- R. Kohavi, A Study of Cross-Validation and Bootstrap for Accuracy Estimation and Model Selection, Intl. Jnt. Conf. AI\n",
    "\n",
    "- R. Bharat Rao, G. Fung, R. Rosales, On the Dangers of Cross-Validation. An Experimental Evaluation, SIAM 2008;\n",
    "\n",
    "- G. James, D. Witten, T. Hastie, R Tibshirani, An Introduction to Statistical Learning, Springer 2013.\n",
    "\n",
    "\n",
    "See more at https://scikit-learn.org/stable/modules/cross_validation.html#leave-one-out"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd804065",
   "metadata": {},
   "source": [
    "#### _We are making the following very transparent and explict du to the small sample size ..._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6d25f933",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_true, yhatLR, yhatRF, yhatSVC (k=0): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=1): IBS_ IBS HC IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=2): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=3): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=4): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=5): HC_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=6): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=7): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=8): IBS_ IBS HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=9): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=10): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=11): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=12): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=13): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=14): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=15): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=16): HC_ HC IBS HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=17): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=18): IBS_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=19): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=20): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=21): IBS_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=22): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=23): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=24): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=25): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=26): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=27): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=28): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=29): HC_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=30): IBS_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=31): HC_ IBS HC IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=32): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=33): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=34): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=35): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=36): HC_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=37): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=38): HC_ HC IBS HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=39): IBS_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=40): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=41): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=42): HC_ HC IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=43): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=44): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=45): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=46): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=47): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=48): IBS_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=49): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=50): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=51): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=52): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=53): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=54): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=55): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=56): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=57): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=58): IBS_ IBS IBS HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=59): IBS_ HC HC IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=60): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=61): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=62): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=63): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=64): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=65): HC_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=66): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=67): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=68): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=69): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=70): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=71): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=72): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=73): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=74): HC_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=75): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=76): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=77): HC_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=78): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=79): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=80): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=81): IBS_ IBS IBS IBS\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=82): HC_ HC HC HC\n",
      "y_true, yhatLR, yhatRF, yhatSVC (k=83): IBS_ IBS HC IBS\n",
      "Accuracy LR: 0.845\n",
      "Accuracy RF: 0.786\n",
      "Accuracy SVC: 0.821\n",
      "Balanced accuracy LR: 0.839\n",
      "Balanced accuracy RF: 0.780\n",
      "Balanced accuracy SVC: 0.814\n",
      "CPU times: user 7.47 s, sys: 117 ms, total: 7.59 s\n",
      "Wall time: 7.58 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# create loocv procedure\n",
    "cv = LeaveOneOut()\n",
    "\n",
    "# enumerate splits\n",
    "y_true, y_predRF, y_predLR, y_predSVC = list(), list(), list(), list()\n",
    "\n",
    "k=0\n",
    "for train_ix, test_ix in cv.split(X):\n",
    "    # split data\n",
    "    X_train, X_test = X[train_ix, :], X[test_ix, :]\n",
    "    y_train, y_test = y[train_ix], y[test_ix]\n",
    "    # fit models\n",
    "    modelLR = LogisticRegression(random_state=1, max_iter=1000)\n",
    "    modelLR.fit(X_train, y_train)\n",
    "    \n",
    "    modelRF = RandomForestClassifier(random_state=1)\n",
    "    modelRF.fit(X_train, y_train)\n",
    "    \n",
    "    modelSVC = SVC(random_state=1)\n",
    "    modelSVC.fit(X_train, y_train)\n",
    "    \n",
    "    # evaluate model\n",
    "    yhatLR = modelLR.predict(X_test)\n",
    "    yhatRF = modelRF.predict(X_test)\n",
    "    yhatSVC = modelSVC.predict(X_test)\n",
    "    \n",
    "    print('y_true, yhatLR, yhatRF, yhatSVC (k=%d):' % k, f\"{y_test[0]}_\", yhatLR[0], yhatRF[0], yhatSVC[0])\n",
    "    # store\n",
    "    y_true.append(y_test[0])\n",
    "    y_predLR.append(yhatLR[0])\n",
    "    y_predRF.append(yhatRF[0])\n",
    "    y_predSVC.append(yhatSVC[0])\n",
    "    \n",
    "    k=k+1\n",
    "# calculate accuracy\n",
    "accLR = accuracy_score(y_true, y_predLR)\n",
    "accRF = accuracy_score(y_true, y_predRF)\n",
    "accSVC = accuracy_score(y_true, y_predSVC)\n",
    "\n",
    "baccLR = balanced_accuracy_score(y_true, y_predLR)\n",
    "baccRF = balanced_accuracy_score(y_true, y_predRF)\n",
    "baccSVC = balanced_accuracy_score(y_true, y_predSVC)\n",
    "\n",
    "print('Accuracy LR: %.3f' % accLR)\n",
    "print('Accuracy RF: %.3f' % accRF)\n",
    "print('Accuracy SVC: %.3f' % accSVC)\n",
    "\n",
    "print('Balanced accuracy LR: %.3f' % baccLR)\n",
    "print('Balanced accuracy RF: %.3f' % baccRF)\n",
    "print('Balanced accuracy SVC: %.3f' % baccSVC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d04df819",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1], dtype=int8)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat = ['HC','IBS']\n",
    "pd.Series(cat).astype('category').cat.codes.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6b5a2e17",
   "metadata": {},
   "outputs": [],
   "source": [
    "def numberfycat(str):\n",
    "    if str == 'HC':\n",
    "        return 0\n",
    "    elif str == 'IBS':\n",
    "        return 1\n",
    "    else:\n",
    "        return 99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f94b8c3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_y_true = list(map(numberfycat, y_true))\n",
    "num_y_predLR = list(map(numberfycat, y_predLR))\n",
    "num_y_predRF = list(map(numberfycat, y_predRF))\n",
    "num_y_predSVC = list(map(numberfycat, y_predSVC))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "da099f3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy LR: 0.845\n",
      "Accuracy RF: 0.786\n",
      "Accuracy SVC: 0.821\n",
      "Balanced accuracy LR: 0.839\n",
      "Balanced accuracy RF: 0.780\n",
      "Balanced accuracy SVC: 0.814\n",
      "F1-score LR: 0.869\n",
      "F1-score RF: 0.816\n",
      "F1-score SVC: 0.848\n",
      "Precision LR: 0.860\n",
      "Precision RF: 0.816\n",
      "Precision SVC: 0.840\n",
      "Recall LR: 0.878\n",
      "Recall RF: 0.816\n",
      "Recall SVC: 0.857\n"
     ]
    }
   ],
   "source": [
    "# calculate performance metrics\n",
    "accLR = accuracy_score(num_y_true, num_y_predLR)\n",
    "accRF = accuracy_score(num_y_true, num_y_predRF)\n",
    "accSVC = accuracy_score(num_y_true, num_y_predSVC)\n",
    "\n",
    "baccLR = balanced_accuracy_score(num_y_true, num_y_predLR)\n",
    "baccRF = balanced_accuracy_score(num_y_true, num_y_predRF)\n",
    "baccSVC = balanced_accuracy_score(num_y_true, num_y_predSVC)\n",
    "\n",
    "f1LR = f1_score(num_y_true, num_y_predLR)\n",
    "f1RF = f1_score(num_y_true, num_y_predRF)\n",
    "f1SVC = f1_score(num_y_true, num_y_predSVC)\n",
    "                \n",
    "precLR = precision_score(num_y_true, num_y_predLR)\n",
    "precRF = precision_score(num_y_true, num_y_predRF)\n",
    "precSVC = precision_score(num_y_true, num_y_predSVC)\n",
    "\n",
    "recallLR = recall_score(num_y_true, num_y_predLR)\n",
    "recallRF = recall_score(num_y_true, num_y_predRF)\n",
    "recallSVC = recall_score(num_y_true, num_y_predSVC)\n",
    "\n",
    "print('Accuracy LR: %.3f' % accLR)\n",
    "print('Accuracy RF: %.3f' % accRF)\n",
    "print('Accuracy SVC: %.3f' % accSVC)\n",
    "\n",
    "print('Balanced accuracy LR: %.3f' % baccLR)\n",
    "print('Balanced accuracy RF: %.3f' % baccRF)\n",
    "print('Balanced accuracy SVC: %.3f' % baccSVC)\n",
    "\n",
    "print('F1-score LR: %.3f' % f1LR)\n",
    "print('F1-score RF: %.3f' % f1RF)\n",
    "print('F1-score SVC: %.3f' % f1SVC)\n",
    "\n",
    "print('Precision LR: %.3f' % precLR)\n",
    "print('Precision RF: %.3f' % precRF)\n",
    "print('Precision SVC: %.3f' % precSVC)\n",
    "\n",
    "print('Recall LR: %.3f' % recallLR)\n",
    "print('Recall RF: %.3f' % recallRF)\n",
    "print('Recall SVC: %.3f' % recallSVC)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  },
  "vscode": {
   "interpreter": {
    "hash": "c21bae82bb21577bd8c242b8b74adb4da250e502dbbd54b12619dba12b653253"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}