examples/gp_regression/demo_gp_regression_meps_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import pandas as pd\n",
    "from copy import deepcopy\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from uq360.algorithms.homoscedastic_gaussian_process_regression import HomoscedasticGPRegression\n",
    "from uq360.algorithms.ucc_recalibration import UCCRecalibration\n",
    "from uq360.metrics import picp, mpiw, compute_regression_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from uq360.datasets import MEPSDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = MEPSDataset().data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_features = ['REGION','SEX','MARRY',\n",
    "                                 'FTSTU','ACTDTY','HONRDC','RTHLTH','MNHLTH','HIBPDX','CHDDX','ANGIDX',\n",
    "                                 'MIDX','OHRTDX','STRKDX','EMPHDX','CHBRON','CHOLDX','CANCERDX','DIABDX',\n",
    "                                 'JTPAIN','ARTHDX','ARTHTYPE','ASTHDX','ADHDADDX','PREGNT','WLKLIM',\n",
    "                                 'ACTLIM','SOCLIM','COGLIM','DFHEAR42','DFSEE42','ADSMOK42',\n",
    "                                 'PHQ242','EMPST','POVCAT','INSCOV']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.get_dummies(data, columns=categorical_features, prefix_sep='=')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[data[\"UTILIZATION\"] >=0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AGE</th>\n",
       "      <th>RACE</th>\n",
       "      <th>PCS42</th>\n",
       "      <th>MCS42</th>\n",
       "      <th>K6SUM42</th>\n",
       "      <th>UTILIZATION</th>\n",
       "      <th>REGION=1</th>\n",
       "      <th>REGION=2</th>\n",
       "      <th>REGION=3</th>\n",
       "      <th>REGION=4</th>\n",
       "      <th>...</th>\n",
       "      <th>EMPST=3</th>\n",
       "      <th>EMPST=4</th>\n",
       "      <th>POVCAT=1</th>\n",
       "      <th>POVCAT=2</th>\n",
       "      <th>POVCAT=3</th>\n",
       "      <th>POVCAT=4</th>\n",
       "      <th>POVCAT=5</th>\n",
       "      <th>INSCOV=1</th>\n",
       "      <th>INSCOV=2</th>\n",
       "      <th>INSCOV=3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>53</td>\n",
       "      <td>White</td>\n",
       "      <td>25.93</td>\n",
       "      <td>58.47</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>56</td>\n",
       "      <td>White</td>\n",
       "      <td>20.42</td>\n",
       "      <td>26.57</td>\n",
       "      <td>17</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>White</td>\n",
       "      <td>53.12</td>\n",
       "      <td>50.33</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>White</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>-1</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27</td>\n",
       "      <td>Non-White</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15808</th>\n",
       "      <td>25</td>\n",
       "      <td>Non-White</td>\n",
       "      <td>56.71</td>\n",
       "      <td>62.39</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15809</th>\n",
       "      <td>25</td>\n",
       "      <td>Non-White</td>\n",
       "      <td>56.71</td>\n",
       "      <td>62.39</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15810</th>\n",
       "      <td>2</td>\n",
       "      <td>White</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>-1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15811</th>\n",
       "      <td>54</td>\n",
       "      <td>Non-White</td>\n",
       "      <td>43.97</td>\n",
       "      <td>42.45</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15812</th>\n",
       "      <td>73</td>\n",
       "      <td>Non-White</td>\n",
       "      <td>42.68</td>\n",
       "      <td>43.46</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15813 rows × 139 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       AGE       RACE  PCS42  MCS42  K6SUM42  UTILIZATION  REGION=1  REGION=2  \\\n",
       "0       53      White  25.93  58.47        3           15         0         1   \n",
       "1       56      White  20.42  26.57       17           10         0         1   \n",
       "2       23      White  53.12  50.33        7            3         0         1   \n",
       "3        3      White  -1.00  -1.00       -1            7         0         1   \n",
       "4       27  Non-White  -1.00  -1.00       -1            0         0         0   \n",
       "...    ...        ...    ...    ...      ...          ...       ...       ...   \n",
       "15808   25  Non-White  56.71  62.39        0            0         0         0   \n",
       "15809   25  Non-White  56.71  62.39        0            2         0         0   \n",
       "15810    2      White  -1.00  -1.00       -1            3         0         0   \n",
       "15811   54  Non-White  43.97  42.45       24            8         1         0   \n",
       "15812   73  Non-White  42.68  43.46        0            2         1         0   \n",
       "\n",
       "       REGION=3  REGION=4  ...  EMPST=3  EMPST=4  POVCAT=1  POVCAT=2  \\\n",
       "0             0         0  ...        0        1         1         0   \n",
       "1             0         0  ...        0        1         0         0   \n",
       "2             0         0  ...        0        0         0         1   \n",
       "3             0         0  ...        0        0         0         1   \n",
       "4             1         0  ...        0        0         0         0   \n",
       "...         ...       ...  ...      ...      ...       ...       ...   \n",
       "15808         1         0  ...        0        0         1         0   \n",
       "15809         1         0  ...        0        1         1         0   \n",
       "15810         1         0  ...        0        0         1         0   \n",
       "15811         0         0  ...        0        0         0         0   \n",
       "15812         0         0  ...        0        1         0         0   \n",
       "\n",
       "       POVCAT=3  POVCAT=4  POVCAT=5  INSCOV=1  INSCOV=2  INSCOV=3  \n",
       "0             0         0         0         0         1         0  \n",
       "1             1         0         0         0         1         0  \n",
       "2             0         0         0         0         1         0  \n",
       "3             0         0         0         0         1         0  \n",
       "4             1         0         0         1         0         0  \n",
       "...         ...       ...       ...       ...       ...       ...  \n",
       "15808         0         0         0         1         0         0  \n",
       "15809         0         0         0         1         0         0  \n",
       "15810         0         0         0         0         1         0  \n",
       "15811         1         0         0         0         1         0  \n",
       "15812         1         0         0         0         1         0  \n",
       "\n",
       "[15813 rows x 139 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset index to consecutive integers\n",
    "data.reset_index(drop=True, inplace=True)\n",
    "# Drop panel number (not meant to be predictive) and sample weights\n",
    "data.drop(columns = ['PERWT15F'], inplace=True)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def race(row):\n",
    "    if row['RACE'] == 'White':\n",
    "        return 1.0\n",
    "    return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"RACE\"] = data.apply(lambda row: race(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate target variable\n",
    "y = np.log( 1.0 + data.pop('UTILIZATION'))\n",
    "a = data['RACE']\n",
    "# Split data into training and test sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "df_train, df_test, y_train, y_test, a_train, a_test = train_test_split(data, y, a, random_state=0)\n",
    "df_train, df_val, y_train, y_val, a_train, a_val = train_test_split(df_train, y_train, a_train, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#kwargs = {'cumulative': False}\n",
    "#sns.distplot(y, hist_kws=kwargs, kde_kws=kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalizer_y = StandardScaler()\n",
    "y_train = normalizer_y.fit_transform(y_train.values.reshape(-1, 1))\n",
    "y_test = normalizer_y.transform(y_test.values.reshape(-1, 1))\n",
    "normalizer_X = MinMaxScaler()\n",
    "X_train = normalizer_X.fit_transform(df_train)\n",
    "X_test = normalizer_X.transform(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#kwargs = {'cumulative': False}\n",
    "#sns.distplot(y_test, hist_kws=kwargs, kde_kws=kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_res_df(method_name, y_test, y_mean, y_lower, y_upper):\n",
    "    results = {\"Overall\": compute_regression_metrics(y_test, y_mean, y_lower, y_upper)}\n",
    "    results[\"White\"] = compute_regression_metrics(y_test[a_test==1.0], y_mean[a_test==1.0], y_lower[a_test==1.0], y_upper[a_test==1.0])\n",
    "    results[\"Non-white\"] = compute_regression_metrics(y_test[a_test==0.0], y_mean[a_test==0.0], y_lower[a_test==0.0], y_upper[a_test==0.0])\n",
    "\n",
    "    total = [method_name,\n",
    "             \"Overall\",\n",
    "             results[\"Overall\"][\"rmse\"],\n",
    "             results[\"Overall\"][\"r2\"],\n",
    "             results[\"Overall\"][\"picp\"],\n",
    "             results[\"Overall\"][\"mpiw\"],\n",
    "             100*results[\"Overall\"][\"auucc_gain\"]]\n",
    "    white = [method_name,\n",
    "             \"White\",\n",
    "             results[\"White\"][\"rmse\"],\n",
    "             results[\"White\"][\"r2\"],\n",
    "             results[\"White\"][\"picp\"],\n",
    "             results[\"White\"][\"mpiw\"],\n",
    "             100*results[\"White\"][\"auucc_gain\"]]\n",
    "    nonwhite = [method_name,\n",
    "             \"Non-white\",\n",
    "             results[\"Non-white\"][\"rmse\"],\n",
    "             results[\"Non-white\"][\"r2\"],\n",
    "             results[\"Non-white\"][\"picp\"],\n",
    "             results[\"Non-white\"][\"mpiw\"],\n",
    "             100*results[\"Non-white\"][\"auucc_gain\"]]\n",
    "    res = pd.DataFrame([total, white, nonwhite], columns=['Method', 'Group', 'rmse', 'r2', 'Avg. Coverage', 'Avg. Width', '% AUUCC Gain'])\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results = pd.DataFrame(columns=['Method', 'Group', 'rmse', 'r2', 'Avg. Coverage', 'Avg. Width', '% AUUCC Gain'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train GP Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "gp_model = HomoscedasticGPRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = np.random.randint(X_train.shape[0], size=1000)\n",
    "gp_model = gp_model.fit(X_train[ids], y_train[ids])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "gp_y_test_mean, gp_y_test_lower, gp_y_test_upper = gp_model.predict(X_test)[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "gp_res = create_res_df(\"GP\", y_test.squeeze(), gp_y_test_mean, gp_y_test_lower, gp_y_test_upper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Group</th>\n",
       "      <th>rmse</th>\n",
       "      <th>r2</th>\n",
       "      <th>Avg. Coverage</th>\n",
       "      <th>Avg. Width</th>\n",
       "      <th>% AUUCC Gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GP</td>\n",
       "      <td>Overall</td>\n",
       "      <td>0.798368</td>\n",
       "      <td>0.377280</td>\n",
       "      <td>0.950683</td>\n",
       "      <td>3.080116</td>\n",
       "      <td>2.974789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GP</td>\n",
       "      <td>White</td>\n",
       "      <td>0.849492</td>\n",
       "      <td>0.367940</td>\n",
       "      <td>0.937111</td>\n",
       "      <td>3.143033</td>\n",
       "      <td>1.709259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GP</td>\n",
       "      <td>Non-white</td>\n",
       "      <td>0.767312</td>\n",
       "      <td>0.330276</td>\n",
       "      <td>0.958516</td>\n",
       "      <td>3.043801</td>\n",
       "      <td>3.353928</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Method      Group      rmse        r2  Avg. Coverage  Avg. Width  \\\n",
       "0     GP    Overall  0.798368  0.377280       0.950683    3.080116   \n",
       "1     GP      White  0.849492  0.367940       0.937111    3.143033   \n",
       "2     GP  Non-white  0.767312  0.330276       0.958516    3.043801   \n",
       "\n",
       "   % AUUCC Gain  \n",
       "0      2.974789  \n",
       "1      1.709259  \n",
       "2      3.353928  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gp_res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Post-hoc calibration using a held out set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us now train UCCRecalibration models to calibrate the predictions on white group separately to achieve better coverage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val = normalizer_y.transform(y_val.values.reshape(-1, 1))\n",
    "X_val = normalizer_X.transform(df_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "calib_gp_for_white = UCCRecalibration(base_model=gp_model).fit(X_val[a_val==1.0], y_val[a_val==1.0])\n",
    "calib_gp_y_test_mean_white, calib_gp_y_test_lower_white, calib_gp_y_test_upper_white = calib_gp_for_white.predict(X_test[a_test==1.0], missrate=0.03)[:3]\n",
    "\n",
    "res_calib_gp_white = compute_regression_metrics(y_test[a_test==1.0].squeeze(), calib_gp_y_test_mean_white, calib_gp_y_test_lower_white, calib_gp_y_test_upper_white)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Group</th>\n",
       "      <th>rmse</th>\n",
       "      <th>r2</th>\n",
       "      <th>Avg. Coverage</th>\n",
       "      <th>Avg. Width</th>\n",
       "      <th>% AUUCC Gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GP</td>\n",
       "      <td>Overall</td>\n",
       "      <td>0.798368</td>\n",
       "      <td>0.377280</td>\n",
       "      <td>0.950683</td>\n",
       "      <td>3.080116</td>\n",
       "      <td>2.974789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GP</td>\n",
       "      <td>White</td>\n",
       "      <td>0.849492</td>\n",
       "      <td>0.367940</td>\n",
       "      <td>0.937111</td>\n",
       "      <td>3.143033</td>\n",
       "      <td>1.709259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GP</td>\n",
       "      <td>Non-white</td>\n",
       "      <td>0.767312</td>\n",
       "      <td>0.330276</td>\n",
       "      <td>0.958516</td>\n",
       "      <td>3.043801</td>\n",
       "      <td>3.353928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GP</td>\n",
       "      <td>Calib White</td>\n",
       "      <td>0.849492</td>\n",
       "      <td>0.367940</td>\n",
       "      <td>0.968901</td>\n",
       "      <td>3.743788</td>\n",
       "      <td>1.692031</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Method        Group      rmse        r2  Avg. Coverage  Avg. Width  \\\n",
       "0     GP      Overall  0.798368  0.377280       0.950683    3.080116   \n",
       "1     GP        White  0.849492  0.367940       0.937111    3.143033   \n",
       "2     GP    Non-white  0.767312  0.330276       0.958516    3.043801   \n",
       "3     GP  Calib White  0.849492  0.367940       0.968901    3.743788   \n",
       "\n",
       "   % AUUCC Gain  \n",
       "0      2.974789  \n",
       "1      1.709259  \n",
       "2      3.353928  \n",
       "3      1.692031  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gp_res.append(pd.Series([\"GP\",\n",
    "             \"Calib White\",\n",
    "             res_calib_gp_white[\"rmse\"],\n",
    "             res_calib_gp_white[\"r2\"],\n",
    "             res_calib_gp_white[\"picp\"],\n",
    "             res_calib_gp_white[\"mpiw\"],\n",
    "             100*res_calib_gp_white[\"auucc_gain\"]], index = gp_res.columns), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}