2_ML_Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from joblib import dump, load\n",
    "np.random.seed(42)\n",
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with np.load('output/Xy.npz') as f:\n",
    "    X = f['X']\n",
    "    y = f['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform temporal split of data into train/test sets\n",
    "pop = pd.read_csv('data/population.csv').set_index('BMT_ID')\n",
    "\n",
    "split_idx = -85\n",
    "assert pop[:split_idx].index.str.startswith('train_').all()\n",
    "assert pop[split_idx:].index.str.startswith('test_').all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing, model_selection, metrics, utils\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from tqdm import tqdm\n",
    "from joblib import Parallel, delayed\n",
    "from sklearn.base import clone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify hyperparameters and cv parameters\n",
    "base_estimator = LogisticRegression(penalty='l2', class_weight='balanced', solver='liblinear')\n",
    "param_grid = {\n",
    "    'C': [10. ** n for n in range(-6, 7)],\n",
    "    'penalty': ['l2'],\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model with baseline+vitals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtr, Xte = X[:split_idx], X[split_idx:]\n",
    "ytr, yte = y[:split_idx], y[split_idx:]\n",
    "\n",
    "cv_splits, cv_repeat = 5, 20\n",
    "cv = model_selection.RepeatedStratifiedKFold(cv_splits, cv_repeat, random_state=0)\n",
    "clf = model_selection.GridSearchCV(\n",
    "    clone(base_estimator), param_grid, \n",
    "    cv=cv, scoring='roc_auc', n_jobs=5,\n",
    ")\n",
    "clf.fit(Xtr, ytr)\n",
    "test_score = metrics.roc_auc_score(yte, clf.decision_function(Xte))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test AUC: 0.658 (0.536, 0.784)\n"
     ]
    }
   ],
   "source": [
    "y_true = yte\n",
    "y_score = clf.decision_function(Xte)\n",
    "\n",
    "def boostrap_func(i, y_true, y_score):\n",
    "    yte_true_b, yte_pred_b = utils.resample(y_true, y_score, replace=True, random_state=i)\n",
    "    return metrics.roc_curve(yte_true_b, yte_pred_b), metrics.roc_auc_score(yte_true_b, yte_pred_b)\n",
    "\n",
    "roc_curves, auc_scores = zip(*Parallel(n_jobs=4)(delayed(boostrap_func)(i, y_true, y_score) for i in range(1000)))\n",
    "print('Test AUC: {:.3f} ({:.3f}, {:.3f})'.format(np.median(auc_scores), np.percentile(auc_scores, 2.5), np.percentile(auc_scores, 97.5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['output/model_combined.joblib']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dump(clf, 'output/model_combined.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model with baseline features only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tangsp/jco_cci_agvhd_prediction/venv/lib/python3.7/site-packages/sklearn/model_selection/_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "Xtr, Xte = X[:split_idx, :52], X[split_idx:, :52]\n",
    "ytr, yte = y[:split_idx], y[split_idx:]\n",
    "\n",
    "cv_splits, cv_repeat = 5, 20\n",
    "cv = model_selection.RepeatedStratifiedKFold(cv_splits, cv_repeat, random_state=0)\n",
    "clf = model_selection.GridSearchCV(\n",
    "    clone(base_estimator), param_grid, \n",
    "    cv=cv, scoring='roc_auc', n_jobs=5,\n",
    ")\n",
    "clf.fit(Xtr, ytr)\n",
    "test_score = metrics.roc_auc_score(yte, clf.decision_function(Xte))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test AUC: 0.512 (0.364, 0.643)\n"
     ]
    }
   ],
   "source": [
    "y_true = yte\n",
    "y_score = clf.decision_function(Xte)\n",
    "\n",
    "def boostrap_func(i, y_true, y_score):\n",
    "    yte_true_b, yte_pred_b = utils.resample(y_true, y_score, replace=True, random_state=i)\n",
    "    return metrics.roc_curve(yte_true_b, yte_pred_b), metrics.roc_auc_score(yte_true_b, yte_pred_b)\n",
    "\n",
    "roc_curves, auc_scores = zip(*Parallel(n_jobs=4)(delayed(boostrap_func)(i, y_true, y_score) for i in range(1000)))\n",
    "print('Test AUC: {:.3f} ({:.3f}, {:.3f})'.format(np.median(auc_scores), np.percentile(auc_scores, 2.5), np.percentile(auc_scores, 97.5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['output/model_baseline.joblib']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dump(clf, 'output/model_baseline.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model with vitals features only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtr, Xte = X[:split_idx, 52:], X[split_idx:, 52:]\n",
    "ytr, yte = y[:split_idx], y[split_idx:]\n",
    "\n",
    "cv_splits, cv_repeat = 5, 20\n",
    "cv = model_selection.RepeatedStratifiedKFold(cv_splits, cv_repeat, random_state=0)\n",
    "clf = model_selection.GridSearchCV(\n",
    "    clone(base_estimator), param_grid, \n",
    "    cv=cv, scoring='roc_auc', n_jobs=5,\n",
    ")\n",
    "clf.fit(Xtr, ytr)\n",
    "test_score = metrics.roc_auc_score(yte, clf.decision_function(Xte))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test AUC: 0.633 (0.507, 0.757)\n"
     ]
    }
   ],
   "source": [
    "y_true = yte\n",
    "y_score = clf.decision_function(Xte)\n",
    "\n",
    "def boostrap_func(i, y_true, y_score):\n",
    "    yte_true_b, yte_pred_b = utils.resample(y_true, y_score, replace=True, random_state=i)\n",
    "    return metrics.roc_curve(yte_true_b, yte_pred_b), metrics.roc_auc_score(yte_true_b, yte_pred_b)\n",
    "\n",
    "roc_curves, auc_scores = zip(*Parallel(n_jobs=4)(delayed(boostrap_func)(i, y_true, y_score) for i in range(1000)))\n",
    "print('Test AUC: {:.3f} ({:.3f}, {:.3f})'.format(np.median(auc_scores), np.percentile(auc_scores, 2.5), np.percentile(auc_scores, 97.5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['output/model_vitals.joblib']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dump(clf, 'output/model_vitals.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gvhd_python3",
   "language": "python",
   "name": "gvhd_venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}