experiments/01_airline_GPU.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Experiment 01: Airline dataset (GPU version)\n",
    "\n",
    "In this experiment we use [the airline dataset](http://kt.ijs.si/elena_ikonomovska/data.html) to predict arrival delay. The dataset consists of a large amount of records, containing flight arrival and departure details for all the commercial flights within the USA, from October 1987 to April 2008. Its size is around 116 million records and 5.76 GB of memory.\n",
    "\n",
    "The details of the machine we used and the version of the libraries can be found in [experiment 01](01_airline.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "System version: 3.5.3 |Anaconda 4.4.0 (64-bit)| (default, Mar  6 2017, 11:58:13) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "XGBoost version: 0.6\n",
      "LightGBM version: 0.2\n"
     ]
    }
   ],
   "source": [
    "import os,sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from libs.metrics import classification_metrics_binary, classification_metrics_binary_prob, binarize_prediction\n",
    "from libs.loaders import load_airline\n",
    "from libs.conversion import convert_cols_categorical_to_numeric, convert_related_cols_categorical_to_numeric\n",
    "from libs.timer import Timer\n",
    "from libs.utils import get_number_processors\n",
    "import pkg_resources\n",
    "import json\n",
    "import warnings\n",
    "\n",
    "print(\"System version: {}\".format(sys.version))\n",
    "print(\"XGBoost version: {}\".format(pkg_resources.get_distribution('xgboost').version))\n",
    "print(\"LightGBM version: {}\".format(pkg_resources.get_distribution('lightgbm').version))\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# 1) XGBoost vs LightGBM benchmark\n",
    "In the next section we compare both libraries speed, accuracy and other metrics for the dataset of airline arrival delay. \n",
    "\n",
    "### Data loading and management"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:libs.loaders:MOUNT_POINT not found in environment. Defaulting to /fileshare\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(115069017, 14)\n",
      "CPU times: user 1min 31s, sys: 12 s, total: 1min 43s\n",
      "Wall time: 4min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_plane = load_airline()\n",
    "print(df_plane.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>DayofMonth</th>\n",
       "      <th>DayofWeek</th>\n",
       "      <th>CRSDepTime</th>\n",
       "      <th>CRSArrTime</th>\n",
       "      <th>UniqueCarrier</th>\n",
       "      <th>FlightNum</th>\n",
       "      <th>ActualElapsedTime</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Dest</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Diverted</th>\n",
       "      <th>ArrDelay</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>556</td>\n",
       "      <td>AA</td>\n",
       "      <td>190</td>\n",
       "      <td>247</td>\n",
       "      <td>SFO</td>\n",
       "      <td>ORD</td>\n",
       "      <td>1846</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>114</td>\n",
       "      <td>EA</td>\n",
       "      <td>57</td>\n",
       "      <td>74</td>\n",
       "      <td>LAX</td>\n",
       "      <td>SFO</td>\n",
       "      <td>337</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>35</td>\n",
       "      <td>HP</td>\n",
       "      <td>351</td>\n",
       "      <td>167</td>\n",
       "      <td>ICT</td>\n",
       "      <td>LAS</td>\n",
       "      <td>987</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>40</td>\n",
       "      <td>DL</td>\n",
       "      <td>251</td>\n",
       "      <td>35</td>\n",
       "      <td>MCO</td>\n",
       "      <td>PBI</td>\n",
       "      <td>142</td>\n",
       "      <td>0</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>517</td>\n",
       "      <td>UA</td>\n",
       "      <td>500</td>\n",
       "      <td>208</td>\n",
       "      <td>LAS</td>\n",
       "      <td>ORD</td>\n",
       "      <td>1515</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year  Month  DayofMonth  DayofWeek  CRSDepTime  CRSArrTime UniqueCarrier  \\\n",
       "0  1987     10           1          4           1         556            AA   \n",
       "1  1987     10           1          4           5         114            EA   \n",
       "2  1987     10           1          4           5          35            HP   \n",
       "3  1987     10           1          4           5          40            DL   \n",
       "4  1987     10           1          4           8         517            UA   \n",
       "\n",
       "   FlightNum  ActualElapsedTime Origin Dest  Distance  Diverted  ArrDelay  \n",
       "0        190                247    SFO  ORD      1846         0        27  \n",
       "1         57                 74    LAX  SFO       337         0         5  \n",
       "2        351                167    ICT  LAS       987         0        17  \n",
       "3        251                 35    MCO  PBI       142         0        -2  \n",
       "4        500                208    LAS  ORD      1515         0        17  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_plane.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The first step is to convert the categorical features to numeric features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 46s, sys: 7.26 s, total: 1min 53s\n",
      "Wall time: 1min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_plane_numeric = convert_related_cols_categorical_to_numeric(df_plane, col_list=['Origin','Dest'])\n",
    "del df_plane"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>DayofMonth</th>\n",
       "      <th>DayofWeek</th>\n",
       "      <th>CRSDepTime</th>\n",
       "      <th>CRSArrTime</th>\n",
       "      <th>UniqueCarrier</th>\n",
       "      <th>FlightNum</th>\n",
       "      <th>ActualElapsedTime</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Dest</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Diverted</th>\n",
       "      <th>ArrDelay</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>556</td>\n",
       "      <td>AA</td>\n",
       "      <td>190</td>\n",
       "      <td>247</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>1846</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>114</td>\n",
       "      <td>EA</td>\n",
       "      <td>57</td>\n",
       "      <td>74</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>337</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>35</td>\n",
       "      <td>HP</td>\n",
       "      <td>351</td>\n",
       "      <td>167</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>987</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>40</td>\n",
       "      <td>DL</td>\n",
       "      <td>251</td>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>41</td>\n",
       "      <td>142</td>\n",
       "      <td>0</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>517</td>\n",
       "      <td>UA</td>\n",
       "      <td>500</td>\n",
       "      <td>208</td>\n",
       "      <td>4</td>\n",
       "      <td>33</td>\n",
       "      <td>1515</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year  Month  DayofMonth  DayofWeek  CRSDepTime  CRSArrTime UniqueCarrier  \\\n",
       "0  1987     10           1          4           1         556            AA   \n",
       "1  1987     10           1          4           5         114            EA   \n",
       "2  1987     10           1          4           5          35            HP   \n",
       "3  1987     10           1          4           5          40            DL   \n",
       "4  1987     10           1          4           8         517            UA   \n",
       "\n",
       "   FlightNum  ActualElapsedTime  Origin  Dest  Distance  Diverted  ArrDelay  \n",
       "0        190                247       0    33      1846         0        27  \n",
       "1         57                 74       1     0       337         0         5  \n",
       "2        351                167       2     4       987         0        17  \n",
       "3        251                 35       3    41       142         0        -2  \n",
       "4        500                208       4    33      1515         0        17  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_plane_numeric.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 1s, sys: 7.51 s, total: 1min 8s\n",
      "Wall time: 1min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_plane_numeric = convert_cols_categorical_to_numeric(df_plane_numeric, col_list='UniqueCarrier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>DayofMonth</th>\n",
       "      <th>DayofWeek</th>\n",
       "      <th>CRSDepTime</th>\n",
       "      <th>CRSArrTime</th>\n",
       "      <th>UniqueCarrier</th>\n",
       "      <th>FlightNum</th>\n",
       "      <th>ActualElapsedTime</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Dest</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Diverted</th>\n",
       "      <th>ArrDelay</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>556</td>\n",
       "      <td>0</td>\n",
       "      <td>190</td>\n",
       "      <td>247</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>1846</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>114</td>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>74</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>337</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>35</td>\n",
       "      <td>2</td>\n",
       "      <td>351</td>\n",
       "      <td>167</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>987</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>40</td>\n",
       "      <td>3</td>\n",
       "      <td>251</td>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>41</td>\n",
       "      <td>142</td>\n",
       "      <td>0</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>517</td>\n",
       "      <td>4</td>\n",
       "      <td>500</td>\n",
       "      <td>208</td>\n",
       "      <td>4</td>\n",
       "      <td>33</td>\n",
       "      <td>1515</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year  Month  DayofMonth  DayofWeek  CRSDepTime  CRSArrTime  UniqueCarrier  \\\n",
       "0  1987     10           1          4           1         556              0   \n",
       "1  1987     10           1          4           5         114              1   \n",
       "2  1987     10           1          4           5          35              2   \n",
       "3  1987     10           1          4           5          40              3   \n",
       "4  1987     10           1          4           8         517              4   \n",
       "\n",
       "   FlightNum  ActualElapsedTime  Origin  Dest  Distance  Diverted  ArrDelay  \n",
       "0        190                247       0    33      1846         0        27  \n",
       "1         57                 74       1     0       337         0         5  \n",
       "2        351                167       2     4       987         0        17  \n",
       "3        251                 35       3    41       142         0        -2  \n",
       "4        500                208       4    33      1515         0        17  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_plane_numeric.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "To simplify the pipeline, we are going to set a classification problem where the goal is to classify wheather a flight has arrived delayed or not. For that we need to binarize the variable `ArrDelay`.\n",
    "\n",
    "If you want to extend this experiment, you can set a regression problem and try to identify the number of minutes of delay a fight has. Both XGBoost and LightGBM have regression classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_plane_numeric = df_plane_numeric.apply(lambda x: x.astype('int16'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 396 ms, sys: 372 ms, total: 768 ms\n",
      "Wall time: 771 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_plane_numeric['ArrDelayBinary'] = 1*(df_plane_numeric['ArrDelay'] > 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>DayofMonth</th>\n",
       "      <th>DayofWeek</th>\n",
       "      <th>CRSDepTime</th>\n",
       "      <th>CRSArrTime</th>\n",
       "      <th>UniqueCarrier</th>\n",
       "      <th>FlightNum</th>\n",
       "      <th>ActualElapsedTime</th>\n",
       "      <th>Origin</th>\n",
       "      <th>Dest</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Diverted</th>\n",
       "      <th>ArrDelay</th>\n",
       "      <th>ArrDelayBinary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>556</td>\n",
       "      <td>0</td>\n",
       "      <td>190</td>\n",
       "      <td>247</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>1846</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>114</td>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>74</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>337</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>35</td>\n",
       "      <td>2</td>\n",
       "      <td>351</td>\n",
       "      <td>167</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>987</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>40</td>\n",
       "      <td>3</td>\n",
       "      <td>251</td>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>41</td>\n",
       "      <td>142</td>\n",
       "      <td>0</td>\n",
       "      <td>-2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1987</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>517</td>\n",
       "      <td>4</td>\n",
       "      <td>500</td>\n",
       "      <td>208</td>\n",
       "      <td>4</td>\n",
       "      <td>33</td>\n",
       "      <td>1515</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year  Month  DayofMonth  DayofWeek  CRSDepTime  CRSArrTime  UniqueCarrier  \\\n",
       "0  1987     10           1          4           1         556              0   \n",
       "1  1987     10           1          4           5         114              1   \n",
       "2  1987     10           1          4           5          35              2   \n",
       "3  1987     10           1          4           5          40              3   \n",
       "4  1987     10           1          4           8         517              4   \n",
       "\n",
       "   FlightNum  ActualElapsedTime  Origin  Dest  Distance  Diverted  ArrDelay  \\\n",
       "0        190                247       0    33      1846         0        27   \n",
       "1         57                 74       1     0       337         0         5   \n",
       "2        351                167       2     4       987         0        17   \n",
       "3        251                 35       3    41       142         0        -2   \n",
       "4        500                208       4    33      1515         0        17   \n",
       "\n",
       "   ArrDelayBinary  \n",
       "0               1  \n",
       "1               1  \n",
       "2               1  \n",
       "3               0  \n",
       "4               1  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_plane_numeric.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Once the features are prepared, let's split the dataset into train, validation and test set. We won't use validation for this example (however, you can try to add it)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def split_train_val_test_df(df, val_size=0.2, test_size=0.2):\n",
    "    train, validate, test = np.split(df.sample(frac=1), \n",
    "                                     [int((1-val_size-test_size)*len(df)), int((1-test_size)*len(df))])\n",
    "    return train, validate, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(92055213, 15)\n",
      "(0, 15)\n",
      "(23013804, 15)\n",
      "CPU times: user 36.3 s, sys: 2.82 s, total: 39.1 s\n",
      "Wall time: 38.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, validate, test = split_train_val_test_df(df_plane_numeric, val_size=0, test_size=0.2)\n",
    "\n",
    "#smaller dataset depending on the size of your GPU\n",
    "#df_plane_numeric_small = df_plane_numeric.sample(n=1e6).reset_index(drop=True)\n",
    "#train, validate, test = split_train_val_test_df(df_plane_numeric_small, val_size=0, test_size=0.2)\n",
    "\n",
    "print(train.shape)\n",
    "print(validate.shape)\n",
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def generate_feables(df):\n",
    "    X = df[df.columns.difference(['ArrDelay', 'ArrDelayBinary'])]\n",
    "    y = df['ArrDelayBinary']\n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.12 s, sys: 440 ms, total: 1.56 s\n",
      "Wall time: 1.56 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_train, y_train = generate_feables(train)\n",
    "X_test, y_test = generate_feables(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Training and Evaluation\n",
    "Now we are going to create two pipelines, one of XGBoost and one for LightGBM. The technology behind both libraries is different, so it is difficult to compare them in the exact same model setting. XGBoost grows the trees depth-wise and controls model complexity with `max_depth`. Instead, LightGBM uses a leaf-wise algorithm and controls the model complexity by `num_leaves`. As a tradeoff, we use XGBoost with `max_depth=8`, which will have max number leaves of 255, and compare it with LightGBM with `num_leaves=255`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict = dict()\n",
    "num_rounds = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def train_xgboost(parameters, X, y, num_rounds=50):\n",
    "    ddata = xgb.DMatrix(data=X, label=y)\n",
    "    with Timer() as t:\n",
    "        clf = xgb.train(parameters, ddata, num_boost_round=num_rounds)\n",
    "    return clf, t.interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def test_xgboost(clf, X, y):\n",
    "    ddata = xgb.DMatrix(data=X, label=y)\n",
    "    with Timer() as t:\n",
    "        y_pred = clf.predict(ddata)\n",
    "    return y_pred, t.interval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's start with the XGBoost model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xgb_params = {'max_depth':8, #'max_depth':2, \n",
    "              'objective':'binary:logistic', \n",
    "              'min_child_weight':30, \n",
    "              'eta':0.1, \n",
    "              'scale_pos_weight':2, \n",
    "              'gamma':0.1, \n",
    "              'reg_lamda':1, \n",
    "              'subsample':1,\n",
    "              'tree_method':'exact', \n",
    "              'updater':'grow_gpu'\n",
    "             }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "*NOTE: We got an out of memory error with xgb due to the big size of the dataset and the tree. At the end of the notebook we perform a benchmark with different data sizes.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "source": [
    "```python\n",
    "xgb_clf_pipeline, t_train = train_xgboost(xgb_params, X_train, y_train, num_rounds)\n",
    "\n",
    "results_dict['xgb']={ 'train_time': t_train }\n",
    "\n",
    "y_prob_xgb, t_test = test_xgboost(xgb_clf_pipeline, X_test, y_test)\n",
    "\n",
    "y_pred_xgb = binarize_prediction(y_prob_xgb)\n",
    "\n",
    "results_dict['xgb']['test_time'] = t_test\n",
    "\n",
    "del xgb_clf_pipeline\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Training XGBoost model with leaf-wise growth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xgb_hist_params = {'max_depth':0, \n",
    "                  'max_leaves':2**8, \n",
    "                  'objective':'binary:logistic', \n",
    "                  'min_child_weight':30, \n",
    "                  'eta':0.1, \n",
    "                  'scale_pos_weight':2, \n",
    "                  'gamma':0.1, \n",
    "                  'reg_lamda':1, \n",
    "                  'subsample':1,\n",
    "                  'tree_method':'hist', \n",
    "                  'grow_policy':'lossguide', \n",
    "                  'updater':'grow_gpu_hist'\n",
    "                 }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xgb_hist_clf_pipeline, t_train = train_xgboost(xgb_hist_params, X_train, y_train, num_rounds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']={\n",
    "    'train_time': t_train\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_prob_xgb_hist, t_test = test_xgboost(xgb_hist_clf_pipeline, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_pred_xgb_hist = binarize_prediction(y_prob_xgb_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']['test_time'] = t_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "del xgb_hist_clf_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Training LightGBM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def train_lightgbm(parameters, X, y, num_rounds=50):\n",
    "    ddata = lgb.Dataset(X.values, y.values, free_raw_data=False)\n",
    "    with Timer() as t:\n",
    "        clf = lgb.train(parameters, ddata, num_boost_round=num_rounds)\n",
    "    return clf, t.interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def test_lightgbm(clf, X):\n",
    "    with Timer() as t:\n",
    "        y_pred = clf.predict(X.values)\n",
    "    return y_pred, t.interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "lgbm_params = {'num_leaves': 2**8,\n",
    "               'learning_rate': 0.1,\n",
    "               'scale_pos_weight': 2,\n",
    "               'min_split_gain': 0.1,\n",
    "               'min_child_weight': 30,\n",
    "               'reg_lambda': 1,\n",
    "               'subsample': 1,\n",
    "               'objective':'binary',\n",
    "               'device': 'gpu',\n",
    "               'task': 'train'\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "lgbm_clf_pipeline, t_train = train_lightgbm(lgbm_params, X_train, y_train, num_rounds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']={\n",
    "    'train_time': t_train\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_prob_lgbm, t_test = test_lightgbm(lgbm_clf_pipeline, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_pred_lgbm = binarize_prediction(y_prob_lgbm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']['test_time'] = t_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "del lgbm_clf_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "As it can be seen in the results, given the specific versions and parameters of both XGBoost and LightGBM and in this specific dataset, LightGBM is faster. \n",
    "\n",
    "In general terms, leaf-wise algorithms are more efficient, they converge much faster than depth-wise. However, it may cause over-fitting when the data is small or there are too many leaves."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Metrics\n",
    "We are going to obtain some metrics to evaluate the performance of each of the models."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "source": [
    "```python\n",
    "report_xgb = classification_metrics_binary(y_test, y_pred_xgb)\n",
    "report2_xgb = classification_metrics_binary_prob(y_test, y_prob_xgb)\n",
    "report_xgb.update(report2_xgb)\n",
    "results_dict['xgb']['performance'] = report_xgb\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report_xgb_hist = classification_metrics_binary(y_test, y_pred_xgb_hist)\n",
    "report2_xgb_hist = classification_metrics_binary_prob(y_test, y_prob_xgb_hist)\n",
    "report_xgb_hist.update(report2_xgb_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']['performance'] = report_xgb_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report_lgbm = classification_metrics_binary(y_test, y_pred_lgbm)\n",
    "report2_lgbm = classification_metrics_binary_prob(y_test, y_prob_lgbm)\n",
    "report_lgbm.update(report2_lgbm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']['performance'] = report_lgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"lgbm\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.8420633400407397,\n",
      "            \"Accuracy\": 0.7253679574224235,\n",
      "            \"F1\": 0.7459552507319687,\n",
      "            \"Precision\": 0.6647341223976702,\n",
      "            \"Recall\": 0.8497872609318392\n",
      "        },\n",
      "        \"test_time\": 48.380596682982286,\n",
      "        \"train_time\": 645.403558799997\n",
      "    },\n",
      "    \"xgb_hist\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.8411899192614771,\n",
      "            \"Accuracy\": 0.7245983323747782,\n",
      "            \"F1\": 0.745273441487613,\n",
      "            \"Precision\": 0.6640651298572008,\n",
      "            \"Recall\": 0.8491109466751164\n",
      "        },\n",
      "        \"test_time\": 31.72845547500765,\n",
      "        \"train_time\": 1271.9167441620084\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Results\n",
    "print(json.dumps(results_dict, indent=4, sort_keys=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The experiment shows a similar performance in XGBoost hist and LightGBM. Under the parameters we used and this big dataset, LightGBM is faster. We couldn't compute the standard version of XGBoost because we got an out of memory."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# 2) Data size benchmark\n",
    "Now we are going to analyze the performance of the libraries with different data sizes. The depth-wise implementation needs much more memory than the leaf-wise implementation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sizes = [1e4, 1e5, 1e6, 1e7]\n",
    "num_rounds = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def generate_partial_datasets(df, num_rows, test_size=0.2):\n",
    "    df_small = df.sample(n=int(num_rows)).reset_index(drop=True)\n",
    "    train, _, test = split_train_val_test_df(df_small, val_size=0, test_size=test_size)\n",
    "    X_train, y_train = generate_feables(train)\n",
    "    X_test, y_test = generate_feables(test)\n",
    "    return X_train, y_train, X_test, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "del X_train, y_train, X_test, y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's loop for the different data sizes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computed XGBoost with 1e+04 samples in 6.793s with AUC=0.728\n",
      "Computed XGBoost hist with 1e+04 samples in 4.296s with AUC=0.724\n",
      "Computed LightGBM with 1e+04 samples in 4.632s with AUC=0.729\n",
      "\n",
      "Computed XGBoost with 1e+05 samples in 15.313s with AUC=0.799\n",
      "Computed XGBoost hist with 1e+05 samples in 33.554s with AUC=0.790\n",
      "Computed LightGBM with 1e+05 samples in 29.671s with AUC=0.791\n",
      "\n",
      "Computed XGBoost with 1e+06 samples in 127.482s with AUC=0.832\n",
      "Computed XGBoost hist with 1e+06 samples in 54.013s with AUC=0.837\n",
      "Computed LightGBM with 1e+06 samples in 41.903s with AUC=0.838\n",
      "\n",
      "Computed XGBoost with 1e+07 samples in 1363.350s with AUC=0.843\n",
      "Computed XGBoost hist with 1e+07 samples in 238.144s with AUC=0.854\n",
      "Computed LightGBM with 1e+07 samples in 134.127s with AUC=0.855\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for s in sizes:\n",
    "    X_train, y_train, X_test, y_test = generate_partial_datasets(df_plane_numeric, s)\n",
    "    clf_xgb, train_time_xgb = train_xgboost(xgb_params, X_train, y_train, num_rounds)\n",
    "    y_pred, test_time_xgb = test_xgboost(clf_xgb, X_test, y_test)\n",
    "    auc_xgb = roc_auc_score(y_test, y_pred)\n",
    "    del clf_xgb #free GPU memory\n",
    "    print(\"Computed XGBoost with {:.0e} samples in {:.3f}s with AUC={:.3f}\".format(s, train_time_xgb, auc_xgb))\n",
    "    \n",
    "    clf_xgb_hist, train_time_xgb_hist = train_xgboost(xgb_hist_params, X_train, y_train, num_rounds)\n",
    "    y_pred, test_time_xgb = test_xgboost(clf_xgb_hist, X_test, y_test)\n",
    "    auc_xgb_hist = roc_auc_score(y_test, y_pred)\n",
    "    del clf_xgb_hist\n",
    "    print(\"Computed XGBoost hist with {:.0e} samples in {:.3f}s with AUC={:.3f}\".format(s, train_time_xgb_hist, auc_xgb_hist))\n",
    "\n",
    "    clf_lgbm, train_time_lgbm = train_lightgbm(lgbm_params, X_train, y_train, num_rounds)\n",
    "    y_pred, test_time_lgbm = test_lightgbm(clf_lgbm, X_test)\n",
    "    auc_lgbm = roc_auc_score(y_test, y_pred)\n",
    "    del clf_lgbm\n",
    "    print(\"Computed LightGBM with {:.0e} samples in {:.3f}s with AUC={:.3f}\\n\".format(s, train_time_lgbm, auc_lgbm))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}