withAdditionalFields.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "pd.set_option(\"display.max_columns\", 100)\n",
    "from sklearn.metrics import f1_score\n",
    "DATA_PATH = Path.cwd().parent / \"ML project\" \n",
    "from imblearn.combine import SMOTETomek, SMOTEENN\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder, MinMaxScaler\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import Normalizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import MissingIndicator\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "from imblearn.over_sampling import SMOTE, SVMSMOTE\n",
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "from sklearn.preprocessing import Normalizer\n",
    "RANDOM_SEED = 6    # Set a random seed for reproducibility!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from math import cos, asin, sqrt, pi\n",
    "\n",
    "def distanceLong(lat1, lon1, lat2, lon2):\n",
    "    p = pi/180\n",
    "    latDiff = (lat2-lat1)*p\n",
    "    lonDiff = (lon2-lon1)*p\n",
    "    coslat1 = np.cos(lat1*p)\n",
    "    coslat2 = np.cos(lat2*p)\n",
    "    coslondiff = np.cos(lonDiff)\n",
    "    coslatdiff = np.cos(latDiff)\n",
    "    a = 0.5 - coslatdiff/2 + coslat1 * coslat2 * (1-coslondiff)/2\n",
    "    return 12742 * (np.arcsin(a**0.5)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def formatdata(dataFrame):\n",
    "     \n",
    "    dataFrame['pickup_time'] = dataFrame['pickup_time']+ '.099'\n",
    "    dataFrame['pickup_time'] = dataFrame['pickup_time'].apply(lambda x:  x.replace(\"/\",\"-\"))\n",
    "    dataFrame['drop_time'] = dataFrame['drop_time']+ '.099'\n",
    "\n",
    "    pd.to_datetime(dataFrame['pickup_time'], format='%m-%d-%Y %H:%M:%S.%f', errors='ignore')\n",
    "    pd.to_datetime(dataFrame['drop_time'], format='%m/%d/%Y %H:%M:%S.%f', errors='ignore')\n",
    "    \n",
    "    dataFrame['rush'] = (pd.to_datetime(dataFrame['pickup_time'], errors='ignore').dt.hour)\n",
    "    dataFrame['rush'] = dataFrame['rush'].apply(lambda x: ((x >= 7 and x < 9) or (x >= 17 and x < 19)))\n",
    "    dataFrame['pickup_time'] = dataFrame['pickup_time'].apply(lambda x: datetime.timestamp(datetime.strptime(x,'%m-%d-%Y %H:%M.%f')))\n",
    "    dataFrame['drop_time'] = dataFrame['drop_time'].apply(lambda x: datetime.timestamp(datetime.strptime(x,'%m/%d/%Y %H:%M.%f')))\n",
    "    dataFrame['timediff'] =  dataFrame['drop_time'] - dataFrame['pickup_time']\n",
    "   # dataFrame['timediff1'] =  dataFrame['timediff']**2\n",
    "    dataFrame['distance'] = distanceLong(dataFrame['drop_lat'],dataFrame['drop_lon'],dataFrame['pick_lat'],dataFrame['pick_lon'])*100\n",
    "    #dataFrame['distance'] = ((dataFrame['drop_lat']-dataFrame['pick_lat'])**2+(dataFrame['drop_lon']-dataFrame['pick_lon'])**2)**0.5\n",
    "    #dataFrame['unit_meter_waiting_fair'] = dataFrame['meter_waiting_fare'] / dataFrame['meter_waiting']\n",
    "    #print(dataFrame['unit_meter_waiting_fair'])\n",
    "    \n",
    "    #dataFrame['_fare'] = dataFrame['fare'] - dataFrame['additional_fare'] \n",
    "    #dataFrame['tripDuration'] = dataFrame['duration'] - dataFrame['meter_waiting']\n",
    "    \n",
    "    \n",
    "    return dataFrame\n",
    "\n",
    " \n",
    "bestDegree = 0\n",
    "bestBias = False\n",
    "bestInteraction = False\n",
    "bestscore = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "C:\\Users\\User\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n",
      "C:\\Users\\User\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "\n",
    "solver = 'lbfgs'\n",
    "alpha = 0.2\n",
    "hidden_layer_sizes=(10,2 )\n",
    "random_state=2\n",
    "max_iter= 10000000\n",
    "activation='tanh'\n",
    "#estimatorsX=MLPClassifier(solver=solver, alpha=alpha, hidden_layer_sizes=hidden_layer_sizes, random_state=random_state,\n",
    " #                        max_iter=max_iter, activation=activation)\n",
    "\n",
    "min_child_weight=1.5\n",
    "max_depth=5\n",
    "gamma=2.5\n",
    "colsample_bytree=0.8\n",
    "subsample=0.8\n",
    "n_estimators=200\n",
    "threads = 4\n",
    "estimatorsXX = XGBClassifier(min_child_weight=min_child_weight,max_depth=max_depth, \n",
    "                           gamma=gamma,colsample_bytree=colsample_bytree,subsample=subsample,\n",
    "                           n_estimators=n_estimators, n_jobs=threads\n",
    "                          )\n",
    "params = { 'min_child_weight': [1.5,2,2.5],'max_depth':[8,9,7]}\n",
    "\n",
    "#estimatorsY = LGBMClassifier(boosting_type='gbdt', num_leaves=40, max_depth=6,learning_rate=0.2,\n",
    " #                          n_estimators=1000,random_state=42)\n",
    "\n",
    "estimators = BaggingClassifier(base_estimator=estimatorsXX,n_estimators=10,max_samples=0.65,\n",
    "                             max_features=1.0, random_state=1,bootstrap=False)\n",
    "#estimators = VotingClassifier( [('xgb', estimatorsX), ('mlp', estimatorsY)], weights=[1,1])\n",
    "#estimators = GridSearchCV(estimator=estimatorsXX, param_grid=params, cv=10)\n",
    "\n",
    "\n",
    "train_df = pd.read_csv(\n",
    "    DATA_PATH / \"train.csv\", \n",
    "    index_col=\"tripid\"\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "train_df['label'] = train_df['label'].map({\"correct\":1, \"incorrect\":0})\n",
    "train_df_cols = train_df.columns.values\n",
    "li = list(train_df_cols)\n",
    "features_df = train_df[li[0:-1]]\n",
    "labels_df = train_df[li[-1]]\n",
    "features_df = formatdata(features_df)\n",
    "features_df = features_df.fillna(features_df.mean())\n",
    "numeric_cols = features_df.columns[features_df.dtypes != object].values\n",
    "\n",
    "\n",
    "\n",
    "def removeColumns(npArry, columnNames):\n",
    "    li = list(npArry)\n",
    "    for i in columnNames:\n",
    "        li.remove(i)\n",
    "    return np.array(li)\n",
    "\n",
    "\n",
    "def powerAll(data_frame, exceptList,power):\n",
    "    allColumns = data_frame.columns.values\n",
    "    for i in allColumns:\n",
    "        if(i not in exceptList ):\n",
    "            data_frame[i] = data_frame[i]**power\n",
    " \n",
    "\n",
    "#nothing\n",
    "exceptList = []\n",
    "power = 1\n",
    "\n",
    "numeric_cols = removeColumns(numeric_cols,['drop_time', 'pickup_time','drop_lat','drop_lon','pick_lat','pick_lon'\n",
    "                                             ])  \n",
    "\n",
    "#powerAll(features_df, exceptList,1)\n",
    "#exceptList = features_df.columns[features_df.dtypes == \"object\"].values\n",
    "#polyList1 = ['meter_waiting_fare','fare']\n",
    "polyList1 = ['additional_fare','distance','meter_waiting']\n",
    "polyList2 = ['timediff','distance']\n",
    "polyList3 = ['meter_waiting','duration']\n",
    "polyList4 = ['timediff','fare']\n",
    "polyList5 = ['distance','fare']\n",
    "polyList6 = ['meter_waiting_fare','additional_fare']\n",
    "numeric_preprocessing_steps = Pipeline([ \n",
    "    ('standard_scaler', StandardScaler()),\n",
    "])\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "feature_transformer2 = PolynomialFeatures(degree=4, include_bias=False,interaction_only=False)        \n",
    "feature_transformer3 = PolynomialFeatures(degree=4, include_bias=True,interaction_only=False)\n",
    "feature_transformer4 = PolynomialFeatures(degree=4, include_bias=True,interaction_only=False)\n",
    "feature_transformer5 = PolynomialFeatures(degree=4, include_bias=True,interaction_only=False)\n",
    "feature_transformer6 = PolynomialFeatures(degree=7, include_bias=False,interaction_only=False)\n",
    "            \n",
    "normalizer_transformer = Normalizer(norm ='l2')\n",
    "            # create the preprocessor stage of final pipeline\n",
    "            # each entry in the transformer list is a tuple of\n",
    "            # (name you choose, sklearn transformer, list of columns)\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers = [\n",
    "        ('poly2', feature_transformer2,polyList2),\n",
    "        ('poly3', feature_transformer3,polyList3),\n",
    "        ('poly4', feature_transformer4,polyList4),\n",
    "        ('poly5', feature_transformer5,polyList5),\n",
    "        ('poly6', feature_transformer6,polyList6),\n",
    "        (\"numeric\", numeric_preprocessing_steps, numeric_cols),\n",
    "        (\"nomalizer\", normalizer_transformer, numeric_cols)\n",
    "    ],\n",
    "    remainder = \"drop\"\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "print(features_df.loc[213182176,'rush'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-64-7eeeeada28ab>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     25\u001b[0m             \u001b[1;33m(\u001b[0m\u001b[1;34m\"estimators\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mestimators\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m         ])\n\u001b[1;32m---> 27\u001b[1;33m         \u001b[0mfull_pipeline\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     28\u001b[0m         \u001b[0mpreds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfull_pipeline\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_eval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m         y_preds = pd.DataFrame(\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[0;32m    352\u001b[0m                                  self._log_message(len(self.steps) - 1)):\n\u001b[0;32m    353\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_final_estimator\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m'passthrough'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 354\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_final_estimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    355\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    356\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\ensemble\\_bagging.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    241\u001b[0m         \u001b[0mself\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mobject\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    242\u001b[0m         \"\"\"\n\u001b[1;32m--> 243\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_samples\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    244\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    245\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_parallel_args\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\ensemble\\_bagging.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, max_samples, max_depth, sample_weight)\u001b[0m\n\u001b[0;32m    378\u001b[0m                 \u001b[0mtotal_n_estimators\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    379\u001b[0m                 verbose=self.verbose)\n\u001b[1;32m--> 380\u001b[1;33m             for i in range(n_jobs))\n\u001b[0m\u001b[0;32m    381\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    382\u001b[0m         \u001b[1;31m# Reduce\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m   1002\u001b[0m             \u001b[1;31m# remaining jobs.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1003\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1004\u001b[1;33m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1005\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1006\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[1;34m(self, iterator)\u001b[0m\n\u001b[0;32m    833\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    834\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 835\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    836\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    837\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    752\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    753\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 754\u001b[1;33m             \u001b[0mjob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    755\u001b[0m             \u001b[1;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    756\u001b[0m             \u001b[1;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[1;34m(self, func, callback)\u001b[0m\n\u001b[0;32m    207\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    208\u001b[0m         \u001b[1;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 209\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    210\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    211\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    588\u001b[0m         \u001b[1;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    589\u001b[0m         \u001b[1;31m# arguments in memory\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 590\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    591\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    592\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    254\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    255\u001b[0m             return [func(*args, **kwargs)\n\u001b[1;32m--> 256\u001b[1;33m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[0;32m    257\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    258\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    254\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    255\u001b[0m             return [func(*args, **kwargs)\n\u001b[1;32m--> 256\u001b[1;33m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[0;32m    257\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    258\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\ensemble\\_bagging.py\u001b[0m in \u001b[0;36m_parallel_build_estimators\u001b[1;34m(n_estimators, ensemble, X, y, sample_weight, seeds, total_n_estimators, verbose)\u001b[0m\n\u001b[0;32m    108\u001b[0m                 \u001b[0mcurr_sample_weight\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnot_indices_mask\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    109\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 110\u001b[1;33m             \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeatures\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcurr_sample_weight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    111\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    112\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set, callbacks)\u001b[0m\n\u001b[0;32m    730\u001b[0m                               \u001b[0mevals_result\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals_result\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    731\u001b[0m                               \u001b[0mverbose_eval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxgb_model\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mxgb_model\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 732\u001b[1;33m                               callbacks=callbacks)\n\u001b[0m\u001b[0;32m    733\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    734\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobjective\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mxgb_options\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"objective\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[0;32m    214\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    215\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 216\u001b[1;33m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[0;32m    217\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    218\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[0;32m     72\u001b[0m         \u001b[1;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     73\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m2\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[0;32m   1107\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1108\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[1;32m-> 1109\u001b[1;33m                                                     dtrain.handle))\n\u001b[0m\u001b[0;32m   1110\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1111\u001b[0m             \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "best_df_list = []\n",
    "statesList = range(100)\n",
    "for i in statesList:    \n",
    "    kf = KFold(n_splits=10, random_state=i, shuffle=True)\n",
    "\n",
    "\n",
    "    kf.get_n_splits(features_df)\n",
    "    test_features_df = pd.read_csv(DATA_PATH / \"test.csv\", \n",
    "                                   index_col=\"tripid\")\n",
    "\n",
    "    test_features_df = formatdata(test_features_df)\n",
    "    powerAll(test_features_df, exceptList,power)\n",
    "\n",
    "    bestscore = 0\n",
    "    bestModelx = \"\"\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    for train_index, test_index in kf.split(features_df):\n",
    "\n",
    "        X_train, X_eval, y_train, y_eval = features_df.iloc[train_index], features_df.iloc[test_index],labels_df.iloc[train_index], labels_df.iloc[test_index]\n",
    "        full_pipeline = Pipeline([\n",
    "            (\"preprocessor\", preprocessor),\n",
    "            (\"estimators\", estimators),\n",
    "        ])\n",
    "        full_pipeline.fit(X_train, y_train)\n",
    "        preds = full_pipeline.predict(X_eval)\n",
    "        y_preds = pd.DataFrame(\n",
    "            preds,\n",
    "            index = y_eval.index\n",
    "        )\n",
    "        test_probas = full_pipeline.predict(test_features_df)\n",
    "        test_preds = pd.DataFrame(\n",
    "            {'prediction':test_probas},\n",
    "            index = test_features_df.index\n",
    "        )\n",
    "        f1 = f1_score(y_eval, y_preds)\n",
    "        if(f1 > bestscore):\n",
    "            bestscore = f1\n",
    "            bestModelx = full_pipeline\n",
    "\n",
    "\n",
    "    \n",
    "    test_probas_state_best = bestModelx.predict(test_features_df)\n",
    "    test_preds_state_best = pd.DataFrame(\n",
    "        {'prediction':test_probas_state_best},\n",
    "        index = test_features_df.index\n",
    "    )\n",
    "    best_df_list.append(test_preds_state_best)\n",
    "    \n",
    "    print(test_preds_state_best.prediction.value_counts())\n",
    "    print('bestscore', bestscore)\n",
    "    print(\"state\", i)\n",
    "    print('')\n",
    "\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8157\n",
      "0     419\n",
      "Name: prediction, dtype: int64\n",
      "1    8134\n",
      "0     442\n",
      "Name: prediction, dtype: int64\n",
      "1    8143\n",
      "0     433\n",
      "Name: prediction, dtype: int64\n",
      "1    8143\n",
      "0     433\n",
      "Name: prediction, dtype: int64\n",
      "1    8148\n",
      "0     428\n",
      "Name: prediction, dtype: int64\n",
      "1    8142\n",
      "0     434\n",
      "Name: prediction, dtype: int64\n",
      "1    8140\n",
      "0     436\n",
      "Name: prediction, dtype: int64\n",
      "1    8147\n",
      "0     429\n",
      "Name: prediction, dtype: int64\n",
      "1    8141\n",
      "0     435\n",
      "Name: prediction, dtype: int64\n",
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n",
      "1    8151\n",
      "0     425\n",
      "Name: prediction, dtype: int64\n",
      "1    8154\n",
      "0     422\n",
      "Name: prediction, dtype: int64\n",
      "1    8152\n",
      "0     424\n",
      "Name: prediction, dtype: int64\n",
      "1    8143\n",
      "0     433\n",
      "Name: prediction, dtype: int64\n",
      "1    8148\n",
      "0     428\n",
      "Name: prediction, dtype: int64\n",
      "1    8161\n",
      "0     415\n",
      "Name: prediction, dtype: int64\n",
      "1    8153\n",
      "0     423\n",
      "Name: prediction, dtype: int64\n",
      "1    8150\n",
      "0     426\n",
      "Name: prediction, dtype: int64\n",
      "1    8157\n",
      "0     419\n",
      "Name: prediction, dtype: int64\n",
      "1    8151\n",
      "0     425\n",
      "Name: prediction, dtype: int64\n",
      "1    8153\n",
      "0     423\n",
      "Name: prediction, dtype: int64\n",
      "1    8152\n",
      "0     424\n",
      "Name: prediction, dtype: int64\n",
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n",
      "1    8152\n",
      "0     424\n",
      "Name: prediction, dtype: int64\n",
      "1    8145\n",
      "0     431\n",
      "Name: prediction, dtype: int64\n",
      "1    8145\n",
      "0     431\n",
      "Name: prediction, dtype: int64\n",
      "1    8144\n",
      "0     432\n",
      "Name: prediction, dtype: int64\n",
      "1    8146\n",
      "0     430\n",
      "Name: prediction, dtype: int64\n",
      "1    8145\n",
      "0     431\n",
      "Name: prediction, dtype: int64\n",
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n",
      "1    8157\n",
      "0     419\n",
      "Name: prediction, dtype: int64\n",
      "1    8147\n",
      "0     429\n",
      "Name: prediction, dtype: int64\n",
      "1    8140\n",
      "0     436\n",
      "Name: prediction, dtype: int64\n",
      "1    8150\n",
      "0     426\n",
      "Name: prediction, dtype: int64\n",
      "1    8147\n",
      "0     429\n",
      "Name: prediction, dtype: int64\n",
      "1    8157\n",
      "0     419\n",
      "Name: prediction, dtype: int64\n",
      "1    8146\n",
      "0     430\n",
      "Name: prediction, dtype: int64\n",
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n",
      "1    8132\n",
      "0     444\n",
      "Name: prediction, dtype: int64\n",
      "1    8157\n",
      "0     419\n",
      "Name: prediction, dtype: int64\n",
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n",
      "1    8148\n",
      "0     428\n",
      "Name: prediction, dtype: int64\n",
      "1    8145\n",
      "0     431\n",
      "Name: prediction, dtype: int64\n",
      "1    8152\n",
      "0     424\n",
      "Name: prediction, dtype: int64\n",
      "1    8142\n",
      "0     434\n",
      "Name: prediction, dtype: int64\n",
      "1    8153\n",
      "0     423\n",
      "Name: prediction, dtype: int64\n",
      "1    8161\n",
      "0     415\n",
      "Name: prediction, dtype: int64\n",
      "1    8140\n",
      "0     436\n",
      "Name: prediction, dtype: int64\n",
      "1    8146\n",
      "0     430\n",
      "Name: prediction, dtype: int64\n",
      "1    8159\n",
      "0     417\n",
      "Name: prediction, dtype: int64\n",
      "1    8139\n",
      "0     437\n",
      "Name: prediction, dtype: int64\n",
      "1    8154\n",
      "0     422\n",
      "Name: prediction, dtype: int64\n",
      "1    8146\n",
      "0     430\n",
      "Name: prediction, dtype: int64\n",
      "1    8143\n",
      "0     433\n",
      "Name: prediction, dtype: int64\n",
      "1    8139\n",
      "0     437\n",
      "Name: prediction, dtype: int64\n",
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n",
      "1    8162\n",
      "0     414\n",
      "Name: prediction, dtype: int64\n",
      "1    8154\n",
      "0     422\n",
      "Name: prediction, dtype: int64\n",
      "1    8144\n",
      "0     432\n",
      "Name: prediction, dtype: int64\n",
      "1    8146\n",
      "0     430\n",
      "Name: prediction, dtype: int64\n",
      "1    8164\n",
      "0     412\n",
      "Name: prediction, dtype: int64\n",
      "1    8144\n",
      "0     432\n",
      "Name: prediction, dtype: int64\n",
      "1    8145\n",
      "0     431\n",
      "Name: prediction, dtype: int64\n",
      "1    8151\n",
      "0     425\n",
      "Name: prediction, dtype: int64\n",
      "1    8154\n",
      "0     422\n",
      "Name: prediction, dtype: int64\n",
      "1    8151\n",
      "0     425\n",
      "Name: prediction, dtype: int64\n",
      "1    8153\n",
      "0     423\n",
      "Name: prediction, dtype: int64\n",
      "1    8142\n",
      "0     434\n",
      "Name: prediction, dtype: int64\n",
      "1    8148\n",
      "0     428\n",
      "Name: prediction, dtype: int64\n",
      "1    8156\n",
      "0     420\n",
      "Name: prediction, dtype: int64\n",
      "1    8142\n",
      "0     434\n",
      "Name: prediction, dtype: int64\n",
      "1    8155\n",
      "0     421\n",
      "Name: prediction, dtype: int64\n",
      "1    8156\n",
      "0     420\n",
      "Name: prediction, dtype: int64\n",
      "1    8153\n",
      "0     423\n",
      "Name: prediction, dtype: int64\n",
      "1    8156\n",
      "0     420\n",
      "Name: prediction, dtype: int64\n",
      "1    8146\n",
      "0     430\n",
      "Name: prediction, dtype: int64\n",
      "1    8151\n",
      "0     425\n",
      "Name: prediction, dtype: int64\n",
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n",
      "1    8145\n",
      "0     431\n",
      "Name: prediction, dtype: int64\n",
      "1    8148\n",
      "0     428\n",
      "Name: prediction, dtype: int64\n",
      "1    8156\n",
      "0     420\n",
      "Name: prediction, dtype: int64\n",
      "1    8159\n",
      "0     417\n",
      "Name: prediction, dtype: int64\n",
      "1    8143\n",
      "0     433\n",
      "Name: prediction, dtype: int64\n",
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n",
      "1    8145\n",
      "0     431\n",
      "Name: prediction, dtype: int64\n",
      "1    8133\n",
      "0     443\n",
      "Name: prediction, dtype: int64\n",
      "1    8154\n",
      "0     422\n",
      "Name: prediction, dtype: int64\n",
      "1    8154\n",
      "0     422\n",
      "Name: prediction, dtype: int64\n",
      "1    8155\n",
      "0     421\n",
      "Name: prediction, dtype: int64\n",
      "1    8147\n",
      "0     429\n",
      "Name: prediction, dtype: int64\n",
      "1    8155\n",
      "0     421\n",
      "Name: prediction, dtype: int64\n",
      "1    8157\n",
      "0     419\n",
      "Name: prediction, dtype: int64\n",
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n",
      "1    8151\n",
      "0     425\n",
      "Name: prediction, dtype: int64\n",
      "1    8133\n",
      "0     443\n",
      "Name: prediction, dtype: int64\n",
      "1    8148\n",
      "0     428\n",
      "Name: prediction, dtype: int64\n",
      "1    8140\n",
      "0     436\n",
      "Name: prediction, dtype: int64\n",
      "1    8141\n",
      "0     435\n",
      "Name: prediction, dtype: int64\n",
      "1    8146\n",
      "0     430\n",
      "Name: prediction, dtype: int64\n",
      "1    8151\n",
      "0     425\n",
      "Name: prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "for i in best_df_list:\n",
    "    print(i.prediction.value_counts())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8148\n",
      "0     428\n",
      "Name: prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "best_result = best_df_list[0]\n",
    "\n",
    "for index, row in best_df_list[0].iterrows():\n",
    "    t_row_sum = 0\n",
    "    for i in best_df_list:\n",
    "        t_row_sum += i.loc[index].prediction\n",
    "    if t_row_sum> 50:\n",
    "        best_result.loc[index].prediction = 1\n",
    "    else:\n",
    "        best_result.loc[index].prediction = 0\n",
    "\n",
    "print(best_result.prediction.value_counts())\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8149\n",
      "0     427\n",
      "Name: prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "best_result_from_10 = best_df_list[0]\n",
    "best_model_states = [3,6,12,13,33,37,45,49,60,96,78]\n",
    "for index, row in best_df_list[0].iterrows():\n",
    "    t_row_sum = 0\n",
    "    for i in best_model_states:\n",
    "        t_row_sum += best_df_list[i].loc[index].prediction\n",
    "    if t_row_sum> 5:\n",
    "        best_result_from_10.loc[index].prediction = 1\n",
    "    else:\n",
    "        best_result_from_10.loc[index].prediction = 0\n",
    "print(best_result_from_10.prediction.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original 1    8161\n",
      "0     415\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8161\n",
      "0     415\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9825119236883942\n",
      "state 3\n",
      "\n",
      "original 1    8168\n",
      "0     408\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8163\n",
      "0     413\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9803416048984853\n",
      "state 6\n",
      "\n",
      "original 1    8167\n",
      "0     409\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8157\n",
      "0     419\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9810964083175803\n",
      "state 12\n",
      "\n",
      "original 1    8162\n",
      "0     414\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8150\n",
      "0     426\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9818065751675711\n",
      "state 13\n",
      "\n",
      "original 1    8161\n",
      "0     415\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8161\n",
      "0     415\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9829974811083123\n",
      "state 33\n",
      "\n",
      "original 1    8162\n",
      "0     414\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8162\n",
      "0     414\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9835338822039265\n",
      "state 37\n",
      "\n",
      "original 1    8158\n",
      "0     418\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8152\n",
      "0     424\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9823899371069182\n",
      "state 45\n",
      "\n",
      "original 1    8161\n",
      "0     415\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8163\n",
      "0     413\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9820245979186376\n",
      "state 49\n",
      "\n",
      "original 1    8159\n",
      "0     417\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8163\n",
      "0     413\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9810246679316889\n",
      "state 60\n",
      "\n",
      "original 1    8163\n",
      "0     413\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8158\n",
      "0     418\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9812400635930048\n",
      "state 96\n",
      "\n",
      "original 1    8154\n",
      "0     422\n",
      "Name: prediction, dtype: int64\n",
      "test 1    8166\n",
      "0     410\n",
      "Name: prediction, dtype: int64\n",
      "bestscore 0.9807145115396776\n",
      "state 78\n",
      "\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "best_df_list_bagging = []\n",
    "test_list = []\n",
    "statesList = [3,6,12,13,33,37,45,49,60,96,78]\n",
    "for i in statesList:    \n",
    "    kf = KFold(n_splits=10, random_state=i, shuffle=True)\n",
    "\n",
    "\n",
    "    kf.get_n_splits(features_df)\n",
    "    test_features_df = pd.read_csv(DATA_PATH / \"test.csv\", \n",
    "                                   index_col=\"tripid\")\n",
    "\n",
    "    test_features_df = formatdata(test_features_df)\n",
    "    powerAll(test_features_df, exceptList,power)\n",
    "\n",
    "    bestscore = 0\n",
    "    bestModelXX = \"\"\n",
    "    test_probas_state_best =''\n",
    "\n",
    "\n",
    "    for train_index, test_index in kf.split(features_df):\n",
    "\n",
    "        X_train, X_eval, y_train, y_eval = features_df.iloc[train_index], features_df.iloc[test_index],labels_df.iloc[train_index], labels_df.iloc[test_index]\n",
    "        full_pipeline = Pipeline([\n",
    "            (\"preprocessor\", preprocessor),\n",
    "            (\"estimators\", estimators),\n",
    "        ])\n",
    "        full_pipeline.fit(X_train, y_train)\n",
    "        preds = full_pipeline.predict(X_eval)\n",
    "        y_preds = pd.DataFrame(\n",
    "            preds,\n",
    "            index = y_eval.index\n",
    "        )\n",
    "        test_probas = full_pipeline.predict(test_features_df)\n",
    "        test_preds = pd.DataFrame(\n",
    "            {'prediction':test_probas},\n",
    "            index = test_features_df.index\n",
    "        )\n",
    "        f1 = f1_score(y_eval, y_preds)\n",
    "        if(f1 > bestscore):\n",
    "            bestscore = f1\n",
    "            bestModelXX = full_pipeline\n",
    "            test_probas_state_best = full_pipeline.predict(test_features_df)\n",
    "\n",
    "    test_probas_state_best_test = bestModelXX.predict(test_features_df)\n",
    "    test_preds_state_best = pd.DataFrame(\n",
    "        {'prediction':test_probas_state_best},\n",
    "        index = test_features_df.index\n",
    "    )\n",
    "    test_preds_state_best_test = pd.DataFrame(\n",
    "        {'prediction':test_probas_state_best_test},\n",
    "        index = test_features_df.index\n",
    "    )\n",
    "    best_df_list_bagging.append(test_preds_state_best)\n",
    "    test_list.append(test_preds_state_best_test)\n",
    "    \n",
    "    print(\"original\",test_preds_state_best.prediction.value_counts())\n",
    "    print(\"test\",test_preds_state_best_test.prediction.value_counts())\n",
    "    print('bestscore', bestscore)\n",
    "    print(\"state\", i)\n",
    "    print('')\n",
    "\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8165\n",
      "0     411\n",
      "Name: prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "best_result_from_10_bagging = best_df_list_bagging[0]\n",
    "for index, row in best_df_list_bagging[0].iterrows():\n",
    "    t_row_sum = 0\n",
    "    for i in best_df_list_bagging:\n",
    "        t_row_sum += i.loc[index].prediction\n",
    "    if t_row_sum> 5:\n",
    "        best_result_from_10_bagging.loc[index].prediction = 1\n",
    "    else:\n",
    "        best_result_from_10_bagging.loc[index].prediction = 0\n",
    "print(best_result_from_10_bagging.prediction.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8162\n",
      "0     414\n",
      "Name: prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "best_result_from_10_bagging_test = test_list[0]\n",
    "for index, row in test_list[0].iterrows():\n",
    "    t_row_sum = 0\n",
    "    for i in test_list:\n",
    "        t_row_sum += i.loc[index].prediction\n",
    "    if t_row_sum> 5:\n",
    "        best_result_from_10_bagging_test.loc[index].prediction = 1\n",
    "    else:\n",
    "        best_result_from_10_bagging_test.loc[index].prediction = 0\n",
    "print(best_result_from_10_bagging_test.prediction.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#0.9828134945894335 state 3 k=10\n",
    "#0.9816188326346339 state 6 k=10\n",
    "best_result_from_10_bagging_test.to_csv(DATA_PATH / 'my_submission.csv', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "solver = 'lbfgs'\n",
    "alpha = 0.2\n",
    "hidden_layer_sizes=(10,2 )\n",
    "random_state=2\n",
    "max_iter= 10000000\n",
    "activation='tanh'\n",
    "estimatorsX=MLPClassifier(solver=solver, alpha=alpha, hidden_layer_sizes=hidden_layer_sizes, random_state=random_state,\n",
    "                         max_iter=max_iter, activation=activation)\n",
    "\n",
    "min_child_weight=1.5\n",
    "max_depth=5\n",
    "gamma=2.5\n",
    "colsample_bytree=0.8\n",
    "subsample=0.8\n",
    "n_estimators=200\n",
    "threads = 4\n",
    "estimators = XGBClassifier(min_child_weight=min_child_weight,max_depth=max_depth, \n",
    "                           gamma=gamma,colsample_bytree=colsample_bytree,subsample=subsample,\n",
    "                           n_estimators=n_estimators, n_jobs=threads\n",
    "                          )\n",
    "params = { 'max_features': [0.8,0.85,0.9,0.95,1.0],'max_samples':[0.6,0.65,0.7,0.75,0.8]}\n",
    "\n",
    "#estimatorsY = LGBMClassifier(boosting_type='gbdt', num_leaves=40, max_depth=6,learning_rate=0.2,\n",
    " #                          n_estimators=1000,random_state=42)\n",
    "\n",
    "#estimators = BaggingClassifier(base_estimator=estimatorsY,n_estimators=10,max_samples=0.65,\n",
    " #                            max_features=1.0, random_state=1,bootstrap=False)\n",
    "#estimators = VotingClassifier( [('xgb', estimatorsX), ('mlp', estimatorsY)], weights=[1,1])\n",
    "#estimators = GridSearchCV(estimator=estimatorsB, param_grid=params, cv=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_eval, y_train, y_eval = train_test_split(\n",
    "    features_df,\n",
    "    labels_df,\n",
    "    test_size=0.33,\n",
    "    shuffle=True,\n",
    "    stratify=labels_df,\n",
    "    random_state=2\n",
    ")\n",
    "smt = SMOTE(sampling_strategy='all', random_state=None, k_neighbors=5, n_jobs=1)\n",
    "svmus = SVMSMOTE(sampling_strategy='auto',  k_neighbors=10, random_state=42,\n",
    "                                n_jobs=4, out_step=0.1)\n",
    "smottomek = SMOTETomek(sampling_strategy ='all', random_state=2, n_jobs=1)\n",
    "\n",
    "#X_train, y_train = svmus.fit_resample(X_train, y_train)\n",
    "full_pipeline = Pipeline([\n",
    "    (\"preprocessor\", preprocessor),\n",
    "    (\"estimators\", estimators),\n",
    "])\n",
    "\n",
    "full_pipeline.fit(X_train, y_train)\n",
    "preds = full_pipeline.predict(X_eval)\n",
    "y_preds = pd.DataFrame(\n",
    "    preds,\n",
    "    index = y_eval.index\n",
    ")\n",
    "\n",
    "f1 = f1_score(y_eval, y_preds)\n",
    "print(f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(full_pipeline.named_steps['estimators'].best_params_, full_pipeline.named_steps['estimators'].best_score_)\n",
    "print(f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_features_df = pd.read_csv(DATA_PATH / \"test.csv\", \n",
    "                               index_col=\"tripid\")\n",
    "\n",
    "test_features_df = formatdata(test_features_df)\n",
    "powerAll(test_features_df, exceptList,power)\n",
    "\n",
    "\n",
    "test_probas = full_pipeline.predict(test_features_df)\n",
    "test_preds = pd.DataFrame(\n",
    "    {'prediction':test_probas},\n",
    "    index = test_features_df.index\n",
    ")\n",
    "\n",
    "test_preds.to_csv(DATA_PATH / 'my_submission.csv', index=True)\n",
    "\n",
    "print(confusion_matrix(y_eval, y_preds))\n",
    "print(test_preds.prediction.value_counts())\n",
    "print(f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_eval, y_preds))\n",
    "print(test_preds.prediction.value_counts())\n",
    "print(f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#median\n",
    "#0.9762502413593357\n",
    "#1    8113\n",
    "#0     463\n",
    "#[[ 367  188]\n",
    " #[  58 5056]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mean\n",
    "#0.9754447022428461\n",
    "#1    8124\n",
    "#0     452\n",
    "#[[ 370  185]\n",
    "# [  69 5045]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#most_frequent\n",
    "#0.9740121727369336\n",
    "#1    8126\n",
    "#0     450\n",
    "#[[ 359  196]\n",
    "# [  73 5041]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#constant\n",
    "0.9746713070378964\n",
    "#1    8123\n",
    "#0     453\n",
    "#[[ 366  189]\n",
    "# [  73 5041]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}