Otto Group Product Classification

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import keras\n",
    "from keras.utils import np_utils\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import LabelEncoder \n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import data\n",
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prep_data(data, train=True):\n",
    "    \n",
    "    #if training data...\n",
    "    if train:\n",
    "        \n",
    "        #shuffle data\n",
    "        data = data.reindex(np.random.permutation(data.index))\n",
    "        \n",
    "        #prepare feature/target datasets\n",
    "        X = data.drop(['id','target'],axis=1)\n",
    "        y = data['target']\n",
    "        \n",
    "        #convert type\n",
    "        X = X.astype('float')\n",
    "        \n",
    "        #standardize feature set\n",
    "        scaler = StandardScaler()\n",
    "        scaler.fit(X)\n",
    "        X = scaler.transform(X)\n",
    "        \n",
    "        #encode classes as ints\n",
    "        encoder = LabelEncoder()\n",
    "        encoder.fit(y)\n",
    "        y = encoder.transform(y).astype(np.int32)\n",
    "        \n",
    "        #convert target vector to categorical matrix\n",
    "        y = np_utils.to_categorical(y) \n",
    "        \n",
    "        return [X,y]\n",
    "        \n",
    "    #if testing data...    \n",
    "    else:\n",
    "        \n",
    "        #drop 'id' field\n",
    "        X = data.drop(['id'],axis=1)\n",
    "        \n",
    "        #convert type\n",
    "        X = X.astype('float')\n",
    "        \n",
    "        #standardize feature set\n",
    "        scaler = StandardScaler()\n",
    "        scaler.fit(X)\n",
    "        X = scaler.transform(X)\n",
    "        \n",
    "        return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_data = train.copy()\n",
    "\n",
    "X, y = prep_data(training_data,train=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93 dimensions\n",
      "9 classes\n"
     ]
    }
   ],
   "source": [
    "dims = X.shape[1]\n",
    "print(dims, 'dimensions')\n",
    "\n",
    "nb_classes = len(y[0])\n",
    "print(nb_classes, 'classes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#prep for model\n",
    "fBestModel = 'best_model.h5' \n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=1) \n",
    "best_model = ModelCheckpoint(fBestModel, verbose=0, save_best_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def baseline_model():\n",
    "    #create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(100, input_shape=(dims,)))\n",
    "    model.add(Dense(nb_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(optimizer='sgd', loss='categorical_crossentropy')\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model():\n",
    "    #create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(1000, input_shape=(dims,)))\n",
    "    model.add(Dropout(0.05))\n",
    "    model.add(Dense(1000, activation='relu'))\n",
    "    model.add(Dropout(0.05))\n",
    "    model.add(Dense(100, activation='relu'))\n",
    "    model.add(Dropout(0.05))\n",
    "    model.add(Dense(nb_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(optimizer='sgd', loss='categorical_crossentropy')\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross-Validating Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#cross-validation\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building model...\n",
      "Training cross-validated model...\n",
      "Train on 44706 samples, validate on 7890 samples\n",
      "Epoch 1/100\n",
      "44706/44706 [==============================] - 5s - loss: 1.9829 - val_loss: 1.7555\n",
      "Epoch 2/100\n",
      "44706/44706 [==============================] - 5s - loss: 1.6540 - val_loss: 1.5086\n",
      "Epoch 3/100\n",
      "44706/44706 [==============================] - 5s - loss: 1.4501 - val_loss: 1.3391\n",
      "Epoch 4/100\n",
      "44706/44706 [==============================] - 5s - loss: 1.3075 - val_loss: 1.2172\n",
      "Epoch 5/100\n",
      "44706/44706 [==============================] - 5s - loss: 1.2001 - val_loss: 1.1273\n",
      "Epoch 6/100\n",
      "44706/44706 [==============================] - 5s - loss: 1.1211 - val_loss: 1.0584\n",
      "Epoch 7/100\n",
      "44706/44706 [==============================] - 5s - loss: 1.0598 - val_loss: 1.0042\n",
      "Epoch 8/100\n",
      "44706/44706 [==============================] - 5s - loss: 1.0116 - val_loss: 0.9608\n",
      "Epoch 9/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.9718 - val_loss: 0.9252\n",
      "Epoch 10/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.9393 - val_loss: 0.8954\n",
      "Epoch 11/100\n",
      "44706/44706 [==============================] - 6s - loss: 0.9106 - val_loss: 0.8701\n",
      "Epoch 12/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.8874 - val_loss: 0.8481\n",
      "Epoch 13/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.8664 - val_loss: 0.8290\n",
      "Epoch 14/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.8479 - val_loss: 0.8128\n",
      "Epoch 15/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.8323 - val_loss: 0.7981\n",
      "Epoch 16/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.8193 - val_loss: 0.7853\n",
      "Epoch 17/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.8047 - val_loss: 0.7740\n",
      "Epoch 18/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7934 - val_loss: 0.7635\n",
      "Epoch 19/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7854 - val_loss: 0.7545\n",
      "Epoch 20/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7763 - val_loss: 0.7461\n",
      "Epoch 21/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7674 - val_loss: 0.7386\n",
      "Epoch 22/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7608 - val_loss: 0.7316\n",
      "Epoch 23/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7532 - val_loss: 0.7249\n",
      "Epoch 24/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7486 - val_loss: 0.7192\n",
      "Epoch 25/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7414 - val_loss: 0.7136\n",
      "Epoch 26/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7360 - val_loss: 0.7085\n",
      "Epoch 27/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7312 - val_loss: 0.7039\n",
      "Epoch 28/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7258 - val_loss: 0.6993\n",
      "Epoch 29/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7209 - val_loss: 0.6949\n",
      "Epoch 30/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7167 - val_loss: 0.6909\n",
      "Epoch 31/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7145 - val_loss: 0.6873\n",
      "Epoch 32/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7081 - val_loss: 0.6838\n",
      "Epoch 33/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7040 - val_loss: 0.6806\n",
      "Epoch 34/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.7025 - val_loss: 0.6771\n",
      "Epoch 35/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6982 - val_loss: 0.6739\n",
      "Epoch 36/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6955 - val_loss: 0.6712\n",
      "Epoch 37/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6914 - val_loss: 0.6682\n",
      "Epoch 38/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6882 - val_loss: 0.6657\n",
      "Epoch 39/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6863 - val_loss: 0.6630\n",
      "Epoch 40/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6836 - val_loss: 0.6605\n",
      "Epoch 41/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6811 - val_loss: 0.6580\n",
      "Epoch 42/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6798 - val_loss: 0.6558\n",
      "Epoch 43/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6751 - val_loss: 0.6535\n",
      "Epoch 44/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6734 - val_loss: 0.6514\n",
      "Epoch 45/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6725 - val_loss: 0.6492\n",
      "Epoch 46/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6681 - val_loss: 0.6471\n",
      "Epoch 47/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6652 - val_loss: 0.6457\n",
      "Epoch 48/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6649 - val_loss: 0.6432\n",
      "Epoch 49/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6629 - val_loss: 0.6415\n",
      "Epoch 50/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6611 - val_loss: 0.6398\n",
      "Epoch 51/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6579 - val_loss: 0.6381\n",
      "Epoch 52/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6571 - val_loss: 0.6364\n",
      "Epoch 53/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6555 - val_loss: 0.6345\n",
      "Epoch 54/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6529 - val_loss: 0.6329\n",
      "Epoch 55/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6522 - val_loss: 0.6313\n",
      "Epoch 56/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6492 - val_loss: 0.6299\n",
      "Epoch 57/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6469 - val_loss: 0.6287\n",
      "Epoch 58/100\n",
      "44706/44706 [==============================] - 6s - loss: 0.6471 - val_loss: 0.6272\n",
      "Epoch 59/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6448 - val_loss: 0.6255\n",
      "Epoch 60/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6428 - val_loss: 0.6239\n",
      "Epoch 61/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6416 - val_loss: 0.6230\n",
      "Epoch 62/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6405 - val_loss: 0.6216\n",
      "Epoch 63/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6385 - val_loss: 0.6200\n",
      "Epoch 64/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6377 - val_loss: 0.6188\n",
      "Epoch 65/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6368 - val_loss: 0.6178\n",
      "Epoch 66/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6342 - val_loss: 0.6164\n",
      "Epoch 67/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6332 - val_loss: 0.6153\n",
      "Epoch 68/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6315 - val_loss: 0.6141\n",
      "Epoch 69/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6301 - val_loss: 0.6131\n",
      "Epoch 70/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6304 - val_loss: 0.6123\n",
      "Epoch 71/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6277 - val_loss: 0.6108\n",
      "Epoch 72/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6265 - val_loss: 0.6099\n",
      "Epoch 73/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6270 - val_loss: 0.6091\n",
      "Epoch 74/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6239 - val_loss: 0.6076\n",
      "Epoch 75/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6236 - val_loss: 0.6064\n",
      "Epoch 76/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6226 - val_loss: 0.6055\n",
      "Epoch 77/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6221 - val_loss: 0.6048\n",
      "Epoch 78/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6193 - val_loss: 0.6036\n",
      "Epoch 79/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6176 - val_loss: 0.6026\n",
      "Epoch 80/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6169 - val_loss: 0.6020\n",
      "Epoch 81/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6172 - val_loss: 0.6008\n",
      "Epoch 82/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6161 - val_loss: 0.5998\n",
      "Epoch 83/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6144 - val_loss: 0.5988\n",
      "Epoch 84/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6125 - val_loss: 0.5981\n",
      "Epoch 85/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6112 - val_loss: 0.5969\n",
      "Epoch 86/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6109 - val_loss: 0.5963\n",
      "Epoch 87/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6104 - val_loss: 0.5955\n",
      "Epoch 88/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6089 - val_loss: 0.5949\n",
      "Epoch 89/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6086 - val_loss: 0.5940\n",
      "Epoch 90/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6075 - val_loss: 0.5932\n",
      "Epoch 91/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6074 - val_loss: 0.5926\n",
      "Epoch 92/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6046 - val_loss: 0.5916\n",
      "Epoch 93/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6047 - val_loss: 0.5907\n",
      "Epoch 94/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6028 - val_loss: 0.5898\n",
      "Epoch 95/100\n",
      "44706/44706 [==============================] - 6s - loss: 0.6013 - val_loss: 0.5892\n",
      "Epoch 96/100\n",
      "44706/44706 [==============================] - 6s - loss: 0.6007 - val_loss: 0.5891\n",
      "Epoch 97/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6000 - val_loss: 0.5879\n",
      "Epoch 98/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.6007 - val_loss: 0.5872\n",
      "Epoch 99/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.5989 - val_loss: 0.5865\n",
      "Epoch 100/100\n",
      "44706/44706 [==============================] - 5s - loss: 0.5981 - val_loss: 0.5858\n",
      "CV Training Complete.\n"
     ]
    }
   ],
   "source": [
    "print('Building model...')\n",
    "model = model()\n",
    "\n",
    "print('Training cross-validated model...')\n",
    "model.fit(x_train, y_train, epochs=100,\n",
    "    batch_size=2000, verbose=True, validation_split=0.15, \n",
    "    callbacks=[best_model, early_stop])\n",
    "print('CV Training Complete.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model...\n",
      "8900/9282 [===========================>..] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.590264311460451"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Evaluating model...')\n",
    "model.evaluate(x_test,y_test, batch_size=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data = test.copy()\n",
    "\n",
    "X_test = prep_data(test_data,train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "144000/144368 [============================>.] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "predictions = model.predict_classes(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results = np_utils.to_categorical(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rng = list(range(1,len(results)+1))\n",
    "df['id']= rng\n",
    "df = df.set_index('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cols = []\n",
    "\n",
    "for i in range(1,10):\n",
    "    cols.append(\"Class_\"+str(i))\n",
    "    \n",
    "df.columns = cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('results_7_19_2017_06_00_00.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}