deep_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from PIL import Image\n",
    "from numpy import array\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "# for neural network layers\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "from keras import regularizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(35425,) (35425, 22, 24, 4)\n",
      "(28340, 22, 24, 4)\n",
      "(22672, 22, 24, 4)\n",
      "('x_train shape: ', (22672, 22, 24, 4))\n",
      "(22672, 'train samples')\n",
      "(7085, 'test samples')\n"
     ]
    }
   ],
   "source": [
    "data = np.load(\"zdmdtImages.npz\")\n",
    "print data['Y'][:-3].shape,data['X'].shape\n",
    "\n",
    "batch_size = 32\n",
    "num_classes = 2\n",
    "epochs = 12\n",
    "# splitting data into train and test\n",
    "x_train, x_test, y_train, y_test= train_test_split(data['X'], data['Y'][:-3], test_size=0.20, random_state=42, stratify=data['Y'][:-3])\n",
    "# splitting train into valid dataset\n",
    "print x_train.shape\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.20, random_state=42, stratify=y_train)\n",
    "print x_train.shape\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "print('x_train shape: ', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "y_valid = keras.utils.to_categorical(y_valid, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_4 (Conv2D)            (None, 21, 23, 32)        544       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 11, 12, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 11, 12, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 7, 8, 64)          51264     \n",
      "_________________________________________________________________\n",
      "conv2d_6 (Conv2D)            (None, 3, 4, 128)         204928    \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 1536)              0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 512)               786944    \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 2)                 514       \n",
      "=================================================================\n",
      "Total params: 1,175,522\n",
      "Trainable params: 1,175,522\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 22672 samples, validate on 5668 samples\n",
      "Epoch 1/50\n",
      "22672/22672 [==============================] - 57s 3ms/step - loss: 2.3678 - acc: 0.8665 - val_loss: 2.2512 - val_acc: 0.8679\n",
      "Epoch 2/50\n",
      "22672/22672 [==============================] - 55s 2ms/step - loss: 2.1931 - acc: 0.8679 - val_loss: 2.1616 - val_acc: 0.8679\n",
      "Epoch 3/50\n",
      "22672/22672 [==============================] - 51s 2ms/step - loss: 2.1492 - acc: 0.8679 - val_loss: 2.1420 - val_acc: 0.8679\n",
      "Epoch 4/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 2.1375 - acc: 0.8679 - val_loss: 2.1354 - val_acc: 0.8679\n",
      "Epoch 5/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 2.1331 - acc: 0.8679 - val_loss: 2.1326 - val_acc: 0.8679\n",
      "Epoch 6/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 2.1312 - acc: 0.8679 - val_loss: 2.1314 - val_acc: 0.8679\n",
      "Epoch 7/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 2.1303 - acc: 0.8679 - val_loss: 2.1307 - val_acc: 0.8679\n",
      "Epoch 8/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 2.1299 - acc: 0.8679 - val_loss: 2.1304 - val_acc: 0.8679\n",
      "Epoch 9/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 2.1315 - acc: 0.8679 - val_loss: 2.1535 - val_acc: 0.8679\n",
      "Epoch 10/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 2.1290 - acc: 0.8604 - val_loss: 2.1501 - val_acc: 0.8679\n",
      "Epoch 11/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.4123 - acc: 0.8679 - val_loss: 2.1394 - val_acc: 0.8679\n",
      "Epoch 12/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 0.4002 - acc: 0.8679 - val_loss: 2.1363 - val_acc: 0.8679\n",
      "Epoch 13/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 0.3913 - acc: 0.8679 - val_loss: 2.1197 - val_acc: 0.8679\n",
      "Epoch 14/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.3722 - acc: 0.8686 - val_loss: 2.1469 - val_acc: 0.8613\n",
      "Epoch 15/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 0.3471 - acc: 0.8730 - val_loss: 3.0103 - val_acc: 0.8029\n",
      "Epoch 16/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 0.3343 - acc: 0.8775 - val_loss: 2.5342 - val_acc: 0.8373\n",
      "Epoch 17/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.3284 - acc: 0.8787 - val_loss: 2.4669 - val_acc: 0.8365\n",
      "Epoch 18/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.3219 - acc: 0.8810 - val_loss: 2.5148 - val_acc: 0.8335\n",
      "Epoch 19/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.3204 - acc: 0.8817 - val_loss: 2.1751 - val_acc: 0.8585\n",
      "Epoch 20/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.3126 - acc: 0.8840 - val_loss: 3.1407 - val_acc: 0.7915\n",
      "Epoch 21/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 0.3098 - acc: 0.8873 - val_loss: 1.7498 - val_acc: 0.8890\n",
      "Epoch 22/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.3079 - acc: 0.8872 - val_loss: 2.2436 - val_acc: 0.8546\n",
      "Epoch 23/50\n",
      "22672/22672 [==============================] - 52s 2ms/step - loss: 0.3043 - acc: 0.8882 - val_loss: 1.8118 - val_acc: 0.8841\n",
      "Epoch 24/50\n",
      "22672/22672 [==============================] - 51s 2ms/step - loss: 0.3013 - acc: 0.8884 - val_loss: 1.6793 - val_acc: 0.8933\n",
      "Epoch 25/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 0.3006 - acc: 0.8898 - val_loss: 2.5190 - val_acc: 0.8352\n",
      "Epoch 26/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 0.2999 - acc: 0.8888 - val_loss: 1.7498 - val_acc: 0.8860\n",
      "Epoch 27/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.2979 - acc: 0.8912 - val_loss: 1.6506 - val_acc: 0.8931\n",
      "Epoch 28/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.2947 - acc: 0.8926 - val_loss: 1.7702 - val_acc: 0.8857\n",
      "Epoch 29/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 0.2942 - acc: 0.8934 - val_loss: 2.0058 - val_acc: 0.8705\n",
      "Epoch 30/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.2931 - acc: 0.8927 - val_loss: 2.2403 - val_acc: 0.8525\n",
      "Epoch 31/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.2927 - acc: 0.8924 - val_loss: 2.2085 - val_acc: 0.8578\n",
      "Epoch 32/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.2895 - acc: 0.8937 - val_loss: 2.0965 - val_acc: 0.8619\n",
      "Epoch 33/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.2908 - acc: 0.8929 - val_loss: 1.6992 - val_acc: 0.8903\n",
      "Epoch 34/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 0.2895 - acc: 0.8937 - val_loss: 3.0404 - val_acc: 0.7996\n",
      "Epoch 35/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.2877 - acc: 0.8944 - val_loss: 2.3492 - val_acc: 0.8479\n",
      "Epoch 36/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.2883 - acc: 0.8952 - val_loss: 1.8490 - val_acc: 0.8802\n",
      "Epoch 37/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 0.2883 - acc: 0.8943 - val_loss: 1.9394 - val_acc: 0.8747\n",
      "('Test loss:', 0.2860291074764249)\n",
      "('Test accuracy:', 0.89908256882416504)\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(2, 2),\n",
    "                 activation = 'relu',\n",
    "                 input_shape = (22, 24, 4)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2), padding=\"same\"))\n",
    "model.add(Dropout(0.1))\n",
    "model.add(Conv2D(64,kernel_size=(5, 5), activation = 'relu'))\n",
    "model.add(Conv2D(128,kernel_size=(5, 5), activation = 'relu'))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512, activation='relu',kernel_regularizer=regularizers.l2(0.01)))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "model.summary()\n",
    "class_weight = {0: 1.,\n",
    "                1: 1.}\n",
    "\n",
    "#try mean squared error for loss, patience to 10, kernel size consistent to (3,3), categorical crossentropy loss\n",
    "# and look at upsilon on github, increase epocs marginally, binary classification for other classes, \n",
    "# the number of color filters should be 3 instead of 4, try taking images directly as input to NN\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adagrad(),\n",
    "              metrics=['accuracy'])\n",
    "earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',min_delta=1e-6,patience=10,verbose=0, mode='auto')\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=128,\n",
    "          epochs=50,\n",
    "          verbose=1,\n",
    "          validation_data=(x_valid,y_valid),\n",
    "          callbacks=[earlyStopping],\n",
    "         class_weight=class_weight)\n",
    "\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6015,  134],\n",
       "       [ 581,  355]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ypred = model.predict(x_test)\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_test.argmax(axis=1),ypred.argmax(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.91191631,  0.72597137]),\n",
       " array([ 0.97820784,  0.3792735 ]),\n",
       " array([ 0.94389957,  0.49824561]),\n",
       " array([6149,  936]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "precision_recall_fscore_support(y_test.argmax(axis=1), ypred.argmax(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_13 (Conv2D)           (None, 20, 22, 32)        1184      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2 (None, 10, 11, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 10, 11, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, 6, 7, 64)          51264     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_12 (LeakyReLU)   (None, 6, 7, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, 2, 3, 128)         204928    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_13 (LeakyReLU)   (None, 2, 3, 128)         0         \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, 768)               0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 512)               393728    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_14 (LeakyReLU)   (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 2)                 514       \n",
      "=================================================================\n",
      "Total params: 782,946\n",
      "Trainable params: 782,946\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 22672 samples, validate on 5668 samples\n",
      "Epoch 1/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 2.2888 - acc: 0.8631 - val_loss: 2.1419 - val_acc: 0.8679\n",
      "Epoch 2/50\n",
      "22672/22672 [==============================] - 45s 2ms/step - loss: 2.1374 - acc: 0.8679 - val_loss: 2.1357 - val_acc: 0.8679\n",
      "Epoch 3/50\n",
      "22672/22672 [==============================] - 45s 2ms/step - loss: 2.1337 - acc: 0.8679 - val_loss: 2.1333 - val_acc: 0.8679\n",
      "Epoch 4/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 2.1320 - acc: 0.8679 - val_loss: 2.1321 - val_acc: 0.8679\n",
      "Epoch 5/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 2.1310 - acc: 0.8679 - val_loss: 2.1314 - val_acc: 0.8679\n",
      "Epoch 6/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 1.5065 - acc: 0.8637 - val_loss: 2.1398 - val_acc: 0.8679\n",
      "Epoch 7/50\n",
      "22672/22672 [==============================] - 45s 2ms/step - loss: 0.3865 - acc: 0.8679 - val_loss: 2.1369 - val_acc: 0.8679\n",
      "Epoch 8/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.3438 - acc: 0.8691 - val_loss: 2.2790 - val_acc: 0.8559\n",
      "Epoch 9/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.3248 - acc: 0.8792 - val_loss: 1.8025 - val_acc: 0.8878\n",
      "Epoch 10/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.3154 - acc: 0.8816 - val_loss: 1.7154 - val_acc: 0.8913\n",
      "Epoch 11/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.3085 - acc: 0.8857 - val_loss: 1.8488 - val_acc: 0.8830\n",
      "Epoch 12/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.3022 - acc: 0.8883 - val_loss: 1.9878 - val_acc: 0.8746\n",
      "Epoch 13/50\n",
      "22672/22672 [==============================] - 46s 2ms/step - loss: 0.2982 - acc: 0.8908 - val_loss: 1.7706 - val_acc: 0.8881\n",
      "Epoch 14/50\n",
      "22672/22672 [==============================] - 45s 2ms/step - loss: 0.2929 - acc: 0.8931 - val_loss: 1.6625 - val_acc: 0.8940\n",
      "Epoch 15/50\n",
      "22672/22672 [==============================] - 45s 2ms/step - loss: 0.2915 - acc: 0.8938 - val_loss: 1.7715 - val_acc: 0.8892\n",
      "Epoch 16/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.2865 - acc: 0.8962 - val_loss: 1.7261 - val_acc: 0.8910\n",
      "Epoch 17/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.2853 - acc: 0.8968 - val_loss: 2.6289 - val_acc: 0.8296\n",
      "Epoch 18/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 0.2809 - acc: 0.8993 - val_loss: 1.7423 - val_acc: 0.8906\n",
      "Epoch 19/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.2793 - acc: 0.8993 - val_loss: 1.5671 - val_acc: 0.9007\n",
      "Epoch 20/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 0.2776 - acc: 0.8998 - val_loss: 1.6185 - val_acc: 0.8970\n",
      "Epoch 21/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.2755 - acc: 0.9007 - val_loss: 1.6035 - val_acc: 0.8987\n",
      "Epoch 22/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.2728 - acc: 0.9025 - val_loss: 1.5606 - val_acc: 0.9016\n",
      "Epoch 23/50\n",
      "22672/22672 [==============================] - 50s 2ms/step - loss: 0.2713 - acc: 0.9015 - val_loss: 1.9392 - val_acc: 0.8749\n",
      "Epoch 24/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 0.2706 - acc: 0.9034 - val_loss: 1.6507 - val_acc: 0.8952\n",
      "Epoch 25/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.2678 - acc: 0.9026 - val_loss: 2.1253 - val_acc: 0.8638\n",
      "Epoch 26/50\n",
      "22672/22672 [==============================] - 49s 2ms/step - loss: 0.2664 - acc: 0.9025 - val_loss: 2.3581 - val_acc: 0.8483\n",
      "Epoch 27/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.2638 - acc: 0.9053 - val_loss: 1.6425 - val_acc: 0.8952\n",
      "Epoch 28/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.2629 - acc: 0.9061 - val_loss: 1.6903 - val_acc: 0.8927\n",
      "Epoch 29/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.2624 - acc: 0.9056 - val_loss: 1.6464 - val_acc: 0.8940\n",
      "Epoch 30/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 0.2606 - acc: 0.9054 - val_loss: 1.6808 - val_acc: 0.8922\n",
      "Epoch 31/50\n",
      "22672/22672 [==============================] - 48s 2ms/step - loss: 0.2602 - acc: 0.9075 - val_loss: 1.9361 - val_acc: 0.8731\n",
      "Epoch 32/50\n",
      "22672/22672 [==============================] - 47s 2ms/step - loss: 0.2562 - acc: 0.9074 - val_loss: 1.7101 - val_acc: 0.8892\n",
      "('Test loss:', 0.26663793237931138)\n",
      "('Test accuracy:', 0.90458715597171557)\n"
     ]
    }
   ],
   "source": [
    "# new attempt to deep network\n",
    "from keras.layers.advanced_activations import LeakyReLU, PReLU\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(3, 3),\n",
    "                 activation = 'relu',\n",
    "                 input_shape = (22, 24, 4)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2), padding=\"same\"))\n",
    "model.add(Dropout(0.1))\n",
    "model.add(Conv2D(64,kernel_size=(5, 5), activation = 'relu'))\n",
    "model.add(LeakyReLU(alpha=0.1))\n",
    "model.add(Conv2D(128,kernel_size=(5, 5), activation = 'relu'))\n",
    "model.add(LeakyReLU(alpha=0.1))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512, activation='relu',kernel_regularizer=regularizers.l2(0.01)))\n",
    "model.add(LeakyReLU(alpha=0.1))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "model.summary()\n",
    "\n",
    "#try mean squared error for loss, patience to 10, kernel size consistent to (3,3), categorical crossentropy loss\n",
    "# and look at upsilon on github, increase epocs marginally, binary classification for \n",
    "# other classes, the number of color filters should be 3 instead of 4, try taking \n",
    "# images directly as input to NN\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adagrad(),\n",
    "              metrics=['accuracy'])\n",
    "earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',min_delta=1e-6,patience=10,verbose=0, mode='auto')\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=128,\n",
    "          epochs=50,\n",
    "          verbose=1,\n",
    "          validation_data=(x_valid,y_valid),\n",
    "          callbacks=[earlyStopping])\n",
    "\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6008,  141],\n",
       "       [ 535,  401]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ypred = model.predict(x_test)\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_test.argmax(axis=1),ypred.argmax(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.9256917 ,  0.61315789]),\n",
       " array([ 0.95218735,  0.49786325]),\n",
       " array([ 0.93875261,  0.5495283 ]),\n",
       " array([6149,  936]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "precision_recall_fscore_support(y_test.argmax(axis=1), ypred.argmax(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'numpy.ndarray' object has no attribute 'history'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-78fad3019956>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mu'matplotlib inline'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0maccuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mval_accuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'history'"
     ]
    }
   ],
   "source": [
    "# To be tried ...\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "accuracy = x_train.history['acc']\n",
    "val_accuracy = x_train.history['val_acc']\n",
    "loss = x_train.history['loss']\n",
    "val_loss = x_train.history['val_loss']\n",
    "epochs = range(len(accuracy))\n",
    "plt.plot(epochs, accuracy, 'bo', label='Training accuracy')\n",
    "plt.plot(epochs, val_accuracy, 'b', label='Validation accuracy')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}