optimized CNN - SampleDataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pretrained model only"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "train_filename = \"data_preprocessed/train_data_sample_rgb.p\"\n",
    "(train_labels, train_data, train_tensors) = pickle.load(open(train_filename, mode='rb'))\n",
    "\n",
    "valid_filename = \"data_preprocessed/valid_data_sample_rgb.p\"\n",
    "(valid_labels, valid_data, valid_tensors) = pickle.load(open(valid_filename, mode='rb'))\n",
    "\n",
    "test_filename = \"data_preprocessed/test_data_sample_rgb.p\"\n",
    "(test_labels, test_data, test_tensors) = pickle.load(open(test_filename, mode='rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 64, 64, 3)         0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 64, 64, 64)        1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 64, 64, 64)        36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 32, 32, 64)        0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 32, 32, 128)       73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 32, 32, 128)       147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 16, 16, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 16, 16, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 16, 16, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 8, 8, 512)         1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 8, 8, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 8, 8, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 4, 4, 512)         0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 2, 2, 512)         0         \n",
      "_________________________________________________________________\n",
      "sequential_3 (Sequential)    (None, 1)                 537445    \n",
      "=================================================================\n",
      "Total params: 15,252,133\n",
      "Trainable params: 15,252,133\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_3 (Flatten)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 256)               524544    \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 50)                12850     \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 1)                 51        \n",
      "=================================================================\n",
      "Total params: 537,445\n",
      "Trainable params: 537,445\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, Dropout, Flatten, Dense\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras import regularizers, applications, optimizers, initializers\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "# VGG16\n",
    "# resnet50.ResNet50\n",
    "# inception_v3.InceptionV3 299x299\n",
    "# inception_resnet_v2.InceptionResNetV2 299x299\n",
    "\n",
    "base_model = applications.VGG16(weights='imagenet', \n",
    "                                include_top=False, \n",
    "                                input_shape=train_tensors.shape[1:])\n",
    "\n",
    "add_model = Sequential()\n",
    "add_model.add(Flatten(input_shape=base_model.output_shape[1:]))\n",
    "# add_model.add(Conv2D(filters=512, \n",
    "#                  kernel_size=4, \n",
    "#                  strides=2, \n",
    "# #                  kernel_regularizer=regularizers.l2(0.01),\n",
    "# #                  activity_regularizer=regularizers.l1(0.01),\n",
    "#                  kernel_initializer=initializers.random_normal(stddev=0.01), \n",
    "#                  padding='same', \n",
    "#                  activation='relu', \n",
    "#                  input_shape=base_model.output_shape[1:]))\n",
    "# # add_model.add(MaxPooling2D(pool_size=2))\n",
    "# add_model.add(BatchNormalization())\n",
    "# add_model.add(Flatten())\n",
    "# add_model.add(Dropout(0.2))\n",
    "# add_model.add(Dense(1024, activation='relu'))\n",
    "add_model.add(Dropout(0.2))\n",
    "add_model.add(Dense(256, activation='relu'))\n",
    "add_model.add(Dropout(0.2))\n",
    "add_model.add(Dense(50, activation='relu'))\n",
    "add_model.add(Dropout(0.2))\n",
    "add_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model = Model(inputs=base_model.input, outputs=add_model(base_model.output))\n",
    "\n",
    "model.summary()\n",
    "add_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def binary_accuracy(y_true, y_pred):\n",
    "    return K.mean(K.equal(y_true, K.round(y_pred)))\n",
    "\n",
    "def precision_threshold(threshold = 0.5):\n",
    "    def precision(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(y_pred)\n",
    "        precision_ratio = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision_ratio\n",
    "    return precision\n",
    "\n",
    "def recall_threshold(threshold = 0.5):\n",
    "    def recall(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.clip(y_true, 0, 1))\n",
    "        recall_ratio = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall_ratio\n",
    "    return recall\n",
    "\n",
    "def fbeta_score_threshold(beta = 1, threshold = 0.5):\n",
    "    def fbeta_score(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        beta_value = beta\n",
    "        p = precision_threshold(threshold_value)(y_true, y_pred)\n",
    "        r = recall_threshold(threshold_value)(y_true, y_pred)\n",
    "        bb = beta_value ** 2\n",
    "        fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
    "        return fbeta_score\n",
    "    return fbeta_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras.backend as K\n",
    "\n",
    "model.compile(optimizer=optimizers.SGD(lr=1e-4, decay=1e-6, momentum=0.9, nesterov=True), \n",
    "              loss='binary_crossentropy', \n",
    "              metrics=[binary_accuracy,\n",
    "                       precision_threshold(threshold = 0.4), \n",
    "                       recall_threshold(threshold = 0.4), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.4),\n",
    "                      precision_threshold(threshold = 0.5), \n",
    "                       recall_threshold(threshold = 0.5), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.5),\n",
    "                      precision_threshold(threshold = 0.6), \n",
    "                       recall_threshold(threshold = 0.6), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.6)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6447 - binary_accuracy: 0.6342 - precision_1: 0.5578 - recall_1: 0.7575 - fbeta_score_1: 0.5836 - precision_2: 0.6173 - recall_2: 0.5807 - fbeta_score_2: 0.5958 - precision_3: 0.6798 - recall_3: 0.3690 - fbeta_score_3: 0.5517Epoch 00001: val_loss improved from inf to 0.62077, saving model to saved_models/pretrainedVGG.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 16s 151ms/step - loss: 0.6449 - binary_accuracy: 0.6333 - precision_1: 0.5553 - recall_1: 0.7575 - fbeta_score_1: 0.5813 - precision_2: 0.6140 - recall_2: 0.5799 - fbeta_score_2: 0.5929 - precision_3: 0.6775 - recall_3: 0.3703 - fbeta_score_3: 0.5508 - val_loss: 0.6208 - val_binary_accuracy: 0.6691 - val_precision_1: 0.6438 - val_recall_1: 0.6767 - val_fbeta_score_1: 0.6444 - val_precision_2: 0.7247 - val_recall_2: 0.4713 - val_fbeta_score_2: 0.6413 - val_precision_3: 0.7505 - val_recall_3: 0.2084 - val_fbeta_score_3: 0.4703\n",
      "Epoch 2/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6388 - binary_accuracy: 0.6497 - precision_1: 0.5654 - recall_1: 0.7681 - fbeta_score_1: 0.5930 - precision_2: 0.6397 - recall_2: 0.5754 - fbeta_score_2: 0.6183 - precision_3: 0.6734 - recall_3: 0.3287 - fbeta_score_3: 0.5388Epoch 00002: val_loss improved from 0.62077 to 0.60743, saving model to saved_models/pretrainedVGG.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 16s 150ms/step - loss: 0.6389 - binary_accuracy: 0.6498 - precision_1: 0.5637 - recall_1: 0.7693 - fbeta_score_1: 0.5916 - precision_2: 0.6381 - recall_2: 0.5766 - fbeta_score_2: 0.6172 - precision_3: 0.6718 - recall_3: 0.3303 - fbeta_score_3: 0.5385 - val_loss: 0.6074 - val_binary_accuracy: 0.6900 - val_precision_1: 0.6501 - val_recall_1: 0.6871 - val_fbeta_score_1: 0.6507 - val_precision_2: 0.6961 - val_recall_2: 0.5752 - val_fbeta_score_2: 0.6631 - val_precision_3: 0.7252 - val_recall_3: 0.3738 - val_fbeta_score_3: 0.5960\n",
      "Epoch 3/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6402 - binary_accuracy: 0.6411 - precision_1: 0.5665 - recall_1: 0.7516 - fbeta_score_1: 0.5892 - precision_2: 0.6189 - recall_2: 0.5729 - fbeta_score_2: 0.5971 - precision_3: 0.6701 - recall_3: 0.3696 - fbeta_score_3: 0.5557Epoch 00003: val_loss did not improve\n",
      "106/106 [==============================] - 15s 146ms/step - loss: 0.6397 - binary_accuracy: 0.6418 - precision_1: 0.5671 - recall_1: 0.7529 - fbeta_score_1: 0.5899 - precision_2: 0.6201 - recall_2: 0.5741 - fbeta_score_2: 0.5985 - precision_3: 0.6708 - recall_3: 0.3694 - fbeta_score_3: 0.5562 - val_loss: 0.6100 - val_binary_accuracy: 0.6900 - val_precision_1: 0.6402 - val_recall_1: 0.7095 - val_fbeta_score_1: 0.6472 - val_precision_2: 0.7095 - val_recall_2: 0.5528 - val_fbeta_score_2: 0.6644 - val_precision_3: 0.7375 - val_recall_3: 0.3093 - val_fbeta_score_3: 0.5624\n",
      "Epoch 4/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6365 - binary_accuracy: 0.6488 - precision_1: 0.5764 - recall_1: 0.7591 - fbeta_score_1: 0.6002 - precision_2: 0.6334 - recall_2: 0.5839 - fbeta_score_2: 0.6139 - precision_3: 0.6809 - recall_3: 0.3726 - fbeta_score_3: 0.5618Epoch 00004: val_loss did not improve\n",
      "106/106 [==============================] - 15s 146ms/step - loss: 0.6365 - binary_accuracy: 0.6483 - precision_1: 0.5758 - recall_1: 0.7600 - fbeta_score_1: 0.5999 - precision_2: 0.6324 - recall_2: 0.5844 - fbeta_score_2: 0.6133 - precision_3: 0.6807 - recall_3: 0.3732 - fbeta_score_3: 0.5622 - val_loss: 0.6086 - val_binary_accuracy: 0.6900 - val_precision_1: 0.6368 - val_recall_1: 0.7110 - val_fbeta_score_1: 0.6449 - val_precision_2: 0.6963 - val_recall_2: 0.5750 - val_fbeta_score_2: 0.6627 - val_precision_3: 0.7376 - val_recall_3: 0.3423 - val_fbeta_score_3: 0.5847\n",
      "Epoch 5/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6323 - binary_accuracy: 0.6586 - precision_1: 0.5755 - recall_1: 0.7596 - fbeta_score_1: 0.6006 - precision_2: 0.6443 - recall_2: 0.5963 - fbeta_score_2: 0.6254 - precision_3: 0.7007 - recall_3: 0.3983 - fbeta_score_3: 0.5891Epoch 00005: val_loss did not improve\n",
      "106/106 [==============================] - 16s 147ms/step - loss: 0.6319 - binary_accuracy: 0.6595 - precision_1: 0.5742 - recall_1: 0.7592 - fbeta_score_1: 0.5995 - precision_2: 0.6442 - recall_2: 0.5966 - fbeta_score_2: 0.6255 - precision_3: 0.6995 - recall_3: 0.3980 - fbeta_score_3: 0.5884 - val_loss: 0.6141 - val_binary_accuracy: 0.6873 - val_precision_1: 0.6662 - val_recall_1: 0.6584 - val_fbeta_score_1: 0.6585 - val_precision_2: 0.7181 - val_recall_2: 0.5290 - val_fbeta_score_2: 0.6610 - val_precision_3: 0.7746 - val_recall_3: 0.2935 - val_fbeta_score_3: 0.5549\n",
      "Epoch 6/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6272 - binary_accuracy: 0.6580 - precision_1: 0.5809 - recall_1: 0.7516 - fbeta_score_1: 0.6023 - precision_2: 0.6481 - recall_2: 0.6005 - fbeta_score_2: 0.6271 - precision_3: 0.6957 - recall_3: 0.3897 - fbeta_score_3: 0.5818Epoch 00006: val_loss improved from 0.60743 to 0.60369, saving model to saved_models/pretrainedVGG.best.from_scratch.hdf5\n",
      "106/106 [==============================] - 16s 151ms/step - loss: 0.6270 - binary_accuracy: 0.6580 - precision_1: 0.5804 - recall_1: 0.7517 - fbeta_score_1: 0.6019 - precision_2: 0.6476 - recall_2: 0.5992 - fbeta_score_2: 0.6265 - precision_3: 0.6967 - recall_3: 0.3890 - fbeta_score_3: 0.5820 - val_loss: 0.6037 - val_binary_accuracy: 0.6955 - val_precision_1: 0.6374 - val_recall_1: 0.7146 - val_fbeta_score_1: 0.6451 - val_precision_2: 0.7027 - val_recall_2: 0.5904 - val_fbeta_score_2: 0.6715 - val_precision_3: 0.7477 - val_recall_3: 0.3676 - val_fbeta_score_3: 0.6047\n",
      "Epoch 7/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6287 - binary_accuracy: 0.6491 - precision_1: 0.5690 - recall_1: 0.7633 - fbeta_score_1: 0.5947 - precision_2: 0.6276 - recall_2: 0.5909 - fbeta_score_2: 0.6115 - precision_3: 0.7034 - recall_3: 0.3800 - fbeta_score_3: 0.5773Epoch 00007: val_loss did not improve\n",
      "106/106 [==============================] - 16s 147ms/step - loss: 0.6287 - binary_accuracy: 0.6486 - precision_1: 0.5692 - recall_1: 0.7636 - fbeta_score_1: 0.5950 - precision_2: 0.6270 - recall_2: 0.5910 - fbeta_score_2: 0.6111 - precision_3: 0.7024 - recall_3: 0.3802 - fbeta_score_3: 0.5770 - val_loss: 0.6064 - val_binary_accuracy: 0.6945 - val_precision_1: 0.6398 - val_recall_1: 0.6906 - val_fbeta_score_1: 0.6430 - val_precision_2: 0.7121 - val_recall_2: 0.5702 - val_fbeta_score_2: 0.6710 - val_precision_3: 0.7597 - val_recall_3: 0.3315 - val_fbeta_score_3: 0.5841\n",
      "Epoch 8/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6353 - binary_accuracy: 0.6443 - precision_1: 0.5693 - recall_1: 0.7463 - fbeta_score_1: 0.5927 - precision_2: 0.6201 - recall_2: 0.5853 - fbeta_score_2: 0.6030 - precision_3: 0.6542 - recall_3: 0.3590 - fbeta_score_3: 0.5438Epoch 00008: val_loss did not improve\n",
      "106/106 [==============================] - 16s 147ms/step - loss: 0.6354 - binary_accuracy: 0.6436 - precision_1: 0.5686 - recall_1: 0.7474 - fbeta_score_1: 0.5922 - precision_2: 0.6192 - recall_2: 0.5867 - fbeta_score_2: 0.6026 - precision_3: 0.6548 - recall_3: 0.3619 - fbeta_score_3: 0.5453 - val_loss: 0.6136 - val_binary_accuracy: 0.6718 - val_precision_1: 0.5468 - val_recall_1: 0.8886 - val_fbeta_score_1: 0.5892 - val_precision_2: 0.6273 - val_recall_2: 0.7364 - val_fbeta_score_2: 0.6410 - val_precision_3: 0.7157 - val_recall_3: 0.5036 - val_fbeta_score_3: 0.6514\n",
      "Epoch 9/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6188 - binary_accuracy: 0.6634 - precision_1: 0.5960 - recall_1: 0.7756 - fbeta_score_1: 0.6189 - precision_2: 0.6477 - recall_2: 0.6233 - fbeta_score_2: 0.6324 - precision_3: 0.6975 - recall_3: 0.4358 - fbeta_score_3: 0.6031Epoch 00009: val_loss did not improve\n",
      "106/106 [==============================] - 16s 148ms/step - loss: 0.6189 - binary_accuracy: 0.6624 - precision_1: 0.5939 - recall_1: 0.7767 - fbeta_score_1: 0.6171 - precision_2: 0.6445 - recall_2: 0.6217 - fbeta_score_2: 0.6295 - precision_3: 0.6945 - recall_3: 0.4349 - fbeta_score_3: 0.6008 - val_loss: 0.6056 - val_binary_accuracy: 0.6882 - val_precision_1: 0.6470 - val_recall_1: 0.6939 - val_fbeta_score_1: 0.6494 - val_precision_2: 0.6987 - val_recall_2: 0.5629 - val_fbeta_score_2: 0.6608 - val_precision_3: 0.7435 - val_recall_3: 0.3439 - val_fbeta_score_3: 0.5853\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/20\n",
      "105/106 [============================>.] - ETA: 0s - loss: 0.6242 - binary_accuracy: 0.6625 - precision_1: 0.5819 - recall_1: 0.7703 - fbeta_score_1: 0.6074 - precision_2: 0.6422 - recall_2: 0.6125 - fbeta_score_2: 0.6281 - precision_3: 0.6840 - recall_3: 0.3855 - fbeta_score_3: 0.5763Epoch 00010: val_loss did not improve\n",
      "106/106 [==============================] - 16s 148ms/step - loss: 0.6242 - binary_accuracy: 0.6627 - precision_1: 0.5822 - recall_1: 0.7684 - fbeta_score_1: 0.6074 - precision_2: 0.6440 - recall_2: 0.6100 - fbeta_score_2: 0.6284 - precision_3: 0.6854 - recall_3: 0.3852 - fbeta_score_3: 0.5770 - val_loss: 0.6050 - val_binary_accuracy: 0.6900 - val_precision_1: 0.6509 - val_recall_1: 0.6840 - val_fbeta_score_1: 0.6516 - val_precision_2: 0.6994 - val_recall_2: 0.5737 - val_fbeta_score_2: 0.6636 - val_precision_3: 0.7384 - val_recall_3: 0.3724 - val_fbeta_score_3: 0.6039\n",
      "Epoch 00010: early stopping\n",
      "training time: 2.62 minutes\n"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping\n",
    "import numpy as np\n",
    "\n",
    "epochs = 20\n",
    "batch_size = 32\n",
    "\n",
    "earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=4, verbose=1, mode='auto')\n",
    "log = CSVLogger('saved_models/log_pretrained_CNN.csv')\n",
    "checkpointer = ModelCheckpoint(filepath='saved_models/pretrainedVGG.best.from_scratch.hdf5', \n",
    "                               verbose=1, save_best_only=True)\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "# model.fit(train_tensors, train_labels, \n",
    "#           validation_data=(valid_tensors, valid_labels),\n",
    "#           epochs=epochs, batch_size=batch_size, callbacks=[checkpointer, log, earlystop], verbose=1)\n",
    "\n",
    "def train_generator(x, y, batch_size):\n",
    "        train_datagen = ImageDataGenerator(\n",
    "                        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "                        samplewise_center=False,  # set each sample mean to 0\n",
    "                        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "                        samplewise_std_normalization=False,  # divide each input by its std\n",
    "                        zca_whitening=False,  # apply ZCA whitening\n",
    "                        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "                        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n",
    "                        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n",
    "                        horizontal_flip=True,  # randomly flip images\n",
    "                        vertical_flip=False)  # randomly flip images\n",
    "        generator = train_datagen.flow(x, y, batch_size=batch_size)\n",
    "        while 1:\n",
    "            x_batch, y_batch = generator.next()\n",
    "            yield [x_batch, y_batch]\n",
    "\n",
    "# Training with data augmentation. If shift_fraction=0., also no augmentation.\n",
    "model.fit_generator(generator=train_generator(train_tensors, train_labels, batch_size),\n",
    "                    steps_per_epoch=int(train_labels.shape[0] / batch_size),\n",
    "                    validation_data=(valid_tensors, valid_labels),\n",
    "                    epochs=epochs, callbacks=[checkpointer, log, earlystop], verbose=1)\n",
    "\n",
    "# Show total training time\n",
    "print(\"training time: %.2f minutes\"%((time.time()-start)/60))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('saved_models/pretrainedVGG.best.from_scratch.hdf5')\n",
    "prediction = model.predict(test_tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.644550 %\n",
      "Recall: 0.555102 %\n",
      "Fscore: 0.624426 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.66696835"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.eval(binary_accuracy(K.variable(value=test_labels),\n",
    "                       K.variable(value=prediction)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.48835951],\n",
       "       [ 0.26106429],\n",
       "       [ 0.41754654],\n",
       "       [ 0.64929706],\n",
       "       [ 0.32506767],\n",
       "       [ 0.60315681],\n",
       "       [ 0.25606936],\n",
       "       [ 0.56809974],\n",
       "       [ 0.21197747],\n",
       "       [ 0.29700339],\n",
       "       [ 0.45825043],\n",
       "       [ 0.699117  ],\n",
       "       [ 0.19021288],\n",
       "       [ 0.15274318],\n",
       "       [ 0.57559574],\n",
       "       [ 0.66964573],\n",
       "       [ 0.4788951 ],\n",
       "       [ 0.63720137],\n",
       "       [ 0.57080215],\n",
       "       [ 0.19700085],\n",
       "       [ 0.49120948],\n",
       "       [ 0.79610401],\n",
       "       [ 0.2416992 ],\n",
       "       [ 0.60802132],\n",
       "       [ 0.56850374],\n",
       "       [ 0.55580062],\n",
       "       [ 0.72331935],\n",
       "       [ 0.37036556],\n",
       "       [ 0.64609927],\n",
       "       [ 0.49083158]], dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction[:30]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extra data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "train_filename = \"data_preprocessed/train_data_sample_rgb.p\"\n",
    "(train_labels, train_data, train_tensors) = pickle.load(open(train_filename, mode='rb'))\n",
    "\n",
    "valid_filename = \"data_preprocessed/valid_data_sample_rgb.p\"\n",
    "(valid_labels, valid_data, valid_tensors) = pickle.load(open(valid_filename, mode='rb'))\n",
    "\n",
    "test_filename = \"data_preprocessed/test_data_sample_rgb.p\"\n",
    "(test_labels, test_data, test_tensors) = pickle.load(open(test_filename, mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_6 (InputLayer)            (None, 64, 64, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1 (Conv2D)           (None, 64, 64, 64)   1792        input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2 (Conv2D)           (None, 64, 64, 64)   36928       block1_conv1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1_pool (MaxPooling2D)      (None, 32, 32, 64)   0           block1_conv2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2_conv1 (Conv2D)           (None, 32, 32, 128)  73856       block1_pool[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block2_conv2 (Conv2D)           (None, 32, 32, 128)  147584      block2_conv1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block2_pool (MaxPooling2D)      (None, 16, 16, 128)  0           block2_conv2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block3_conv1 (Conv2D)           (None, 16, 16, 256)  295168      block2_pool[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block3_conv2 (Conv2D)           (None, 16, 16, 256)  590080      block3_conv1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block3_conv3 (Conv2D)           (None, 16, 16, 256)  590080      block3_conv2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block3_pool (MaxPooling2D)      (None, 8, 8, 256)    0           block3_conv3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block4_conv1 (Conv2D)           (None, 8, 8, 512)    1180160     block3_pool[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block4_conv2 (Conv2D)           (None, 8, 8, 512)    2359808     block4_conv1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block4_conv3 (Conv2D)           (None, 8, 8, 512)    2359808     block4_conv2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block4_pool (MaxPooling2D)      (None, 4, 4, 512)    0           block4_conv3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block5_conv1 (Conv2D)           (None, 4, 4, 512)    2359808     block4_pool[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block5_conv2 (Conv2D)           (None, 4, 4, 512)    2359808     block5_conv1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block5_conv3 (Conv2D)           (None, 4, 4, 512)    2359808     block5_conv2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block5_pool (MaxPooling2D)      (None, 2, 2, 512)    0           block5_conv3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "sequential_5 (Sequential)       (None, 2048)         0           block5_pool[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "input_7 (InputLayer)            (None, 5)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 2053)         0           sequential_5[1][0]               \n",
      "                                                                 input_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_12 (Dropout)            (None, 2053)         0           concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 256)          525824      dropout_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_13 (Dropout)            (None, 256)          0           dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 1)            257         dropout_13[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 15,240,769\n",
      "Trainable params: 15,240,769\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:25: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Tensor(\"in..., inputs=Tensor(\"in...)`\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:39: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Tensor(\"de..., inputs=[<tf.Tenso...)`\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D\n",
    "from keras.layers import Dropout, Flatten, Dense\n",
    "from keras.models import Sequential\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras import regularizers\n",
    "from keras import applications\n",
    "from keras.models import Model\n",
    "from keras import optimizers\n",
    "from keras.layers import Input, merge, concatenate\n",
    "\n",
    "base_model = applications.VGG16(weights='imagenet', \n",
    "                                include_top=False, \n",
    "                                input_shape=train_tensors.shape[1:])\n",
    "\n",
    "add_model = Sequential()\n",
    "add_model.add(Flatten(input_shape=base_model.output_shape[1:]))\n",
    "\n",
    "added_model = Model(inputs=base_model.input, outputs=add_model(base_model.output))\n",
    "\n",
    "\n",
    "inp = Input(batch_shape=(None, train_data.shape[1]))\n",
    "# out = Dense(8)(inp)\n",
    "extra_model = Model(input=inp, output=inp)\n",
    "\n",
    "x = concatenate([added_model.output,\n",
    "           extra_model.output])\n",
    "\n",
    "# x = Dropout(0.5)(x)\n",
    "# x = Dense(1024, activation='relu')(x)\n",
    "x = Dropout(0.2)(x)\n",
    "x = Dense(256, activation='relu')(x)\n",
    "x = Dropout(0.2)(x)\n",
    "x = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "model =  Model(input=[added_model.input,\n",
    "                extra_model.input],\n",
    "                output=x)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def binary_accuracy(y_true, y_pred):\n",
    "    return K.mean(K.equal(y_true, K.round(y_pred)))\n",
    "\n",
    "def precision_threshold(threshold = 0.5):\n",
    "    def precision(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(y_pred)\n",
    "        precision_ratio = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision_ratio\n",
    "    return precision\n",
    "\n",
    "def recall_threshold(threshold = 0.5):\n",
    "    def recall(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.clip(y_true, 0, 1))\n",
    "        recall_ratio = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall_ratio\n",
    "    return recall\n",
    "\n",
    "def fbeta_score_threshold(beta = 1, threshold = 0.5):\n",
    "    def fbeta_score(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        beta_value = beta\n",
    "        p = precision_threshold(threshold_value)(y_true, y_pred)\n",
    "        r = recall_threshold(threshold_value)(y_true, y_pred)\n",
    "        bb = beta_value ** 2\n",
    "        fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
    "        return fbeta_score\n",
    "    return fbeta_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer='sgd', loss='binary_crossentropy', \n",
    "              metrics=[binary_accuracy,\n",
    "                       precision_threshold(threshold = 0.4), \n",
    "                       recall_threshold(threshold = 0.4), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.4),\n",
    "                      precision_threshold(threshold = 0.5), \n",
    "                       recall_threshold(threshold = 0.5), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.5),\n",
    "                      precision_threshold(threshold = 0.6), \n",
    "                       recall_threshold(threshold = 0.6), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.6)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3400 samples, validate on 1100 samples\n",
      "Epoch 1/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.7016 - binary_accuracy: 0.5451 - precision_1: 0.4784 - recall_1: 0.7324 - fbeta_score_1: 0.5001 - precision_2: 0.4891 - recall_2: 0.3659 - fbeta_score_2: 0.3961 - precision_3: 0.3322 - recall_3: 0.1313 - fbeta_score_3: 0.2090Epoch 00001: val_loss improved from inf to 0.72926, saving model to saved_models/pretrained_extradata_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 16s 5ms/step - loss: 0.7016 - binary_accuracy: 0.5453 - precision_1: 0.4785 - recall_1: 0.7321 - fbeta_score_1: 0.5001 - precision_2: 0.4903 - recall_2: 0.3660 - fbeta_score_2: 0.3970 - precision_3: 0.3337 - recall_3: 0.1315 - fbeta_score_3: 0.2098 - val_loss: 0.7293 - val_binary_accuracy: 0.4709 - val_precision_1: 0.4618 - val_recall_1: 1.0000 - val_fbeta_score_1: 0.5153 - val_precision_2: 0.4650 - val_recall_2: 0.9792 - val_fbeta_score_2: 0.5174 - val_precision_3: 0.5316 - val_recall_3: 0.8313 - val_fbeta_score_3: 0.5692\n",
      "Epoch 2/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6729 - binary_accuracy: 0.5949 - precision_1: 0.5333 - recall_1: 0.7591 - fbeta_score_1: 0.5502 - precision_2: 0.5894 - recall_2: 0.4592 - fbeta_score_2: 0.5136 - precision_3: 0.5284 - recall_3: 0.2140 - fbeta_score_3: 0.3492Epoch 00002: val_loss improved from 0.72926 to 0.64395, saving model to saved_models/pretrained_extradata_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 16s 5ms/step - loss: 0.6728 - binary_accuracy: 0.5953 - precision_1: 0.5332 - recall_1: 0.7590 - fbeta_score_1: 0.5502 - precision_2: 0.5897 - recall_2: 0.4599 - fbeta_score_2: 0.5141 - precision_3: 0.5295 - recall_3: 0.2147 - fbeta_score_3: 0.3504 - val_loss: 0.6439 - val_binary_accuracy: 0.6436 - val_precision_1: 0.5386 - val_recall_1: 0.8599 - val_fbeta_score_1: 0.5783 - val_precision_2: 0.5990 - val_recall_2: 0.7253 - val_fbeta_score_2: 0.6148 - val_precision_3: 0.6481 - val_recall_3: 0.5603 - val_fbeta_score_3: 0.6210\n",
      "Epoch 3/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6552 - binary_accuracy: 0.6288 - precision_1: 0.5755 - recall_1: 0.7446 - fbeta_score_1: 0.5869 - precision_2: 0.6321 - recall_2: 0.5399 - fbeta_score_2: 0.5743 - precision_3: 0.6494 - recall_3: 0.3045 - fbeta_score_3: 0.4676Epoch 00003: val_loss did not improve\n",
      "3400/3400 [==============================] - 15s 4ms/step - loss: 0.6554 - binary_accuracy: 0.6285 - precision_1: 0.5755 - recall_1: 0.7443 - fbeta_score_1: 0.5869 - precision_2: 0.6322 - recall_2: 0.5395 - fbeta_score_2: 0.5744 - precision_3: 0.6479 - recall_3: 0.3038 - fbeta_score_3: 0.4665 - val_loss: 0.6808 - val_binary_accuracy: 0.5591 - val_precision_1: 0.4683 - val_recall_1: 0.9859 - val_fbeta_score_1: 0.5212 - val_precision_2: 0.5147 - val_recall_2: 0.8775 - val_fbeta_score_2: 0.5577 - val_precision_3: 0.6033 - val_recall_3: 0.7618 - val_fbeta_score_3: 0.6243\n",
      "Epoch 4/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6441 - binary_accuracy: 0.6415 - precision_1: 0.5744 - recall_1: 0.7062 - fbeta_score_1: 0.5830 - precision_2: 0.6455 - recall_2: 0.5444 - fbeta_score_2: 0.5972 - precision_3: 0.6759 - recall_3: 0.3659 - fbeta_score_3: 0.5368Epoch 00004: val_loss improved from 0.64395 to 0.61386, saving model to saved_models/pretrained_extradata_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 16s 5ms/step - loss: 0.6440 - binary_accuracy: 0.6415 - precision_1: 0.5745 - recall_1: 0.7069 - fbeta_score_1: 0.5832 - precision_2: 0.6455 - recall_2: 0.5454 - fbeta_score_2: 0.5974 - precision_3: 0.6760 - recall_3: 0.3674 - fbeta_score_3: 0.5373 - val_loss: 0.6139 - val_binary_accuracy: 0.6809 - val_precision_1: 0.6028 - val_recall_1: 0.7703 - val_fbeta_score_1: 0.6253 - val_precision_2: 0.6674 - val_recall_2: 0.6236 - val_fbeta_score_2: 0.6499 - val_precision_3: 0.7038 - val_recall_3: 0.4775 - val_fbeta_score_3: 0.6324\n",
      "Epoch 5/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6361 - binary_accuracy: 0.6453 - precision_1: 0.5891 - recall_1: 0.7230 - fbeta_score_1: 0.5982 - precision_2: 0.6478 - recall_2: 0.5574 - fbeta_score_2: 0.6029 - precision_3: 0.7134 - recall_3: 0.3891 - fbeta_score_3: 0.5627Epoch 00005: val_loss improved from 0.61386 to 0.60760, saving model to saved_models/pretrained_extradata_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 16s 5ms/step - loss: 0.6359 - binary_accuracy: 0.6456 - precision_1: 0.5895 - recall_1: 0.7231 - fbeta_score_1: 0.5986 - precision_2: 0.6487 - recall_2: 0.5572 - fbeta_score_2: 0.6034 - precision_3: 0.7118 - recall_3: 0.3882 - fbeta_score_3: 0.5613 - val_loss: 0.6076 - val_binary_accuracy: 0.6764 - val_precision_1: 0.6472 - val_recall_1: 0.6913 - val_fbeta_score_1: 0.6489 - val_precision_2: 0.6778 - val_recall_2: 0.5678 - val_fbeta_score_2: 0.6453 - val_precision_3: 0.6973 - val_recall_3: 0.4784 - val_fbeta_score_3: 0.6297\n",
      "Epoch 6/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6286 - binary_accuracy: 0.6562 - precision_1: 0.6071 - recall_1: 0.7316 - fbeta_score_1: 0.6148 - precision_2: 0.6535 - recall_2: 0.5826 - fbeta_score_2: 0.6174 - precision_3: 0.7090 - recall_3: 0.4185 - fbeta_score_3: 0.5795Epoch 00006: val_loss did not improve\n",
      "3400/3400 [==============================] - 15s 5ms/step - loss: 0.6290 - binary_accuracy: 0.6556 - precision_1: 0.6068 - recall_1: 0.7311 - fbeta_score_1: 0.6145 - precision_2: 0.6528 - recall_2: 0.5818 - fbeta_score_2: 0.6166 - precision_3: 0.7073 - recall_3: 0.4175 - fbeta_score_3: 0.5781 - val_loss: 0.6138 - val_binary_accuracy: 0.6718 - val_precision_1: 0.6419 - val_recall_1: 0.6857 - val_fbeta_score_1: 0.6449 - val_precision_2: 0.6890 - val_recall_2: 0.5087 - val_fbeta_score_2: 0.6358 - val_precision_3: 0.7320 - val_recall_3: 0.3894 - val_fbeta_score_3: 0.6089\n",
      "Epoch 7/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6182 - binary_accuracy: 0.6722 - precision_1: 0.6168 - recall_1: 0.7310 - fbeta_score_1: 0.6242 - precision_2: 0.6745 - recall_2: 0.5993 - fbeta_score_2: 0.6359 - precision_3: 0.7090 - recall_3: 0.4458 - fbeta_score_3: 0.5999Epoch 00007: val_loss did not improve\n",
      "3400/3400 [==============================] - 15s 5ms/step - loss: 0.6184 - binary_accuracy: 0.6718 - precision_1: 0.6165 - recall_1: 0.7305 - fbeta_score_1: 0.6239 - precision_2: 0.6741 - recall_2: 0.5991 - fbeta_score_2: 0.6356 - precision_3: 0.7089 - recall_3: 0.4460 - fbeta_score_3: 0.5999 - val_loss: 0.6160 - val_binary_accuracy: 0.6691 - val_precision_1: 0.6480 - val_recall_1: 0.6744 - val_fbeta_score_1: 0.6467 - val_precision_2: 0.7011 - val_recall_2: 0.4902 - val_fbeta_score_2: 0.6382 - val_precision_3: 0.7549 - val_recall_3: 0.3048 - val_fbeta_score_3: 0.5635\n",
      "Epoch 8/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6086 - binary_accuracy: 0.6831 - precision_1: 0.6181 - recall_1: 0.7187 - fbeta_score_1: 0.6251 - precision_2: 0.6817 - recall_2: 0.6030 - fbeta_score_2: 0.6434 - precision_3: 0.7282 - recall_3: 0.4536 - fbeta_score_3: 0.6095Epoch 00008: val_loss did not improve\n",
      "3400/3400 [==============================] - 15s 5ms/step - loss: 0.6083 - binary_accuracy: 0.6829 - precision_1: 0.6178 - recall_1: 0.7194 - fbeta_score_1: 0.6249 - precision_2: 0.6812 - recall_2: 0.6039 - fbeta_score_2: 0.6432 - precision_3: 0.7283 - recall_3: 0.4548 - fbeta_score_3: 0.6099 - val_loss: 0.7873 - val_binary_accuracy: 0.5818 - val_precision_1: 0.7599 - val_recall_1: 0.2156 - val_fbeta_score_1: 0.4809 - val_precision_2: 0.7316 - val_recall_2: 0.1311 - val_fbeta_score_2: 0.3535 - val_precision_3: 0.4718 - val_recall_3: 0.0750 - val_fbeta_score_3: 0.2128\n",
      "Epoch 00008: early stopping\n",
      "training time: 2.07 minutes\n"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping\n",
    "import numpy as np\n",
    "\n",
    "epochs = 20\n",
    "batch_size = 32\n",
    "\n",
    "earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')\n",
    "log = CSVLogger('saved_models/log_pretrained_extradata_CNN.csv')\n",
    "checkpointer = ModelCheckpoint(filepath='saved_models/pretrained_extradata_CNN.best.from_scratch.hdf5', \n",
    "                               verbose=1, save_best_only=True)\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "model.fit([train_tensors,  train_data], train_labels, \n",
    "          validation_data=([valid_tensors, valid_data], valid_labels),\n",
    "          epochs=epochs, batch_size=batch_size, callbacks=[checkpointer, log, earlystop], verbose=1)\n",
    "\n",
    "# def train_generator(x1, x2, y, batch_size):\n",
    "#         train_datagen = ImageDataGenerator(\n",
    "#                         featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "#                         samplewise_center=False,  # set each sample mean to 0\n",
    "#                         featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "#                         samplewise_std_normalization=False,  # divide each input by its std\n",
    "#                         zca_whitening=False,  # apply ZCA whitening\n",
    "#                         rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "#                         width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n",
    "#                         height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n",
    "#                         horizontal_flip=True,  # randomly flip images\n",
    "#                         vertical_flip=False)  # randomly flip images\n",
    "#         generator = train_datagen.flow((x1, x2), y, batch_size=batch_size)\n",
    "#         while 1:\n",
    "#             (x1_batch, x2_batch), y_batch = generator.next()\n",
    "#             yield [[x1_batch, x2_batch], y_batch]\n",
    "\n",
    "# # Training with data augmentation. If shift_fraction=0., also no augmentation.\n",
    "# model.fit_generator(generator=train_generator(train_tensors, train_data, train_labels, batch_size),\n",
    "#                     steps_per_epoch=int(train_labels.shape[0] / batch_size),\n",
    "#                     validation_data=([valid_tensors, valid_data], valid_labels),\n",
    "#                     epochs=epochs, callbacks=[checkpointer, log, earlystop], verbose=1)\n",
    "\n",
    "# Show total training time\n",
    "print(\"training time: %.2f minutes\"%((time.time()-start)/60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('saved_models/pretrained_extradata_CNN.best.from_scratch.hdf5')\n",
    "prediction = model.predict([test_tensors, test_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.647191 %\n",
      "Recall: 0.587755 %\n",
      "Fscore: 0.634361 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.67511314"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.eval(binary_accuracy(K.variable(value=test_labels),\n",
    "                       K.variable(value=prediction)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.43854904],\n",
       "       [ 0.28271237],\n",
       "       [ 0.32896599],\n",
       "       [ 0.76249766],\n",
       "       [ 0.31618291],\n",
       "       [ 0.77488339],\n",
       "       [ 0.3089357 ],\n",
       "       [ 0.72916156],\n",
       "       [ 0.21439876],\n",
       "       [ 0.38674796],\n",
       "       [ 0.4488712 ],\n",
       "       [ 0.7688958 ],\n",
       "       [ 0.28179762],\n",
       "       [ 0.23615587],\n",
       "       [ 0.70831102],\n",
       "       [ 0.72348529],\n",
       "       [ 0.36050457],\n",
       "       [ 0.72639018],\n",
       "       [ 0.43309873],\n",
       "       [ 0.25428677],\n",
       "       [ 0.41418654],\n",
       "       [ 0.80132085],\n",
       "       [ 0.32700819],\n",
       "       [ 0.79180586],\n",
       "       [ 0.75714654],\n",
       "       [ 0.73956227],\n",
       "       [ 0.75789005],\n",
       "       [ 0.25184935],\n",
       "       [ 0.62190068],\n",
       "       [ 0.51263815]], dtype=float32)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction[:30]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train with extra data and spacial transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "train_filename = \"data_preprocessed/train_data_sample_rgb.p\"\n",
    "(train_labels, train_data, train_tensors) = pickle.load(open(train_filename, mode='rb'))\n",
    "\n",
    "valid_filename = \"data_preprocessed/valid_data_sample_rgb.p\"\n",
    "(valid_labels, valid_data, valid_tensors) = pickle.load(open(valid_filename, mode='rb'))\n",
    "\n",
    "test_filename = \"data_preprocessed/test_data_sample_rgb.p\"\n",
    "(test_labels, test_data, test_tensors) = pickle.load(open(test_filename, mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "lambda_3_input (InputLayer)     (None, 64, 64, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 64, 64, 3)    0           lambda_3_input[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 64, 64, 3)    12          lambda_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "spatial_transformer_3 (SpatialT (None, 64, 64, 3)    247270      batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "model_18 (Model)                (None, 2048)         14714688    spatial_transformer_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "input_13 (InputLayer)           (None, 5)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (None, 2053)         0           model_18[1][0]                   \n",
      "                                                                 input_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_18 (Dropout)            (None, 2053)         0           concatenate_5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_27 (Dense)                (None, 256)          525824      dropout_18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_19 (Dropout)            (None, 256)          0           dense_27[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_28 (Dense)                (None, 1)            257         dropout_19[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 15,488,051\n",
      "Trainable params: 15,488,045\n",
      "Non-trainable params: 6\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:59: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Tensor(\"in..., inputs=Tensor(\"in...)`\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:73: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Tensor(\"de..., inputs=[<tf.Tenso...)`\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "\n",
    "from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, Lambda\n",
    "from keras.layers import Dropout, Flatten, Dense\n",
    "from keras.models import Sequential\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras import regularizers\n",
    "from keras import applications\n",
    "from keras.models import Model\n",
    "from keras import optimizers\n",
    "from keras.layers import Input, merge, concatenate\n",
    "from spatial_transformer import SpatialTransformer\n",
    "\n",
    "def locnet():\n",
    "    b = np.zeros((2, 3), dtype='float32')\n",
    "    b[0, 0] = 1\n",
    "    b[1, 1] = 1\n",
    "    W = np.zeros((64, 6), dtype='float32')\n",
    "    weights = [W, b.flatten()]\n",
    "    locnet = Sequential()\n",
    "\n",
    "    locnet.add(Conv2D(16, (7, 7), padding='valid', input_shape=train_tensors.shape[1:]))\n",
    "    locnet.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    locnet.add(Conv2D(32, (5, 5), padding='valid'))\n",
    "    locnet.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    locnet.add(Conv2D(64, (3, 3), padding='valid'))\n",
    "    locnet.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    locnet.add(Flatten())\n",
    "    locnet.add(Dense(128, activation='elu'))\n",
    "    locnet.add(Dense(64, activation='elu'))\n",
    "    locnet.add(Dense(6, weights=weights))\n",
    "\n",
    "    return locnet\n",
    "\n",
    "base_model = applications.VGG16(weights='imagenet', \n",
    "                                include_top=False, \n",
    "                                input_shape=train_tensors.shape[1:])\n",
    "\n",
    "add_model = Sequential()\n",
    "add_model.add(Flatten(input_shape=base_model.output_shape[1:]))\n",
    "\n",
    "added0_model = Model(inputs=base_model.input, outputs=add_model(base_model.output))\n",
    "\n",
    "stn_model = Sequential()\n",
    "stn_model.add(Lambda(\n",
    "    lambda x: 2*x - 1.,\n",
    "    input_shape=train_tensors.shape[1:],\n",
    "    output_shape=train_tensors.shape[1:]))\n",
    "stn_model.add(BatchNormalization())\n",
    "stn_model.add(SpatialTransformer(localization_net=locnet(),\n",
    "                                 output_size=train_tensors.shape[1:3]))\n",
    "\n",
    "added_model = Model(inputs=stn_model.input, outputs=added0_model(stn_model.output))\n",
    "\n",
    "inp = Input(batch_shape=(None, train_data.shape[1]))\n",
    "# out = Dense(8)(inp)\n",
    "extra_model = Model(input=inp, output=inp)\n",
    "\n",
    "x = concatenate([added_model.output,\n",
    "           extra_model.output])\n",
    "\n",
    "# x = Dropout(0.5)(x)\n",
    "# x = Dense(1024, activation='relu')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "x = Dense(256, activation='relu')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "x = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "model =  Model(input=[added_model.input,\n",
    "                extra_model.input],\n",
    "                output=x)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def binary_accuracy(y_true, y_pred):\n",
    "    return K.mean(K.equal(y_true, K.round(y_pred)))\n",
    "\n",
    "def precision_threshold(threshold = 0.5):\n",
    "    def precision(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(y_pred)\n",
    "        precision_ratio = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision_ratio\n",
    "    return precision\n",
    "\n",
    "def recall_threshold(threshold = 0.5):\n",
    "    def recall(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.clip(y_true, 0, 1))\n",
    "        recall_ratio = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall_ratio\n",
    "    return recall\n",
    "\n",
    "def fbeta_score_threshold(beta = 1, threshold = 0.5):\n",
    "    def fbeta_score(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        beta_value = beta\n",
    "        p = precision_threshold(threshold_value)(y_true, y_pred)\n",
    "        r = recall_threshold(threshold_value)(y_true, y_pred)\n",
    "        bb = beta_value ** 2\n",
    "        fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
    "        return fbeta_score\n",
    "    return fbeta_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=optimizers.SGD(lr=1e-4, decay=1e-6, momentum=0.9, nesterov=True), \n",
    "              loss='binary_crossentropy', \n",
    "              metrics=[binary_accuracy,\n",
    "                       precision_threshold(threshold = 0.4), \n",
    "                       recall_threshold(threshold = 0.4), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.4),\n",
    "                      precision_threshold(threshold = 0.5), \n",
    "                       recall_threshold(threshold = 0.5), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.5),\n",
    "                      precision_threshold(threshold = 0.6), \n",
    "                       recall_threshold(threshold = 0.6), \n",
    "                       fbeta_score_threshold(beta=0.5, threshold = 0.6)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3400 samples, validate on 1100 samples\n",
      "Epoch 1/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.7859 - binary_accuracy: 0.5292 - precision_1: 0.4774 - recall_1: 0.6244 - fbeta_score_1: 0.4952 - precision_2: 0.4906 - recall_2: 0.4578 - fbeta_score_2: 0.4734 - precision_3: 0.5071 - recall_3: 0.3049 - fbeta_score_3: 0.4289Epoch 00001: val_loss improved from inf to 0.66229, saving model to saved_models/log_pretrained_extradata_stn_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 19s 6ms/step - loss: 0.7861 - binary_accuracy: 0.5291 - precision_1: 0.4770 - recall_1: 0.6241 - fbeta_score_1: 0.4949 - precision_2: 0.4906 - recall_2: 0.4579 - fbeta_score_2: 0.4735 - precision_3: 0.5074 - recall_3: 0.3053 - fbeta_score_3: 0.4293 - val_loss: 0.6623 - val_binary_accuracy: 0.5609 - val_precision_1: 0.5511 - val_recall_1: 0.8046 - val_fbeta_score_1: 0.5840 - val_precision_2: 0.5571 - val_recall_2: 0.2083 - val_fbeta_score_2: 0.3974 - val_precision_3: 0.0000e+00 - val_recall_3: 0.0000e+00 - val_fbeta_score_3: 0.0000e+00\n",
      "Epoch 2/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.7129 - binary_accuracy: 0.5587 - precision_1: 0.5041 - recall_1: 0.6603 - fbeta_score_1: 0.5251 - precision_2: 0.5224 - recall_2: 0.4469 - fbeta_score_2: 0.4968 - precision_3: 0.5542 - recall_3: 0.2646 - fbeta_score_3: 0.4379Epoch 00002: val_loss improved from 0.66229 to 0.64536, saving model to saved_models/log_pretrained_extradata_stn_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.7123 - binary_accuracy: 0.5591 - precision_1: 0.5053 - recall_1: 0.6611 - fbeta_score_1: 0.5262 - precision_2: 0.5235 - recall_2: 0.4474 - fbeta_score_2: 0.4978 - precision_3: 0.5553 - recall_3: 0.2656 - fbeta_score_3: 0.4390 - val_loss: 0.6454 - val_binary_accuracy: 0.6236 - val_precision_1: 0.6014 - val_recall_1: 0.6869 - val_fbeta_score_1: 0.6111 - val_precision_2: 0.6530 - val_recall_2: 0.3810 - val_fbeta_score_2: 0.5624 - val_precision_3: 0.4073 - val_recall_3: 0.0370 - val_fbeta_score_3: 0.1296\n",
      "Epoch 3/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6882 - binary_accuracy: 0.5884 - precision_1: 0.5271 - recall_1: 0.7267 - fbeta_score_1: 0.5529 - precision_2: 0.5578 - recall_2: 0.5183 - fbeta_score_2: 0.5404 - precision_3: 0.5804 - recall_3: 0.2905 - fbeta_score_3: 0.4622Epoch 00003: val_loss improved from 0.64536 to 0.63555, saving model to saved_models/log_pretrained_extradata_stn_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6885 - binary_accuracy: 0.5882 - precision_1: 0.5268 - recall_1: 0.7262 - fbeta_score_1: 0.5526 - precision_2: 0.5577 - recall_2: 0.5183 - fbeta_score_2: 0.5403 - precision_3: 0.5790 - recall_3: 0.2898 - fbeta_score_3: 0.4611 - val_loss: 0.6355 - val_binary_accuracy: 0.6336 - val_precision_1: 0.6230 - val_recall_1: 0.6870 - val_fbeta_score_1: 0.6298 - val_precision_2: 0.6530 - val_recall_2: 0.4306 - val_fbeta_score_2: 0.5836 - val_precision_3: 0.5309 - val_recall_3: 0.0896 - val_fbeta_score_3: 0.2490\n",
      "Epoch 4/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6635 - binary_accuracy: 0.6108 - precision_1: 0.5475 - recall_1: 0.7373 - fbeta_score_1: 0.5730 - precision_2: 0.5843 - recall_2: 0.5412 - fbeta_score_2: 0.5670 - precision_3: 0.6039 - recall_3: 0.3220 - fbeta_score_3: 0.4961Epoch 00004: val_loss improved from 0.63555 to 0.63160, saving model to saved_models/log_pretrained_extradata_stn_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6637 - binary_accuracy: 0.6106 - precision_1: 0.5472 - recall_1: 0.7374 - fbeta_score_1: 0.5728 - precision_2: 0.5841 - recall_2: 0.5411 - fbeta_score_2: 0.5668 - precision_3: 0.6041 - recall_3: 0.3224 - fbeta_score_3: 0.4964 - val_loss: 0.6316 - val_binary_accuracy: 0.6500 - val_precision_1: 0.6453 - val_recall_1: 0.6612 - val_fbeta_score_1: 0.6431 - val_precision_2: 0.6726 - val_recall_2: 0.4655 - val_fbeta_score_2: 0.6101 - val_precision_3: 0.6750 - val_recall_3: 0.2228 - val_fbeta_score_3: 0.4606\n",
      "Epoch 5/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6632 - binary_accuracy: 0.6235 - precision_1: 0.5429 - recall_1: 0.7552 - fbeta_score_1: 0.5717 - precision_2: 0.5956 - recall_2: 0.5806 - fbeta_score_2: 0.5848 - precision_3: 0.6431 - recall_3: 0.3117 - fbeta_score_3: 0.5073Epoch 00005: val_loss did not improve\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6630 - binary_accuracy: 0.6235 - precision_1: 0.5421 - recall_1: 0.7558 - fbeta_score_1: 0.5709 - precision_2: 0.5948 - recall_2: 0.5816 - fbeta_score_2: 0.5841 - precision_3: 0.6416 - recall_3: 0.3109 - fbeta_score_3: 0.5061 - val_loss: 0.6447 - val_binary_accuracy: 0.6309 - val_precision_1: 0.6502 - val_recall_1: 0.6021 - val_fbeta_score_1: 0.6355 - val_precision_2: 0.6555 - val_recall_2: 0.3793 - val_fbeta_score_2: 0.5619 - val_precision_3: 0.6737 - val_recall_3: 0.1303 - val_fbeta_score_3: 0.3379\n",
      "Epoch 6/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6421 - binary_accuracy: 0.6365 - precision_1: 0.5658 - recall_1: 0.7568 - fbeta_score_1: 0.5915 - precision_2: 0.6224 - recall_2: 0.5726 - fbeta_score_2: 0.6030 - precision_3: 0.6722 - recall_3: 0.3207 - fbeta_score_3: 0.5306Epoch 00006: val_loss improved from 0.63160 to 0.62771, saving model to saved_models/log_pretrained_extradata_stn_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6421 - binary_accuracy: 0.6362 - precision_1: 0.5657 - recall_1: 0.7566 - fbeta_score_1: 0.5913 - precision_2: 0.6217 - recall_2: 0.5721 - fbeta_score_2: 0.6024 - precision_3: 0.6706 - recall_3: 0.3200 - fbeta_score_3: 0.5293 - val_loss: 0.6277 - val_binary_accuracy: 0.6764 - val_precision_1: 0.6029 - val_recall_1: 0.7267 - val_fbeta_score_1: 0.6197 - val_precision_2: 0.6770 - val_recall_2: 0.5603 - val_fbeta_score_2: 0.6433 - val_precision_3: 0.6399 - val_recall_3: 0.2600 - val_fbeta_score_3: 0.4786\n",
      "Epoch 7/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6369 - binary_accuracy: 0.6397 - precision_1: 0.5573 - recall_1: 0.7846 - fbeta_score_1: 0.5877 - precision_2: 0.6138 - recall_2: 0.5900 - fbeta_score_2: 0.6018 - precision_3: 0.6729 - recall_3: 0.3720 - fbeta_score_3: 0.5634Epoch 00007: val_loss did not improve\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6364 - binary_accuracy: 0.6400 - precision_1: 0.5579 - recall_1: 0.7851 - fbeta_score_1: 0.5883 - precision_2: 0.6147 - recall_2: 0.5898 - fbeta_score_2: 0.6023 - precision_3: 0.6737 - recall_3: 0.3723 - fbeta_score_3: 0.5640 - val_loss: 0.6285 - val_binary_accuracy: 0.6636 - val_precision_1: 0.6152 - val_recall_1: 0.7108 - val_fbeta_score_1: 0.6274 - val_precision_2: 0.6826 - val_recall_2: 0.5005 - val_fbeta_score_2: 0.6287 - val_precision_3: 0.6350 - val_recall_3: 0.1884 - val_fbeta_score_3: 0.4102\n",
      "Epoch 8/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6386 - binary_accuracy: 0.6445 - precision_1: 0.5634 - recall_1: 0.7621 - fbeta_score_1: 0.5903 - precision_2: 0.6212 - recall_2: 0.5643 - fbeta_score_2: 0.6015 - precision_3: 0.6865 - recall_3: 0.3161 - fbeta_score_3: 0.5330Epoch 00008: val_loss improved from 0.62771 to 0.61430, saving model to saved_models/log_pretrained_extradata_stn_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6389 - binary_accuracy: 0.6441 - precision_1: 0.5644 - recall_1: 0.7615 - fbeta_score_1: 0.5909 - precision_2: 0.6221 - recall_2: 0.5638 - fbeta_score_2: 0.6018 - precision_3: 0.6872 - recall_3: 0.3161 - fbeta_score_3: 0.5335 - val_loss: 0.6143 - val_binary_accuracy: 0.6764 - val_precision_1: 0.5883 - val_recall_1: 0.7874 - val_fbeta_score_1: 0.6157 - val_precision_2: 0.6633 - val_recall_2: 0.6082 - val_fbeta_score_2: 0.6452 - val_precision_3: 0.7138 - val_recall_3: 0.4011 - val_fbeta_score_3: 0.6043\n",
      "Epoch 9/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6350 - binary_accuracy: 0.6418 - precision_1: 0.5694 - recall_1: 0.7882 - fbeta_score_1: 0.5983 - precision_2: 0.6142 - recall_2: 0.6059 - fbeta_score_2: 0.6053 - precision_3: 0.6753 - recall_3: 0.3598 - fbeta_score_3: 0.5556Epoch 00009: val_loss did not improve\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6346 - binary_accuracy: 0.6424 - precision_1: 0.5700 - recall_1: 0.7887 - fbeta_score_1: 0.5988 - precision_2: 0.6151 - recall_2: 0.6063 - fbeta_score_2: 0.6061 - precision_3: 0.6760 - recall_3: 0.3601 - fbeta_score_3: 0.5563 - val_loss: 0.6247 - val_binary_accuracy: 0.6664 - val_precision_1: 0.6649 - val_recall_1: 0.6452 - val_fbeta_score_1: 0.6546 - val_precision_2: 0.6999 - val_recall_2: 0.4833 - val_fbeta_score_2: 0.6325 - val_precision_3: 0.6884 - val_recall_3: 0.2756 - val_fbeta_score_3: 0.5143\n",
      "Epoch 10/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6295 - binary_accuracy: 0.6577 - precision_1: 0.5713 - recall_1: 0.7640 - fbeta_score_1: 0.5964 - precision_2: 0.6358 - recall_2: 0.5988 - fbeta_score_2: 0.6178 - precision_3: 0.6964 - recall_3: 0.3765 - fbeta_score_3: 0.5761Epoch 00010: val_loss improved from 0.61430 to 0.60826, saving model to saved_models/log_pretrained_extradata_stn_CNN.best.from_scratch.hdf5\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6291 - binary_accuracy: 0.6585 - precision_1: 0.5713 - recall_1: 0.7645 - fbeta_score_1: 0.5965 - precision_2: 0.6367 - recall_2: 0.5997 - fbeta_score_2: 0.6187 - precision_3: 0.6971 - recall_3: 0.3780 - fbeta_score_3: 0.5771 - val_loss: 0.6083 - val_binary_accuracy: 0.6855 - val_precision_1: 0.5988 - val_recall_1: 0.7864 - val_fbeta_score_1: 0.6251 - val_precision_2: 0.6770 - val_recall_2: 0.6196 - val_fbeta_score_2: 0.6584 - val_precision_3: 0.6995 - val_recall_3: 0.4005 - val_fbeta_score_3: 0.5941\n",
      "Epoch 11/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6169 - binary_accuracy: 0.6642 - precision_1: 0.5837 - recall_1: 0.7764 - fbeta_score_1: 0.6096 - precision_2: 0.6432 - recall_2: 0.6118 - fbeta_score_2: 0.6304 - precision_3: 0.6952 - recall_3: 0.3938 - fbeta_score_3: 0.5862Epoch 00011: val_loss did not improve\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6163 - binary_accuracy: 0.6647 - precision_1: 0.5847 - recall_1: 0.7761 - fbeta_score_1: 0.6103 - precision_2: 0.6440 - recall_2: 0.6120 - fbeta_score_2: 0.6310 - precision_3: 0.6959 - recall_3: 0.3937 - fbeta_score_3: 0.5865 - val_loss: 0.6083 - val_binary_accuracy: 0.6755 - val_precision_1: 0.5809 - val_recall_1: 0.8382 - val_fbeta_score_1: 0.6149 - val_precision_2: 0.6334 - val_recall_2: 0.7015 - val_fbeta_score_2: 0.6410 - val_precision_3: 0.7005 - val_recall_3: 0.5652 - val_fbeta_score_3: 0.6603\n",
      "Epoch 12/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6198 - binary_accuracy: 0.6657 - precision_1: 0.5774 - recall_1: 0.7804 - fbeta_score_1: 0.6044 - precision_2: 0.6416 - recall_2: 0.6296 - fbeta_score_2: 0.6303 - precision_3: 0.6759 - recall_3: 0.4013 - fbeta_score_3: 0.5743Epoch 00012: val_loss did not improve\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6201 - binary_accuracy: 0.6653 - precision_1: 0.5776 - recall_1: 0.7809 - fbeta_score_1: 0.6046 - precision_2: 0.6412 - recall_2: 0.6287 - fbeta_score_2: 0.6298 - precision_3: 0.6743 - recall_3: 0.4004 - fbeta_score_3: 0.5729 - val_loss: 0.6115 - val_binary_accuracy: 0.6736 - val_precision_1: 0.6249 - val_recall_1: 0.7430 - val_fbeta_score_1: 0.6403 - val_precision_2: 0.6877 - val_recall_2: 0.5594 - val_fbeta_score_2: 0.6489 - val_precision_3: 0.7401 - val_recall_3: 0.3004 - val_fbeta_score_3: 0.5548\n",
      "Epoch 13/20\n",
      "3392/3400 [============================>.] - ETA: 0s - loss: 0.6199 - binary_accuracy: 0.6736 - precision_1: 0.5902 - recall_1: 0.7990 - fbeta_score_1: 0.6172 - precision_2: 0.6594 - recall_2: 0.6323 - fbeta_score_2: 0.6401 - precision_3: 0.7104 - recall_3: 0.3584 - fbeta_score_3: 0.5652Epoch 00013: val_loss did not improve\n",
      "3400/3400 [==============================] - 18s 5ms/step - loss: 0.6199 - binary_accuracy: 0.6738 - precision_1: 0.5898 - recall_1: 0.7995 - fbeta_score_1: 0.6168 - precision_2: 0.6590 - recall_2: 0.6332 - fbeta_score_2: 0.6399 - precision_3: 0.7099 - recall_3: 0.3587 - fbeta_score_3: 0.5651 - val_loss: 0.6087 - val_binary_accuracy: 0.6773 - val_precision_1: 0.6341 - val_recall_1: 0.7091 - val_fbeta_score_1: 0.6430 - val_precision_2: 0.6914 - val_recall_2: 0.5602 - val_fbeta_score_2: 0.6530 - val_precision_3: 0.7192 - val_recall_3: 0.3573 - val_fbeta_score_3: 0.5852\n",
      "Epoch 00013: early stopping\n",
      "training time: 3.91 minutes\n"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping\n",
    "\n",
    "epochs = 20\n",
    "batch_size = 32\n",
    "\n",
    "earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')\n",
    "log = CSVLogger('saved_models/log_pretrained_extradata_stn_CNN.csv')\n",
    "checkpointer = ModelCheckpoint(filepath='saved_models/log_pretrained_extradata_stn_CNN.best.from_scratch.hdf5', \n",
    "                               verbose=1, save_best_only=True)\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "model.fit([train_tensors, train_data], train_labels, \n",
    "          validation_data=([valid_tensors, valid_data], valid_labels),\n",
    "          epochs=epochs, batch_size=batch_size, callbacks=[checkpointer, log, earlystop], verbose=1)\n",
    "\n",
    "# def train_generator(x, y, batch_size):\n",
    "#         train_datagen = ImageDataGenerator(\n",
    "#                         featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "#                         samplewise_center=False,  # set each sample mean to 0\n",
    "#                         featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "#                         samplewise_std_normalization=False,  # divide each input by its std\n",
    "#                         zca_whitening=False,  # apply ZCA whitening\n",
    "#                         rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "#                         width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n",
    "#                         height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n",
    "#                         horizontal_flip=True,  # randomly flip images\n",
    "#                         vertical_flip=False)  # randomly flip images\n",
    "#         generator = train_datagen.flow(x, y, batch_size=batch_size)\n",
    "#         while 1:\n",
    "#             x_batch, y_batch = generator.next()\n",
    "#             yield [x_batch, y_batch]\n",
    "\n",
    "# # Training with data augmentation. If shift_fraction=0., also no augmentation.\n",
    "# model.fit_generator(generator=train_generator(train_tensors, train_labels, batch_size),\n",
    "#                     steps_per_epoch=int(train_labels.shape[0] / batch_size),\n",
    "#                     validation_data=(valid_tensors, valid_labels),\n",
    "#                     epochs=epochs, callbacks=[checkpointer, log, earlystop], verbose=1)\n",
    "\n",
    "# Show total training time\n",
    "print(\"training time: %.2f minutes\"%((time.time()-start)/60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('saved_models/log_pretrained_extradata_stn_CNN.best.from_scratch.hdf5')\n",
    "prediction = model.predict([test_tensors, test_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.641791 %\n",
      "Recall: 0.614286 %\n",
      "Fscore: 0.636095 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction)))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6769231"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.eval(binary_accuracy(K.variable(value=test_labels),\n",
    "                       K.variable(value=prediction)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}