hand_gesture_vgg_16.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load training, validation and testing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from data_helper import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------------------------------------\n",
      "[Train (Head)] number of image paths: 12744\n",
      "[Train (Hand)] number of image paths: 12744\n",
      "[Train (Label)] number of labels: 12744\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[Validation (Head)] number of image paths: 2248\n",
      "[Validation (Hand)] number of image paths: 2248\n",
      "[Validation (Label)] number of labels: 2248\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[Test (Head)] number of image paths: 12776\n",
      "[Test (Hand)] number of image paths: 12776\n",
      "[Test (Label)] number of labels: 12776\n"
     ]
    }
   ],
   "source": [
    "IMAGE_FOLDER_PATH = 'dataset/resized/frames/'\n",
    "LABEL_FOLDER_PATH = 'dataset/labels/'\n",
    "\n",
    "train_head_image_paths, train_hand_image_paths, train_labels, \\\n",
    "val_head_image_paths, val_hand_image_paths, val_labels, \\\n",
    "test_head_image_paths, test_hand_image_paths, test_labels = load_dataset(image_folder_path=IMAGE_FOLDER_PATH,\n",
    "                                                                         label_folder_path=LABEL_FOLDER_PATH,\n",
    "                                                                         label_type='ges',\n",
    "                                                                         hand_types=['left', 'right'],\n",
    "                                                                         with_head=True,\n",
    "                                                                         validation_split_ratio=0.15)\n",
    "\n",
    "# Only take hand image paths for baseline\n",
    "train_image_paths =  train_hand_image_paths\n",
    "val_image_paths = val_hand_image_paths\n",
    "test_image_paths = test_hand_image_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use Tensorflow to build computational graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "import tensorflow.contrib.slim.nets\n",
    "import vgg_preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PRETRAINED_VGG_MODEL_PATH = 'model/vgg_16.ckpt'\n",
    "MODEL_PATH = 'model/hand_gesture_vgg_16/hand_gesture_vgg_16_model'\n",
    "\n",
    "num_classes = 24\n",
    "batch_size = 32\n",
    "num_workers = 20\n",
    "max_epochs1 = 30\n",
    "max_epochs2 = 30\n",
    "max_patience = 5 # For early stopping\n",
    "learning_rate1 = 1e-3\n",
    "learning_rate2 = 1e-5\n",
    "dropout_keep_prob = 0.5\n",
    "weight_decay = 5e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 996 ms, sys: 11.9 ms, total: 1.01 s\n",
      "Wall time: 1.01 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def dataset_map_fn(image_path, label, is_training):\n",
    "    # Load image\n",
    "    image_string = tf.read_file(image_path)\n",
    "    image_decoded = tf.image.decode_png(image_string, channels=3)\n",
    "    image = tf.cast(image_decoded, tf.float32)\n",
    "    # Preprocess image\n",
    "    preprocessed_image = tf.cond(is_training,\n",
    "                                 true_fn=lambda: vgg_preprocessing.preprocess_image(image, 224, 224, is_training=True),\n",
    "                                 false_fn=lambda: vgg_preprocessing.preprocess_image(image, 224, 224, is_training=False))\n",
    "    return preprocessed_image, label\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    # ---------------------------------------------------------------------\n",
    "    # Indicates whether we are in training or in test mode\n",
    "    # Since VGG16 has applied `dropout`, we need to disable it when testing.\n",
    "    is_training = tf.placeholder(dtype=tf.bool, name='is_training')\n",
    "    \n",
    "    # Training, validation, testing data to feed in.\n",
    "    image_paths = tf.placeholder(dtype=tf.string, shape=(None,), name='image_paths')\n",
    "    labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='labels')\n",
    "    \n",
    "    # Use dataset API to automatically generate batch data by iterator.\n",
    "    dataset = tf.contrib.data.Dataset.from_tensor_slices((image_paths, labels))\n",
    "    dataset = dataset.map(lambda image_path, label: dataset_map_fn(image_path, label, is_training))\n",
    "    dataset = dataset.shuffle(buffer_size=10000)\n",
    "    batched_dataset = dataset.batch(batch_size)\n",
    "    \n",
    "    # Now we define an iterator that can operator on dataset.\n",
    "    # The iterator can be reinitialized by calling:\n",
    "    # sess.run(dataset_init_op, feed_dict={image_paths=train_image_paths, labels=train_labels}) \n",
    "    # for 1 epoch on the training set.\n",
    "    \n",
    "    # Once this is done, we don't need to feed any value for images and labels\n",
    "    # as they are automatically pulled out from the iterator queues.\n",
    "\n",
    "    # A reinitializable iterator is defined by its structure. We could use the\n",
    "    # `output_types` and `output_shapes` properties of dataset.\n",
    "    # The dataset will be fed with training, validation or testing data.\n",
    "    iterator = tf.contrib.data.Iterator.from_structure(batched_dataset.output_types,\n",
    "                                                       batched_dataset.output_shapes)\n",
    "    \n",
    "    # A batch of data to feed into the networks.\n",
    "    batch_images, batch_labels = iterator.get_next()\n",
    "    dataset_init_op = iterator.make_initializer(batched_dataset)\n",
    "    \n",
    "    # ---------------------------------------------------------------------\n",
    "    # Now that we have set up the data, it's time to set up the model.\n",
    "    # For this example, we'll use VGG-16 pretrained on ImageNet. We will remove the\n",
    "    # last fully connected layer (fc8) and replace it with our own, with an\n",
    "    # output size `num_classes`\n",
    "    # We will first train the last layer for a few epochs.\n",
    "    # Then we will train the entire model on our dataset for a few epochs.\n",
    "\n",
    "    # Get the pretrained model, specifying the num_classes argument to create a new\n",
    "    # fully connected replacing the last one, called \"vgg_16/fc8\"\n",
    "    # Each model has a different architecture, so \"vgg_16/fc8\" will change in another model.\n",
    "    # Here, logits gives us directly the predicted scores we wanted from the images.\n",
    "    # We pass a scope to initialize \"vgg_16/fc8\" weights with he_initializer\n",
    "    vgg = tf.contrib.slim.nets.vgg\n",
    "    with slim.arg_scope(vgg.vgg_arg_scope(weight_decay=weight_decay)):\n",
    "        logits, _ = vgg.vgg_16(batch_images, num_classes=num_classes, is_training=is_training,\n",
    "                               dropout_keep_prob=dropout_keep_prob)\n",
    "    \n",
    "    # Restore only the layers up to fc7 (included)\n",
    "    # Calling function `init_fn(sess)` will load all the pretrained weights.\n",
    "    variables_to_restore = tf.contrib.framework.get_variables_to_restore(exclude=['vgg_16/fc8'])\n",
    "    init_fn = tf.contrib.framework.assign_from_checkpoint_fn(PRETRAINED_VGG_MODEL_PATH, variables_to_restore)\n",
    "\n",
    "    # Initialization operation from scratch for the new \"fc8\" layers\n",
    "    # `get_variables` will only return the variables whose name starts with the given pattern\n",
    "    fc8_variables = tf.contrib.framework.get_variables('vgg_16/fc8')\n",
    "    fc8_init = tf.variables_initializer(fc8_variables)\n",
    "    \n",
    "    # ---------------------------------------------------------------------\n",
    "    # Using tf.losses, any loss is added to the tf.GraphKeys.LOSSES collection\n",
    "    # We can then call the total loss easily\n",
    "    tf.losses.sparse_softmax_cross_entropy(labels=batch_labels, logits=logits)\n",
    "    loss = tf.losses.get_total_loss()\n",
    "    \n",
    "    # First we want to train only the reinitialized last layer fc8 for a few epochs.\n",
    "    # We run minimize the loss only with respect to the fc8 variables (weight and bias).\n",
    "    fc8_optimizer = tf.train.GradientDescentOptimizer(learning_rate1)\n",
    "    fc8_train_op = fc8_optimizer.minimize(loss, var_list=fc8_variables)\n",
    "    \n",
    "    # Then we want to finetune the entire model for a few epochs.\n",
    "    # We run minimize the loss only with respect to all the variables.\n",
    "    full_optimizer = tf.train.GradientDescentOptimizer(learning_rate2)\n",
    "    full_train_op = full_optimizer.minimize(loss)\n",
    "\n",
    "    # Evaluation metrics\n",
    "    prediction = tf.to_int32(tf.argmax(logits, 1))\n",
    "    correct_prediction = tf.equal(prediction, batch_labels)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))    \n",
    "    \n",
    "    # 'Saver' op to save and restore all the variables\n",
    "    saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(sess, loss, correct_prediction, dataset_init_op, feed_dict):\n",
    "    \"\"\"\n",
    "        Evaluation in training loop.\n",
    "        Check the performance of the model on either train, val or test (depending on `dataset_init_op`)\n",
    "        Note: The arguments are tensorflow operators defined in the graph.\n",
    "    \"\"\"\n",
    "    \n",
    "    # Initialize the correct dataset.\n",
    "    sess.run(dataset_init_op, feed_dict=feed_dict)\n",
    "\n",
    "    data_loss = 0\n",
    "    num_correct = 0\n",
    "    num_samples = 0\n",
    "    \n",
    "    # Evaluate on every batch.\n",
    "    while True:\n",
    "        try:\n",
    "            # Disable `is_training` since we have `dropout` in VGG net.\n",
    "            _loss, _correct_prediction = sess.run([loss, correct_prediction], feed_dict={is_training: False})\n",
    "\n",
    "            data_loss += _loss\n",
    "            num_correct += _correct_prediction.sum() # e.g: [True, False, True].sum() = 2\n",
    "            num_samples += _correct_prediction.shape[0] # Batch size\n",
    "            \n",
    "        except tf.errors.OutOfRangeError:\n",
    "            break\n",
    "\n",
    "    data_loss = data_loss / num_samples\n",
    "    acc = num_correct / num_samples\n",
    "\n",
    "    return data_loss, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# --------------------------------------------------------------------------\n",
    "# Now that we have built the graph and finalized it, we define the session.\n",
    "# The session is the interface to *run* the computational graph.\n",
    "# We can call our training operations with `sess.run(train_op)` for instance\n",
    "sess = tf.Session(graph=graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from model/hand_gesture_vgg_16/hand_gesture_vgg_16_model\n"
     ]
    }
   ],
   "source": [
    "# init_fn(sess) # load the pretrained weights\n",
    "# sess.run(fc8_init)  # initialize the new fc8 layer\n",
    "\n",
    "saver.restore(sess, MODEL_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Only train 'fc8' layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/30 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 1/30\n",
      "[Train] loss: 0.06228925008969867 | accuracy: 0.5976145637162587\n",
      "[Validation] loss: 0.08147881431935945 | accuracy: 0.41725978647686834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  3%|▎         | 1/30 [05:38<2:43:45, 338.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/hand_gesture_vgg_16.ckpt\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 2/30\n",
      "[Train] loss: 0.058167793675181406 | accuracy: 0.6263339610797238\n",
      "[Validation] loss: 0.07669812652988366 | accuracy: 0.44483985765124556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  7%|▋         | 2/30 [11:18<2:38:13, 339.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/hand_gesture_vgg_16.ckpt\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 3/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 3/30 [16:54<2:32:09, 338.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.05608863585113506 | accuracy: 0.6345731324544884\n",
      "[Validation] loss: 0.07903092160980048 | accuracy: 0.4435053380782918\n",
      "Model not improved at epoch 3/30. Patience: 1/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 4/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 13%|█▎        | 4/30 [22:30<2:26:16, 337.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.05424258221415552 | accuracy: 0.6541902071563088\n",
      "[Validation] loss: 0.07752268435267791 | accuracy: 0.4288256227758007\n",
      "Model not improved at epoch 4/30. Patience: 2/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 5/30\n",
      "[Train] loss: 0.05321637331990423 | accuracy: 0.6611738857501569\n",
      "[Validation] loss: 0.07711983029316329 | accuracy: 0.4470640569395018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|█▋        | 5/30 [28:12<2:21:08, 338.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/hand_gesture_vgg_16.ckpt\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 6/30\n",
      "[Train] loss: 0.0521653358639222 | accuracy: 0.6711393596986818\n",
      "[Validation] loss: 0.07633064681949141 | accuracy: 0.46174377224199287\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 6/30 [33:51<2:15:32, 338.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/hand_gesture_vgg_16.ckpt\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 7/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 23%|██▎       | 7/30 [39:28<2:09:41, 338.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.051627347614132076 | accuracy: 0.6738072818581293\n",
      "[Validation] loss: 0.07687992127976807 | accuracy: 0.445729537366548\n",
      "Model not improved at epoch 7/30. Patience: 1/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 8/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 27%|██▋       | 8/30 [45:04<2:03:51, 337.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.052398323142820384 | accuracy: 0.665646578782172\n",
      "[Validation] loss: 0.07734628148986776 | accuracy: 0.4314946619217082\n",
      "Model not improved at epoch 8/30. Patience: 2/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 9/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 9/30 [50:41<1:58:03, 337.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.05076000679983557 | accuracy: 0.6792215944758317\n",
      "[Validation] loss: 0.07721551001284047 | accuracy: 0.45062277580071175\n",
      "Model not improved at epoch 9/30. Patience: 3/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 10/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 10/30 [56:16<1:52:17, 336.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.0504849471540026 | accuracy: 0.6858913998744507\n",
      "[Validation] loss: 0.07524671758196956 | accuracy: 0.45462633451957296\n",
      "Model not improved at epoch 10/30. Patience: 4/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 11/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 37%|███▋      | 11/30 [1:01:51<1:46:29, 336.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.05033740173026742 | accuracy: 0.6835373509102323\n",
      "[Validation] loss: 0.07703210652086659 | accuracy: 0.4443950177935943\n",
      "Model not improved at epoch 11/30. Patience: 5/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 12/30\n",
      "[Train] loss: 0.050016819944339805 | accuracy: 0.684557438794727\n",
      "[Validation] loss: 0.07466981154319655 | accuracy: 0.445729537366548\n",
      "Model not improved at epoch 12/30. Patience: 6/5\n",
      "Max patience exceeded. Early stopping.\n"
     ]
    }
   ],
   "source": [
    "max_acc = 0.0\n",
    "patience = 0\n",
    "\n",
    "# Update only the last layer for a few epochs.\n",
    "for epoch in tqdm(range(max_epochs1)):\n",
    "    # Run an epoch over the training data.\n",
    "    print('-'*110)\n",
    "    print('Starting epoch {}/{}'.format(epoch+1, max_epochs1))\n",
    "    # Here we initialize the iterator with the training set.\n",
    "    # This means that we can go through an entire epoch until the iterator becomes empty.\n",
    "    sess.run(dataset_init_op, feed_dict={image_paths: train_image_paths,\n",
    "                                         labels: train_labels,\n",
    "                                         is_training: True})\n",
    "    while True:\n",
    "        try:\n",
    "            _ = sess.run(fc8_train_op, feed_dict={is_training: True})\n",
    "        except tf.errors.OutOfRangeError:\n",
    "            break\n",
    "\n",
    "    # Check performance every epoch\n",
    "    train_loss, train_acc = evaluate(sess, loss, correct_prediction, dataset_init_op,\n",
    "                                     feed_dict={image_paths: train_image_paths,\n",
    "                                                labels: train_labels,\n",
    "                                                is_training: True})\n",
    "    \n",
    "    val_loss, val_acc = evaluate(sess, loss, correct_prediction, dataset_init_op,\n",
    "                                 feed_dict={image_paths: val_image_paths,\n",
    "                                            labels: val_labels,\n",
    "                                            is_training: False})\n",
    "    \n",
    "    print('[Train] loss: {} | accuracy: {}'.format(train_loss, train_acc))\n",
    "    print('[Validation] loss: {} | accuracy: {}'.format(val_loss, val_acc))\n",
    "    \n",
    "    # Save checkpoint\n",
    "    if val_acc > max_acc:\n",
    "        patience = 0\n",
    "        max_acc = val_acc\n",
    "        save_path = saver.save(sess, MODEL_PATH)\n",
    "        print(\"Model updated and saved in file: %s\" % save_path)\n",
    "    else:\n",
    "        patience += 1\n",
    "        print('Model not improved at epoch {}/{}. Patience: {}/{}'.format(epoch+1, max_epochs1, patience, max_patience))\n",
    "    # Early stopping.\n",
    "    if patience > max_patience:\n",
    "        print('Max patience exceeded. Early stopping.')\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train all layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/30 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 1/30\n",
      "[Train] loss: 0.046402791256584315 | accuracy: 0.7221437539234149\n",
      "[Validation] loss: 0.0718847906144064 | accuracy: 0.47330960854092524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  3%|▎         | 1/30 [08:41<4:12:02, 521.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/hand_gesture_vgg_16/hand_gesture_vgg_16_model\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 2/30\n",
      "[Train] loss: 0.04632800730496028 | accuracy: 0.7218298807281858\n",
      "[Validation] loss: 0.07255974247574382 | accuracy: 0.474644128113879\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  7%|▋         | 2/30 [17:24<4:03:33, 521.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/hand_gesture_vgg_16/hand_gesture_vgg_16_model\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 3/30\n",
      "[Train] loss: 0.04683835052388761 | accuracy: 0.7186911487758946\n",
      "[Validation] loss: 0.07246557540952947 | accuracy: 0.47597864768683273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 3/30 [26:05<3:54:48, 521.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/hand_gesture_vgg_16/hand_gesture_vgg_16_model\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 4/30\n",
      "[Train] loss: 0.04689487364968637 | accuracy: 0.7203389830508474\n",
      "[Validation] loss: 0.07189593037252325 | accuracy: 0.4786476868327402\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 13%|█▎        | 4/30 [34:49<3:46:17, 522.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/hand_gesture_vgg_16/hand_gesture_vgg_16_model\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 5/30\n",
      "[Train] loss: 0.04651352033552252 | accuracy: 0.7266949152542372\n",
      "[Validation] loss: 0.07225807133094271 | accuracy: 0.4795373665480427\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|█▋        | 5/30 [43:30<3:37:26, 521.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/hand_gesture_vgg_16/hand_gesture_vgg_16_model\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 6/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 6/30 [52:08<3:28:22, 520.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.046100814996915124 | accuracy: 0.732187696170747\n",
      "[Validation] loss: 0.07216650332420321 | accuracy: 0.476423487544484\n",
      "Model not improved at epoch 6/30. Patience: 1/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 7/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 23%|██▎       | 7/30 [1:00:46<3:19:20, 520.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.04611234199454481 | accuracy: 0.7281073446327684\n",
      "[Validation] loss: 0.07207183317145419 | accuracy: 0.47686832740213525\n",
      "Model not improved at epoch 7/30. Patience: 2/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 8/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 27%|██▋       | 8/30 [1:09:24<3:10:26, 519.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.04612294859057017 | accuracy: 0.7270087884494664\n",
      "[Validation] loss: 0.0725403787614612 | accuracy: 0.47731316725978645\n",
      "Model not improved at epoch 8/30. Patience: 3/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 9/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 9/30 [1:18:04<3:01:49, 519.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.046086961435059366 | accuracy: 0.7277934714375393\n",
      "[Validation] loss: 0.07204095918512854 | accuracy: 0.47686832740213525\n",
      "Model not improved at epoch 9/30. Patience: 4/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 10/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 10/30 [1:26:43<2:53:09, 519.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.04592442379683275 | accuracy: 0.7277934714375393\n",
      "[Validation] loss: 0.07214404016839228 | accuracy: 0.47820284697508897\n",
      "Model not improved at epoch 10/30. Patience: 5/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 11/30\n",
      "[Train] loss: 0.04605437530823197 | accuracy: 0.7268518518518519\n",
      "[Validation] loss: 0.0721421920001719 | accuracy: 0.4786476868327402\n",
      "Model not improved at epoch 11/30. Patience: 6/5\n",
      "Max patience exceeded. Early stopping.\n"
     ]
    }
   ],
   "source": [
    "# Train the entire model for a few more epochs, continuing with the *same* weights.\n",
    "max_acc = 0.0\n",
    "patience = 0\n",
    "for epoch in tqdm(range(max_epochs2)):\n",
    "    # Run an epoch over the training data.\n",
    "    print('-'*110)\n",
    "    print('Starting epoch {}/{}'.format(epoch+1, max_epochs2))\n",
    "    # Here we initialize the iterator with the training set.\n",
    "    # This means that we can go through an entire epoch until the iterator becomes empty.\n",
    "    sess.run(dataset_init_op, feed_dict={image_paths: train_image_paths,\n",
    "                                         labels: train_labels,\n",
    "                                         is_training: True})\n",
    "    while True:\n",
    "        try:\n",
    "            _ = sess.run(full_train_op, feed_dict={is_training: True})    \n",
    "        except tf.errors.OutOfRangeError:\n",
    "            break\n",
    "\n",
    "    # Check performance every epoch\n",
    "    train_loss, train_acc = evaluate(sess, loss, correct_prediction, dataset_init_op,\n",
    "                                     feed_dict={image_paths: train_image_paths,\n",
    "                                                labels: train_labels,\n",
    "                                                is_training: True})\n",
    "    \n",
    "    val_loss, val_acc = evaluate(sess, loss, correct_prediction, dataset_init_op,\n",
    "                                 feed_dict={image_paths: val_image_paths,\n",
    "                                            labels: val_labels,\n",
    "                                            is_training: False})\n",
    "    \n",
    "    print('[Train] loss: {} | accuracy: {}'.format(train_loss, train_acc))\n",
    "    print('[Validation] loss: {} | accuracy: {}'.format(val_loss, val_acc))\n",
    "    \n",
    "    # Save checkpoint\n",
    "    if val_acc > max_acc:\n",
    "        patience = 0\n",
    "        max_acc = val_acc\n",
    "        save_path = saver.save(sess, MODEL_PATH)\n",
    "        print(\"Model updated and saved in file: %s\" % save_path)\n",
    "    else:\n",
    "        patience += 1\n",
    "        print('Model not improved at epoch {}/{}. Patience: {}/{}'.format(epoch+1, max_epochs1, patience, max_patience))\n",
    "    # Early stopping.\n",
    "    if patience > max_patience:\n",
    "        print('Max patience exceeded. Early stopping.')\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test] loss: 0.0594797047996939 | accuracy: 0.6193644333124608\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = evaluate(sess, loss, correct_prediction, dataset_init_op,\n",
    "                               feed_dict={image_paths: test_image_paths,\n",
    "                                          labels: test_labels,\n",
    "                                          is_training: False})\n",
    "\n",
    "print('[Test] loss: {} | accuracy: {}'.format(test_loss, test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}