Convolutional.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Convolutional Network "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "import dataset\n",
    "import tfutil as tfu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Network Construction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "ALPHA=100\n",
    "\n",
    "def conv_inference_op(images, reg_terms, train=True, share=False):\n",
    "    # reshape flat input vectors to 3D 'images' (height x width x channel depth)\n",
    "    h = tf.reshape(images, [-1] + list(dataset.image_dim(include_channels=True)))\n",
    "    \n",
    "    # dropout probability: 50% chance of dropout during training; disabled during evaluation/prediction\n",
    "    keep_prob = 0.5 if train else 1.0\n",
    "    \n",
    "    with tf.variable_scope('conv', reuse=share):\n",
    "        h = tfu.conv_op(h, size=10, channels=[3, 32], stride=1, name='conv1.1')\n",
    "        h = tfu.conv_op(h, size=10, channels=[32, 80], stride=2, name='conv1.2')\n",
    "        h = tfu.pool_op(h, size=2, stride=2, mode='max', name='pool1')\n",
    "        \n",
    "        h = tfu.conv_op(h, size=5, channels=[80, 80], stride=1, name='conv2.1')\n",
    "        h = tfu.conv_op(h, size=5, channels=[80, 160], stride=2, name='conv2.2')\n",
    "        h = tfu.pool_op(h, size=2, stride=2, mode='max', name='pool2')\n",
    "        \n",
    "        h = tfu.conv_op(h, size=3, channels=[160, 160], stride=1, name='conv3.1')\n",
    "        h = tfu.conv_op(h, size=3, channels=[160, 160], stride=1, name='conv3.2')\n",
    "        h = tfu.pool_op(h, size=2, stride=2, mode='max', name='pool3')\n",
    "        \n",
    "        h = tfu.conv_op(h, size=3, channels=[160, 160], stride=1, name='conv4.1')\n",
    "        h = tfu.conv_op(h, size=3, channels=[160, 200], stride=1, name='conv4.2')\n",
    "        \n",
    "        h = tfu.conv_op(h, size=3, channels=[200, 300], stride=1, name='conv5.1')\n",
    "        h = tfu.conv_op(h, size=3, channels=[300, 300], stride=1, name='conv5.2')\n",
    "        h = tfu.pool_op(h, size=2, stride=2, mode='max', name='pool4')\n",
    "        \n",
    "        FC_IN_SIZE = 5 * 5 * 300\n",
    "        FC_HIDDEN_SIZE = 4096\n",
    "        \n",
    "        # flatten output back to 1D vector\n",
    "        h = tf.reshape(h, [-1, FC_IN_SIZE])\n",
    "        \n",
    "        h = tf.nn.dropout(h, keep_prob=keep_prob, name='dropout1')\n",
    "        h = tfu.fc_op(h, channels_in=FC_IN_SIZE, channels_out=FC_HIDDEN_SIZE, name='fc1', reg_terms=reg_terms, alpha=ALPHA, relu=False)\n",
    "\n",
    "        h = tf.nn.dropout(h, keep_prob=keep_prob, name='dropout2')\n",
    "        h = tfu.fc_op(h, channels_in=FC_HIDDEN_SIZE, channels_out=1, name='out', reg_terms=reg_terms, alpha=ALPHA, relu=False)\n",
    "        \n",
    "    return h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training and Evaluation\n",
    "\n",
    "First, I set up some general settings and the arguments to be passed to the training, evaluation and prediction functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "NAME = 'conv.100'\n",
    "conv_reg_terms = {}\n",
    "\n",
    "args = {\n",
    "    'name': NAME,\n",
    "    'inference_op': conv_inference_op,\n",
    "    'inputs': dataset.inputs,\n",
    "    'reg_terms': conv_reg_terms,\n",
    "}\n",
    "\n",
    "training_args = {\n",
    "    'optimizer': tf.train.AdamOptimizer,\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The cell below erases any saved logs, checkpoints and prediction files. Run this to start with the model from scratch instead of resuming from the most recent checkpoint."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tfu.run_cleanup(name=NAME)\n",
    "tfu.run_setup(name=NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Finally, the cells below run training itself. The helper library function `tfu.run_training` allows us to run training multiple times with different learning rates, each time picking up from the last checkpoint saved by the previous operation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 49.4%\n",
      "Validation Accuracy: 51.6%\n",
      "Train Loss: 418838.243\n",
      "Validation Loss: 418838.238\n",
      "Cross Entropy: 418838.812\n",
      "Cross Entropy: 18511.455\n",
      "Cross Entropy: 266.296\n",
      "Cross Entropy: 1.738\n",
      "Train Accuracy: 64.0%\n",
      "Validation Accuracy: 66.0%\n",
      "Train Loss: 0.715\n",
      "Validation Loss: 0.721\n",
      "Cross Entropy: 0.755\n",
      "Cross Entropy: 0.805\n",
      "Cross Entropy: 0.703\n",
      "Cross Entropy: 0.670\n",
      "Train Accuracy: 69.9%\n",
      "Validation Accuracy: 71.8%\n",
      "Train Loss: 0.677\n",
      "Validation Loss: 0.675\n",
      "Cross Entropy: 0.691\n",
      "Cross Entropy: 0.717\n",
      "Cross Entropy: 0.823\n",
      "Cross Entropy: 0.830\n",
      "Train Accuracy: 63.4%\n",
      "Validation Accuracy: 61.1%\n",
      "Train Loss: 0.740\n",
      "Validation Loss: 0.742\n",
      "Cross Entropy: 0.744\n",
      "Cross Entropy: 0.609\n",
      "Cross Entropy: 0.696\n",
      "Cross Entropy: 0.519\n",
      "Done training for 3944 steps.\n",
      "Train Accuracy: 81.3%\n",
      "Validation Accuracy: 82.1%\n",
      "Train Loss: 0.593\n",
      "Validation Loss: 0.594\n"
     ]
    }
   ],
   "source": [
    "final_step = tfu.run_training(\n",
    "    learning_rate=1e-4,\n",
    "    num_epochs=10,\n",
    "    **training_args,\n",
    "    **args,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 81.1%\n",
      "Validation Accuracy: 80.4%\n",
      "Train Loss: 0.467\n",
      "Validation Loss: 0.473\n",
      "Cross Entropy: 0.421\n",
      "Cross Entropy: 0.455\n",
      "Cross Entropy: 0.389\n",
      "Cross Entropy: 0.613\n",
      "Train Accuracy: 83.0%\n",
      "Validation Accuracy: 83.2%\n",
      "Train Loss: 0.415\n",
      "Validation Loss: 0.433\n",
      "Cross Entropy: 0.612\n",
      "Cross Entropy: 0.571\n",
      "Cross Entropy: 0.498\n",
      "Cross Entropy: 0.419\n",
      "Train Accuracy: 81.7%\n",
      "Validation Accuracy: 83.2%\n",
      "Train Loss: 0.399\n",
      "Validation Loss: 0.424\n",
      "Cross Entropy: 0.585\n",
      "Cross Entropy: 0.391\n",
      "Cross Entropy: 0.373\n",
      "Cross Entropy: 0.548\n",
      "Train Accuracy: 84.0%\n",
      "Validation Accuracy: 83.7%\n",
      "Train Loss: 0.399\n",
      "Validation Loss: 0.417\n",
      "Cross Entropy: 0.332\n",
      "Cross Entropy: 0.377\n",
      "Cross Entropy: 0.283\n",
      "Cross Entropy: 0.430\n",
      "Done training for 7889 steps.\n",
      "Train Accuracy: 85.0%\n",
      "Validation Accuracy: 85.2%\n",
      "Train Loss: 0.374\n",
      "Validation Loss: 0.416\n"
     ]
    }
   ],
   "source": [
    "final_step = tfu.run_training(\n",
    "    learning_rate=1e-5,\n",
    "    num_epochs=10,\n",
    "    **training_args,\n",
    "    **args,\n",
    "    step=final_step,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Evaluation and prediction restore the checkpoint saved previously by training. Here, I finally evaluate on the test set to check the model generalises, before generating submissions for Kaggle. Note that the Kaggle submission file is written in a random order (though with each prediction labelled by the image id). On Linux, one can sort it for submission using `sort -t, -nk1 file.csv > file_sorted.csv`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 84.9%\n",
      "Validation Accuracy: 84.8%\n",
      "Test Accuracy: 85.6%\n",
      "Train Loss: 0.389\n",
      "Validation Loss: 0.402\n",
      "Test Loss: 0.382\n",
      "Wrote predictions to ./data/conv.100.csv\n",
      "Wrote predictions to ./data/conv.100_clipped.csv\n"
     ]
    }
   ],
   "source": [
    "tfu.run_eval(**args)\n",
    "tfu.run_prediction(**args)\n",
    "tfu.run_prediction(**args, clip=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "For reference my original network architecture is recorded below. Due to the large convolutional filters, this network seemed to have problems with overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# original convnet architecture that got ~80% on validation set\n",
    "def conv_old_inference_op(images, train=True):\n",
    "    h = tf.reshape(images, [-1,] + list(dataset.image_dim(include_channels=True)))\n",
    "    keep_prob = 0.5 if train else 1.0\n",
    "    \n",
    "    with tf.variable_scope('conv', reuse=(not train)):\n",
    "        h = tfu.conv_op(h, size=40, channels=[3, 16], stride=2, name='conv1')\n",
    "        h = tfu.conv_op(h, size=25, channels=[16, 64], stride=2, name='conv2')\n",
    "        h = tfu.pool_op(h, size=2, stride=2, mode='avg', name='pool1')\n",
    "        # size is now 38 x 38 x 64\n",
    "        h = tfu.conv_op(h, size=16, channels=[64, 128], stride=1, name='conv3')\n",
    "        h = tfu.conv_op(h, size=7, channels=[128, 256], stride=2, name='conv4', padding='VALID')\n",
    "        h = tfu.pool_op(h, size=2, stride=2, mode='max', name='pool2')\n",
    "        \n",
    "        # now size is:\n",
    "        FC_IN_SIZE = 8 * 8 * 256\n",
    "        h = tf.reshape(h, [-1, FC_IN_SIZE])\n",
    "        \n",
    "        h = tfu.fc_op(h, FC_IN_SIZE, 1024, name='fc1')\n",
    "        h = tfu.fc_op(h, 1024, 1024, name='fc2')\n",
    "        h = tf.nn.dropout(h, keep_prob=keep_prob)\n",
    "        \n",
    "        logits = tfu.fc_op(h, 1024, 1, relu=False, name='out')\n",
    "        \n",
    "    return logits"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}