HW7_FCN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Homework 7\n",
    "This homework focuses on fully convolutional networks.\n",
    "\n",
    "## Part 0: Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import util\n",
    "\n",
    "# Colors to visualize the labeling\n",
    "COLORS = np.array([(0,0,0), (255,0,0), (0,255,0), (255,255,0), (0,0,255), (255,255,255)], dtype=np.uint8)\n",
    "CROP_SIZE = 64\n",
    "\n",
    "def parser(record):\n",
    "    # Parse the TF record\n",
    "    parsed = tf.parse_single_example(record, features={\n",
    "        'height': tf.FixedLenFeature([], tf.int64),\n",
    "        'width': tf.FixedLenFeature([], tf.int64),\n",
    "        'image_raw': tf.FixedLenFeature([], tf.string),\n",
    "        'label_raw': tf.FixedLenFeature([], tf.string)\n",
    "    })\n",
    "    # Load the data and format it\n",
    "    H = tf.cast(parsed['height'], tf.int32)\n",
    "    W = tf.cast(parsed['width'], tf.int32)\n",
    "    image = tf.reshape(tf.decode_raw(parsed[\"image_raw\"], tf.uint8), [H,W,3])\n",
    "    label = tf.reshape(tf.decode_raw(parsed[\"label_raw\"], tf.uint8), [H,W])\n",
    "    \n",
    "    ## Data augmentation\n",
    "    # Stack the image and labels to make sure the same operations are applied\n",
    "    data = tf.concat([image, label[:,:,None]], axis=-1)\n",
    "    \n",
    "    # TODO: Apply the data augmentation (you should both crop the images randomly and flip them)\n",
    "    data= tf.image.random_flip_left_right(data)\n",
    "    \n",
    "    return data[:,:,:-1], data[:,:,-1]\n",
    "\n",
    "def load_dataset(tfrecord):\n",
    "    # Load the dataset\n",
    "    dataset = tf.contrib.data.TFRecordDataset(tfrecord)\n",
    "\n",
    "    # Parse the tf record entries\n",
    "    dataset = dataset.map(parser, num_threads=8, output_buffer_size=1024)\n",
    "\n",
    "    # Shuffle the data, batch it and run this for multiple epochs\n",
    "    dataset = dataset.shuffle(buffer_size=10000)\n",
    "    dataset = dataset.batch(32) # note make this 32, my gpu did not work with 32 \n",
    "    dataset = dataset.repeat()\n",
    "    return dataset\n",
    "\n",
    "# We still have 6 classes\n",
    "num_classes = 6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 1: Define your convnet\n",
    "Important note. The label frequency is horribly inbalanced for this task. On the training set\n",
    "```[ 0.66839117, 0.00382957, 0.00092516, 0.00345217, 0.00339063, 0.3200113 ]```\n",
    "On the validation set\n",
    "```[ 0.68367316, 0.00392016, 0.00165766, 0.00194697, 0.0034067, 0.30539535]```\n",
    "Tux, bonus, objects and enemies make up less than 1.5% of all labels overall.\n",
    "You should reweight the loss to address this, if you don't your model will likely ignore all but background and tile labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a new log directory (if you run low on disk space you can either disable this or delete old logs)\n",
    "# run: `tensorboard --logdir log` to see all the nice summaries\n",
    "for n_model in range(1000):\n",
    "    LOG_DIR = 'log/model_%d'%n_model\n",
    "    from os import path\n",
    "    if not path.exists(LOG_DIR):\n",
    "        break\n",
    "\n",
    "# Lets clear the tensorflow graph, so that you don't have to restart the notebook every time you change the network\n",
    "tf.reset_default_graph()\n",
    "\n",
    "TF_COLORS = tf.constant(COLORS)\n",
    "\n",
    "train_data = load_dataset('train.tfrecord')\n",
    "valid_data = load_dataset('valid.tfrecord')\n",
    "\n",
    "# Create an iterator for the datasets\n",
    "# The iterator allows us to quickly switch between training and validataion\n",
    "iterator = tf.contrib.data.Iterator.from_structure(train_data.output_types, ((None,None,None,3), (None,None,None)))\n",
    "\n",
    "# and fetch the next images from the dataset (every time next_image is evaluated a new image set of 32 images is returned)\n",
    "next_image, next_label = iterator.get_next()\n",
    "\n",
    "# Define operations that switch between train and valid\n",
    "switch_train_op = iterator.make_initializer(train_data)\n",
    "switch_valid_op = iterator.make_initializer(valid_data)\n",
    "\n",
    "# Convert the input\n",
    "image = tf.cast(next_image, tf.float32)\n",
    "label = tf.cast(next_label, tf.int32)\n",
    "\n",
    "# Whiten the input\n",
    "inputs = tf.identity(image, name='inputs')\n",
    "white_inputs = (inputs - 100.) / 72.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Conv/Relu:0\", shape=(?, ?, ?, 10), dtype=float32)\n",
      "Tensor(\"Conv_1/Relu:0\", shape=(?, ?, ?, 25), dtype=float32)\n",
      "Tensor(\"Conv_2/Relu:0\", shape=(?, ?, ?, 25), dtype=float32)\n",
      "Tensor(\"Conv_3/Relu:0\", shape=(?, ?, ?, 50), dtype=float32)\n",
      "Tensor(\"Conv2d_transpose/Relu:0\", shape=(?, ?, ?, 25), dtype=float32)\n",
      "Tensor(\"Conv2d_transpose_1/Relu:0\", shape=(?, ?, ?, 10), dtype=float32)\n",
      "Tensor(\"Conv2d_transpose_2/Relu:0\", shape=(?, ?, ?, 50), dtype=float32)\n",
      "Tensor(\"Conv2d_transpose_3/Relu:0\", shape=(?, ?, ?, 10), dtype=float32)\n",
      "Tensor(\"Conv2d_transpose_4/Relu:0\", shape=(?, ?, ?, 50), dtype=float32)\n",
      "Tensor(\"Conv2d_transpose_6/Relu:0\", shape=(?, ?, ?, 10), dtype=float32)\n",
      "filter_concat: Tensor(\"concat:0\", shape=(?, ?, ?, 40), dtype=float32)\n",
      "output: Tensor(\"Conv_4/Relu:0\", shape=(?, ?, ?, 6), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# TODO: Define your convnet here\n",
    "# white_inputs = tf.placeholder(tf.float32, shape=(None,32,32,3))\n",
    "h1 = tf.contrib.layers.conv2d(white_inputs, num_outputs=10, kernel_size=(5, 5), stride=(2, 2))\n",
    "print(h1)\n",
    "h2 = tf.contrib.layers.conv2d(h1, num_outputs=25, kernel_size=(5, 5), stride=(2, 2))\n",
    "print(h2)\n",
    "\n",
    "h3 = tf.contrib.layers.conv2d(h2, num_outputs=25, kernel_size=(3, 3), stride=(2, 2))\n",
    "print(h3)\n",
    "\n",
    "h4 = tf.contrib.layers.conv2d(h3, num_outputs=50, kernel_size=(3, 3), stride=(2, 2))\n",
    "print(h4)\n",
    "\n",
    "h1t = tf.contrib.layers.conv2d_transpose(h4, num_outputs=25, kernel_size=(5, 5), stride=(2, 2))\n",
    "h1t_skip = tf.contrib.layers.conv2d_transpose(h4, num_outputs=10, kernel_size=(5, 5), stride=(16, 16))\n",
    "print(h1t)\n",
    "print(h1t_skip)\n",
    "\n",
    "h2t = tf.contrib.layers.conv2d_transpose(h1t, num_outputs=50,kernel_size=(5, 5), stride=(2, 2))\n",
    "h2t_skip = tf.contrib.layers.conv2d_transpose(h3, num_outputs=10, kernel_size=(5, 5), stride=(8, 8))\n",
    "print(h2t)\n",
    "print(h2t_skip)\n",
    "\n",
    "h3t = tf.contrib.layers.conv2d_transpose(h2t, num_outputs=50,kernel_size=(5, 5), stride=(2, 2))\n",
    "h3t_skip = tf.contrib.layers.conv2d_transpose(h1, num_outputs=10,kernel_size=(5, 5), stride=(2, 2))\n",
    "print(h3t)\n",
    "\n",
    "h4t = tf.contrib.layers.conv2d_transpose(h3t, num_outputs=10, kernel_size=(5, 5), stride=(2, 2))\n",
    "print(h4t)\n",
    "\n",
    "\n",
    "filter_concat = tf.concat([h1t_skip, h2t_skip, h3t_skip, h4t], axis=-1)\n",
    "print('filter_concat: ' + str(filter_concat))\n",
    "\n",
    "logit = tf.contrib.layers.conv2d(filter_concat, num_outputs=6, kernel_size=(1, 1), stride=(1, 1))\n",
    "print('output: ' + str(logit))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of variables used  183146\n"
     ]
    }
   ],
   "source": [
    "# Let's compute the output labeling\n",
    "output = tf.identity(tf.argmax(logit, axis=-1), name='output')\n",
    "\n",
    "# TODO: Define a weight per class here (try a function of the class frequencies)\n",
    "# This is one of the most important steps to get the class accuracy higher\n",
    "loss_weight = tf.constant([ 1., 100., 500., 100., 100., 1. ])\n",
    "# Broadcast the weights spatially\n",
    "weight = tf.gather_nd(loss_weight,label[:,:,:,None])\n",
    "\n",
    "# Define the loss function\n",
    "loss = tf.reduce_sum(weight * tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logit, labels=label)) / tf.reduce_sum(weight)\n",
    "\n",
    "# Let's weight the regularization loss down, otherwise it will hurt the model performance\n",
    "# You can tune this weight if you wish\n",
    "regularization_loss = tf.losses.get_regularization_loss()\n",
    "total_loss = loss + 1e-6 * regularization_loss\n",
    "\n",
    "# Adam will likely converge much faster than SGD for this assignment.\n",
    "optimizer = tf.train.AdamOptimizer(0.001, 0.9, 0.999)\n",
    "\n",
    "# use that optimizer on your loss function (control_dependencies makes sure any \n",
    "# batch_norm parameters are properly updated)\n",
    "with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):\n",
    "    opt = optimizer.minimize(total_loss)\n",
    "confusion = tf.confusion_matrix(labels=tf.reshape(label,[-1]), predictions=tf.reshape(output,[-1]), num_classes=num_classes)\n",
    "\n",
    "# Let's define some summaries for tensorboard\n",
    "colored_label = tf.gather_nd(TF_COLORS, label[:,:,:,None])\n",
    "colored_output = tf.gather_nd(TF_COLORS, output[:,:,:,None])\n",
    "tf.summary.image('confusion', tf.cast(confusion[None,:,:,None], tf.float32), max_outputs=1)\n",
    "tf.summary.image('image', next_image, max_outputs=3)\n",
    "tf.summary.image('label', colored_label, max_outputs=3)\n",
    "tf.summary.image('output', colored_output, max_outputs=3)\n",
    "tf.summary.scalar('loss', tf.placeholder(tf.float32, name='loss'))\n",
    "tf.summary.scalar('accuracy', tf.placeholder(tf.float32, name='accuracy'))\n",
    "tf.summary.scalar('class_accuracy', tf.placeholder(tf.float32, name='class_accuracy'))\n",
    "tf.summary.scalar('jaccard', tf.placeholder(tf.float32, name='jaccard'))\n",
    "tf.summary.scalar('val_accuracy', tf.placeholder(tf.float32, name='val_accuracy'))\n",
    "tf.summary.scalar('val_class_accuracy', tf.placeholder(tf.float32, name='val_class_accuracy'))\n",
    "tf.summary.scalar('val_jaccard', tf.placeholder(tf.float32, name='val_jaccard'))\n",
    "\n",
    "merged_summary = tf.summary.merge_all()\n",
    "summary_writer = tf.summary.FileWriter(LOG_DIR, tf.get_default_graph())\n",
    "\n",
    "# Let's compute the model size\n",
    "print( \"Total number of variables used \", np.sum([v.get_shape().num_elements() for v in tf.trainable_variables()]) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 2: Training\n",
    "\n",
    "Training might take up to 20 min depending on your architecture (and if you have a GPU or not)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  0] Loss: 1.775  \t  A.: 0.413  CA.: 0.190  J.: 0.085  \t  Val A.: 0.561  CA.: 0.204  J.: 0.103\n",
      "[  1] Loss: 1.731  \t  A.: 0.579  CA.: 0.206  J.: 0.109  \t  Val A.: 0.620  CA.: 0.190  J.: 0.117\n",
      "[  2] Loss: 1.664  \t  A.: 0.589  CA.: 0.249  J.: 0.113  \t  Val A.: 0.633  CA.: 0.253  J.: 0.127\n",
      "saving\n",
      "[  3] Loss: 1.599  \t  A.: 0.581  CA.: 0.309  J.: 0.117  \t  Val A.: 0.650  CA.: 0.322  J.: 0.130\n",
      "saving\n",
      "[  4] Loss: 1.521  \t  A.: 0.586  CA.: 0.354  J.: 0.121  \t  Val A.: 0.638  CA.: 0.370  J.: 0.133\n",
      "saving\n",
      "[  5] Loss: 1.447  \t  A.: 0.595  CA.: 0.384  J.: 0.124  \t  Val A.: 0.607  CA.: 0.413  J.: 0.134\n",
      "saving\n",
      "[  6] Loss: 1.350  \t  A.: 0.579  CA.: 0.448  J.: 0.125  \t  Val A.: 0.582  CA.: 0.437  J.: 0.129\n",
      "saving\n",
      "[  7] Loss: 1.339  \t  A.: 0.553  CA.: 0.466  J.: 0.122  \t  Val A.: 0.631  CA.: 0.405  J.: 0.136\n",
      "[  8] Loss: 1.307  \t  A.: 0.561  CA.: 0.483  J.: 0.130  \t  Val A.: 0.574  CA.: 0.424  J.: 0.133\n",
      "[  9] Loss: 1.240  \t  A.: 0.549  CA.: 0.514  J.: 0.132  \t  Val A.: 0.651  CA.: 0.421  J.: 0.161\n",
      "[ 10] Loss: 1.171  \t  A.: 0.533  CA.: 0.539  J.: 0.135  \t  Val A.: 0.686  CA.: 0.418  J.: 0.146\n",
      "[ 11] Loss: 1.137  \t  A.: 0.551  CA.: 0.541  J.: 0.146  \t  Val A.: 0.586  CA.: 0.455  J.: 0.151\n",
      "saving\n",
      "[ 12] Loss: 1.074  \t  A.: 0.566  CA.: 0.565  J.: 0.144  \t  Val A.: 0.637  CA.: 0.470  J.: 0.156\n",
      "saving\n",
      "[ 13] Loss: 1.035  \t  A.: 0.556  CA.: 0.579  J.: 0.144  \t  Val A.: 0.552  CA.: 0.461  J.: 0.140\n",
      "[ 14] Loss: 1.083  \t  A.: 0.537  CA.: 0.562  J.: 0.136  \t  Val A.: 0.584  CA.: 0.482  J.: 0.149\n",
      "saving\n",
      "[ 15] Loss: 1.010  \t  A.: 0.579  CA.: 0.566  J.: 0.146  \t  Val A.: 0.657  CA.: 0.493  J.: 0.151\n",
      "saving\n",
      "[ 16] Loss: 1.063  \t  A.: 0.558  CA.: 0.573  J.: 0.142  \t  Val A.: 0.622  CA.: 0.485  J.: 0.154\n",
      "[ 17] Loss: 0.975  \t  A.: 0.585  CA.: 0.602  J.: 0.153  \t  Val A.: 0.608  CA.: 0.487  J.: 0.160\n",
      "[ 18] Loss: 1.015  \t  A.: 0.540  CA.: 0.602  J.: 0.150  \t  Val A.: 0.634  CA.: 0.496  J.: 0.159\n",
      "[ 19] Loss: 0.917  \t  A.: 0.554  CA.: 0.628  J.: 0.142  \t  Val A.: 0.618  CA.: 0.501  J.: 0.150\n",
      "saving\n",
      "[ 20] Loss: 0.931  \t  A.: 0.565  CA.: 0.636  J.: 0.150  \t  Val A.: 0.639  CA.: 0.520  J.: 0.163\n",
      "saving\n",
      "[ 21] Loss: 0.911  \t  A.: 0.581  CA.: 0.636  J.: 0.156  \t  Val A.: 0.613  CA.: 0.472  J.: 0.178\n",
      "[ 22] Loss: 0.865  \t  A.: 0.557  CA.: 0.661  J.: 0.166  \t  Val A.: 0.588  CA.: 0.538  J.: 0.168\n",
      "saving\n",
      "[ 23] Loss: 0.880  \t  A.: 0.584  CA.: 0.661  J.: 0.166  \t  Val A.: 0.672  CA.: 0.564  J.: 0.180\n",
      "saving\n",
      "[ 24] Loss: 0.928  \t  A.: 0.589  CA.: 0.600  J.: 0.167  \t  Val A.: 0.574  CA.: 0.519  J.: 0.181\n",
      "[ 25] Loss: 0.819  \t  A.: 0.575  CA.: 0.674  J.: 0.176  \t  Val A.: 0.616  CA.: 0.510  J.: 0.174\n",
      "[ 26] Loss: 0.870  \t  A.: 0.572  CA.: 0.660  J.: 0.168  \t  Val A.: 0.622  CA.: 0.591  J.: 0.191\n",
      "saving\n",
      "[ 27] Loss: 0.830  \t  A.: 0.572  CA.: 0.676  J.: 0.187  \t  Val A.: 0.613  CA.: 0.542  J.: 0.179\n",
      "[ 28] Loss: 0.823  \t  A.: 0.606  CA.: 0.683  J.: 0.179  \t  Val A.: 0.648  CA.: 0.535  J.: 0.195\n",
      "[ 29] Loss: 0.814  \t  A.: 0.596  CA.: 0.676  J.: 0.183  \t  Val A.: 0.643  CA.: 0.540  J.: 0.188\n",
      "[ 30] Loss: 0.825  \t  A.: 0.566  CA.: 0.691  J.: 0.182  \t  Val A.: 0.646  CA.: 0.497  J.: 0.206\n",
      "[ 31] Loss: 0.796  \t  A.: 0.585  CA.: 0.694  J.: 0.167  \t  Val A.: 0.651  CA.: 0.565  J.: 0.202\n",
      "[ 32] Loss: 0.749  \t  A.: 0.588  CA.: 0.703  J.: 0.195  \t  Val A.: 0.645  CA.: 0.509  J.: 0.206\n",
      "[ 33] Loss: 0.816  \t  A.: 0.590  CA.: 0.681  J.: 0.181  \t  Val A.: 0.564  CA.: 0.561  J.: 0.192\n",
      "[ 34] Loss: 0.734  \t  A.: 0.597  CA.: 0.711  J.: 0.197  \t  Val A.: 0.630  CA.: 0.552  J.: 0.202\n",
      "[ 35] Loss: 0.761  \t  A.: 0.584  CA.: 0.700  J.: 0.184  \t  Val A.: 0.575  CA.: 0.574  J.: 0.194\n",
      "[ 36] Loss: 0.712  \t  A.: 0.597  CA.: 0.724  J.: 0.203  \t  Val A.: 0.640  CA.: 0.577  J.: 0.211\n",
      "[ 37] Loss: 0.770  \t  A.: 0.590  CA.: 0.696  J.: 0.193  \t  Val A.: 0.667  CA.: 0.586  J.: 0.216\n",
      "[ 38] Loss: 0.715  \t  A.: 0.605  CA.: 0.714  J.: 0.206  \t  Val A.: 0.644  CA.: 0.567  J.: 0.196\n",
      "[ 39] Loss: 0.772  \t  A.: 0.604  CA.: 0.694  J.: 0.206  \t  Val A.: 0.649  CA.: 0.578  J.: 0.187\n",
      "[ 40] Loss: 0.646  \t  A.: 0.600  CA.: 0.720  J.: 0.191  \t  Val A.: 0.602  CA.: 0.561  J.: 0.177\n",
      "[ 41] Loss: 0.715  \t  A.: 0.602  CA.: 0.711  J.: 0.203  \t  Val A.: 0.652  CA.: 0.550  J.: 0.192\n",
      "[ 42] Loss: 0.707  \t  A.: 0.603  CA.: 0.707  J.: 0.202  \t  Val A.: 0.554  CA.: 0.568  J.: 0.190\n",
      "[ 43] Loss: 0.806  \t  A.: 0.585  CA.: 0.697  J.: 0.198  \t  Val A.: 0.669  CA.: 0.543  J.: 0.202\n",
      "[ 44] Loss: 0.697  \t  A.: 0.617  CA.: 0.723  J.: 0.205  \t  Val A.: 0.645  CA.: 0.586  J.: 0.207\n",
      "[ 45] Loss: 0.675  \t  A.: 0.621  CA.: 0.720  J.: 0.202  \t  Val A.: 0.658  CA.: 0.571  J.: 0.196\n",
      "[ 46] Loss: 0.677  \t  A.: 0.576  CA.: 0.726  J.: 0.195  \t  Val A.: 0.666  CA.: 0.554  J.: 0.196\n",
      "[ 47] Loss: 0.690  \t  A.: 0.606  CA.: 0.721  J.: 0.190  \t  Val A.: 0.625  CA.: 0.578  J.: 0.199\n",
      "[ 48] Loss: 0.638  \t  A.: 0.626  CA.: 0.727  J.: 0.206  \t  Val A.: 0.642  CA.: 0.570  J.: 0.204\n",
      "[ 49] Loss: 0.606  \t  A.: 0.617  CA.: 0.731  J.: 0.191  \t  Val A.: 0.572  CA.: 0.617  J.: 0.197\n",
      "saving\n",
      "[ 50] Loss: 0.647  \t  A.: 0.603  CA.: 0.731  J.: 0.201  \t  Val A.: 0.642  CA.: 0.546  J.: 0.187\n",
      "[ 51] Loss: 0.650  \t  A.: 0.595  CA.: 0.740  J.: 0.190  \t  Val A.: 0.628  CA.: 0.581  J.: 0.207\n",
      "[ 52] Loss: 0.672  \t  A.: 0.617  CA.: 0.716  J.: 0.195  \t  Val A.: 0.555  CA.: 0.576  J.: 0.186\n",
      "[ 53] Loss: 0.606  \t  A.: 0.621  CA.: 0.744  J.: 0.197  \t  Val A.: 0.658  CA.: 0.599  J.: 0.218\n",
      "[ 54] Loss: 0.673  \t  A.: 0.610  CA.: 0.729  J.: 0.200  \t  Val A.: 0.679  CA.: 0.592  J.: 0.207\n",
      "[ 55] Loss: 0.573  \t  A.: 0.631  CA.: 0.751  J.: 0.204  \t  Val A.: 0.655  CA.: 0.632  J.: 0.198\n",
      "saving\n",
      "[ 56] Loss: 0.574  \t  A.: 0.623  CA.: 0.750  J.: 0.194  \t  Val A.: 0.652  CA.: 0.553  J.: 0.184\n",
      "[ 57] Loss: 0.588  \t  A.: 0.601  CA.: 0.749  J.: 0.200  \t  Val A.: 0.668  CA.: 0.616  J.: 0.202\n",
      "[ 58] Loss: 0.562  \t  A.: 0.643  CA.: 0.749  J.: 0.208  \t  Val A.: 0.636  CA.: 0.670  J.: 0.197\n",
      "saving\n",
      "[ 59] Loss: 0.548  \t  A.: 0.638  CA.: 0.756  J.: 0.217  \t  Val A.: 0.635  CA.: 0.596  J.: 0.193\n",
      "[ 60] Loss: 0.568  \t  A.: 0.614  CA.: 0.753  J.: 0.203  \t  Val A.: 0.634  CA.: 0.601  J.: 0.195\n",
      "[ 61] Loss: 0.622  \t  A.: 0.612  CA.: 0.741  J.: 0.191  \t  Val A.: 0.659  CA.: 0.626  J.: 0.200\n",
      "[ 62] Loss: 0.609  \t  A.: 0.625  CA.: 0.746  J.: 0.211  \t  Val A.: 0.680  CA.: 0.659  J.: 0.233\n",
      "[ 63] Loss: 0.552  \t  A.: 0.715  CA.: 0.788  J.: 0.266  \t  Val A.: 0.836  CA.: 0.687  J.: 0.351\n",
      "saving\n",
      "[ 64] Loss: 0.511  \t  A.: 0.761  CA.: 0.820  J.: 0.332  \t  Val A.: 0.841  CA.: 0.704  J.: 0.349\n",
      "saving\n",
      "[ 65] Loss: 0.518  \t  A.: 0.811  CA.: 0.831  J.: 0.346  \t  Val A.: 0.854  CA.: 0.723  J.: 0.376\n",
      "saving\n",
      "[ 66] Loss: 0.436  \t  A.: 0.827  CA.: 0.859  J.: 0.364  \t  Val A.: 0.854  CA.: 0.701  J.: 0.348\n",
      "[ 67] Loss: 0.444  \t  A.: 0.851  CA.: 0.861  J.: 0.381  \t  Val A.: 0.896  CA.: 0.736  J.: 0.399\n",
      "saving\n",
      "[ 68] Loss: 0.412  \t  A.: 0.872  CA.: 0.876  J.: 0.403  \t  Val A.: 0.871  CA.: 0.731  J.: 0.370\n",
      "[ 69] Loss: 0.489  \t  A.: 0.847  CA.: 0.852  J.: 0.374  \t  Val A.: 0.801  CA.: 0.714  J.: 0.343\n",
      "[ 70] Loss: 0.393  \t  A.: 0.867  CA.: 0.879  J.: 0.411  \t  Val A.: 0.889  CA.: 0.722  J.: 0.388\n",
      "[ 71] Loss: 0.368  \t  A.: 0.855  CA.: 0.889  J.: 0.390  \t  Val A.: 0.897  CA.: 0.728  J.: 0.412\n",
      "[ 72] Loss: 0.361  \t  A.: 0.872  CA.: 0.890  J.: 0.412  \t  Val A.: 0.864  CA.: 0.744  J.: 0.373\n",
      "saving\n",
      "[ 73] Loss: 0.388  \t  A.: 0.875  CA.: 0.885  J.: 0.403  \t  Val A.: 0.900  CA.: 0.757  J.: 0.399\n",
      "saving\n",
      "[ 74] Loss: 0.373  \t  A.: 0.886  CA.: 0.885  J.: 0.404  \t  Val A.: 0.908  CA.: 0.765  J.: 0.435\n",
      "saving\n",
      "[ 75] Loss: 0.422  \t  A.: 0.880  CA.: 0.869  J.: 0.402  \t  Val A.: 0.898  CA.: 0.751  J.: 0.408\n",
      "[ 76] Loss: 0.387  \t  A.: 0.862  CA.: 0.884  J.: 0.392  \t  Val A.: 0.884  CA.: 0.744  J.: 0.383\n",
      "[ 77] Loss: 0.300  \t  A.: 0.878  CA.: 0.907  J.: 0.434  \t  Val A.: 0.887  CA.: 0.772  J.: 0.401\n",
      "saving\n",
      "[ 78] Loss: 0.385  \t  A.: 0.874  CA.: 0.884  J.: 0.397  \t  Val A.: 0.896  CA.: 0.770  J.: 0.412\n",
      "[ 79] Loss: 0.325  \t  A.: 0.888  CA.: 0.902  J.: 0.425  \t  Val A.: 0.900  CA.: 0.800  J.: 0.427\n",
      "saving\n",
      "[ 80] Loss: 0.312  \t  A.: 0.890  CA.: 0.901  J.: 0.409  \t  Val A.: 0.906  CA.: 0.759  J.: 0.418\n",
      "[ 81] Loss: 0.369  \t  A.: 0.888  CA.: 0.877  J.: 0.416  \t  Val A.: 0.903  CA.: 0.733  J.: 0.394\n",
      "[ 82] Loss: 0.368  \t  A.: 0.874  CA.: 0.895  J.: 0.408  \t  Val A.: 0.883  CA.: 0.791  J.: 0.406\n",
      "[ 83] Loss: 0.358  \t  A.: 0.875  CA.: 0.892  J.: 0.419  \t  Val A.: 0.847  CA.: 0.733  J.: 0.351\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 84] Loss: 0.305  \t  A.: 0.888  CA.: 0.906  J.: 0.427  \t  Val A.: 0.874  CA.: 0.736  J.: 0.396\n",
      "[ 85] Loss: 0.320  \t  A.: 0.888  CA.: 0.902  J.: 0.410  \t  Val A.: 0.874  CA.: 0.745  J.: 0.400\n",
      "[ 86] Loss: 0.317  \t  A.: 0.883  CA.: 0.903  J.: 0.425  \t  Val A.: 0.879  CA.: 0.769  J.: 0.432\n",
      "[ 87] Loss: 0.323  \t  A.: 0.883  CA.: 0.905  J.: 0.420  \t  Val A.: 0.906  CA.: 0.777  J.: 0.412\n",
      "[ 88] Loss: 0.297  \t  A.: 0.902  CA.: 0.910  J.: 0.441  \t  Val A.: 0.902  CA.: 0.736  J.: 0.402\n",
      "[ 89] Loss: 0.289  \t  A.: 0.898  CA.: 0.906  J.: 0.428  \t  Val A.: 0.908  CA.: 0.760  J.: 0.393\n",
      "[ 90] Loss: 0.263  \t  A.: 0.895  CA.: 0.917  J.: 0.423  \t  Val A.: 0.883  CA.: 0.747  J.: 0.399\n",
      "[ 91] Loss: 0.299  \t  A.: 0.907  CA.: 0.908  J.: 0.440  \t  Val A.: 0.913  CA.: 0.720  J.: 0.385\n",
      "[ 92] Loss: 0.306  \t  A.: 0.890  CA.: 0.908  J.: 0.409  \t  Val A.: 0.903  CA.: 0.774  J.: 0.380\n",
      "[ 93] Loss: 0.326  \t  A.: 0.888  CA.: 0.902  J.: 0.415  \t  Val A.: 0.896  CA.: 0.731  J.: 0.372\n",
      "[ 94] Loss: 0.313  \t  A.: 0.895  CA.: 0.902  J.: 0.438  \t  Val A.: 0.881  CA.: 0.729  J.: 0.412\n",
      "[ 95] Loss: 0.318  \t  A.: 0.877  CA.: 0.906  J.: 0.416  \t  Val A.: 0.919  CA.: 0.713  J.: 0.441\n",
      "[ 96] Loss: 0.313  \t  A.: 0.893  CA.: 0.901  J.: 0.426  \t  Val A.: 0.910  CA.: 0.715  J.: 0.421\n",
      "[ 97] Loss: 0.279  \t  A.: 0.899  CA.: 0.917  J.: 0.430  \t  Val A.: 0.909  CA.: 0.807  J.: 0.413\n",
      "saving\n",
      "[ 98] Loss: 0.263  \t  A.: 0.901  CA.: 0.921  J.: 0.434  \t  Val A.: 0.894  CA.: 0.740  J.: 0.379\n",
      "[ 99] Loss: 0.247  \t  A.: 0.906  CA.: 0.923  J.: 0.462  \t  Val A.: 0.901  CA.: 0.761  J.: 0.383\n"
     ]
    }
   ],
   "source": [
    "def accuracy(confusion):\n",
    "    # Overall pixelwise accuracy\n",
    "    # This metric heavily favors tiles and background (as they are most frequent)\n",
    "    return np.sum(np.diag(confusion)) / np.sum(confusion)\n",
    "\n",
    "def class_accuracy(confusion):\n",
    "    # Class wise accuracy\n",
    "    # This metric normalizes for class frequencies and favors small classes\n",
    "    return np.mean(np.diag(confusion) / (np.sum(confusion, axis=1) + 1e-10))\n",
    "\n",
    "def jaccard(confusion):\n",
    "    # Jaccard index\n",
    "    # A mix of the above, neither favors small or large classes much\n",
    "    D = np.diag(confusion)\n",
    "    return np.mean( D / (np.sum(confusion, axis=1) + np.sum(confusion, axis=0) - D + 1e-10))\n",
    "\n",
    "# Start a session\n",
    "sess = tf.Session()\n",
    "\n",
    "# Set up training\n",
    "sess.run(tf.global_variables_initializer())\n",
    "creation=0\n",
    "# Run the training for some iterations\n",
    "for it in range(100):\n",
    "    sess.run(switch_train_op)\n",
    "\n",
    "    total_confusion = np.zeros((num_classes, num_classes))\n",
    "    loss_vals = []\n",
    "    # Run 10 training iterations and 1 validation iteration\n",
    "    for i in range(10):\n",
    "        confusion_val, loss_val, _ = sess.run([confusion, loss, opt])\n",
    "        total_confusion += confusion_val\n",
    "        loss_vals.append(loss_val)\n",
    "    \n",
    "    sess.run(switch_valid_op)\n",
    "    confusion_val = sess.run([confusion])\n",
    "    confusion_val=np.ravel(confusion_val).reshape(6,6)\n",
    "    \n",
    "    # Let's update tensorboard\n",
    "    summary_writer.add_summary( sess.run(merged_summary, {'loss:0': np.mean(loss_vals), 'accuracy:0': accuracy(total_confusion), 'class_accuracy:0': class_accuracy(total_confusion), 'jaccard:0': jaccard(total_confusion), 'val_accuracy:0': accuracy(confusion_val), 'val_class_accuracy:0': class_accuracy(confusion_val), 'val_jaccard:0': jaccard(confusion_val)}), it )\n",
    "    print('[%3d] Loss: %0.3f  \\t  A.: %0.3f  CA.: %0.3f  J.: %0.3f  \\t  Val A.: %0.3f  CA.: %0.3f  J.: %0.3f'%(it, np.mean(loss_vals), accuracy(total_confusion), class_accuracy(total_confusion), jaccard(total_confusion), accuracy(confusion_val), class_accuracy(confusion_val), jaccard(confusion_val)))    \n",
    "    \n",
    "    if creation==0:\n",
    "        present_cal_acc= class_accuracy(confusion_val)\n",
    "        creation+=1\n",
    "    if present_cal_acc+0.005 < class_accuracy(confusion_val):\n",
    "        present_cal_acc= class_accuracy(confusion_val)\n",
    "        print(\"saving\")\n",
    "        util.save('assignment7_best_'+str(present_cal_acc)+'.tfg', session=sess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 3: Evaluation\n",
    "### Compute the validation accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean class accuracy 0.757314108743\n"
     ]
    }
   ],
   "source": [
    "total_lbl, total_cor = np.zeros(6)+1e-10, np.zeros(6)\n",
    "for it in tf.python_io.tf_record_iterator('valid.tfrecord'):\n",
    "    example = tf.train.Example()\n",
    "    example.ParseFromString(it)\n",
    "    I = np.frombuffer(example.features.feature['image_raw'].bytes_list.value[0], dtype=np.uint8).reshape(256, 256, 3)\n",
    "    L = np.frombuffer(example.features.feature['label_raw'].bytes_list.value[0], dtype=np.uint8).reshape(256, 256)\n",
    "    \n",
    "    P = sess.run('output:0', {'inputs:0':I[None]})\n",
    "    total_lbl += np.bincount(L.flat, minlength=6)\n",
    "    total_cor += np.bincount(L.flat, (P==L).flat, minlength=6)\n",
    "print( 'Mean class accuracy', np.mean(total_cor / total_lbl) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 4: Save Model\n",
    "Please note that we also want you to turn in your ipynb for this assignment.  Zip up the ipynb along with the tfg for your submission."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "util.save('assignment7_757.tfg', session=sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "160px",
    "width": "253px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}