Project 2 Deep Learning-Baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.eager as tfe\n",
    "import numpy as np\n",
    "import math\n",
    "import timeit\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.misc\n",
    "import _pickle as cPickle\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './cifar-10/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unzip(filename):\n",
    "    fo=open(filename,'rb')\n",
    "    data=cPickle.load(fo, encoding='latin1')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=unzip(path+'train_data')\n",
    "validate=unzip(path+'validate_data')\n",
    "test=unzip(path+'test_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape:  (49000, 32, 32, 3)\n",
      "Train labels shape:  (49000,) int32\n",
      "Validation data shape:  (1000, 32, 32, 3)\n",
      "Validation labels shape:  (1000,)\n",
      "Test data shape:  (10000, 32, 32, 3)\n",
      "Test labels shape:  (10000,)\n"
     ]
    }
   ],
   "source": [
    "def load_cifar10(num_training=49000, num_validation=1000, num_test=10000):\n",
    "    \"\"\"\n",
    "    Fetch the CIFAR-10 dataset from the web and perform preprocessing to prepare\n",
    "    it for the two-layer neural net classifier. These are the same steps as\n",
    "    we used for the SVM, but condensed to a single function.\n",
    "    \"\"\"\n",
    "#     # Load the raw CIFAR-10 dataset and use appropriate data types and shapes\n",
    "#     X_train=np.asarray(train['data'],dtype=np.float32)\n",
    "#     X_train=X_train.reshape(X_train.shape[0], 32, 32, 3)\n",
    "#     y_train=np.asarray(train['labels'],dtype=np.int32)\n",
    "#     X_test=np.asarray(test['data'],dtype=np.float32)\n",
    "#     X_test=X_test.reshape(X_test.shape[0], 32, 32, 3)\n",
    "#     y_test=np.asarray(test['labels'],dtype=np.int32)\n",
    "    \n",
    "    cifar10 = tf.keras.datasets.cifar10.load_data()\n",
    "    (X_train, y_train), (X_test, y_test) = cifar10\n",
    "    X_train = np.asarray(X_train, dtype=np.float32)\n",
    "    y_train = np.asarray(y_train, dtype=np.int32).flatten()\n",
    "    X_test = np.asarray(X_test, dtype=np.float32)\n",
    "    y_test = np.asarray(y_test, dtype=np.int32).flatten()\n",
    "\n",
    "    # Subsample the data\n",
    "    mask = range(num_training, num_training + num_validation)\n",
    "    X_val = X_train[mask]\n",
    "    y_val = y_train[mask]\n",
    "    mask = range(num_training)\n",
    "    X_train = X_train[mask]\n",
    "    y_train = y_train[mask]\n",
    "    mask = range(num_test)\n",
    "    X_test = X_test[mask]\n",
    "    y_test = y_test[mask]\n",
    "\n",
    "    # Normalize the data: subtract the mean pixel and divide by std\n",
    "    mean_pixel = X_train.mean(axis=(0, 1, 2), keepdims=True)\n",
    "    std_pixel = X_train.std(axis=(0, 1, 2), keepdims=True)\n",
    "    X_train = (X_train - mean_pixel) / std_pixel\n",
    "    X_val = (X_val - mean_pixel) / std_pixel\n",
    "    X_test = (X_test - mean_pixel) / std_pixel\n",
    "\n",
    "    return X_train, y_train, X_val, y_val, X_test, y_test\n",
    "\n",
    "\n",
    "# Invoke the above function to get our data.\n",
    "NHW = (0, 1, 2)\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = load_cifar10()\n",
    "print('Train data shape: ', X_train.shape)\n",
    "print('Train labels shape: ', y_train.shape, y_train.dtype)\n",
    "print('Validation data shape: ', X_val.shape)\n",
    "print('Validation labels shape: ', y_val.shape)\n",
    "print('Test data shape: ', X_test.shape)\n",
    "print('Test labels shape: ', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_examples_with_label = {}\n",
    "valid_examples_with_label = {}\n",
    "test_examples_with_label = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(10):\n",
    "    train_examples_with_label[i] = np.array([index for index, value in enumerate(y_train) if value == i])\n",
    "    test_examples_with_label[i] = np.array([index for index, value in enumerate(y_test) if value == i])\n",
    "    valid_examples_with_label[i] = np.array([index for index, value in enumerate(y_val) if value == i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set_size = 100\n",
    "test_set_size = 100\n",
    "valid_set_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = []\n",
    "test_labels = []\n",
    "valid_labels = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "for i in range(10):\n",
    "    train_labels = train_labels + random.sample(list(train_examples_with_label[i]), int(training_set_size/10))\n",
    "    test_labels = test_labels + random.sample(list(test_examples_with_label[i]), int(test_set_size/10))\n",
    "    valid_labels = valid_labels + random.sample(list(valid_examples_with_label[i]), int(valid_set_size/10))\n",
    "random.shuffle(train_labels)\n",
    "random.shuffle(test_labels)\n",
    "random.shuffle(valid_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train[train_labels]\n",
    "y_train = y_train[train_labels]\n",
    "X_test = X_test[test_labels]\n",
    "y_test = y_test[test_labels]\n",
    "X_val = X_val[valid_labels]\n",
    "y_val = y_val[valid_labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resize_datasets(X):\n",
    "    X_new = []\n",
    "    for i in range(X.shape[0]):\n",
    "        if(i % 100 == 0):\n",
    "            print(i)\n",
    "        img = scipy.ndimage.interpolation.zoom(X[i], (8.0, 8.0, 1.0))\n",
    "        X_new.append(img)\n",
    "    X_new = np.asarray(X_new, dtype=np.float32)\n",
    "    return X_new.reshape([X.shape[0], 256, 256, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Resizing Validation...')\n",
    "X_val = resize_datasets(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Resizing Test...')\n",
    "X_test = resize_datasets(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Resizing Training...\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "print('Resizing Training...')\n",
    "X_train = resize_datasets(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 256, 256, 3)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, pickle\n",
    "file_path = \"train_dataset.pkl\"\n",
    "max_bytes = 2**31 - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## write\n",
    "bytes_out = pickle.dumps(X_train)\n",
    "with open(file_path, 'wb') as f_out:\n",
    "    for idx in range(0, len(bytes_out), max_bytes):\n",
    "        f_out.write(bytes_out[idx:idx+max_bytes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "EOFError",
     "evalue": "Ran out of input",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mEOFError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-d763e9fff846>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mpickle_in\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"dataset.pickle\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"rb\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpickle_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0my_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpickle_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpickle_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mEOFError\u001b[0m: Ran out of input"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "pickle_in = open(\"dataset.pickle\",\"rb\")\n",
    "X_train = pickle.load(pickle_in)\n",
    "y_train = pickle.load(pickle_in)\n",
    "X_test = pickle.load(pickle_in)\n",
    "y_test = pickle.load(pickle_in)\n",
    "X_val = pickle.load(pickle_in)\n",
    "y_val = pickle.load(pickle_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## read\n",
    "bytes_in = bytearray(0)\n",
    "input_size = os.path.getsize(file_path)\n",
    "with open(file_path, 'rb') as f_in:\n",
    "    for _ in range(0, input_size, max_bytes):\n",
    "        bytes_in += f_in.read(max_bytes)\n",
    "X_train = pickle.loads(bytes_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "EOFError",
     "evalue": "Ran out of input",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mEOFError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-d763e9fff846>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mpickle_in\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"dataset.pickle\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"rb\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpickle_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0my_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpickle_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpickle_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mEOFError\u001b[0m: Ran out of input"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "pickle_in = open(\"dataset.pickle\",\"rb\")\n",
    "X_train = pickle.load(pickle_in)\n",
    "y_train = pickle.load(pickle_in)\n",
    "X_test = pickle.load(pickle_in)\n",
    "y_test = pickle.load(pickle_in)\n",
    "X_val = pickle.load(pickle_in)\n",
    "y_val = pickle.load(pickle_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset(object):\n",
    "    def __init__(self, X, y, batch_size, shuffle=False):\n",
    "        \"\"\"\n",
    "        Construct a Dataset object to iterate over data X and labels y\n",
    "        \n",
    "        Inputs:\n",
    "        - X: Numpy array of data, of any shape\n",
    "        - y: Numpy array of labels, of any shape but with y.shape[0] == X.shape[0]\n",
    "        - batch_size: Integer giving number of elements per minibatch\n",
    "        - shuffle: (optional) Boolean, whether to shuffle the data on each epoch\n",
    "        \"\"\"\n",
    "        assert X.shape[0] == y.shape[0], 'Got different numbers of data and labels'\n",
    "        self.X, self.y = X, y\n",
    "        self.batch_size, self.shuffle = batch_size, shuffle\n",
    "\n",
    "    def __iter__(self):\n",
    "        N, B = self.X.shape[0], self.batch_size\n",
    "        idxs = np.arange(N)\n",
    "        if self.shuffle:\n",
    "            np.random.shuffle(idxs)\n",
    "        return iter((self.X[i:i+B], self.y[i:i+B]) for i in range(0, N, B))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten(x):\n",
    "    \"\"\"    \n",
    "    Input:\n",
    "    - TensorFlow Tensor of shape (N, D1, ..., DM)\n",
    "    \n",
    "    Output:\n",
    "    - TensorFlow Tensor of shape (N, D1 * ... * DM)\n",
    "    \"\"\"\n",
    "    N = tf.shape(x)[0]\n",
    "    return tf.reshape(x, (N, -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = tf.get_variable(name = 'input', shape=[1,256,256,3], dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#globals used in the assignment\n",
    "IMG_SHAPE = [256, 256, 3] # height, width, channels\n",
    "NUM_CLASSES = 10\n",
    "\n",
    "# training hyperparameters\n",
    "LEARNING_RATE = 1e-2\n",
    "MOMENTUM = 0.9\n",
    "BATCH_SIZE = 16\n",
    "EPOCHS = 10\n",
    "DISPLAY_STEP = 1\n",
    "VALIDATION_STEP = 1000\n",
    "SAVE_STEP = 1000\n",
    "CKPT_PATH = './ckpt'\n",
    "SUMMARY_PATH = './summary'\n",
    "\n",
    "# net architecture hyperparamaters\n",
    "LAMBDA = 5e-4 #for weight decay\n",
    "DROPOUT = 0.5\n",
    "\n",
    "# test hyper parameters\n",
    "K_PATCHES = 5\n",
    "TOP_K = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AlexNet(tfe.Network):\n",
    "    def __init__(self, training):\n",
    "        super(AlexNet, self).__init__()\n",
    "        self.training = training\n",
    "        \n",
    "        # convolutional layers\n",
    "        conv_init = tf.contrib.layers.xavier_initializer_conv2d()\n",
    "        \n",
    "        self.conv1 = self.track_layer(\n",
    "            tf.layers.Conv2D(\n",
    "                96,\n",
    "                11,\n",
    "                4,\n",
    "                'SAME',\n",
    "                activation=tf.nn.relu,\n",
    "                kernel_initializer=conv_init))\n",
    "        self.pool1 = self.track_layer(tf.layers.MaxPooling2D(3, 2, 'VALID'))\n",
    "\n",
    "        self.conv2 = self.track_layer(\n",
    "            tf.layers.Conv2D(\n",
    "                256,\n",
    "                5,\n",
    "                1,\n",
    "                'SAME',\n",
    "                activation=tf.nn.relu,\n",
    "                kernel_initializer=conv_init))\n",
    "        self.pool2 = self.track_layer(tf.layers.MaxPooling2D(3, 2, 'VALID'))\n",
    "\n",
    "        self.conv3 = self.track_layer(\n",
    "            tf.layers.Conv2D(\n",
    "                384,\n",
    "                3,\n",
    "                1,\n",
    "                'SAME',\n",
    "                activation=tf.nn.relu,\n",
    "                kernel_initializer=conv_init))\n",
    "\n",
    "        self.conv4 = self.track_layer(\n",
    "            tf.layers.Conv2D(\n",
    "                384,\n",
    "                3,\n",
    "                1,\n",
    "                'SAME',\n",
    "                activation=tf.nn.relu,\n",
    "                kernel_initializer=conv_init))\n",
    "\n",
    "        self.conv5 = self.track_layer(\n",
    "            tf.layers.Conv2D(\n",
    "                256,\n",
    "                3,\n",
    "                1,\n",
    "                'SAME',\n",
    "                activation=tf.nn.relu,\n",
    "                kernel_initializer=conv_init))\n",
    "        self.pool5 = self.track_layer(tf.layers.MaxPooling2D(3, 2, 'VALID'))\n",
    "\n",
    "        # fully connected layers\n",
    "\n",
    "        fc_init = tf.contrib.layers.xavier_initializer()\n",
    "\n",
    "        self.fc1 = self.track_layer(\n",
    "            tf.layers.Dense(\n",
    "                4096, activation=tf.nn.relu, kernel_initializer=fc_init))\n",
    "        self.drop1 = self.track_layer(tf.layers.Dropout(DROPOUT))\n",
    "\n",
    "        self.fc2 = self.track_layer(\n",
    "            tf.layers.Dense(\n",
    "                4096, activation=tf.nn.relu, kernel_initializer=fc_init))\n",
    "        self.drop2 = self.track_layer(tf.layers.Dropout(DROPOUT))\n",
    "        \n",
    "        self.out = self.track_layer(\n",
    "            tf.layers.Dense(NUM_CLASSES, kernel_initializer=fc_init))\n",
    "\n",
    "    def call(self, x):\n",
    "        \"\"\" Function that executes the model \"\"\"\n",
    "        output = self.conv1(x)\n",
    "#         print(\"After conv1 : \" + str(output.numpy().shape))\n",
    "        output = tf.nn.lrn(\n",
    "            output, depth_radius=2, bias=1.0, alpha=2e-05, beta=0.75)\n",
    "#         print(\"After normalisation1 : \" + str(output.numpy().shape))\n",
    "        output = self.pool1(output)\n",
    "#         print(\"After pool1 : \" + str(output.numpy().shape))\n",
    "\n",
    "        output = self.conv2(output)\n",
    "#         print(\"After conv2 : \" + str(output.numpy().shape))\n",
    "        output = tf.nn.lrn(\n",
    "            output, depth_radius=2, bias=1.0, alpha=2e-05, beta=0.75)\n",
    "#         print(\"After normalisation2 : \" + str(output.numpy().shape))\n",
    "        output = self.pool2(output)\n",
    "#         print(\"After pool2 : \" + str(output.numpy().shape))\n",
    "\n",
    "        output = self.conv3(output)\n",
    "#         print(\"After conv3 : \" + str(output.numpy().shape))\n",
    "\n",
    "        output = self.conv4(output)\n",
    "#         print(\"After conv4 : \" + str(output.numpy().shape))\n",
    "        conv4_output = output\n",
    "\n",
    "        output = self.conv5(output)\n",
    "#         print(\"After conv5 : \" + str(output.numpy().shape))\n",
    "        conv5_output = output\n",
    "        \n",
    "        output = self.pool5(output)\n",
    "#         print(\"After pool5 : \" + str(output.numpy().shape))\n",
    "\n",
    "        output = tf.layers.flatten(output)\n",
    "#         print(\"After flattening : \" + str(output.numpy().shape))\n",
    "\n",
    "        output = self.fc1(output)\n",
    "#         print(\"After fc1 : \" + str(output.numpy().shape))\n",
    "        if self.training:\n",
    "            output = self.drop1(output)\n",
    "#             print(\"After drop1 : \" + str(output.numpy().shape))\n",
    "\n",
    "        output = self.fc2(output)\n",
    "#         print(\"After fc2 : \" + str(output.numpy().shape))\n",
    "        if self.training:\n",
    "            output = self.drop2(output)\n",
    "#             print(\"After drop2 : \" + str(output.numpy().shape))\n",
    "        \n",
    "        output = self.out(output)\n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss(net, mode, x, y):\n",
    "    \"\"\"\n",
    "    Computes the loss for a given batch of examples\n",
    "    Args:\n",
    "        mode, string 'train' or 'val'\n",
    "        x, tf tensor representing a batch of images\n",
    "        y, tf tensor representing a batch of labels\n",
    "    Returns:\n",
    "        the loss between the predictions on the images and the groundtruths\n",
    "    \"\"\"\n",
    "    pred = net(x)\n",
    "    y = tf.one_hot(y, NUM_CLASSES)\n",
    "\n",
    "    loss_value = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=pred)\n",
    "    weight_decay = tf.reduce_sum(LAMBDA * tf.stack([tf.nn.l2_loss(v) for v in net.variables]))\n",
    "\n",
    "    total_loss = loss_value + weight_decay\n",
    "\n",
    "    tf.contrib.summary.scalar(mode + '/loss', total_loss)\n",
    "\n",
    "    return total_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(net, mode, x, y):\n",
    "    \"\"\"\n",
    "    Computes the accuracy for a given batch of examples\n",
    "    Args:\n",
    "        mode, string 'train' or 'val'\n",
    "        x, tf tensor representing a batch of images\n",
    "        y, tf tensor representing a batch of labels\n",
    "    Returns:\n",
    "        the accuracy of the predictions on the images and the groundtruths\n",
    "    \"\"\"\n",
    "    pred = tf.nn.softmax(net(x))\n",
    "    y = tf.one_hot(y, NUM_CLASSES)\n",
    "\n",
    "    accuracy_value = tf.reduce_sum(\n",
    "                tf.cast(\n",
    "                    tf.equal(\n",
    "                        tf.argmax(pred, axis=1, output_type=tf.int64),\n",
    "                        tf.argmax(y, axis=1, output_type=tf.int64)\n",
    "                    ),\n",
    "                    dtype=tf.float32\n",
    "                ) \n",
    "            ) / float(pred.shape[0].value)\n",
    "\n",
    "    tf.contrib.summary.scalar(mode +'/accuracy', accuracy_value)\n",
    "\n",
    "    return accuracy_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dset = Dataset(X_train, y_train, batch_size=BATCH_SIZE, shuffle=True)\n",
    "val_dset = Dataset(X_val, y_val, batch_size=valid_set_size, shuffle=False)\n",
    "test_dset = Dataset(X_test, y_test, batch_size=test_set_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 (16, 256, 256, 3) (16,)\n",
      "1 (16, 256, 256, 3) (16,)\n",
      "2 (16, 256, 256, 3) (16,)\n",
      "3 (16, 256, 256, 3) (16,)\n",
      "4 (16, 256, 256, 3) (16,)\n",
      "5 (16, 256, 256, 3) (16,)\n",
      "6 (4, 256, 256, 3) (4,)\n"
     ]
    }
   ],
   "source": [
    "# We can iterate through a dataset like this:\n",
    "for t, (x, y) in enumerate(train_dset):\n",
    "    print(t, x.shape, y.shape)\n",
    "#     print(y)\n",
    "    if t > 5: break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "DISPLAY_STEP = 1\n",
    "VALIDATION_STEP = 5\n",
    "SAVE_STEP = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#used for printing the log with the timestamp\n",
    "logging = tf.logging\n",
    "logging.set_verbosity(logging.INFO)\n",
    "def log_msg(msg):\n",
    "       logging.info(f'{time.ctime()}: {msg}') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "def train(net):\n",
    "    \"\"\"\n",
    "    Training procedure\n",
    "    \"\"\"\n",
    "    start_time = time.time()\n",
    "    step_time = 0.0\n",
    "    \n",
    "    writer = tf.contrib.summary.create_summary_file_writer(SUMMARY_PATH)\n",
    "#     optimizer = tf.train.MomentumOptimizer(learning_rate=LEARNING_RATE, momentum=MOMENTUM)\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate = LEARNING_RATE)\n",
    "    global_step = tf.train.get_or_create_global_step()\n",
    "    epoch = tfe.Variable(0, name='epoch', dtype=tf.float32, trainable=False)\n",
    "    all_variables = (net.variables + optimizer.variables() + [global_step] + [epoch])\n",
    "    \n",
    "    with writer.as_default():\n",
    "        with tf.contrib.summary.record_summaries_every_n_global_steps(DISPLAY_STEP):\n",
    "            for e in range(int(epoch.numpy()), EPOCHS):\n",
    "                tf.assign(epoch, e)\n",
    "                for (batch_i, (images, labels)) in enumerate(train_dset):\n",
    "                    global_step = tf.train.get_global_step()\n",
    "                    step = global_step.numpy() + 1\n",
    "                    images = tf.to_float(images)\n",
    "                    step_start_time = int(round(time.time() * 1000))\n",
    "                    \n",
    "                    optimizer.minimize(lambda: loss(net, 'train', images, labels), global_step=global_step)\n",
    "\n",
    "                    step_end_time = int(round(time.time() * 1000))\n",
    "                    step_time += step_end_time - step_start_time\n",
    "\n",
    "                    if (step % DISPLAY_STEP) == 0:\n",
    "                        l = loss(net, 'train', images, labels)\n",
    "                        a = accuracy(net, 'train', images, labels).numpy()\n",
    "                        log_msg('Epoch: {:03d} Step/Batch: {:09d} Step mean time: {:04d}ms \\nLoss: {:.7f} Training accuracy: {:.4f}'.format(e, step, int(step_time / step), l, a))\n",
    "\n",
    "#                     if (step % VALIDATION_STEP) == 0:\n",
    "#                         for (_, (val_images, val_labels)) in enumerate(val_dset):\n",
    "# #                             val_images, val_labels = tfe.Iterator(val_dset).next()\n",
    "#                             l = loss(net, 'val', val_images, val_labels)\n",
    "#                             a = accuracy(net, 'val', val_images, val_labels).numpy()\n",
    "#                             int_time = time.time() - start_time\n",
    "#                             log_msg('Elapsed time: {:04d}ms --- Loss: {:.7f} Validation accuracy: {:.4f}'.format(int(int_time), l, a))\n",
    "\n",
    "#                     if (step % SAVE_STEP) == 0:\n",
    "#                         tfe.Saver(net.variables).save(os.path.join(CKPT_PATH, 'net.ckpt'), global_step=global_step)\n",
    "#                         log_msg('Variables saved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-22-e7ecc533a2aa>:3: Network.__init__ (from tensorflow.contrib.eager.python.network) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please inherit from `tf.keras.Model`, and see its documentation for details. `tf.keras.Model` should be a drop-in replacement for `tfe.Network` in most cases, but note that `track_layer` is no longer necessary or supported. Instead, `Layer` instances are tracked on attribute assignment (see the section of `tf.keras.Model`'s documentation on subclassing). Since the output of `track_layer` is often assigned to an attribute anyway, most code can be ported by simply removing the `track_layer` calls.\n",
      "\n",
      "`tf.keras.Model` works with all TensorFlow `Layer` instances, including those from `tf.layers`, but switching to the `tf.keras.layers` versions along with the migration to `tf.keras.Model` is recommended, since it will preserve variable names. Feel free to import it with an alias to avoid excess typing :).\n",
      "WARNING:tensorflow:** tfe.Network is deprecated and will be removed in a future version.\n",
      "\n",
      "Please inherit from `tf.keras.Model`, and see its documentation for details. `tf.keras.Model` should be a drop-in replacement for `tfe.Network` in most cases, but note that `track_layer` is no longer necessary or supported. Instead, `Layer` instances are tracked on attribute assignment (see the section of `tf.keras.Model`'s documentation on subclassing). Since the output of `track_layer` is often assigned to an attribute anyway, most code can be ported by simply removing the `track_layer` calls.\n",
      "\n",
      "`tf.keras.Model` works with all TensorFlow `Layer` instances, including those from `tf.layers`, but switching to the `tf.keras.layers` versions along with the migration to `tf.keras.Model` is recommended, since it will preserve variable names. Feel free to import it with an alias to avoid excess typing :).\n"
     ]
    }
   ],
   "source": [
    "alexnet = AlexNet(training=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Deprecation Warning: create_summary_file_writer was renamed to create_file_writer\n",
      "INFO:tensorflow:Mon Nov 26 15:52:40 2018: Epoch: 000 Step/Batch: 000000001 Step mean time: 3799ms \n",
      "Loss: 7665465.5000000 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:52:44 2018: Epoch: 000 Step/Batch: 000000002 Step mean time: 3185ms \n",
      "Loss: 363.0084839 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:52:48 2018: Epoch: 000 Step/Batch: 000000003 Step mean time: 2957ms \n",
      "Loss: 61.6835480 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:52:52 2018: Epoch: 000 Step/Batch: 000000004 Step mean time: 2849ms \n",
      "Loss: 4718.1572266 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:52:56 2018: Epoch: 000 Step/Batch: 000000005 Step mean time: 2783ms \n",
      "Loss: 377.3768921 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:52:59 2018: Epoch: 000 Step/Batch: 000000006 Step mean time: 2740ms \n",
      "Loss: 22544.8417969 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:53:01 2018: Epoch: 000 Step/Batch: 000000007 Step mean time: 2500ms \n",
      "Loss: 329.3940735 Training accuracy: 0.2500\n",
      "INFO:tensorflow:Mon Nov 26 15:53:05 2018: Epoch: 001 Step/Batch: 000000008 Step mean time: 2505ms \n",
      "Loss: 11.4313450 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:53:09 2018: Epoch: 001 Step/Batch: 000000009 Step mean time: 2504ms \n",
      "Loss: 11.7026329 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:53:12 2018: Epoch: 001 Step/Batch: 000000010 Step mean time: 2504ms \n",
      "Loss: 12.5358028 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:53:16 2018: Epoch: 001 Step/Batch: 000000011 Step mean time: 2504ms \n",
      "Loss: 22.4292908 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:53:20 2018: Epoch: 001 Step/Batch: 000000012 Step mean time: 2535ms \n",
      "Loss: 730.0519409 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:53:24 2018: Epoch: 001 Step/Batch: 000000013 Step mean time: 2547ms \n",
      "Loss: 14.0692282 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:53:26 2018: Epoch: 001 Step/Batch: 000000014 Step mean time: 2442ms \n",
      "Loss: 14.3464251 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:53:30 2018: Epoch: 002 Step/Batch: 000000015 Step mean time: 2457ms \n",
      "Loss: 14.7673225 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:53:34 2018: Epoch: 002 Step/Batch: 000000016 Step mean time: 2487ms \n",
      "Loss: 15.0972366 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:53:38 2018: Epoch: 002 Step/Batch: 000000017 Step mean time: 2500ms \n",
      "Loss: 15.6188602 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:53:42 2018: Epoch: 002 Step/Batch: 000000018 Step mean time: 2505ms \n",
      "Loss: 15.9779882 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:53:46 2018: Epoch: 002 Step/Batch: 000000019 Step mean time: 2508ms \n",
      "Loss: 16.1495037 Training accuracy: 0.2500\n",
      "INFO:tensorflow:Mon Nov 26 15:53:50 2018: Epoch: 002 Step/Batch: 000000020 Step mean time: 2511ms \n",
      "Loss: 16.4518471 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:53:52 2018: Epoch: 002 Step/Batch: 000000021 Step mean time: 2441ms \n",
      "Loss: 16.5794106 Training accuracy: 0.2500\n",
      "INFO:tensorflow:Mon Nov 26 15:53:55 2018: Epoch: 003 Step/Batch: 000000022 Step mean time: 2447ms \n",
      "Loss: 17.1012573 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:53:59 2018: Epoch: 003 Step/Batch: 000000023 Step mean time: 2452ms \n",
      "Loss: 17.4406815 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:54:03 2018: Epoch: 003 Step/Batch: 000000024 Step mean time: 2462ms \n",
      "Loss: 813.9719849 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:54:07 2018: Epoch: 003 Step/Batch: 000000025 Step mean time: 2471ms \n",
      "Loss: 17.7504215 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:54:11 2018: Epoch: 003 Step/Batch: 000000026 Step mean time: 2479ms \n",
      "Loss: 22.2847443 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:54:15 2018: Epoch: 003 Step/Batch: 000000027 Step mean time: 2480ms \n",
      "Loss: 18.8446007 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:54:17 2018: Epoch: 003 Step/Batch: 000000028 Step mean time: 2428ms \n",
      "Loss: 138.5944214 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:54:20 2018: Epoch: 004 Step/Batch: 000000029 Step mean time: 2432ms \n",
      "Loss: 19.0403633 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:54:25 2018: Epoch: 004 Step/Batch: 000000030 Step mean time: 2443ms \n",
      "Loss: 19.5255718 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:54:29 2018: Epoch: 004 Step/Batch: 000000031 Step mean time: 2452ms \n",
      "Loss: 19.9522820 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:54:33 2018: Epoch: 004 Step/Batch: 000000032 Step mean time: 2458ms \n",
      "Loss: 19.6790733 Training accuracy: 0.2500\n",
      "INFO:tensorflow:Mon Nov 26 15:54:36 2018: Epoch: 004 Step/Batch: 000000033 Step mean time: 2459ms \n",
      "Loss: 20.6144009 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:54:40 2018: Epoch: 004 Step/Batch: 000000034 Step mean time: 2460ms \n",
      "Loss: 625.1127930 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:54:42 2018: Epoch: 004 Step/Batch: 000000035 Step mean time: 2419ms \n",
      "Loss: 21.9552307 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:54:45 2018: Epoch: 005 Step/Batch: 000000036 Step mean time: 2421ms \n",
      "Loss: 21.5066452 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:54:49 2018: Epoch: 005 Step/Batch: 000000037 Step mean time: 2422ms \n",
      "Loss: 355.3714600 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:54:53 2018: Epoch: 005 Step/Batch: 000000038 Step mean time: 2424ms \n",
      "Loss: 495.6416931 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:54:57 2018: Epoch: 005 Step/Batch: 000000039 Step mean time: 2426ms \n",
      "Loss: 2571.1486816 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:55:00 2018: Epoch: 005 Step/Batch: 000000040 Step mean time: 2428ms \n",
      "Loss: 23.6523705 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:55:04 2018: Epoch: 005 Step/Batch: 000000041 Step mean time: 2429ms \n",
      "Loss: 146.4314117 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:55:06 2018: Epoch: 005 Step/Batch: 000000042 Step mean time: 2396ms \n",
      "Loss: 24.8577938 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:55:10 2018: Epoch: 006 Step/Batch: 000000043 Step mean time: 2399ms \n",
      "Loss: 793.7995605 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:55:13 2018: Epoch: 006 Step/Batch: 000000044 Step mean time: 2401ms \n",
      "Loss: 23.8523064 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:55:17 2018: Epoch: 006 Step/Batch: 000000045 Step mean time: 2403ms \n",
      "Loss: 24.2050705 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:55:21 2018: Epoch: 006 Step/Batch: 000000046 Step mean time: 2405ms \n",
      "Loss: 22.7567158 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:55:25 2018: Epoch: 006 Step/Batch: 000000047 Step mean time: 2407ms \n",
      "Loss: 22.9023476 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:55:28 2018: Epoch: 006 Step/Batch: 000000048 Step mean time: 2409ms \n",
      "Loss: 23.3343925 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:55:30 2018: Epoch: 006 Step/Batch: 000000049 Step mean time: 2380ms \n",
      "Loss: 24.2217026 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:55:34 2018: Epoch: 007 Step/Batch: 000000050 Step mean time: 2384ms \n",
      "Loss: 24.2052860 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:55:38 2018: Epoch: 007 Step/Batch: 000000051 Step mean time: 2388ms \n",
      "Loss: 24.3295040 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:55:41 2018: Epoch: 007 Step/Batch: 000000052 Step mean time: 2390ms \n",
      "Loss: 24.6648750 Training accuracy: 0.0000\n",
      "INFO:tensorflow:Mon Nov 26 15:55:45 2018: Epoch: 007 Step/Batch: 000000053 Step mean time: 2393ms \n",
      "Loss: 24.7344322 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:55:49 2018: Epoch: 007 Step/Batch: 000000054 Step mean time: 2395ms \n",
      "Loss: 201.1267548 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:55:53 2018: Epoch: 007 Step/Batch: 000000055 Step mean time: 2396ms \n",
      "Loss: 53.3326874 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:55:54 2018: Epoch: 007 Step/Batch: 000000056 Step mean time: 2371ms \n",
      "Loss: 24.8314686 Training accuracy: 0.2500\n",
      "INFO:tensorflow:Mon Nov 26 15:55:58 2018: Epoch: 008 Step/Batch: 000000057 Step mean time: 2374ms \n",
      "Loss: 25.2325592 Training accuracy: 0.0625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Mon Nov 26 15:56:02 2018: Epoch: 008 Step/Batch: 000000058 Step mean time: 2376ms \n",
      "Loss: 24.8975620 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:56:06 2018: Epoch: 008 Step/Batch: 000000059 Step mean time: 2378ms \n",
      "Loss: 25.0748138 Training accuracy: 0.1875\n",
      "INFO:tensorflow:Mon Nov 26 15:56:09 2018: Epoch: 008 Step/Batch: 000000060 Step mean time: 2380ms \n",
      "Loss: 138.1916809 Training accuracy: 0.2500\n",
      "INFO:tensorflow:Mon Nov 26 15:56:13 2018: Epoch: 008 Step/Batch: 000000061 Step mean time: 2382ms \n",
      "Loss: 25.2194157 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:56:17 2018: Epoch: 008 Step/Batch: 000000062 Step mean time: 2384ms \n",
      "Loss: 25.3128147 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:56:18 2018: Epoch: 008 Step/Batch: 000000063 Step mean time: 2362ms \n",
      "Loss: 25.5467224 Training accuracy: 0.2500\n",
      "INFO:tensorflow:Mon Nov 26 15:56:22 2018: Epoch: 009 Step/Batch: 000000064 Step mean time: 2366ms \n",
      "Loss: 25.5251312 Training accuracy: 0.1250\n",
      "INFO:tensorflow:Mon Nov 26 15:56:26 2018: Epoch: 009 Step/Batch: 000000065 Step mean time: 2367ms \n",
      "Loss: 25.5196018 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:56:30 2018: Epoch: 009 Step/Batch: 000000066 Step mean time: 2369ms \n",
      "Loss: 25.5535488 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:56:34 2018: Epoch: 009 Step/Batch: 000000067 Step mean time: 2371ms \n",
      "Loss: 25.7365036 Training accuracy: 0.2500\n",
      "INFO:tensorflow:Mon Nov 26 15:56:37 2018: Epoch: 009 Step/Batch: 000000068 Step mean time: 2373ms \n",
      "Loss: 25.8625736 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:56:41 2018: Epoch: 009 Step/Batch: 000000069 Step mean time: 2375ms \n",
      "Loss: 25.8512573 Training accuracy: 0.0625\n",
      "INFO:tensorflow:Mon Nov 26 15:56:43 2018: Epoch: 009 Step/Batch: 000000070 Step mean time: 2355ms \n",
      "Loss: 26.1434937 Training accuracy: 0.0000\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(CKPT_PATH):\n",
    "    os.makedirs(CKPT_PATH)\n",
    "train(alexnet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (_, (test_images, test_labels)) in enumerate(test_dset):\n",
    "    l = loss(alexnet, 'val', test_images, test_labels)\n",
    "    a = accuracy(alexnet, 'val', test_images, test_labels).numpy()\n",
    "log_msg('Final Test Loss: {:.7f} Test accuracy: {:.4f}'.format(l, a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}