lab-10-2-1-mnist_nn_xavier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Eager mode on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load & pre-processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (class version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(2):\n",
    "            self.model.add(dense(256, weight_init))\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (function version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(2) :\n",
    "        model.add(dense(256, weight_init))\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data & hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size).\\\n",
    "    repeat()\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x)).\\\n",
    "    repeat()\n",
    "\n",
    "train_iterator = train_dataset.make_one_shot_iterator()\n",
    "test_iterator = test_dataset.make_one_shot_iterator()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model & optimizer & writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_xavier'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restore checkpoint & start train or test phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 0.3464, train_loss: 2.08131313, train_accuracy: 0.3750, test_Accuracy: 0.2719\n",
      "Epoch: [ 0] [    1/  468] time: 0.4808, train_loss: 2.02861357, train_accuracy: 0.4766, test_Accuracy: 0.4322\n",
      "Epoch: [ 0] [    2/  468] time: 0.5887, train_loss: 1.91424179, train_accuracy: 0.5234, test_Accuracy: 0.5423\n",
      "Epoch: [ 0] [    3/  468] time: 0.6909, train_loss: 1.76309013, train_accuracy: 0.6250, test_Accuracy: 0.5716\n",
      "Epoch: [ 0] [    4/  468] time: 0.8009, train_loss: 1.71893096, train_accuracy: 0.5781, test_Accuracy: 0.6158\n",
      "Epoch: [ 0] [    5/  468] time: 0.9101, train_loss: 1.68493593, train_accuracy: 0.6172, test_Accuracy: 0.6568\n",
      "Epoch: [ 0] [    6/  468] time: 1.0192, train_loss: 1.45470881, train_accuracy: 0.6953, test_Accuracy: 0.6954\n",
      "Epoch: [ 0] [    7/  468] time: 1.1196, train_loss: 1.30661106, train_accuracy: 0.7188, test_Accuracy: 0.7245\n",
      "Epoch: [ 0] [    8/  468] time: 1.2275, train_loss: 1.26174331, train_accuracy: 0.7188, test_Accuracy: 0.7501\n",
      "Epoch: [ 0] [    9/  468] time: 1.3379, train_loss: 1.09275603, train_accuracy: 0.7812, test_Accuracy: 0.7683\n",
      "Epoch: [ 0] [   10/  468] time: 1.4318, train_loss: 0.96765471, train_accuracy: 0.8203, test_Accuracy: 0.7946\n",
      "Epoch: [ 0] [   11/  468] time: 1.5289, train_loss: 0.98256129, train_accuracy: 0.8047, test_Accuracy: 0.8078\n",
      "Epoch: [ 0] [   12/  468] time: 1.6289, train_loss: 0.89085370, train_accuracy: 0.7891, test_Accuracy: 0.8088\n",
      "Epoch: [ 0] [   13/  468] time: 1.7168, train_loss: 0.93801010, train_accuracy: 0.7578, test_Accuracy: 0.8148\n",
      "Epoch: [ 0] [   14/  468] time: 1.8196, train_loss: 0.75926411, train_accuracy: 0.8281, test_Accuracy: 0.8229\n",
      "Epoch: [ 0] [   15/  468] time: 1.9273, train_loss: 0.71489519, train_accuracy: 0.8125, test_Accuracy: 0.8272\n",
      "Epoch: [ 0] [   16/  468] time: 2.0274, train_loss: 0.63756931, train_accuracy: 0.8594, test_Accuracy: 0.8313\n",
      "Epoch: [ 0] [   17/  468] time: 2.1315, train_loss: 0.74543703, train_accuracy: 0.7656, test_Accuracy: 0.8307\n",
      "Epoch: [ 0] [   18/  468] time: 2.2383, train_loss: 0.58154160, train_accuracy: 0.8125, test_Accuracy: 0.8234\n",
      "Epoch: [ 0] [   19/  468] time: 2.3248, train_loss: 0.53775477, train_accuracy: 0.8203, test_Accuracy: 0.8217\n",
      "Epoch: [ 0] [   20/  468] time: 2.4349, train_loss: 0.60482156, train_accuracy: 0.7891, test_Accuracy: 0.8350\n",
      "Epoch: [ 0] [   21/  468] time: 2.5196, train_loss: 0.49007818, train_accuracy: 0.8750, test_Accuracy: 0.8510\n",
      "Epoch: [ 0] [   22/  468] time: 2.6230, train_loss: 0.50485349, train_accuracy: 0.8438, test_Accuracy: 0.8567\n",
      "Epoch: [ 0] [   23/  468] time: 2.7086, train_loss: 0.52184165, train_accuracy: 0.8203, test_Accuracy: 0.8501\n",
      "Epoch: [ 0] [   24/  468] time: 2.8121, train_loss: 0.54150540, train_accuracy: 0.8281, test_Accuracy: 0.8482\n",
      "Epoch: [ 0] [   25/  468] time: 2.9122, train_loss: 0.56265199, train_accuracy: 0.8438, test_Accuracy: 0.8479\n",
      "Epoch: [ 0] [   26/  468] time: 3.0140, train_loss: 0.43164819, train_accuracy: 0.8750, test_Accuracy: 0.8556\n",
      "Epoch: [ 0] [   27/  468] time: 3.1153, train_loss: 0.37945485, train_accuracy: 0.8828, test_Accuracy: 0.8635\n",
      "Epoch: [ 0] [   28/  468] time: 3.2033, train_loss: 0.38673082, train_accuracy: 0.8438, test_Accuracy: 0.8593\n",
      "Epoch: [ 0] [   29/  468] time: 3.2918, train_loss: 0.41602033, train_accuracy: 0.8984, test_Accuracy: 0.8650\n",
      "Epoch: [ 0] [   30/  468] time: 3.3887, train_loss: 0.45831743, train_accuracy: 0.8516, test_Accuracy: 0.8760\n",
      "Epoch: [ 0] [   31/  468] time: 3.4768, train_loss: 0.50757456, train_accuracy: 0.8672, test_Accuracy: 0.8814\n",
      "Epoch: [ 0] [   32/  468] time: 3.5717, train_loss: 0.47103453, train_accuracy: 0.8672, test_Accuracy: 0.8818\n",
      "Epoch: [ 0] [   33/  468] time: 3.6660, train_loss: 0.37903887, train_accuracy: 0.9062, test_Accuracy: 0.8818\n",
      "Epoch: [ 0] [   34/  468] time: 3.7568, train_loss: 0.34824729, train_accuracy: 0.9062, test_Accuracy: 0.8831\n",
      "Epoch: [ 0] [   35/  468] time: 3.8462, train_loss: 0.34317690, train_accuracy: 0.9141, test_Accuracy: 0.8884\n",
      "Epoch: [ 0] [   36/  468] time: 3.9341, train_loss: 0.31115791, train_accuracy: 0.9062, test_Accuracy: 0.8912\n",
      "Epoch: [ 0] [   37/  468] time: 4.0377, train_loss: 0.40703526, train_accuracy: 0.8906, test_Accuracy: 0.8927\n",
      "Epoch: [ 0] [   38/  468] time: 4.1313, train_loss: 0.37342438, train_accuracy: 0.9062, test_Accuracy: 0.8890\n",
      "Epoch: [ 0] [   39/  468] time: 4.2256, train_loss: 0.36000398, train_accuracy: 0.8828, test_Accuracy: 0.8899\n",
      "Epoch: [ 0] [   40/  468] time: 4.3266, train_loss: 0.33263433, train_accuracy: 0.8906, test_Accuracy: 0.8878\n",
      "Epoch: [ 0] [   41/  468] time: 4.4296, train_loss: 0.41964778, train_accuracy: 0.8906, test_Accuracy: 0.8855\n",
      "Epoch: [ 0] [   42/  468] time: 4.5283, train_loss: 0.30988994, train_accuracy: 0.8750, test_Accuracy: 0.8868\n",
      "Epoch: [ 0] [   43/  468] time: 4.6217, train_loss: 0.31905419, train_accuracy: 0.8906, test_Accuracy: 0.8824\n",
      "Epoch: [ 0] [   44/  468] time: 4.7148, train_loss: 0.33644655, train_accuracy: 0.8828, test_Accuracy: 0.8816\n",
      "Epoch: [ 0] [   45/  468] time: 4.8085, train_loss: 0.41382462, train_accuracy: 0.8984, test_Accuracy: 0.8890\n",
      "Epoch: [ 0] [   46/  468] time: 4.9076, train_loss: 0.25993216, train_accuracy: 0.9375, test_Accuracy: 0.8979\n",
      "Epoch: [ 0] [   47/  468] time: 5.0039, train_loss: 0.31408986, train_accuracy: 0.9297, test_Accuracy: 0.9023\n",
      "Epoch: [ 0] [   48/  468] time: 5.0947, train_loss: 0.42341414, train_accuracy: 0.8906, test_Accuracy: 0.8995\n",
      "Epoch: [ 0] [   49/  468] time: 5.1832, train_loss: 0.28597045, train_accuracy: 0.9375, test_Accuracy: 0.8946\n",
      "Epoch: [ 0] [   50/  468] time: 5.2695, train_loss: 0.32379079, train_accuracy: 0.8906, test_Accuracy: 0.8923\n",
      "Epoch: [ 0] [   51/  468] time: 5.3587, train_loss: 0.34739187, train_accuracy: 0.9062, test_Accuracy: 0.8958\n",
      "Epoch: [ 0] [   52/  468] time: 5.4443, train_loss: 0.51041901, train_accuracy: 0.8516, test_Accuracy: 0.9019\n",
      "Epoch: [ 0] [   53/  468] time: 5.5312, train_loss: 0.28932089, train_accuracy: 0.9219, test_Accuracy: 0.9023\n",
      "Epoch: [ 0] [   54/  468] time: 5.6217, train_loss: 0.44202149, train_accuracy: 0.8828, test_Accuracy: 0.8997\n",
      "Epoch: [ 0] [   55/  468] time: 5.7109, train_loss: 0.14494072, train_accuracy: 0.9688, test_Accuracy: 0.8949\n",
      "Epoch: [ 0] [   56/  468] time: 5.8000, train_loss: 0.25086522, train_accuracy: 0.9375, test_Accuracy: 0.8950\n",
      "Epoch: [ 0] [   57/  468] time: 5.8979, train_loss: 0.47147399, train_accuracy: 0.8906, test_Accuracy: 0.9015\n",
      "Epoch: [ 0] [   58/  468] time: 5.9940, train_loss: 0.28977692, train_accuracy: 0.9297, test_Accuracy: 0.9056\n",
      "Epoch: [ 0] [   59/  468] time: 6.0859, train_loss: 0.31472224, train_accuracy: 0.9062, test_Accuracy: 0.9000\n",
      "Epoch: [ 0] [   60/  468] time: 6.1842, train_loss: 0.35319340, train_accuracy: 0.9297, test_Accuracy: 0.8957\n",
      "Epoch: [ 0] [   61/  468] time: 6.2750, train_loss: 0.49346113, train_accuracy: 0.8516, test_Accuracy: 0.8905\n",
      "Epoch: [ 0] [   62/  468] time: 6.3674, train_loss: 0.44487795, train_accuracy: 0.8828, test_Accuracy: 0.8955\n",
      "Epoch: [ 0] [   63/  468] time: 6.4618, train_loss: 0.26763377, train_accuracy: 0.9141, test_Accuracy: 0.9026\n",
      "Epoch: [ 0] [   64/  468] time: 6.5559, train_loss: 0.17779678, train_accuracy: 0.9375, test_Accuracy: 0.8997\n",
      "Epoch: [ 0] [   65/  468] time: 6.6497, train_loss: 0.32253796, train_accuracy: 0.8828, test_Accuracy: 0.8961\n",
      "Epoch: [ 0] [   66/  468] time: 6.7448, train_loss: 0.30409384, train_accuracy: 0.9062, test_Accuracy: 0.8907\n",
      "Epoch: [ 0] [   67/  468] time: 6.8395, train_loss: 0.35309738, train_accuracy: 0.8984, test_Accuracy: 0.8949\n",
      "Epoch: [ 0] [   68/  468] time: 6.9314, train_loss: 0.31055999, train_accuracy: 0.8984, test_Accuracy: 0.9015\n",
      "Epoch: [ 0] [   69/  468] time: 7.0235, train_loss: 0.25381264, train_accuracy: 0.9453, test_Accuracy: 0.9070\n",
      "Epoch: [ 0] [   70/  468] time: 7.1211, train_loss: 0.22321716, train_accuracy: 0.9297, test_Accuracy: 0.9103\n",
      "Epoch: [ 0] [   71/  468] time: 7.2182, train_loss: 0.28898698, train_accuracy: 0.8984, test_Accuracy: 0.9005\n",
      "Epoch: [ 0] [   72/  468] time: 7.3082, train_loss: 0.45250621, train_accuracy: 0.8281, test_Accuracy: 0.8980\n",
      "Epoch: [ 0] [   73/  468] time: 7.4077, train_loss: 0.48727876, train_accuracy: 0.8594, test_Accuracy: 0.9000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   74/  468] time: 7.4990, train_loss: 0.31315297, train_accuracy: 0.8750, test_Accuracy: 0.9070\n",
      "Epoch: [ 0] [   75/  468] time: 7.5926, train_loss: 0.24490650, train_accuracy: 0.9141, test_Accuracy: 0.9129\n",
      "Epoch: [ 0] [   76/  468] time: 7.6862, train_loss: 0.41736907, train_accuracy: 0.8984, test_Accuracy: 0.9096\n",
      "Epoch: [ 0] [   77/  468] time: 7.7752, train_loss: 0.37905023, train_accuracy: 0.8750, test_Accuracy: 0.9039\n",
      "Epoch: [ 0] [   78/  468] time: 7.8668, train_loss: 0.39331663, train_accuracy: 0.8984, test_Accuracy: 0.9031\n",
      "Epoch: [ 0] [   79/  468] time: 7.9565, train_loss: 0.34171745, train_accuracy: 0.8906, test_Accuracy: 0.9062\n",
      "Epoch: [ 0] [   80/  468] time: 8.0505, train_loss: 0.33353072, train_accuracy: 0.9062, test_Accuracy: 0.9071\n",
      "Epoch: [ 0] [   81/  468] time: 8.1385, train_loss: 0.28421435, train_accuracy: 0.9219, test_Accuracy: 0.9117\n",
      "Epoch: [ 0] [   82/  468] time: 8.2317, train_loss: 0.30892992, train_accuracy: 0.9062, test_Accuracy: 0.9177\n",
      "Epoch: [ 0] [   83/  468] time: 8.3228, train_loss: 0.30811739, train_accuracy: 0.9219, test_Accuracy: 0.9175\n",
      "Epoch: [ 0] [   84/  468] time: 8.4104, train_loss: 0.26866558, train_accuracy: 0.9219, test_Accuracy: 0.9148\n",
      "Epoch: [ 0] [   85/  468] time: 8.4999, train_loss: 0.31868207, train_accuracy: 0.8906, test_Accuracy: 0.9108\n",
      "Epoch: [ 0] [   86/  468] time: 8.5911, train_loss: 0.20260486, train_accuracy: 0.9375, test_Accuracy: 0.9072\n",
      "Epoch: [ 0] [   87/  468] time: 8.6815, train_loss: 0.28861430, train_accuracy: 0.9062, test_Accuracy: 0.9105\n",
      "Epoch: [ 0] [   88/  468] time: 8.7717, train_loss: 0.19095375, train_accuracy: 0.9453, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [   89/  468] time: 8.8663, train_loss: 0.25284028, train_accuracy: 0.9062, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [   90/  468] time: 8.9560, train_loss: 0.15927224, train_accuracy: 0.9609, test_Accuracy: 0.9198\n",
      "Epoch: [ 0] [   91/  468] time: 9.0443, train_loss: 0.27550778, train_accuracy: 0.9141, test_Accuracy: 0.9177\n",
      "Epoch: [ 0] [   92/  468] time: 9.1333, train_loss: 0.27598596, train_accuracy: 0.9453, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [   93/  468] time: 9.2239, train_loss: 0.26506966, train_accuracy: 0.8828, test_Accuracy: 0.9162\n",
      "Epoch: [ 0] [   94/  468] time: 9.3161, train_loss: 0.28603137, train_accuracy: 0.9297, test_Accuracy: 0.9156\n",
      "Epoch: [ 0] [   95/  468] time: 9.4051, train_loss: 0.22638276, train_accuracy: 0.9297, test_Accuracy: 0.9166\n",
      "Epoch: [ 0] [   96/  468] time: 9.5069, train_loss: 0.25426316, train_accuracy: 0.9297, test_Accuracy: 0.9178\n",
      "Epoch: [ 0] [   97/  468] time: 9.5971, train_loss: 0.19514802, train_accuracy: 0.9453, test_Accuracy: 0.9170\n",
      "Epoch: [ 0] [   98/  468] time: 9.6882, train_loss: 0.25055534, train_accuracy: 0.9219, test_Accuracy: 0.9176\n",
      "Epoch: [ 0] [   99/  468] time: 9.7850, train_loss: 0.19839583, train_accuracy: 0.9375, test_Accuracy: 0.9192\n",
      "Epoch: [ 0] [  100/  468] time: 9.8738, train_loss: 0.38003069, train_accuracy: 0.8984, test_Accuracy: 0.9202\n",
      "Epoch: [ 0] [  101/  468] time: 9.9694, train_loss: 0.24758489, train_accuracy: 0.9375, test_Accuracy: 0.9219\n",
      "Epoch: [ 0] [  102/  468] time: 10.0585, train_loss: 0.34801197, train_accuracy: 0.8828, test_Accuracy: 0.9225\n",
      "Epoch: [ 0] [  103/  468] time: 10.1516, train_loss: 0.14773294, train_accuracy: 0.9453, test_Accuracy: 0.9217\n",
      "Epoch: [ 0] [  104/  468] time: 10.2482, train_loss: 0.19008571, train_accuracy: 0.9453, test_Accuracy: 0.9235\n",
      "Epoch: [ 0] [  105/  468] time: 10.3417, train_loss: 0.30642676, train_accuracy: 0.8906, test_Accuracy: 0.9264\n",
      "Epoch: [ 0] [  106/  468] time: 10.4369, train_loss: 0.26043612, train_accuracy: 0.9297, test_Accuracy: 0.9284\n",
      "Epoch: [ 0] [  107/  468] time: 10.5391, train_loss: 0.16147411, train_accuracy: 0.9688, test_Accuracy: 0.9285\n",
      "Epoch: [ 0] [  108/  468] time: 10.6364, train_loss: 0.22109115, train_accuracy: 0.9141, test_Accuracy: 0.9252\n",
      "Epoch: [ 0] [  109/  468] time: 10.7300, train_loss: 0.35140508, train_accuracy: 0.8906, test_Accuracy: 0.9233\n",
      "Epoch: [ 0] [  110/  468] time: 10.8248, train_loss: 0.14034277, train_accuracy: 0.9453, test_Accuracy: 0.9213\n",
      "Epoch: [ 0] [  111/  468] time: 10.9188, train_loss: 0.19303375, train_accuracy: 0.9531, test_Accuracy: 0.9219\n",
      "Epoch: [ 0] [  112/  468] time: 11.0138, train_loss: 0.25988269, train_accuracy: 0.9453, test_Accuracy: 0.9207\n",
      "Epoch: [ 0] [  113/  468] time: 11.1057, train_loss: 0.23602737, train_accuracy: 0.9453, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  114/  468] time: 11.1978, train_loss: 0.21546224, train_accuracy: 0.9297, test_Accuracy: 0.9211\n",
      "Epoch: [ 0] [  115/  468] time: 11.2887, train_loss: 0.25059819, train_accuracy: 0.9062, test_Accuracy: 0.9209\n",
      "Epoch: [ 0] [  116/  468] time: 11.3835, train_loss: 0.18910718, train_accuracy: 0.9453, test_Accuracy: 0.9184\n",
      "Epoch: [ 0] [  117/  468] time: 11.4701, train_loss: 0.24980441, train_accuracy: 0.9453, test_Accuracy: 0.9181\n",
      "Epoch: [ 0] [  118/  468] time: 11.5549, train_loss: 0.28510952, train_accuracy: 0.9297, test_Accuracy: 0.9197\n",
      "Epoch: [ 0] [  119/  468] time: 11.6494, train_loss: 0.22606258, train_accuracy: 0.9375, test_Accuracy: 0.9302\n",
      "Epoch: [ 0] [  120/  468] time: 11.7404, train_loss: 0.28852320, train_accuracy: 0.8984, test_Accuracy: 0.9343\n",
      "Epoch: [ 0] [  121/  468] time: 11.8335, train_loss: 0.22165547, train_accuracy: 0.9219, test_Accuracy: 0.9309\n",
      "Epoch: [ 0] [  122/  468] time: 11.9249, train_loss: 0.24310501, train_accuracy: 0.9297, test_Accuracy: 0.9276\n",
      "Epoch: [ 0] [  123/  468] time: 12.0185, train_loss: 0.32299939, train_accuracy: 0.9062, test_Accuracy: 0.9270\n",
      "Epoch: [ 0] [  124/  468] time: 12.1159, train_loss: 0.21235263, train_accuracy: 0.9375, test_Accuracy: 0.9257\n",
      "Epoch: [ 0] [  125/  468] time: 12.2108, train_loss: 0.21930704, train_accuracy: 0.9375, test_Accuracy: 0.9285\n",
      "Epoch: [ 0] [  126/  468] time: 12.2991, train_loss: 0.13138054, train_accuracy: 0.9766, test_Accuracy: 0.9306\n",
      "Epoch: [ 0] [  127/  468] time: 12.3914, train_loss: 0.23018163, train_accuracy: 0.9141, test_Accuracy: 0.9297\n",
      "Epoch: [ 0] [  128/  468] time: 12.4829, train_loss: 0.17856160, train_accuracy: 0.9375, test_Accuracy: 0.9295\n",
      "Epoch: [ 0] [  129/  468] time: 12.5770, train_loss: 0.19415392, train_accuracy: 0.9453, test_Accuracy: 0.9275\n",
      "Epoch: [ 0] [  130/  468] time: 12.6732, train_loss: 0.23739624, train_accuracy: 0.9375, test_Accuracy: 0.9260\n",
      "Epoch: [ 0] [  131/  468] time: 12.7635, train_loss: 0.27067456, train_accuracy: 0.9297, test_Accuracy: 0.9252\n",
      "Epoch: [ 0] [  132/  468] time: 12.8520, train_loss: 0.23540795, train_accuracy: 0.9453, test_Accuracy: 0.9258\n",
      "Epoch: [ 0] [  133/  468] time: 12.9449, train_loss: 0.36756539, train_accuracy: 0.8906, test_Accuracy: 0.9277\n",
      "Epoch: [ 0] [  134/  468] time: 13.0444, train_loss: 0.24896756, train_accuracy: 0.9375, test_Accuracy: 0.9286\n",
      "Epoch: [ 0] [  135/  468] time: 13.1357, train_loss: 0.29105309, train_accuracy: 0.9141, test_Accuracy: 0.9292\n",
      "Epoch: [ 0] [  136/  468] time: 13.2263, train_loss: 0.20618796, train_accuracy: 0.9453, test_Accuracy: 0.9302\n",
      "Epoch: [ 0] [  137/  468] time: 13.3199, train_loss: 0.23260257, train_accuracy: 0.9219, test_Accuracy: 0.9293\n",
      "Epoch: [ 0] [  138/  468] time: 13.4101, train_loss: 0.22066930, train_accuracy: 0.9375, test_Accuracy: 0.9298\n",
      "Epoch: [ 0] [  139/  468] time: 13.5014, train_loss: 0.12740844, train_accuracy: 0.9609, test_Accuracy: 0.9301\n",
      "Epoch: [ 0] [  140/  468] time: 13.5904, train_loss: 0.14079398, train_accuracy: 0.9531, test_Accuracy: 0.9309\n",
      "Epoch: [ 0] [  141/  468] time: 13.6821, train_loss: 0.20803982, train_accuracy: 0.9531, test_Accuracy: 0.9310\n",
      "Epoch: [ 0] [  142/  468] time: 13.7737, train_loss: 0.23418832, train_accuracy: 0.9453, test_Accuracy: 0.9323\n",
      "Epoch: [ 0] [  143/  468] time: 13.8673, train_loss: 0.29647946, train_accuracy: 0.8828, test_Accuracy: 0.9308\n",
      "Epoch: [ 0] [  144/  468] time: 13.9597, train_loss: 0.14158839, train_accuracy: 0.9766, test_Accuracy: 0.9310\n",
      "Epoch: [ 0] [  145/  468] time: 14.0509, train_loss: 0.34076756, train_accuracy: 0.9141, test_Accuracy: 0.9322\n",
      "Epoch: [ 0] [  146/  468] time: 14.1417, train_loss: 0.33726948, train_accuracy: 0.9297, test_Accuracy: 0.9326\n",
      "Epoch: [ 0] [  147/  468] time: 14.2371, train_loss: 0.16124934, train_accuracy: 0.9688, test_Accuracy: 0.9327\n",
      "Epoch: [ 0] [  148/  468] time: 14.3288, train_loss: 0.27362016, train_accuracy: 0.9375, test_Accuracy: 0.9306\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  149/  468] time: 14.4213, train_loss: 0.15053803, train_accuracy: 0.9609, test_Accuracy: 0.9304\n",
      "Epoch: [ 0] [  150/  468] time: 14.5153, train_loss: 0.26566446, train_accuracy: 0.9297, test_Accuracy: 0.9297\n",
      "Epoch: [ 0] [  151/  468] time: 14.6067, train_loss: 0.19817568, train_accuracy: 0.9609, test_Accuracy: 0.9319\n",
      "Epoch: [ 0] [  152/  468] time: 14.7006, train_loss: 0.22187383, train_accuracy: 0.9219, test_Accuracy: 0.9330\n",
      "Epoch: [ 0] [  153/  468] time: 14.7871, train_loss: 0.25859040, train_accuracy: 0.9375, test_Accuracy: 0.9347\n",
      "Epoch: [ 0] [  154/  468] time: 14.8762, train_loss: 0.16941282, train_accuracy: 0.9766, test_Accuracy: 0.9358\n",
      "Epoch: [ 0] [  155/  468] time: 14.9704, train_loss: 0.15483929, train_accuracy: 0.9375, test_Accuracy: 0.9334\n",
      "Epoch: [ 0] [  156/  468] time: 15.0619, train_loss: 0.14636864, train_accuracy: 0.9609, test_Accuracy: 0.9324\n",
      "Epoch: [ 0] [  157/  468] time: 15.1600, train_loss: 0.28390071, train_accuracy: 0.8984, test_Accuracy: 0.9310\n",
      "Epoch: [ 0] [  158/  468] time: 15.2513, train_loss: 0.17361002, train_accuracy: 0.9531, test_Accuracy: 0.9314\n",
      "Epoch: [ 0] [  159/  468] time: 15.3433, train_loss: 0.29500329, train_accuracy: 0.9062, test_Accuracy: 0.9333\n",
      "Epoch: [ 0] [  160/  468] time: 15.4337, train_loss: 0.16383907, train_accuracy: 0.9297, test_Accuracy: 0.9357\n",
      "Epoch: [ 0] [  161/  468] time: 15.5265, train_loss: 0.25463119, train_accuracy: 0.8984, test_Accuracy: 0.9355\n",
      "Epoch: [ 0] [  162/  468] time: 15.6173, train_loss: 0.24688101, train_accuracy: 0.8984, test_Accuracy: 0.9358\n",
      "Epoch: [ 0] [  163/  468] time: 15.7096, train_loss: 0.19996038, train_accuracy: 0.9531, test_Accuracy: 0.9355\n",
      "Epoch: [ 0] [  164/  468] time: 15.8035, train_loss: 0.30949557, train_accuracy: 0.9062, test_Accuracy: 0.9356\n",
      "Epoch: [ 0] [  165/  468] time: 15.8929, train_loss: 0.31261370, train_accuracy: 0.8906, test_Accuracy: 0.9367\n",
      "Epoch: [ 0] [  166/  468] time: 15.9837, train_loss: 0.16512513, train_accuracy: 0.9688, test_Accuracy: 0.9379\n",
      "Epoch: [ 0] [  167/  468] time: 16.0717, train_loss: 0.28232175, train_accuracy: 0.9375, test_Accuracy: 0.9386\n",
      "Epoch: [ 0] [  168/  468] time: 16.1620, train_loss: 0.13466254, train_accuracy: 0.9531, test_Accuracy: 0.9383\n",
      "Epoch: [ 0] [  169/  468] time: 16.2540, train_loss: 0.22133082, train_accuracy: 0.9297, test_Accuracy: 0.9366\n",
      "Epoch: [ 0] [  170/  468] time: 16.3450, train_loss: 0.16852337, train_accuracy: 0.9688, test_Accuracy: 0.9354\n",
      "Epoch: [ 0] [  171/  468] time: 16.4349, train_loss: 0.22088450, train_accuracy: 0.9219, test_Accuracy: 0.9334\n",
      "Epoch: [ 0] [  172/  468] time: 16.5244, train_loss: 0.14325905, train_accuracy: 0.9531, test_Accuracy: 0.9333\n",
      "Epoch: [ 0] [  173/  468] time: 16.6157, train_loss: 0.25582230, train_accuracy: 0.9219, test_Accuracy: 0.9343\n",
      "Epoch: [ 0] [  174/  468] time: 16.7073, train_loss: 0.28740048, train_accuracy: 0.9062, test_Accuracy: 0.9370\n",
      "Epoch: [ 0] [  175/  468] time: 16.7972, train_loss: 0.19892699, train_accuracy: 0.9453, test_Accuracy: 0.9379\n",
      "Epoch: [ 0] [  176/  468] time: 16.8892, train_loss: 0.19704694, train_accuracy: 0.9375, test_Accuracy: 0.9381\n",
      "Epoch: [ 0] [  177/  468] time: 16.9796, train_loss: 0.18026724, train_accuracy: 0.9297, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [  178/  468] time: 17.0780, train_loss: 0.13751727, train_accuracy: 0.9688, test_Accuracy: 0.9332\n",
      "Epoch: [ 0] [  179/  468] time: 17.1723, train_loss: 0.33129814, train_accuracy: 0.9062, test_Accuracy: 0.9316\n",
      "Epoch: [ 0] [  180/  468] time: 17.2654, train_loss: 0.39356548, train_accuracy: 0.8984, test_Accuracy: 0.9355\n",
      "Epoch: [ 0] [  181/  468] time: 17.3619, train_loss: 0.32323924, train_accuracy: 0.9219, test_Accuracy: 0.9384\n",
      "Epoch: [ 0] [  182/  468] time: 17.4536, train_loss: 0.16380522, train_accuracy: 0.9609, test_Accuracy: 0.9405\n",
      "Epoch: [ 0] [  183/  468] time: 17.5451, train_loss: 0.15580972, train_accuracy: 0.9531, test_Accuracy: 0.9413\n",
      "Epoch: [ 0] [  184/  468] time: 17.6341, train_loss: 0.21018203, train_accuracy: 0.9375, test_Accuracy: 0.9396\n",
      "Epoch: [ 0] [  185/  468] time: 17.7259, train_loss: 0.19324729, train_accuracy: 0.9609, test_Accuracy: 0.9392\n",
      "Epoch: [ 0] [  186/  468] time: 17.8206, train_loss: 0.27947348, train_accuracy: 0.8828, test_Accuracy: 0.9374\n",
      "Epoch: [ 0] [  187/  468] time: 17.9101, train_loss: 0.27580097, train_accuracy: 0.9062, test_Accuracy: 0.9376\n",
      "Epoch: [ 0] [  188/  468] time: 18.0037, train_loss: 0.23804173, train_accuracy: 0.9297, test_Accuracy: 0.9396\n",
      "Epoch: [ 0] [  189/  468] time: 18.1005, train_loss: 0.14533588, train_accuracy: 0.9375, test_Accuracy: 0.9392\n",
      "Epoch: [ 0] [  190/  468] time: 18.1952, train_loss: 0.20993337, train_accuracy: 0.9219, test_Accuracy: 0.9404\n",
      "Epoch: [ 0] [  191/  468] time: 18.2878, train_loss: 0.19065937, train_accuracy: 0.9531, test_Accuracy: 0.9403\n",
      "Epoch: [ 0] [  192/  468] time: 18.3761, train_loss: 0.23568121, train_accuracy: 0.9375, test_Accuracy: 0.9416\n",
      "Epoch: [ 0] [  193/  468] time: 18.4688, train_loss: 0.34642923, train_accuracy: 0.8906, test_Accuracy: 0.9431\n",
      "Epoch: [ 0] [  194/  468] time: 18.5623, train_loss: 0.13795380, train_accuracy: 0.9688, test_Accuracy: 0.9436\n",
      "Epoch: [ 0] [  195/  468] time: 18.6515, train_loss: 0.22928451, train_accuracy: 0.9531, test_Accuracy: 0.9422\n",
      "Epoch: [ 0] [  196/  468] time: 18.7447, train_loss: 0.26650548, train_accuracy: 0.9219, test_Accuracy: 0.9404\n",
      "Epoch: [ 0] [  197/  468] time: 18.8386, train_loss: 0.26467636, train_accuracy: 0.8984, test_Accuracy: 0.9410\n",
      "Epoch: [ 0] [  198/  468] time: 18.9347, train_loss: 0.17476206, train_accuracy: 0.9531, test_Accuracy: 0.9379\n",
      "Epoch: [ 0] [  199/  468] time: 19.0422, train_loss: 0.17509304, train_accuracy: 0.9531, test_Accuracy: 0.9366\n",
      "Epoch: [ 0] [  200/  468] time: 19.1410, train_loss: 0.27229106, train_accuracy: 0.9297, test_Accuracy: 0.9379\n",
      "Epoch: [ 0] [  201/  468] time: 19.2417, train_loss: 0.21822841, train_accuracy: 0.9141, test_Accuracy: 0.9406\n",
      "Epoch: [ 0] [  202/  468] time: 19.3414, train_loss: 0.21305689, train_accuracy: 0.9297, test_Accuracy: 0.9422\n",
      "Epoch: [ 0] [  203/  468] time: 19.4420, train_loss: 0.11930017, train_accuracy: 0.9844, test_Accuracy: 0.9404\n",
      "Epoch: [ 0] [  204/  468] time: 19.5368, train_loss: 0.23749068, train_accuracy: 0.9141, test_Accuracy: 0.9415\n",
      "Epoch: [ 0] [  205/  468] time: 19.6332, train_loss: 0.10066749, train_accuracy: 0.9766, test_Accuracy: 0.9421\n",
      "Epoch: [ 0] [  206/  468] time: 19.7263, train_loss: 0.26082665, train_accuracy: 0.9219, test_Accuracy: 0.9443\n",
      "Epoch: [ 0] [  207/  468] time: 19.8206, train_loss: 0.11230231, train_accuracy: 0.9766, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [  208/  468] time: 19.9192, train_loss: 0.18405330, train_accuracy: 0.9453, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  209/  468] time: 20.0154, train_loss: 0.22886160, train_accuracy: 0.9297, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [  210/  468] time: 20.1086, train_loss: 0.15737042, train_accuracy: 0.9531, test_Accuracy: 0.9420\n",
      "Epoch: [ 0] [  211/  468] time: 20.1988, train_loss: 0.12520310, train_accuracy: 0.9766, test_Accuracy: 0.9412\n",
      "Epoch: [ 0] [  212/  468] time: 20.2905, train_loss: 0.20214227, train_accuracy: 0.9375, test_Accuracy: 0.9418\n",
      "Epoch: [ 0] [  213/  468] time: 20.3836, train_loss: 0.22413504, train_accuracy: 0.9453, test_Accuracy: 0.9424\n",
      "Epoch: [ 0] [  214/  468] time: 20.4770, train_loss: 0.18987381, train_accuracy: 0.9297, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [  215/  468] time: 20.5688, train_loss: 0.23846255, train_accuracy: 0.9297, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  216/  468] time: 20.6673, train_loss: 0.14048746, train_accuracy: 0.9609, test_Accuracy: 0.9463\n",
      "Epoch: [ 0] [  217/  468] time: 20.7621, train_loss: 0.26302075, train_accuracy: 0.9297, test_Accuracy: 0.9442\n",
      "Epoch: [ 0] [  218/  468] time: 20.8521, train_loss: 0.21730779, train_accuracy: 0.9375, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [  219/  468] time: 20.9532, train_loss: 0.18664944, train_accuracy: 0.9375, test_Accuracy: 0.9425\n",
      "Epoch: [ 0] [  220/  468] time: 21.0485, train_loss: 0.15799572, train_accuracy: 0.9531, test_Accuracy: 0.9424\n",
      "Epoch: [ 0] [  221/  468] time: 21.1367, train_loss: 0.31995437, train_accuracy: 0.9297, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [  222/  468] time: 21.2337, train_loss: 0.22879161, train_accuracy: 0.9297, test_Accuracy: 0.9447\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  223/  468] time: 21.3319, train_loss: 0.26018322, train_accuracy: 0.9375, test_Accuracy: 0.9461\n",
      "Epoch: [ 0] [  224/  468] time: 21.4210, train_loss: 0.18056758, train_accuracy: 0.9297, test_Accuracy: 0.9458\n",
      "Epoch: [ 0] [  225/  468] time: 21.5114, train_loss: 0.21827528, train_accuracy: 0.9297, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [  226/  468] time: 21.6023, train_loss: 0.11887397, train_accuracy: 0.9844, test_Accuracy: 0.9434\n",
      "Epoch: [ 0] [  227/  468] time: 21.6993, train_loss: 0.20365638, train_accuracy: 0.9297, test_Accuracy: 0.9411\n",
      "Epoch: [ 0] [  228/  468] time: 21.7960, train_loss: 0.29492781, train_accuracy: 0.9453, test_Accuracy: 0.9407\n",
      "Epoch: [ 0] [  229/  468] time: 21.8949, train_loss: 0.19285861, train_accuracy: 0.9531, test_Accuracy: 0.9413\n",
      "Epoch: [ 0] [  230/  468] time: 21.9877, train_loss: 0.28439391, train_accuracy: 0.9375, test_Accuracy: 0.9431\n",
      "Epoch: [ 0] [  231/  468] time: 22.0838, train_loss: 0.13235317, train_accuracy: 0.9688, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  232/  468] time: 22.1772, train_loss: 0.09989302, train_accuracy: 0.9766, test_Accuracy: 0.9463\n",
      "Epoch: [ 0] [  233/  468] time: 22.2693, train_loss: 0.19653225, train_accuracy: 0.9375, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  234/  468] time: 22.3656, train_loss: 0.21012762, train_accuracy: 0.9375, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  235/  468] time: 22.4585, train_loss: 0.15242752, train_accuracy: 0.9609, test_Accuracy: 0.9486\n",
      "Epoch: [ 0] [  236/  468] time: 22.5571, train_loss: 0.18626434, train_accuracy: 0.9297, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  237/  468] time: 22.6463, train_loss: 0.24803974, train_accuracy: 0.9297, test_Accuracy: 0.9454\n",
      "Epoch: [ 0] [  238/  468] time: 22.7382, train_loss: 0.10757142, train_accuracy: 0.9688, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [  239/  468] time: 22.8341, train_loss: 0.16821730, train_accuracy: 0.9375, test_Accuracy: 0.9469\n",
      "Epoch: [ 0] [  240/  468] time: 22.9316, train_loss: 0.19030359, train_accuracy: 0.9531, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  241/  468] time: 23.0291, train_loss: 0.18297476, train_accuracy: 0.9609, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  242/  468] time: 23.1277, train_loss: 0.15668178, train_accuracy: 0.9531, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  243/  468] time: 23.2279, train_loss: 0.11631300, train_accuracy: 0.9609, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  244/  468] time: 23.3231, train_loss: 0.15603405, train_accuracy: 0.9531, test_Accuracy: 0.9482\n",
      "Epoch: [ 0] [  245/  468] time: 23.4153, train_loss: 0.15281302, train_accuracy: 0.9531, test_Accuracy: 0.9458\n",
      "Epoch: [ 0] [  246/  468] time: 23.5115, train_loss: 0.13033679, train_accuracy: 0.9531, test_Accuracy: 0.9435\n",
      "Epoch: [ 0] [  247/  468] time: 23.6074, train_loss: 0.11969022, train_accuracy: 0.9688, test_Accuracy: 0.9426\n",
      "Epoch: [ 0] [  248/  468] time: 23.6991, train_loss: 0.24841458, train_accuracy: 0.9375, test_Accuracy: 0.9441\n",
      "Epoch: [ 0] [  249/  468] time: 23.7910, train_loss: 0.21869159, train_accuracy: 0.9688, test_Accuracy: 0.9452\n",
      "Epoch: [ 0] [  250/  468] time: 23.8838, train_loss: 0.18063073, train_accuracy: 0.9531, test_Accuracy: 0.9485\n",
      "Epoch: [ 0] [  251/  468] time: 23.9800, train_loss: 0.14080256, train_accuracy: 0.9531, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  252/  468] time: 24.0738, train_loss: 0.19490042, train_accuracy: 0.9297, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  253/  468] time: 24.1652, train_loss: 0.15045117, train_accuracy: 0.9531, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  254/  468] time: 24.2580, train_loss: 0.14128518, train_accuracy: 0.9453, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  255/  468] time: 24.3572, train_loss: 0.16136524, train_accuracy: 0.9531, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  256/  468] time: 24.4585, train_loss: 0.09107592, train_accuracy: 0.9844, test_Accuracy: 0.9472\n",
      "Epoch: [ 0] [  257/  468] time: 24.5573, train_loss: 0.17826797, train_accuracy: 0.9609, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [  258/  468] time: 24.6478, train_loss: 0.15570675, train_accuracy: 0.9609, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  259/  468] time: 24.7358, train_loss: 0.14355507, train_accuracy: 0.9375, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  260/  468] time: 24.8371, train_loss: 0.13642795, train_accuracy: 0.9453, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  261/  468] time: 24.9345, train_loss: 0.21696745, train_accuracy: 0.9297, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  262/  468] time: 25.0344, train_loss: 0.07822695, train_accuracy: 0.9766, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  263/  468] time: 25.1255, train_loss: 0.16284502, train_accuracy: 0.9609, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  264/  468] time: 25.2244, train_loss: 0.13609183, train_accuracy: 0.9609, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  265/  468] time: 25.3222, train_loss: 0.16918372, train_accuracy: 0.9531, test_Accuracy: 0.9469\n",
      "Epoch: [ 0] [  266/  468] time: 25.4158, train_loss: 0.13584411, train_accuracy: 0.9609, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  267/  468] time: 25.5156, train_loss: 0.16701257, train_accuracy: 0.9375, test_Accuracy: 0.9458\n",
      "Epoch: [ 0] [  268/  468] time: 25.6122, train_loss: 0.18434879, train_accuracy: 0.9375, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  269/  468] time: 25.7070, train_loss: 0.24730003, train_accuracy: 0.9297, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  270/  468] time: 25.8020, train_loss: 0.16642824, train_accuracy: 0.9219, test_Accuracy: 0.9487\n",
      "Epoch: [ 0] [  271/  468] time: 25.9010, train_loss: 0.20120904, train_accuracy: 0.9219, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  272/  468] time: 25.9932, train_loss: 0.06561626, train_accuracy: 0.9844, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  273/  468] time: 26.0851, train_loss: 0.16433755, train_accuracy: 0.9297, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  274/  468] time: 26.1880, train_loss: 0.16405025, train_accuracy: 0.9453, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  275/  468] time: 26.2839, train_loss: 0.11696424, train_accuracy: 0.9766, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  276/  468] time: 26.3843, train_loss: 0.09282570, train_accuracy: 0.9844, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  277/  468] time: 26.4752, train_loss: 0.10368338, train_accuracy: 0.9609, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  278/  468] time: 26.5699, train_loss: 0.17190316, train_accuracy: 0.9297, test_Accuracy: 0.9500\n",
      "Epoch: [ 0] [  279/  468] time: 26.6622, train_loss: 0.14216769, train_accuracy: 0.9609, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  280/  468] time: 26.7550, train_loss: 0.13475290, train_accuracy: 0.9531, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  281/  468] time: 26.8530, train_loss: 0.21275416, train_accuracy: 0.9531, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  282/  468] time: 26.9461, train_loss: 0.19107495, train_accuracy: 0.9609, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  283/  468] time: 27.0324, train_loss: 0.09483656, train_accuracy: 0.9766, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  284/  468] time: 27.1274, train_loss: 0.16322446, train_accuracy: 0.9297, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  285/  468] time: 27.2248, train_loss: 0.11215127, train_accuracy: 0.9688, test_Accuracy: 0.9506\n",
      "Epoch: [ 0] [  286/  468] time: 27.3235, train_loss: 0.18578446, train_accuracy: 0.9297, test_Accuracy: 0.9505\n",
      "Epoch: [ 0] [  287/  468] time: 27.4172, train_loss: 0.09970901, train_accuracy: 0.9766, test_Accuracy: 0.9505\n",
      "Epoch: [ 0] [  288/  468] time: 27.5099, train_loss: 0.09468104, train_accuracy: 0.9844, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  289/  468] time: 27.6062, train_loss: 0.09671310, train_accuracy: 0.9688, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  290/  468] time: 27.7006, train_loss: 0.08451357, train_accuracy: 0.9844, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  291/  468] time: 27.8007, train_loss: 0.19337669, train_accuracy: 0.9609, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  292/  468] time: 27.8974, train_loss: 0.11031670, train_accuracy: 0.9844, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  293/  468] time: 28.0134, train_loss: 0.12109719, train_accuracy: 0.9922, test_Accuracy: 0.9487\n",
      "Epoch: [ 0] [  294/  468] time: 28.1073, train_loss: 0.11617881, train_accuracy: 0.9688, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  295/  468] time: 28.1995, train_loss: 0.13238803, train_accuracy: 0.9688, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  296/  468] time: 28.2924, train_loss: 0.18202579, train_accuracy: 0.9375, test_Accuracy: 0.9528\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  297/  468] time: 28.3874, train_loss: 0.14165691, train_accuracy: 0.9688, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  298/  468] time: 28.4815, train_loss: 0.21757787, train_accuracy: 0.9453, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  299/  468] time: 28.5740, train_loss: 0.20907217, train_accuracy: 0.9297, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  300/  468] time: 28.6668, train_loss: 0.25615394, train_accuracy: 0.9297, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  301/  468] time: 28.7611, train_loss: 0.10746391, train_accuracy: 0.9766, test_Accuracy: 0.9525\n",
      "Epoch: [ 0] [  302/  468] time: 28.8576, train_loss: 0.09740644, train_accuracy: 0.9766, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  303/  468] time: 28.9526, train_loss: 0.10737349, train_accuracy: 0.9766, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  304/  468] time: 29.0455, train_loss: 0.05210468, train_accuracy: 1.0000, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  305/  468] time: 29.1387, train_loss: 0.15345418, train_accuracy: 0.9375, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  306/  468] time: 29.2327, train_loss: 0.11043887, train_accuracy: 0.9688, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  307/  468] time: 29.3255, train_loss: 0.14470920, train_accuracy: 0.9688, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  308/  468] time: 29.4206, train_loss: 0.12930959, train_accuracy: 0.9688, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  309/  468] time: 29.5101, train_loss: 0.13129772, train_accuracy: 0.9766, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  310/  468] time: 29.6075, train_loss: 0.11531464, train_accuracy: 0.9766, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  311/  468] time: 29.7028, train_loss: 0.11416353, train_accuracy: 0.9688, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  312/  468] time: 29.7976, train_loss: 0.09733407, train_accuracy: 0.9766, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  313/  468] time: 29.8892, train_loss: 0.15790482, train_accuracy: 0.9375, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  314/  468] time: 29.9816, train_loss: 0.16544469, train_accuracy: 0.9375, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  315/  468] time: 30.0743, train_loss: 0.14782631, train_accuracy: 0.9609, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  316/  468] time: 30.1702, train_loss: 0.10924754, train_accuracy: 0.9609, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  317/  468] time: 30.2625, train_loss: 0.09947362, train_accuracy: 0.9688, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  318/  468] time: 30.3516, train_loss: 0.22656162, train_accuracy: 0.9375, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  319/  468] time: 30.4473, train_loss: 0.24682581, train_accuracy: 0.9375, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  320/  468] time: 30.5366, train_loss: 0.12068794, train_accuracy: 0.9453, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  321/  468] time: 30.6334, train_loss: 0.16469187, train_accuracy: 0.9453, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  322/  468] time: 30.7254, train_loss: 0.07812736, train_accuracy: 0.9766, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  323/  468] time: 30.8161, train_loss: 0.12036864, train_accuracy: 0.9766, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  324/  468] time: 30.9066, train_loss: 0.22440061, train_accuracy: 0.9141, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  325/  468] time: 31.0075, train_loss: 0.16236687, train_accuracy: 0.9531, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  326/  468] time: 31.1022, train_loss: 0.09051025, train_accuracy: 0.9844, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  327/  468] time: 31.1953, train_loss: 0.20182064, train_accuracy: 0.9297, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  328/  468] time: 31.2865, train_loss: 0.30354691, train_accuracy: 0.8984, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  329/  468] time: 31.3836, train_loss: 0.09293675, train_accuracy: 0.9609, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  330/  468] time: 31.4707, train_loss: 0.12426201, train_accuracy: 0.9688, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  331/  468] time: 31.5613, train_loss: 0.12378941, train_accuracy: 0.9609, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  332/  468] time: 31.6554, train_loss: 0.17971429, train_accuracy: 0.9375, test_Accuracy: 0.9500\n",
      "Epoch: [ 0] [  333/  468] time: 31.7518, train_loss: 0.16872874, train_accuracy: 0.9609, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  334/  468] time: 31.8416, train_loss: 0.09860858, train_accuracy: 0.9766, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  335/  468] time: 31.9300, train_loss: 0.10164489, train_accuracy: 0.9531, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  336/  468] time: 32.0266, train_loss: 0.16305032, train_accuracy: 0.9453, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  337/  468] time: 32.1236, train_loss: 0.14755411, train_accuracy: 0.9609, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  338/  468] time: 32.2208, train_loss: 0.18294966, train_accuracy: 0.9609, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  339/  468] time: 32.3140, train_loss: 0.07855458, train_accuracy: 0.9766, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  340/  468] time: 32.4093, train_loss: 0.15700281, train_accuracy: 0.9531, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  341/  468] time: 32.4993, train_loss: 0.10285521, train_accuracy: 0.9766, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  342/  468] time: 32.5932, train_loss: 0.25385275, train_accuracy: 0.9219, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  343/  468] time: 32.6819, train_loss: 0.23827715, train_accuracy: 0.9375, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  344/  468] time: 32.7751, train_loss: 0.13436720, train_accuracy: 0.9688, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  345/  468] time: 32.8719, train_loss: 0.20879668, train_accuracy: 0.9297, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  346/  468] time: 32.9657, train_loss: 0.11772773, train_accuracy: 0.9766, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  347/  468] time: 33.0599, train_loss: 0.07584949, train_accuracy: 0.9844, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  348/  468] time: 33.1573, train_loss: 0.26229209, train_accuracy: 0.9375, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  349/  468] time: 33.2554, train_loss: 0.06404651, train_accuracy: 0.9922, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  350/  468] time: 33.3528, train_loss: 0.12854971, train_accuracy: 0.9531, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  351/  468] time: 33.4448, train_loss: 0.09635796, train_accuracy: 0.9609, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  352/  468] time: 33.5358, train_loss: 0.18623272, train_accuracy: 0.9453, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  353/  468] time: 33.6368, train_loss: 0.07022749, train_accuracy: 0.9766, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  354/  468] time: 33.7341, train_loss: 0.14418104, train_accuracy: 0.9688, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  355/  468] time: 33.8284, train_loss: 0.14375952, train_accuracy: 0.9531, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  356/  468] time: 33.9305, train_loss: 0.17658396, train_accuracy: 0.9453, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  357/  468] time: 34.0243, train_loss: 0.10892484, train_accuracy: 0.9453, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  358/  468] time: 34.1253, train_loss: 0.12968092, train_accuracy: 0.9609, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  359/  468] time: 34.2345, train_loss: 0.11743514, train_accuracy: 0.9766, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  360/  468] time: 34.3362, train_loss: 0.10919717, train_accuracy: 0.9688, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  361/  468] time: 34.4449, train_loss: 0.11372110, train_accuracy: 0.9766, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  362/  468] time: 34.5557, train_loss: 0.20966202, train_accuracy: 0.9531, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  363/  468] time: 34.6503, train_loss: 0.09545442, train_accuracy: 0.9766, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  364/  468] time: 34.7452, train_loss: 0.08669692, train_accuracy: 0.9844, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  365/  468] time: 34.8484, train_loss: 0.09671341, train_accuracy: 0.9844, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  366/  468] time: 34.9484, train_loss: 0.13567430, train_accuracy: 0.9688, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  367/  468] time: 35.0464, train_loss: 0.14713578, train_accuracy: 0.9609, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  368/  468] time: 35.1496, train_loss: 0.14114757, train_accuracy: 0.9531, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  369/  468] time: 35.2551, train_loss: 0.14879674, train_accuracy: 0.9531, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  370/  468] time: 35.3576, train_loss: 0.11512967, train_accuracy: 0.9766, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  371/  468] time: 35.4524, train_loss: 0.13123053, train_accuracy: 0.9531, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  372/  468] time: 35.5526, train_loss: 0.12674409, train_accuracy: 0.9922, test_Accuracy: 0.9550\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  373/  468] time: 35.6519, train_loss: 0.15533739, train_accuracy: 0.9453, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  374/  468] time: 35.7475, train_loss: 0.13455072, train_accuracy: 0.9688, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  375/  468] time: 35.8445, train_loss: 0.19740234, train_accuracy: 0.9609, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  376/  468] time: 35.9392, train_loss: 0.05739752, train_accuracy: 0.9844, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  377/  468] time: 36.0372, train_loss: 0.14384778, train_accuracy: 0.9688, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  378/  468] time: 36.1375, train_loss: 0.17199636, train_accuracy: 0.9531, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  379/  468] time: 36.2401, train_loss: 0.20914838, train_accuracy: 0.9375, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  380/  468] time: 36.3364, train_loss: 0.12735286, train_accuracy: 0.9453, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  381/  468] time: 36.4384, train_loss: 0.10665734, train_accuracy: 0.9609, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  382/  468] time: 36.5396, train_loss: 0.16688518, train_accuracy: 0.9531, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  383/  468] time: 36.6418, train_loss: 0.09276950, train_accuracy: 0.9688, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  384/  468] time: 36.7379, train_loss: 0.14570035, train_accuracy: 0.9609, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  385/  468] time: 36.8327, train_loss: 0.11157576, train_accuracy: 0.9609, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  386/  468] time: 36.9352, train_loss: 0.10461429, train_accuracy: 0.9844, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  387/  468] time: 37.0390, train_loss: 0.04334654, train_accuracy: 0.9922, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  388/  468] time: 37.1424, train_loss: 0.08026519, train_accuracy: 0.9766, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  389/  468] time: 37.2412, train_loss: 0.17242165, train_accuracy: 0.9375, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  390/  468] time: 37.3431, train_loss: 0.17716745, train_accuracy: 0.9375, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  391/  468] time: 37.4456, train_loss: 0.16823140, train_accuracy: 0.9297, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  392/  468] time: 37.5384, train_loss: 0.14040041, train_accuracy: 0.9531, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  393/  468] time: 37.6374, train_loss: 0.24275473, train_accuracy: 0.9375, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  394/  468] time: 37.7359, train_loss: 0.14790840, train_accuracy: 0.9688, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  395/  468] time: 37.8383, train_loss: 0.09989472, train_accuracy: 0.9844, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  396/  468] time: 37.9370, train_loss: 0.07190669, train_accuracy: 0.9766, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  397/  468] time: 38.0336, train_loss: 0.12072396, train_accuracy: 0.9688, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  398/  468] time: 38.1315, train_loss: 0.15926820, train_accuracy: 0.9688, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  399/  468] time: 38.2329, train_loss: 0.14357489, train_accuracy: 0.9375, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  400/  468] time: 38.3278, train_loss: 0.14838317, train_accuracy: 0.9531, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  401/  468] time: 38.4231, train_loss: 0.10014865, train_accuracy: 0.9766, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  402/  468] time: 38.5180, train_loss: 0.16273221, train_accuracy: 0.9453, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  403/  468] time: 38.6122, train_loss: 0.13867223, train_accuracy: 0.9609, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  404/  468] time: 38.7090, train_loss: 0.06281906, train_accuracy: 0.9922, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  405/  468] time: 38.8044, train_loss: 0.15696798, train_accuracy: 0.9531, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  406/  468] time: 38.8988, train_loss: 0.10878406, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  407/  468] time: 38.9894, train_loss: 0.11465613, train_accuracy: 0.9766, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  408/  468] time: 39.0832, train_loss: 0.08751480, train_accuracy: 0.9766, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  409/  468] time: 39.1789, train_loss: 0.14924821, train_accuracy: 0.9531, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  410/  468] time: 39.2721, train_loss: 0.10911728, train_accuracy: 0.9688, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  411/  468] time: 39.3694, train_loss: 0.13747746, train_accuracy: 0.9531, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  412/  468] time: 39.4665, train_loss: 0.13302544, train_accuracy: 0.9297, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  413/  468] time: 39.5637, train_loss: 0.10974400, train_accuracy: 0.9766, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  414/  468] time: 39.6608, train_loss: 0.06411657, train_accuracy: 0.9844, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  415/  468] time: 39.7546, train_loss: 0.09286235, train_accuracy: 0.9844, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  416/  468] time: 39.8511, train_loss: 0.06227522, train_accuracy: 0.9766, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  417/  468] time: 39.9407, train_loss: 0.13771737, train_accuracy: 0.9609, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  418/  468] time: 40.0307, train_loss: 0.04342861, train_accuracy: 0.9844, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  419/  468] time: 40.1252, train_loss: 0.11788864, train_accuracy: 0.9609, test_Accuracy: 0.9568\n",
      "Epoch: [ 0] [  420/  468] time: 40.2186, train_loss: 0.11105556, train_accuracy: 0.9688, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  421/  468] time: 40.3087, train_loss: 0.11287695, train_accuracy: 0.9609, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  422/  468] time: 40.4067, train_loss: 0.13888338, train_accuracy: 0.9688, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  423/  468] time: 40.5119, train_loss: 0.08711173, train_accuracy: 0.9844, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  424/  468] time: 40.6183, train_loss: 0.13991621, train_accuracy: 0.9609, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  425/  468] time: 40.7180, train_loss: 0.15043409, train_accuracy: 0.9766, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  426/  468] time: 40.8128, train_loss: 0.11323555, train_accuracy: 0.9844, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  427/  468] time: 40.9094, train_loss: 0.06476830, train_accuracy: 0.9922, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  428/  468] time: 41.0006, train_loss: 0.25110468, train_accuracy: 0.9219, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  429/  468] time: 41.0980, train_loss: 0.19771086, train_accuracy: 0.9609, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  430/  468] time: 41.1943, train_loss: 0.11073121, train_accuracy: 0.9688, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  431/  468] time: 41.2927, train_loss: 0.08224127, train_accuracy: 0.9688, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  432/  468] time: 41.3861, train_loss: 0.11397314, train_accuracy: 0.9688, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  433/  468] time: 41.4797, train_loss: 0.14226925, train_accuracy: 0.9453, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  434/  468] time: 41.5748, train_loss: 0.07928927, train_accuracy: 0.9688, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  435/  468] time: 41.6696, train_loss: 0.08509672, train_accuracy: 0.9844, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  436/  468] time: 41.7645, train_loss: 0.11262645, train_accuracy: 0.9609, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  437/  468] time: 41.8623, train_loss: 0.10298819, train_accuracy: 0.9688, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  438/  468] time: 41.9565, train_loss: 0.10322271, train_accuracy: 0.9609, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  439/  468] time: 42.0481, train_loss: 0.06852045, train_accuracy: 0.9844, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  440/  468] time: 42.1505, train_loss: 0.08357573, train_accuracy: 0.9922, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  441/  468] time: 42.2482, train_loss: 0.08509928, train_accuracy: 0.9766, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  442/  468] time: 42.3429, train_loss: 0.09727608, train_accuracy: 0.9844, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  443/  468] time: 42.4357, train_loss: 0.12336783, train_accuracy: 0.9609, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  444/  468] time: 42.5320, train_loss: 0.07565853, train_accuracy: 0.9766, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  445/  468] time: 42.6300, train_loss: 0.18554243, train_accuracy: 0.9453, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  446/  468] time: 42.7244, train_loss: 0.03525577, train_accuracy: 1.0000, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  447/  468] time: 42.8233, train_loss: 0.18023542, train_accuracy: 0.9531, test_Accuracy: 0.9603\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  448/  468] time: 42.9204, train_loss: 0.12465809, train_accuracy: 0.9688, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  449/  468] time: 43.0195, train_loss: 0.12483499, train_accuracy: 0.9609, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  450/  468] time: 43.1135, train_loss: 0.09034453, train_accuracy: 0.9688, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  451/  468] time: 43.2109, train_loss: 0.12152860, train_accuracy: 0.9688, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  452/  468] time: 43.3006, train_loss: 0.05697597, train_accuracy: 0.9766, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  453/  468] time: 43.3974, train_loss: 0.08105034, train_accuracy: 0.9844, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  454/  468] time: 43.4874, train_loss: 0.14425385, train_accuracy: 0.9766, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  455/  468] time: 43.5841, train_loss: 0.07521962, train_accuracy: 0.9766, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  456/  468] time: 43.6834, train_loss: 0.07860439, train_accuracy: 0.9688, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  457/  468] time: 43.7862, train_loss: 0.09478126, train_accuracy: 0.9688, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  458/  468] time: 43.8786, train_loss: 0.08770430, train_accuracy: 0.9766, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  459/  468] time: 43.9747, train_loss: 0.15784821, train_accuracy: 0.9609, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  460/  468] time: 44.0670, train_loss: 0.07348727, train_accuracy: 0.9844, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  461/  468] time: 44.1616, train_loss: 0.04955758, train_accuracy: 0.9844, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  462/  468] time: 44.2551, train_loss: 0.13794906, train_accuracy: 0.9844, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  463/  468] time: 44.3528, train_loss: 0.09997927, train_accuracy: 0.9688, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  464/  468] time: 44.4518, train_loss: 0.05327693, train_accuracy: 0.9922, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  465/  468] time: 44.5507, train_loss: 0.22266154, train_accuracy: 0.9219, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  466/  468] time: 44.6478, train_loss: 0.07454156, train_accuracy: 0.9766, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  467/  468] time: 44.7433, train_loss: 0.11013886, train_accuracy: 0.9844, test_Accuracy: 0.9606\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.contrib.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)\n",
    "    global_step = tf.train.create_global_step()\n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)\n",
    "        start_iteration = checkpoint_counter - start_epoch * training_iterations\n",
    "        counter = checkpoint_counter\n",
    "        global_step.assign(checkpoint_counter)\n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default(), tf.contrib.summary.always_record_summaries():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx in range(start_iteration, training_iterations):\n",
    "                train_input, train_label = train_iterator.get_next()\n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables), global_step=global_step)\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "\n",
    "                test_input, test_label = test_iterator.get_next()\n",
    "                test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.contrib.summary.scalar(name='train_loss', tensor=train_loss)\n",
    "                tf.contrib.summary.scalar(name='train_accuracy', tensor=train_accuracy)\n",
    "                tf.contrib.summary.scalar(name='test_accuracy', tensor=test_accuracy)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1\n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    test_input, test_label = test_iterator.get_next()\n",
    "    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test accuracy : 96.55 %"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}