numpy_tf_validate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from vanilla_impl.two_layer_mlp import MLPNet\n",
    "from tf_impl.two_layer_mlp import two_layer_mlp_net, xent_loss, sgd_optimize\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "net = MLPNet(feature_dim=784, hidden_size=256, num_classes=10, learning_rate=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x, y = mnist.train.next_batch(batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_ph = tf.placeholder(dtype=tf.float32, shape=(None, 784))\n",
    "y_ph = tf.placeholder(dtype=tf.float32, shape=(None, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "outputs = two_layer_mlp_net(x_ph, 256, 10)\n",
    "loss = xent_loss(outputs, y_ph)\n",
    "train_op, pairs = sgd_optimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "net.hidden_0_weight[:] = sess.run(pairs[0][1])\n",
    "net.hidden_0_bias[:] = sess.run(pairs[1][1])\n",
    "net.hidden_1_weight[:] = sess.run(pairs[2][1])\n",
    "net.hidden_1_bias[:] = sess.run(pairs[3][1])\n",
    "net.output_weight[:] = sess.run(pairs[4][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(<tf.Tensor 'gradients/dense/MatMul_grad/MatMul_1:0' shape=(784, 256) dtype=float32>,\n",
       "  <tf.Variable 'dense/kernel:0' shape=(784, 256) dtype=float32_ref>),\n",
       " (<tf.Tensor 'gradients/dense/BiasAdd_grad/BiasAddGrad:0' shape=(256,) dtype=float32>,\n",
       "  <tf.Variable 'dense/bias:0' shape=(256,) dtype=float32_ref>),\n",
       " (<tf.Tensor 'gradients/dense_2/MatMul_grad/MatMul_1:0' shape=(256, 256) dtype=float32>,\n",
       "  <tf.Variable 'dense_1/kernel:0' shape=(256, 256) dtype=float32_ref>),\n",
       " (<tf.Tensor 'gradients/dense_2/BiasAdd_grad/BiasAddGrad:0' shape=(256,) dtype=float32>,\n",
       "  <tf.Variable 'dense_1/bias:0' shape=(256,) dtype=float32_ref>),\n",
       " (<tf.Tensor 'gradients/dense_3/MatMul_grad/MatMul_1:0' shape=(256, 10) dtype=float32>,\n",
       "  <tf.Variable 'dense_2/kernel:0' shape=(256, 10) dtype=float32_ref>)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "_, y_label = np.where(y)\n",
    "net.forward(x, y_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current xent: 2.45210981369\n",
      "current xent: 2.34260310577\n",
      "current xent: 2.32653062016\n",
      "current xent: 2.31941086368\n",
      "current xent: 2.31627211712\n",
      "current xent: 2.31441176695\n",
      "current xent: 2.31201324242\n",
      "current xent: 2.30948698792\n",
      "current xent: 2.30845377338\n",
      "current xent: 2.30726800239\n",
      "current xent: 2.30653545581\n",
      "current xent: 2.30519256989\n",
      "current xent: 2.30464438267\n",
      "current xent: 2.30346326887\n",
      "current xent: 2.30262939339\n",
      "current xent: 2.3014641588\n",
      "current xent: 2.3004943718\n",
      "current xent: 2.29992253903\n",
      "current xent: 2.29900225515\n",
      "current xent: 2.29819961255\n",
      "current xent: 2.29743256064\n",
      "current xent: 2.29644675717\n",
      "current xent: 2.29586973813\n",
      "current xent: 2.29507597867\n",
      "current xent: 2.2942350724\n",
      "current xent: 2.29329166169\n",
      "current xent: 2.29253745921\n",
      "current xent: 2.29157867481\n",
      "current xent: 2.29103838836\n",
      "current xent: 2.29029486298\n",
      "current xent: 2.28935731013\n",
      "current xent: 2.2883669648\n",
      "current xent: 2.28765984808\n",
      "current xent: 2.28685316537\n",
      "current xent: 2.28605716741\n",
      "current xent: 2.28520466748\n",
      "current xent: 2.28446538705\n",
      "current xent: 2.28362997768\n",
      "current xent: 2.2828692194\n",
      "current xent: 2.28204500626\n",
      "current xent: 2.28119068927\n",
      "current xent: 2.28025134497\n",
      "current xent: 2.27943672031\n",
      "current xent: 2.27860941076\n",
      "current xent: 2.27746334713\n",
      "current xent: 2.27660073078\n",
      "current xent: 2.27561742155\n",
      "current xent: 2.27464833531\n",
      "current xent: 2.2736860469\n",
      "current xent: 2.27271838756\n",
      "current xent: 2.27185791021\n",
      "current xent: 2.27103410059\n",
      "current xent: 2.27016872539\n",
      "current xent: 2.26932033982\n",
      "current xent: 2.26837875277\n",
      "current xent: 2.26746530484\n",
      "current xent: 2.26646609327\n",
      "current xent: 2.2654520198\n",
      "current xent: 2.26446514964\n",
      "current xent: 2.26337886341\n",
      "current xent: 2.26238521046\n",
      "current xent: 2.26135573006\n",
      "current xent: 2.2602996986\n",
      "current xent: 2.25932406275\n",
      "current xent: 2.25822109032\n",
      "current xent: 2.25701970236\n",
      "current xent: 2.25589701958\n",
      "current xent: 2.25476653579\n",
      "current xent: 2.25361501224\n",
      "current xent: 2.25254350466\n",
      "current xent: 2.25142818107\n",
      "current xent: 2.25038534846\n",
      "current xent: 2.24918277591\n",
      "current xent: 2.24800567872\n",
      "current xent: 2.24671125618\n",
      "current xent: 2.2454973796\n",
      "current xent: 2.24423385156\n",
      "current xent: 2.24296488767\n",
      "current xent: 2.2416475473\n",
      "current xent: 2.24017447656\n",
      "current xent: 2.23862525203\n",
      "current xent: 2.23729710212\n",
      "current xent: 2.23589156896\n",
      "current xent: 2.23458404968\n",
      "current xent: 2.23312018577\n",
      "current xent: 2.23173081023\n",
      "current xent: 2.23012574547\n",
      "current xent: 2.22871639799\n",
      "current xent: 2.22705600333\n",
      "current xent: 2.22553924768\n",
      "current xent: 2.2238440037\n",
      "current xent: 2.22226946226\n",
      "current xent: 2.22061233031\n",
      "current xent: 2.21880573244\n",
      "current xent: 2.2169228848\n",
      "current xent: 2.21526930815\n",
      "current xent: 2.21358997923\n",
      "current xent: 2.21174173309\n",
      "current xent: 2.20986449602\n",
      "current xent: 2.20808305903\n",
      "current xent: 2.2062137553\n",
      "current xent: 2.20433642759\n",
      "current xent: 2.20256459816\n",
      "current xent: 2.20049910752\n",
      "current xent: 2.19840459162\n",
      "current xent: 2.19637997543\n",
      "current xent: 2.19439969329\n",
      "current xent: 2.19225952715\n",
      "current xent: 2.19008514516\n",
      "current xent: 2.18762406429\n",
      "current xent: 2.18535909407\n",
      "current xent: 2.18316665743\n",
      "current xent: 2.18090593636\n",
      "current xent: 2.17866286224\n",
      "current xent: 2.17635632209\n",
      "current xent: 2.17393795721\n",
      "current xent: 2.17159065511\n",
      "current xent: 2.16914363027\n",
      "current xent: 2.16670524848\n",
      "current xent: 2.1641748735\n",
      "current xent: 2.16162803852\n",
      "current xent: 2.1590568612\n",
      "current xent: 2.15630508403\n",
      "current xent: 2.15366927607\n",
      "current xent: 2.15098391433\n",
      "current xent: 2.14830175094\n",
      "current xent: 2.14556257417\n",
      "current xent: 2.1428641192\n",
      "current xent: 2.13990871544\n",
      "current xent: 2.13683408488\n",
      "current xent: 2.13400891512\n",
      "current xent: 2.13117606732\n",
      "current xent: 2.12816987111\n",
      "current xent: 2.12511218508\n",
      "current xent: 2.12205240233\n",
      "current xent: 2.11872702798\n",
      "current xent: 2.11550345375\n",
      "current xent: 2.11221786845\n",
      "current xent: 2.1088583705\n",
      "current xent: 2.10571786703\n",
      "current xent: 2.10250020494\n",
      "current xent: 2.09910441131\n",
      "current xent: 2.0956606447\n",
      "current xent: 2.09237771163\n",
      "current xent: 2.08899345516\n",
      "current xent: 2.08551521769\n",
      "current xent: 2.08202105955\n",
      "current xent: 2.078515122\n",
      "current xent: 2.07512071931\n",
      "current xent: 2.07157191716\n",
      "current xent: 2.06797242317\n",
      "current xent: 2.06437249065\n",
      "current xent: 2.06071133812\n",
      "current xent: 2.05713443979\n",
      "current xent: 2.05353381045\n",
      "current xent: 2.04996679722\n",
      "current xent: 2.04623681208\n",
      "current xent: 2.04249363776\n",
      "current xent: 2.03885699046\n",
      "current xent: 2.03536421795\n",
      "current xent: 2.03172285603\n",
      "current xent: 2.02799284736\n",
      "current xent: 2.02431872482\n",
      "current xent: 2.02052786551\n",
      "current xent: 2.01676667303\n",
      "current xent: 2.0128414524\n",
      "current xent: 2.00920914179\n",
      "current xent: 2.00546900269\n",
      "current xent: 2.00170349845\n",
      "current xent: 1.99773909642\n",
      "current xent: 1.99382434691\n",
      "current xent: 1.98993894776\n",
      "current xent: 1.98576119397\n",
      "current xent: 1.98182162572\n",
      "current xent: 1.97778515574\n",
      "current xent: 1.97390520763\n",
      "current xent: 1.96988552447\n",
      "current xent: 1.96577471449\n",
      "current xent: 1.96181275105\n",
      "current xent: 1.95790951424\n",
      "current xent: 1.95393983395\n",
      "current xent: 1.94985571774\n",
      "current xent: 1.94585816435\n",
      "current xent: 1.94185292611\n",
      "current xent: 1.93784769733\n",
      "current xent: 1.93390296807\n",
      "current xent: 1.92995983417\n",
      "current xent: 1.92603726697\n",
      "current xent: 1.92198539835\n",
      "current xent: 1.91803373057\n",
      "current xent: 1.91410800699\n",
      "current xent: 1.91037241743\n",
      "current xent: 1.90637249021\n",
      "current xent: 1.90243549516\n",
      "current xent: 1.89848229456\n",
      "current xent: 1.89458016331\n",
      "current xent: 1.89066693978\n",
      "current xent: 1.88646378613\n",
      "current xent: 1.88253825519\n",
      "current xent: 1.87857239404\n",
      "current xent: 1.87465717437\n",
      "current xent: 1.87065746311\n",
      "current xent: 1.86669192224\n",
      "current xent: 1.8628353938\n",
      "current xent: 1.85885401215\n",
      "current xent: 1.85496446412\n",
      "current xent: 1.85120678934\n",
      "current xent: 1.84723016621\n",
      "current xent: 1.84348388801\n",
      "current xent: 1.83950250419\n",
      "current xent: 1.83562734467\n",
      "current xent: 1.8314406444\n",
      "current xent: 1.82757439904\n",
      "current xent: 1.82367880025\n",
      "current xent: 1.81957095797\n",
      "current xent: 1.81587640811\n",
      "current xent: 1.81220093024\n",
      "current xent: 1.80832523532\n",
      "current xent: 1.80461152521\n",
      "current xent: 1.80088663913\n",
      "current xent: 1.79713716755\n",
      "current xent: 1.79350508093\n",
      "current xent: 1.78965197121\n",
      "current xent: 1.78576595331\n",
      "current xent: 1.78212639804\n",
      "current xent: 1.77844449886\n",
      "current xent: 1.77457379875\n",
      "current xent: 1.77086903746\n",
      "current xent: 1.76718494966\n",
      "current xent: 1.76361089207\n",
      "current xent: 1.76002822554\n",
      "current xent: 1.75642080875\n",
      "current xent: 1.75266281136\n",
      "current xent: 1.74893757029\n",
      "current xent: 1.74532324782\n",
      "current xent: 1.74171307515\n",
      "current xent: 1.73803275564\n",
      "current xent: 1.73460276989\n",
      "current xent: 1.73102777085\n",
      "current xent: 1.72745798397\n",
      "current xent: 1.72383268554\n",
      "current xent: 1.72040160462\n",
      "current xent: 1.71683145748\n",
      "current xent: 1.7134302984\n",
      "current xent: 1.71003920119\n",
      "current xent: 1.70681892779\n",
      "current xent: 1.70341904876\n",
      "current xent: 1.69996618041\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-7dcdf1fb5655>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwhere\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0maverage_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/pu/work/PycharmProjects/neural_nets_exercise/vanilla_impl/two_layer_mlp.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     50\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_0_act_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmanual_grad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxw_plus_b_grad_x\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_1_grad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_1_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_0_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmanual_grad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msigmoid_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_0_act\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_0_act_grad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_weight_0_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmanual_grad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxw_plus_b_grad_w\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_0_grad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_bias_0_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_0_grad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/pu/work/PycharmProjects/neural_nets_exercise/vanilla_impl/manual_grad.py\u001b[0m in \u001b[0;36mxw_plus_b_grad_w\u001b[0;34m(grad, x)\u001b[0m\n\u001b[1;32m     36\u001b[0m   \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m   \u001b[0mgrad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "average_loss = 0\n",
    "counts = 0\n",
    "for i in range(100000):\n",
    "    x, y = mnist.train.next_batch(batch_size=32)\n",
    "    _, y = np.where(y)\n",
    "    net.forward(x, y)\n",
    "    net.backward()\n",
    "    net.update()\n",
    "    average_loss += net.xent.mean()\n",
    "    counts += 1\n",
    "    if i % 25 == 0:\n",
    "        print ('current xent:', average_loss / counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}