mnist_mlp.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST with a simple *MLP*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "using Knet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36mINFO: \u001b[39m\u001b[22m\u001b[36mLoading MNIST...\n",
      "\u001b[39m"
     ]
    }
   ],
   "source": [
    "include(Knet.dir(\"data\",\"mnist.jl\"))\n",
    "xtrn, ytrn, xtst, ytst = mnist();\n",
    "dtrn = minibatch(xtrn, ytrn, 100);\n",
    "dtst = minibatch(xtst, ytst, 100);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = prod(size(xtrn)[1:2]);\n",
    "classifiers = 10;\n",
    "weights = [128, 64];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "predict (generic function with 1 method)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function predict(w, x)\n",
    "    for i=1:2:length(w)\n",
    "        x = w[i]*mat(x) .+ w[i+1]\n",
    "        if i < length(w)-1\n",
    "            x = relu.(x)\n",
    "        end\n",
    "    end\n",
    "    return x\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss(w,x,y) = nll(predict(w,x),y)\n",
    "lossgradient = grad(loss);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train (generic function with 1 method)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function train(w, dtrn; lr=.1, epochs=20)\n",
    "    for epoch= 1:epochs\n",
    "        for (x,y) in dtrn\n",
    "            g = lossgradient(w, x, y)\n",
    "            update!(w, g; lr=lr)\n",
    "        end\n",
    "    end\n",
    "    return w\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "make_weights (generic function with 1 method)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function make_weights(h...; atype=Array{Float32}, winit=0.1)\n",
    "    w = Any[]\n",
    "    x = features\n",
    "    for y in [h..., classifiers]\n",
    "        push!(w, convert(atype, winit*randn(y,x)))\n",
    "        push!(w, convert(atype, zeros(y, 1)))\n",
    "        x=y\n",
    "    end\n",
    "    return w\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epochs = 10;\n",
    "learning_rate = 0.1;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x training size: (28, 28, 1, 60000)\n",
      "y training size: (28, 28, 1, 10000)\n"
     ]
    }
   ],
   "source": [
    "@printf(\"x training size: %s\\ny training size: %s\\n\",size(xtrn), size(xtst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = make_weights(weights...);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(:epoch, 0, :trn, 0.09523333333333334, :test, 0.0941)\n",
      "(:epoch, 1, :trn, 0.9195166666666666, :test, 0.9218)\n",
      "(:epoch, 2, :trn, 0.9452, :test, 0.9434)\n",
      "(:epoch, 3, :trn, 0.9581, :test, 0.9554)\n",
      "(:epoch, 4, :trn, 0.9651833333333333, :test, 0.9598)\n",
      "(:epoch, 5, :trn, 0.9699, :test, 0.9644)\n",
      "(:epoch, 6, :trn, 0.9737, :test, 0.9657)\n",
      "(:epoch, 7, :trn, 0.9764333333333334, :test, 0.9675)\n",
      "(:epoch, 8, :trn, 0.9793333333333333, :test, 0.9693)\n",
      "(:epoch, 9, :trn, 0.9817833333333333, :test, 0.9702)\n",
      "(:epoch, 10, :trn, 0.98395, :test, 0.9703)\n",
      " 21.859452 seconds (5.66 M allocations: 10.432 GiB, 3.27% gc time)\n"
     ]
    }
   ],
   "source": [
    "println((:epoch, 0, :trn, accuracy(w,dtrn,predict), :test, accuracy(w,dtst,predict)))\n",
    "@time for epoch=1:epochs\n",
    "   train(w, dtrn; lr=learning_rate, epochs=1)\n",
    "   println((:epoch, epoch, :trn, accuracy(w,dtrn,predict), :test, accuracy(w,dtst,predict)))\n",
    "end"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.2",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}