normal_flattening.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import classification_report\n",
    "import matplotlib.pyplot as plt\n",
    "import h5py\n",
    "import time\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load MNIST data\n",
    "opt={\"n_h\":64,\"n_x\":784,\"lr\":0.5,\"beta\":0.9,\"batch_size\":64,\"epochs\":50}\n",
    "MNIST_data = h5py.File(\"MNISTdata.hdf5\", 'r')\n",
    "x_train = np.float32(MNIST_data['x_train'][:])\n",
    "y_train = np.int32(np.array(MNIST_data['y_train'][:, 0])).reshape(-1, 1)\n",
    "x_test  = np.float32(MNIST_data['x_test'][:])\n",
    "y_test  = np.int32(np.array(MNIST_data['y_test'][:, 0])).reshape(-1, 1)\n",
    "MNIST_data.close()\n",
    "\n",
    "\n",
    "# stack together for next step\n",
    "X = np.vstack((x_train, x_test))\n",
    "y = np.vstack((y_train, y_test))\n",
    "\n",
    "\n",
    "# one-hot encoding\n",
    "digits = 10\n",
    "examples = y.shape[0]\n",
    "y = y.reshape(1, examples)\n",
    "Y_new = np.eye(digits)[y.astype('int32')]\n",
    "Y_new = Y_new.T.reshape(digits, examples)\n",
    "\n",
    "\n",
    "# number of training set\n",
    "m = 60000\n",
    "m_test = X.shape[0] - m\n",
    "X_train, X_test = X[:m].T, X[m:].T\n",
    "Y_train, Y_test = Y_new[:, :m], Y_new[:, m:]\n",
    "\n",
    "\n",
    "# shuffle training set\n",
    "shuffle_index = np.random.permutation(m)\n",
    "X_train, Y_train = X_train[:, shuffle_index], Y_train[:, shuffle_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAANgUlEQVR4nO3df4wc9XnH8c+Hq+1QA40NtusYpxDXiJJGdZqrDaGqSFGQgxIMqkhxReREaU0lUKHijyAaKfyJUhyEqgbJCVYcSIhICLVbIYJ1onVpZJczMrap4xosJza++CCuZBxU/3z6xw3VYd/Onndmd9Y875d02t15dmYere5zM7ff2f06IgTg/e+8phsA0BuEHUiCsANJEHYgCcIOJPEbvdzZVE+LD2h6L3cJpPK/+rWOxVFPVKsUdttLJT0iaUDStyPiwbLnf0DTtcTXV9klgBKbY6hlrePTeNsDkv5R0mckXSVpue2rOt0egO6q8j/7YkmvRcSeiDgm6QeSltXTFoC6VQn7PEn7xj3eXyx7D9srbQ/bHj6uoxV2B6CKKmGf6E2AM669jYjVETEYEYNTNK3C7gBUUSXs+yXNH/f4UkkHqrUDoFuqhP0lSQttX257qqTbJK2vpy0Adet46C0iTti+S9JPNDb0tiYiXq2tMwC1qjTOHhHPSnq2pl4AdBGXywJJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSqDRls+29kt6WdFLSiYgYrKMpAPWrFPbCpyLirRq2A6CLOI0Hkqga9pD0vO0ttldO9ATbK20P2x4+rqMVdwegU1VP46+NiAO2Z0vaYPtnEbFx/BMiYrWk1ZJ0kWdGxf0B6FClI3tEHChuRyU9I2lxHU0BqF/HYbc93faF796XdIOkHXU1BqBeVU7j50h6xva72/l+RDxXS1cAatdx2CNij6Q/qLEXAF3E0BuQBGEHkiDsQBKEHUiCsANJ1PFBmHPCkVuXlNZ/tGpVaf2VYxe3rD18+5+X73zTtvI60AMc2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgiTTj7L+8xqX1AZfXP3X+kZa1k088Vbru3/zLF0vrV67aX1o/sa+8DkwGR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSMIRvZuk5SLPjCW+vmf7OxuvP3R1af3fP/9Qy9rMgWml657X5m/qV0c/UVr/tza9XfziG6X1MicPHCytD3xoTsfblqQ48uuWNV8wvdK2qzj15q/K6++806NO6rU5hnQ4Dk140QhHdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2STq29I9a1+4pH7P914/9sLR+Sqc66qkOt772udL6D3/3nytt/4nD81vWbr9oX+m67a5PqPK6fWzjX5bWL1/+SsfbblKlcXbba2yP2t4xbtlM2xts7y5uZ9TZMID6TeY0/juSlp627D5JQxGxUNJQ8RhAH2sb9ojYKOnQaYuXSVpb3F8r6eaa+wJQs07foJsTESOSVNzObvVE2yttD9sePq6jHe4OQFVdfzc+IlZHxGBEDE5R+QdGAHRPp2E/aHuuJBW3o/W1BKAbOg37ekkrivsrJK2rpx0A3dJ2nN32k5Kuk3SJpIOSvibpnyQ9JenDkn4h6daIOP1NvDOcy+PsVYyuu7K0vmnw8R51cqZujmVX1c3ernz+r0vrV3xpS8fbblLZOHvbSSIiYnmLUr7UAucwLpcFkiDsQBKEHUiCsANJEHYgiTRTNjdp9rKfldZvUuuPz0rSyL2fLK2frHBh4m9+8q3S+vLLhjvfuKQXDy1oWdv13MLSdWddd6C0vuGjT3fUkyRd/NOpHa97ruLIDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMM5+Dpi76qeN7fsnuqjiFt5sWbm0pCZJez54TWn91EfLP+L61YOLW9bmPFf+NdYnSqvnJo7sQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AE4+zoWxtv+/s2zyj/IP8zLyxpWVuwb1MHHZ3bOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs6MxAws/Ulq/ZKB82uR2UzbP33DyrHt6P2t7ZLe9xvao7R3jlj1g+w3bW4ufG7vbJoCqJnMa/x1JSydY/nBELCp+nq23LQB1axv2iNgo6VAPegHQRVXeoLvL9rbiNH9GqyfZXml72PbwcR2tsDsAVXQa9kclLZC0SNKIpFWtnhgRqyNiMCIGp7T54AKA7uko7BFxMCJORsQpSd+S1PprPAH0hY7CbnvuuIe3SNrR6rkA+kPbcXbbT0q6TtIltvdL+pqk62wvkhSS9kq6o4s94n1q3hMHK63/Z7tvKq1PG3qlZS0q7fnc1DbsEbF8gsWPdaEXAF3E5bJAEoQdSIKwA0kQdiAJwg4kwUdc0VXv3NL665y/eek3S9ed4oHS+p4Nl5fW5x8fKa1nw5EdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1dtf+zrb/Oud1XQe88Vv41ZvOHjnTUU1Yc2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZUcnhv7i6tL75hpaTBUmaWrruHXffU1o/f9N/ltbxXhzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtlRyehgef23zms9lv6VX15Tuu756xhHr1PbI7vt+bZfsL3T9qu27y6Wz7S9wfbu4nZG99sF0KnJnMafkHRvRPyepKsl3Wn7Kkn3SRqKiIWShorHAPpU27BHxEhEvFzcf1vSTknzJC2TtLZ42lpJN3erSQDVndUbdLYvk/RxSZslzYmIEWnsD4Kk2S3WWWl72PbwcZV/pxiA7pl02G1fIOlpSfdExOHJrhcRqyNiMCIGp2haJz0CqMGkwm57isaC/r2I+HGx+KDtuUV9rqTR7rQIoA5th95sW9JjknZGxDfGldZLWiHpweJ2XVc6RKPOW3RVaf3rn/1+x9veeWf5tqVtHW8bZ5rMOPu1kr4gabvtrcWy+zUW8qdsf1nSLyTd2p0WAdShbdgj4kVJblG+vt52AHQLl8sCSRB2IAnCDiRB2IEkCDuQBB9xRalZj+4vrd80/X9K61/6+adbFzcxjt5LHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2ZMbmDWrtL7owtdL66d0qrT+H7sWtKxdoS2l66JeHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2ZN74/aFpfU7ZzxbafvXXLGnZe1XlbaMs8WRHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSmMz87PMlfVfSb0s6JWl1RDxi+wFJfyXpzeKp90dEtUFZ9Nzf3vGjrm5/++jclrUPqfw751GvyVxUc0LSvRHxsu0LJW2xvaGoPRwRD3WvPQB1mcz87COSRor7b9veKWletxsDUK+z+p/d9mWSPi5pc7HoLtvbbK+xPaPFOittD9sePq6jlZoF0LlJh932BZKelnRPRByW9KikBZIWaezIv2qi9SJidUQMRsTgFE2roWUAnZhU2G1P0VjQvxcRP5akiDgYEScj4pSkb0la3L02AVTVNuy2LekxSTsj4hvjlo9/m/UWSTvqbw9AXSbzbvy1kr4gabvtrcWy+yUtt71IUkjaK+mOrnSIrnpk15+W1pcPPl5a/8Q/3F1a//C3d7WsnSxdE3WbzLvxL0ryBCXG1IFzCFfQAUkQdiAJwg4kQdiBJAg7kARhB5JwRPRsZxd5Zizx9T3bH5DN5hjS4Tg00VA5R3YgC8IOJEHYgSQIO5AEYQeSIOxAEoQdSKKn4+y235T083GLLpH0Vs8aODv92lu/9iXRW6fq7O13ImLWRIWehv2MndvDETHYWAMl+rW3fu1LordO9ao3TuOBJAg7kETTYV/d8P7L9Gtv/dqXRG+d6klvjf7PDqB3mj6yA+gRwg4k0UjYbS+1vcv2a7bva6KHVmzvtb3d9lbbww33ssb2qO0d45bNtL3B9u7idsI59hrq7QHbbxSv3VbbNzbU23zbL9jeaftV23cXyxt97Ur66snr1vP/2W0PSPpvSZ+WtF/SS5KWR8R/9bSRFmzvlTQYEY1fgGH7TyQdkfTdiPj9YtnXJR2KiAeLP5QzIuIrfdLbA5KOND2NdzFb0dzx04xLulnSF9Xga1fS1+fVg9etiSP7YkmvRcSeiDgm6QeSljXQR9+LiI2SDp22eJmktcX9tRr7Zem5Fr31hYgYiYiXi/tvS3p3mvFGX7uSvnqiibDPk7Rv3OP96q/53kPS87a32F7ZdDMTmBMRI9LYL4+k2Q33c7q203j30mnTjPfNa9fJ9OdVNRH2ib4fq5/G/66NiD+U9BlJdxanq5icSU3j3SsTTDPeFzqd/ryqJsK+X9L8cY8vlXSggT4mFBEHittRSc+o/6aiPvjuDLrF7WjD/fy/fprGe6JpxtUHr12T0583EfaXJC20fbntqZJuk7S+gT7OYHt68caJbE+XdIP6byrq9ZJWFPdXSFrXYC/v0S/TeLeaZlwNv3aNT38eET3/kXSjxt6Rf13S3zXRQ4u+PiLpleLn1aZ7k/Skxk7rjmvsjOjLki6WNCRpd3E7s496e1zSdknbNBasuQ319sca+9dwm6Stxc+NTb92JX315HXjclkgCa6gA5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk/g+oohPazXrNvwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAMk0lEQVR4nO3dUawcZRnG8ecRSwmVJq3YpiJBLFxITKzmpC2pNBgiIDeFC629kJqQFBJJ0JgowQu4JERovDBgkYZiEDDRhl4QtTkxqRhoOJAKhaq0WLS2OZX0oqCxFHi9OFNzKLszy87Mzva8/1+y2d35dnfebs7TmZ1vvvkcEQIw932k6wIAjAZhB5Ig7EAShB1IgrADSXx0lCs72/PjHC0Y5SqBVP6rf+vtOOFebbXCbvtaST+WdJakn0XE3WWvP0cLtMpX1VklgBK7Y7Jv29C78bbPkvQTSV+VdJmkDbYvG/bzALSrzm/2lZL2R8RrEfG2pMclrWumLABNqxP2CyT9Y9bzQ8Wy97G9yfaU7amTOlFjdQDqqBP2XgcBPnDubURsiYiJiJiYp/k1VgegjjphPyTpwlnPPyXpcL1yALSlTtifk3Sp7Yttny3pG5J2NFMWgKYN3fUWEe/YvlXSbzXT9bY1Il5urDIAjarVzx4RT0l6qqFaALSI02WBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSGKkUzajt/2bV5e2H1j/QGn7ja+vHXrdf3y23lycl3z32Vrvx+iwZQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJOhnnwMeuWjX8G+u815JWl/evPyJW/q20Uc/WrXCbvugpDclvSvpnYiYaKIoAM1rYsv+5Yh4o4HPAdAifrMDSdQNe0j6ne3nbW/q9QLbm2xP2Z46qRM1VwdgWHV349dExGHbSyTttP3niHjfEZ+I2CJpiyQt9OKouT4AQ6q1ZY+Iw8X9UUnbJa1soigAzRs67LYX2D7v1GNJV0va21RhAJpVZzd+qaTttk99zi8i4jeNVIU5o2ws/hW7bi5977nbdzddTmpDhz0iXpP0+QZrAdAiut6AJAg7kARhB5Ig7EAShB1IgiGuY6BqqOdy9R8mWtea1a+UttcaPlvhDz/5aWn7FaJrrkls2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCUeM7uIxC704Vvmqka0P7Vv6zMLS9jr99FVTUU9ffnzoz56rdsekjscx92pjyw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTCeHbVU9XXf+Ez/vvI2x8rjg9iyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAS9LNjbFX1w1+jFSOqZG6o3LLb3mr7qO29s5Yttr3T9qvF/aJ2ywRQ1yC78Q9Luva0ZbdLmoyISyVNFs8BjLHKsEfELknHTlu8TtK24vE2Sdc3XBeAhg17gG5pRByRpOJ+Sb8X2t5ke8r21EmdGHJ1AOpq/Wh8RGyJiImImJin+W2vDkAfw4Z92vYySSrujzZXEoA2DBv2HZI2Fo83SnqymXIAtKWyn932Y5KulHS+7UOS7pR0t6Rf2r5J0t8lfa3NIrP7zw2rStsPr+15mfCBVM3PXqXNMelV142XuG78h1EZ9ojY0KeJ2R6AMwinywJJEHYgCcIOJEHYgSQIO5AEQ1zHwP7Nq0vbD6x/YESVnFmquiTP3b57RJWcGdiyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjoiRrWyhF8cqM1judL89vKfrEuakaz6Z71LTu2NSx+NYzzHPbNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnGs6OWqss9//HZy/q2VV3Guu5lqpc+s7Bv2/Tl+S5DzZYdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgPPsYqLr++cXf31faXtaX3bZLvvtsa59d1k8u1euHX/7ELaXtbf672lRrPLvtrbaP2t47a9ldtv9pe09xu67JggE0b5Dd+IclXdtj+eaIWFHcnmq2LABNqwx7ROySdGwEtQBoUZ0DdLfafrHYzV/U70W2N9mesj11UidqrA5AHcOG/X5JyyWtkHRE0r39XhgRWyJiIiIm5mn+kKsDUNdQYY+I6Yh4NyLek/SgpJXNlgWgaUOF3fayWU9vkLS332sBjIfK8ey2H5N0paTzbR+SdKekK22vkBSSDkq6ucUa57yqecSnt5e//xKdmX3CVSrPH6g53j2byrBHxIYeix9qoRYALeJ0WSAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmDKZnSm6hLaB9Y/MKJKcmDLDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ0M+OWqr6yg+v7Tl7sKT2+9FvfH1t37YzdUrmOtiyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAS9LMPaP/m1X3b1qx+pfS9VVMPd9nnW6efXOp2zPnyJ24pbc/Yl16mcstu+0Lbv7e9z/bLtm8rli+2vdP2q8X9ovbLBTCsQXbj35H0vYj4rKTVkr5t+zJJt0uajIhLJU0WzwGMqcqwR8SRiHihePympH2SLpC0TtK24mXbJF3fVpEA6vtQB+hsf1rSFyTtlrQ0Io5IM/8hSFrS5z2bbE/ZnjqpE/WqBTC0gcNu+2OSfiXpOxFxfND3RcSWiJiIiIl5mj9MjQAaMFDYbc/TTNAfjYhfF4unbS8r2pdJOtpOiQCaUNn1ZtuSHpK0LyLum9W0Q9JGSXcX90+2UuGY+OSu6Nv2yPpd5W++qKJ9/RAFzVI2lPORqnVrT72Vt4iutWYN0s++RtI3Jb1k+9Rfxh2aCfkvbd8k6e+SvtZOiQCaUBn2iHhaUr8zK65qthwAbeF0WSAJwg4kQdiBJAg7kARhB5JwRP/+46Yt9OJY5bl3AL9s+KuUe+rhsnMApi8f+ERMDGh3TOp4HOvZe8aWHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeS4FLSDagaV33FrptL26su11x1qerqMev9lfWDD+Jv93y2tP3c7btrfT6aw5YdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgPDswhzCeHQBhB7Ig7EAShB1IgrADSRB2IAnCDiRRGXbbF9r+ve19tl+2fVux/C7b/7S9p7hd1365AIY1yMUr3pH0vYh4wfZ5kp63vbNo2xwRP2qvPABNGWR+9iOSjhSP37S9T9IFbRcGoFkf6je77U9L+oKkU9cautX2i7a32l7U5z2bbE/ZnjqpE7WKBTC8gcNu+2OSfiXpOxFxXNL9kpZLWqGZLf+9vd4XEVsiYiIiJuZpfgMlAxjGQGG3PU8zQX80In4tSRExHRHvRsR7kh6UtLK9MgHUNcjReEt6SNK+iLhv1vJls152g6S9zZcHoCmDHI1fI+mbkl6yvadYdoekDbZXSApJByWVXy8ZQKcGORr/tKRe42Ofar4cAG3hDDogCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASI52y2fa/JL0+a9H5kt4YWQEfzrjWNq51SdQ2rCZruygiPtGrYaRh/8DK7amImOisgBLjWtu41iVR27BGVRu78UAShB1Iouuwb+l4/WXGtbZxrUuitmGNpLZOf7MDGJ2ut+wARoSwA0l0Enbb19r+i+39tm/vooZ+bB+0/VIxDfVUx7VstX3U9t5Zyxbb3mn71eK+5xx7HdU2FtN4l0wz3ul31/X05yP/zW77LEl/lfQVSYckPSdpQ0S8MtJC+rB9UNJERHR+AobttZLekvRIRHyuWHaPpGMRcXfxH+WiiPjBmNR2l6S3up7Gu5itaNnsacYlXS/pW+rwuyup6+sawffWxZZ9paT9EfFaRLwt6XFJ6zqoY+xFxC5Jx05bvE7StuLxNs38sYxcn9rGQkQciYgXisdvSjo1zXin311JXSPRRdgvkPSPWc8Pabzmew9Jv7P9vO1NXRfTw9KIOCLN/PFIWtJxPaernMZ7lE6bZnxsvrthpj+vq4uw95pKapz6/9ZExBclfVXSt4vdVQxmoGm8R6XHNONjYdjpz+vqIuyHJF046/mnJB3uoI6eIuJwcX9U0naN31TU06dm0C3uj3Zcz/+N0zTevaYZ1xh8d11Of95F2J+TdKnti22fLekbknZ0UMcH2F5QHDiR7QWSrtb4TUW9Q9LG4vFGSU92WMv7jMs03v2mGVfH313n059HxMhvkq7TzBH5A5J+2EUNfer6jKQ/FbeXu65N0mOa2a07qZk9opskfVzSpKRXi/vFY1TbzyW9JOlFzQRrWUe1fUkzPw1flLSnuF3X9XdXUtdIvjdOlwWS4Aw6IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjif1Eo93l3nAzsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAOq0lEQVR4nO3df4xV5Z3H8c8HGNBFVKYsiEjxZ2lprWin2Eaz0RCNsnZRq41k0+JqFjfRXd10u2tsNvqH27Jdf6ztNt3gikXaUpu0FmKtypKmrqs1DgYFllrQoPJjQUu6Il1xGL77x1w2I8557nDPuT/geb+Syb1zvvec883NfObce59z7uOIEIAj34h2NwCgNQg7kAnCDmSCsAOZIOxAJka1cmejPSaO0thW7hLIyrvao/dir4eqlQq77Usk3SdppKR/i4iFqccfpbE617PL7BJAwnOxqrDW8Mt42yMlfVvSpZJmSJpne0aj2wPQXGXes8+StCkiXo2I9yT9UNLcatoCULUyYZ8i6Y1Bv2+pLXsf2wts99ru7dPeErsDUEaZsA/1IcAHzr2NiEUR0RMRPV0aU2J3AMooE/YtkqYO+v0kSdvKtQOgWcqE/XlJZ9g+xfZoSddIWlFNWwCq1vDQW0Tss32TpCc0MPS2OCLWV9YZgEqVGmePiMckPVZRLwCaiNNlgUwQdiAThB3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUyUmsUVaKeN930mWX/5qm83vO1P/+NfJuuTvvlMw9tul1Jht71Z0m5J/ZL2RURPFU0BqF4VR/YLI+KtCrYDoIl4zw5komzYQ9KTtlfbXjDUA2wvsN1ru7dPe0vuDkCjyr6MPy8ittmeKGml7V9HxFODHxARiyQtkqRj3R0l9wegQaWO7BGxrXa7U9IjkmZV0RSA6jUcdttjbY87cF/SxZLWVdUYgGqVeRk/SdIjtg9s5wcR8XglXaEyI8ePT9Zf/euPJusr5t+V3r7S78z65WS9jEkjn03W93Mayfs0/GxExKuSzqqwFwBNxNAbkAnCDmSCsAOZIOxAJgg7kAnGJo4Av7/y3MLaW2eOTK774vX31dn66GR1RJ3jxX7tr7P99ni5rz9ZP/6VvhZ10joc2YFMEHYgE4QdyARhBzJB2IFMEHYgE4QdyATj7IeB/efPTNYfuPeewtq0Uelx8rIuXHt1sn70nccW1l6bc3Ry3bXzv9lQTwese6/48tsb//6W5LrH/exXpfbdiTiyA5kg7EAmCDuQCcIOZIKwA5kg7EAmCDuQCcbZDwNbb9mXrKfG0nf1p6fc+tLGecl6/9cnJutj/311sv77K4qvtf/TP/5lct2y/uzF+YW1yd878sbR6+HIDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJhhn7wD9F56TrC/91L/W2ULxtMjP702Pk4+Y/Ua6rnR95OmnJOsP/fPdhbUTR41JrlvPOf9yc7J+8oOvFNbSZy4cmeoe2W0vtr3T9rpBy7ptr7S9sXabngQcQNsN52X8dyVdctCyWyWtiogzJK2q/Q6gg9UNe0Q8JWnXQYvnSlpSu79E0uUV9wWgYo1+QDcpIrZLUu228I2h7QW2e2339il9njaA5mn6p/ERsSgieiKip0vlPpAB0LhGw77D9mRJqt3urK4lAM3QaNhXSDpw/eB8ScuraQdAs9QdZ7e9TNIFkibY3iLpdkkLJf3I9vWSXpeU/vJwJI3ZlH5h9PPdn0zWP/GhtYW1sSPSn5OMOmFSsr716tOS9S/d8HiyXmYs/dW+9BzpU365J1nf9987Gt73kahu2COi6NsNZlfcC4Am4nRZIBOEHcgEYQcyQdiBTBB2IBNc4toB9r2xJVn/+bYZyfpXEkNv5x/1bnLd5Y8efNnD+503enOyftm44n1L0mdXX1dYe/ZT30uue+XzNyTrH37mxWQd78eRHcgEYQcyQdiBTBB2IBOEHcgEYQcyQdiBTDDOfhg4/i/SX3y8cPlZhbVbJ6THov9p8jMN9XTARx+9JVk/fWnxZapPPHhcct1pX9ufrEeyioNxZAcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBOOaN1o5bHujnPNl9JWbc/jpxbWVp35cFP3PaLO8WLuJy8qrPX/Nn0tPQ7dc7FKb8euIefw5sgOZIKwA5kg7EAmCDuQCcIOZIKwA5kg7EAmuJ79CDDCxedK1BsHL6vLI5P1386ZXlg7fumzVbeDhLp/CbYX295pe92gZXfY3mp7Te1nTnPbBFDWcP7tf1fSJUMsvzciZtZ+Hqu2LQBVqxv2iHhKEuc1Aoe5Mm/obrL9Uu1l/viiB9leYLvXdm+f9pbYHYAyGg37dySdJmmmpO2S7i56YEQsioieiOjp0pgGdwegrIbCHhE7IqI/IvZLul/SrGrbAlC1hsJue/KgX6+QtK7osQA6Q91xdtvLJF0gaYLtLZJul3SB7Zka+OruzZLSE2mjlFEnfzhZn33Cy4W1/Up/9/r1rxVfby5JU47+XbJ+58TVyfrPvnZXYe3Srr9Jrtu9mHH4KtUNe0TMG2LxA03oBUATcboskAnCDmSCsAOZIOxAJgg7kAkuce0Ao049OVk/5eHtyXpqWubLfn1lct2uz7+drG/4wmeT9TtvTw+9jRsxurD2zqXvJNftXpws4xBxZAcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBOMs7fAyNNPSdanLUuPo9994tPJ+rLdUwprXde8m1y3/3f/k6xPWJS+zPRbf3VGsn7j+OLLb9FaHNmBTBB2IBOEHcgEYQcyQdiBTBB2IBOEHcgE4+wt8AcP7k7W7z3xP0pt/xtLryqsTX3zmVLbHjFzRrI+/aiflNo+WocjO5AJwg5kgrADmSDsQCYIO5AJwg5kgrADmWCcvQKjTiq+nlySrppYbqx7y769yfqE9fsa3rZHpf8ENn91ZLJ+8dF7kvXUhNF7t41Nrotq1T2y255q+xe2N9heb/vm2vJu2yttb6zdjm9+uwAaNZyX8fskfTkiPibpM5JutD1D0q2SVkXEGZJW1X4H0KHqhj0itkfEC7X7uyVtkDRF0lxJS2oPWyLp8mY1CaC8Q/qAzvbJks6W9JykSRGxXRr4hyBpYsE6C2z32u7tU/q9J4DmGXbYbR8j6ceSbomI9GyAg0TEoojoiYieLo1ppEcAFRhW2G13aSDo34+IA5c57bA9uVafLGlnc1oEUIW6Q2+2LekBSRsi4p5BpRWS5ktaWLtd3pQODwNbPj8tWb/imJ+W2v7n7v/bZH3qT4uH9vzpM5Pr9n09/VXSaz5Wb97k9PHirP+8rrA2/ba1yXVTw3Y4dMMZZz9P0hclrbW9prbsNg2E/Ee2r5f0uqSrm9MigCrUDXtEPC3JBeXZ1bYDoFk4XRbIBGEHMkHYgUwQdiAThB3IBJe4VmDin7zR1O1/5KJXkvWNZ3+8sPaVjz+RXHfeuK0N9XTAk/+bvkz11AWvF9b696Qvj0W1OLIDmSDsQCYIO5AJwg5kgrADmSDsQCYIO5AJxtkrsOm1Scl63/T+ZL3L6a9rfvj0R9MNnJ7Yd6T3vakvkvVr181P1rsv+02yLqWvl0frcGQHMkHYgUwQdiAThB3IBGEHMkHYgUwQdiATjLNX4CPX9SbrD66fnqwvOH5Tqf2f86trC2vxwnHJdaf+Q3o66W7VG0fH4YIjO5AJwg5kgrADmSDsQCYIO5AJwg5kgrADmXBE+npm21MlPSTpBA1Mmb0oIu6zfYekP5f0Zu2ht0XEY6ltHevuONdM/Ao0y3OxSm/HriFnXR7OSTX7JH05Il6wPU7Satsra7V7I+KuqhoF0DzDmZ99u6Tttfu7bW+QNKXZjQGo1iG9Z7d9sqSzJT1XW3ST7ZdsL7Y9vmCdBbZ7bff2aW+pZgE0bthht32MpB9LuiUi3pb0HUmnSZqpgSP/3UOtFxGLIqInInq6NKaClgE0Ylhht92lgaB/PyJ+IkkRsSMi+iNiv6T7Jc1qXpsAyqobdtuW9ICkDRFxz6Dlkwc97ApJ66pvD0BVhvNp/HmSvihpre01tWW3SZpne6akkLRZ0g1N6RBAJYbzafzTkoYat0uOqQPoLJxBB2SCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZqPtV0pXuzH5T0muDFk2Q9FbLGjg0ndpbp/Yl0VujquxtWkT84VCFlob9Azu3eyOip20NJHRqb53al0RvjWpVb7yMBzJB2IFMtDvsi9q8/5RO7a1T+5LorVEt6a2t79kBtE67j+wAWoSwA5loS9htX2L7ZdubbN/ajh6K2N5se63tNbZ729zLYts7ba8btKzb9krbG2u3Q86x16be7rC9tfbcrbE9p029TbX9C9sbbK+3fXNteVufu0RfLXneWv6e3fZISb+RdJGkLZKelzQvIv6rpY0UsL1ZUk9EtP0EDNt/JOkdSQ9FxCdqy74haVdELKz9oxwfEX/XIb3dIemddk/jXZutaPLgacYlXS7pWrXxuUv09QW14Hlrx5F9lqRNEfFqRLwn6YeS5rahj44XEU9J2nXQ4rmSltTuL9HAH0vLFfTWESJie0S8ULu/W9KBacbb+twl+mqJdoR9iqQ3Bv2+RZ0133tIetL2atsL2t3MECZFxHZp4I9H0sQ293OwutN4t9JB04x3zHPXyPTnZbUj7ENNJdVJ43/nRcQ5ki6VdGPt5SqGZ1jTeLfKENOMd4RGpz8vqx1h3yJp6qDfT5K0rQ19DCkittVud0p6RJ03FfWOAzPo1m53trmf/9dJ03gPNc24OuC5a+f05+0I+/OSzrB9iu3Rkq6RtKINfXyA7bG1D05ke6yki9V5U1GvkDS/dn++pOVt7OV9OmUa76JpxtXm567t059HRMt/JM3RwCfyr0j6ajt6KOjrVEkv1n7Wt7s3Scs08LKuTwOviK6X9CFJqyRtrN12d1BvSyWtlfSSBoI1uU29na+Bt4YvSVpT+5nT7ucu0VdLnjdOlwUywRl0QCYIO5AJwg5kgrADmSDsQCYIO5AJwg5k4v8ACRo+x1Va1OsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAANp0lEQVR4nO3df6zV9X3H8dfL2wtMKobfMqX+KihmjbrdoK22czPtLDbFZnEpXTbWuNEsuNjMLDP2jzqTJWapNXbrXKgSaeN0dtbKEuJK0YYZHeHqEHBsRRltEcalZSnQ8Jv3/rhfmgve8z2X8z3fcw68n4/k5pzzfX/P9/POgdf9nns+55yPI0IAzn3ndbsBAJ1B2IEkCDuQBGEHkiDsQBLv6+Rg4zw+JmhiJ4cEUjmkX+hIHPZotUpht32bpEcl9Ul6PCIeKtt/gibqBt9aZUgAJdbFmoa1lp/G2+6T9HVJn5R0jaRFtq9p9XgA6lXlb/b5kt6OiG0RcUTSM5IWtqctAO1WJewXS/rJiNs7im2nsL3E9qDtwaM6XGE4AFVUCftoLwK85723EbEsIgYiYqBf4ysMB6CKKmHfIWn2iNuXSNpZrR0AdakS9vWS5ti+3PY4SZ+VtLI9bQFot5an3iLimO27Jf2rhqfelkfEW23rDEBbVZpnj4hVkla1qRcANeLtskAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkKi3ZbHu7pP2Sjks6FhED7WgKQPtVCnvhtyLip204DoAa8TQeSKJq2EPS92y/bnvJaDvYXmJ70PbgUR2uOByAVlV9Gn9TROy0PUPSatv/FRFrR+4QEcskLZOkSZ4SFccD0KJKZ/aI2FlcDkl6XtL8djQFoP1aDrvtibYvOHld0ickbW5XYwDaq8rT+JmSnrd98jj/GBEvtqUrdMx5EyaU1vf8/vWl9dce/LuWx75l052l9YkPXlBa96tvtjx2Ri2HPSK2Sbq2jb0AqBFTb0AShB1IgrADSRB2IAnCDiTRjg/CoMv6Jk1qWPO0KaX3nfrU3tL6yg+UT62d0InSepmXPvRPpfV7H725tP7OR8unDU8cOnTGPZ3LOLMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBLMs58F+ubNKa0f/NqRhrUXr/l2xdG7dz54+FdfKa3Pe/ju0vqcpeva2c5ZjzM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTBPHsP6Js2tbR++3P/Xlr/4wu3tbOdU7x8sPwz42sPXF1a//SFbzSsXTuupZZ+ae68HaV1lh86FWd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCefYO6Js+vbS+71vlSxPXOY9+9cql5fV/2F9aP2/o/0rr89e+07B27bifl94X7dX0zG57ue0h25tHbJtie7XtrcXl5HrbBFDVWJ7GPynpttO23SdpTUTMkbSmuA2ghzUNe0SslXT6GkELJa0orq+QdEeb+wLQZq2+QDczInZJUnE5o9GOtpfYHrQ9eFSHWxwOQFW1vxofEcsiYiAiBvo1vu7hADTQath3254lScXlUPtaAlCHVsO+UtLi4vpiSS+0px0AdWk6z277aUm3SJpme4ekL0t6SNKztu+S9GNJd9bZ5NnuFzdeXlpf86Gv1zb2vO+Wf7f6Vfe8Xlo/cexYaX3bX3+4tP475/9Lab2KoW9/oLQ+Xe/WNvbZqGnYI2JRg9Ktbe4FQI14uyyQBGEHkiDsQBKEHUiCsANJ8BHXDhj6w4O1Hv+fD1zUsHb1g/9Tet/jTabWjv32b5TWly+qb9pw8HBfaX3G4IHSOl8lfSrO7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBPPsHXD5tJ/Vevy/+o/bG4+9e2OlY7/vS7tL6wPjj1c6fpnHh36ztB7rN9U29rmIMzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME8+zng+x/5+4a1z3//c6X3/bNLXyqt335+k6+aLq1W84ONV5fW52p9jaOfezizA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASzLN3wA8HLy3fYW6148/sG9+wtmrec9UOXuP5YN6aL5TWr1r6Zmmd74U/M03/JW0vtz1ke/OIbQ/Yftf2huJnQb1tAqhqLL+2n5R02yjbH4mI64qfVe1tC0C7NQ17RKyVtLcDvQCoUZU/yO62vbF4mj+50U62l9getD14VIcrDAegilbD/pikKyVdJ2mXpIcb7RgRyyJiICIG+tX4hSQA9Wop7BGxOyKOR8QJSd+QNL+9bQFot5bCbnvWiJufkbS50b4AekPTeXbbT0u6RdI02zskfVnSLbav0/BU53ZJ5ROmyc19fE9pfdmCD5bW77pwa2m93+XrmFfR7NhHm0x2v3xwQsPa3L89WnrfOHqk/OA4I03DHhGLRtn8RA29AKgRb5cFkiDsQBKEHUiCsANJEHYgCUd07oOCkzwlbvCtHRvvXLHnTz9cWt//0YMNa9/9yGOl9/1gf/mEzHlNzgcnmnyZ9Pyv3NOwdtEjr5beF2duXazRvtjr0Wqc2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCb5K+iww/bHXmtQb1/78xvJPHz/5bOPlniVpWt+vlNYPxbHSev8+vvC5V3BmB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkmGc/x+274vzS+gRX+32/7tCk0vrUJ8rfI4DO4cwOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwz36O+/mV5b/Pzz+vv0OdoNuantltz7b9su0ttt+yfU+xfYrt1ba3FpeT628XQKvG8jT+mKR7I2KepBslLbV9jaT7JK2JiDmS1hS3AfSopmGPiF0R8UZxfb+kLZIulrRQ0opitxWS7qirSQDVndELdLYvk3S9pHWSZkbELmn4F4KkGQ3us8T2oO3BozpcrVsALRtz2G2/X9Jzkr4YEfvGer+IWBYRAxEx0K/xrfQIoA3GFHbb/RoO+lMR8Z1i827bs4r6LElD9bQIoB2aTr3ZtqQnJG2JiK+OKK2UtFjSQ8XlC7V0iKb65l7ZsPa5O1/qYCfoZWOZZ79J0h9I2mR7Q7Htfg2H/Fnbd0n6saQ762kRQDs0DXtEvCJp1MXdJd3a3nYA1IW3ywJJEHYgCcIOJEHYgSQIO5AEH3E9B+z41MyGtb+Y+kytYz/+vx9rssfPah0fY8eZHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYJ4dlWx+8arS+my92qFO0AxndiAJwg4kQdiBJAg7kARhB5Ig7EAShB1Ignn2c8DvLv5Bbcf+/I/Kv0D4sq9tLq0fb2czqIQzO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kMZb12WdL+qakiySdkLQsIh61/YCkP5G0p9j1/ohYVVejaOyScXtrO/a6tfNK61fse622sdFeY3lTzTFJ90bEG7YvkPS67dVF7ZGI+Ep97QFol7Gsz75L0q7i+n7bWyRdXHdjANrrjP5mt32ZpOslrSs23W17o+3ltic3uM8S24O2B4/qcKVmAbRuzGG3/X5Jz0n6YkTsk/SYpCslXafhM//Do90vIpZFxEBEDPRrfBtaBtCKMYXddr+Gg/5URHxHkiJid0Qcj4gTkr4haX59bQKoqmnYbVvSE5K2RMRXR2yfNWK3z0gq//gTgK5yRJTvYN8s6d8kbdLw1Jsk3S9pkYafwoek7ZK+ULyY19AkT4kbXP6RSQCtWxdrtC/2erTaWF6Nf0XSaHdmTh04i/AOOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJNP8/e1sHsPZJ+NGLTNEk/7VgDZ6ZXe+vVviR6a1U7e7s0IqaPVuho2N8zuD0YEQNda6BEr/bWq31J9NaqTvXG03ggCcIOJNHtsC/r8vhlerW3Xu1LordWdaS3rv7NDqBzun1mB9AhhB1Ioitht32b7f+2/bbt+7rRQyO2t9veZHuD7cEu97Lc9pDtzSO2TbG92vbW4nLUNfa61NsDtt8tHrsNthd0qbfZtl+2vcX2W7bvKbZ39bEr6asjj1vH/2a33Sfph5I+LmmHpPWSFkXEf3a0kQZsb5c0EBFdfwOG7Y9JOiDpmxHxa8W2v5G0NyIeKn5RTo6Iv+yR3h6QdKDby3gXqxXNGrnMuKQ7JP2RuvjYlfT1e+rA49aNM/t8SW9HxLaIOCLpGUkLu9BHz4uItZL2nrZ5oaQVxfUVGv7P0nENeusJEbErIt4oru+XdHKZ8a4+diV9dUQ3wn6xpJ+MuL1DvbXee0j6nu3XbS/pdjOjmHlyma3ickaX+zld02W8O+m0ZcZ75rFrZfnzqroR9tGWkuql+b+bIuLXJX1S0tLi6SrGZkzLeHfKKMuM94RWlz+vqhth3yFp9ojbl0ja2YU+RhURO4vLIUnPq/eWot59cgXd4nKoy/38Ui8t4z3aMuPqgceum8ufdyPs6yXNsX257XGSPitpZRf6eA/bE4sXTmR7oqRPqPeWol4paXFxfbGkF7rYyyl6ZRnvRsuMq8uPXdeXP4+Ijv9IWqDhV+TfkfSlbvTQoK8rJL1Z/LzV7d4kPa3hp3VHNfyM6C5JUyWtkbS1uJzSQ719S8NLe2/UcLBmdam3mzX8p+FGSRuKnwXdfuxK+urI48bbZYEkeAcdkARhB5Ig7EAShB1IgrADSRB2IAnCDiTx/15q/zCXm3KyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    x=Y_train.argmax(axis=0).T[i]\n",
    "    print(x)\n",
    "    plt.imshow(X_train.T[i].reshape(int(784**0.5),-1))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialization\n",
    "params = {\"W1\": np.random.randn(opt[\"n_h\"], opt[\"n_x\"]) * np.sqrt(1. / opt[\"n_x\"]),\n",
    "          \"b1\": np.zeros((opt[\"n_h\"], 1)) * np.sqrt(1. / opt[\"n_x\"]),\n",
    "          \"W2\": np.random.randn(digits, opt[\"n_h\"]) * np.sqrt(1. / opt[\"n_h\"]),\n",
    "          \"b2\": np.zeros((digits, 1)) * np.sqrt(1. / opt[\"n_h\"])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    \"\"\"\n",
    "    sigmoid activation function.\n",
    "\n",
    "    inputs: z\n",
    "    \n",
    "    outputs: sigmoid(z)\n",
    "    \"\"\"\n",
    "    s = 1. / (1. + np.exp(-z))\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_loss(Y, Y_hat):\n",
    "    \"\"\"\n",
    "    compute loss function\n",
    "    \"\"\"\n",
    "    L_sum = np.sum(np.multiply(Y, np.log(Y_hat)))\n",
    "    m = Y.shape[1]\n",
    "    L = -(1./m) * L_sum\n",
    "\n",
    "    return L"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feed_forward(X, params):\n",
    "    \"\"\"\n",
    "    feed forward network: 2 - layer neural net\n",
    "\n",
    "    inputs:\n",
    "        params: dictionay a dictionary contains all the weights and biases\n",
    "\n",
    "    return:\n",
    "        cache: dictionay a dictionary contains all the fully connected units and activations\n",
    "    \"\"\"\n",
    "    cache = {}\n",
    "\n",
    "    # Z1 = W1.dot(x) + b1\n",
    "    cache[\"Z1\"] = np.matmul(params[\"W1\"], X) + params[\"b1\"]\n",
    "\n",
    "    # A1 = sigmoid(Z1)\n",
    "    cache[\"A1\"] = sigmoid(cache[\"Z1\"])\n",
    "\n",
    "    # Z2 = W2.dot(A1) + b2\n",
    "    cache[\"Z2\"] = np.matmul(params[\"W2\"], cache[\"A1\"]) + params[\"b2\"]\n",
    "\n",
    "    # A2 = softmax(Z2)\n",
    "    cache[\"A2\"] = np.exp(cache[\"Z2\"]) / np.sum(np.exp(cache[\"Z2\"]), axis=0)\n",
    "\n",
    "    return cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def back_propagate(X, Y, params, cache, m_batch):\n",
    "    \"\"\"\n",
    "    back propagation\n",
    "\n",
    "    inputs:\n",
    "        params: dictionay a dictionary contains all the weights and biases\n",
    "        cache: dictionay a dictionary contains all the fully connected units and activations\n",
    "\n",
    "    return:\n",
    "        grads: dictionay a dictionary contains the gradients of corresponding weights and biases\n",
    "    \"\"\"\n",
    "    # error at last layer\n",
    "    dZ2 = cache[\"A2\"] - Y\n",
    "\n",
    "    # gradients at last layer (Py2 need 1. to transform to float)\n",
    "    dW2 = (1. / m_batch) * np.matmul(dZ2, cache[\"A1\"].T)\n",
    "    db2 = (1. / m_batch) * np.sum(dZ2, axis=1, keepdims=True)\n",
    "\n",
    "    # back propgate through first layer\n",
    "    dA1 = np.matmul(params[\"W2\"].T, dZ2)\n",
    "    dZ1 = dA1 * sigmoid(cache[\"Z1\"]) * (1 - sigmoid(cache[\"Z1\"]))\n",
    "\n",
    "    # gradients at first layer (Py2 need 1. to transform to float)\n",
    "    dW1 = (1. / m_batch) * np.matmul(dZ1, X.T)\n",
    "    db1 = (1. / m_batch) * np.sum(dZ1, axis=1, keepdims=True)\n",
    "\n",
    "    grads = {\"dW1\": dW1, \"db1\": db1, \"dW2\": dW2, \"db2\": db2}\n",
    "\n",
    "    return grads\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: training loss = 0.807676055598168, test loss = 0.7892788000667379\n",
      "Epoch 2: training loss = 0.5239935966997753, test loss = 0.5046638133843891\n",
      "Epoch 3: training loss = 0.47197982873196603, test loss = 0.45495938100228134\n",
      "Epoch 4: training loss = 0.3790309952083612, test loss = 0.3603143194681983\n",
      "Epoch 5: training loss = 0.3597105457873166, test loss = 0.3421829196785463\n",
      "Epoch 6: training loss = 0.34433793872291557, test loss = 0.3262335645987243\n",
      "Epoch 7: training loss = 0.3239133178699333, test loss = 0.31117966306657296\n",
      "Epoch 8: training loss = 0.3014529628139515, test loss = 0.2899977040605068\n",
      "Epoch 9: training loss = 0.29275050287637117, test loss = 0.28063200803169797\n",
      "Epoch 10: training loss = 0.2841156027665997, test loss = 0.2708937259432107\n",
      "Epoch 11: training loss = 0.2784368221453087, test loss = 0.26646471071872974\n",
      "Epoch 12: training loss = 0.27186602894554107, test loss = 0.2626087433105066\n",
      "Epoch 13: training loss = 0.2657858622932657, test loss = 0.26101513440273205\n",
      "Epoch 14: training loss = 0.2755725914492817, test loss = 0.2641143772794899\n",
      "Epoch 15: training loss = 0.2471794901979022, test loss = 0.23739628711672933\n",
      "Epoch 16: training loss = 0.24793470018613503, test loss = 0.24363300751240902\n",
      "Epoch 17: training loss = 0.23991849814526198, test loss = 0.23669172473127761\n",
      "Epoch 18: training loss = 0.22821898967890467, test loss = 0.22524550834102075\n",
      "Epoch 19: training loss = 0.22861066815657685, test loss = 0.22680272539906238\n",
      "Epoch 20: training loss = 0.22200993641146652, test loss = 0.2189067620461367\n",
      "Epoch 21: training loss = 0.21506730234222346, test loss = 0.21187441203685609\n",
      "Epoch 22: training loss = 0.20950006439961097, test loss = 0.20683779609630412\n",
      "Epoch 23: training loss = 0.21174689442414477, test loss = 0.2099352778648622\n",
      "Epoch 24: training loss = 0.20373804604838133, test loss = 0.2039968568987018\n",
      "Epoch 25: training loss = 0.2074453008621499, test loss = 0.20562008743900292\n",
      "Epoch 26: training loss = 0.19833947206686547, test loss = 0.1970858189255695\n",
      "Epoch 27: training loss = 0.19562934145159358, test loss = 0.19806776664749215\n",
      "Epoch 28: training loss = 0.19615034155766425, test loss = 0.19675108181083403\n",
      "Epoch 29: training loss = 0.1897295899661452, test loss = 0.19090048138186086\n",
      "Epoch 30: training loss = 0.18697232211683526, test loss = 0.18832573260759516\n",
      "Epoch 31: training loss = 0.1905586630028625, test loss = 0.19543868016673033\n",
      "Epoch 32: training loss = 0.18468554831224296, test loss = 0.1898627197977667\n",
      "Epoch 33: training loss = 0.1778919909224111, test loss = 0.17861994859519448\n",
      "Epoch 34: training loss = 0.1765057194056431, test loss = 0.1805668177683067\n",
      "Epoch 35: training loss = 0.17196496843836115, test loss = 0.17719124487382454\n",
      "Epoch 36: training loss = 0.17237071682968966, test loss = 0.17890054970200556\n",
      "Epoch 37: training loss = 0.16866178810857857, test loss = 0.17241515934296445\n",
      "Epoch 38: training loss = 0.16681181253583346, test loss = 0.17251274299390945\n",
      "Epoch 39: training loss = 0.16690176454493733, test loss = 0.17160129004325106\n",
      "Epoch 40: training loss = 0.16021932961207397, test loss = 0.16267584992638248\n",
      "Epoch 41: training loss = 0.15723532397294965, test loss = 0.16211578272170793\n",
      "Epoch 42: training loss = 0.15778727437842385, test loss = 0.16196630302720508\n",
      "Epoch 43: training loss = 0.1543644204154151, test loss = 0.16319285557434499\n",
      "Epoch 44: training loss = 0.15178603610047864, test loss = 0.16049779801536004\n",
      "Epoch 45: training loss = 0.15152512040997432, test loss = 0.15717914635222519\n",
      "Epoch 46: training loss = 0.1508716245629926, test loss = 0.15835175973835688\n",
      "Epoch 47: training loss = 0.14595030109909382, test loss = 0.15472298425675854\n",
      "Epoch 48: training loss = 0.14344325439162264, test loss = 0.15253832731969033\n",
      "Epoch 49: training loss = 0.143734019468215, test loss = 0.1541004340865985\n",
      "Epoch 50: training loss = 0.139386355340473, test loss = 0.15079291920700552\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "time_i=time.time()\n",
    "for i in range(opt[\"epochs\"]):\n",
    "\n",
    "    # shuffle training set\n",
    "    permutation = np.random.permutation(X_train.shape[1])\n",
    "    X_train_shuffled = X_train[:, permutation]\n",
    "    Y_train_shuffled = Y_train[:, permutation]\n",
    "\n",
    "    for j in range(opt[\"batch_size\"]):\n",
    "\n",
    "        # get mini-batch\n",
    "        begin = j * opt[\"batch_size\"]\n",
    "        end = min(begin + opt[\"batch_size\"], X_train.shape[1] - 1)\n",
    "        X = X_train_shuffled[:, begin:end]\n",
    "        Y = Y_train_shuffled[:, begin:end]\n",
    "        m_batch = end - begin\n",
    "\n",
    "        # forward and backward\n",
    "        cache = feed_forward(X, params)\n",
    "        grads = back_propagate(X, Y, params, cache, m_batch)\n",
    "\n",
    "        # with momentum (optional)\n",
    "        grads[\"dW1\"] = (opt[\"beta\"] * grads[\"dW1\"] + (1. - opt[\"beta\"]) * grads[\"dW1\"])\n",
    "        grads[\"db1\"] = (opt[\"beta\"] * grads[\"db1\"] + (1. - opt[\"beta\"]) * grads[\"db1\"])\n",
    "        grads[\"dW2\"] = (opt[\"beta\"] * grads[\"dW2\"] + (1. - opt[\"beta\"]) * grads[\"dW2\"])\n",
    "        grads[\"db2\"] = (opt[\"beta\"] * grads[\"db2\"] + (1. - opt[\"beta\"]) * grads[\"db2\"])\n",
    "\n",
    "        # gradient descent\n",
    "        params[\"W1\"] = params[\"W1\"] - opt[\"lr\"] * grads[\"dW1\"]\n",
    "        params[\"b1\"] = params[\"b1\"] - opt[\"lr\"] * grads[\"db1\"]\n",
    "        params[\"W2\"] = params[\"W2\"] - opt[\"lr\"] * grads[\"dW2\"]\n",
    "        params[\"b2\"] = params[\"b2\"] - opt[\"lr\"] * grads[\"db2\"]\n",
    "\n",
    "    # forward pass on training set\n",
    "    cache = feed_forward(X_train, params)\n",
    "    train_loss = compute_loss(Y_train, cache[\"A2\"])\n",
    "\n",
    "    # forward pass on test set\n",
    "    cache = feed_forward(X_test, params)\n",
    "    test_loss = compute_loss(Y_test, cache[\"A2\"])\n",
    "    print(\"Epoch {}: training loss = {}, test loss = {}\".format(\n",
    "        i + 1, train_loss, test_loss))\n",
    "time_f=time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.97       980\n",
      "           1       0.98      0.98      0.98      1135\n",
      "           2       0.96      0.94      0.95      1032\n",
      "           3       0.95      0.96      0.95      1010\n",
      "           4       0.95      0.96      0.96       982\n",
      "           5       0.95      0.93      0.94       892\n",
      "           6       0.95      0.96      0.96       958\n",
      "           7       0.96      0.94      0.95      1028\n",
      "           8       0.95      0.95      0.95       974\n",
      "           9       0.95      0.92      0.94      1009\n",
      "\n",
      "    accuracy                           0.95     10000\n",
      "   macro avg       0.95      0.95      0.95     10000\n",
      "weighted avg       0.96      0.95      0.95     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cache = feed_forward(X_test, params)\n",
    "Y_pred=cache[\"A2\"]\n",
    "print(classification_report(y_pred=Y_pred.argmax(axis=0),y_true=Y_test.argmax(axis=0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time of training 45.26097583770752\n"
     ]
    }
   ],
   "source": [
    "print(\"time of training\",time_f-time_i)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}