Notebook_2_linear_regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow for ICT Applications\n",
    "\n",
    "## Lab session 2: Linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Content of this lab session:**\n",
    "\n",
    "1. Installing TensorFlow 2.0.\n",
    "2. Defining and manipulating tensors (with NumPy interoperability).\n",
    "3. Automatic differentiation in TensorFlow.\n",
    "4. A simple linear regression example."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Installing TensorFlow 2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "^C\n"
     ]
    }
   ],
   "source": [
    "# Need the beta release until official release\n",
    "!pip install tensorflow==2.0.0-beta0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0-beta0'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check version\n",
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This should be true at startup - otherwise, you are running an old version\n",
    "tf.executing_eagerly()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check if a GPU is available on the computer\n",
    "tf.test.is_gpu_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining tensors and operating with NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(2.0, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "x = tf.constant(2.0)\n",
    "print(x) # Check the default type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Casting operation\n",
    "x = tf.cast(x, tf.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(5.0, shape=(), dtype=float64)\n"
     ]
    }
   ],
   "source": [
    "# Full interoperability with NumPy\n",
    "import numpy as np\n",
    "y = np.array(3)\n",
    "c = x + y\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.0\n"
     ]
    }
   ],
   "source": [
    "c = x.numpy() + y # Result is now a NumPy array\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Matrix definition\n",
    "X = tf.random.uniform((4, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=17, shape=(3,), dtype=float32, numpy=array([0.8978696 , 0.82085824, 0.7804252 ], dtype=float32)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Similar indexing as in NumPy\n",
    "X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=20, shape=(), dtype=float32, numpy=7.79669>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Equivalent to X.sum() in NumPy\n",
    "tf.reduce_sum(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Automatic differentiation with TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a simple function working on TF tensors\n",
    "def J(w):\n",
    "    return 2.0 * tf.cos(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    tape.watch(x)\n",
    "    y = J(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Automatic differentiation\n",
    "ygrad = tape.gradient(y, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=38, shape=(), dtype=float64, numpy=-1.8185948536513634>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ygrad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the gradient is numerically correct\n",
    "ygrad.numpy() == -2.0*np.sin(x.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Variables are automatically watched\n",
    "v = tf.Variable(tf.random.uniform((4, 3)))\n",
    "with tf.GradientTape() as tape:\n",
    "    y = J(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[-1.1009622  -0.79421335 -0.07293252]\n",
      " [-0.7007517  -1.2572982  -0.34718406]\n",
      " [-0.8851586  -0.80971026 -1.4044869 ]\n",
      " [-0.28858966 -0.8795029  -0.91865987]], shape=(4, 3), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "ygrad = tape.gradient(y, v)\n",
    "print(ygrad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's try with two variables\n",
    "v1 = v\n",
    "v2 = tf.Variable(3.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Numerical functions can also have if/while control loops\n",
    "is_cos = True\n",
    "with tf.GradientTape() as tape:\n",
    "  y = J(v1)\n",
    "  if is_cos:\n",
    "    y = y * tf.cos(v2)\n",
    "  else:\n",
    "    y = y * tf.sin(v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = tape.gradient(y, [v1, v2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: id=119, shape=(4, 3), dtype=float32, numpy=\n",
       " array([[1.0899442 , 0.78626525, 0.07220265],\n",
       "        [0.69373894, 1.2447158 , 0.34370962],\n",
       "        [0.8763004 , 0.8016071 , 1.3904315 ],\n",
       "        [0.2857016 , 0.87070125, 0.9094664 ]], dtype=float32)>,\n",
       " <tf.Tensor: id=116, shape=(), dtype=float32, numpy=-3.0340908>]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Higher-order gradients\n",
    "with tf.GradientTape() as outer_tape:\n",
    "    with tf.GradientTape() as inner_tape:\n",
    "        y = J(v)\n",
    "    ygrad = inner_tape.gradient(y, v)\n",
    "ygradgrad = outer_tape.gradient(ygrad, v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.6696954, -1.835545 , -1.9986697],\n",
       "       [-1.8732183, -1.5553782, -1.9696352],\n",
       "       [-1.7934587, -1.8287617, -1.4238738],\n",
       "       [-1.9790695, -1.7962391, -1.7765316]], dtype=float32)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ygradgrad.numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear regression with automatic differentiation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the data\n",
    "import numpy as np\n",
    "N = 25\n",
    "# y = wx + b + epsilon\n",
    "w = -0.3\n",
    "b = 0.5\n",
    "X = np.random.rand(N)\n",
    "y = X * w + b + np.random.randn(N)*0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAE7RJREFUeJzt3X+s3XV9x/Hn2wvoXUTrbN3obWvZVhoZLHaeoQvJxABr4Q/aIHNgiLAQmrjhP2oTiIYZzAKzWdyWsGmjRCVRBGW10bqbDTAaA6aXdMKKqXb4g3trRlXKP1yldO/9cU/h9vb0/jrf+/1+7/k8H0nD+X7Ph/P53E++5/X9ns/3xycyE0lSWV7VdAMkSfUz/CWpQIa/JBXI8JekAhn+klQgw1+SCmT4S1KBDH9JKpDhL0kFOqPpBpzOypUrc/369U03Q5KWlccff/wXmblqrnKtDf/169czNjbWdDMkaVmJiJ/Op5zDPpJUIMNfkgpk+EtSgQx/SSqQ4S9JBTL8JalAhr8kFai11/kPqt37J9g5epDDRydZvWKYHZs3sm3TSNPNklQYw79Gu/dPcNuDTzJ57DgAE0cnue3BJwHcAUiqlcM+Ndo5evDl4D9h8thxdo4ebKhFkkpl+Nfo8NHJBa2XpKVi+Ndo9YrhBa2XpKVi+Ndox+aNDJ85dNK64TOH2LF5Y0MtklQqT/jW6MRJXa/2kdQ0w79m2zaNGPaSGuewjyQVyPCXpAJVEv4RcU9EPBsR/32a9yMi/jkiDkXEExHxx1XUK0lanKqO/D8HbJnl/SuADd1/24F/raheSdIiVBL+mflt4FezFNkKfCGnPAasiIhzqqhbkrRwdY35jwDPTFse766TJDWgrvCPHuvylEIR2yNiLCLGjhw5UkOzJKlMdYX/OLB22vIa4PDMQpm5KzM7mdlZtWpVTU2TpPLUFf57gPd1r/p5B/B8Zv68prolSTNUcodvRHwJuARYGRHjwN8CZwJk5qeAvcCVwCHgBeCvqqhXkrQ4lYR/Zl43x/sJ/E0VdUmS+ucdvpJUIMNfkgpk+EtSgQx/SSqQ4S9JBTL8JalAhr8kFcjwl6QCGf6SVCDDX5IKZPhLUoEMf0kqkOEvSQUy/CWpQIa/JBXI8JekAhn+klQgw1+SCmT4S1KBDH9JKpDhL0kFMvwlqUCGvyQVqJLwj4gtEXEwIg5FxK093l8XEY9ExP6IeCIirqyiXknS4vQd/hExBNwNXAGcD1wXEefPKPZR4P7M3ARcC/xLv/VKkhaviiP/i4BDmfl0Zr4I3AdsnVEmgdd1X78eOFxBvZKkRaoi/EeAZ6Ytj3fXTfcx4PqIGAf2Ah/o9UERsT0ixiJi7MiRIxU0TZLUSxXhHz3W5Yzl64DPZeYa4Erg3og4pe7M3JWZnczsrFq1qoKmSZJ6qSL8x4G105bXcOqwzk3A/QCZ+SjwGmBlBXVLkhahivDfB2yIiHMj4iymTujumVHmZ8ClABHxFqbC33EdSWpI3+GfmS8BtwCjwA+YuqrnQETcERFXdYt9CLg5Ir4PfAm4MTNnDg1JkmpyRhUfkpl7mTqRO33d7dNePwVcXEVdkqT+eYevJBXI8JekAhn+klQgw1+SCmT4S1KBDH9JKpDhL0kFMvwlqUCV3OSl8uzeP8HO0YMcPjrJ6hXD7Ni8kW2bZj7MVVJbGf5asN37J7jtwSeZPHYcgImjk9z24JMA7gCkZcJhHy3YztGDLwf/CZPHjrNz9GBDLZK0UIa/Fuzw0ckFrZfUPoa/Fmz1iuEFrZfUPoa/FmzH5o0Mnzl00rrhM4fYsXljQy2StFCe8NWCnTip69U+0vJl+GtRtm0aMeylZcxhH0kqkOEvSQUy/CWpQIa/JBXI8JekAhn+klSgSsI/IrZExMGIOBQRt56mzHsi4qmIOBARX6yiXknS4vR9nX9EDAF3A5cD48C+iNiTmU9NK7MBuA24ODOfi4g39VuvJGnxqjjyvwg4lJlPZ+aLwH3A1hllbgbuzsznADLz2QrqlSQtUhV3+I4Az0xbHgfePqPMeQAR8V1gCPhYZv77zA+KiO3AdoB169ZV0DT1y0lbpMFURfhHj3XZo54NwCXAGuA7EXFBZh496X/K3AXsAuh0OjM/QzVz0hZpcFUx7DMOrJ22vAY43KPM1zLzWGb+GDjI1M5ALeakLdLgqiL89wEbIuLciDgLuBbYM6PMbuBdABGxkqlhoKcrqFtLyElbpMHV97BPZr4UEbcAo0yN59+TmQci4g5gLDP3dN/784h4CjgO7MjMX/Zb90I5fr0wq1cMM9Ej6J20RVr+IrOdQ+udTifHxsYq+7yZ49cwNQHJnVdf6A7gNOwzafmJiMczszNXuWLu8HX8euG2bRrhzqsvZGTFMAGMrBg2+KUBUcxkLo5fL46TtkiDqZgjfycdl6RXFBP+TjouSa8oZtjHSccl6RXFhD84fi1JJxQz7CNJeoXhL0kFMvwlqUCGvyQVyPCXpAIZ/pJUIMNfkgo00Nf5+whnSeptYMPfKQjL485emr+BDf/ZHuG8kEAwUE7Vxj5xZy8tzMCGfxWPcDZQTtXWPqlqZ1+FNu4cF2oQ/gbNbmBP+FbxCGcngDlVW/ukLfM1nNg5ThydJHll57h7/0St7ejHIPwNmtvAhn8Vj3BuS6C0SVv7pC3zNbR157gQg/A3aG4DG/5VTEHYlkBpk7b2SVvma2jrznEhBuFv0NwGdswf+n+E847NG3tOYF7yBDBt7ZO2zNewesUwEz1Csumd40IMwt+guQ10+PerLYHSJm3ukzbM19DWneNCDMLfoLlFZvb/IRFbgH8ChoDPZOZdpyl3DfAA8CeZOTbbZ3Y6nRwbm7WI1EqDcKXMIPwNpYqIxzOzM2e5fsM/IoaAHwKXA+PAPuC6zHxqRrmzgW8AZwG3GP6SVL35hn8VJ3wvAg5l5tOZ+SJwH7C1R7mPA58Afl1BnZKkPlQx5j8CPDNteRx4+/QCEbEJWJuZX4+ID1dQZ2X8eavlym1X/agi/KPHupfHkiLiVcAngRvn/KCI7cB2gHXr1lXQtNm19W5VaS5uu+pXFcM+48DaactrgMPTls8GLgC+FRE/Ad4B7ImIU8akMnNXZnYys7Nq1aoKmjY7b2bRcuW2q35VceS/D9gQEecCE8C1wHtPvJmZzwMrTyxHxLeAD891wrcO3syi5cptd3A0NXzX95F/Zr4E3AKMAj8A7s/MAxFxR0Rc1e/nL6W23q0qzcVtdzA0+RylSh7vkJl7M/O8zPz9zPy77rrbM3NPj7KXtOGoH9rzSABpodx2B0OTw3dF3+Hb5rtVpdm47Q6GJofvig5/aMcjAaTFcNtd/pp8jtLAPtVTkmaze/8EF9/1MOfe+g0uvuvhRuYraHL4zvCXVJy2TFizbdMI737bCEMxdbvUUATvfls9v+gMf0mtUdfReFvuk9i9f4KvPj7B8e4z1o5n8tXHJ5bP1T6S1K86j8bbcp9Ekzshw19SK9QZhG25T6LJnZDhL6kV6gzCttwn0eROyPCX1Ap1BmEVc3xXocmdUPHX+Utqh7qnj2zDfRJN3qxn+EtqhVLvWm5qJ2T4S2qNNhyNl8Ixf0kqkOEvSQUy/CWpQIa/JBXI8JekAhn+klQgL/Vcxpqa+FnS8mf4L1MnnoB44m7IE09ABNwBaFnxIKYZhv8yNdsTEGd+cfxyqa08iGmOY/7L1HyfgNiWGYukXtoyqUqJKgn/iNgSEQcj4lBE3Nrj/Q9GxFMR8UREPBQRb66i3pLN9wmIfrm0WHXMqtWWSVVK1Hf4R8QQcDdwBXA+cF1EnD+j2H6gk5l/BHwF+ES/9ZZuvo+C9culxajrF2NbJlUpURVH/hcBhzLz6cx8EbgP2Dq9QGY+kpkvdBcfA9ZUUG/R5vs8cr9cWoy6fjG2ZVKVElVxwncEeGba8jjw9lnK3wR8s4J6izefJyDW/Yx0DYa6fjGW+hjnNlyEUUX4R4912bNgxPVAB3jnad7fDmwHWLduXQVNU6lfLvVn9YphJnoE/VLNqlXS9tiWK5yqCP9xYO205TXA4ZmFIuIy4CPAOzPzN70+KDN3AbsAOp1Ozx2IFq60L5f65y/GpbOQy7SXUhXhvw/YEBHnAhPAtcB7pxeIiE3Ap4EtmflsBXVKWkIl/WKsewimLRdh9B3+mflSRNwCjAJDwD2ZeSAi7gDGMnMPsBN4LfBARAD8LDOv6rduSUunhF+MTQzB1DmkNptK7vDNzL3A3hnrbp/2+rIq6pGkKjUxBNOWITUf7yAVrg1XnjSliSGYtgypGf5Swdpy5UlTmhqCacOQms/2kQpW+uM/Sr7JzCN/qWBtufKkKW0ZgmmC4S8VrC1XnjSpDUMwTXDYRypYycMepfPIXypYycMepTP8pcKVOuxROod9JKlAhr8kFcjwl6QCGf6SVCDDX5IKZPhLUoEMf0kqkOEvSQUy/CWpQIa/JBXI8JekAhn+klQgw1+SCuRTPaXClTyBe8kqOfKPiC0RcTAiDkXErT3ef3VEfLn7/vciYn0V9Urqz4kJ3CeOTpK8MoH77v0TTTdNS6zv8I+IIeBu4ArgfOC6iDh/RrGbgOcy8w+ATwJ/32+9kvpX+gTuJaviyP8i4FBmPp2ZLwL3AVtnlNkKfL77+ivApRERFdQtqQ+lT+BesirCfwR4ZtryeHddzzKZ+RLwPPDGCuqW1IfTTdRe0gTupaoi/HsdweciyhAR2yNiLCLGjhw5UkHTJM3GCdzLVUX4jwNrpy2vAQ6frkxEnAG8HvjVzA/KzF2Z2cnMzqpVqypomqTZbNs0wp1XX8jIimECGFkxzJ1XX+jVPgWo4lLPfcCGiDgXmACuBd47o8we4AbgUeAa4OHMPOXIX1L9nMC9TH2Hf2a+FBG3AKPAEHBPZh6IiDuAsczcA3wWuDciDjF1xH9tv/VKkhavkpu8MnMvsHfGutunvf418BdV1CVJ6p+Pd5CkAhn+klQgw1+SCmT4S1KBDH9JKpDhL0kFMvwlqUCGvyQVyPCXpAIZ/pJUIMNfkgpk+EtSgQx/SSqQ4S9JBTL8JalAhr8kFcjwl6QCGf6SVCDDX5IKVMkcvtJC7N4/wc7Rgxw+OsnqFcPs2LyRbZtGmm6WVBTDX7XavX+C2x58ksljxwGYODrJbQ8+CeAOQKqRwz6q1c7Rgy8H/wmTx46zc/RgQy2SytRX+EfEb0fEf0TEj7r/fUOPMm+NiEcj4kBEPBERf9lPnVreDh+dXNB6SUuj3yP/W4GHMnMD8FB3eaYXgPdl5h8CW4B/jIgVfdarZWr1iuEFrZe0NPoN/63A57uvPw9sm1kgM3+YmT/qvj4MPAus6rNeLVM7Nm9k+Myhk9YNnznEjs0bG2qRVKZ+T/j+Tmb+HCAzfx4Rb5qtcERcBJwF/E+f9WqZOnFS16t9pGbNGf4R8Z/A7/Z46yMLqSgizgHuBW7IzP87TZntwHaAdevWLeTjtYxs2zRi2EsNmzP8M/Oy070XEf8bEed0j/rPYWpIp1e51wHfAD6amY/NUtcuYBdAp9PJudomSVqcfsf89wA3dF/fAHxtZoGIOAv4N+ALmflAn/VJkirQb/jfBVweET8CLu8uExGdiPhMt8x7gD8DboyI/+r+e2uf9UqS+hCZ7Rxd6XQ6OTY21nQzJGlZiYjHM7MzVznv8JWkAhn+klQgw1+SCtTaMf+IOAL8dJYiK4Ff1NSc5cD+OJn9cSr75GSD2h9vzsw5n6LQ2vCfS0SMzeekRinsj5PZH6eyT05Wen847CNJBTL8JalAyzn8dzXdgJaxP05mf5zKPjlZ0f2xbMf8JUmLt5yP/CVJi9T68I+ILRFxMCIORcQpM4VFxKsj4svd978XEevrb2V95tEfH4yIp7pTZj4UEW9uop11mas/ppW7JiIyIgb66o759EdEvKe7jRyIiC/W3ca6zeM7sy4iHomI/d3vzZVNtLN2mdnaf8AQUxO//B5Tk8B8Hzh/Rpm/Bj7VfX0t8OWm291wf7wL+K3u6/eX3h/dcmcD3wYeAzpNt7vh7WMDsB94Q3f5TU23uwV9sgt4f/f1+cBPmm53Hf/afuR/EXAoM5/OzBeB+5iaOnK66VNJfgW4NCKixjbWac7+yMxHMvOF7uJjwJqa21in+WwfAB8HPgH8us7GNWA+/XEzcHdmPgeQmT3n4Bgg8+mTBF7Xff164HCN7WtM28N/BHhm2vJ4d13PMpn5EvA88MZaWle/+fTHdDcB31zSFjVrzv6IiE3A2sz8ep0Na8h8to/zgPMi4rsR8VhEbKmtdc2YT598DLg+IsaBvcAH6mlas/qdw3ep9TqCn3l50nzKDIp5/60RcT3QAd65pC1q1qz9ERGvAj4J3FhXgxo2n+3jDKaGfi5h6lfhdyLigsw8usRta8p8+uQ64HOZ+Q8R8afAvd0+6Tnd7KBo+5H/OLB22vIaTv1J9nKZiDiDqZ9tv6qldfWbT38QEZcxNcfyVZn5m5ra1oS5+uNs4ALgWxHxE+AdwJ4BPuk73+/L1zLzWGb+GDjI1M5gUM2nT24C7gfIzEeB1zD13J+B1vbw3wdsiIhzu9NBXsvU1JHTTZ9K8hrg4eyeuRlAc/ZHd5jj00wF/6CP587aH5n5fGauzMz1mbmeqXMgV2XmoM4SNJ/vy26mLgogIlYyNQz0dK2trNd8+uRnwKUAEfEWpsL/SK2tbECrw787hn8LMAr8ALg/Mw9ExB0RcVW32GeBN0bEIeCDwGkv91vu5tkfO4HXAg90p8ycuaEPjHn2RzHm2R+jwC8j4ingEWBHZv6ymRYvvXn2yYeAmyPi+8CXgBsH+ADyZd7hK0kFavWRvyRpaRj+klQgw1+SCmT4S1KBDH9JKpDhL0kFMvwlqUCGvyQV6P8BnbrouW7ZCjEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Some initial plot\n",
    "import matplotlib.pyplot as plt\n",
    "plt.figure()\n",
    "plt.scatter(X, y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0-beta0'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__ # Should be '2.0.0-beta0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cast the data to TF tensors\n",
    "X = tf.constant(X)\n",
    "y = tf.constant(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cast the data to tf.float32 for compatibility with default in tf.Variable\n",
    "X = tf.cast(X, tf.float32)\n",
    "y = tf.cast(y, tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Random initial guess for our linear regression parameters\n",
    "west = tf.Variable(0.05)\n",
    "best = tf.Variable(0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear(xb):\n",
    "    # Output of the linear model\n",
    "    return xb * west + best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=171, shape=(25,), dtype=float32, numpy=\n",
       "array([0.00410321, 0.03124942, 0.0022194 , 0.02474734, 0.00160496,\n",
       "       0.00289566, 0.0359645 , 0.0113263 , 0.04059012, 0.01985775,\n",
       "       0.04526726, 0.03207052, 0.01434694, 0.016639  , 0.01427351,\n",
       "       0.04524369, 0.02859161, 0.00987695, 0.04489909, 0.03126319,\n",
       "       0.03199722, 0.00705291, 0.01356084, 0.03629699, 0.04029265],\n",
       "      dtype=float32)>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check it is returning predictions\n",
    "linear(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We will use this to save loss evolution and w's gradient throughout iterations\n",
    "all_losses = []\n",
    "all_grads = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0610 11:14:54.498608 15036 deprecation.py:323] From C:\\Users\\Simone\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1205: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "for i in range(1000):\n",
    "    \n",
    "    with tf.GradientTape() as tape:\n",
    "    \n",
    "        # Compute the output of the model\n",
    "        ypred = linear(X)\n",
    "    \n",
    "        # Compute the error of the model\n",
    "        l = tf.reduce_mean((ypred - y)**2)\n",
    "        all_losses.append(l.numpy())\n",
    "    \n",
    "    # Compute the gradients of the error\n",
    "    g = tape.gradient(l, [west, best])\n",
    "    all_grads.append(g[0].numpy())\n",
    "    \n",
    "    # Optimize the parameters using the gradients\n",
    "    west.assign_add(- 0.01 * g[0])\n",
    "    best.assign_add(- 0.01 * g[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x20d171c4fd0>]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHo9JREFUeJzt3XuQXGd95vHvr+/Tcx9prLuQbMs2MsbGSAJzc4pwsanFTig7yE42NsuW2QLvsrvZTZnKFmSd3UqyZLkVrsQOYAKs4ziEgANKFMcQbgVGspEvstDFli2NriONRnO/9Mxv/zinRz09PTM90kitOef5VHV1n/e83fOeOdLzvvOet7vN3RERkXhI1LoBIiJy4Sj0RURiRKEvIhIjCn0RkRhR6IuIxIhCX0QkRhT6IiIxotAXEYkRhb6ISIykat2AcosXL/Y1a9bUuhkiIgvK008/fcLd22erd9GF/po1a9i+fXutmyEisqCY2avV1NP0johIjCj0RURiRKEvIhIjCn0RkRhR6IuIxIhCX0QkRhT6IiIxEpnQ7xsu8Jkn9rDjYHetmyIictGKTOiPFMb5wpN72XHgVK2bIiJy0YpM6OczSQAGRsdq3BIRkYtXZEI/mwoOZWhEoS8iMp3IhL6ZUZdOMqiRvojItCIT+hBM8QxopC8iMq1IhX5OI30RkRlFKvTzmSSDGumLiEwrUqFfl9FIX0RkJpEK/VxaI30RkZlEKvTzGumLiMyoqtA3s5vMbLeZ7TOz+yrsf4eZPWNmBTO7rWzfajP7ZzPbZWYvmtma+Wn6VHUa6YuIzGjW0DezJPAAcDOwHrjDzNaXVTsA3A08UuElvgZ82t1fC2wCjp9Lg2eidfoiIjOr5ovRNwH73P1lADN7FLgVeLFYwd1fCfeNlz4x7BxS7v5EWK9vfppdWZ1W74iIzKia6Z0VwMGS7Y6wrBpXAN1m9i0z+6WZfTr8y+G80EhfRGRm1YS+VSjzKl8/Bbwd+G/ARuBSgmmgyT/A7B4z225m2zs7O6t86amKSzbdq22eiEi8VBP6HcCqku2VwOEqX78D+KW7v+zuBeDbwPXlldz9IXff4O4b2tvbq3zpqeoySdxhuDA+e2URkRiqJvS3AevMbK2ZZYDNwONVvv42oNXMikn+TkquBcy3unQwc6R5fRGRymYN/XCEfi+wFdgFPObuO83sfjO7BcDMNppZB3A78KCZ7QyfO0YwtfOkmT1PMFX0l+fnUEpCX/P6IiIVVbN6B3ffAmwpK/tkyeNtBNM+lZ77BPD6c2hj1eqKX6Sikb6ISEWRekducaQ/pJG+iEhF0Qr9jKZ3RERmEqnQz2t6R0RkRpEK/ZxW74iIzChSoX9m9U6hxi0REbk4RSr085lgMdLgiN6cJSJSSaRCX+v0RURmFq3QL67eGdH0johIJZEK/XTSSCZMI30RkWlEKvTNLPz2LM3pi4hUEqnQh+LHK2t6R0SkkuiFfjqpN2eJiEwjcqFfn03RP6zQFxGpJHqhn0nSP6zpHRGRSqIX+tkUA1qyKSJSUQRDP0mfRvoiIhVFL/QzKV3IFRGZRvRCP5vSSF9EZBoRDP3gQq6717opIiIXnQiGfopxh+GC3pUrIlIueqEffryypnhERKaKXuhng9Af0Bu0RESmiFzoN2SDj1fWSF9EZKrIhX7x27P69QYtEZEpqgp9M7vJzHab2T4zu6/C/neY2TNmVjCz2yrsbzKzQ2b2xflo9EyK0zv6KAYRkalmDX0zSwIPADcD64E7zGx9WbUDwN3AI9O8zB8BPzz7ZlavPpze0YeuiYhMVc1IfxOwz91fdvcR4FHg1tIK7v6Kuz8HTFknaWZvBJYA/zwP7Z1VcfWORvoiIlNVE/orgIMl2x1h2azMLAH8X+C/z1LvHjPbbmbbOzs7q3npaTVkNacvIjKdakLfKpRV+3bXjwJb3P3gTJXc/SF33+DuG9rb26t86cryE9M7Cn0RkXKpKup0AKtKtlcCh6t8/RuAt5vZR4EGIGNmfe4+5WLwfMmmkqSTRr8+dE1EZIpqQn8bsM7M1gKHgM3AndW8uLv/dvGxmd0NbDifgV+Uz6Q00hcRqWDW6R13LwD3AluBXcBj7r7TzO43s1sAzGyjmXUAtwMPmtnO89no2TTokzZFRCqqZqSPu28BtpSVfbLk8TaCaZ+ZXuOrwFfn3MKzUJ9N6mMYREQqiNw7ciGc3tHqHRGRKSIZ+g1ZzemLiFQSydDPZ5J6R66ISAWRDP2GnC7kiohUEsnQb8ql6RkarXUzREQuOpEM/cZwpD8+ru/JFREpFcnQb8qlcYc+reAREZkkkqHfmAveftA7pNAXESkV0dBPA9CreX0RkUkiGfpNdcFIv2dQI30RkVKRDH2N9EVEKoto6GtOX0SkkkiGflM40tdafRGRySIZ+hrpi4hUFsnQz6WTZJIJegY10hcRKRXJ0IdgBU+PRvoiIpNENvQbc2mt3hERKRPh0NdIX0SkXGRDv0kjfRGRKSIb+o25lFbviIiUiXToa/WOiMhkkQ39YHpHI30RkVKRDf3GXJrB0TFGx8Zr3RQRkYtGVaFvZjeZ2W4z22dm91XY/w4ze8bMCmZ2W0n5dWb2MzPbaWbPmdkH57PxMznzSZua4hERKZo19M0sCTwA3AysB+4ws/Vl1Q4AdwOPlJUPAL/r7lcDNwGfM7OWc210NVrywefvdCv0RUQmpKqoswnY5+4vA5jZo8CtwIvFCu7+Srhv0lyKu+8peXzYzI4D7UD3Obd8Fi35DADdAwp9EZGiaqZ3VgAHS7Y7wrI5MbNNQAZ4aa7PPRutE6E/ciF+nIjIglBN6FuFMp/LDzGzZcDXgQ+5+5Qrq2Z2j5ltN7PtnZ2dc3npabXUhdM7GumLiEyoJvQ7gFUl2yuBw9X+ADNrAr4H/A93/3mlOu7+kLtvcPcN7e3t1b70jIoj/VMa6YuITKgm9LcB68xsrZllgM3A49W8eFj/74Gvufvfnn0z564xlyJhGumLiJSaNfTdvQDcC2wFdgGPuftOM7vfzG4BMLONZtYB3A48aGY7w6f/FvAO4G4z2xHerjsvR1ImkTCa69J0D2qkLyJSVM3qHdx9C7ClrOyTJY+3EUz7lD/vG8A3zrGNZ601n+GURvoiIhMi+45cCNbqa/WOiMgZkQ791nxGc/oiIiUiHfrN+bRCX0SkRKRDPxjpa3pHRKQo4qGfpn9kjJGCPmlTRAQiHvrN+igGEZFJIh36rfqkTRGRSSIe+uFHMfRrpC8iAnEJfU3viIgAEQ/9xQ1B6Hf2KfRFRCDiod9Wn8EMTvQO17opIiIXhUiHfiqZoDWf4WS/Ql9EBCIe+gCL6jOc6NX0jogIxCD0FzdkOdGnkb6ICMQh9BsV+iIiRZEP/UX1GU5q9Y6ICBCD0G9vzNI7XGBodKzWTRERqbnIh35xrb6meEREYhD6i+qzAJriEREhBqG/uDEIfY30RUTiEPqa3hERmRCD0C+O9DW9IyIS+dDPpZM05VIc7xmqdVNERGquqtA3s5vMbLeZ7TOz+yrsf4eZPWNmBTO7rWzfXWa2N7zdNV8Nn4ulzTmOnFboi4jMGvpmlgQeAG4G1gN3mNn6smoHgLuBR8qe2wZ8CngTsAn4lJm1nnuz52Zpcx1HNdIXEalqpL8J2OfuL7v7CPAocGtpBXd/xd2fA8q/gfy9wBPu3uXup4AngJvmod1zsrQpy1GN9EVEqgr9FcDBku2OsKwa5/LcebO0uY7OvmFGx8r7JBGReKkm9K1CmVf5+lU918zuMbPtZra9s7Ozypeu3tKmHO7QqS9TEZGYqyb0O4BVJdsrgcNVvn5Vz3X3h9x9g7tvaG9vr/Klq7esOQegi7kiEnvVhP42YJ2ZrTWzDLAZeLzK198KvMfMWsMLuO8Jyy6opWHoa15fROJu1tB39wJwL0FY7wIec/edZna/md0CYGYbzawDuB140Mx2hs/tAv6IoOPYBtwfll1QS5vC0NcKHhGJuVQ1ldx9C7ClrOyTJY+3EUzdVHruV4CvnEMbz1lLPk02leDo6cFaNkNEpOYi/45cADPTG7RERIhJ6ENwMfdwt0b6IhJvsQn9Va15Dp5S6ItIvMUm9Fe35ensHWZwRF+bKCLxFZvQX9WWB6Dj1ECNWyIiUjuxC/2DCn0RibEYhX4dAAe7NK8vIvEVm9Bvb8iSSyc40KWRvojEV2xC38yCFTwKfRGJsdiEPgQreDTSF5E4i1Xor2oLRvru1X4ytIhItMQq9C9rr6d/ZIxjPfpcfRGJp5iFfgMA+4731bglIiK1EavQv/ySIPRf6lToi0g8xSr02xuzNOZSGumLSGzFKvTNjMvaGzTSF5HYilXoQzDFo5G+iMRV7EL/svYGjvcO0zM0WuumiIhccLEL/SuWBBdz9x7rrXFLREQuvNiF/vrlTQDsPNxT45aIiFx4sQv9pU052uozvHDodK2bIiJywcUu9M2Mq5c3aaQvIrEUu9CHYIpnz7FeRgrjtW6KiMgFFcvQv3p5M6Njzt7jupgrIvFSVeib2U1mttvM9pnZfRX2Z83sb8L9T5nZmrA8bWZ/ZWbPm9kuM/vE/Db/7FwdXszVvL6IxM2soW9mSeAB4GZgPXCHma0vq/Zh4JS7Xw58FvjTsPx2IOvu1wBvBD5S7BBqae2ielryaZ5+9VStmyIickFVM9LfBOxz95fdfQR4FLi1rM6twF+Fj78J/LqZGeBAvZmlgDpgBKj5FdREwnjj6la2K/RFJGaqCf0VwMGS7Y6wrGIddy8Ap4FFBB1AP3AEOAD8mbt3nWOb58Ub17Tycmc/Xf0jtW6KiMgFU03oW4Wy8q+emq7OJmAMWA6sBX7PzC6d8gPM7jGz7Wa2vbOzs4omnbuNa9oANMUjIrFSTeh3AKtKtlcCh6erE07lNANdwJ3AP7n7qLsfB34KbCj/Ae7+kLtvcPcN7e3tcz+Ks3DNimYyyQTbXrko/vAQEbkgqgn9bcA6M1trZhlgM/B4WZ3HgbvCx7cB3/fgi2gPAO+0QD3wZuBX89P0c5NLJ7ludQs/2Xui1k0REblgZg39cI7+XmArsAt4zN13mtn9ZnZLWO3LwCIz2wf8V6C4rPMBoAF4gaDzeNjdn5vnYzhrN17RzotHeujs1Xfmikg8pKqp5O5bgC1lZZ8seTxEsDyz/Hl9lcovFjde0c6nt+7mx3s7+cD1K2vdHBGR8y6W78gtWr+siUX1GX6058JcPBYRqbVYh34iYdx4ZTs/2N3J6Jg+h0dEoi/WoQ/wvtct4/TgKD/dpwu6IhJ9sQ/9t1+xmMZsiu89d6TWTREROe9iH/rZVJJ3X72ErTuPMjQ6VuvmiIicV7EPfYDffMMKeoYKbN15tNZNERE5rxT6wFsvW8zqtjyPPHWg1k0RETmvFPoEq3g2b1rFU/u72HtMX6wiItGl0A99cMMqcukEf/HDl2vdFBGR80ahH1rUkGXzxtV8e8chDnYN1Lo5IiLnhUK/xEduvJSkGZ/7l721boqIyHmh0C+xrLmOD71tDX/3TAc7DnbXujkiIvNOoV/mP75zHe2NWf7w8Z2MjZd/V4yIyMKm0C/TkE3xB+97LTsOdvMXP3yp1s0REZlXCv0Kbr1uOe+/djmfeWIPzxzQ1ymKSHQo9CswM/7Xb7yO5S057vna01rNIyKRodCfRnNdmofv3shIYYy7Hv4Fx3uGat0kEZFzptCfweWXNPKluzZy9PQQtz/4M434RWTBU+jPYtPaNr7x79/Eqf4RbvniT/QtWyKyoCn0q3D96la+c+/buKQxx10P/4L//b0XGRgp1LpZIiJzptCv0trF9fz9x97C5o2r+csf7+fdn/kR33vuCONayy8iC4hCfw7ymRR//IFreOwjN1CfTfKxR57hfV/4Mf/4/BEK+o5dEVkAzP3iGqlu2LDBt2/fXutmzGps3PmHZw/z+Sf3sv9EP8uac9y5aTW/tXEVS5pytW6eiMSMmT3t7htmrVdN6JvZTcDngSTwJXf/k7L9WeBrwBuBk8AH3f2VcN/rgQeBJmAc2Oju065/XCihX1QYG+fJXx3nGz9/lR/vPYEZbFzTxvuvXc7Nr1vK4oZsrZsoIjEwb6FvZklgD/BuoAPYBtzh7i+W1Pko8Hp3/w9mthn4TXf/oJmlgGeAf+vuz5rZIqDb3af9MtqFFvql9p/o5zs7DvHd546w73gfZvD6lS382hXt3HhlO9eubCGZsFo3U0QiaD5D/wbgD939veH2JwDc/Y9L6mwN6/wsDPqjQDtwM3Cnu/9OtQ1fyKFf5O7sPtbL1heO8cM9x9lxsJtxh5Z8mrdctohNa9rYuLaNq5Y2qRMQkXlRbeinqnitFcDBku0O4E3T1XH3gpmdBhYBVwAedgrtwKPu/n+q+JkLmplx1dImrlraxMfftY5T/SP8ZN8J/nV3J0/tP8mW54MvYG/Mpdi4po1Na9vY8JpWrl7eTF0mWePWi0iUVRP6lYai5X8eTFcnBbwN2AgMAE+GvdGTk55sdg9wD8Dq1auraNLC0lqf4f3XBh/iBnCoe5Bt+7t4an8Xv9h/ku//6jgAyYRx5ZJGrlvdwnUrW7h2VQuXX9KgvwZEZN5UE/odwKqS7ZXA4WnqdITTO81AV1j+Q3c/AWBmW4DrgUmh7+4PAQ9BML0z98NYWFa01LHiDSv4jTesAOBE3zA7DnTzbEc3Ow528w/PHuaRpw4AUJ9Jcs3KZq5d2cL65U1cvbyZtYvr1RGIyFmpJvS3AevMbC1wCNgM3FlW53HgLuBnwG3A9929OK3z+2aWB0aAG4HPzlfjo2JxQ5Z3rV/Cu9YvAWB83Nl/sp9nD3bz7MGgI3j4p68wEr4XoC6d5KpljVy9vIn1y5q5enkTVy5tJJfW1JCIzGzW0A/n6O8FthIs2fyKu+80s/uB7e7+OPBl4Otmto9ghL85fO4pM/sMQcfhwBZ3/955OpbISCSMy9obuKy9gQ9cvxKA0bFx9h3vY+fhHnYePs2Lh3v4zo7DfOPnwV8EyYRxWXs9Vy9v5qqljVyxtJErlzSyrDmHmf4qEJGA3py1gLk7B7sGefHI6bAzCDqEYz3DE3UasynWLWngiiWNXLGkkSuXNrJuSQPtDVl1BiIRMq9vzrqQFPrnrntghD3H+th9rJe9x3rZfbSXPcd6OTUwOlGnNZ+e6Aguv6SBS9vrubS9gWVNORK6XiCy4Mznkk1ZYFryGTatDZaCFrk7J/pG2HOst+TWx7d/eYje4TOfGJpLJ1i7OOgELlscdATFDqEhq38uIgud/hfHhJnR3pilvTHLWy9fPFHu7nT2DvNSZz8vn+jj5c5+Xu7s44VDp/nH549Q+iGilzRmJzqANYvyrG7Ls7qtntcsylOvDkFkQdD/1JgzMy5pynFJU44bLls0ad9wYYwDJwemdAhbnj9Cd8lUEcDihkzYCeRZvaie17Tlec2iPKsX5XX9QOQiotCXaWVTSdYtaWTdksYp+04PjnLg5AAHugZ4taufAycHePXkANteOcXjzx6e9BdCXTrJ6rY8q9rqWNFSx/KWOla0Bo9XtNaxuD6r6wgiF4hCX85Kc12aa1Y2c83K5in7RgrjdJwKOoQDXUFn8OrJATpODfCL/V30DE3+1rFMKsHy5hwrWutY3lzSIYSdwrLmOjIpffWDyHxQ6Mu8y6QS4QXghor7e4dGOdQ9yKFTgxzuHqSj5PGP9nZyvHeY8kVli+ozLGnKsbQ5x5KmbPC4KceS5uB+aVOOlnxa00gis1DoywXXmEtz1dI0Vy1tqrh/pDDO0dNDdHQPcOjUIEdOD3G0Z4hj4f1zHd2c6BuZ8rxsKsGSpsmdwtLmHO2NWRY3BLf2xiwtdWlNJ0lsKfTlopNJJVgdXgSezkhhnOO9QxzrGeLo6WGO9gxxvCfoFI6eHuKFQ6f5l13HGBqd+jWWqYSxqCEzqSMIHmeCFU4NWRaH983qICRiFPqyIGVSCVa25lnZOn3H4O70DBbo7BvmRN8wnb3B/ZnHI3T2DrPnWC8n+oYZHZv6RsVUwmirz9BWn6E1H97Xp2nLZ2gtKy/e9BlIcjFT6EtkmRnN+TTN+TSXX1L5+kKRu3N6cJQTfcMcDzuEE73DdPYN09U3QtfACKf6R9h1tIdT/SN0D45Oue5QVJdOTnQOEx1FPkNzXXryLT95W52FXAgKfRGCDqIln6Eln+HyS6YuUS03Nh50El39I5waGAnu+890Dl39oxPlB7oG6OobmfTO50oyqcTUjiG8NZVtN+ZSwS2bpiGXoiGb0gonqYpCX+QsJEumfapVGBund6jA6cHRireesu1jPUPsOdbL6cFReodm7jAguJDdGHYADSUdQmNxO5eioaSsvG4+m6Q+kyKXTmgVVIQp9EUukFQyQWt9cC1grsbGnd6hMx1C71CB3qECfcMFeodG6Ss+Hg7Lh0bpGy5wsGtgol7fcIGx8dk/YNEM8ukk+WyK+kySfCZFPlNpO+gk8mFZfbZkXyZJffbMvlw6QS6V1EXxi4BCX2QBSCbOTD+dLXdnaHSc3uHRsGModhpBxzE4Okb/8BgDIwUGRoL7YDt4fHpwlKOnByfq9I+MMVKYujpqJplUgrp0kly6eF+8lW9PLis+Jzdpe3JZNpUgm06QTSbJpBJkUgl9w1wFCn2RmDAz6jJJ6jJJqrhsUZXC2DgDo2MMlHQW/cOFibL+kQIDwwWGCuMMjY4xNFq8H2Nw4j4oO9k/UlJ+pl6lVVXVSiWMbNgBZFIJsqlkeB+WJRNk08nwPkE2mZi0v1i/YlkyQSZlpJOJiVsmmSAdlmUmyo106sx2rTsihb6InLVUMkFTMkFTLn3efkZhbHyi0xgcGWO4MMbgyDhDhbGJssHR4K+O4cI4I4VxRsbGGR4dZ2RsLLwfn7gP6o0xHNbvGRwNnzc25fnDhTGqmBGbk4RR0kGEnUK4vX55E1+88/r5/YFlFPoiclFLJRM0JBM1+z6Hwtj0ncbomDM6Ns5o2FFMbJc9HimUbY+NM1oo2x5zVrXWnffjUeiLiMwglUyQSiY4h8spFxUt7BURiRGFvohIjCj0RURiRKEvIhIjCn0RkRhR6IuIxIhCX0QkRhT6IiIxYj7dN0HUiJl1Aq+ew0ssBk7MU3MWCh1z9MXteEHHPFevcff22SpddKF/rsxsu7tvqHU7LiQdc/TF7XhBx3y+aHpHRCRGFPoiIjESxdB/qNYNqAEdc/TF7XhBx3xeRG5OX0REphfFkb6IiEwjMqFvZjeZ2W4z22dm99W6PfPFzFaZ2Q/MbJeZ7TSzj4flbWb2hJntDe9bw3Izsy+Ev4fnzOz8fg3PeWRmSTP7pZl9N9xea2ZPhcf8N2aWCcuz4fa+cP+aWrb7bJlZi5l908x+FZ7vG6J+ns3sv4T/rl8ws782s1zUzrOZfcXMjpvZCyVlcz6vZnZXWH+vmd11tu2JROibWRJ4ALgZWA/cYWbra9uqeVMAfs/dXwu8GfhYeGz3AU+6+zrgyXAbgt/BuvB2D/DnF77J8+bjwK6S7T8FPhse8yngw2H5h4FT7n458Nmw3kL0eeCf3P0q4FqCY4/seTazFcB/Aja4++uAJLCZ6J3nrwI3lZXN6byaWRvwKeBNwCbgU8WOYs7cfcHfgBuArSXbnwA+Uet2nadj/Q7wbmA3sCwsWwbsDh8/CNxRUn+i3kK6ASvD/wzvBL4LGMGbVlLl5xzYCtwQPk6F9azWxzDH420C9pe3O8rnGVgBHATawvP2XeC9UTzPwBrghbM9r8AdwIMl5ZPqzeUWiZE+Z/7xFHWEZZES/jn7BuApYIm7HwEI7y8Jq0Xld/E54PeB8XB7EdDt7oVwu/S4Jo453H86rL+QXAp0Ag+HU1pfMrN6Inye3f0Q8GfAAeAIwXl7mmif56K5ntd5O99RCX2rUBapZUlm1gD8HfCf3b1npqoVyhbU78LM/g1w3N2fLi2uUNWr2LdQpIDrgT939zcA/Zz5k7+SBX/M4fTErcBaYDlQTzC9US5K53k20x3jvB17VEK/A1hVsr0SOFyjtsw7M0sTBP7/c/dvhcXHzGxZuH8ZcDwsj8Lv4q3ALWb2CvAowRTP54AWM0uFdUqPa+KYw/3NQNeFbPA86AA63P2pcPubBJ1AlM/zu4D97t7p7qPAt4C3EO3zXDTX8zpv5zsqob8NWBde9c8QXAx6vMZtmhdmZsCXgV3u/pmSXY8DxSv4dxHM9RfLfzdcBfBm4HTxz8iFwt0/4e4r3X0Nwbn8vrv/NvAD4LawWvkxF38Xt4X1F9QI0N2PAgfN7Mqw6NeBF4nweSaY1nmzmeXDf+fFY47seS4x1/O6FXiPmbWGfyG9Jyybu1pf4JjHCyXvA/YALwF/UOv2zONxvY3gz7jngB3h7X0Ec5lPAnvD+7awvhGsZHoJeJ5gZUTNj+Mcjv/XgO+Gjy8FfgHsA/4WyIbluXB7X7j/0lq3+yyP9Tpge3iuvw20Rv08A/8T+BXwAvB1IBu18wz8NcE1i1GCEfuHz+a8Av8uPPZ9wIfOtj16R66ISIxEZXpHRESqoNAXEYkRhb6ISIwo9EVEYkShLyISIwp9EZEYUeiLiMSIQl9EJEb+P+B3TiGsBkjuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the loss\n",
    "plt.figure()\n",
    "plt.plot(all_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x20d17233240>]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmQnHed3/H3d6ane+5T10jyWDKWb4zwDsIsgbC2BfYuWbs2zmKWgACDajew2WSP2C6TJXGWCmRTMUlBWLQ2WNzGZlmrDIXLFvaGLbDxGMv3ocOHRhpppLmvnqu/+aOfHnW3uudQ90xr5vm8qrqe6/dM/x499nzm9/s9h7k7IiIiKWWlroCIiJxdFAwiIpJBwSAiIhkUDCIikkHBICIiGRQMIiKSQcEgIiIZFAwiIpJBwSAiIhkipa7AmVi1apVv2rSp1NUQEVlWnnrqqZPuvnqucssyGDZt2kRHR0epqyEisqyY2RvzKaeuJBERyaBgEBGRDAoGERHJoGAQEZEMCgYREclQlGAws2vN7BUzO2Bmt+bYHjOze4PtT5jZpmD9JjMbM7N9wefvilEfERE5cwVfrmpm5cBXge1AJ/Ckme1x9xfTit0M9Ln7+WZ2E/Al4EPBtoPuvrXQeoiISHEU4z6GbcABdz8EYGY/AK4H0oPheuC/BPP3A18xMyvCd4fS6MQUnX1jHBuIMxifZGBskqH4FNMJx91JOLhDrKKMqory5CdaTnW0nMbqKM01UZqro9RVRigr02kQkUzFCIYNwOG05U7gnfnKuPuUmQ0ALcG2zWb2NDAIfM7df1GEOq0YfSMT/PJgD88e6ee5zgFePT7EyeGJovzs8jKjqbqCpuooa+srWddQybr6StYG09aGStbWV9JSE1WAiIRIMYIh128Mn2eZLqDN3XvM7LeAfzSzS9198LQvMdsJ7ARoa2srsMpntxND4/z46U5+9vwx9h3uJ+FQUW5c3FrP1Retpa2lmo1NVbQ2VNFYXUF9ZQV1lREi5UaZJT8A41PTjE1MMzaZnA6PT9E/NknfyAS9IxP0jU7QOzJJz/A4x4fGObD/JN1DcRJZZ6+i3FhbX8nGpio2NCa/e2NTFRuaqjinqZp1DZVUlOs6BpGVohjB0Amck7a8ETiap0ynmUWABqDX3R0YB3D3p8zsIHABcNrzLtx9F7ALoL29PTt4VoRnO/v52mMHefjF40wlnLduaOBPr9rCv7xwNZeurycWKV/Qz6uORqiOLuwUTyeck8PjHBuIc2wwPjPt6h+js2+MXx48ybHBOJ52BsoM1tVXsrGpeiYw0kOktbFywXUXkdIpRjA8CWwxs83AEeAm4I+yyuwBdgC/Am4Efu7ubmarSQbEtJmdB2wBDhWhTsvKwRPD/PefvsQjL3XTUFXBJ969iQ+9o43z19QueV3Ky5Ktg7X1lbwtT5mJqQRdA2Mc6UuGRWffKJ1BcDzxWi9d+8YyWh1msKYuxsamajY0VqWFx6nlygoFh8jZouBgCMYMPgs8BJQD33D3F8zsDqDD3fcAdwPfNrMDQC/J8AB4L3CHmU0B08Afu3tvoXVaLiamEvzdPx3kKz8/QKyijL/YfgEff/cm6iorSl21WUUjZZzbUsO5LTU5t09OJzg2EKezb4wj/cngSIXIvsP9/PS5Lqay+qtW1cZmWhobc4RHTWxZPu9RZFky9+XXK9Pe3u7L/emqR/rH+Hff/Q3PHO7ng5e38tf/6hLW1FWWulpLYjrhHB+MnxYaR/pPTSemEhn7NFVX5G9xNFdRf5aHqcjZwMyecvf2ucrpz7AS6Hi9l09/q4PJaedrH7mC697aWuoqLanyMmN9YxXrG6t4x6bm07YngnGOVPfUkVR3Vd8YB04M89ir3cQnM4OjrjKSERypz/rGKtY1VLKqJqYrq0TmScGwxB59uZs/+e5TtDZUcfeOds5bvfTjCGe7sjJjTX0la+oruaKt6bTt7k7vyETOrqrOvlEeP9TD8PhUxj7R8jLWNsRorU8Ohrc2VNHakLwkNxUeLTVRdHuNiIJhST32Sjef/lYHF7XWcc8ntrGqNlbqKi1LZkZLbYyW2hhvO6fxtO3uzuDYFIf7RukaiNM1MMbR/jjHBsY4OhDnN2/2cWygi8npzG7UaKSM1uAejvWNp4KjteFUmDRVVyg8ZMVTMCyRp9/s40++8xsuWFvH9z59pfrEF5GZ0VBdQUN1A5dtaMhZJpFwekYmMkKjayDO0YHk/K9f6+X4YPy0QfLKirLkTYAznxhrg9bN2rrYzPqqqK6ykuVLwbAEjvaPcfPuDlbXxbjnk+9QKJwFysqM1XUxVtfFuHxj7jLTCadneJyjA8n7OFKhcXQgzonBcZ7p7OfYQJzxrIFySI55zARHXRAc9bGMQFldF9P9HXJWUjAsssnpBJ/93m8Yn5zmvj9+V2iuPFoJytPGOrbm6LKCoNsqPkX3YJzjg+McH4xzfChOd2p+MM4Tr/XSPRQ/resKoLkmypqgpbGmLsaa+hirapOhkT6tr4yoC0uWjIJhkf2Pn73Mb97s5yt/9HbeooHmFcfMaKiqoKGqgi1r6/KWSySc/rHJmbDoTguR44PjdA/GefnYID3DE6d1X0Fy/GN1bYxVdTFW10aTrZ2Z5cxpTbRcISIFUTAsoqfe6OWuf36Nf3tlGx+8fH2pqyMlVFZmyafa1kS5uLU+b7lUgJwYGufk8HjG9EQwTd4oOEDvyPhpz7UCqKooZ1VdNBkUaa2OVbVRWmpjNNdEaQnq0lgdpVyX8UoWBcMiGZ+a5pYfPcf6hipuve7iUldHlon0ALmQ/C0QSI6B9I5MnBYeJ1PT4XFe7xmh440+ekdyP5G3zKAp9Sj2migttVFaamIz86n1q4JAaVKQhIKCYZF87bGDHOge5p5PvINaPc5BFkF52gD6XCanE/SNTNATPFm3Z2SCnuHxmfne4eT6V44N0TvSQ9/oZM6fYwaNVRVBcMRmWh6paVPQCmmqrqCxKkpjTQV1MY2PLDf6jbUIjg/G+fo/HeL33trK+y5cU+rqiFBRXjYzkD4fU9MJ+kYng+BIBkjvyAQnhyfoDZZ7hic40D1MT/AI93xP14mUGY3VFacCI5g2VUcz1jUG61LL0Yge5V4qCoZFcOfDrzKVSHDLtReVuioiZyRSXpbWGpm9SwuS3VoDY5P0jU7QPzpB30hqPjntG52kP1g+3DvKc53J9bku9U2pCd442FSTGSKpwf76qrT5yorkvStVFRp8LwIFQ5G9enyIH3Yc5uO/vZm2lupSV0dkSZSnjY0sxNjEdBAcWSEycipMUusO947SNzrJYHwyb+sEki2U+rTwqK+MzARI9qc+a74uptfdgoKh6L7y8wNUVZTzp1edX+qqiJz1qqLlVEWTDzucr0TCGRqfYnAs+b7z1DT9k3wX+tTMcmff2Mz8dK5LuQJlBnWVFdRXnQqT2liEuuAtiXWVyaCpq4xQG0utO7W+tjJCVcXyb7EoGIrozZ5RHnz2KJ96z3k0LfAvJxGZn7KyU/eOnDN38QzuzujE9GlBkgqY7JAZjE9xYmiEofgUw/EphrIezphLpMyoTQVGVnhkz9fGIjOv5p1ZVxmhJhop6dVfCoYi+vtfHKK8zPjkuzeXuioikoOZUROLUBOLLKiVkpJIOMMTUwzFpxiKT84ExmAwn75+KD7J8PgUg/EpjvTHGYoPzayfpdEyoyZaTk0sGRR1QZ1rYxE+//uXsuEM6r4QCoYi6Rke54cdh/mDt29kXYMeeyGyEpWVGfWVFcHzzs7sl3Oq1TI8ngyJwfjpQTM8HnziUwxPnFrXOzLKUrxcTcFQJD/s6GR8KsGn36vWgojkl95qWTvPy4eXmi4ULoJEwrn3yTfZtrmZ89fMfWmfiMjZTMFQBI8f6uH1nlE+vG2hQ2EiImcfBUMRfP/Jw9RXRrjusnC9u1lEViYFQ4H6Ryd46Plj/MEVG6ms0EtXRGT5UzAU6KEXjjExneBfX5HnNWAiIsuMgqFADz7bRVtzNZdtyP+MfRGR5aQowWBm15rZK2Z2wMxuzbE9Zmb3BtufMLNNadtuC9a/YmYfKEZ9lkrP8Di/PNjD713euuxvgRcRSSk4GMysHPgqcB1wCfBhM7skq9jNQJ+7nw/cCXwp2PcS4CbgUuBa4P8GP29ZeOiF40wnnA9erkFnEVk5itFi2AYccPdD7j4B/AC4PqvM9cDuYP5+4GpL/ol9PfADdx9399eAA8HPWxZ+8txRNq+q4ZJZXtUoIrLcFCMYNgCH05Y7g3U5y7j7FDAAtMxz37PSwNgkjx/q5brL1qkbSURWlGIEQ67fitkP88hXZj77Jn+A2U4z6zCzjhMnTiywisX3z/tPMp1wrrpIb2gTkZWlGMHQCRlPv90IHM1XxswiQAPQO899AXD3Xe7e7u7tq1evLkK1C/PoK900VFWw9ZzGUldFRKSoihEMTwJbzGyzmUVJDibvySqzB9gRzN8I/NyTjwjcA9wUXLW0GdgC/LoIdVpUiYTz2CsneO8Fq4mU64pfEVlZCn66qrtPmdlngYeAcuAb7v6Cmd0BdLj7HuBu4NtmdoBkS+GmYN8XzOyHwIvAFPAZd58utE6L7cWuQU4Oj/M7F5a+5SIiUmxFeey2u/8U+GnWur9Om48D/ybPvl8AvlCMeiyVR1/uBuC9FygYRGTlUT/IGfh/+09w+cYGVtXGSl0VEZGiUzAs0NjENPsO9/Out7SUuioiIotCwbBAT7/Zx+S0c+VmBYOIrEwKhgV6/LVeygzaNzWVuioiIotCwbBATxzq4dL1DdRVVpS6KiIii0LBsADxyWmePtzPOzc3l7oqIiKLRsGwAM8c7mdiKsE7z9P4goisXAqGBfj1a72YwbZNajGIyMqlYFiApw/3c/7qWhqqNb4gIiuXgmGe3J1nDvfroXkisuIpGOaps2+MnpEJ3qZgEJEVTsEwT8909gOoxSAiK56CYZ6eOdxPNFLGhevqSl0VEZFFpWCYp2cOD3DZ+noq9P4FEVnh9FtuHqamEzx3ZEDjCyISCgqGedjfPczY5LTGF0QkFBQM8/BsMPB8+UYFg4isfAqGeXjx6CA10XLOba4udVVERBadgmEeXuoa4qLWesrKrNRVERFZdAqGObg7Lx0b5CJdpioiIaFgmMOR/jGG4lNc3Fpf6qqIiCwJBcMcXuoaAlAwiEhoKBjm8HLXIIC6kkQkNBQMc3jp2CDntlRTE4uUuioiIkuioGAws2Yze9jM9gfTpjzldgRl9pvZjrT1j5nZK2a2L/isKaQ+i+HlriEuXqduJBEJj0JbDLcCe919C7A3WM5gZs3A54F3AtuAz2cFyEfcfWvw6S6wPkUVn5zm9Z4RLlA3koiESKHBcD2wO5jfDdyQo8wHgIfdvdfd+4CHgWsL/N4l8XrPCAmH89fUlroqIiJLptBgWOvuXQDBNFdX0AbgcNpyZ7Au5ZtBN9J/NrOz6g6yg90jALxldU2JayIisnTmHFE1s0eAdTk23T7P78j1y96D6Ufc/YiZ1QE/Aj4KfCtPPXYCOwHa2trm+dWFOdA9jBmct0otBhEJjzmDwd2vybfNzI6bWau7d5lZK5BrjKATeF/a8kbgseBnHwmmQ2b2PZJjEDmDwd13AbsA2tvbPVeZYjt4YpgNjVVURcuX4utERM4KhXYl7QFSVxntAB7IUeYh4P1m1hQMOr8feMjMIma2CsDMKoAPAs8XWJ+iOtA9rPEFEQmdQoPhi8B2M9sPbA+WMbN2M7sLwN17gf8GPBl87gjWxUgGxLPAPuAI8PcF1qdoEgnn0Mlh3rJawSAi4VLQXVvu3gNcnWN9B/CptOVvAN/IKjMC/FYh37+YjvSPEZ9MqMUgIqGjO5/zOHBiGNClqiISPgqGPA52J4NBXUkiEjYKhjwOnhimuSZKc0201FUREVlSCoY8Dp4Y4bxVurFNRMJHwZDHGz0jnNuiYBCR8FEw5BCfnOb44DjntlSXuioiIktOwZDD4d5RAAWDiISSgiGHN3qSwdDWrGAQkfBRMOTwRq+CQUTCS8GQw5s9I9TGIrpUVURCScGQw5u9o7Q1V3OWvR5CRGRJKBhyeKN3VAPPIhJaCoYs0wmns3eMNgWDiISUgiHLscE4E9MJzm3WzW0iEk4Khixv9CTf86wrkkQkrBQMWXRzm4iEnYIhyxs9o0TKjNaGylJXRUSkJBQMWY70j7G+sYpIuf5pRCSc9Nsvy9H+MbUWRCTUFAxZjvbH2dBYVepqiIiUjIIhzdR0gmODcdYrGEQkxBQMabqHxplOuIJBREJNwZCma2AMgNZGjTGISHgpGNIc6Y8DaIxBREKtoGAws2Yze9jM9gfTpjzlfmZm/Wb2YNb6zWb2RLD/vWZW0udcH+0PWgy6KklEQqzQFsOtwF533wLsDZZz+VvgoznWfwm4M9i/D7i5wPoU5Gj/GPWVEeoqK0pZDRGRkio0GK4Hdgfzu4EbchVy973AUPo6S77s4Crg/rn2XypHg5vbRETCrNBgWOvuXQDBdM0C9m0B+t19KljuBDYUWJ+CHO3XpaoiIpG5CpjZI8C6HJtuL/C7c70ezWepx05gJ0BbW1uBX53b0YExrji3cVF+tojIcjFnMLj7Nfm2mdlxM2t19y4zawW6F/DdJ4FGM4sErYaNwNFZ6rEL2AXQ3t6eN0DO1Mj4FP2jk2oxiEjoFdqVtAfYEczvAB6Y747u7sCjwI1nsn+xpe5h0KWqIhJ2hQbDF4HtZrYf2B4sY2btZnZXqpCZ/QK4D7jazDrN7APBpluAPzezAyTHHO4usD5nLHUPQ2uDgkFEwm3OrqTZuHsPcHWO9R3Ap9KW35Nn/0PAtkLqUCxduodBRATQnc8zjg+OA7C2XsEgIuGmYAgcH4rTXBMlGtE/iYiEm34LBroHx1lTFyt1NURESk7BEOgeirNG3UgiIgqGlO7BcdaqxSAiomAAmE44J4bHWVOvYBARUTAAvSMTTCdcVySJiKBgAOD4YPLmtjV1CgYREQUDyYFnQF1JIiIoGIDkwDPo5jYREVAwAKfuel5dqxaDiIiCgWRXku56FhFJ0m9Cki0G3fUsIpKkYABO6K5nEZEZCgaSLQbd9SwikhT6YNBdzyIimUIfDLrrWUQkU+iD4dRdz2oxiIiAgoGTw8E9DHochogIoGCgZ3gCgJaaaIlrIiJydlAwjCRbDC21CgYREVAw0DMyQTRSRm0sUuqqiIicFRQMwxOsqoliZqWuiojIWUHBMDxOix6eJyIyo6BgMLNmM3vYzPYH06Y85X5mZv1m9mDW+nvM7DUz2xd8thZSnzPRMzKh8QURkTSFthhuBfa6+xZgb7Ccy98CH82z7a/cfWvw2VdgfRasZ3iClhq1GEREUgoNhuuB3cH8buCGXIXcfS8wVOB3FZ27c3J4nFVqMYiIzCg0GNa6exdAMF1zBj/jC2b2rJndaWZL+qf7yMQ041MJdSWJiKSZ8xpNM3sEWJdj0+1F+P7bgGNAFNgF3ALckaceO4GdAG1tbUX46uTAM6CuJBGRNHMGg7tfk2+bmR03s1Z37zKzVqB7IV+eam0A42b2TeAvZym7i2R40N7e7gv5nnxOpu56VotBRGRGoV1Je4AdwfwO4IGF7ByECZa8ieAG4PkC67MgqRbDKl2uKiIyo9Bg+CKw3cz2A9uDZcys3czuShUys18A9wFXm1mnmX0g2PRdM3sOeA5YBfxNgfVZkJ4RtRhERLIV9BwId+8Brs6xvgP4VNrye/Lsf1Uh31+oVIuhWQ/QExGZEeo7n08OT1BXGSEWKS91VUREzhqhDoaekQmNL4iIZAl3MAyP6z0MIiJZQh4Mek6SiEi2cAfDiJ6sKiKSLbTBMJ1wekeS72IQEZFTQhsMA2OTJByaFAwiIhlCGwx9o8mb23QPg4hIptAGQ38QDA1VFSWuiYjI2SW0wdA3MglAU7VaDCIi6UIbDP1jCgYRkVzCGwxBV1JjjbqSRETShTYY+kYnKC8z6mIFPUdQRGTFCXEwTNJYVUHyVRAiIpIS2mDoH52gsVrdSCIi2UIbDH0jkxp4FhHJIbzBMDpBo4JBROQ0oQ2G/tFJmtSVJCJymtAGQ9/ohJ6TJCKSQyiDIT45zfhUQo/DEBHJIZTBkHqAngafRUROF85gmHlOkloMIiLZQhkMM4/DUItBROQ0oQyGvtGgxaDnJImInKagYDCzZjN72Mz2B9OmHGW2mtmvzOwFM3vWzD6Utm2zmT0R7H+vmS3Jn/AaYxARya/QFsOtwF533wLsDZazjQIfc/dLgWuBL5tZY7DtS8Cdwf59wM0F1mde9JIeEZH8Cg2G64Hdwfxu4IbsAu7+qrvvD+aPAt3Aaks+ve4q4P7Z9l8MfaOTVFWUU1lRvhRfJyKyrBQaDGvdvQsgmK6ZrbCZbQOiwEGgBeh396lgcyewocD6zEvf6ISuSBIRyWPOlxGY2SPAuhybbl/IF5lZK/BtYIe7Jyz38659lv13AjsB2traFvLVpxkYndQVSSIiecwZDO5+Tb5tZnbczFrdvSv4xd+dp1w98BPgc+7+eLD6JNBoZpGg1bARODpLPXYBuwDa29vzBsh8JB+HoRaDiEguhXYl7QF2BPM7gAeyCwRXGv0Y+Ja735da7+4OPArcONv+i6F/dJLGKrUYRERyKTQYvghsN7P9wPZgGTNrN7O7gjJ/CLwX+LiZ7Qs+W4NttwB/bmYHSI453F1gfeZlYGySBo0xiIjkVNALj929B7g6x/oO4FPB/HeA7+TZ/xCwrZA6LJS7Mxif1KWqIiJ5hO7O57HJaSannfpKBYOISC6hC4bBseTVsfVVBTWWRERWrPAFQzz5nCS1GEREcgtfMIwFwaAxBhGRnMIXDEGLQYPPIiK5hS4YBlIthkqNMYiI5BK6YDg1+KwWg4hILiEMhmSLoU4tBhGRnMIXDPFJKivKiEX0yG0RkVzCFwxjUxp4FhGZReiCYWBsUvcwiIjMInTBMBif1MCziMgswhkMGngWEckrfMEwNqUWg4jILMIXDHGNMYiIzCZUweDuDI7pXQwiIrMJVTAMj0+RcD1yW0RkNqEKhsF48DgMdSWJiOQVrmDQI7dFROYUzmBQi0FEJK9wBUPQlaTBZxGR/EIVDDPvYtDgs4hIXqEKBnUliYjMLVzBENe7GERE5lJQMJhZs5k9bGb7g2lTjjJbzexXZvaCmT1rZh9K23aPmb1mZvuCz9ZC6jOXwbEpaqLlRMpDlYciIgtS6G/IW4G97r4F2BssZxsFPubulwLXAl82s8a07X/l7luDz74C6zOrwbjuehYRmUuhwXA9sDuY3w3ckF3A3V919/3B/FGgG1hd4PeekYExPXJbRGQuhQbDWnfvAgima2YrbGbbgChwMG31F4IupjvNLFZgfWa19ZxG3nfhrFUUEQm9OUdhzewRYF2OTbcv5IvMrBX4NrDD3RPB6tuAYyTDYhdwC3BHnv13AjsB2traFvLVMz7zO+ef0X4iImEyZzC4+zX5tpnZcTNrdfeu4Bd/d55y9cBPgM+5++NpP7srmB03s28CfzlLPXaRDA/a29t9rnqLiMiZKbQraQ+wI5jfATyQXcDMosCPgW+5+31Z21qDqZEcn3i+wPqIiEiBCg2GLwLbzWw/sD1YxszazeyuoMwfAu8FPp7jstTvmtlzwHPAKuBvCqyPiIgUyNyXX69Me3u7d3R0lLoaIiLLipk95e7tc5XTnV4iIpJBwSAiIhkUDCIikkHBICIiGZbl4LOZnQDeOMPdVwEni1id5UDHHA465nAo5JjPdfc5H0m0LIOhEGbWMZ9R+ZVExxwOOuZwWIpjVleSiIhkUDCIiEiGMAbDrlJXoAR0zOGgYw6HRT/m0I0xiIjI7MLYYhARkVmEKhjM7Foze8XMDphZrteQLjtmdo6ZPWpmLwXv1f6zYH3O93Fb0v8J/g2eNbMrSnsEZ87Mys3saTN7MFjebGZPBMd8b/BkX8wsFiwfCLZvKmW9z5SZNZrZ/Wb2cnC+37XSz7OZ/cfgv+vnzez7Zla50s6zmX3DzLrN7Pm0dQs+r2a2Iyi/38x25Pqu+QpNMJhZOfBV4DrgEuDDZnZJaWtVFFPAX7j7xcCVwGeC48r3Pu7rgC3BZyfwtaWvctH8GfBS2vKXgDuDY+4Dbg7W3wz0ufv5wJ1BueXofwM/c/eLgLeRPPYVe57NbAPw74F2d78MKAduYuWd53uAa7PWLei8mlkz8HngncA24POpMDkj7h6KD/Au4KG05duA20pdr0U4zgdIPgL9FaA1WNcKvBLMfx34cFr5mXLL6QNsDP6HuQp4EDCSN/1Ess838BDwrmA+EpSzUh/DAo+3Hngtu94r+TwDG4DDQHNw3h4EPrASzzOwCXj+TM8r8GHg62nrM8ot9BOaFgOn/iNL6QzWrRhB0/ntwBPkfx/3Svl3+DLwn4DUa2JbgH53nwqW049r5piD7QNB+eXkPOAE8M2g++wuM6thBZ9ndz8C/E/gTaCL5Hl7ipV9nlMWel6Ler7DFAyWY92KuSTLzGqBHwH/wd0HZyuaY92y+ncwsw8C3e7+VPrqHEV9HtuWiwhwBfA1d387MMKp7oVclv0xB10h1wObgfVADcmulGwr6TzPJd8xFvXYwxQMncA5acsbgaMlqktRmVkFyVD4rrv/Q7D6eNqrU9Pfx70S/h3eDfy+mb0O/IBkd9KXgUYzS73HPP24Zo452N4A9C5lhYugE+h09yeC5ftJBsVKPs/XAK+5+wl3nwT+AfhtVvZ5TlnoeS3q+Q5TMDwJbAmuaIiSHMTaU+I6FczMDLgbeMnd/1fapnzv494DfCy4uuFKYCDVZF0u3P02d9/o7ptInsefu/tHgEeBG4Ni2cec+re4MSi/rP6SdPdjwGEzuzBYdTXwIiv4PJPsQrrSzKqD/85Tx7xiz3OahZ7Xh4D3m1lT0NJ6f7DuzJR60GWJB3h+F3gVOAjcXur6FOmY/gXJJuOzwL7g87sk+1b3AvuDaXNQ3khenXWQ5Lu220t9DAUe//uAB4P584BfAweA+4BYsL4yWD4QbD+v1PU+w2PdCnQE5/ofgaaVfp6B/wq8DDwPfBsdiKzpAAAAV0lEQVSIrbTzDHyf5BjKJMm//G8+k/MKfDI49gPAJwqpk+58FhGRDGHqShIRkXlQMIiISAYFg4iIZFAwiIhIBgWDiIhkUDCIiEgGBYOIiGRQMIiISIb/D6OCHcjsOEb/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the gradients of w\n",
    "plt.figure()\n",
    "plt.plot(all_grads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x20d17290c50>]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF35JREFUeJzt3X2QVNWdxvHvj+HFSXwZFYw6MIIbnJXoxtERk3V9WzGguyuEVQOWG90YSbIhFfNCAhvLWJgKRiprsikTQyWuibWKmhjCKjpJfImuEWXIKAZ0dASVmWFlIoIaRwT87R/TwNA0M/1y+97bfZ5PFWXf22f6nD52P/f2uS/H3B0REQnLkKQbICIi8VP4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiARqadAP2ZeTIkT527NikmyEiUlFWrlz5Z3cfNVi51Ib/2LFjaW1tTboZIiIVxcxezqechn1ERAKk8BcRCZDCX0QkQAp/EZEAKfxFRAKk8BcRCZDCX0QkQKk9z79aLWnrYmFLO92bezmyrpY5kxuZ1lSfdLNEJDAK/xgtaeti3t3P0LttBwBdm3uZd/czANoAiEisNOwTo4Ut7buCf6febTtY2NKeUItEJFQK/xh1b+4taL2ISLko/GN0ZF1tQetFRMpF4R+jOZMbqR1Ws8e62mE1zJncmFCLRCRUOuAbo50HdXW2j4gkTeEfs2lN9Qp7EUmchn1ERAKk8BcRCVAk4W9mN5vZRjP70z6eNzP7TzPrMLNVZnZiFPWKiEhxotrzvwWYMsDz5wLjM/9mAT+KqF4RESlCJOHv7o8AmwYoMhX4ufdZDtSZ2RFR1C0iIoWLa8y/Hljfb7kzs05ERBIQV/hbjnW+VyGzWWbWamatPT09MTRLRCRMcYV/JzCm3/JooDu7kLsvcvdmd28eNWpUTE0TEQlPXOG/FPhk5qyfjwBb3H1DTHWLiEiWSK7wNbPbgTOBkWbWCXwTGAbg7jcBy4DzgA7gbeBfo6hXRESKE0n4u/vMQZ534PNR1CUiIqXTFb4iIgFS+IuIBEjhLyISIIW/iEiAFP4iIgFS+IuIBEjhLyISIIW/iEiAFP4iIgFS+IuIBEjhLyISIIW/iEiAFP4iIgFS+IuIBEjhLyISIIW/iEiAFP4iIgFS+IuIBEjhLyISIIW/iEiAFP4iIgFS+IuIBEjhLyISoEjC38ymmFm7mXWY2dwczzeY2UNm1mZmq8zsvCjqFRGR4pQc/mZWA9wInAtMAGaa2YSsYlcBd7p7EzAD+GGp9YqISPGi2POfCHS4+1p3fxdYDEzNKuPAgZnHBwHdEdQrIiJFiiL864H1/ZY7M+v6uwa4xMw6gWXAF3K9kJnNMrNWM2vt6emJoGkiIpJLFOFvOdZ51vJM4BZ3Hw2cB9xqZnvV7e6L3L3Z3ZtHjRoVQdNERCSXKMK/ExjTb3k0ew/rXA7cCeDujwP7ASMjqFtERIoQRfivAMab2TgzG07fAd2lWWVeAc4GMLNj6Qt/jeuIiCSk5PB39+3AbKAFeJa+s3pWm9l8Mzs/U+wrwBVm9jRwO3CZu2cPDYmISEyGRvEi7r6MvgO5/ddd3e/xGuDUKOoSEZHS6QpfEZEAKfxFRAKk8BcRCZDCX0QkQAp/EZEAKfxFRAKk8BcRCZDCX0QkQJFc5CXhWdLWxcKWdro393JkXS1zJjcyrSn7Zq4iklYKfynYkrYu5t39DL3bdgDQtbmXeXc/A6ANgEiF0LCPFGxhS/uu4N+pd9sOFra0J9QiESmUwl8K1r25t6D1IpI+Cn8p2JF1tQWtF5H0UfhLweZMbqR2WM0e62qH1TBncmNCLRKRQumArxRs50Fdne0jUrkU/lKUaU31CnuRCqZhHxGRACn8RUQCpPAXEQmQwl9EJEAKfxGRACn8RUQCFEn4m9kUM2s3sw4zm7uPMheZ2RozW21mt0VRr4iIFKfk8/zNrAa4ETgH6ARWmNlSd1/Tr8x4YB5wqru/bmaHlVqviIgUL4o9/4lAh7uvdfd3gcXA1KwyVwA3uvvrAO6+MYJ6RUSkSFFc4VsPrO+33AmcklXmGAAzewyoAa5x9/uzX8jMZgGzABoaGiJompRKk7aIVKcowt9yrPMc9YwHzgRGA4+a2XHuvnmPP3JfBCwCaG5uzn4NiZkmbRGpXlEM+3QCY/otjwa6c5T5tbtvc/d1QDt9GwNJMU3aIlK9ogj/FcB4MxtnZsOBGcDSrDJLgLMAzGwkfcNAayOoW8pIk7aIVK+Sh33cfbuZzQZa6BvPv9ndV5vZfKDV3ZdmnvuYma0BdgBz3P21UusulMavC3NkXS1dOYJek7aIVD5zT+fQenNzs7e2tkb2etnj19A3AcmC6cdrA7AP6jORymNmK929ebBywVzhq/Hrwk1rqmfB9OOpr6vFgPq6WgW/SJUIZjIXjV8XR5O2iFSnYPb8Nem4iMhuwYS/Jh0XEdktmGEfTTouIrJbMOEPGr8WEdkpmGEfERHZTeEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISICq+jx/3cJZRCS3qg1/TUEYHm3sRfJXteE/0C2cCwkEBcre0tgn2tiLFKZqwz+KWzgrUPaW1j6JamMfhTRuHAtVDe9BBla1B3yjuIWzJoDZW1r7JC3zNezcOHZt7sXZvXFc0tYVaztKUQ3vQQZXteGf6xbOI2qG8MWzx+f9GmkJlDRJa5+kZb6GtG4cC1EN70EGV7XDPjt/ol55x1O71m3d8R5f++UqvvbLVQP+7cRxhzC9qZ7DD9qPDVve2ev5kCeASeuk7nMmN+acbzju+RrSunEsRDW8Bxlc1YY/9G0ATjrqYH6z5lWe3fAGz254g9Xdbwz6d0+u28ST6zbt8/muzb2MnXsvACP3H86s04/m4lOOYv8RVd2dQHpCNlta5mtI68axENXwHmRw5u5JtyGn5uZmb21tja2+5/7vDX71xy7ubuui582tZalj9MG1fOb0o7ngpDHUDq8Z/A9SSgcD9y37gDj0bRwraeL7angPITOzle7ePGi5KMLfzKYA3wdqgJ+4+3X7KHcBcBdwsrsPmOxxh3++3J0n1m1i0SNrefC5jWWt6zNnHM2XJh3DfsMqd0MRomrYOFbDewhVbOFvZjXA88A5QCewApjp7muyyh0A3AsMB2ZXavgXwt15+PkeFv1+LY+vfa2sdV140mi+9fHjGDFUGwqRkOUb/lEMUk8EOtx9babixcBUYE1WuWuB64GvRlBnRTAzzmo8jLMaDxu07PYd73HLH17iW/c+W1Rdd63s5K6VnXmVnfyhD3DjxScytKZqT/YSkUFEEf71wPp+y53AKf0LmFkTMMbd7zGzVIV/Wn7eDq0ZwqdPO5pPn3b0oGW3bt/Bjx5+ke/97oWi6mpZ/Sof/MZ9eZWdObGBb3/8OMysqLqkfNLy2ZXKFEX450qFXWNJZjYEuAG4bNAXMpsFzAJoaGiIoGkDS+vVqoMZMbSGKycdw5WTjhm07JvvbOPKxU/xQJHHJ25/8hVuf/KVvMqe2TiK/7rsZG0oYlCpn11JjyjG/D8KXOPukzPL8wDcfUFm+SDgReCtzJ8cDmwCzh9o3D+OMf9Tr3sw5ylt9XW1PDb378tadxptfOMdzrnhEbb0bit7XTMnNrBg+vFlr6da6bMr+xLnmP8KYLyZjQO6gBnAxTufdPctwMh+DXsY+OpgB3zjoItZ9nTYgfvx9Dc/llfZl1/7C2csfLjougr5RXH1P07gU383rui6qpE+u9UjqeG7ksPf3beb2Wyghb5TPW9299VmNh9odfelpdZRLrqYpXhHHfp+XrruH/Iqu3zta8xYtLzouubfs4b592SfP5Dbf1z0YaafOLrouiqFPrvVIcnhu6Av8tLFLOnz6As9/MtPn4ylru/POIGpJ1Tm/2d9dqtDOYbvYr3IqxziOs9fZ0xUrntWdTP7trZY6vr5pyZy+jGjYqkrX/rsVr5xc+8lVwIbsC7PX9Z7/a3CX2S3Hz7cwfX3x3NXyiWfP5UTxtTFUpdUNu3556Dwl6TM/5813PzYuljqemTOWTQc+r5Y6pI9peGXUzmG7xT+IjH4/G1/5N5VG2Kp68l/P5vDDtwvlrqqXZqOmVy15Bluf2I9O9ypMWPmKWP41rTiT4NW+IukiLvzxcVPsfTp7rLXZQZPXf0xDqodVva6ohbX3nharpPQnn8OCn8J1Y73nE/e/ASPdZT3ZoAAow4YwSNzzkrFLcbj3Bsvx4HWYiQ55l/9s4+IVJiaIcZ/f/ojeZXdun0H//SD/+X5V98avHAOPW9u5dir78+r7KRjP8BNl5TvhoADTR8Zdfin5TqJJC/WU/iLVLARQ2v4zZfOyKvsW1u3c9p3HuT1t4u7fcfvns3/hoCfaB7DgunHM2RI/vd5ijMI0zIjXZIbIYW/SCD2HzGUtqvzu33Hpr+8y4nX/rbouu5oXc8dresHLwh89oy/4utTGmMNwrRM+5nkRkhj/iJSko1vvMPZ3/09b27dXva6vnvhh/nnk6rr9h1RH+TWAV8RSZ1SbwhYiJsuOYkpxx0eS11povAXkYr2Ys9bzFy0nI1vbi17XXd99qOcPPaQstcTB4W/iATjT11bmP6jP/Du9vfKWs+5xx3O16f8NWNHvr+s9ZRC4S8iksOT6zZx0Y8fL3s905vq+erkxthPH1X4i4iU6E9dW/jO/c/x6At/LlsdJ489mAtPGsMFJ40u6NTYfVH4i4jEqPWlTSy47zlWvvx6UX8/cv/htF51Tsnt0BW+IiIxah57CL/83N/mVfbRF3r49rLneHbDG7vW/WDmieVqWk4KfxGRmJ02fhT3fTHZyYEU/hUsDfcjF5HKpPCvUElO/CwSJe3EJEPhX6EKuQOivlySVtqJSU557s0qZZfvHRB3frm6Nvfi7P5yLWnriqGVIgMbaCdGyiuS8DezKWbWbmYdZjY3x/NfNrM1ZrbKzB4ws6OiqDdk+7pwJHu9vlxSrCVtXZx63YOMm3svp173YFl2GJK8n33oSg5/M6sBbgTOBSYAM81sQlaxNqDZ3f8G+AVwfan1hm7O5EZqh+05+1KuW8HqyyXFiOsXY747MRK9KPb8JwId7r7W3d8FFgNT+xdw94fc/e3M4nKguu7JmoBpTfUsmH489XW1GH3TvuWa7k5fLilGXL8Y892JkehFccC3Hug/a0MncMoA5S8H8psOSAY0ral+0INiaZmxSCpLXL8Y0zKpStzScBJGFOGf62YUOe8ZYWaXAM1AznnnzGwWMAugoaEhgqZJqF8uKU3cs2qF9HlMyxlOUYR/JzCm3/JooDu7kJlNAr4BnOHuOW/Q7e6LgEXQd2+fCNomhPflktLpF2P5xDlR/UCiCP8VwHgzGwd0ATOAi/sXMLMm4MfAFHffGEGdIlJGIf1ijHsIJi0nYZQc/u6+3cxmAy1ADXCzu682s/lAq7svBRYC+wN3mRnAK+5+fql1i0j5hPCLMYkhmDiH1AYSyRW+7r4MWJa17up+jydFUY+ISJSSGIJJy5Cabu8gErg0nHmSlCSGYNIypKbwFwlYWs48SUpSQzBpGFLTvX1EAhb67T9CvshMe/4iAUvLmSdJScsQTBIU/iIBS8uZJ0lKwxBMEjTsIxKwkIc9Qqc9f5GAhTzsETqFv0jgQh32CJ2GfUREAqTwFxEJkMJfRCRACn8RkQAp/EVEAqTwFxEJkMJfRCRACn8RkQAp/EVEAqTwFxEJkMJfRCRACn8RkQAp/EVEAqS7eooELuQJ3EMWyZ6/mU0xs3Yz6zCzuTmeH2Fmd2Sef8LMxkZRr4iUZucE7l2be3F2T+C+pK0r6aZJmZUc/mZWA9wInAtMAGaa2YSsYpcDr7v7B4EbgO+UWq+IlC70CdxDFsWe/0Sgw93Xuvu7wGJgalaZqcDPMo9/AZxtZhZB3SJSgtAncA9ZFOFfD6zvt9yZWZezjLtvB7YAh0ZQt4iUYF8TtYc0gXuoogj/XHvwXkQZzGyWmbWaWWtPT08ETRORgWgC93BFEf6dwJh+y6OB7n2VMbOhwEHApuwXcvdF7t7s7s2jRo2KoGkiMpBpTfUsmH489XW1GFBfV8uC6cfrbJ8ARHGq5wpgvJmNA7qAGcDFWWWWApcCjwMXAA+6+157/iISP03gHqaSw9/dt5vZbKAFqAFudvfVZjYfaHX3pcBPgVvNrIO+Pf4ZpdYrIiLFi+QiL3dfBizLWnd1v8fvABdGUZeIiJROt3cQEQmQwl9EJEAKfxGRACn8RUQCpPAXEQmQwl9EJEAKfxGRACn8RUQCpPAXEQmQwl9EJEAKfxGRACn8RUQCpPAXEQmQwl9EJEAKfxGRACn8RUQCpPAXEQmQwl9EJEAKfxGRAEUyh69IIZa0dbGwpZ3uzb0cWVfLnMmNTGuqT7pZIkFR+EuslrR1Me/uZ+jdtgOArs29zLv7GQBtAERipGEfidXClvZdwb9T77YdLGxpT6hFImEqKfzN7BAz+62ZvZD578E5ypxgZo+b2WozW2VmnyilTqls3Zt7C1ovIuVR6p7/XOABdx8PPJBZzvY28El3/xAwBfiemdWVWK9UqCPragtaLyLlUWr4TwV+lnn8M2BadgF3f97dX8g87gY2AqNKrFcq1JzJjdQOq9ljXe2wGuZMbkyoRSJhKvWA7wfcfQOAu28ws8MGKmxmE4HhwIsl1isVaudBXZ3tI5KsQcPfzH4HHJ7jqW8UUpGZHQHcClzq7u/to8wsYBZAQ0NDIS8vFWRaU73CXiRhg4a/u0/a13Nm9qqZHZHZ6z+CviGdXOUOBO4FrnL35QPUtQhYBNDc3OyDtU1ERIpT6pj/UuDSzONLgV9nFzCz4cCvgJ+7+10l1iciIhEoNfyvA84xsxeAczLLmFmzmf0kU+Yi4HTgMjN7KvPvhBLrFRGREph7OkdXmpubvbW1NelmiIhUFDNb6e7Ng5XTFb4iIgFS+IuIBEjhLyISoNSO+ZtZD/DyAEVGAn+OqTmVQP2xJ/XH3tQne6rW/jjK3Qe9i0Jqw38wZtaaz0GNUKg/9qT+2Jv6ZE+h94eGfUREAqTwFxEJUCWH/6KkG5Ay6o89qT/2pj7ZU9D9UbFj/iIiUrxK3vMXEZEipT78zWyKmbWbWYeZ7TVTmJmNMLM7Ms8/YWZj429lfPLojy+b2ZrMlJkPmNlRSbQzLoP1R79yF5iZm1lVn92RT3+Y2UWZz8hqM7st7jbGLY/vTIOZPWRmbZnvzXlJtDN27p7af0ANfRO/HE3fJDBPAxOyyvwbcFPm8QzgjqTbnXB/nAW8L/P4c6H3R6bcAcAjwHKgOel2J/z5GA+0AQdnlg9Lut0p6JNFwOcyjycALyXd7jj+pX3PfyLQ4e5r3f1dYDF9U0f2138qyV8AZ5uZxdjGOA3aH+7+kLu/nVlcDoyOuY1xyufzAXAtcD3wTpyNS0A+/XEFcKO7vw7g7jnn4Kgi+fSJAwdmHh8EdMfYvsSkPfzrgfX9ljsz63KWcfftwBbg0FhaF798+qO/y4H7ytqiZA3aH2bWBIxx93vibFhC8vl8HAMcY2aPmdlyM5sSW+uSkU+fXANcYmadwDLgC/E0LVmlzuFbbrn24LNPT8qnTLXI+72a2SVAM3BGWVuUrAH7w8yGADcAl8XVoITl8/kYSt/Qz5n0/Sp81MyOc/fNZW5bUvLpk5nALe7+XTP7KHBrpk9yTjdbLdK+598JjOm3PJq9f5LtKmNmQ+n72bYpltbFL5/+wMwm0TfH8vnuvjWmtiVhsP44ADgOeNjMXgI+Aiyt4oO++X5ffu3u29x9HdBO38agWuXTJ5cDdwK4++PAfvTd96eqpT38VwDjzWxcZjrIGfRNHdlf/6kkLwAe9MyRmyo0aH9khjl+TF/wV/t47oD94e5b3H2ku49197H0HQM5392rdZagfL4vS+g7KQAzG0nfMNDaWFsZr3z65BXgbAAzO5a+8O+JtZUJSHX4Z8bwZwMtwLPAne6+2szmm9n5mWI/BQ41sw7gy8A+T/erdHn2x0Jgf+CuzJSZ2R/0qpFnfwQjz/5oAV4zszXAQ8Acd38tmRaXX5598hXgCjN7GrgduKyKdyB30RW+IiIBSvWev4iIlIfCX0QkQAp/EZEAKfxFRAKk8BcRCZDCX0QkQAp/EZEAKfxFRAL0/2G4p8ilztn1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the final predictions\n",
    "plt.figure()\n",
    "plt.scatter(X, y)\n",
    "plt.plot(X, linear(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-0.20068194>\n",
      "<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.446361>\n"
     ]
    }
   ],
   "source": [
    "# Check the values of the parameters\n",
    "print(west)\n",
    "print(best)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}