S4_LSTM_PARAMS.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jofyc9OC4Qcf"
      },
      "source": [
        "#Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ahBVnrNc3E0U"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "%matplotlib inline\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "from IPython import display\n",
        "plt.style.use('seaborn-white')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "crQSAaIz4SkA"
      },
      "source": [
        "# Read and process data. \n",
        "\n",
        "Download the file from this URL: https://drive.google.com/file/d/1UWWIi-sz9g0x3LFvkIZjvK1r2ZaCqgGS/view?usp=sharing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rgOGxPDP3Wpp"
      },
      "outputs": [],
      "source": [
        "data = open('text.txt', 'r').read()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZeXXMLRb4kXb"
      },
      "source": [
        "Process data and calculate indices"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E5TKeiOp4jtl",
        "outputId": "3301a988-01cc-494d-bc17-7194fb4b1825"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Corona Virus article has 10223 characters, 75 unique characters\n"
          ]
        }
      ],
      "source": [
        "chars = list(set(data))\n",
        "data_size, X_size = len(data), len(chars)\n",
        "print(\"Corona Virus article has %d characters, %d unique characters\" %(data_size, X_size))\n",
        "char_to_idx = {ch:i for i,ch in enumerate(chars)}\n",
        "idx_to_char = {i:ch for i,ch in enumerate(chars)}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4C53MB135LRY"
      },
      "source": [
        "# Constants and Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dfj21ORa49Ps"
      },
      "outputs": [],
      "source": [
        "Hidden_Layer_size = 10 #size of the hidden layer\n",
        "Time_steps = 10 # Number of time steps (length of the sequence) used for training\n",
        "learning_rate = 1e-1 # Learning Rate\n",
        "weight_sd = 0.1 #Standard deviation of weights for initialization\n",
        "z_size = Hidden_Layer_size + X_size #Size of concatenation(H, X) vector"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OdmJf4Du5uhb"
      },
      "source": [
        "# Activation Functions and Derivatives"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "seGHei_D5FGk"
      },
      "outputs": [],
      "source": [
        "def sigmoid(x): # sigmoid function\n",
        "  return (1. /(1 + np.exp(-x)))\n",
        "\n",
        "def dsigmoid(y): # derivative of sigmoid function\n",
        "  return (y * (1. - y))\n",
        "\n",
        "def tanh(x): # tanh function\n",
        "  return (np.tanh(x))\n",
        "\n",
        "def dtanh(y): # derivative of tanh\n",
        "  return (1. - y * y)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KeCvVH1v6Me-"
      },
      "source": [
        "# Quiz Question 1\n",
        "\n",
        "What is the value of sigmoid(0) calculated from  your code? (Answer up to 1 decimal point, e.g. 4.2 and NOT 4.29999999, no rounding off).\n",
        "\n",
        "# Quiz Question 2\n",
        "\n",
        "What is the value of dsigmoid(sigmoid(0)) calculated from your code?? (Answer up to 2 decimal point, e.g. 4.29 and NOT 4.29999999, no rounding off). \n",
        "\n",
        "# Quiz Question 3\n",
        "\n",
        "What is the value of tanh(dsigmoid(sigmoid(0))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off).\n",
        "\n",
        "# Quiz Question 4\n",
        "\n",
        "What is the value of dtanh(tanh(dsigmoid(sigmoid(0)))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QJX5bgUHS3mN",
        "outputId": "0ebdd995-e369-4ad1-d827-e848887b1fc8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.5\n",
            "0.25\n",
            "0.24492\n",
            "0.94001\n"
          ]
        }
      ],
      "source": [
        "print(float(\"{:.2f}\".format(sigmoid(0))))\n",
        "\n",
        "print(float(\"{:.2f}\".format(dsigmoid(sigmoid(0)))))\n",
        "\n",
        "print(float(\"{:.5f}\".format(tanh(dsigmoid(sigmoid(0))))))\n",
        "\n",
        "print(float(\"{:.5f}\".format(dtanh(tanh(dsigmoid(sigmoid(0)))))))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EeSVipDu8iKE"
      },
      "source": [
        "# Parameters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ICbWNemE6LGV"
      },
      "outputs": [],
      "source": [
        "class Param:\n",
        "    def __init__(self, name, value):\n",
        "      self.name = name\n",
        "      self.v = value # parameter value\n",
        "      self.d = np.zeros_like(value) # derivative\n",
        "      self.m = np.zeros_like(value) # momentum for Adagrad"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j83pZNPE8212"
      },
      "source": [
        "We use random weights with normal distribution (0, weight_sd) for  tanh  activation function and (0.5, weight_sd) for  `sigmoid`  activation function.\n",
        "\n",
        "Biases are initialized to zeros."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swHwLXOI9E7V"
      },
      "source": [
        "# LSTM \n",
        "You are making this network, please note f, i, c and o (also \"v\") in the image below:\n",
        "![alt text](http://blog.varunajayasiri.com/ml/lstm.svg)\n",
        "\n",
        "Please note that we are concatenating the old_hidden_vector and new_input."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0DBzNY-90s5"
      },
      "source": [
        "# Quiz Question 4\n",
        "\n",
        "In the class definition below, what should be size_a, size_b, and size_c? ONLY use the variables defined above."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SFuHhqVq6Wge"
      },
      "outputs": [],
      "source": [
        "size_a =  Hidden_Layer_size\n",
        "size_b =  z_size \n",
        "size_c =  X_size\n",
        "\n",
        "class Parameters:\n",
        "    def __init__(self):\n",
        "        self.W_f = Param('W_f', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_f = Param('b_f', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_i = Param('W_i', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_i = Param('b_i', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_C = Param('W_C', np.random.randn(size_a, size_b) * weight_sd)\n",
        "        self.b_C = Param('b_C', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_o = Param('W_o', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_o = Param('b_o', np.zeros((size_a, 1)))\n",
        "\n",
        "        #For final layer to predict the next character\n",
        "        self.W_v = Param('W_v', np.random.randn(X_size, size_a) * weight_sd)\n",
        "        self.b_v = Param('b_v', np.zeros((size_c, 1)))\n",
        "        \n",
        "    def all(self):\n",
        "        return [self.W_f, self.W_i, self.W_C, self.W_o, self.W_v,\n",
        "               self.b_f, self.b_i, self.b_C, self.b_o, self.b_v]\n",
        "\n",
        "parameters = Parameters()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RzmfGLZt_xVs"
      },
      "source": [
        "Look at these operations which we'll be writing:\n",
        "\n",
        "**Concatenation of h and x:**\n",
        "\n",
        "$z\\:=\\:\\left[h_{t-1},\\:x\\right]$\n",
        "\n",
        "$f_t=\\sigma\\left(W_f\\cdot z\\:+\\:b_f\\:\\right)$\n",
        "\n",
        "$i_i=\\sigma\\left(W_i\\cdot z\\:+\\:b_i\\right)$\n",
        "\n",
        "$\\overline{C_t}=\\tanh\\left(W_C\\cdot z\\:+\\:b_C\\right)$\n",
        "\n",
        "$C_t=f_t\\ast C_{t-1}+i_t\\ast \\overline{C}_t$\n",
        "\n",
        "$o_t=\\sigma\\left(W_o\\cdot z\\:+\\:b_i\\right)$\n",
        "\n",
        "$h_t=o_t\\ast\\tanh\\left(C_t\\right)$\n",
        "\n",
        "**Logits:**\n",
        "\n",
        "$v_t=W_v\\cdot h_t+b_v$\n",
        "\n",
        "**Softmax:**\n",
        "\n",
        "$\\hat{y}=softmax\\left(v_t\\right)$\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-bUkseNnDott"
      },
      "outputs": [],
      "source": [
        "def forward(x, h_prev, C_prev, p = parameters):\n",
        "    assert x.shape == (X_size, 1)\n",
        "    assert h_prev.shape == (Hidden_Layer_size, 1)\n",
        "    assert C_prev.shape == (Hidden_Layer_size, 1)\n",
        "    \n",
        "    z = np.row_stack((h_prev, x))\n",
        "    f = sigmoid(np.dot(p.W_f.v, z) + p.b_f.v)\n",
        "    i = sigmoid(np.dot(p.W_i.v, z) + p.b_i.v)\n",
        "    C_bar = tanh(np.dot(p.W_i.v, z) + p.b_C.v)\n",
        "\n",
        "    C = f * C_prev + i * C_bar\n",
        "    o = sigmoid(np.dot(p.W_o.v, z) + p.b_i.v)\n",
        "    h = o * tanh(C)\n",
        "    \n",
        "    v = np.dot(p.W_v.v, h) + p.b_v.v\n",
        "    y = np.exp(v) / np.sum(np.exp(v)) #softmax\n",
        "\n",
        "    return z, f, i, C_bar, C, o, h, v, y"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jZrDhZIjFpdI"
      },
      "source": [
        "You must finish the function above before you can attempt the questions below. \n",
        "\n",
        "# Quiz Question 5\n",
        "\n",
        "What is the output of 'print(len(forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)), parameters)))'?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7153-pJItkEo",
        "outputId": "e9f631b2-f11a-4e43-c271-a3a522cc2166"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9\n"
          ]
        }
      ],
      "source": [
        "\n",
        "\n",
        "print(\n",
        "    len(\n",
        "      forward(\n",
        "        np.zeros((X_size, 1)),\n",
        "        np.zeros((Hidden_Layer_size, 1)),\n",
        "        np.zeros((Hidden_Layer_size, 1)),\n",
        "        parameters\n",
        "      )\n",
        "    )\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XV-YVl_GGiX8"
      },
      "source": [
        "# Quiz Question 6. \n",
        "\n",
        "Assuming you have fixed the forward function, run this command: \n",
        "z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))\n",
        "\n",
        "Now, find these values:\n",
        "\n",
        "\n",
        "1.   print(z.shape)\n",
        "2.   print(np.sum(z))\n",
        "3.   print(np.sum(f))\n",
        "\n",
        "Copy and paste exact values you get in the logs into the quiz.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1GvKVWmTDt3H"
      },
      "outputs": [],
      "source": [
        "z, f, i, C_bar, C, o, h, v, y = forward(\n",
        "    np.zeros((X_size, 1)), \n",
        "    np.zeros((Hidden_Layer_size, 1)), \n",
        "    np.zeros((Hidden_Layer_size, 1))\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RMcwQJnNaJHx",
        "outputId": "bc42b6a4-42db-4c82-9c94-e9f4bc3385af"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(85, 1)\n",
            "0.0\n",
            "5.0\n"
          ]
        }
      ],
      "source": [
        "print(z.shape)\n",
        "print(np.sum(z))\n",
        "print(np.sum(f))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NeSvhkqwILsG"
      },
      "source": [
        "# Backpropagation\n",
        "\n",
        "Here we are defining the backpropagation. It's too complicated, here is the whole code. (Please note that this would work only if your earlier code is perfect)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zIa1jUZiGPmF"
      },
      "outputs": [],
      "source": [
        "def backward(target, dh_next, dC_next, C_prev,\n",
        "             z, f, i, C_bar, C, o, h, v, y,\n",
        "             p = parameters):\n",
        "    \n",
        "    assert z.shape == (X_size + Hidden_Layer_size, 1)\n",
        "    assert v.shape == (X_size, 1)\n",
        "    assert y.shape == (X_size, 1)\n",
        "    \n",
        "    for param in [dh_next, dC_next, C_prev, f, i, C_bar, C, o, h]:\n",
        "        assert param.shape == (Hidden_Layer_size, 1)\n",
        "        \n",
        "    dv = np.copy(y)\n",
        "    dv[target] -= 1\n",
        "\n",
        "    p.W_v.d += np.dot(dv, h.T)\n",
        "    p.b_v.d += dv\n",
        "\n",
        "    dh = np.dot(p.W_v.v.T, dv)        \n",
        "    dh += dh_next\n",
        "    do = dh * tanh(C)\n",
        "    do = dsigmoid(o) * do\n",
        "    p.W_o.d += np.dot(do, z.T)\n",
        "    p.b_o.d += do\n",
        "\n",
        "    dC = np.copy(dC_next)\n",
        "    dC += dh * o * dtanh(tanh(C))\n",
        "    dC_bar = dC * i\n",
        "    dC_bar = dtanh(C_bar) * dC_bar\n",
        "    p.W_C.d += np.dot(dC_bar, z.T)\n",
        "    p.b_C.d += dC_bar\n",
        "\n",
        "    di = dC * C_bar\n",
        "    di = dsigmoid(i) * di\n",
        "    p.W_i.d += np.dot(di, z.T)\n",
        "    p.b_i.d += di\n",
        "\n",
        "    df = dC * C_prev\n",
        "    df = dsigmoid(f) * df\n",
        "    p.W_f.d += np.dot(df, z.T)\n",
        "    p.b_f.d += df\n",
        "\n",
        "    dz = (np.dot(p.W_f.v.T, df)\n",
        "         + np.dot(p.W_i.v.T, di)\n",
        "         + np.dot(p.W_C.v.T, dC_bar)\n",
        "         + np.dot(p.W_o.v.T, do))\n",
        "    dh_prev = dz[:Hidden_Layer_size, :]\n",
        "    dC_prev = f * dC\n",
        "    \n",
        "    return dh_prev, dC_prev"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tnc7WpRkIU5S"
      },
      "source": [
        "# Forward and Backward Combined Pass\n",
        "\n",
        "Let's first clear the gradients before each backward pass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OJWoC3U1ITf8"
      },
      "outputs": [],
      "source": [
        "def clear_gradients(params = parameters):\n",
        "    for p in params.all():\n",
        "        p.d.fill(0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7XN93UnjIgmA"
      },
      "source": [
        "Clip gradients to mitigate exploding gradients"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0LTsublxIfFl"
      },
      "outputs": [],
      "source": [
        "def clip_gradients(params = parameters):\n",
        "    for p in params.all():\n",
        "        np.clip(p.d, -1, 1, out=p.d)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T7XUpDTWIl_Y"
      },
      "source": [
        "Calculate and store the values in forward pass. Accumulate gradients in backward pass and clip gradients to avoid exploding gradients.\n",
        "\n",
        "input, target are list of integers, with character indexes.\n",
        "h_prev is the array of initial h at  h−1  (size H x 1)\n",
        "C_prev is the array of initial C at  C−1  (size H x 1)\n",
        "Returns loss, final  hT  and  CT"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CQNxjTuZIia_"
      },
      "outputs": [],
      "source": [
        "def forward_backward(inputs, targets, h_prev, C_prev):\n",
        "    global paramters\n",
        "    \n",
        "    # To store the values for each time step\n",
        "    x_s, z_s, f_s, i_s,  = {}, {}, {}, {}\n",
        "    C_bar_s, C_s, o_s, h_s = {}, {}, {}, {}\n",
        "    v_s, y_s =  {}, {}\n",
        "    \n",
        "    # Values at t - 1\n",
        "    h_s[-1] = np.copy(h_prev)\n",
        "    C_s[-1] = np.copy(C_prev)\n",
        "    \n",
        "    loss = 0\n",
        "    # Loop through time steps\n",
        "    assert len(inputs) == Time_steps\n",
        "    for t in range(len(inputs)):\n",
        "        x_s[t] = np.zeros((X_size, 1))\n",
        "        x_s[t][inputs[t]] = 1 # Input character\n",
        "        \n",
        "        (z_s[t], f_s[t], i_s[t],\n",
        "        C_bar_s[t], C_s[t], o_s[t], h_s[t],\n",
        "        v_s[t], y_s[t]) = \\\n",
        "            forward(x_s[t], h_s[t - 1], C_s[t - 1]) # Forward pass\n",
        "            \n",
        "        loss += -np.log(y_s[t][targets[t], 0]) # Loss for at t\n",
        "        \n",
        "    clear_gradients()\n",
        "\n",
        "    dh_next = np.zeros_like(h_s[0]) #dh from the next character\n",
        "    dC_next = np.zeros_like(C_s[0]) #dh from the next character\n",
        "\n",
        "    for t in reversed(range(len(inputs))):\n",
        "        # Backward pass\n",
        "        dh_next, dC_next = \\\n",
        "            backward(target = targets[t], dh_next = dh_next,\n",
        "                     dC_next = dC_next, C_prev = C_s[t-1],\n",
        "                     z = z_s[t], f = f_s[t], i = i_s[t], C_bar = C_bar_s[t],\n",
        "                     C = C_s[t], o = o_s[t], h = h_s[t], v = v_s[t],\n",
        "                     y = y_s[t])\n",
        "\n",
        "    clip_gradients()\n",
        "        \n",
        "    return loss, h_s[len(inputs) - 1], C_s[len(inputs) - 1]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tcy5u_vRItkV"
      },
      "source": [
        "# Sample the next character"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p8SrtJiwIsSm"
      },
      "outputs": [],
      "source": [
        "def sample(h_prev, C_prev, first_char_idx, sentence_length):\n",
        "    x = np.zeros((X_size, 1))\n",
        "    x[first_char_idx] = 1\n",
        "\n",
        "    h = h_prev\n",
        "    C = C_prev\n",
        "\n",
        "    indexes = []\n",
        "    \n",
        "    for t in range(sentence_length):\n",
        "        _, _, _, _, C, _, h, _, p = forward(x, h, C)\n",
        "        idx = np.random.choice(range(X_size), p=p.ravel())\n",
        "        x = np.zeros((X_size, 1))\n",
        "        x[idx] = 1\n",
        "        indexes.append(idx)\n",
        "\n",
        "    return indexes"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SiWFaWLNIx_L"
      },
      "source": [
        "# Training (Adagrad)\n",
        "\n",
        "Update the graph and display a sample output\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ENQYU-7AIw0t"
      },
      "outputs": [],
      "source": [
        "def update_status(inputs, h_prev, C_prev):\n",
        "    #initialized later\n",
        "    global plot_iter, plot_loss\n",
        "    global smooth_loss\n",
        "    \n",
        "    # Get predictions for 200 letters with current model\n",
        "\n",
        "    sample_idx = sample(h_prev, C_prev, inputs[0], 200)\n",
        "    txt = ''.join(idx_to_char[idx] for idx in sample_idx)\n",
        "\n",
        "    # Clear and plot\n",
        "    plt.plot(plot_iter, plot_loss)\n",
        "    display.clear_output(wait=True)\n",
        "    plt.show()\n",
        "\n",
        "    #Print prediction and loss\n",
        "    print(\"----\\n %s \\n----\" % (txt, ))\n",
        "    print(\"iter %d, loss %f\" % (iteration, smooth_loss))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ACXcASJuI73a"
      },
      "source": [
        "# Update Parameters\n",
        "\n",
        "\\begin{align}\n",
        "\\theta_i &= \\theta_i - \\eta\\frac{d\\theta_i}{\\sum dw_{\\tau}^2} \\\\\n",
        "d\\theta_i &= \\frac{\\partial L}{\\partial \\theta_i}\n",
        "\\end{align}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bR08TvcjI4Pf"
      },
      "outputs": [],
      "source": [
        "def update_paramters(params = parameters):\n",
        "    for p in params.all():\n",
        "        p.m += p.d * p.d # Calculate sum of gradients\n",
        "        #print(learning_rate * dparam)\n",
        "        p.v += -(learning_rate * p.d / np.sqrt(p.m + 1e-8))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "La9vyJ6RJLFK"
      },
      "source": [
        "To delay the keyboard interrupt to prevent the training from stopping in the middle of an iteration\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZVDHbMb7JNGT"
      },
      "outputs": [],
      "source": [
        "# Exponential average of loss\n",
        "# Initialize to a error of a random model\n",
        "smooth_loss = -np.log(1.0 / X_size) * Time_steps\n",
        "\n",
        "iteration, pointer = 0, 0\n",
        "\n",
        "# For the graph\n",
        "plot_iter = np.zeros((0))\n",
        "plot_loss = np.zeros((0))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HF6vS0VWJqsS"
      },
      "source": [
        "# Training Loop"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 422
        },
        "id": "OQyNSL0iJOxH",
        "outputId": "8e4c09f2-4bc5-474d-e606-7dfb1de5add8"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAD1CAYAAACWXdT/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deYAUxfXHvz2zuywLLJcLCCIgooAgeICgoFyioMYjUZQrJt4oAQ0iIiHml0QQj4AY0XhgvFEwaCICAV1Fg4uAoiDXosi5ywILu7DnzPTvj5nqqa6u6q6eY2dmpz7/7E53T3VNdderV6/ee6Xpuq5DoVAoFEmNJ9EVUCgUCoUzSlgrFApFCqCEtUKhUKQASlgrFApFCqCEtUKhUKQAGfEotKqqCps3b0ZeXh68Xm88bqFQKBT1Dr/fj5KSEvTo0QPZ2dmmc3ER1ps3b8aYMWPiUbRCoVDUe958801ceOGFpmNxEdZ5eXnGDdu0aROPWygUCkW9o6ioCGPGjDFkKE1chDUxfbRp0wannXZaPG6hUCgU9Rae+VgtMCoUCkUKoIS1QqFQpABKWCsUCkUKoIS1QqFQpABKWCsUCkUKoIS1QqFQpABKWCsUKUZJeTU6TvsIq7cWJ7oqijpECWuFIsXYfOA4AOC1tT8nuCaKukQJa4VCoUgBlLBWKBSKFEAJa4VCoUgBlLBWKBSKFEAJa4VCoUgBlLBWKBSKFEAJa4VCoUgBlLBWKFINPdEVUCQCJawVCoUiBVDCWqFQKFIAJawVihRF0xJdA0VdooS1QqFQpABKWCsUCkUKoIS1QqFQpAAZThcUFBRg0qRJ6NKlCwDgrLPOwu23346pU6fC7/cjLy8PTzzxBLKysuJeWYVCoUhXHIU1APTt2xfPPPOM8fnhhx/G6NGjMWLECDz99NNYvHgxRo8eHbdKKhQKRboTkRmkoKAAQ4cOBQAMHjwYa9eujWmlFAqFGF1FxaQlUpp1YWEh7r77bhw/fhz33XcfKisrDbNHy5YtUVJSEtdKKhQKRbrjKKw7duyI++67DyNGjMDevXsxfvx4+P1+47yuq1FeoVAo4o2jGaR169YYOXIkNE3D6aefjlNOOQXHjx9HVVUVAKC4uBitWrWKe0UVCoUinXEU1h9++CFefvllAEBJSQmOHDmCG264AStWrAAArFy5EgMHDoxvLWPAobIqXDn3cxw4VpnoqigUCoVrHM0gQ4YMwZQpU7B69WrU1tbi0UcfRbdu3fDQQw9h0aJFaNu2La677rq6qGtUvLt+L7YVlePNgp/x4BVdE10dhUKhcIWjsG7cuDGef/55y/GFCxfGpUIKhUKhsKIiGBUKhSIFUMJaoVAoUgAlrBUKhSIFSBthrdzBFQpFKpM2wlqhqG+ovQfSi7QR1mpXDYVCkcqkjbBWKBSKVEYJa4VCoUgBlLBWKBSKFCBthLWsN8g/Pt+FFz7bFd/KKBQKhUuk8lknml0lJ9C4QQZa52bH/V6PLdsGALjrss5xv5dCoVDIkhLCeuhTnwEAds++KsE1USgUisSQNmYQ5bqnUChSmbQR1gpFfUFF46YnaSOs3b7garsyhUKRTKSksN57tCLiHV80ySDdylq/80UKhUJRR6SksB4451NcPPuTuN6jskYJa4VCkTykpLCOBh1y5g1lBFEoFMlE2ghrt94gymStUCiSibQR1gqFQpHKpI2wdu0NogwhCoUiiZCKYKyqqsLVV1+NCRMmYN26ddiyZQuaNWsGALjtttswaNCgeNYxpsh6gyhZrVAokgkpYb1gwQI0bdrU+PzAAw9g8ODBcatUMqBktSLZ0VRYblrhaAbZtWsXCgsLU0p7tkPaG0RJa4VCkUQ4CuvHH38c06ZNMx174403MH78eNx///04evRo3CoXS5QSokhnqlSQV8pjK6yXLl2K3r17o3379saxa6+9FlOmTMFrr72Gbt264dlnn417JWOBWmBUpCurfihG1z8sx6a9xxJdFUUU2Arr/Px8rF69GjfddBPee+89PPfcc9B1Hd26dQMADBkyBDt27KiTisYK2QVGZQZR1Bc+21ECANi0TwnrVMZ2gXHu3LnG//Pnz0e7du3w9ttvo3379mjfvj0KCgrQpUuXuFeSsHxzEQadnVcn91KyWqFQJBOuNx8YM2YMJk+ejIYNGyInJwezZs2KR7243P3GBvy6f4c6uZfKuhc7hv/tMzTLycK7d/VPdFXqBZG+muqVTm2khfXEiRON/5csWRKXysiwrzSybHsEZYuue3YUn0h0FdIatbheP0i6CMbCQ+VYvrlIeL6uRK3SQhQKRTKRdML69bU/Y8bS7+NWvnQEo0KhUCQRSSesAcAXSLxaqzRrhUKRTCSdsNY0DQEbYV1XC3/Ktp0a7D9Wifmrd6blgrCaI6YXSSislVarkOfu1zfgqf/uwI+HTya6KgpFXEk6Ye3RtLjqtCo3SP2iosYHQLlaKuo/rv2s441HAwJJ0PESX4P0oaS8Gk0bZiIrQ1532HqwzNX1CjWgpTpJKKw1W2Ed7esmH26uXuy6os9fV+HKc9rg+XEXSH9nxLw1AIAz8hrFq1r1BmXbrh8kn2qiAUngDKI06zpm+Raxb70cSiQp6jdJJ6w9mqYkZRoRuxmMemkU9ZskFNZJYrNOfBXSgmjbWenTinQhCYW1g8060iQ2MfrG8cpaPPrhFlT7VDL3WBDtwKzGVEW6kHTCWtO05LBZC+rwt//uwKv/243FG/bVbYWSDJ8/gPfW77UNYJIhdo86fXTsJOgeigSQdN4gpMvpuh7TDUHdliTqELX+AIDkWARNJC+u+QmPL98GHcBNF7Z3vF6EMjfVHaqpU5uk06w9IQEt6sTqhUsOjpyoBgAcq6iJqhwV1h9/1C7o9YMkFNbBv4leZFQanz2x6v+qnRUKOZJPWIektRszg88fwPai8pjWw1HjU1JGoVDUIUknrLUINOs5K7bjirmf48cS8Y4kbkWrzO2Xby5C4SG1C0o0JHoGpVCkCsknrGFvs+ax4edSAMDRk9HZT2kc769puPuNDRj29Gcxu2c8OFntw+GQfTkZiVpWK1mvSALeXrcH97yxIa73SDphTWzWbhaeiHZmZ0dN1yWWkc+swYV/WRXzcjWHhWBZZL++9Jv9uGb+Fzb1ia4eCjkKD53AN3tKE12NpOPh97/HxzbbEcYCKde9qqoqXH311ZgwYQL69++PqVOnwu/3Iy8vD0888QSysrJiViHiDSKyWfPCk8OHYtdj64vN+ucjFXEp13CxjLIc2XDzyYu+dSgnyooopCAzyd2zr0pwTdIPKc16wYIFaNq0KQDgmWeewejRo/HWW2+hQ4cOWLx4cUwr5GSz5h0mhzwSsvpEtQ93vrYeh8qrbK+j7+MP6Ph27zFT/RTu0XUdz3+2C/uPhXeoF8nYnw6fxPGKWldlK+xRTZTaOArrXbt2obCwEIMGDQIAFBQUYOjQoQCAwYMHY+3atbGtEJleB/jneUKcdFSPjSQl33p/4z6s/KEY81cXStdp/ic7cd3fv1TTvyjZe7QSsz/ehjv+ud44Rp4z++gGP5mPkc+skS47neSQGpjSE0dh/fjjj2PatGnG58rKSsPs0bJlS5SUlMS0Qk6aNX18w89HTcfcaL1urv3hQBkAoLisWmknUeAPNR7Z3QWwNzfRGrhxvYsZV32F/FQ1y0ssH3y7H7vrcDs5W2G9dOlS9O7dG+3b88OJ4zHCG5q14B70x18uWGs6JrOxgGyN49n5tx4sQ2VN+iaCopvWbTuLI1vTSForkoJJ73xrbIJRF9guMObn52Pv3r3Iz89HUVERsrKykJOTg6qqKmRnZ6O4uBitWrWKaYXYCEa2c/LNIMG/Ut4gOvNZgKjzR6vNlFfVYsS8Na53RklW6sobRHR9eFCPrh6pRDr91mSnsrbulC5bYT137lzj//nz56Ndu3b45ptvsGLFClx77bVYuXIlBg4cGNMKaYY3SEhYM+d5XiIBgc26xhew3aevqtaPqYu/w0MjuqJds4amc/wOET5YUh6Z73JVbdBIuz5kwklZIhi0eF9xOzsLXh8u6afQNFQF1yjqO679rCdOnIilS5di9OjROHbsGK677rqYVojIW9L3rGYQOc165ZYinDXjY8PezOvKn2w7hA83HcCf//2DtUyHej7zifwCJY/6IluiNT9Eq1kbx+tJe8qRVj9WEUI6RerEiRON/xcuXBiXygDWrHvsa+nnuu5ZFxhXbz0EANi07xi6t821fEfTNCnzSaypL4tCshsPO2EsDkten15CObaopkttkjaCUWSzttOs6VOGAGevBecajqSg71NfBGxS4naBUfCFdBLibn+ren/rB0knrK02a7E3CIEn2EVaMy2E7bxI0qjvR0WsFhhlcy7H0htk3qqd6DjtI9ffUygS4euedMKamEH+V3gEgDtvELrDGkIgJIh54dF2UerxfhapPhgkWz7rSHbu+duqHbG5eR2T6u9OfSARM7mkE9ZEBkxd8h33vJ/TK3nuW3pYWpuvpTRu3dZeqrqEW3z+AJ79ZCdOVvucLw7hViMWatZR9J5o95GMN1fO/RyPLdtqfE4nk0+ykohHkHTC2uNQI25uEJ4ZhLFZs1+LZIFM16NfWItlR9N1nTt4JYp/f3cAT67cgadWymusroNiRDZrd8WY8Neh9Kuq9bseWLYVleMfn/8YpxopIkGZQWD1lbYsMHK6JZFXpnOGBm0uj/5IyublFIlXpBz5vtPDPlReZSSPEvHC5z+i8/RlOF4pn/AonhAfcjeaday8QaLpO3U14JWerEHXPyzHc/m7oipHRWsmnkToSEknrFnhyr6YvI4l8hwBON4gPO2bZ7Pm1o1z0C2SD3nE3DW47u9f2l7z7td7AUQeoBMNsXIwcK9Zuz/jhK+Oet6h0HP64Nv9dXI/Fl3XsXJLkUoEFQMSMWAmn7BmPltd96zfIcfoxcewl4HgPhrtDSIu0+mYW2TlwpEY7noTDXuOVOCBRd+i1m9OgxgOXqJcHCXLjKYdRYImEnlL3ET9POf9OBCeRSTGlPZWwR7c+foGvLdhX1T3jwW6rqM0Sd7xSFALjBDnfhB9BiibNeeYxXWPuopo6ZqmYc3OEqzffdTyfcC5c/n8ATyXX4gqiTwBhhnE8crEcaisysj3/dCS7/D+N/ux7qfow+P5/uy8Y+7150g6jzckrX0BQT7eOBHpDO2lNT/irEc+dn+/0Pt78HjwmRYdt8/l7oZAIDJtfcFnu3Den/+LA5zMigo+SSes2ZV59iXguu4Z11qPWa4NnThZ7TOmvxqAcS+vw6+e5+fmpgU8T3Av3rAPc5Zvx/xPdgruar1/TGCzU8WIvo+tRt+/rg6V7M6m7BaeKcpOS46lNwhZq6grm3W0z/4vH21FjT8Q8dOOR3DM61/9jDtf34B/fePOtLPqh2IASIiwPlntw8Pvf4/yqsjXepRmDWvHYdvEzmZNXy0KeCEv7Dtf74WPTO0lbdYiSOatE1XhhbUaXwCHyqwaTKolHBK5QBK+KDwcnducS81a9GAiqQHRrGvrSlgbA1O0ZpAoF7lj+HOJtn7QpbbOpkKuS1793268vW4PXvgscg8bZbOG1Y3KEhTD87M2bNbUsdBfawRj+P9aG1slfR0t8HkPiffiPbTkO/R9bDVqfOYpdorJaktwEctXPx7F+xvNWpWbkPCwrzvdxnb1kS/bCS/RrOvIZm23RhIPPt1+CB2nfYQ9R0+a7htLQeMNSRC3vurGJiMJcKsIm00jv7fSrMF5eMxHnk+s4bon0YD0JcRWyQ83l38avN1tlod2Oo7EHsrTyLn3dV1yBBABw9yMbrPCkhPcawRFOR6LZPYRkRkkxWzWhIDTdCfE0pBpYuMesn9obHakp3Ha4FqElkDNOhYkot5JJ6xZNypWaPJfitBIacr74TzlJJq1n9dZBU+DJ9gNjUXCg8TOzZAwNLSDdDIgY7Nmd71xY1d263Uj9n93DzGDpIrNmhDp2BKrHelNZTK5fNzWJZFmwXh4JcWTpBPWdtt4ASKbdeha+nuhvxYhYzKDhII4ot1iy4WWIPOMy6vkg0pky4yUcHi+WFzLeMEAovaRE+D2ZUTWBkQrjJWf9fC/fYZZH28VnicDn93Gzou+3oMZS7+3vU/EEZdxmIp5IxTWRhukqGqtgmLgvMDI+vsCYQFvemFYa4oh0MMnfCHN+gRHOMpO2YGwv67M+xpLTSLahSoAePq/O/De+r3C8yLbP/2Z3dpIVC1ZLdrOBCXcMDeCXk/srbHSrHcUn7BdtAoYA5+4jIeWfI83vtpjex+3Wp1VYYm9zdptG4ZNhzGrimt4z6HoeBU6TvsI79r0CQDihe44ak7JJ6yp3xoI6JYfb6dZ1/p1VPuCgoN1CeN1ZiL4yXdo3LQ5MY3wcmCzwjmRisSWA8fRcdpH2Lin1Dj2zOqdeHAxP2kWQC8AipE1g/B+vfE4ZV33RMeTQLOWJdohNtrqxscM4u574UX55FKtCw8F11+WOrgiJiKvetIJa3qBcd7qnfh+/3HTeR9n5Z4IlF+/sg5nz1geOhY8Fxak1ntVhzw1anjaegQLjLx7WNZL4/AwZYvM314CAPhvyMfVTdl22iDpeE7BQ7JatG1QjOBUJDOWsJ913SwwxkrrinR2Zre2EinE7s+r047icrzwGT8PSjJo1nbt4LhYrswgZi1n3uqduHXh16bzXMFqM71mTWNm171gWZv3l9mW6fTg7FyiZPaQjJRINTQ33xPZrOlPXi+bz0VQll351DF7zVrW8u2MEcHIUQD2Hq0I++HHiPDAF51uHamA0+KgzRruj5xKXfvsl5j18Taue17YM6XupZ7b9n/3673oOO0jHDkRzsHj5h2PFUknrCNy2+IeY6bvnJB01gdaWL5Dlew0a/ZYeNCQ+53sdfuPVeKddfY2zVgiXKil8DIv/89HTuJtTh2lPT/smiaGvUTkDXKorAoD53yKx5Ztc1+oDaIUCJGWkwx4bDRrdi2DJh5aviz26Qys594Mvct7jlY4lpFWNuuGmV7X33Ej4OlreVo6wU2TE+8Nus+LXJPcPktWKRnz4leYxoTKypYZ0Ysk8R0i9Ahf7y7Fw+9bPRq4Mw+uHduuM4mOR2IGCf5lbdYk5exnOw5ZvlPrDxih0pEiI6s7TvvIsJ+yJMtmCX3+ugovrwkuqNrViXfGbi0pGXA06bk8HgschXVlZSUmTZqEsWPH4sYbb8Snn36KadOm4ZprrsG4ceMwbtw45Ofnx6xCY/t1cP0dO42Ndeujr7XTrEUb5vK0or98tDV0D5sFNOOzu8fJCtjDJ2pC9xIvYsYSmam76IzIDdO0/RrXjm1TH8FJt00w+Ml87CoJRvaxQTF2/tfzPynE7a+tx2c7SoRlFx2vwqfbrILebR1X/lDEPR65GUS+Hmt3HcHXu+2Td5WUV+NAKMzczp2QvAfbisrw8PvfIxDQE6pZ2yHrNhqr99ANGU4XfPrpp+jRowfuuOMO7N+/H7/97W9x3nnn4YEHHsDgwYNjXqGsDPfKvl1yp3vf2ojepw/hNiLPDZD9vuW4yym6aMNf2Wcqus5s43UuTdd1ix1fRkPj2ZRZ/IKLdJ2/2QO3fJM3iLtpavA7NhXk8NPhk8b/bNqBzJA/Gi8dwb7S4FTYLof4Dc99iQPHq7B79lWm46Q0Oz9rFt5syHUASuh2boJibnnxKwCw/AYRduZ9cr/b/7ke+0orcc9lnSOOfIyE3YdP4mhFDc4/vTkAe8UjrJyEj/EGlli6kMriKKxHjhxp/H/w4EG0bt06bpWJFLuFKwDBFI6cq6olbdZu67HlwHHjf39AR3lVLZpkZ4aucatZ29/L7hpROWSKZ2cGCt9HUDj1Not8bGsDATyxbLttXfiLsjb1EWo0kXcS9ruaYR6xtk8GsdHaSJkDgqRG5DtubNayAxwPkfYaaVPZe+nYnQvVhzNDrQv7+6An8wHIDzwsspuTAPHVrKXV2JtvvhlTpkzB9OnTAQBvvPEGxo8fj/vvvx9Hj0af6zga+A/cPNV2q1lHMkDquo4NP5fiqme+QEXI93j2x9vQ89GVOBHa6sq9zVq0kMH/X1g3zjEpYW2YkxiBJqgLzdpdR/DSFz9RdZATzBHlsxZ+wxmReyXPSyScA1tuNmN3H0JVrR/vb9wnrUVH7LoXdbY/8Tm3s6FI/bOThaR23XvnnXewYMECPPjgg7j22msxZcoUvPbaa+jWrRueffbZeNbREScvAx0CYWVnsxYIFrv3PaCHp8mED749ACC8aGXUK8qHreu6oSGLOsrCL8OCMqBbf5GMN4yM2UakWbPCTtY+bW+zdh683GKJmiXCmvO7wvZs+bYLfw5p1oy96PHl2/DAu5ssdnBd5wszGQGn67rJ1GM6x7Twe+v34oudhx3LrLX5zbZ6D6e+4RaQe3A/HT4ZkRYu6/Fl1EbWSysZg2I2b96MgwcPAgC6desGv9+Ps846C926dQMADBkyBDt2yO9mHQ/sbNaA+AFIp0iVXMjTIRZcZAocrRkkHFpvvi+PP/37B2455PfYziwQ1Ph+OFhm+v6SDfvw+HKzS5tocUkmh4Vbm6z4VOS9hL0f+czzs7bzK3aqEWkPdsAvDmVZPFnNiaR16S1DeHHNj9i0zxxQFnZjNV/74OLvMPblAscy7fpLpLlBZAaeNTtLMPjJfNcbHADBdAqRwJ+FmGfrPOJps3YU1uvXr8crr7wCADh8+DAqKiowc+ZM7N0bjJ0vKChAly5d4lZBGbjamYTLnL03iPV/XVAOfU9RJ2a9U2QRdQL6uNQCI3RL3Z20jtIKeo+84Jd//94mLGB25zZssWwdBRqruV4IfVezvc6JaDQay4AY+svblID4FdNy6y//+QE9/7jCcq11EAj+ZeUAUVg9PNuoSzMRYePPxyzHos2dVGvzvrgdYN14pmwvKgcAbDlgDV5z4uBxdzvR8KpjL7aZ43HUrB0XGG+++WY88sgjGD16NKqqqjBz5kzk5ORg8uTJaNiwIXJycjBr1qz41VACp+m1rvNHPF5OEPv7OGvWQhtz6P5up3KiqwO67mqRxrzAGMRJWNPfYeUWLXBEGrQ1NTnPtGTVNiPRrEWD4NaDZbj2718if8ogtG3WkHsNO8DaadYZlBlk9sfbsHprMXaK/KEFGrvoOp4yx7dZc4uRILqoQbuZmN1Mw27DDhlFg5TNG8yciJfwFHuDxA9HYZ2dnY2nnnrKcnzJkiVxqVA80KFzW9HWDIJgMpfJi75Fn45Blx8nm7Wu68KO5DfMIO6QsdH6A8Aj//oeYy7qgO5tc6XLtvv9gHXAEyE0/UjMbmSPhetkPxiyvFWwBzW+AFZtLcb4/h0l62k1NREMzToAPC/IexEuh/1MZiCi8Hzry8Vri8j96qMTJXYL0hFr1hL3JcqAJxJpbUM0rSE0g8RRtU66CMZYwWqFvCa0c93TdR1vFQTDTH8MBU842aOCC0L8a8hiFW8ndjuchD8AFJVV4c2CPfjNq+uoulgFJVt/J7srrVnKuWaZO5NMxkG+CUtcJ3En4R+3y/8h+q7ZBGY+6SZ/M3sNaU4P0+vIPXiyKFLN2i6QK1J5Qg/uMtkwjWstn3VXuUHIJW780yOGvMtOl8VThRaQ8sL6c04kWVDDpV8s/kvhFBTDjuhOD0jXxf635F6uH7Jw2i/uOMHzbDG0HYS4n9mbQej2sau2vGbNM4NYv2ebz9rlcZnpNmvGoX9OcZk5+CVDQvjzyqHrYNGsbYpyvaGwBJF+2zx4m88F9GBOlZPVnNzwzO/WdXc54Mn7xeagiRa70ni32vBzqZHmIZYJxWRJeWE9/pV1lmOdHl6GNZQrkqhhff4A+p/RUnBOx7aQJ4RXcuVah3iBkXRu97La2QwiswgVHLDM1zhq1tR5q581FRRDZgsCTdKoA+ceMgON0/V2x2WS49vZlvvNWo0lG/YZn8MLjDIaofNgBTi8ExGaQXiXlFVG5utPqLEZvAMBHX0fW41rnv3CWhf2Wj0cbu5mhsLOPK742+f4zUJr/7e7t+w5GjILeGzZNkx4c2Pwuy5neLEg5YW1DDxBBQSFUavcBtzvzFmxzdjuy+uRm7IFdLOXAM1z+YVBjd/l6pDocp43iN2CIK8Yp8AOk/bIXGpaEAxY708fN4rgCp5QeabrItCsBSdkhKtT/hba/SvDhZ+1tdzgX4s3iM0AFMsFRiJsI3UvszODkE/EZGg6x2kHY/MBiaqQ94i1WW8vLsen28U5WpxYkL8LFTXmmYBT22wO5dcXXqWEdfSIhJVoavXzkXBwS4Y3ejPIii3F2FdaadQj0hSphP98dxDbQi5NMqYEU2Kq0F8nzdpkarG5jghCZ+8HjhmEe0yMUKMRfMsQrnY+wg6DSlFZFR79cAt8/oBpgdEJkV2Xte3bm0E4wlpCWkdiMfjp8Em8+Ll4WzJZs5gFVnHQdeMllNOsg3+jNYM8uWK75ZnM/6SQe60m+N/jYGtPqJ91fYBOYkTjD+hSK8zyWxDxNSFCQFAP+xL5PLFiu+01lkUzznUmMwdHANDnbQWKwNOF1Wa5ZfAGGttGEmmh5s9EY5IxYTl5rfgDOl793278b9cRlwuM/PtY/NHJcY4w4t3lBRuBKsM//7cbAFBWVYtxVDDMqBfW4q/LthqpEVhoP2vroqzdbIgdxOk+FeR/uw7jg2/5QS+x8gZ59tNCbD1YbjrG2tidHivPi+UYFY+gzCBRsvPQCaGglRmtZdM56rqzLdPt4hAtFA6fqObuxC6TP0LXYfwA8pPpqTwvlJjWvJ0W6O58bT1eofKAANYBgHzcV1qJO19bj2qfPxwUQz0HuyaSsRW+t34vus9cgV0lJ2JiBiFU+wK26VOt5UoMVjbH7eriBp4ZAgA++u6gaW1HJKQJZps13wwid386RWrw5OgXCzDpnW+53zfMIDFYYKxiYitYJwNSV1EeFY1jvllAuXDGUVanh7D+4NsDjjZNO8hXA7r9CnJAwiatM38fX76Nu0hq+QKAMS/yQ4J57oASFgiTTZrnc00LJLtfFQgA3+w9ZgkOsROCK38oxjd7jtnasXmITtHHyR6TO4vLqcpo9gEAACAASURBVPBwudzldveo8QWMzhqJsBYFv9gtIkcqq51ysJPy+cf5J8w2a/m6WGZcAZ3a/9L5+4Y3iIS0OnyiGs9/tivcJxwWeWW8enjpUsUzvPiJa8egmPqOzAtAEuLoDoYQHRIeI8x5NnSbVyZbDxbeC2/1cbYXxrW+ANBAfN7uJQzofC8YUWQg/Vl295hwPcR14CFjY3aqJ6HG7zfawcntEYClP/sFGqJdUW77/vLNRThR7cOy7/kbF8ggqo/dnpT2syHzIKXr1OyO+eLRkzVo0SjLXJ/QJTKa9ZT3NiF/ewku6tQC54XyV/PKIuw8dAK6rgs16YoaH77eXWp8dsppojTrOOJm0UJGELtZtNtRXG5zpfV6ETytkZf20zA5wJrmkxed5mM0a5I5MFgGfX++sHYyAwQCfG8QW+El6g7UYfqRErPF3tIKy8p/uJ5MUSLN0hdec5DRCEWpV9k3zm5wcmsGufuNDZjy3iZX3zHqQX6bcLCKbBu88HsXJKCLg2JGvbDW8v2w655zXy0LvaNOCdUI3+49hsWUayZb10OMnz0vp0m0eW1kSRthLXp4rBkk02vzQjjZo+EcdqtT/z+10pqY3+6Wok69t5STrIYVFJzv0W3CyxNCv9iVNX70+tNKbnl+gfnHUVjruqmzlp6swe/f3YSTAqHKKyNcH/4JMhh/9N1BY/s1t/UkVPsDxrUyHhnyuUHIfZ3bMJaI2kw0o7Izg8hEuBLovDZsv+TlWXk1tCDK7vUZrpfVS0Uk13mP7TsqOyE7CxDlcTdtTSf4P9akjbBe++MR7nFWs+7etqmwDKfHoOu6re+yDvNLLeX+ZSqff80/ON4BPx42v/S8PSXpuvKiOenzG38uNZ0zLT4G+L/baeGOzbE9b/VOLNm4D++s22spK/w7xMf9AXMOZ103d3D2N9D1t6s3YfnmgzgU2s5LJijGKevekg370HHaR9gfGmx54fnx1NREBEzvKP8diWQQ2R1yhw16g8D4XxbR8tLAxz/lHDV7mxB4g4pp6zn2esu1Dm68SrOOniLBVkvsaJ1hs+BIb/TJQ9ft7XrVPj93FGf/Z+/plkBAx/XP/c9cN851tPmk2hew3Is+f4jZc9DUoXWdK7xkbNY880UkezDqCAr7wU/mmwQ2/XwbhPb35O3gMm/VTnSc9hGOnKjGTZypOAB8WXgEL4c8XqTyWTOXhPNZB+u0NOSqtv9YZahM5zIiQbYM0rb0bxMN6G6EtQ7dtFO7rofjGyocPFBoRHblorJw33aqlrMpk51lMZq1kxnEvvioSBthXVnLT4fKmkFEUy1AQrOGbpvJ7sq5a0zRcPQLb/e94xW1+HavNT+xCJ4ZgedZQHfEEfPWYPjcz03naeHBCiezZs0fVJyCTX776npTQnny0tsmBRKcCug6CkKzJ3pg4Xn78Kbl//g8uNBLNltwQmZbL1FuFFIjVvjwBjw7oSi72iIzCzBdT/02kalMZI8HgNVbi5mTZl/kgA5kZ3kBAF/uct6hhipGGrEZxL6U8Gm+Bh32s677KU/aCGuRAGDNIHaa9aKv99o+ohpfQM5LgNSJKkyUW1vXgRtf+B+u+/uXjuWRF6u8iiOsKV8WUQRjIWMvpH8L+7toARAQmH9YIcF7Bh9uCm57Bi08zXUrXADzYEQHXNDPl5TK0/DJd2SEMCA347EuqNp/h3c+FiKBW65NG9ODHf0sTG1jY+K67Z/rcag8rO2yC5P0jEo04+UhlaHP4Tzv3TItbrODEHMtL0zeZEZRZpD44Uaz3lZUbjsyn6z2o9Yn97R0mBflqn0BrPqhGHuOVFiu21F8IvS/HDxhzTM3OPmYLqW0Xotm7eCjDfBswfZaokciNFy8wEh7DYSP81wz2SICejiS1W43FBoZM8i6n44yM6ngX00DCn48YskYydsLUtbcUFXrxxkPf8Q9xyvCH9BRXcv/rb9cEDahmZ6zT2wGYT//akHYlDTs6c8s15Krj1EeRk7Q75PTnpHGK2CZAei2Qp/1hbdo1pzvyDgBxIKU8rM+vUUO9hytcL7QBW40a8B+d5WKGp/tpqIs9Ate7Qvg9tfWG3bV8DX86+0gaRxpeN90mgXQSXJYgey0OMleAzjbC3kLnyxi1z2d6mhh7Ye2J9KuY2y9yGN32pCBIKP9T13yHQBgwqDOyM70Gvf9svAIVmwptlzPK1NWUztUVi1sX957M3fVTjz7qTkvBu9esjZr9lWi+2kVMygEZ0HB7x+vrJUa+IL3DP7d8HOpeM9Ihwaj3UUJtDmK/K6Dxyvx0pofccmZp5iudUpApTTrEB4N+Me4C2JaJqt5ZThEyRw4Jp62naj2S0VEEeiXtDpkU2c3RDAvQsqVyzWD6LB4WchO+QF7m7XIhGPVFO3vRzrC9/vF9nlREf5AWIybp7VU+wnKCFARdU6bCBPsFpJZiAZL6iIK63Zrrlj5Q7FhbrDT6HiDAG/zWV4JdPvVUO82W1U3pis6R46u85ULAp0vhNTl6Mka0eUGosXIgC7O30NnOdy8vwx/+WirNRaCswhu500SS1JKWGuahuHntMGdl54RszItZhAHx3u7F+tktU+6swNWzZqHu4cfvLqCkz9Eh44joZdc14NCyc7cwGJnsxbVndVSnTo0aXk24b9M2LsvEBYAHo6dmoa3wEjeA7vAD/Z+sgyc8wnGvlTgOLPgLzDaf+ehxd853p9XRpNsuUm1SLN2s1MMy7iX15meC0+52PBzKTpO+4ibL8TWI8vh3gFd3Kazl2/D/YvMAUWsWZJNQGW5fxxV69QS1qG/sWwQVjh77YJiYL8VWGWtX3oaDZhfGqGwjuC3OrnRPfXfHejyyMeoEHjIcMu0sVmLtV1GwDt0aFGEGh2RJ7I51vgDxnEj/zh0boi+nRnEaRNh436S1wFAWZUPXxQedjRjWbxnHBMchAeXjzeLw8udMipSN7Rwyz++woafjwJgzSDMPVy+p/TlvO+uYj1KqOvs90EN/hVdovPcokK8+uVuy7GDZeaZNE8G0fdSZhBCqFWa5WTZX+cCdkExMwqbNSA/jQbMwquSow0D7h++ruvYc8QqoE5WW8v/stC6SCMSqG61ZACoZcpyGshYGyoAbC8qN03ZdfDbpNbHF2t0wFBJeTWumf8F9paatSV/IOz3K20GicD/fR8v0pSCV6TTEgixyc/+eJtNuTyNXa7+e45W4PfvBgfLWpMZJHLNGjCbbXjf5fVCNqjI7X1IGaKq8pwL2P7OS5EaT9MHjeNcqLKyEtOmTcORI0dQXV2NCRMmoGvXrpg6dSr8fj/y8vLwxBNPICsrdgJUBGnKsRd1wPrdR6PaJYLAanNeajfTgV1OMaWQBOw1awD4jLMnJI/gSxN+zKK8FW5fhFe+3I0nV+6wHD/OWXU/VlGLvp1aYN1PR41jsvZnmc5ZxWjubuy8hCsY32+RjPEFwqHg7Ia0hIPHq3DweJUleRadyEdWY47kt5ANmEXI5FdhkRFc/EFA/s3SOAOZW2HNnjVr1s7X8+7Jv4/D7IVjsyZtyDOBss85qRcYP/30U/To0QNvvPEG5s6di9mzZ+OZZ57B6NGj8dZbb6FDhw5YvHhx/GpIQV6apjmZ+NUF7Y3jj17T3fZ7t17cUXiOHU3pj4PPbmW5Xtam6QgjrP/8nx+4l7mdXm7cww+pLuMIa58/gIaZXtMx4R6SzHEZzfL9jeZFLLvv2Gnd5j7E745BM4j52PT3v+eWt5/RcAN6WMDLatZuzF2y8J61KJiLEGmOZ67niUDQkTuYbdbma5zeUzd7YIoImzicfzMpkTUPBYU1/zu8ACrxcxaYQRKZG2TkyJG44447AAAHDx5E69atUVBQgKFDhwIABg8ejLVr+eG5sYZulAzKtpyZEbk1h31A9MvPS+rkxlZpR3Bz3fDn3cxCRvdTcyO6H+v6RyjjLIxW+6zCWvQiW+3P7tvBThu1GwQzKVWZdvuioTPikaqV8fzNYR00/HrYDFIjKYTdBD/Jwgqtr3cfxY3P2/etSPPxu6q+4dYoFtZOg7c1mlVsUqFuyS9DwmYd0PXg5ha69TwrUInw51lA2cE7qTVrws0334wpU6Zg+vTpqKysNMweLVu2RElJ9OYImoFdTuEep1/MLMrFLtPB3c7uhWYfEC28vZz5dKyEtZNi2ji0Ws/z7LBjFydrGcDX0Kp9ATTMYoS1SLNm/awj0CztOrSd5kUPmjrEPuPhjYPt65adaX6uOu0NIm0GiX2vZMcrOo+yiIg1azdmkNDfGp9YwJ4QDIwE9tnTn2QHDglZHS5f13H4hNXFz25rPa7N2u9ss04617133nkHCxYswIMPPiiVgCgaXrm1D/e4WesNV12kTRLspk1eTRNGvGVwNGunaaks9NZGonoBwPdU4icnqmsD2CS4nheZV+3zWwSXyOZu0UYjWGCLZFEOMM+cRK/btqJyavdue344YM7/EQjAtZ91zMxhdD1YW6rEdyLdltCNX7QOYNayrdhK5U1h68om+nK6H/3x6f9aUwXzxiDbvSqNgZpcCxzm1CmgWxWS99bvxZeFh7kDHzt4G5sP0O6kVHEJdd3bvHkzDh48CADo1q0b/H4/GjVqhKqqoEtLcXExWrWy2najwUlTBsyC1Em7sNWsPZrp+/T/57TNdayHE3lNGnCP67p9vcgo/9dl/BzMPKpdmhlq/ToaZJg168mL+PvgscI5EsEbyaIcYH4f/vXNPnzGWVhe99NR/FgS9IJxsoGy5hGTGSRCzToW7wrbxnJKc2TSuoQjyETNdqyiFi98/qORHRBwnhmy8NwSCau2HpIqo8rnx/R/fY9STlCMRXPXg1t8WerBsVmXV/sw5qUCrmbNDt6J9AZxlIrr16/HK6+8AgA4fPgwKioqcPHFF2PFihUAgJUrV2LgwIHxrWUITaBZ2wnrDI/9ckRQs7YK65aNskymlkjp0qpxRN+LZCdnO61QJIQyPJrU77QExYTe+P+79hzp+kWqWdP1e3vdXtz+2nrb693e5kSVD4UlQfORqA3PPc2c55xtD9Gg7AaLsJYQxKu2FuOlNdHtdk5w83zcapA8YeqWZd8fxFsFe7iL8WTwJMUGdJ2/uXRAxytf/sQtX0az5uWzNmnZtr8gOhx76c0334yjR49i9OjRuPPOOzFz5kxMnDgRS5cuxejRo3Hs2DFcd911caxiGLop6Q7csnEWBp+dh74dW5iuf/CKs7F+xjBHDZa3uajHowlDVvt0tO7tRpNLRYfZ3dtOuJKJQ9um2bb3orHrAKKFM69Hs98dJwQbVVjjDyDL60FudqZ0/SK18/LMUXa4FQTLtxQZglJk3mBne6w9NBYDe6S7woh2wIkVdj7PsrAh4k5f5w1URJjy1nFqA8F87MRUE9B17kzOrt48zZp11dUQHFS/2xdOifAC5csfzwVGRz/r7OxsPPXUU5bjCxcujEuFCIPOzkM+M92lBR/dgbMyPFj4m754csV2rNsd9hk+tWm2YwCNx6OZHtLBUO4PjyYWtNmMBwVLu+Y5KAu9NHbaEcmmx4PUqXmjLBxwkUZSRKXAj9vr0YI2YZcLmbX+AKC5E6RuklzRyJjFaKKxG9KuWh4t3LntsjEC7gcUHqxmHak9OtZEE1RDeJhxo4zkERFTFS8WwOfXTdGGus6fKbjd2IK3wPje+r2YJQxCSqDNOlH8Y9yFlmN0kAXdgYlrl2jRhGjIvIVI1gxyZshswR6ncRTWzRpS97a9VIjbIA0nXlzDn/pleDTXwhAAvtlzDBqADFEECodINeu9LjMtRrNvYTX1jtHP3ylnTKReGTTs+8ubxieCSIJ1nHH/fWIa5GnHPn/AnLtc59d7xtLNwvJ5uoTFDAL7GXFSuO7VNVkcwUpPpehpJ9FqRG5n5BFOHnaW5VymV8MZeY0AAF88NBg3XRgMttEYLxFT3RyEW5fWkdmpacIpO2PvdWC6j0dzDLEXoWkOGwwzkC2x3OIUNcoSoWkcgHmXHXrdwGlMctK8ZYhkC7e6gBthGGVVo9GsedQGdNMzEplB7OD1NZ43SGObJFhJ4bqXDJRWhAM76IRLRGAI3cmMFVzr+UYNMrDw1j54afyFOK15jnGNl/ESmTaia7g4h355welhmzZrRweAa3q1tS+AIh6RcjQZHs0xeZUIDZpjStlEEI0ZhA7Lp3ObO2nOsRDWcR6XhZzWvKHteV4AkFv/fxbiuRMrfP6AyeQo2r3IDp5SwDOD2D2ntNSsncimNG8yFRc9HPIQeQ3ZMMuLlo0bYFj31gDCAp9deGzcIAO/PP80AECv05rZ1q0NtSh47+AzLefdTCHj4c9L4/V4HKf4Itxq1vHiut7mwS+aDkMLa5FLJ49I25AmetNCZDjN3njmgdv++XVU96TXlnh8ss3qzme35lHrN2dj1OE+FoBncuR5g9jNgBIabp6stGwcdpUyUmIKbdbicnLYCD4SJaUxNktKc2qak8nVmAnNcsIeElwXPKqaZ7VujLsv6ywsK95mEK/H3JZuicTeHWs6nWI2O0Uj9Moqw2YQ+r0ZfHae7fdiYgZJmLC2vy9vLYi7dVwM4W1cbNcXav0BUz113X7zah68hUuezdp2JyOlWfPpHLI1k0YWLjCG/vKEeU6m2f7UuEHwc+/TmgmFNfSwQL65T3uwNG1o785Gj77tm+dYBoxgnYP3O1Yhv0ddJHg9nogDOipq/I7boLnlmVvOc/0ddryIpr/QOVTo3/briztiwJn8NAjBOsTeG6SucNp3MlH1YrHb39TnNwe7BALu89fwfOVZwezRNNvoz7QV1u/e1d/2/AvjLsQtfU83NCvRwEtrSAt/0wdP39TL+MzmxmjTNBsf3HsJHruhp2mBkRVKRFjzwmzZ5Eg057TNNT1Q1nWQUFdb3Wd4NDRzGFzsiLVmfXXPU43/+50hnr3QsLOXaDRU07OhXhxN09AqVzwDSWXNOt6mtlhhq1mH/KwJkdisO7Rs5HiNptkvBKetGaRvJ/vOemarxph1Q0+jozitput6MO0pXS5Pq+3VvhmyM72moBivR8M9gzqjc14jDOveGj3aBSPauO6ANh33o98NNHVKr8YX1jLRa7HA49G4njeyxMK/mIYI3l/0aivt1cHai9kNWqOtC4E3i+jdvhm+nXl5bFz3JH7wn11EjNLYme3ibWpzi6gd7AaVr348wuwy7t5VtNoXMDzDRAQXGJVmHTWi6UnPdsEFwW6htKOnNc8xztlpwWbN2oMzWzXG6t8PQotGWRjXrwNev60vhnQN50W5+tygViiKfCTQ1dQ0/uJUDPq+FBkezZIfxN33Y/8K7fzrCMwd1Vt6+i2j1V7kMPBzy7XZmIKQ4dHQLCcrJuYgGZnJq4MMI3q2EZ4bxMnbnkhEg4ed8J2zfDt2HQ4HmekRaNY1voDjoOvRNO57eXnIQYGXjyRW1CthLdKsr+zRBp89OMjw+ACAzx4chKdv6mWbg8Nsszaf0zQNA7vkmUb7uaN64/tHh1vKeeaW89CyUZZhj6VrWeMLcOsgM0K3bBT97jzeKDXrWIRZs2R6PfB4NGk/WafBEYDhP+8G9rHwPF98lPdQtMiYQSIdFOhnzG7G8dyY83HvYPEidyz5zSUdHa8RCWunGcBmKuNkQHdvs67x+aWiRnlKYf8zWgIAbl34NQ4et9++LVLqlbC2M/yz9qgOLRvhhpArngizsOY3VTU15c7wetAklCtj3s298c/f9gUQnNJv+MPl+EXIv5pe6Kzy+cGzJMhMTb+aPhT/mnCx43UA0LNdU7xyqzUq1KsljxnkgcvNQUuympGoCm1ywy6UzRu5t8uP6nO6+T6cnkx8kMmA+6sL7N8pO37kbO7LEumgQA+q9w0xu5NmZ3rRUcJeGwsaZTnvqs7bLxRwfh+Ky8Ph5oGQN0gjjplTRHUUmjX9WPrP+sRVamNZ6pewDjXiZWcF3azOat0kZmXT7ng0okWna3u3M+rBQo8pVbUBbgcsYnZVnvPLcy3XZHo90rbSrAwPhnQNzyyIDTPDK5d1T0QshXXnPLMLXrRmkHZUsEeOQEhcde6p3OMAcCujBfK0WjI1JyaT9pSJzS2b9h5zvMZNe9MmPnpAJh5PkZYbbwY8/klE36P1GxJu7mYXKRkziGiBkX0Hy6tj78VVr4Q1mUbeeOFp2DBjmLEIGCn0ym4fwQLNVT3FnV1cbhhfQOdq7V3bmN3prunVlrsYKqtpNWI6qLFJqKQZRPQOZ8bQZs3+FtmFL5Epi7Y5i36jXTCL16PhiV+da6SBJc9pYJdTDJdN1gziJql/JLjRrP8+5jxc3Dk4PacXwnmL4rFYIJVBZrYUaSrdt9eFNyQm3iBuvJUOlVeb/LtJ29F4NI1bP/YddNoQJRLqlbA2og81LapADxY7v2kZeykLbQbx+QNcO1nXNk3w42Mjjc8Ns7xYMPYCzv3l7tmCmRmQTi9rBhHZSmOpkbH3kNWsRR2SHkdEncc2fa6m4cYL22N8/46m+jVtmIkRoUGa1JGci3d+D95zEEWRalo4SRf9nvLe2boS1kdPhhfg4nnL5z7dhVp/wDHvzfmni6ORx/fvwD1OlEJ6tst6b0WzaC+ingnr4N9IEvfzIC96zyg1dBa6P/sDOjcngY7wvoAkfzb94s0d1RuAVdN6YZxVoAOwpIotDplZmuVkccPnf8HkLxF15lj6WbM5SkQRaOcxHUwkrOi2EQlrdsZBw04aSHk5WV7jnkT799SRZs3zvhG1k4bw83EaRMhvu/KcNrjr0jOiq6QNB4+b05jGi+3F5Vi764hj3hu7QCdeW1fW+PGvb/Yjw6PhJiogjn3u0awDiahXwvqyUEgwa/uMlEYNMrDknv5YMPb8mJRHYM0gvH0dybPf/pcr8fYd/QAA/Tu3xENXdsWmmcNx3XntAFiF6BXn8F20mjPCesrws3HuaU3Rt1MLtGmajcV3hwOQLu7c0hJJKNKsYyms2XvQZpCrKdsya3rJ8HiwYcYwfHDvJabjdNtkefmaTnPBWgT7fbp+DTK8hp3fz5hB4q1Zu0m65dE0ZGXwB5Gt/3el5VogZIII3YLdHScWdDrF/ULmnZeeYaQudkNFjc/RtbSBjesur63X/ngE+0orLaYQ9rnHw0uqXgnrsRedjm/+cHlED1bEBR1aGB4esYI2gwQCuilPt3FN6G+DDK+R2U7TgoE5TencI9T79PEk6/Zq9wwKumRd3ctsWx/R81R8eN+AcF4V6hyZ5tHCk/WMIMTCZU1UFu0WSS/WZmaYr8v0Bs1e9GLvFee0NmvWmfxX3e7ZWvysQ503K8NjDFJEqx0UUhSGn9Ma8cSN614w0VYoyRmjfbORuyQH+zltc40p/ZU9rAN/tI97+shuOMtlCuH+Z7SMyGWxxhdAhkfDgjFiZcvOtuxmPYY12SnN2gFN09A8Br7HdYkvoHMj7mSniLT2x/PT7dqmCXbPvspxtmHaUy70f4tQW668/1LMuKqbXIU49Gpvn6WQwGpBdM6KLpRnD6vNk890Wzxzy3nmBUaBpkNn2WNhzWmkjWhhTXx5u7bJxe7ZV+GCDu6Db9zgZnD0aBpOCa3dOK0t9DytKT687xL8bmiX8KawnHcwWtt2dqaXm4nSDlFKBidq/IFQ9kzrd+fd3BvfPTrcVli7uSfb99QCYz2BfrB+oWYtJ63pzsPrXLKLdKy2D4SFda2fH7gjywf3XmLaPUcEK09pzfqctrk4NZR6NptZvAnPPMLHGmR4TXUWaTokmEEGsrbQIMNjmBci2f1G5NIpg5uIUQ1Bc9eMq7phRA+r19LHkwbiq4eHGp/PPa1ZKI978DPPpGMnrFtIKkojXXpQeTUtosXIWr+ODK94E5Hc7Ez07SR+/m7S/7LCOi016z9e093QDpKVd+7sh0lDu0hfT28y6wsE0KiB1W4mq1nToz9Ps76gg/3mvsb9qP9Jp/vbqN64vHtrdGkV1GqvtvFJpuFNO2VkDDvDoBfOvJpmTN2bMDt1kIVXVpDQmjVP0/nq4aG49Kw8vPqbPlIzB5LdkdasI3Ezc4pU5G1OQQSHG21PC7XZ7QPP4H6v26m5ptzrBOKTXkEpEWeTmY3N7VdMvlSqH7g1aWR4NW5ObanvejzcWQVJfXp2mybY8qcrLOdPaZzlqq1ZL9O0FNa/uaQT1s8Yluhq2NLvjJa4/3LrlmEiZt3QE78OuQX5AzomDumCP/0isgQ9tHxiFxF3z75KKpMYYB4cnvhVMCtht1Nz8eL4C40X76/X95Qri3PsV+c7h3uXOZgkSLQou62SlxHWtA85gbejTeuQjXvQ2a1w+8AzHHdMIR08y0sLa6sUueuyMzCwS9jLgKwbEJxmO8O6mXN1zLu5t/F83NqsIyE31L708yBl2RWZ16SB5bfy6+WuYs1zsowBzq05LsOjcaMm6VkbG7+w7c9X4stpQ1zNYiyadaIWGOfMmYNRo0bhl7/8JVauXIlp06bhmmuuwbhx4zBu3Djk5+fHvGL1mWY5WYbdzhfQkZ3pxa+ZfA2RbE3VvkXk0XNk38j5t5xnWsCkkZUTA7qcgu6nmoN6fjf0TDSh3OS6nZqLK5jFuN42Pq9AuIOxi4JEcyL1Ixq1k+mGFRpOTU68U5w064dHdMONoVwkV597Kh66sqvpvJM2Xs3MMIZ3b2MIA7c260jIDcUVlHE2GBCVuXv2VRHdS4bmjTKNZ8PzPrLzMPF6+Hsm0psKsO9BdqY3tLDvRrM2P9NI4i+ccAzU/+qrr7Bz504sWrQIpaWluP7669GvXz888MADGDx4cMwrlC4QbdUvsHnKympynYxN2I5TGjdw7HCynT83OxPLJg1Ex2kfGcc0TUOT7AyUVwcFwGnNG+KFcRca18h0dtLBmjRgNWtz4AcR0nZ97Y3bLrIcczJP0Jo10ZxEXyEDBq9MO/e+TK+GgWeZfX81LTxbcSNAItesQ8K6shadaqBUTgAAEQpJREFU8xrheGWt8eydxop4BNfQmjVPWPfp2Bw/CfKqZHo93PB63hZeLG5mMXWxQYOjZt2nTx/MmzcPAJCbm4vKykr4/dFtlqkIT8tFQRSnt4xcS44X0brp0X6rbl7uVQ9cBiDcwSxmEE1Os15yT9iXfEAXazCE0wBJ7t8g02NxH2S5uHPLoFngMqvng51mvfOvI3FqU/PA69G0yMwg0leaITumNMnOwKoHLsPXjwwz1hycNMZ4RCVmej1hd1LOYGX3Knk9GldY92R8yEnSNRo32nFdbBzhKKy9Xi9ycoKCY/Hixbj00kvh9XrxxhtvYPz48bj//vtx9Kj95pcKK0Qz68JJNrXknv6WCEIR5B2hzWvv3tXf1rc0Uuh3t2ubJlj78BDT+Vk39MQ7d/YTfp+2Acrk/Vh0Zz88cPlZht88WYhlO58o0pCEiROc3OqcOhzJg94mt6FjMFDzRln4+pFhhlCYNiJsCpn9y54Y2tU+h/TvqTUQWj67yWcd6VS8R7um+NuoXvjr9T2haUHXN+J77VRkvMLWDbdJTrvbPbYMjhnk25mX4+LO5sGa76EjL4Bv7suPQ4gl0k9+1apVWLx4MWbOnIlrr70WU6ZMwWuvvYZu3brh2WefjWcd6yVZGR68fUc/vHprH8u5Czq0kO5o5DLaw6RvpxZG7opYQntXLJ98qUUDvKXv6ehn4wpH51iR0awvOqMlfkd5Fyy6qz8evaa7ZaWdDAKkRLJA1lvSv5vgVKVJw7rg1d/0Qf/O7oM0iDDI8nrQtU0uXqae+/a/XGm5fiL1u2kBaHff5ZMH4j8TBxgCLRq5ef15p5meF7mtcwpRd/cZ2rUVnrqxl/D8fyYOAABbM4jd+o6Xs7kGm3pBhBtluW3TbPx9dOwVJBrn5LIA1qxZg+effx4vvfQSmjRpgv79w9PJIUOG4NFHH41X/eo1/TlZvdzSvkUOHhnZzTbVZ6xwqzW9P+Fi00r882MvwJT3NuGLwsMR+Sd3zmuMznmN8d8fik3HSf/Nzc7E9JFdMby7eFcUO5wWdTO9HmNXFbdaK2k7nu+uU9If+lY8M8CAM0+BpoUzNXY8JQc7ik/EVMvVJG3Wbtvl5Vv7GHlqeJDMmWQg5ZtB7NYAgi/H6t9fhqFPfeZYH9rF080bGkyaFZ9ZBcFRWJeXl2POnDl49dVX0axZUFOZOHEipk6divbt26OgoABdusj7GCtizx0uE+/IBi+wuO37559u9vFu0zQbk4Z1CQrrSB1nYbXF0kLpzkvtXcfevqMfvt7NN9tFYnUc209u+ksGFDq/cpvcbFPe8lyO1wJgFoC8dYM3bjcvlhoujFI1k4NUoWnDTJRWxDZXs8ygQmZiPP9lJ5s1EBzoF97ax5QClWXdI0PRgMohc3qLHLTObYDiMrmtuuLhW03jKKyXLVuG0tJSTJ482Th2ww03YPLkyWjYsCFycnIwa9asuFYyXbjkzJb4svBIXO/xye8vk54GshCh0YFZ/LzynDZYvqVIqoxogkkIrNnWLhkPS//OLYUzmvNPb45VW4u553i4cVcLa9bhyn8+dbChFT59Uy9cKBGqLuP7G/bciJ24JmXNvKY78reX4LW1P8ew7PBf0WtBokdzOM/a7k2izUaDu7bCYJu1glZNzAFC2ZleFEwfZvJqsiMevtU0jsJ61KhRGDVqlOX49ddfH5cKpTOv/faiuLsAnRFlRsI3b7/IsgPP38ecL11v0nmIGeS2AZ1QeOiE3VcskEHjnLa5uPXijlG7LRKeuaU3us9cEZOyWIiGR3doWhNz2mKOLUfmGtmUBTKQ2zZukInfDe0SY2EdLDzD4zF86U9tmm1Kp0oWpBtmeTG0ayus3nbIOMeaQf42qhc+/PYAPt1eUqc74IiShcUKKZt1MvB/155jGfnqG94IE9bUJZdw8v+6qTfpPMQM8oeru7uuA+ncLRplGcEnsSAnKwMF04fiosdWx6xMAhnMorVryixsGrm1YzjwE2+QgO5u9xUZWBfLXqc1xWu3XWSKoDSEdaYXL9/aB13/8LGRnoBda+h+alN8sye4RVqku8E7seQe696ndpuUxIKUEdasG5YiNSHT+GjMIKRrx8O1tXVufBQCkuckWkEnMygaiZhi2EB0Jr5YT/dJfXXoWHn/pWjTNBu52Zkm4Ud83LNDZpC/3dQb97y5EQAseUM8WridI90N3glezp3cOAvrpM8NoqhfsGaQSCACq452oooJRL61c8g/4oSM8CEulqwr+5ThZ9luY2UHmc3ouh6ThbSPJw3Ev+8bwJQd3OQ6l5NjnAzuJI/HiJ6n4vnQpiC8QckQ1nVoBom3Zq2EtaJOIR09mg5/UacW+HX/Dnics+N7snJmqyZ44lfnYt6o85wvtsHr0bDy/kuNz5tmDrdcIzKD3DekC96fcInlehk0Q1uPzYYT3U7NNQKGeBtgiKA3TSBrFzpgykWjg8pSGMMR/TmHQLN47LtIo4S1ok45tWk2fn/5WXjl19ZgIFkyvB786doeaBujhcW64sYL2wuTZMmiaZppgZdXnpfSgmMF2TWG9QSKBWETi3N96VzmtLa/bNJAYwcaXac2pIihGUR2B6oRnB12YkHK2KwV9QNN00zReQo5Jg3tgnmrdxqfn7qxl9BGStbUYrl577h+HXD9ee2MjIfDurV25eYIBFPH8vBozpr1OW1zseVAmUn4eihtny6HXgSN5YDFplLl8eNjI+NmnlPCWqFIUv4zcYARxHH/5WeZcqb/8gKxq59H45tBoiGYNZG/96csD4/g56L2UjZrEW/d0c8S6UgLZ5qArhu2/Vh6wuZw8mKzxFKTZ1HCWlGvefeu/thztCLR1YiIHu2aGuHWbuh0SiOs2Xk45hs909ABN3a5PYBg+t237rCmpCVoWjDF7+Rh4hlX04aZ1gU8gWat60BlaJebWO6FKKNZxxMlrBX1mr6dWqBvp/huYptsTB/ZDQO75Elv6RYJRFb/ffT5jnlperdvZgmkMpel4ctpQ4TnRXgY2zwdyl9aUQPAuntSNGS7iJSNB2qBUaGoZ2RnenF599bOF0ZB2M4c/zzO4joE/xIzyLO3nIfbB3RC91NzcSyUv6RZlAu6yYQS1gqFwj2MCcL20jiZcT2Mrbt9ixzMuLo7PB7NMB91Y7aXS2WUGUShULiGNUEkAo3RrGl+c3FHDOvWSnrD6EhZeu8ldbYmooS1QsFwbe+2MV2Yqo8YIeI2spq4G8bLP4JEOvL87T0eLS6C+rYBnUxpbXu3b+Z6k4tIUcJaoWCYd3N0UYbpABHAdvlH4m2C6NGuKZ4fez4u5W7JFR8iSTwWK5SwVigUrmHtxXzibyK5skf8d0hKFtRcT6FQuMdFZr9USriVzChhrVAoXOMUIt6uWcO4pLBNZ5QZRKFQuCa8wGiVyBtmDEN2phfbioKh8nYBMQp5lLBWKBSuyWvSAEBwmy+Wlo2D5y7o0AKL7uwX10jKdEIJa4VC4ZrfDe2CDi0aYWRP+3SgF53B35xY4R4lrBUKhWsaZHhxU5/Y7X+pcEZKWM+ZMwcbNmyAz+fDXXfdhZ49e2Lq1Knw+/3Iy8vDE088gays2CVMUSgUCoUZR2H91VdfYefOnVi0aBFKS0tx/fXXo3///hg9ejRGjBiBp59+GosXL8bo0aPror4KhUKRlji67vXp0wfz5s0DAOTm5qKyshIFBQUYOnQoAGDw4MFYu3ZtfGupUCgUaY6jsPZ6vcjJCe67tnjxYlx66aWorKw0zB4tW7ZESUlJfGupUCgUCaJJdkZSBPZILzCuWrUKixcvxiuvvILhw8M7Kicy65ZCoVDEm/UzhiVFgI9UBOOaNWvw/PPP48UXX0STJk2Qk5ODqqpg5qni4mK0atUqrpVUKBSKRNEgw5vwXWIACWFdXl6OOXPm4IUXXkCzZsFUgBdffDFWrFgBAFi5ciUGDhwY31oqFApFmuNoBlm2bBlKS0sxefJk49js2bMxY8YMLFq0CG3btsV1110X10oqFApFuuMorEeNGoVRo0ZZji9cuDAuFVIoFAqFFZV1T6FQKFIAJawVCoUiBVDCWqFQKFKAuCRy8vv9AICioqJ4FK9QKBT1EiIziQyliYuwJhGNY8aMiUfxCoVCUa8pKSlBhw4dTMc0PQ4hiFVVVdi8eTPy8vLg9SbemVyhUChSAb/fj5KSEvTo0QPZ2dmmc3ER1gqFQqGILWqBUaFQKFKApNsp5rHHHsOmTZugaRqmT5+Oc889N9FVqhNkN3j48MMP8c9//hMejwc33XQTbrzxxkRXPS5UVVXh6quvxoQJE9C/f/+0bosPP/wQL730EjIyMvC73/0OZ599dtq2x8mTJ/HQQw/h+PHjqK2txb333ou8vDw8+uijAICzzz4bf/rTnwAAL730EpYvXw5N03DffffhsssuS2DNY4CeRBQUFOh33nmnruu6XlhYqN90000JrlHdsHbtWv3222/XdV3Xjx49ql922WX6tGnT9GXLlum6rutPPfWU/uabb+onT57Uhw8frpeVlemVlZX6VVddpZeWliay6nHj6aef1m+44QZ9yZIlad0WR48e1YcPH66Xl5frxcXF+owZM9K6PV5//XX9ySef1HVd14uKivQrrrhCHzt2rL5p0yZd13X9gQce0PPz8/U9e/bo119/vV5dXa0fOXJEv+KKK3Sfz5fIqkdNUplB1q5di2HDhgEAOnfujOPHj+PEiRMJrlX8kd3gYdOmTejZsyeaNGmC7OxsnH/++di4cWMiqx4Xdu3ahcLCQgwaNAgA0rot1q5di/79+6Nx48Zo1aoV/vznP6d1ezRv3hzHjh0DAJSVlaFZs2bYv3+/MQMn7VFQUICBAwciKysLLVq0QLt27VBYWJjIqkdNUgnrw4cPo3nz8Lb1LVq0SIuNDWQ3eDh8+DBatGhhfK++ts/jjz+OadOmGZ/TuS327duHqqoq3H333Rg9ejTWrl2b1u1x1VVX4cCBA7j88ssxduxYTJ06Fbm5ucb5+tweSWezptHTzFHF7QYP9bF9li5dit69e6N9e/7O2enUFoRjx47h2WefxYEDBzB+/HjTb0239vjggw/Qtm1bvPzyy9i2bRvuvfdeNGnSxDhfn9sjqYR1q1atcPjwYePzoUOHkJeXl8Aa1R1kg4eXXnrJtMFDdna2scEDr3169+6dwFrHnvz8fOzduxf5+fkoKipCVlZW2rYFENQUzzvvPGRkZOD0009Ho0aN4PV607Y9Nm7ciAEDBgAAunbtiurqavh8PuM83R4//fST5Xgqk1RmkEsuucTY1GDLli1o1aoVGjdunOBaxR/ZDR569eqF77//HmVlZTh58iQ2btyICy+8MJFVjzlz587FkiVL8O677+LGG2/EhAkT0rYtAGDAgAH46quvEAgEUFpaioqKirRujw4dOmDTpk0AgP3796NRo0bo3Lkz1q9fDyDcHv369UN+fj5qampQXFyMQ4cO4cwzz0xk1aMm6YJinnzySaxfvx6apuGPf/wjunbtmugqxZ1FixZh/vz56NSpk3GMbPBQXV2Ntm3bYtasWcjMzMTy5cvx8ssvQ9M0jB07Fr/4xS8SWPP4Mn/+fLRr1w4DBgzAQw89lLZt8c4772Dx4sUAgHvuuQc9e/ZM2/Y4efIkpk+fjiNHjsDn82HSpEnIy8vDzJkzEQgE0KtXLzz88MMAgNdffx3//ve/oWkaJk+ejP79+ye49tGRdMJaoVAoFFaSygyiUCgUCj5KWCsUCkUKoIS1QqFQpABKWCsUCkUKoIS1QqFQpABKWCsUCkUKoIS1QqFQpABKWCsUCkUK8P++oO3RZ3a6WgAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "----\n",
            " on  ll reCrea\n",
            "\n",
            "rhconithd \"0setye 9So sre ahtut, nirc \n",
            "mats uh uronhed .l,oucasthh iitannes aecisvh nar Sphe rcrRletWiwot ots s icoth 7e\n",
            "Care\n",
            "ote me thhiouues. les heg P c Io t s,orlinf g htis w yt F-w \n",
            "----\n",
            "iter 900, loss 34.920753\n"
          ]
        }
      ],
      "source": [
        "iter = 1000\n",
        "while iter > 0:\n",
        "  # Reset\n",
        "  if pointer + Time_steps >= len(data) or iteration == 0:\n",
        "      g_h_prev = np.zeros((Hidden_Layer_size, 1))\n",
        "      g_C_prev = np.zeros((Hidden_Layer_size, 1))\n",
        "      pointer = 0\n",
        "\n",
        "\n",
        "  inputs = ([char_to_idx[ch] \n",
        "              for ch in data[pointer: pointer + Time_steps]])\n",
        "  targets = ([char_to_idx[ch] \n",
        "              for ch in data[pointer + 1: pointer + Time_steps + 1]])\n",
        "\n",
        "  loss, g_h_prev, g_C_prev = \\\n",
        "      forward_backward(inputs, targets, g_h_prev, g_C_prev)\n",
        "  smooth_loss = smooth_loss * 0.999 + loss * 0.001\n",
        "\n",
        "  # Print every hundred steps\n",
        "  if iteration % 100 == 0:\n",
        "      update_status(inputs, g_h_prev, g_C_prev)\n",
        "\n",
        "  update_paramters()\n",
        "\n",
        "  plot_iter = np.append(plot_iter, [iteration])\n",
        "  plot_loss = np.append(plot_loss, [loss])\n",
        "\n",
        "  pointer += Time_steps\n",
        "  iteration += 1\n",
        "  iter = iter -1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2AKpa1BGOItQ"
      },
      "source": [
        "# Quiz Question 7. \n",
        "\n",
        "Run the above code for 50000 iterations making sure that you have 100 hidden layers and time_steps is 40. What is the loss value you're seeing?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 777
        },
        "id": "P1D7xYW9a9YT",
        "outputId": "a0f8bbe6-0582-45bf-9dd4-4b820340a389"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD1CAYAAABX2p5TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeW0lEQVR4nO3deVyUd54n8E9RRYkFhQhWcXgED1DkUoLxCh6gBjUx3niwbmZNYlZNxwRF2jDGbmeiop1dY7srYeLRmp12Ui93x3RUHE10WAfpVXpoScfGIzFqVChFQQ45fPaPqnoEKYqiqIJifp/3X8nze47v863n+VBWPfxQSJIkgYiIhOHR1QUQEVHnYvATEQmGwU9EJBgGPxGRYBj8RESCUXXmwWpra1FcXAydTgelUtmZhyYi6pYaGxtRVlaGqKgoeHl5OWWfnRr8xcXFWLp0aWcekojoP4QvvvgC8fHxTtlXpwa/TqcDYDqBoKCgzjw0EVG3dPfuXSxdulTOT2fo1OC3fLwTFBSEfv36deahiYi6NWd+PM4vd4mIBMPgJyISDIOfiEgwDH4iIsEw+ImIBMPgJyISTKc+ztlRoRlfo1dPT8T064W8K0a8mzgEu765il8kheHT01cQ6NsDLwR4448/PMCKCYOQ/a/Xse6Vodie+1cM0nnDW63CpduP8J/HvoAD+TewYcYwfHzsMmL7++FxbT2ulVVh0aj++P3/u4m/fXU4Nv/hL0gI64NrpY/x86NazB4Rgv/z7z/jwxkR+Ptj3+PVmGCcLSlDbX0jkoYF4sR3d+XjpY4ZgC8KfoK3WoWXBvrjm8ulWDMlDP/91BWsmjwYu7+9hj4+aoQHavFv1+7jnYmDsefsNXwwNRyf/EsJBvhrEOCjxp9+eoi/GR+Kfed+xC+nD8OW45cREewLSZJw+W4llowegP9V8BMyZ0bg777+HqMH+uPnRzW4+aAGc+P64kjhbXls2vBAXLhRjofVdZgRHYw//PkOMqYPw9bjl7HgxX7456Kf4aEAXh6iw6nv7yFtajh+8y8leCthIHLyfkBvjSei+pp6v3ryEPz226t4LykMO09fQUgvL/TrrcEff3wgr5+ePBRZJ/6KIXofeHl6oPh2Bf7TmBdw8PwNuYdxA/zwsLoe141VWBjfD/904RY2vjocv/7DXzAxXIe/3q3E3YpavD4iBP/87z/Lr9nrI0Jw+vtS1DU8xeRhOuR+d0/uveX11XqpEDegN86WlMl1WurWaXtgsM4b568/kHu/dlo4dpwsQWiABr00ahTdfIg3xoVi/7896310316oa3iKv96rxOKXBuAf//iTfK2MGxyAnx5U41Z5DeaO7Isjf3rW++lRQci/fh+VtQ14JTIQxy7dxfrkYdh24jIWv9Qfhou3oPLwwPghATj1fal8HayYOAjZZ6/D31uNiGAtzl29j5WTBuN/nLkmX099/XoiqJcXLt4ox5svD8Q//N8f5H0PDdRC6aHAX+5UIHXMABw6/+xaiX+hN4yPn+DH+9WY/2I/GC7eks8laZgef779CGWVTzArNgRHi571fu7Ivjjx3V00PJUwKVyHk3+5J/fOcq36eqkwYkBv/GtJGX6ROASfPnefhgZ4o8DafdrHGz5eKvz5lpX7tF8vVNU14mrpY/k+tVwrLw/pgx+MVbj9sAZzRvbF/27S+5kxwcgrKUN1XSOmDg/E8eJn9+nS0abXUNPkPn1/Sjj+26kSuc8B3moMDTLdp/910mD8zzPP7tP+/j3Rx6cH/vTTQ/yX8QOx99wP8j0VEewLAPj+TgVOvj8B4YHaLk7QZxSd+YdYbt26haSkJJw+fdqh5/hDM752QVVERK7349aZDm3X0dy0hh/1EBEJhsFPRCQYBj8RkWC6TfB//ec7XV0CEZHDlv7D+a4uQdZtgv9I4a2uLoGIyGHnrt7v6hJk3Sb4iYjIORj8RESCYfATEQmm2wT/6culXV0CEdF/CN0m+ImIyDkY/EREgmHwExEJhsFPRCQYBj8RkWAY/EREgmHwExEJxq7gz8rKQkpKCubNm4eTJ08CAH73u98hMjISVVVV8npHjx7FvHnzsGDBAnz55ZeuqZiIiDqkzT+9eP78eVy5cgWHDx9GeXk55syZg+rqaty/fx96vV5er7q6Grt374bBYICnpyfmz5+PqVOnws/Pz6UnQERE7dNm8I8aNQoxMTEAAF9fX9TU1CApKQlarRZfffWVvF5RURGio6Oh1Zr+rmRcXBwKCwuRmJjootKJiMgRbX7Uo1QqodFoAAAGgwETJkyQw70po9EIf39/+f/9/f1RVlbmxFKJiMgZ2nzHb3Hq1CkYDAbs3bvXrvU78W+4ExFRO9j15W5eXh727NmDnJwcq+/2AUCv18NoNMr/X1pa2uw7ACIicg9tBn9lZSWysrKQnZ1t84va2NhYXLp0CRUVFaiqqkJhYSHi4+OdWiwREXVcmx/1HDt2DOXl5VizZo28bPTo0SgoKEBZWRneeustjBgxAunp6UhLS8Py5cuhUCiwatWqVv91QEREXafN4E9JSUFKSkqL5atXr26xLDk5GcnJyc6pjIiIXIK/uUtEJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQlGZc9KWVlZuHjxIhoaGrBixQpER0cjPT0djY2N0Ol02L59O9RqNSIjIxEXFydvt3//fiiVSpcVT0RE7ddm8J8/fx5XrlzB4cOHUV5ejjlz5mDs2LFYsmQJpk+fjk8++QQGgwFLliyBj48PDh482Bl1ExGRg9r8qGfUqFHYuXMnAMDX1xc1NTUoKChAUlISAGDy5MnIz893bZVEROQ0bQa/UqmERqMBABgMBkyYMAE1NTVQq9UAgICAAJSVlQEA6urqkJaWhkWLFmHfvn0uLJuIiBxl12f8AHDq1CkYDAbs3bsX06ZNk5dLkiT/d3p6OmbNmgWFQoHU1FTEx8cjOjrauRUTEVGH2PVUT15eHvbs2YOcnBxotVpoNBrU1tYCAO7duwe9Xg8AWLx4Mby9vaHRaDBmzBiUlJS4rnIiInJIm8FfWVmJrKwsZGdnw8/PDwAwbtw45ObmAgBOnjyJhIQEXL9+HWlpaZAkCQ0NDSgsLERYWJhrqycionZr86OeY8eOoby8HGvWrJGXbd26FZmZmTh8+DBCQkIwe/ZseHp6IigoCPPnz4eHhwcSExMRExPj0uKJiKj92gz+lJQUpKSktFhu7cvbdevWOacqIiJyGf7mLhGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYBj8RESCYfATEQmGwU9EJBgGPxGRYFT2rJSVlYWLFy+ioaEBK1asQHR0NNLT09HY2AidToft27dDrVbj6NGjOHDgADw8PLBw4UIsWLDA1fUTEVE7tRn858+fx5UrV3D48GGUl5djzpw5GDt2LJYsWYLp06fjk08+gcFgwOzZs7F7924YDAZ4enpi/vz5mDp1Kvz8/DrjPIiIyE5tftQzatQo7Ny5EwDg6+uLmpoaFBQUICkpCQAwefJk5Ofno6ioCNHR0dBqtfDy8kJcXBwKCwtdWz0REbVbm8GvVCqh0WgAAAaDARMmTEBNTQ3UajUAICAgAGVlZTAajfD395e38/f3R1lZmYvKJiIiR9n95e6pU6dgMBiwcePGZsslSbK6fmvLiYioa9kV/Hl5edizZw9ycnKg1Wqh0WhQW1sLALh37x70ej30ej2MRqO8TWlpKfR6vWuqJiIih7UZ/JWVlcjKykJ2drb8Re24ceOQm5sLADh58iQSEhIQGxuLS5cuoaKiAlVVVSgsLER8fLxrqycionZr86meY8eOoby8HGvWrJGXbd26FZmZmTh8+DBCQkIwe/ZseHp6Ii0tDcuXL4dCocCqVaug1WpdWjwREbVfm8GfkpKClJSUFsv37dvXYllycjKSk5OdUxkREbkEf3OXiEgwDH4iIsEw+ImIBMPgJyISDIOfiEgwDH4iIsEw+ImIBMPgJyISDIOfiEgwDH4iIsEw+ImIBMPgJyISDIOfiEgwDH4iIsEw+ImIBMPgJyISDIOfiEgwDH4iIsEw+ImIBMPgJyISDIOfiEgwDH4iIsEw+ImIBMPgJyISDIOfiEgwDH4iIsEw+ImIBMPgJyISDIOfiEgwDH4iIsEw+ImIBMPgJyISjMqelUpKSrBy5Uq88cYbSE1NxbVr17Bx40YoFAqEhoZi06ZNUKlUiIyMRFxcnLzd/v37oVQqXVY8ERG1X5vBX11djc2bN2Ps2LHysh07duDtt9/GxIkTsXv3bhw/fhyvvfYafHx8cPDgQZcWTEREHdPmRz1qtRo5OTnQ6/Xyshs3biAmJgYAkJCQgHPnzrmuQiIicqo2g1+lUsHLy6vZsvDwcJw9exYAkJeXB6PRCACoq6tDWloaFi1ahH379rmgXCIi6iiHvtxdv349jh8/jmXLlkGSJEiSBABIT0/Hr3/9a+zduxdfffUVLl265NRiiYio4+z6cvd5wcHByM7OBmB6x19aWgoAWLx4sbzOmDFjUFJSgujoaCeUSUREzuLQO/5PP/0UZ86cAQAcOXIEiYmJuH79OtLS0iBJEhoaGlBYWIiwsDBn1kpERE7Q5jv+4uJibNu2Dbdv34ZKpUJubi7Wrl2LzZs3Y9euXYiPj8ekSZMAAEFBQZg/fz48PDyQmJgofwFMRETuo83gj4qKsvqIpsFgaLFs3bp1zqmKiIhchr+5S0QkGAY/EZFgGPxERIJh8BMRCYbBT0QkGAY/EZFgGPxERIJh8BMRCYbBT0QkGAY/EZFgGPxERIJh8BMRCYbBT0QkGAY/EZFgGPxERIJh8BMRCYbBT0QkGAY/EZFgGPxERIJh8BMRCYbBT0QkGAY/EZFgGPxERIJh8BMRCYbBT0QkGAY/EZFgGPxERIJh8BMRCYbBT0QkGAY/EZFgGPxERIJh8BMRCabbBP8fP0zCrNiQVsd7eipbHfPTeLY6ptP2aHXMp4fKvuJaofJQtDrWQ9V667U2juvvrXZozFYt9lArW69Xo2699716tt57vY3e23o97eGpbP18vTxbPxdfr9Z7H2Cjv7bOU9Gx1kNt41qxdY3auu5t9d7W8exhq/e2Xldbve/j03q9tu4Xe9i6NRztfW8rvd+T+mK76nIlu17hkpISTJkyBYcOHQIAXLt2DUuXLkVqaioyMzPR0NAAADh69CjmzZuHBQsW4Msvv3RqoXqtF7bNiwHQ/MaNG+AHAPi72VEAgBcCNPJYX7+eAICNrw4HAMS/0FseswRZRvIwAMDM6OAWx3x/ajgA4G/Gh7YYWzFxEAAgPXloi7G5I/sCALaa621q/JAAAMDHc6LN5/Xsgg7T+wAANs2KBABEhvjKY5Zg+XBGBAAgcZi+xb7XTjPVsvil/i3G3k0MAwD8InFIi7ElowcAeNbDpqZEBAIAtsw11dv0Yo/t1wsAsPl103aDdN7yWJCvF4BnvR87KEAes9xo6829t/YD/f2ppnrfShjYYmz5y6ZlmTMjWoy9Zt7X1rmm3iub3NWjB/oDAP5+tulcQnp5yWMD+5hqt/Tecm7As3PeYO791OGBLY67dprpWkkdM6DF2OrJpp6/PyW8xVhKfH9zvdEtxiYN1QEAtpivlaZhMjzYdG38ylxveKCPPGYJyb+daep9QlifFvtON/fecq02tWaKqffvTBzcYuyNcaEAgI9eG95ibEZ0kPlcTL1vGpqWe89yjfX37ymPWf77o9cim60LPHuD9MvppnqnRwW1OO4H5t5bamvKcg7W7tN5cf0AQM6Vpiw9s/S+6Q+eoYFaAM96HxH87D61vEaZ5t5bXkMASLZSe1dpM/irq6uxefNmjB07Vl62Y8cOvP322zh06BCCg4Nx/PhxVFdXY/fu3di/fz8OHjyIAwcO4OHDhy4pWpLaN2Z5x9Vs6Lmf8hJa36nVfZp3YKsWezTd/Pl3hrbPs331WjuePWMta3Js5037q3hupzZrau9r3WIdG32ysszqteKkmmxeYzaOZ22d9l0rtmqydTzH7okmO2jX9tY26+p7wtpa7bknOpoPrtJm8KvVauTk5ECvf/YO88aNG4iJMf2UTEhIwLlz51BUVITo6GhotVp4eXkhLi4OhYWFTi3W2j+Znw+RttaXx+w5XjtrsWfDFjeHvWN2FGzzfB3shc3DurT3tvrkUEmu6UEHD+xwTQ6ep33HdfCzKYd779g9YcdhO9yn9rKVR+6kzeBXqVTw8vJqtiw8PBxnz54FAOTl5cFoNMJoNMLf319ex9/fH2VlZU4ul4iIOsqhb3HWr1+P48ePY9myZZAkyeo/dWx+JEBERF3Goa/Dg4ODkZ2dDcD0jr+0tBR6vR5Go1Fep7S0FCNGjHBOlURE5DQOveP/9NNPcebMGQDAkSNHkJiYiNjYWFy6dAkVFRWoqqpCYWEh4uPjnVkrERE5QZvv+IuLi7Ft2zbcvn0bKpUKubm5WLt2LTZv3oxdu3YhPj4ekyZNAgCkpaVh+fLlUCgUWLVqFbRaravrb8Hakwju/KmTo7U5+tSJ7Z26rlGu2LWtp07s2t7K5s58SsvZHL9WnF+Va8/Tsb27Y+/dVZvBHxUVhYMHD7ZYbjAYWixLTk5GcnKycyqzwd7XoKOPXXX0C3prx2vPPtt7rdmz646ep72Pg7andY4+cmnXvptuZ/txq+dqsnevVje3vlW7H0O2/0Qd/1nfvucc29d7++63ju7T5pN99u20XftsD3f9edFtfnO3NXY9Gmj1MVDXHdfRx+E6+gikzX07up0Lauro44Y2t3NsM9vHdeW14vB5dizsXPGYY2ffEx3l+PXbwR80bqDbBz8REbUPg5+ISDAMfiIiwTD4iYgE062C39M8o+b8F/vJsz9aZryL6muaTXFWbAi8PD2gVnng9VjTzIORIb2aravT9sCCF02zIkb2Nc2sN3moaS6i8EAfeYbH2P6mmT/HDjbNLPlSqL88a99o82yTI82zg06JCJRndJxonpHPMmvf/Bf7YYC/adbQaeaZHS31zhkZIs+8+WqMaYbQaPN+ZppnO+zV0xNzzTMJWrZLMs+aGRqgkWcZtIy9bK4xtl8veXbN+FDTdBqjzDNUTgjXYVSoaRbEl4foWvRwiHmm0KRhpu2Hm2cKnRfXT57K2jIbo2W712KCoVZ6oKenErPNsz5axqZFmtYN7uWFheYZKS37nGTu/bAgrTxLqqWvllk9xwzyl2c2HTfYdH4j+pvqfyUyEFF9m+/L0vuFL/aXZ2l9JbJ5718fGQKfHiooPRTyDKFR5mtluvnc/L3VmP+iqb+WayXR3JNBfbwxx3yeMebX7OUhptriBvjJM6i+ZO75KPNrMHmoTp5VNiHM0nvTvueM7ItB5plCLa9dZJMxy9TQM6KbXyuvxgRD6aGATw8V5sQ17/1U8376+vXEolH9zb027dNyrQ4P9kWy+TWKG2Dq6+hBpnrHDwnAGMt/y723XKPBGBakNffFdL6W13X+i/0RbJ4B9ZXn79MRIejpqYRa6YHXzb23nIvlPu3j0wMLWvTedIwheh/5NbP03nJdjArtjQnhOvM5PH+f6uX72rJO02s71Dy7r2UGVstxZ4/s2+p9anktfL1UmNui96Z6Lfe/25A60c2bN6Xw8HDp5s2bDu/jcW291ND4VHpS3yjV1DVIT58+lSpr6yVJkqTK2nqpsfGpVFvfINXWN0iNjc3Hnj59KtXUNUhP6hulhsan0mPzWEVNnSRJUrOxqifNx6qfNEh1DY1SfUOjVP2kocVYfUOjVNdgqqnpWNWTlvVaxh43qdfauVjqtZxL03qfP5fn662ps6/e58eer7fpWHt7b6tea7231Nue3lc9qe9w72vru6b3zrxW2tN7W/Va672ta6W1c7F2rbTW+9buU0fOxZ3vU0c5Izefp5CkzvvVhFu3biEpKQmnT59Gv379OuuwRETdlitys1t91ENERB3H4CciEgyDn4hIMAx+IiLBMPiJiATD4CciEoxDf4HLUY2NjQCAu3fvduZhiYi6LUteWvLTGTo1+C1/fH3p0qWdeVgiom6vrKwML7zwglP21am/wFVbW4vi4mLodDoolcrOOiwRUbfV2NiIsrIyREVFwcvLyyn77NTgJyKirscvd4mIBNOpn/E76uOPP0ZRUREUCgU2bNiAmJiYLq2noKAA7733HsLCwgAA4eHhePPNN5Geno7GxkbodDps374darUaR48exYEDB+Dh4YGFCxdiwYIFqK+vR0ZGBn7++WcolUps2bIF/fv3d1m9JSUlWLlyJd544w2kpqbizp07Ha718uXL2LRpEwBg6NCh+NWvfuXyujMyMvDdd9/Bz880u+Ly5csxadIkt6o7KysLFy9eRENDA1asWIHo6Gi37/XzNX/zzTdu3+eamhpkZGTg/v37ePLkCVauXIlhw4a5da+t1Zybm9s1vXbadG8uUlBQIL399tuSJEnS1atXpYULF3ZxRZJ0/vx56d133222LCMjQzp27JgkSZL0m9/8Rvriiy+kqqoqadq0aVJFRYVUU1MjzZw5UyovL5eOHDkibdq0SZIkScrLy5Pee+89l9VaVVUlpaamSpmZmdLBgwedVmtqaqpUVFQkSZIkffDBB9KZM2dcXvf69eulb775psV67lJ3fn6+9Oabb0qSJEkPHjyQJk6c6Pa9tlazu/dZkiTp66+/lj777DNJkiTp1q1b0rRp09y+19Zq7qpeu/1HPfn5+ZgyZQoAYPDgwXj06BEeP37cxVW1VFBQgKSkJADA5MmTkZ+fj6KiIkRHR0Or1cLLywtxcXEoLCxEfn4+pk6dCgAYN24cCgsLXVaXWq1GTk4O9Hq902qtq6vD7du35X95Wfbh6rqtcae6R40ahZ07dwIAfH19UVNT4/a9tlaztccG3almAJgxYwbeeustAMCdO3cQGBjo9r22VrM1nVGz2we/0WhE79695f/39/eXHwvtSlevXsU777yDxYsX49y5c6ipqYFabfojGQEBASgrK4PRaIS/v7+8jaX2pss9PDygUChQV1fnkjpVKlWLJwE6WqvRaISvr6+8rmUfrq4bAA4dOoRly5bh/fffx4MHD9yqbqVSCY3G9Ac3DAYDJkyY4Pa9tlazUql06z43tWjRIqxduxYbNmxw+15bqxnommu6W3zG35TkBg8hhYaGYvXq1Zg+fTpu3ryJZcuWNXuX1FqN7V3eGZxRa2fV//rrr8PPzw8RERH47LPP8Nvf/hYjR460q5bOrPvUqVMwGAzYu3cvpk2b5lBtrS3vjJqLi4u7RZ8B4Pe//z2+//57rFu3rtlx3LnXTWvesGFDl/Ta7d/x6/V6GI1G+f9LS0uh0+m6sCIgMDAQM2bMgEKhwIABA9CnTx88evQItbW1AIB79+5Br9dbrd2y3PJTub6+HpIkye9UOoNGo+lQrTqdDg8fPpTXtezD1caOHYuIiAgAQGJiIkpKStyu7ry8POzZswc5OTnQarXdotfP19wd+lxcXIw7d+4AACIiItDY2Ahvb2+37rW1msPDw7uk124f/OPHj0dubi4A4LvvvoNer4ePj0+X1nT06FF8/vnnAEy/TXf//n3MnTtXrvPkyZNISEhAbGwsLl26hIqKClRVVaGwsBDx8fEYP348Tpw4AQD49ttvMXr06E6tf9y4cR2q1dPTE4MGDcKFCxea7cPV3n33Xdy8eROA6XuKsLAwt6q7srISWVlZyM7Olp/ScPdeW6vZ3fsMABcuXMDevXsBmD4Orq6udvteW6t548aNXdLrbvELXDt27MCFCxegUCjw0UcfYdiwYV1az+PHj7F27VpUVFSgvr4eq1evRkREBNavX48nT54gJCQEW7ZsgaenJ06cOIHPP/8cCoUCqampmDVrFhobG5GZmYkff/wRarUaW7duRXBwsEtqLS4uxrZt23D79m2oVCoEBgZix44dyMjI6FCtV69excaNG/H06VPExsbil7/8pcvrTk1NxWeffYaePXtCo9Fgy5YtCAgIcJu6Dx8+jF27dmHgwIHysq1btyIzM9Nte22t5rlz5+LQoUNu22fANAvAhx9+iDt37qC2tharV69GVFRUh+9BV9ZtrWaNRoPt27d3eq+7RfATEZHzuP1HPURE5FwMfiIiwTD4iYgEw+AnIhIMg5+ISDAMfiIiwTD4iYgEw+AnIhLM/wdC4BqfzNkhZgAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "----\n",
            " iHcV'B0yBb00RJY 0'–LKFx00bYzKg00'VJgvFG3t4eANvL9JBHM7SfG0B400BB(JB57TeS0()zrKJ,dUl85dL580T39BKT-'0Gbl0qFcF,540UH0'SJmYPTKBR ,c\"F0bFs40.BA?\n",
            "BL0.0“bab0p,4g00e0GK0bL5jkf6bYLCH–aJ05Sy,bJ5cu0J 73f0JJrJ,TK0 \n",
            "----\n",
            "iter 33500, loss 200.973464\n"
          ]
        }
      ],
      "source": [
        "Hidden_Layer_size = 100 #size of the hidden layer\n",
        "Time_steps = 40 # Number of time steps (length of the sequence) used for training\n",
        "learning_rate = 1e-1 # Learning Rate\n",
        "weight_sd = 0.1 #Standard deviation of weights for initialization\n",
        "z_size = Hidden_Layer_size + X_size #Size of concatenation(H, X) vector\n",
        "\n",
        "size_a =  Hidden_Layer_size\n",
        "size_b =  z_size \n",
        "size_c =  X_size\n",
        "\n",
        "class Parameters:\n",
        "    def __init__(self):\n",
        "        self.W_f = Param('W_f', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_f = Param('b_f', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_i = Param('W_i', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_i = Param('b_i', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_C = Param('W_C', np.random.randn(size_a, size_b) * weight_sd)\n",
        "        self.b_C = Param('b_C', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_o = Param('W_o', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_o = Param('b_o', np.zeros((size_a, 1)))\n",
        "\n",
        "        #For final layer to predict the next character\n",
        "        self.W_v = Param('W_v', np.random.randn(X_size, size_a) * weight_sd)\n",
        "        self.b_v = Param('b_v', np.zeros((size_c, 1)))\n",
        "        \n",
        "    def all(self):\n",
        "        return [self.W_f, self.W_i, self.W_C, self.W_o, self.W_v,\n",
        "               self.b_f, self.b_i, self.b_C, self.b_o, self.b_v]\n",
        "\n",
        "parameters = Parameters()\n",
        "\n",
        "def forward(x, h_prev, C_prev, p = parameters):\n",
        "    assert x.shape == (X_size, 1)\n",
        "    assert h_prev.shape == (Hidden_Layer_size, 1)\n",
        "    assert C_prev.shape == (Hidden_Layer_size, 1)\n",
        "    \n",
        "    z = np.row_stack((h_prev, x))\n",
        "    f = sigmoid(np.dot(p.W_f.v, z) + p.b_f.v)\n",
        "    i = sigmoid(np.dot(p.W_i.v, z) + p.b_i.v)\n",
        "    C_bar = tanh(np.dot(p.W_i.v, z) + p.b_C.v)\n",
        "\n",
        "    C = f * C_prev + i * C_bar\n",
        "    o = sigmoid(np.dot(p.W_o.v, z) + p.b_i.v)\n",
        "    h = o * tanh(C)\n",
        "    \n",
        "    v = np.dot(p.W_v.v, h) + p.b_v.v\n",
        "    y = np.exp(v) / np.sum(np.exp(v)) #softmax\n",
        "\n",
        "    return z, f, i, C_bar, C, o, h, v, y\n",
        "\n",
        "def backward(target, dh_next, dC_next, C_prev,\n",
        "             z, f, i, C_bar, C, o, h, v, y,\n",
        "             p = parameters):\n",
        "    \n",
        "    assert z.shape == (X_size + Hidden_Layer_size, 1)\n",
        "    assert v.shape == (X_size, 1)\n",
        "    assert y.shape == (X_size, 1)\n",
        "    \n",
        "    for param in [dh_next, dC_next, C_prev, f, i, C_bar, C, o, h]:\n",
        "        assert param.shape == (Hidden_Layer_size, 1)\n",
        "        \n",
        "    dv = np.copy(y)\n",
        "    dv[target] -= 1\n",
        "\n",
        "    p.W_v.d += np.dot(dv, h.T)\n",
        "    p.b_v.d += dv\n",
        "\n",
        "    dh = np.dot(p.W_v.v.T, dv)        \n",
        "    dh += dh_next\n",
        "    do = dh * tanh(C)\n",
        "    do = dsigmoid(o) * do\n",
        "    p.W_o.d += np.dot(do, z.T)\n",
        "    p.b_o.d += do\n",
        "\n",
        "    dC = np.copy(dC_next)\n",
        "    dC += dh * o * dtanh(tanh(C))\n",
        "    dC_bar = dC * i\n",
        "    dC_bar = dtanh(C_bar) * dC_bar\n",
        "    p.W_C.d += np.dot(dC_bar, z.T)\n",
        "    p.b_C.d += dC_bar\n",
        "\n",
        "    di = dC * C_bar\n",
        "    di = dsigmoid(i) * di\n",
        "    p.W_i.d += np.dot(di, z.T)\n",
        "    p.b_i.d += di\n",
        "\n",
        "    df = dC * C_prev\n",
        "    df = dsigmoid(f) * df\n",
        "    p.W_f.d += np.dot(df, z.T)\n",
        "    p.b_f.d += df\n",
        "\n",
        "    dz = (np.dot(p.W_f.v.T, df)\n",
        "         + np.dot(p.W_i.v.T, di)\n",
        "         + np.dot(p.W_C.v.T, dC_bar)\n",
        "         + np.dot(p.W_o.v.T, do))\n",
        "    dh_prev = dz[:Hidden_Layer_size, :]\n",
        "    dC_prev = f * dC\n",
        "    \n",
        "    return dh_prev, dC_prev\n",
        "\n",
        "parameters = Parameters()\n",
        "\n",
        "\n",
        "# Exponential average of loss\n",
        "# Initialize to a error of a random model\n",
        "smooth_loss = -np.log(1.0 / X_size) * Time_steps\n",
        "\n",
        "iteration, pointer = 0, 0\n",
        "\n",
        "# For the graph\n",
        "plot_iter = np.zeros((0))\n",
        "plot_loss = np.zeros((0))\n",
        "\n",
        "\n",
        "iter = 50000\n",
        "while iter > 0:\n",
        "  # Reset\n",
        "  if pointer + Time_steps >= len(data) or iteration == 0:\n",
        "      g_h_prev = np.zeros((Hidden_Layer_size, 1))\n",
        "      g_C_prev = np.zeros((Hidden_Layer_size, 1))\n",
        "      pointer = 0\n",
        "\n",
        "\n",
        "  inputs = ([char_to_idx[ch] \n",
        "              for ch in data[pointer: pointer + Time_steps]])\n",
        "  targets = ([char_to_idx[ch] \n",
        "              for ch in data[pointer + 1: pointer + Time_steps + 1]])\n",
        "\n",
        "  loss, g_h_prev, g_C_prev = \\\n",
        "      forward_backward(inputs, targets, g_h_prev, g_C_prev)\n",
        "  smooth_loss = smooth_loss * 0.999 + loss * 0.001\n",
        "\n",
        "  # Print every hundred steps\n",
        "  if iteration % 100 == 0:\n",
        "      update_status(inputs, g_h_prev, g_C_prev)\n",
        "\n",
        "  update_paramters()\n",
        "\n",
        "  plot_iter = np.append(plot_iter, [iteration])\n",
        "  plot_loss = np.append(plot_loss, [loss])\n",
        "\n",
        "  pointer += Time_steps\n",
        "  iteration += 1\n",
        "  iter = iter -1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Pnk8hBQ1bnIy"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "S4_LSTM_PARAMS.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}