9. Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "sns.set_style('whitegrid')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class neural_network(object):\n",
    "    \n",
    "    def __init__(self, netarch, n_epochs=1000, eta=0.1, printing=False):\n",
    "        \n",
    "        #initialize layer and unit info\n",
    "        self.num_layers = len(netarch) #number of layers\n",
    "        self.netarch = netarch #number of units in each layer\n",
    "        self.n_epochs = n_epochs #number of epochs\n",
    "        self.eta = eta #learning rate\n",
    "        self.printing = printing #printing options\n",
    "        \n",
    "        #initialize weights and bias matrices\n",
    "        self.initialize_weight_bias()\n",
    "        \n",
    "        #printing options\n",
    "        if printing:\n",
    "            print('#'*40)\n",
    "            print('Neural Net v1.0')\n",
    "            print('#'*40)\n",
    "            \n",
    "            print('Initializing neural network...')\n",
    "            time.sleep(2)\n",
    "            \n",
    "            print('\\nActivation: Sigmoid')\n",
    "            print('Number of layers: {} ({} hidden)'.format(self.num_layers,self.num_layers-2))\n",
    "            print('-'*40)\n",
    "            \n",
    "            \n",
    "            for i, units in enumerate(self.netarch):\n",
    "                if i == 0:\n",
    "                    print('Input Layer: {} units'.format(units))\n",
    "                \n",
    "                elif i == len(netarch)-1:\n",
    "                    print('Output Layer: {} units'.format(units))\n",
    "                \n",
    "                else:\n",
    "                    print('Hidden Layer: {} units'.format(units))\n",
    "            \n",
    "            print('-'*40)\n",
    "            \n",
    "            #for w, b in zip(self.w_keys, self.b_keys):\n",
    "            #    print(w, self.weights[w].shape)\n",
    "            #    print(b, self.biases[b].shape)\n",
    "\n",
    "        print('#'*40)\n",
    "        \n",
    "    def initialize_weight_bias(self):\n",
    "        weights = {}\n",
    "        biases = {}\n",
    "        \n",
    "        #create weight and bias matrix for each layer\n",
    "        for i, layer in enumerate(self.netarch):\n",
    "            \n",
    "            if i == len(self.netarch)-1:\n",
    "                continue\n",
    "            \n",
    "            #if connected to output layer...'out' in name key\n",
    "            if i == len(netarch)-2:\n",
    "                weights[\"wout{0}\".format(i)] = np.random.uniform(size=(self.netarch[i], self.netarch[i+1]))\n",
    "                biases[\"bout{0}\".format(i)] = np.random.uniform(size=(1, self.netarch[i+1]))\n",
    "            \n",
    "            else:\n",
    "                weights[\"wh{0}\".format(i)] = np.random.uniform(size=(self.netarch[i], self.netarch[i+1]))\n",
    "                biases[\"bh{0}\".format(i)] = np.random.uniform(size=(1, self.netarch[i+1]))\n",
    "                \n",
    "        self.weights = weights\n",
    "        self.biases = biases\n",
    "        self.w_keys = list(self.weights.keys())\n",
    "        self.b_keys = list(self.biases.keys())\n",
    "        \n",
    "    def initialize_activations(self):\n",
    "        \n",
    "        layer_activations = {'input': self.X}\n",
    "        \n",
    "        for i, units in enumerate(self.netarch):\n",
    "                if i == 0:\n",
    "                    layer_activations['input'] = self.X\n",
    "                \n",
    "                elif i == len(netarch)-1:\n",
    "                    layer_activations['output'] = None\n",
    "                \n",
    "                else:\n",
    "                    layer_activations['HL{}'.format(i)] = None\n",
    "                    \n",
    "        self.layer_activations = layer_activations\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        \"\"\"Fit training data.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        X : {array-like}, shape = [n_samples, n_features]\n",
    "          :Training vectors, where n_samples is the number of samples and n_features is the number of features.\n",
    "        \n",
    "        y : array-like, shape = [n_samples]\n",
    "          :Target values\n",
    "        \n",
    "        Returns\n",
    "        -------\n",
    "        self : object\n",
    "        \"\"\"  \n",
    "        print('Fitting neural network...')\n",
    "        time.sleep(2)\n",
    "        \n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        \n",
    "        \n",
    "        ws = [None]\n",
    "        bs = [None]\n",
    "        \n",
    "        for w in self.w_keys:\n",
    "            ws.append(w)\n",
    "        for b in self.b_keys:\n",
    "            bs.append(b)\n",
    "        \n",
    "        self.initialize_activations()\n",
    "        #print(len(self.layer_activations), len(ws), len(bs))\n",
    "        \n",
    "        self.epoch_error = []\n",
    "        \n",
    "        activ_update = {}\n",
    "        \n",
    "        for i in range(self.n_epochs):\n",
    "        \n",
    "            ##############################\n",
    "            ## Feed-Forward Propogation ##\n",
    "            ##############################\n",
    "            \n",
    "            #for each layer...feed forward\n",
    "            for i, (weight, bias) in enumerate(zip(ws, bs)):\n",
    "                #print(activ_update.keys())\n",
    "                #print(i, weight, bias)\n",
    "                \n",
    "                #if first layer...no activation to calculate --> X_train\n",
    "                if i == 0:\n",
    "                    pass\n",
    "                \n",
    "                #if first hidden layer...activation is input X_train\n",
    "                elif i == 1:\n",
    "                    input_act = self.X\n",
    "                    activ_update['HL{}'.format(i)] = self.feed_forward(input_act, self.weights[weight], self.biases[bias])\n",
    "                    #print('input_activation:','X', input_act)\n",
    "            \n",
    "                #if last layer, save activation as 'output'\n",
    "                elif i == len(ws)-1:\n",
    "                    #feed prior layer's activation and name 'output'\n",
    "                    inpt = list(activ_update.keys())[i-2]\n",
    "                    input_act = activ_update[inpt]\n",
    "                    activ_update['output'] = self.feed_forward(input_act, self.weights[weight], self.biases[bias])\n",
    "                    #print('input_activation:', inpt, input_act)\n",
    "                \n",
    "                else:\n",
    "                    #feed prior layer's activation and name 'HL_'\n",
    "                    inpt = list(activ_update.keys())[i-2]\n",
    "                    input_act = activ_update[inpt]\n",
    "                    activ_update['HL{}'.format(i)] = self.feed_forward(input_act, self.weights[weight], self.biases[bias])\n",
    "                    #print('input_activation:',inpt, input_act)\n",
    "                    \n",
    "                    \n",
    "                #print()\n",
    "             \n",
    "            self.layer_activations = activ_update\n",
    "            \n",
    "             ##############################\n",
    "             ###### Back-Propogation ######\n",
    "             ##############################\n",
    "                \n",
    "            self.backprop()\n",
    "            self.update_weights_bias()\n",
    "        \n",
    "        print('-'*40)\n",
    "        print('Locked and loaded.')\n",
    "        print('#'*40)\n",
    "        \n",
    "        return self\n",
    "            \n",
    "    \n",
    "    def feed_forward(self, inpt, weight, bias):\n",
    "        #feed forward propogation\n",
    "        layer_input = np.dot(inpt, weight)\n",
    "        layer_input += bias\n",
    "        layer_activation = self.sigmoid(layer_input)\n",
    "        return layer_activation\n",
    "    \n",
    "    def backprop(self):\n",
    "        acts_rev = list(self.layer_activations)[::-1]\n",
    "        output = self.layer_activations['output']\n",
    "    \n",
    "        #calculate output error\n",
    "        output_error = self.y - output #actual - output\n",
    "        \n",
    "        #record epoch MSE\n",
    "        epoch_mse = np.mean(output_error**2) #mean squared error\n",
    "        self.epoch_error.append(epoch_mse) #record epoch error\n",
    "        \n",
    "        #prep errors with zero vectors for each layer\n",
    "        errors= {}\n",
    "        \n",
    "        for activation in acts_rev:\n",
    "            errors['{}_error'.format(activation)] = np.zeros(len(output_error))\n",
    "        \n",
    "        #update 'output_error'\n",
    "        errors['output_error'] = output_error\n",
    "        \n",
    "        ## \n",
    "        ##calculate gradients for each activation layer\n",
    "        ##\n",
    "        \n",
    "        gradients = {}\n",
    "        \n",
    "        for activation in acts_rev:\n",
    "            #derivative of sigmoid activation function\n",
    "            gradients['{}_gradient'.format(activation)] = self.sigmoid_prime(self.layer_activations[activation]) \n",
    "            \n",
    "            \n",
    "        ##\n",
    "        ##compute change factor delta for each layer, dependent on its gradient of error * gradient of its activation \n",
    "        ##\n",
    "        \n",
    "        deltas = {}\n",
    "        \n",
    "        #for each layer, going backwards...\n",
    "        for i, (activation, gradient, error) in enumerate(zip(acts_rev, gradients, errors)):\n",
    "            \n",
    "            #compute layer delta\n",
    "            deltas['delta_{}'.format(activation)] = errors[activation+'_error'] * gradients[activation+'_gradient']\n",
    "        \n",
    "            #if not headed into input layer...(b/c there is no error from the input)\n",
    "            if i == len(acts_rev)-1:\n",
    "                continue\n",
    "            \n",
    "            #compute error from previous layer (i.e if output, find error for last hidden layer)\n",
    "            else:\n",
    "                errors[acts_rev[i+1]+'_error'] = np.dot(deltas['delta_'+activation], self.weights[self.w_keys[::-1][i]].T)\n",
    "        \n",
    "        #save\n",
    "        self.gradients = gradients\n",
    "        self.deltas = deltas\n",
    "        self.errors= errors\n",
    "\n",
    "        ##\n",
    "        ##REFERENCE MATERIAL\n",
    "        ##\n",
    "        \n",
    "        #output wout1 bout1\n",
    "        #HL1 wh0 bh0\n",
    "        \n",
    "        #d_output = output_erro * output_gradient\n",
    "        #HL1_error = d_output.dot(wout.T)\n",
    "        #d_HL = HL1_error * HL1_gradient\n",
    "        \n",
    "        \n",
    "    def update_weights_bias(self):\n",
    "        #get layers backwards\n",
    "        acts_rev = list(self.layer_activations)[::-1]\n",
    "        w_keys_rev = list(self.w_keys)[::-1]\n",
    "        b_keys_rev = list(self.b_keys)[::-1]\n",
    "        \n",
    "        #grab current weight matricies for update\n",
    "        update_weights = self.weights\n",
    "        update_bias = self.biases\n",
    "        \n",
    "        #for each layer, moving backwards...\n",
    "        for i, (activation, weight, bias) in enumerate(zip(acts_rev, w_keys_rev, b_keys_rev)):          \n",
    "            #output wout1 bout1\n",
    "            #HL1 wh0 bh0\n",
    "            \n",
    "            #retrieve layer delta\n",
    "            delta_ = self.deltas['delta_'+activation]\n",
    "            delta_ = np.array(delta_, 'float')\n",
    "            \n",
    "            #update bias vector\n",
    "            update_bias[bias] += np.sum(delta_, axis=0, keepdims=True) * self.eta\n",
    "            \n",
    "            #if headed into input layer...\n",
    "            if i == len(acts_rev)-1:\n",
    "                \n",
    "                #use input X to update weight matrix\n",
    "                update_weights[weight] += X.T.dot(delta_) * self.eta\n",
    "                \n",
    "            #otherwise...\n",
    "            else:\n",
    "                \n",
    "                #use layer's activation feed to...\n",
    "                act = self.layer_activations[acts_rev[i+1]]\n",
    "                act = np.array(act, 'float')\n",
    "                \n",
    "                #update weight matrix\n",
    "                update_weights[weight] += act.T.dot(delta_) * self.eta\n",
    "            \n",
    "        #save updated weight and bias matricies   \n",
    "        self.weights = update_weights\n",
    "        self.biases = update_bias  \n",
    "     \n",
    "    \n",
    "    ##Miscellaneous functions  \n",
    "    def sigmoid(self, z):\n",
    "        \"\"\"Transforms z to real number between 0 and 1\"\"\"\n",
    "        sigmoid = 1.0 / (1.0 + np.exp(-z))\n",
    "        \n",
    "        return sigmoid\n",
    "    \n",
    "    def sigmoid_prime(self, x):\n",
    "        \"\"\"Derivative of sigmoid\"\"\"\n",
    "        sigmoid_prime = x * (1 - x)\n",
    "        \n",
    "        return sigmoid_prime\n",
    "    \n",
    "    def predict(self, X):\n",
    "        \"\"\"Return class label after unit step\"\"\"\n",
    "        return np.where(X >= 0.5, 1, 0)\n",
    "    \n",
    "    def score(self, X, y):\n",
    "        \"\"\"Score the model --> Actual vs Predicted Classes\"\"\"\n",
    "        results = np.equal(X,y)\n",
    "        correct = np.sum(results)\n",
    "        score = float(correct) / float(len(results))\n",
    "        return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import sklearn\n",
    "#from sklearn.datasets import load_breast_cancer\n",
    "#from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "#\n",
    "##load data\n",
    "#raw = sklearn.datasets.load_breast_cancer()\n",
    "##\n",
    "###optional description\n",
    "###[print(line) for line in raw.DESCR.split('\\n')]\n",
    "#\n",
    "###load feature and target d\n",
    "#X = pd.DataFrame(raw.data, columns=raw.feature_names).values\n",
    "#y = pd.Series(raw.target).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Input array\n",
    "X=np.array([[1,0,1,0],[1,0,1,1],[0,1,0,1]])\n",
    "\n",
    "#Output\n",
    "y=np.array([[1],[1],[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#define dimensions and number of classes\n",
    "dims = X.shape[1]\n",
    "nb_classes = pd.DataFrame(y).shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Will\\Anaconda3\\envs\\py35\\lib\\site-packages\\sklearn\\utils\\validation.py:444: DataConversionWarning: Data with input dtype int32 was converted to float64 by StandardScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    }
   ],
   "source": [
    "#standardize the data\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X = scaler.transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#cross-validation\n",
    "#X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, test_size=.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#define network architecture\n",
    "\n",
    "#input layer with same dimensions as feature set\n",
    "input_layer = dims\n",
    "\n",
    "#user-defined units in hidden layer\n",
    "hidden_layer1 = dims * 1.2\n",
    "#hidden_layer2 = dims * 1.2\n",
    "#hidden_layer3 = dims * 1.2\n",
    "        \n",
    "#output layer with 1 unit; one for each class        \n",
    "output_layer = nb_classes\n",
    "\n",
    "#netarch\n",
    "netarch = [input_layer,\n",
    "           hidden_layer1,\n",
    "           #hidden_layer2,\n",
    "           #hidden_layer3,\n",
    "           output_layer]\n",
    "\n",
    "#ensure all elements are integers\n",
    "netarch = [int(layer) for layer in netarch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "########################################\n",
      "Neural Net v1.0\n",
      "########################################\n",
      "Initializing neural network...\n",
      "\n",
      "Activation: Sigmoid\n",
      "Number of layers: 3 (1 hidden)\n",
      "----------------------------------------\n",
      "Input Layer: 4 units\n",
      "Hidden Layer: 4 units\n",
      "Output Layer: 1 units\n",
      "----------------------------------------\n",
      "########################################\n",
      "Fitting neural network...\n",
      "----------------------------------------\n",
      "Locked and loaded.\n",
      "########################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<__main__.neural_network at 0x21141ff7dd8>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#run neural net\n",
    "net = neural_network(netarch, n_epochs=5000, eta=0.1, printing=True)\n",
    "net.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x211420105f8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmsAAAF8CAYAAACQd/xaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYpVVh5/HvvbXd2rvp7qpmETqyHEqFRkEEJIpGxmgk\nIIlbm1FRYtAxmjEzI8ZMnknikkWSMcm4S0yMUYgOk2AiYCQa2TQ0oo1UH2hkaaH3pXqprvXe+eN9\nq/t2dVVXL3Xr3vfe7+eRp97tnHtuHdFfnfO+582VSiUkSZJUm/LVboAkSZJmZ1iTJEmqYYY1SZKk\nGmZYkyRJqmGGNUmSpBpmWJMkSaphzdVugKTqCyGsAB4Hfj3G+Pmy4/8NeF6M8W0L0Ib/BSyNMb5n\n2vG3AZ8CXhhjfKjs+DeAr8UYvzhHvZ8DPh1jXD3t+GXAN4EIlIAcMAH8fozx1qNs+3nA14Eh4OoY\n4xNHU16SDseRNUlTisDHQwhnVbshM8gBXwkhFI6h7OVp+Zk8FmM8L8b4/BjjecA70s9ZdpSf8cvA\nv8UYX2BQkzTfHFmTNGUfcANJWLk4xjhWfjKE0Ar8MfBSoAn4IfDeGOOuEMITwK/GGO9Pr30C+FVg\nK/A9YBBYkZa9BrgKKACdwH+LMd4yR9u+DbQCHwfeM/1kCOFk4K+AU4EW4Ksxxo+GED4CnAR8OYTw\nlhjj9w/3ITHGH4UQhoHTgC0hhA8Bv0Lyh+0TwLtjjM+EEL4DbAfOBm4C3gU0hRDaY4xvDiH8T+BN\nJCN1jwDviTFunFbuU2ndq4GXA33AJ4D+9PfUCbw+xrgmhHAR8CdAG3Ai8K0Y4zvSEdFvA/8CvAg4\nAfhQjPGmEEJzWuY1aTvuSds/Ntv3mqMPJFWJI2uSyn0E2At8dIZz15P8n/75McaVwDPAHx1BnacA\nfxhjPIskcL0CeGmM8VzgQ8AfHEEdJeAtwOtDCK+Z4fyXgBtjjOcDFwKvCCG8Psb4obSdb54rqAGE\nEK4mGWF8OITwFuAc4MJ01O1fgM+XXb4jxvicGOPvA58GbkqD2jXAq0imbc8FHgK+OEO5v0z3V8QY\nnw9cTRKGvxNjvAC4DfjN9Jr3Ab8XY3wR8Bzgl0MI56fnng3cHmO8EPgASUADeDdwPrASeB7QDbzh\nCL6XpBrjyJqk/WKMxRDCrwE/DCHcPu30a4BFwOUhBEiC1+YjqHYCuDet/8kQwluBN4cQzgAuArqO\nsG0bQgjvAG4MIZw7dTyE0EkyEnVCCOEP08NdwHnAzXNUe3oI4cF0uwVYD1wZYxxOQ+GFwP3p920C\nOsrKfm+WOl8F/HWMcW+6/wngQ+nI5Ezl/m/687H0521l+5el228FXh1C+B2SUbmO9DtuA8ZJAhfA\nAySja5CE4i/FGPel+28ACCHcPMf3klRjDGuSDhJjfCqEcB3wN8Dflp1qAt4XY/wmQAihi2QqEw7c\noD+ltWx7NMY4kZZ5AfCPwJ8DdwDfJZkOPNK23RpC+Ie0XeNl7coBl8QYh9PPWQqMHEGVj6WjSzNp\nAv44xviptM42YHHZ+T2zlJs+Y5En+d/aqd/P9HKj5TsxxnEO9T3gRyRB7maSKc+p+sZijMV0u7wf\nJtJ90vb3p22Z63tJqjFOg0o6RIzxH0ielPytssO3A+8JIbSGEPLA54CPpee2ABcApPdXnThL1S8B\n7o8x/hlJULuKJDwcjd8muQ/tF9K27gLuA96ffv4i4G7gyvT6CZJRs6N1O3BtCKEn3f8DkunWIyl3\nTTriB/Be4N9jjKOHKTOrEMJikt/tB2KM/xc4GTiDuX9v/wqsCiG0pf31KZL76I71e0mqEsOapNm8\nF3iybP8PSW5G/yHwMMkIzm+n5z4AvC+dUvx1kpvmZ/IVYGkI4eH0mj0k05fdR9qoGOMISegolR1e\nBVwUQlgDfB/4Sozxy+m5/wfcFEL4T0f6GanPA98A7gsh/AQ4F3jbEZT7AklQ+kEIYRB4AfDmo/zs\n/WKMO0hC8QMhhPuBD5KE0TPmKPoZkt/xamANsAH4C479e0mqklypVJr7KkmSJFWFI2uSJEk1zLAm\nSZJUwwxrkiRJNcywJkmSVMMMa5IkSTUsU4virl692kdXJUlSZpx//vm5ua86vEyFNYDzzz9/7otU\nkwYHBxkYGKh2M3SM7L/ssu+yzf7LrtWrZ1ty8ug4DSpJklTDDGuSJEk1zLAmSZJUwwxrkiRJNcyw\nJkmSVMMMa5IkSTWsIkt3hBDywCeBlcAocG2McV3Z+TcBvwVMAGuAd8cYiyGEB4Bd6WWPxxivqUT7\nJEmSsqJS66xdBRRijBeHEC4CbgCuBAghtAMfBs6JMQ6HEL4CvCaEcAeQizFeVqE2SZIkZU6lpkEv\nBW4DiDHeB1xQdm4UuCTGOJzuNwMjJKNwHSGEO0IId6YhT5IkqaFVKqz1AENl+5MhhGaAGGMxxrgJ\nIITwm0AX8C1gGPg48ErgOuDLU2UkSZIaVaXC0C6gu2w/H2OcmNpJ72n7E+As4FdijKUQwiPAuhhj\nCXgkhLANOBFYX17x4OBghZqsShsZGbH/Msz+yy77LtvsP1UqrN0NXAHcnE5nrpl2/jMk06FXxRiL\n6bG3A+cA7w4hnEQyOrdhesW+Hy27fL9dttl/2WXfZZv9l13z9W7QSoW1W4DLQwj3ADngmhDCKpIp\nz/uBdwDfA+4MIQB8AvgC8MUQwl1ACXh7+WjclMliiab8cb/AXpIkKRMqEtbS0bLrph1eW7Y9271y\nq+aqe+OuEU5e1H6sTZMkScqUzC2Ku3N4rNpNkCRJWjAZDGvj1W6CJEnSgjGsSZIk1bDMhbUdToNK\nkqQGkrmwNrTPkTVJktQ4MhfWdux1ZE2SJDWO7IU171mTJEkNJHNhbWifI2uSJKlxZC6sObImSZIa\nSebCmoviSpKkRpLBsObImiRJahzZC2v7ximVStVuhiRJ0oLIXFibLJbYPTpR7WZIkiQtiMyFNYAh\np0IlSVKDyGRY85VTkiSpUWQ0rDmyJkmSGkMmw5rLd0iSpEaR0bDmyJokSWoMhjVJkqQalrmw1t3W\n7AMGkiSpYWQurC3qbGFonyNrkiSpMWQurC3uaHVkTZIkNYzMhbXe9hbvWZMkSQ0jc2FtcUerS3dI\nkqSGkbmwtqijxUVxJUlSw8hgWGtl18g4k8VStZsiSZJUcdkLa+0tlEqwyydCJUlSA8hcWFvc2QLA\nTsOaJElqAJkLa4s6WgFcvkOSJDWE7IW19mRkbciHDCRJUgPIXFhb7MiaJElqIJkLa4s60nvWHFmT\nJEkNIHNhrafQQi6HC+NKkqSGkLmwls/n6G13YVxJktQYMhfWIH3llEt3SJKkBpDJsLaoo8VpUEmS\n1BCyGdbaW3zAQJIkNYRMhrXFHa0u3SFJkhpCJsNab0eLi+JKkqSGkMmwtrijld2jE4xPFqvdFEmS\npIrKaFhLXznlE6GSJKnOZTKs9aavnPKJUEmSVO8yGdamRtZcGFeSJNW7TIa1Re1TI2uGNUmSVN+y\nGdb2j6w5DSpJkupbpsOay3dIkqR6l8mw1tXWTEtTzpE1SZJU9zIZ1nK5HCd0trJtj2FNkiTVt0yG\nNYAlnW1s2zta7WZIkiRVVHbDWlcrWxxZkyRJdS6zYW1ZVxvb9jiyJkmS6ltmw9qSrla27hmlVCpV\nuymSJEkV01yJSkMIeeCTwEpgFLg2xriu7PybgN8CJoA1wLvTU7OWmW5pVxsj40WGxybpbKvI15Ak\nSaq6So2sXQUUYowXA9cDN0ydCCG0Ax8GXhZjfDHQC7zmcGVmsqSrDcAnQiVJUl2rVFi7FLgNIMZ4\nH3BB2blR4JIY43C63wyMzFHmEEu7kldObfG+NUmSVMcqFdZ6gKGy/ckQQjNAjLEYY9wEEEL4TaAL\n+Nbhysxk6f6RNcOaJEmqX5W62WsX0F22n48xTkztpPe0/QlwFvArMcZSCOGwZaYMDg4CsH1vcmrN\nuid5Vn7HvH8Bzb+RkZH9/afssf+yy77LNvtPlQprdwNXADeHEC4ieYig3GdIpkOvijEWj7AMAAMD\nAwCcPlGErz1Fa9cJDAycWYGvoPk2ODi4v/+UPfZfdtl32Wb/Zdfq1avnpZ5KhbVbgMtDCPcAOeCa\nEMIqkinP+4F3AN8D7gwhAHxipjKH+4DW5jw9hWa2Og0qSZLqWEXCWjpadt20w2vLtme7V256mcNa\n2t3G1r0+DSpJkupXZhfFBVja6VsMJElSfct0WEveYuDImiRJql+ZDmtLfT+oJEmqc5kOa0u6Wtkx\nPM74ZHHuiyVJkjIo02FtamHc7T5kIEmS6lSmw9qy7iSsbd7lVKgkSapPmQ5r/T0FADbvHqlySyRJ\nkioj02Gtb2pkbbcja5IkqT5lOqxN3bPmNKgkSapXmQ5rrc15TuhsdRpUkiTVrUyHNUimQjc5siZJ\nkupU5sPasu42tjiyJkmS6lTmw1p/T8EHDCRJUt3KfFjr625jy+5RisVStZsiSZI07+oirE0US+wY\n9i0GkiSp/mQ/rO1fGNepUEmSVH+yH9bShXE37fIhA0mSVH/qIKw5siZJkupX9sNaTzKytsWwJkmS\n6lDmw1qhpYmeQjObnQaVJEl1KPNhDZKHDJwGlSRJ9ag+wlp3mw8YSJKkulQ3Yc2RNUmSVI/qI6yl\n06Clkm8xkCRJ9aU+wlp3G2MTRXbtm6h2UyRJkuZVfYS1/W8x8L41SZJUX+ojrKVvMfC+NUmSVG/q\nKqz5RKgkSao3dRHW+n2ZuyRJqlN1EdY625rpamt2ZE2SJNWdughrkLwjdPMuR9YkSVJ9qZuw1t9d\ncGRNkiTVnfoJaz1tbHLpDkmSVGfqKKwV2LTLtxhIkqT6Ujdhra+nwNhEkaF949VuiiRJ0rypm7C2\nPF2+Y6P3rUmSpDpSN2Gtv2dqYVyfCJUkSfWjjsJaMrLmE6GSJKme1E1YWzb1flDDmiRJqiN1E9YK\nLU0s6mhxGlSSJNWVuglr4MK4kiSp/tRVWOvraTOsSZKkulJXYW15ujCuJElSvairsNbfU2DLnlEm\ni77FQJIk1Yc6C2ttTBZLbNvr6JokSaoPdRXW+tK11jY7FSpJkupEXYU1F8aVJEn1ps7Cmq+ckiRJ\n9aWuwtrSrjZyOV/mLkmS6kddhbWWpjxLu9p85ZQkSaobdRXWIJkK9Z41SZJUL+ovrHW7MK4kSaof\nzZWoNISQBz4JrARGgWtjjOumXdMBfAt4R4xxbXrsAWBXesnjMcZrjvaz+3oK/OhnO4+n+ZIkSTWj\nImENuAooxBgvDiFcBNwAXDl1MoRwAfBp4JSyYwUgF2O87Hg+uL+nja17xhifLNLSVHcDh5IkqcFU\nKs1cCtwGEGO8D7hg2vk24LXA2rJjK4GOEMIdIYQ705B31KbWWtuy26lQSZKUfZUKaz3AUNn+ZAhh\n/yhejPHuGOP6aWWGgY8DrwSuA75cXuZITa215vIdkiSpHlRqGnQX0F22n48xTsxR5hFgXYyxBDwS\nQtgGnAgcFOoGBwcPW8nw9mRE7YGHH6N9b+dRNluVNDIyMmf/qXbZf9ll32Wb/adKhbW7gSuAm9Pp\nzDVHUObtwDnAu0MIJ5GMzm2YftHAwMBhK1m2ZxRufZqWnqUMDKw42narggYHB+fsP9Uu+y+77Lts\ns/+ya/Xq1fNST6XC2i3A5SGEe4AccE0IYRXQFWP87CxlvgB8MYRwF1AC3n4Eo3GHOKGjleZ8zrXW\nJElSXahIWIsxFknuOyu3dobrLivbHgNWHe9n5/M5+rrbXGtNkiTVhbpc26Kvp8Dm3Y6sSZKk7KvL\nsOYrpyRJUr2o07BWYOOQYU2SJGVf3Ya1XSMT7BubrHZTJEmSjkvdhjXA+9YkSVLm1WlYS95i4BOh\nkiQp6+o0rCUjaz5kIEmSsq4+w1q3YU2SJNWHugxrPe3NtDXnDWuSJCnz6jKs5XI5+nsK3rMmSZIy\nry7DGsDynoIja5IkKfPqNqz19bSxebcja5IkKdvqNqz1pyNrpVKp2k2RJEk6ZnUc1toYHptkz+hE\ntZsiSZJ0zOo4rE0t3+FUqCRJyq66DWt9rrUmSZLqQN2GtQOvnDKsSZKk7KrjsOY0qCRJyr5jCmsh\nhLb5bsh862xrprut2ZE1SZKUaYcNayGEm8q2f7vs1Dcr1qJ5lKy1ZliTJEnZNdfIWl/Z9i+Vbecq\n0JZ55yunJElS1h3NNGh5QMvESrP9vnJKkiRl3FxhrTTLdib09bSxedeobzGQJEmZ1TzH+eeGEP6e\nZFStfPs5FW/ZPOjvLjA2WWTH8DgndLZWuzmSJElHba6w9vqy7U/Psl2zlvceWBjXsCZJkrLosNOg\nMcbvAjvTn/cCzwPOBL63AG07bi6MK0mSsm6upTveD3w2hNAM/ClwOXAO8OcL0LbjNvXKqc0+ESpJ\nkjJqrmnQ1wGXkDxcsAo4M8a4M4RwT8VbNg/6HFmTJEkZN9fToLtjjJPAecBPY4w70+OZWGetrbmJ\nxR0tbHJhXEmSlFFzLt0RQjgLuAa4FSCEcCYwUemGzZf+ngIbh5wGlSRJ2TTXNOjvAl8CNgIfDCG8\nNN1//WFL1ZC+noKvnJIkSZk1V1h7F/ATkmnPvwDaSZ4EfSdwX2WbNj+W97QRN+6qdjMkSZKOyVxh\n7QKSgPZl4B4ycq9auf6eAlt2jzJZLNGUz1zzJUlSg5trnbVzgdcCBeB64GLgsRjj7QvQtnnR11Og\nWIJte7xvTZIkZc+cL3KPMT4UY7w+xvhy4E7gYyGETEyBAvR3Ty3fYViTJEnZM9c0KAAhhG7gauBN\nQCfwd5Vs1Hzq70kWxt24a4Rz6K1yayRJko7OYcNaCOH1wBuB04CvA9fFGJ9YgHbNm6mw5sK4kiQp\ni+YaWfsqsBb4Eclrpj4aQgAgxriqsk2bH0u7WsnnYLNhTZIkZdBcYe1lC9KKCmpuyrO0q8171iRJ\nUiYdNqzFGL+7UA2ppP6egq+ckiRJmTTn06D1oL/HkTVJkpRNDRHW+noK3rMmSZIyqSHCWn93gW17\nxxidmKx2UyRJko5KY4S1nmRh3C27nQqVJEnZ0iBhbWqtNcOaJEnKloYKa963JkmSsqZBwtrU+0EN\na5IkKVsaIqwt7milpSnHJu9ZkyRJGdMQYS2fz9HXXXBkTZIkZU5DhDWAvp42w5okScqchglr/d0F\nnwaVJEmZ0zhhzZE1SZKUQYd9kfuxCiHkgU8CK4FR4NoY47pp13QA3wLeEWNceyRljkd/b4HdIxMM\nj03Q0VqRry1JkjTvKjWydhVQiDFeDFwP3FB+MoRwAfDvwOlHWuZ49XdPrbXmVKgkScqOSoW1S4Hb\nAGKM9wEXTDvfBrwWWHsUZY7LgbcYOBUqSZKyo1JhrQcYKtufDCHsn3uMMd4dY1x/NGWO1/6FcV1r\nTZIkZUilbt7aBXSX7edjjBPzUWZwcPCYGrRnbBKAHz/6JGe2Ds1xtSphZGTkmPtP1Wf/ZZd9l232\nnyoV1u4GrgBuDiFcBKyZrzIDAwPH1KBSqUTha+uh0HvMdej4DA4O+rvPMPsvu+y7bLP/smv16tXz\nUk+lwtotwOUhhHuAHHBNCGEV0BVj/OyRlpnPBuVyOfp7Ck6DSpKkTKlIWIsxFoHrph1eO8N1l81R\nZl719/jKKUmSlC0NsyguJGFts2FNkiRlSGOFte42Nu0apVQqVbspkiRJR6SxwlpPgX3jk+wamevB\nVEmSpNrQUGGtL11rzalQSZKUFQ0V1qbeYrDRsCZJkjKiocLayYvaAXh6x74qt0SSJOnINFRYW95b\nIJ+Dp3ca1iRJUjY0VFhracpzYm+7I2uSJCkzGiqsQTIV+jPDmiRJyojGC2uL250GlSRJmdFwYe2U\nxe1sGNrH+GSx2k2RJEmaU8OFtZMXtVMswcYhl++QJEm1r/HC2uJ0+Q6nQiVJUgY0XFg7ZXEHgA8Z\nSJKkTGi4sHZib/IWA5fvkCRJWdBwYa3Q0kRfdxtP7xyudlMkSZLm1HBhDZL71pwGlSRJWdCYYW2R\na61JkqRsaMiwdsriDp7ZuY9isVTtpkiSJB1WQ4a1kxe3Mz5ZYvPu0Wo3RZIk6bAaMqydekKyfMdT\n233IQJIk1baGDGunpWHtiW17q9wSSZKkw2vIsHby4naa8jme2ubImiRJqm0NGdZamvKcsrjdkTVJ\nklTzGjKsAZy2pJMnHVmTJEk1rmHD2oolHTyxbS+lkst3SJKk2tWwYe3UEzrYPTLBjuHxajdFkiRp\nVg0b1lYs6QTgSe9bkyRJNaxxw9rSZPkO71uTJEm1rGHD2imLO8jlXGtNkiTVtoYNa4WWJk7qbXdk\nTZIk1bSGDWsApy3p4PGtjqxJkqTa1dBhbcXSTh7f6vIdkiSpdjV0WDuzr4uhfeNs2TNa7aZIkiTN\nqKHD2hl9XQCs27ynyi2RJEmaWUOHtTP7ugF4zLAmSZJqVEOHtf6eNrramnnUsCZJkmpUQ4e1XC7H\n6X1dToNKkqSa1dBhDZKHDAxrkiSpVjV8WDujr4vNu0cZ2ucL3SVJUu1p+LB2pk+ESpKkGtbwYW1q\n+Q6fCJUkSbWo4cPaKYs7KLTkiZt2V7spkiRJh2j4sNaUzxGW9zC4YVe1myJJknSIhg9rAM85sYeH\nN+zyHaGSJKnmGNaA55zUw87hcTYMjVS7KZIkSQcxrJGMrAE8/IxToZIkqbYY1oCzl3eTy8HD3rcm\nSZJqjGEN6Gxr5ueWdDqyJkmSao5hLTVwUo8ja5IkqeYY1lLPObGHp7YP+9opSZJUUwxrqZWnLALg\nxz/bWeWWSJIkHWBYS537rF5yOfjhU4Y1SZJUO5orUWkIIQ98ElgJjALXxhjXlZ2/Avg9YAK4Mcb4\nufT4A8DUjWOPxxivqUT7ZtJTaOHMvi5++NSOhfpISZKkOVUkrAFXAYUY48UhhIuAG4ArAUIILcCf\nAy8E9gJ3hxD+CRgCcjHGyyrUpjk9/1mLuf3hjZRKJXK5XLWaIUmStF+lpkEvBW4DiDHeB1xQdm4A\nWBdj3BFjHAPuAl5CMgrXEUK4I4RwZxryFtR5py5i5/A4T2wbXuiPliRJmlGlRtZ6SEbKpkyGEJpj\njBMznNsN9ALDwMeBzwNnAt8MIYS0zH6Dg4MVajL0TowB8M/3/YRfOL27Yp/TqEZGRiraf6os+y+7\n7Ltss/9UqbC2CyhPO/my0DX9XDewE3iEZMStBDwSQtgGnAisL694YGCgQk2Gs4olOm/bwMaJ9op+\nTqMaHBz095ph9l922XfZZv9l1+rVq+elnkpNg94NvBognc5cU3ZuEDgzhHBCCKGVZAr0XuDtJPe2\nEUI4iWQEbkOF2jejpnyOF5y2mO//dPtCfqwkSdKsKhXWbgFGQgj3kDxM8F9DCKtCCO+MMY4D7wdu\nJwlpN8YYnwa+ACwKIdwF3AS8ffoU6EK4+PQlPLp5D1t2jy70R0uSJB2iItOgMcYicN20w2vLzt8K\n3DqtzBiwqhLtORqXnL4UiNz3021csfKkajdHkiQ1OBfFneZ5J/XQ1dbMvT/dVu2mSJIkGdama27K\nc+HPncB9jxnWJElS9RnWZnDxs5fw06172Tg0Uu2mSJKkBmdYm8GlZy4F4LuPbK5ySyRJUqMzrM3g\n7OXdnNRb4NuDhjVJklRdhrUZ5HI5Xj7Qx13rtjIyPlnt5kiSpAZmWJvFL5zdz/DYJN9/3AVyJUlS\n9RjWZnHx6UsotOS5c3BTtZsiSZIamGFtFoWWJn7+zGXc/pNNFIulajdHkiQ1KMPaYbzm3BPZuGuE\n/3jCqVBJklQdhrXDeMVAP4WWPLf++JlqN0WSJDUow9phdLY184qBfv5lzUbGJ4vVbo4kSWpAhrU5\nXLHyJLbvHeOuR7dWuymSJKkBGdbm8LLQx5LOVr7yg6eq3RRJktSADGtzaG3O86sXnMK3125m0y7f\nFSpJkhaWYe0IvOmFpzJZLHHTf6yvdlMkSVKDMawdgRVLO/n5M5fy1R88xYQPGkiSpAVkWDtCb714\nBc8MjfCNH2+odlMkSVIDMawdoZef3cdZ/V186juP+UYDSZK0YAxrRyifz3HdS08nbtrNv8XN1W6O\nJElqEIa1o3DFypM4eVE7//tfH3V0TZIkLQjD2lFoacrz/svPYs3TQ3xjjfeuSZKkyjOsHaWrnn8y\nAyf28Ke3r2V0YrLazZEkSXXOsHaUmvI5Pviqs1m/fR9fuOvxajdHkiTVOcPaMXjJWcv4xecu5xP/\n+ihPbN1b7eZIkqQ6Zlg7Rr9/5XNpbcrzO7esoVTyYQNJklQZhrVj1N9T4PpXn809j23jr+9+otrN\nkSRJdcqwdhxWXXgqrxjo42PfHGTNz4aq3RxJklSHDGvHIZfL8ae/upKlXW285ysPsHN4rNpNkiRJ\ndcawdpwWd7byV6uez4adI7zzS6tdzkOSJM0rw9o8OP+0E/jT153LDx7fzvVfX+PbDSRJ0rxprnYD\n6sWV553M+u3DfPyOR2hrzvPR155DPp+rdrMkSVLGGdbm0X952RmMThT5yzvXUSrBR68+hyYDmyRJ\nOg6GtXmUy+V4/+VnkQP+4s51bNs7xifeeB6dbf6aJUnSsfGetXmWy+V4/38K/P4vP5c7127idZ++\nl6d37qt2syRJUkYZ1irkrZes4AtvfSFPbR/mVf/737ntoQ3VbpIkScogw1oFvezsPv75vZeyYmkn\n1/3dA1z/9R8zNDxe7WZJkqQMMaxV2GlLOvnadZfwGy99Njffv55f+LPv8I8PPu37RCVJ0hExrC2A\n1uY8H3zVAP/0nks5aVE77/vqg/zKp+7h+z/dVu2mSZKkGmdYW0DPO7mXW979Yv7o6nN4ZucIb/js\nfbzlxh9w72PbHGmTJEkzck2JBdaUz/HGC0/lquefzN/e+wSf/u5PedPn7uN5J/dw7aXP5lXnLKet\nuanazZQkSTXCkbUqKbQ08c6XnM4917+cj772HIbHJvmtmx7kwo98m9/7x4dY87MhR9skSZIja9VW\naGli1YtO5Y0vfBZ3rdvK11b/jK/+x3r+9t4nefayTl753OW88rnLOffkXl9fJUlSAzKs1Yh8PsdL\nzlrGS85AonbnAAAM2klEQVRaxtC+cb7x42f45pqNfO7ff8qnvvMYy3sKvPSsZVxyxhIuOX0py7rb\nqt1kSZK0AAxrNai3vYU3v+g03vyi0xgaHufbazdxx0828S8PbeCm+9cDEPq7uejZJ3DeqYs471mL\nWbGkg1zOkTdJkuqNYa3G9Xa0cPULTuHqF5zCZLHEQ08Pcc9j27jnsa3cfP/P+Jt7n0yua29h5bMW\nsfKUXsLybkJ/NyuWdtLS5G2JkiRlmWEtQ5ryuSSQPWsR77rsdCYmizy6eQ8/Wr+TB9N//s+/baGY\nPpfQ2pTn2cs6Ccu7OWNZF6ct7eS0EzpYsaST3o6W6n4ZSZJ0RAxrGdbclGfgxB4GTuzhjReeCsDI\n+CSPbdlD3LibuGk3j2zczX88vp1/fPCZg8r2trewYkkHpy7p5ORF7ZzYW2B5b2H/z6WdbT7QIElS\nDTCs1ZlCSxPPPamX557Ue9Dx4bEJnto+zJPbhnly29705zAPrt/BbQ9tYHzy4GVCWppy9HUn4W1Z\ndxsndLaypKuNpV2tLOlsY0lX6/7t3vYWg50kSRViWGsQHa3NnL28h7OX9xxyrlgssW3vGBuHRtgw\ntI+Nu0bYMDTCxqERntm5j0c372HbnlF27htnpqXfmvI5FrW30NveQk/6T297C73tzfQUDhzfvW0P\n21u20lNooavQTGdrEx1tzXS0NBn2JEmahWFN5PM5lnW3say7jXNO6Z31uonJIjuGx9m2d5Rte8bY\numeU7XvH2LZnjO3DY+zaN87QvnGGhsdYv3042d43zmSxLOF9d/OMdXe0NtHZlga41ma62prpaGui\ns7WZzrYDxwoteQotTbS1NFFoTrYLLU20tzTtP1doydPW3LR/u9DS5IMWkqTMMqzpiDU35feHuiNV\nKpUYHptkaN84Dz78CEuWn8LQvnGGxybZMzrB8NgEe0cn2Ts6wd6x5Ofw2AR7RifYvjcJfXtHJ9k7\nNsHe0QmKx/hSh6Z87qBw19qcp6UpR0tTPt3O05b+bGnK0drcREtTruzYgetam3IHtvcfS7ab8zma\nm3I056e28zTlcwcfb0r2m/LJ5zflc7Tk8zSlx6fOuRSLJAkqFNZCCHngk8BKYBS4Nsa4ruz8FcDv\nARPAjTHGz81VRtmUy+WSEbO2ZoYWtzLw7CXHXFepVGJsssjIeJHR8UlGxouMTEwyMrU9nm5PJNuj\nZcf3lV0/Ol5kfLLI2ET6M93eOzrB2GSR8YkS45NFRsvOj08UGZ9MPn+hTIW2qdA3PeAdCH15mvLQ\nlMuRz+f2/8znkpCazyXlys8nYZBDjufzuf115abKpXXs2L6d/vVx2mck1+enXTtVb66sDblcct2B\n7eS/H8kxyE3bz+dy5Jg6lowAH7RfVld53TPVlSP9zPwsde+v50jqNkRLWliVGlm7CijEGC8OIVwE\n3ABcCRBCaAH+HHghsBe4O4TwT8CLZysjQfJ/km3NTcmL7turs/RIqVRifLJ0UNgrD3UTkyUmiiUm\ni0m4mywm+xOTxfRniYliMTmeXjtRTMpNFkuMF4tMTpYYT+uYKjNZTD5z5vpKFEvJNeU/i0X2l5k6\nXr5dKsHk1PXFUrrNtDqS48UiTBSLFEs7q/J7rzUHAlwSBNP/JOEuDYZTYTAHUB4Iy47n0pO5w5Sf\n+rzZ6p4KmdPLU3bN6OgI7f+69aDyHFRm5jpm2+aQ64/wOx3U1pnLk/6+0loOfP+y3/0h56YVPHAN\nZeVnO3cgfM9U99S15d+7vDyHrftAfdPPMa2ew7Vxy9Yd9G88eOxixt/DHN9jpvYf/B1n/nzK+uaI\nfkccXMFBdU+vZ4bf0XQH9dFBx6d93vTvOsO15Wdmv3aOzzuo3MyVTG12Mz8qFdYuBW4DiDHeF0K4\noOzcALAuxrgDIIRwF/AS4OLDlJFqQi6Xo7U5mQbtbLA3fg0ODjIwMHAgwKUhbnrgO3A+eXhlKviV\nSMJusZQEwtK0n8XSgfPTrzton6njyefPVPdUHdPrLt8/qFzxwDGYdk3p0LqZ/lmUSP+z/3PSyyhR\n2v9gzv76OPgapvZnOHdI3fuPl+/PXh5K7No9QVdXYcbyTGtn+XaxCCWKM3xess9B++XlZ6n7kN/P\nweXZ3+YDPymvi4PPlf9up5+D8s+fvZ7pxzikvtKMnzu9bmY4N3V4tvYfnR3HUkhV9vXXLZ+XeioV\n1nqAobL9yRBCc4xxYoZzu4HeOcrsNzg4WKEmq9JGRkbsvwyrRP9NPfbRdKwV5Dj4T9+aML1B1W/g\nyEgzhUKh2s3QLGYKhPvPAfv2jVAoFA4EwxkC4SEhc4b6pgdJyq6ZXq48fM4eZKeV4dBAO9N1M33H\n0iwJdqa6Z2rnIdceVMn039zs9c712aVZPnDGzy4BzM+tM5UKa7s4ePQvXxa6pp/rBnbOUWa/gYGB\neW6qFsrUyIyyyf7LLvsu2+y/7Fq9evW81FOp9QzuBl4NkN5/tqbs3CBwZgjhhBBCK8kU6L1zlJEk\nSWpIlRpZuwW4PIRwD8kcwDUhhFVAV4zxsyGE9wO3k4TFG2OMT4cQDilTobZJkiRlRkXCWoyxCFw3\n7fDasvO3ArceQRlJkqSG5rLukiRJNcywJkmSVMMMa5IkSTXMsCZJklTDDGuSJEk1zLAmSZJUwwxr\nkiRJNcywJkmSVMMMa5IkSTUsN9ub7mvR6tWrs9NYSZLU8M4///zc8daRqbAmSZLUaJwGlSRJqmGG\nNUmSpBrWXO0GHIkQQh74JLASGAWujTGuq26rVC6E8CLgj2OMl4UQzgC+CJSAh4D/EmMshhB+HfgN\nYAL4cIzxGyGEduDvgD5gN/DWGOOWqnyJBhRCaAFuBFYAbcCHgYex/2peCKEJ+BwQSPrqOmAE+y5T\nQgh9wGrgcpL++SL2XyaEEB4AdqW7jwMfoUL9l5WRtauAQozxYuB64IYqt0dlQgj/A/g8UEgP/Rnw\nuzHGnwdywJUhhOXAe4EXA68EPhZCaAPeBaxJr/1b4HcXuv0N7teAbenv/xeBv8L+y4orAGKMLyb5\nvX8E+y5T0j+WPgPsSw/ZfxkRQigAuRjjZek/11DB/stKWLsUuA0gxngfcEF1m6NpHgOuLts/H/hu\nuv1N4BXAhcDdMcbRGOMQsA44l7K+LbtWC+cfgP+ZbudI/vKz/zIgxvj/gHemu6cBO7HvsubjwKeB\nZ9J9+y87VgIdIYQ7Qgh3hhAuooL9l5Ww1gMMle1PhhAyMYXbCGKMXwfGyw7lYoxTjxnvBno5tA9n\nOj51TAskxrgnxrg7hNANfI3krzv7LyNijBMhhL8B/hL4MvZdZoQQ3gZsiTHeXnbY/suOYZKw/UqS\nWxAq+u9fVsLaLqC7bD8fY5yoVmM0p2LZdjfJX/zT+3Cm41PHtIBCCM8C/g34Uozx77H/MiXG+Fbg\nLJL719rLTtl3te3twOUhhO8A55FMhfWVnbf/atsjwN/FGEsxxkeAbUB/2fl57b+shLW7gVcDpEON\na6rbHM3hhyGEy9LtVwHfA34A/HwIoRBC6AUGSG7A3N+3ZddqgYQQ+oE7gA/EGG9MD9t/GRBC+M8h\nhA+mu8MkIft++y4bYowviTG+NMZ4GfAg8Bbgm/ZfZryd9P75EMJJJCNld1Sq/zKxKG7Z06DnktxX\nc02McW11W6VyIYQVwFdjjBeFEKb+ym8FBoFfjzFOpk/EvJPkj4SPxhi/HkLoAP4GOBEYA1bFGDdW\n5Us0oBDCJ4A3AOX/Pr0P+Avsv5oWQugE/hpYDrQAf0TSX/67lzHp6Np1JIHb/suAEEIryZOfp5I8\n/fkBYCsV6r9MhDVJkqRGlZVpUEmSpIZkWJMkSaphhjVJkqQaZliTJEmqYYY1SZKkGuZbACTVlXSd\no5tJXkg/ZUuM8XXHWe8XSZanuW2uayVpPhnWJNWjO2OMb6x2IyRpPhjWJDWEdOHRtcDZJItrvyHG\nuDGEcAPJS5UB/j7G+IkQwpnA50kWtxwGpoLfb4QQ/gfJe/zeFWP8wUJ+B0mNybAmqR69PA1nU/45\n/XlPjPG6EMK7gd8JIdwB/BxwEcn/Ht4VQrgT+DDwsRjjbSGEXwaen5ZfHWP8cPoS7reRvEpGkirK\nsCapHh0yDRpC+CXgznT3HuBKYD3wvRhjCRgPIdwHPAcIwL0AMcZ/SsuvAlan5TcCHZX+EpIEPg0q\nqbGcn/58MfATkvf3XQoQQmgBLgEeTY+/MD3+5hDCb6blfD+fpAXnyJqkejR9GhSgHXhbCOH9wF7g\nP8cYt4UQLgsh3Etyf9rNMcYHQgj/HfhMCOF3Se5Z+zUOBD1JWlC+yF1SQ0jD23UxxrXVboskHQ2n\nQSVJkmqYI2uSJEk1zJE1SZKkGmZYkyRJqmGGNUmSpBpmWJMkSaphhjVJkqQaZliTJEmqYf8fLQfM\nIigmlXQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x21141fbeb38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(net.epoch_error).plot(figsize=(10,6))\n",
    "plt.title('Neural Net Performance')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('MSE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}