bandit_algorithms(Sutton_2017_Ch_2).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from random import randrange, uniform\n",
    "import gym\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple bandit algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stationary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 14 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 8 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 32 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "13.98\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "ans=[]\n",
    "epoch=100\n",
    "\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset()\n",
    "    Q=[0,0]\n",
    "    N=[0,0]\n",
    "    for t in range(100):\n",
    "        env.render()\n",
    "        e=uniform(0,1)\n",
    "        if e>0.5:\n",
    "            action = np.argmax(Q, axis=0)\n",
    "        else:\n",
    "            action = randrange(2)\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        N[action]+=1\n",
    "        Q[action]=Q[action]+(1/N[action])*(reward-Q[action])\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            ans.append(t+1)\n",
    "            break\n",
    "env.close()\n",
    "print(sum(ans)/epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Non-stationary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 22 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 47 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 32 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 9 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 55 timesteps\n",
      "Episode finished after 42 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 38 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 47 timesteps\n",
      "18.96\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "ans=[]\n",
    "epoch=100\n",
    "\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset()\n",
    "    Q=[0,0]\n",
    "\n",
    "    for t in range(100):\n",
    "        env.render()\n",
    "        e=uniform(0,1)\n",
    "        if e>0.5:\n",
    "            action = np.argmax(Q, axis=0)\n",
    "        else:\n",
    "            action = randrange(2)\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        Q[action]=Q[action]+0.1*(reward-Q[action])\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            ans.append(t+1)\n",
    "            break\n",
    "env.close()\n",
    "print(sum(ans)/epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 15 timesteps\n",
      "Episode finished after 43 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 49 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 54 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 45 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 51 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 49 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 68 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 32 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 43 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 42 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 35 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 39 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "21.88\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "ans=[]\n",
    "epoch=100\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset()\n",
    "    for t in range(100):\n",
    "        env.render()\n",
    "#         print(observation)\n",
    "        action = env.action_space.sample()\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            ans.append(t+1)\n",
    "            break\n",
    "env.close()\n",
    "print(sum(ans)/epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 install Box2D\n",
    "# !pip3 install box2d-py \n",
    "# pip3 install gym[box2d]\n",
    "#pip3 install box2d\n",
    "# pip3 uninstall box2d-kengz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !brew install swig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 install box2d-py box2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lunar Lander - non-stationary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 133 timesteps\n",
      "Episode finished after 145 timesteps\n",
      "Episode finished after 189 timesteps\n",
      "Episode finished after 143 timesteps\n",
      "Episode finished after 177 timesteps\n",
      "Episode finished after 125 timesteps\n",
      "Episode finished after 144 timesteps\n",
      "Episode finished after 106 timesteps\n",
      "Episode finished after 108 timesteps\n",
      "Episode finished after 80 timesteps\n",
      "Episode finished after 128 timesteps\n",
      "Episode finished after 92 timesteps\n",
      "Episode finished after 147 timesteps\n",
      "Episode finished after 137 timesteps\n",
      "Episode finished after 107 timesteps\n",
      "Episode finished after 86 timesteps\n",
      "Episode finished after 117 timesteps\n",
      "Episode finished after 97 timesteps\n",
      "Episode finished after 105 timesteps\n",
      "Episode finished after 139 timesteps\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('LunarLander-v2')\n",
    "\n",
    "epoch=20\n",
    "\n",
    "\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset()\n",
    "    \n",
    "    Q=[0,0,0,0]\n",
    "    for t in range(500):\n",
    "        env.render()\n",
    "        e=uniform(0,1)\n",
    "        if e>0.25:\n",
    "            action = np.argmax(Q, axis=0)\n",
    "        else:\n",
    "            action = randrange(4)\n",
    "        observation, reward, done, info = env.step(action)\n",
    "#         print(reward)\n",
    "        Q[action]=Q[action]+0.01*(reward-Q[action])\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            break\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-0.44347064596776725,\n",
       " -0.4185357146786495,\n",
       " -0.5234913567814106,\n",
       " -1.113154407376888]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ex.2.5 - Sutton, Barto, November 5, 2017 + UCB + Gradient Bandit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "R_out_list=[]\n",
    "for j in range(2000):\n",
    "    Q=np.array([0.0 for x in range(10)])\n",
    "    N=[0 for x in range(10)]\n",
    "   \n",
    "    R_list=[]\n",
    "    for i in range(10000):\n",
    "        e=uniform(0,1)\n",
    "        if e>0.1:\n",
    "            A=np.argmax(Q, axis=0)\n",
    "        else:\n",
    "            A=randrange(10)\n",
    "        R=np.random.normal(Q[A], 1, 1)[0] \n",
    "        N[A]+=1\n",
    "        Q[A]+=(1/N[A])*(R-Q[A]) \n",
    "        Q+=np.random.normal(0, 0.01, 10)\n",
    "        R_list.append(R)\n",
    "    R_out_list.append(R_list) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-3.11671305, -2.16881521, -1.19670943,  0.58927533, -0.08954363,\n",
       "        0.40129174,  0.65733995, -0.57606823, -0.46475376,  0.80576922])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "R_out_list_1=[]\n",
    "for j in range(2000):\n",
    "    Q=np.array([0.0 for x in range(10)])  \n",
    "    R_list=[]\n",
    "    for i in range(10000):\n",
    "        e=uniform(0,1)\n",
    "        if e>0.1:\n",
    "            A=np.argmax(Q, axis=0)\n",
    "        else:\n",
    "            A=randrange(10)\n",
    "        R=np.random.normal(Q[A], 1, 1)[0]\n",
    "        Q[A]+=0.1*(R-Q[A])\n",
    "        Q+=np.random.normal(0, 0.01, 10)       \n",
    "        R_list.append(R)\n",
    "    R_out_list_1.append(R_list) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.11540601, -1.03506049, -1.73590074, -1.65414484, -2.26516075,\n",
       "       -0.31994382, -2.73833746,  7.39214924, -2.60978153,  1.35485555])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "R_out_list_2=[]\n",
    "for j in range(2000):\n",
    "    Q=np.array([0.0 for x in range(10)])\n",
    "    N=np.array([0.0 for x in range(10)])\n",
    "    R_list=[]\n",
    "    for i in range(10000):\n",
    "        e=uniform(0,1)\n",
    "        if e>0.1:    \n",
    "            A = np.argmax(Q+2*np.sqrt(np.divide(np.log(i+1),N+1e-5)), axis=0)           \n",
    "        else:\n",
    "            A = randrange(10)\n",
    "        R=np.random.normal(Q[A], 1, 1)[0]\n",
    "        N[A]+=1\n",
    "        Q[A]+=0.1*(R-Q[A])\n",
    "        Q+=np.random.normal(0, 0.01, 10)       \n",
    "        R_list.append(R)\n",
    "    R_out_list_2.append(R_list) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.15738993, -0.2969734 ,  6.29069639,  0.10616182, -0.43860866,\n",
       "        0.17241244, -3.1386272 ,  0.37855333, -1.85100799,  1.42833028])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "R_out_list_3=[]\n",
    "alpha=0.001 # 11e-20 # 0.1 \n",
    "for j in range(2000):\n",
    "    Q=np.array([0.0 for x in range(10)])\n",
    "    H=np.array([0.0 for x in range(10)])\n",
    "    R_list=[]\n",
    "    for i in range(10000):\n",
    "        e=uniform(0,1)\n",
    "        if e>0.1:\n",
    "            action = np.argmax(H, axis=0)\n",
    "        else:\n",
    "            action = randrange(10)\n",
    "        reward=np.random.normal(H[action],1, 1)[0]  \n",
    "        Q[action]+=alpha*(reward-Q[action])\n",
    "        Q+=np.random.normal(0, 0.01, 10)\n",
    "        \n",
    "        exp=np.exp(H)\n",
    "        prob=np.divide(exp, np.sum(exp))\n",
    "\n",
    "        for a in range(len(H)):\n",
    "            if a==action:\n",
    "                H[action]+=alpha*(reward-Q[action])*(1-prob[action])\n",
    "            else:\n",
    "                H[a]+=alpha*(reward-Q[a])*prob[a]\n",
    "        R_list.append(reward) \n",
    "    R_out_list_3.append(R_list)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.7961708 , -0.47406848,  0.1361272 ,  1.43567135,  0.39664399,\n",
       "        0.61109466,  2.53285569, -1.72584408, -0.45008157,  1.73787739])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlcAAAHgCAYAAACIHEjUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3hUxfrA8e9JdtN7QigpEIy0kJBAAgakN+midJQgFryIBe6VdgEB+YkK13IFO16iSJEAgkgREaRJCYjSBSRA6CQQEkjP+f2xySZhUzbZ3RR4P8+zz+6ZM2fmXRTyZs6cGUVVVYQQQgghhHlYVXYAQgghhBD3E0muhBBCCCHMSJIrIYQQQggzkuRKCCGEEMKMJLkSQgghhDAjSa6EEEIIIcxIU9kBFOTl5aXWq1evssMQQgghhCjVgQMHbqiqWuPe8iqVXNWrV4/Y2NjKDkMIIYQQolSKopwrqlxuCwohhBBCmJEkV0IIIYQQZiTJlRBCCCGEGVWpOVdFyczMJD4+nrS0tMoORYgqy87ODl9fX7RabWWHIoQQD7wqn1zFx8fj7OxMvXr1UBSlssMRospRVZWEhATi4+MJCAio7HCEEOKBV+VvC6alpeHp6SmJlRDFUBQFT09PGd0VQogqosonV4AkVkKUQv6OCCFE1VEtkqvqKj09ncGDBxMYGEirVq2Ii4srst6oUaPw9vamadOm5e4rLi6OJUuWmK2eEEIIIcrHYsmVoigNFUU5VOB1W1GU1yzVX1W0cOFC3N3dOX36NOPGjWPixIlF1hs5ciQbN240qS9JroQQQoiqwWLJlaqqJ1VVDVVVNRRoAdwFVluqP0tavHgxLVu2JDQ0lNGjR5OdnW3UdWvWrCEqKgqAAQMGsGXLFlRVNajXrl07PDw8jI7n119/JTQ0lNDQUMLCwkhOTmbSpEns2LGD0NBQ3n//feLi4mjbti3NmzenefPm7N69G8CgnhBCCCHMq6KeFuwMnFFVtchl4o0184ejHLt020wh6TSp48IbfYKKPX/8+HGWL1/Orl270Gq1jBkzhm+//ZYRI0YwePBgTp48aXDN+PHjGTFiBBcvXsTPzw8AjUaDq6srCQkJeHl5mRTzvHnzWLBgAW3atCElJQU7Ozvefvtt5s2bx7p16wC4e/cumzdvxs7OjlOnTjF06FBiY2MN6gkhhBDCvCoquRoCLK2gvsxqy5YtHDhwgIiICABSU1Px9vYGYPny5ZUSU5s2bRg/fjzDhw/niSeewNfX16BOZmYmY8eO5dChQ1hbW/PXX39VQqRCCCHEg8fiyZWiKDZAX2ByMedfAF4A8Pf3L7GtkkaYLEVVVaKiopgzZ47BudJGrnx8fLhw4QK+vr5kZWWRlJSEp6enyTFNmjSJXr16sX79etq0acOmTZsM6rz//vvUrFmTP/74g5ycHOzs7EzuVwghhBClq4iRqx7AQVVVrxZ1UlXVz4HPAcLDww0nJFWyzp07069fP8aNG4e3tzeJiYkkJydTt27dUkeu+vbtS3R0NJGRkcTExNCpU6cyPTI/f/58AMaOHVuo/MyZMwQHBxMcHMz+/fs5ceIEfn5+JCcn6+skJSXh6+uLlZUV0dHR+nlizs7OheoJIYQQwrwqYimGoVTTW4IATZo0Yfbs2XTr1o2QkBC6du3K5cuXjbr22WefJSEhgcDAQN577z3efvttAC5dukTPnj319YYOHUpkZCQnT57E19eXhQsXAnDixIkiR7o++OADmjZtSkhICFqtlh49ehASEoK1tTXNmjXj/fffZ8yYMURHR9OsWTNOnDiBo6MjgEE9IYQQQpiXUtTTa2ZrXFEcgfNAfVVVk0qrHx4ersbGxhYqO378OI0bN7ZQhFVb7969WbVqFTY2NpUdiqgGHuS/K0IIURkURTmgqmr4veUWvS2oquodwPRJRg8oeaJPCCGEqH5khXYhhBBCVD/ZWTDDFba9U9mRGJDkSgghhBDVT3a67n3XB5UbRxEkuRJCCCFE9VVw7nh8LOz+qPJiyVVRi4gKIYQQQphR7tJGWalw5QikJkJ0H11Z65crLywkuRJCCCFEdfdpm8LHOTlgVXk35+S2oAWlp6czePBgAgMDadWqFXFxcUXWGzVqFN7e3jRt2tSk/rZt20bv3r2Nrr9x40YaNmxIYGCgfg2ue23fvp3mzZuj0WiIiYkxKT4hhBCiXFQV9nwKKdfyy/7aUHz9v7daPqYSSHJlQQsXLsTd3Z3Tp08zbtw4Jk6cWGS9kSNHsnHjxgqNLTs7m5deeokNGzZw7Ngxli5dyrFjxwzq+fv7s2jRIoYNG1ah8QkhhBDcPAepN+HIStg4EVY8Azs/gEW9IWZU8dflZFVcjEWQ5MoIixcvpmXLloSGhjJ69Gj9VjKlWbNmDVFRUQAMGDCALVu2UNSire3atcPDw8PoePbt20dkZCRhYWG0bt26yP0NjWkjMDCQ+vXrY2Njw5AhQ1izZo1BvXr16hESEoJVJQ6vCiGEuE+oqm7phKSLpdfNuAMfhsA79WDls7qyO9fh5zcgbkfJ1x5fa3Kopqhec642TIIrh83bZq1g6FH0LTHQrXq9fPlydu3ahVarZcyYMXz77beMGDGi1I2bL168iJ+fHwAajQZXV1cSEhLw8vIyKeRGjRqxY8cONBoNP//8M1OmTGHlypWF6mzdupVx48YZXOvg4MDu3bsLxQbg6+vL3r17TYpLCCGEKNHVI7DtLTizBR4dBy51oHazwnUSzoCbP2SmGl5/w8jBhN8XQ78FpsdbTtUruaoEW7Zs4cCBA0RERACQmpqKt7c3QKkbN1tKUlISUVFRnDp1CkVRyMzMNKjTsWNHDh06VAnRCSGEEMXIyb3zk5kKS4foPvedD3Vbg+dDkHwVPmquK//X6cqJ0QyqV3JVwgiTpaiqSlRUFHPmzDE4V9rIlY+PDxcuXMDX15esrCySkpKK3Ii5rKZNm0bHjh1ZvXo1cXFxdOjQwaBOaSNXebHliY+Px8fHx+TYhBBCiDJZO1b3PiMJbp3LL58XWDnxmEH1Sq4qQefOnenXrx/jxo3D29ubxMREkpOTqVu3bqkjV3379iU6OprIyEhiYmLo1KkTiqIY3ff8+fMBGDt2bKHypKQkfSK0aNGiIq8tbeQqIiKCU6dOcfbsWXx8fFi2bBlLliwxOjYhhBDCKEnxsPJ5aD8B7N10ZYlnDevNrglZaSZ3t9jFGaecHB43uaXyk1nKpWjSpAmzZ8+mW7duhISE0LVrVy5fvmzUtc8++ywJCQkEBgby3nvv6Zc7uHTpEj179tTXGzp0KJGRkZw8eRJfX18WLlwIwIkTJ4oc6ZowYQKTJ08mLCyMrKzyPRGh0WiYP38+3bt3p3HjxgwaNIigoCAApk+fztq1usmA+/fvx9fXlxUrVjB69Gh9HSGEEKKQnJz8234FbZgI53fDN4/D5x10ZRnJhvXKmFi95+7Ge+66ZO22lcIFjW686B1Pd6bVMP0ukSmUop5eqyzh4eFqbGxsobLjx4/TuHHjSoqocvXu3ZtVq1ZhY2NT2aGIauBB/rsihKgCPmmjm7A+/abu9p6dKzh4wLLhcGKd2bsLDvA3KDtw9jwtcssPR5n5AbgiKIpyQFXV8HvL5bZgFbZunfn/ZxRCCCHMKicHTv6oS6xA9zTg9rngVAue/wUuHjRLN2ruq6Rbbi2KSLgqgyRXQgghhCi/We6Fjw9+rXtPuQLvNylXkz84OZCsWPG2pztfXLlGq7R0Znp5sNLZicNnz5sYsOVJciWEEEII80m5anITU2rkrwf5P1cXWqVdZ6WzEwBzPdxY6+Roch+WJBPahRBCCFG6n2fCX5vgTgJsmQX/61n0BPYyUoF04x+k52tXF25ZW5vcryXJyJUQQgghSnb1KOx8z7B8lvFbtwHEaTT86OTI87eSUAAt8JmbCwvc3fgt7gJORTxkd0mjKXLyelUmyZUQQgghCsvOgrQkcPSEC/thYZdyNfOKtxftUlMJSs/AKSeHF2t5c1Gr4VN3V4O6s7w8mJhwk3NabaHyszZag7pVndwWtKD09HQGDx5MYGAgrVq1Ii4ursh6o0aNwtvbm6ZNm5rU37Zt2+jdu7fR9Tdu3EjDhg0JDAzUr8F1r+3bt9O8eXM0Gg0xMTFGtevk5FToeNGiRYUWQv36669p2rQpwcHBhIWFMW/ePABGjhxJQEAAoaGhNGrUiJkzZxr9XYQQQpjRhtdhbn3diFU5EqtMdHOjtjo6MNPLk0E+tenp58NFbfFjOhucHOlQ15eoOjVNCLxqkOTKghYuXIi7uzunT59m3LhxTJw4sch6I0eOZOPGjRUaW3Z2Ni+99BIbNmzg2LFjLF26lGPHjhnU8/f3Z9GiRQwbNsws/W7YsIEPPviAn376icOHD7Nnzx5cXfN/g5k7dy6HDh3i0KFDREdHc/ZsEav4CiGEsIyz2+G9JnAod8eO83vK3MRSZyeaB/jztauLmYOrPiS5MsLixYtp2bIloaGhjB49muxs4ybwrVmzhqioKAAGDBjAli1bKGrR1nbt2uHhYfx963379hEZGUlYWBitW7cucn9DY9oIDAykfv362NjYMGTIENasWWNQr169eoSEhGBlZZ7/VebMmcO8efOoU6cOALa2tjz//PMG9dLSdCv1OjpW7SdChBCiyrp2HFKuG5YnX4X1EyA7U3ecnaV7AUT3gdsX81dL/3F8mbv9zM3wlt+DplrNuXpn3zucSDxh1jYbeTRiYsuiR5RAt+r18uXL2bVrF1qtljFjxvDtt98yYsSIUjduvnjxIn5+foBuuxlXV1cSEhLw8vIyuKZMMTdqxI4dO9BoNPz8889MmTKFlStXFqpT2sbNBWMD8PX1Ze/evSbFZYwjR47QokWLYs+//vrrzJ49m9OnT/PKK6/g7e1t8ZiEEOK+9PEjYOMMU+ILl6//FxxfC74RULc1fNYOcjJhQly5uzpqo+WAnR1zPd1Lr/wAqFbJVWXYsmULBw4cICIiAoDU1FT9D/zSNm62lKSkJKKiojh16hSKopCZmWlQp7SNmyuasRtWz507lwEDBpCSkkLnzp3ZvXs3rVu3tnB0QghRzZ34ERRraPhY4fK8PfzuJsKej6HDZF1iBbDquaLrFqACX7i60DflDm45OdxRFPbb29H9zl3uKAq/OtjT4W4qQ3xqm/87VWPVKrkqaYTJUlRVJSoqijlz5hicK23kysfHhwsXLuDr60tWVhZJSUlFbsRcVtOmTaNjx46sXr2auLg4OnToYFCntJGrvNjyxMfH4+PjY3JsAPb29mRkZOj3RExMTNSP1gUFBXHgwAE6depUYhtOTk506NCBnTt3SnIlhBBFSTgDqTfBNxyW5c6LnZFkWG//Qtg+D5IvwY7/FN/e24bLHZzVavjIw421zo5c1GjIyv1F2fnKNVY7ObKpCi3mqVE0ZKlZlR0GIHOuStW5c2diYmK4du0aoEsUzp07B+hGrvImXxd8jRgxAoC+ffsSHR0NQExMDJ06dTJ6BAdg/vz5zJ8/36A8KSlJnwgtWrSoyGvzRq7ufe3evRuAiIgITp06xdmzZ8nIyGDZsmX07dvX6NhAd3uyKO3bt2fx4sWAbqTvu+++o2PHjgBMnjyZ119/nStXrgCQkZHBl19+adBGVlYWe/fu5aGHHipTTEII8cD4qDl82bn0ej+O1yVWAGpOsdXSFIWMe8qu5y7WeU6r1SdWAElWVlzVVK3xmfVPrOfQ01Xjjo0kV6Vo0qQJs2fPplu3boSEhNC1a1cuX75s1LXPPvssCQkJBAYG8t577+mXO7h06RI9e/bU1xs6dCiRkZGcPHkSX19fFi5cCMCJEyeKHOmaMGECkydPJiwsjKys8mXpGo2G+fPn0717dxo3bsygQYMICgoCYPr06axdqxs23r9/P76+vqxYsYLRo0fr69y4caPIyfkAH374IatWrSI0NJRHHnmEgQMH0q5dOwB69uzJ2LFj6dKlC0FBQTRv3pzbt2/rr3399dcJDQ0lJCSE4OBgnnjiiXJ9PyGEuK9kpcM3T8CVIyXXy8mGDZPgm/5l7iKinh+9/Oroj790deG52kUvi7DExZlDdrZl7sMS2vq05fenf6e2U22sraxZ1msZ6/uvr9SYlOJ+QFaG8PBwNTY2tlDZ8ePHady4cSVFVLl69+7NqlWr9LfXqpJ169bx999/88orr1R2KCLXg/x3RYj73vk98FV33ee8W38zXPOP8z6PWAtfl+0uRJ68VdD/OHueLn51uF7FRqaK82rzV3ku+LnSK1qAoigHVFUNv7e8evzJPaDWrVtX2SEUqyyLlQohhLCgxALrAa6IMrm5J3xqVYvEqp1vO3oG9KR7ve6VHYoBuS0ohBBCVEUftdBNRr/XDFf48Z/5x/8Nzf+cetPkbs9UwbslRflni3/Sq34vNFZVLxGU5EoIIYSoihJOF1jE856HofYbPghUnKja3rzv7soVa2uuWxf+sX/Uxob/urvyh23VTqiefPjJQscjmoygvlv9SoqmdJJcCSGEEFVZUjzc+KvUaresrPjQ3ZV79xA5aGfHV26udPX3oZO/L+scHQDYb2fLEJ9afOHmylN1alkg8OK52bpRy9H4PkNqhBQ6Vqk688WLIsmVEEIIURG2vaO7pZeTuxzCLC9Yabj9FwAFHzZ7PwjWji21+bc83fnSzZXtDvb6ssQiti6b7O3FWa2GUcU8CVgRFBQ2D9jM4Se3FCqv6VCTw1GH6fdQP9xt3flni38ytdVUg+ur0sN4RZHkSgghhKgIv76je1dzx5ZyMuHwd7rlE9KSYO9nMK+h7tzVowaXH7PRckljXWTTqYrCldxzaYrCQVtbmtXzo31d3yLr9/WtU2R5RdGv+ejkzbvt3tWXj2+huw06+9HZbB+ynZFNRzK40WD9+fa+7Ql0C2R44+EVGm9ZSXJlQenp6QwePJjAwEBatWpFXFxckfVGjRqFt7c3TZs2Nam/bdu2lekpvo0bN9KwYUMCAwP1a3Ddq6TvMGfOHAIDA2nYsCGbNm3Sl5fn+2zcuJGWLVvSqFEjQkNDGTx4MOfPnzf6+qLUq1ePGzduAJi0yvuiRYu4dOlSkedGjhxJQEAAoaGhNGrUiJkzZ5a7n3t16NCBvKVJevbsya1bt7h16xYff/yx2foQQlhQdhYcWZk/CpW3gOfVo/BOQH69lc/qVkffMAFSruhGtz5tY9DcYJ/adPcreieNoXVq8rudHQATvL2IqlOTnDIsWl1Rtg/eDoCrbf7mzj0Ceug/96zf0+AagEC3QAC61u3K6n6r8XUuOmmsKiS5sqCFCxfi7u7O6dOnGTduHBMnFr19z8iRI9m4cWOFxpadnc1LL73Ehg0bOHbsGEuXLuXYsWMG9Yr7DseOHWPZsmUcPXqUjRs3MmbMGLKzs8v1fY4cOcLLL79MdHQ0J06c4NChQwwfPrzIZLS8i6bmrUxfHiUlV6DbDzFvBfzo6GjOnj1bbN3yWr9+PW5ubpJcCVGd7HwfYkbBse9zC3KTrM/bQ2pifr2jq8vc9E0rK07aaJnl6c6LNWtUmyf83O3cmdpqKp91+axM14XUCGHLwC30C+xnocjMS5IrIyxevJiWLVsSGhrK6NGj9UlEadasWUNUlG7NkQEDBrBly5Yi7xO3a9cODw8Po+PZt28fkZGRhIWF0bp16yL3NzSmjcDAQOrXr4+NjQ1DhgxhzZo1Rn+HNWvWMGTIEGxtbQkICCAwMJB9+/aV6/u88847TJkypdACmH379tWv6t6hQwdee+01wsPD+fDDD/nhhx9o1aoVYWFhdOnShatXrwKQkJBAt27dCAoK4rnnniv0Z+3k5KT/PHfuXCIiIggJCeGNN94AIC4ujsaNG/P8888TFBREt27dSE1NJSYmhtjYWIYPH05oaCipqanFfo+0tDQAHB11e23NmjWLiIgImjZtygsvvKCPp0OHDkycOJGWLVvSoEEDduzYAei2ChoyZAiNGzemf//+hfrKG4WbNGkSZ86cITQ0lNdff93oP2MhRAXKuANHv4fb8brju4kl1y+GCuy1sy1y6vZAn1oM8KnNChdndhWYY1WVDGwwEDdbNwBsrfNXcx/caDC1ncq+0bO3g7fZYrO0qrc4RAmuvPUW6cdPmLVN28aNqDVlSrHnjx8/zvLly9m1axdarZYxY8bw7bffMmLEiFI3br548SJ+fn6AbrsZV1dXEhIS9JsYl1ejRo3YsWMHGo2Gn3/+mSlTprBy5cpCdUrbuLlgbAC+vr7s3bvXoH5x3+HixYs88sgjha6/ePFiub7P0aNH+de//lVinYyMDP0tsps3b7Jnzx4UReHLL7/k3Xff5T//+Q8zZ87k0UcfZfr06fz444/6bYQK+umnnzh16hT79u1DVVX69u3L9u3b8ff359SpUyxdupQvvviCQYMGsXLlSp566inmz5/PvHnzCA83WIQX0G3ZM3v2bE6fPs0rr7yCt7fuH4CxY8cyffp0AJ5++mnWrVtHnz59AN0I3L59+1i/fj0zZ87k559/5pNPPsHBwYHjx4/z559/0rx5c4O+3n77bY4cOcKhQ1Vj/ywhRBHWvw6HvoXazXTHOeUbcX/Mtw6XtBqm3EhkaHJKoXNVZV+/w1GHAQiODgagZa2W7Lui+0V7euR0fj73MwBrH1+Lo7b4TZ7fiHwDf2fDjaOrq6rxX6cK27JlCwcOHCAiIgLQjS7k/fBcvnx5pcSUlJREVFQUp06dQlEUMjMzDerkbdxc3SQkJNC5c2fu3r3LCy+8oE+6Bg/On9AYHx/P4MGDuXz5MhkZGQQE6OYubN++nVWrVgHQq1cv3N3dDdr/6aef+OmnnwgLCwMgJSWFU6dO4e/vr587BdCiRYti58jda+7cuQwYMICUlBQ6d+7M7t27ad26NVu3buXdd9/l7t27JCYmEhQUpE+u8vZMLNjP9u3b9dsJhYSEEBISUmR/Qogq7lbufNFruYMBGyboXmV0Sav7Ef2WlwePp9zRl2+thJGqZjWa8cf1P0qt90mXT2ixuIVBuZ3GrtA8q3sNaDDApPiqmmqVXJU0wmQpqqoSFRXFnDlzDM6VNnLl4+PDhQsX8PX1JSsri6SkpCI3Yi6radOm0bFjR1avXk1cXBwdOnQwqFPayFVebHni4+Px8TGcKFncdzD2emMEBQVx8OBBmjVrhqenJ4cOHWLevHmkpOT/ppZ3qw3g5ZdfZvz48fTt25dt27YxY8YMo/tSVZXJkyczevToQuVxcXHY2uYPW1tbW5d4C7AoTk5OdOjQgZ07d9K8eXPGjBlDbGwsfn5+zJgxQ3/bEND3ZW1tXe55ZEKISnZuNyweAOOOgEPuVIj0FEi+rPucnW62rlrWy7/T8ErNGmZr1xgHnzrI+eTzPL7mcb7u8TUjNowAYFbrWfo6gW6BnL51GhtrG6a2mkqot+4XVTuNHZjvj6HaqFbJVWXo3Lkz/fr1Y9y4cXh7e5OYmEhycjJ169YtdeSqb9++REdHExkZSUxMDJ06dcp//NQI8+fPB3S3lwpKSkrSJzKLFi0q8trSRq4iIiI4deoUZ8+excfHh2XLlrFkyRKjv0Pfvn0ZNmwY48eP59KlS5w6dYqWLVuW6/tMmDCB/v3788gjj+jnXd29e7fYdgp+/+joaH15u3btWLJkCVOnTmXDhg3cvGm4DUT37t2ZNm0aw4cPx8nJiYsXL6LVakuM29nZmeTk5BLrgO5W3969e3n55Zf1iZSXlxcpKSnExMQwYEDJv5nlxd+pUyeOHDnCn3/+We5YhBAW9FUP3TIKWgfIvKO7BRg5FjZOgr2fGt3MFWtrYu1s6X2n8L93f9raMLyCF/UsidZay0NuD+lvAW58ciOuNq442eTPZV3dL39SfsGlE77o9gWb4jbhYWf8PNz7gUxoL0WTJk2YPXs23bp1IyQkhK5du3L58mWjrn322WdJSEggMDCQ9957T7/cwaVLl+jZM/9x06FDhxIZGcnJkyfx9fXVzxU6ceJEkSNdEyZMYPLkyYSFhZV71EOj0TB//ny6d+9O48aNGTRoEEFBQQBMnz6dtWvXlvgdgoKCGDRoEE2aNOGxxx5jwYIFWFtbl+v7BAcH8+GHHzJixAgaNmxImzZtOH78OMOGDSsy9hkzZjBw4EBatGhRaP7aG2+8wfbt2wkKCmLVqlX4+xvev+/WrRvDhg0jMjKS4OBgBgwYUGqyMnLkSF588cViJ7S//vrrhIaGEhISQnBwME888QRubm48//zzNG3alO7du+tvK5fkH//4BykpKTRu3Jjp06fTooXh0Lqnpydt2rShadOmMqFdiMqQcQfO74b4/XA59zbZT1N1twLLkFgBPFvbm8neXqQX+J37L62Wj92Kv31WERp7NOazrp/xSZdPmNd+nsF5HyefQolVSeq61OWFkBfMHWKVp1SlVU7Dw8PVvEnLeY4fP17oKbIHSe/evVm1ahU21eQR29Lcb9+nqnmQ/64IUWFiv4J1hlMuyqNFXT8yrBT2x13ATlW5YWVFx2IW/axIeSNUonSKohxQVdXgaScZuarC1q1bd18lIvfb9xFCCFNkWOmGrIbVqUmyojCmVvVZakCUTJIrIYQQoiQ343SrpsftBExf9TxZUQgOyJ+2cMrGhtb1/DhuWzm/fI4OGV16JVEmklwJIYQQBZ3fA0nxhY8Blg6Dda8VeUmcRsOr3l5k5B7Ha6y5YV30j9izNiU/RFPRxoaNpZajbgL9R50+quRo7g8WfVpQURQ34EugKbrFZkepqvpbWdtRVbVMT9kJ8aCpSnMnhai2stJBYwtfddcdR46FhDPw1wbdcXpSsZfO9vJgr70dB+1seSQtnR65ewDuOneBxS4ujL6VRN6WyzkW/ArltXnA5soO4b5i6ZGrD4GNqqo2ApoBx8vagJ2dHQkJCfLDQ4hiqKpKQkICdrmbtgohyuHynzDbW3f7L89v8/MTKyPd+5PqLU8PPnF3ZbOjg77seAXMPe33UP4efC42LvrPLWuVvGSOMA+LjVwpiuIKtANGAqiqmgH6EVOj+fr6Eh8fz/Xr180boBD3ETs7O3x9K/8pIyGqrd+/MelyJY+UMogAACAASURBVDetUu+Zk/Wjk24B5Ne9vWgXdwEHVeUtL8uv+TT70dmsOaPbL7ZbvW7E/BUDwDvt3sFeY8+Kkyv4z4H/WDyOB5UlbwsGANeB/ymK0gw4ALyqquqdki8rTKvV6rc3EUIIISxi3+dlvuSArS2NMzJwUO9NqYrW3t+HmTfKt4lzeXzV/SuO3DhCA/cG+uTKy163NuDIpiPZcXEHtzNuV1g8DxJLJlcaoDnwsqqqexVF+RCYBEwrWElRlBeAF4AiF30UQgghyuxOAqTfBg8jfjk/HFPm5q9bWzGyTk3d5WfPc8tKN6Nqnqcb0ZeuFnlNmpUVE729ijxXVo5aR+5k5o9VaK20vN32bZxsnPSbJUfUiiCiVgSZ2ZlYK9bMajOrUBsLuxtubi/Mw2KLiCqKUgvYo6pqvdzjtsAkVVV7FXdNUYuICiGEECVSVcjOBE2BuUxz/HTJ1YwkyEyFNS9B1zfBtcAeqFkZsGWmbm6VEe4qCgnW1vhlZXFeo6GXXx0ADp09T2hAxQ0OLOm5hPPJ55m0YxIAGisNn3f9nIhape8EIcyruEVELTZyparqFUVRLiiK0lBV1ZNAZ+CYpfoTQgjxgNr2Nvz6NkyOB1tnuHJEl1jl2TABjqwENQe6zITDK6DtP+GLjnD1iFFdJFhZ0SF39fR/3EyieYGN2PfZ2RZ3mVlNj5yOm60bwTWCuXb3GgDBXsEs6WW4L6yoXJbeuPll4FtFUWyAv4FnLNyfEEKIB03eZPTUW7r3T9vkn1vxDBxdpft8dLXuBfDLm2XqokOBbWk+cXcF8p8qfKF2zbJGXGatardiYIOBBuU17GtYvG9RdhZNrlRVPQQYDJcJIYQQZpeVBnPueWo2L7EywkJXZ1qkpROankEO8LutLdc01kww0zwpY73b7l0mbJ+gPz741EGslMIrJ/k4625vhnqHVmhswjiWHrkSQgghyu/3xZB2GzZNhimXwcah+LrzTftd/gMPd0A3QX1YnZocta2Y2333Kjh3qkdAD7TWhiu6N/JoxA+P/0Bdl7oVGZowkmx/I4QQourZ/ZFuQc81L+kSK4C1Y2Hxk3D1num72Znl7ua2lUL2PWXP1PKu8MSqa92u+s9e9l680/adUq+p51pPdi+pomTkSgghRNVy9Sj8NNWw/MhK3ftp3VIDTLkMm6fBnWvl6mZyDU/WOTky+HYyy12c9eWx9hW728HhqMMAjNw4kgNXDwCgGqz1LqoTSa6EEEJUDQe/0Y1OGeut2uXqZoWzI/Y5KutyV08vmFhVJGcbZ1b0WaE//rjzxySm3bPIqORY1ZIkV0IIIaqGH14xa3PfuDhTNzOTdqn5yyZc0GiY5eVp1n7Ko61PW+a2n4uj1lFf5qB1wEGrm1PmbKNL+DztKz9WUXaSXAkhhKh8K57RrUNVTimKwjIXZ0Yl3dZPJn7XM3+COsAljTVHbS2/aXJJfn/6dzRWpf/obevTllmtZ9EjoEcFRCXMTSa0CyGEsLyMu/BZe/gz9zZY6i24chiys2BR7zItmZAnB4hxdiQTmOfpzocebmx1sC+yrgp09/Ph9QpeVuFexiRWAIqi0P/h/thpKnb+lzAPGbkSQghhXu8EgHs9eGGr7vj4D7D8Kd3nVc9ByEBYMRL+3mpSN2ucHJnp5UmilTUpuU/NZSoKGcAvjvlLNgRX4NY0Rdk8YDNX7lyhnku9So1DVBxJroQQQpTPnQSwdwere26CpCbqXqoKfyyD718sfP7rx01OrABu5/Z7yzq//0ygRSUnU/eq5ViLWo61KjsMUYHktqAQQoiyS74Kc+vr9vQrzls+hokVlCmxWuXkSIu6fmTdU35Gq+Gn3NEpFdiU++TflAq47Wevyb/12Ny7eaFzs1rPsnj/ouqT5EoIIUTZpVzRvZ9cX3ydzDsmd/O2pzsZVgqrnJ0ouFTo4751+DN3w+TFri4m92OsKa2m0D+wv/440C2w0Pn+D/dnYsREPO08WdB5AWsfX1thsYmqQ5IrIYQQpruTAB82g8t/mNzUOY2Gq9bWhcre9PLgSzcXljo7VfgcqnEtxuk/e9p5Ut+1vv74yQZPGtR/qslTbBu8jXa+7QhwDaiQGEXVInOuhBBCmCYnBw78D27GwWftTG6ut18dAF5NvEVqgflcH7u7mdx2eYxqOor3D7wPwKM+j2KvsaehR0P9psmHow4THB1cKbGJqkmSKyGEEMXLyYErf0Kd0KLPZ6XD1v+DHfPK1XwWMN7bi65379In5W6hcx96VE4yVZT1T6zHXmOvX+QzL7EqyM/Zr6LDElWUJFdCCCEKy0yD5MvgEQA734Nf3tSVz0jSvS/qDenJus83/ip3YgXwX3c3tjo6sNXRgTPaJK7fczuwIvUP7M/q06uLPFda4hT7VCxWMtNG5JLkSgghRGGrntOtTTX1en5iBTDDFZo+CXE7zNLNr/Z2/M8tfzL6QjdXs7RbXo/UfqRQcrWq7ypqOxq3f6Gtta2lwhLVkCRXQgghCjv+g+69YGKV58hKs3Rx1dpav5RCRZoeOZ2jN47SI6AHdzLvUNelLv/69V+cvnUaP2c/ajrU5OrdqwA87P5whccn7g8yhimEEKJou/9rlmYuaXS3+pKsrLiQ+7mLvw9rnZ3M0n5ZuNu6M6P1DFrVbkUn/0485PYQgxsOBsDfxZ+fB/5c4TGJ+4+MXAkhhNBRVfhvmFmb3Opgzys1a9A8LY2Ddrp98vI2Uq4M7f3aG5QNaTSEIY2G6I83PrmRa3evVWRY4j4jyZUQQgidY9/DzbMmNREc4M9jKXfocecut6yseKOGJ4A+sQJ4w8vDpD6M0d63Pb/G/6o/9rTzZNvgbUZd6+Pkg4+Tj4UiEw8CSa6EEELA7Uu6zZTNYKOTIxtzt6MpyqoKuB0Y6BbIr/G/8nzw82ittDwX8pzF+xQij8y5EkKIB1HKdd374RhIuw0Lu1VuPGaWtx5VDYca/CP0H2ittJUckXiQyMiVEEI8CFQVsjNAYwunt8DiJ6BeW7MtqwDwm13FLUcwuOFglp9crj+O6RNDjprDoHWDABgZNBIbKxsGNhhYYTEJkUeSKyGEeBDMzF3tPHgQOOTOeTJjYgXwas0aZm2vJL3q96KeSz3e2f8OAA09GgKwZeAWUjJTsLG2YWTTkRUWjxAFSXIlhBD3s7uJsGRQ/vHh70xuMsnKijG5idRZrZb+KSl87epSylXmFeYdpnvVDONEwgl9ubeDN954V2gsQtxL5lwJIcT96uY5WNAK4vebtdmNjg78aWfLn3a2JFtbWSyxCnQLBODfrf5dbJ0gzyCebPCkRfoXorxk5EoIIe4XZ7bCrXPQYqTueNkwuGP6ek377WwJyMjE09aVr2yyWOLibHKbJZkYMZFe9XvhbueuL+tdvzeRSyMt2q8Q5iIjV0IIUZ1lZ0JOju7zN4/DD6/mn8tIMUsXo2rX5Ok6NflPUEc+8HDnmsZ8v5cPazRM/7lHQA8AbDW2hRIrACcbJ3YN3WW2foWwJBm5EkKI6uDvbWBtA3VbFy5/0wu0jpB5J79szUvw+2KTu1RzXwDxWi3RCbEmt3mvnvV7svLUStKz02nr05YNZzcQXjO8yLouNi5sHrAZa8Xa7HEIYU6SXAkhRHXwdT/d+4s74dNHIeoHiP1KV1YwsQKzJFYAj/vU5m8by60PNaThEJrVaEZD94b8eeNPutbtSs+AnlhbFZ881XKsZbF4hDAXSa6EEKI62fOp7j26j8lNxWk01MjOxlFV9WWnH+7IuYt72G9nZ9HECqCuS10APu7yMccTj2OnsSvlCiGqB0muhBCiqju5If/zobKPSl211o0E1czOLlTex68OzdLSee/aDTr7+/D2tRtMyjoDFbheFYCrrSuP1H6kQvsUwpJkQrsQQlQVN07lb0uTJ/UmLB1iUrNd/H3o4l/0RsR/2NnymrcXAN87F78foCWoqKVXEqIakuRKCCGqivnh8EGw7nPqLcjOgsw0s3aRCdxRlEJlh3O3rdljb29y+3Uc67Cg8wImRkzUlz0X/ByfdfnMoK6qSnIl7k9yW1AIISrTnyvgxA8w6GvdcVYqfNlFt/Cnez2wdy/x8rJ6pWYNdjrYExt33qzt5tk0YJP+c97WNK821y0PMbfdXII8g9gWv41397+Lt4OspC7uT5JcCSFERbl4AL7uD68cBEcv3fpUq57Tnfv71/x6eSuq34zTvcxop4NudCq8nr9Z2wXd3KmC7t1c+bGAxwAY3ng4fs5+tPdtb/YYhKgK5LagEEJUlF0fQnoSHFsDv38LswqMSn3d1+TmE6yseLp2Ta5ZF72UwR+2Nib3UZKPOn1U6HjqI1M5HHXYoJ6VYkUHvw4o99yeFOJ+ISNXQghhSbcvgb0HqNmQdltX9uN4i3S12tmJQ3a2LHFx4qmkZDxyckgvkMA8Vceya0SFeYdZtH0hqgtJroQQwhJundetRXUzDh7uDud2Q0ayRbvMmx5+VaOhY11fRiTd5rIZt6opyfon1ldIP0JUB5JcCSGEJRz8On++1KlNJVY1F9XVD0hmnZNuSYWvXV0s3uegBoMYEzoGT3tPi/clRHUhc66EEMJUOTmQt6xATg7s+i/cvlzxYTzUwSLtOmgc9JPV7TWFl2t4sdmLklgJcQ8ZuRJCCFPlTUyfkQQnf4TN0yo+hsnxcOJbizS9ut9q6jjVKVTW8buO3Ei9YZH+hKjuJLkSQghTpFzL/zzDtfh65tCoN5xYB14N4cbJQqde2z2NLee3WKTbmg41DcrmtpvLl4e/xMPOwyJ9ClGdyW1BIYQojwv7dbf+5j1ccX3mLSjqUOA2XJeZ3GjUw+yJ1fTI6frPVorhj4rwWuF82vVTrK2KXvZBiAeZjFwJIYSxrh6Dq0cgZBAs7FIxfU5LgC0zubFnPp+SiBLSlVT3uszOvMu1a4fpFfctadnm2yLnqcZPcebWGXoF9MLHyYeVf62U9aiEKCNJroQQoiS/zNbt89drHnwSqSu7YrgwpkU8MgasNdB+IjMS9/Drrdx+k0+yxg7w9wETE6tJLSfx9r639cd9HupDE88mALSu05rWdVqb1L4QDyK5LSiEECXZPhf2f1G4bPd/K6bvtv/Svds68Wv6FZOby9vjL09dl7oMbzyctY+vZWTQSHYO2alPrIQQ5SfJlRBCGOPWhYrt79U/wNG8Sxw8F/wcMX1i9Mfr+q8DIMA1gH+G/9Ngb0AhRPlY9LagoihxQDKQDWSpqhpuyf6EEMJiPmhq3vaca0NyCWthudcjPjkegF/O/2Jyd5NaTgKgoUdDk9sSQpSsIuZcdVRVVRZDEUJUDxl34Oj3kJYE149bpo/pNyH5Euz/Ena+ryubeI6Vu/6P1gHdqO3gDUCPVT3M1mWLmi30n5f1Xoa9tX0JtYUQppAJ7UIIkSftNrztZ942By+G5U8VLrOyAldfaDZMl1wNXMQdjZYZFzfCxY2sf2I9GbfOmC2Edr7taOTRSH8c5BlktraFEIYsPedKBX5SFOWAoigvFFVBUZQXFEWJVRQl9vr16xYORwghivHbAvMnVgD+JTxtV6OBbqmFoP5cSrmkL/7j+h+sP2vaRsg9A3rqPw9uONiktoQQZaOoefthWaJxRfFRVfWioijewGbgZVVVtxdXPzw8XI2NjbVYPEIIUaSsDJhdo/zX12kOlw4alk+9Dhqbwiu3//MvcC684vmttFu0Xd62/P0X4XDUYY4lHOPUzVP0C+xn1raFEDqKohwoaj65RUeuVFW9mPt+DVgNtLRkf0IIUS5/LDXteu8ili/wj9QlVgUc+McvZDp68Pu133lx84t8f/p7jtw4YrbEKqJWRKHjJp5NJLESohJYbM6VoiiOgJWqqsm5n7sBsyzVnxBClFnGHYjuAxcPmNZOYCc4tBgeHQ8739OVPf19/vnnfmHg7kmc2DiSiFoR7L+yH4Bdl3bRwL2BaX0X8FX3r7iUcombaTfN1qYQouwsOaG9JrA6d9sEDbBEVdWNFuxPCCGKl54Mez7RJUCnN0P9DnBut+mJFUDTJ6FhT9DaQ6Ne4FEftHb604leAZxI1S0CmpdY5fnr5l+m919AHac61HGqY9Y2hRBlY7HkSlXVv4FmlmpfCCHKZMss2Pe5biubPQugeRQcjC5fW2FPg1cD2Dwtv0ybu7SBr276xd3Mu3T8riPz2s/j6t2rJgZfsiaeTbiVdsuifQghjCdLMQghHgwZd3Tvexbo3subWIEukWrzSuHkKld8cjzZajZbz2/lbtZdxmwZU/5+jPBs02d5rcVrFu1DCFE2klwJIe5/qgqpZpyHZOeme/eoD4l/FzplzoU/7+Xr5Ms3Pb+h43cdAfBx8pHESogqSJIrIcT9K/EsnP4ZFCs4adq6UYW0y91QecweyMkyX7sl2DpoK05aJ3LUHH2ZglIhfQshykaSKyHE/ef6SVjUC+6Uc2Hiovb9G7EGvs5d1kBjq3+/mXaXF34YiY21De+2e7f8MRdjee/luNq64mXvpS9b0HkBL215iUENB5m9PyGE6SS5EkLcfxaYuKRe11mw6vn84xlJhU6rqoqKyt+3/qb/2v768sdWPmZav/d44uEnaOJpuIZWO992HI46bNa+hBDmI8mVEKL6S0uCXR9C8xHgXs/09kIGQfJlYnf8Hz/XDWOCmkNaVhqP1g/gl/oj+DL2P0QfM2FCfCli+sSQkJpAa58Sts4RQlRZklwJIaq33R/BT1N1n3f8x+TmLoaP4OzFnXg+1JZnTteEjEt0vnqAN/e8SaaaTdsz/0OjmP+fzk5+nfjlwi8ANPRoaPb2hRAVR5IrIUT1pKqwaQrs+dhsTW6IHMmEK7/Az9sMzp1NOqv/nKWafxK7nUa36OiUVlPM3rYQomJZdG9BIYSwmNsXy51YHbXREhzgz2mtlpZ1fYl2cQbgO/V2kfV/v/Z7ucM01uSWk3km6BkGNhho8b6EEJYlyZUQonpIS4IZrhAfC9f/gveDyt3UJkdHAPr71ibVyop5nu7sd3Qh9mpskfU/+v2jcvdVkpg+MfrPbnZujA8fj8ZKbigIUd3J32IhRPWw8jnd+5edTWrmvEbD/9xcDMpHebuZ1G5pXG1dSUpPwl5jT2pWKq82f1U/t6qeSz2L9i2EqFiSXAkhqodyrrB+zEaLrapiBVyxtuZTd1fzxlWC7/t9z+NrHgd0o1Tnbp9j96XdfHXkK32dpb2W4uvkW2ExCSEsT5IrIUTVdDhGN2l91XNlukwFEq2s8MzRrWQ+2Ke2BYIzTk2HmgB0rduVWo61qOVYC1dbV7468hWd/DoB0NSraaXFJ4SwDEmuhBBVT04OrHy2XJfO8XBnqaszg28n0zAjw8yBlY2VYsWOwTtwtHHUlzXyaCQLgApxn5PkSghRtVw9Br8tKNelFzXWLHXVPfm3PPcJwIoQ/Vg0URujADgcdZjEtET+vP4nDloHHLQOFRaHEKJqkORKCFG1LBsGN8+WXq+ANEXhHzVrcNjWxkJBFS/MO0y/mXLeVjUedh508OtQ4bEIIaoGSa6EEJXr9mX44VU4tanMl2YDCdbWdPb3MX9cRvq0y6ccTTgKgL3GvtLiEEJUHZJcCSEqT2YafBgC2WWfG3XQ1pZna3uTpSgWCKx09hp7VvVdVejWX23Hyps8L4SoOiS5EkJUjsS/4bsR5UqsAKLq1DRzQPna+LRh18VdBuWDGgyiec3mrD2zljlt5+Bh5wFAkGcQ77Z7l/a+7S0WkxCi+pDkSghROf4bVuZLTmq1fObuymZHy0wS/3erf3Py5knGho7lROIJajvWpt+afgBMe2QaTzz8BBorDb3q9zK4tkdAD4vEJISofmT7GyFElbXC2ZHgAH8SrXT/VA3wrW2RxKqRRyMAbKxteCPyDTztPWnj04b6bvUJ9goGYFDDQbI1jRDCKPIvhRDC8rIyICcTbBzhzg1Y0LLIahc01gD4ZWVzyNaGWV6eAOxysKNPyl2zh7W672rO3T7HtdRrvLX3Lfyd/Q3qLO65WP80oBBCGEOSKyGE5Zz+GRY/mX/s1wou7C1U5aSNls0ODoy9lURPP91Tf08l3Waxa/7+f1NqeDGlhnlDC6kRQqB7IIHugaiqSkTNCALdAw3qWSlWWCkyyC+EMJ4kV0II85tRzP599yRWAE/VrkmalRWNC6ymXjCxMrcPOn7Aa1tfY3LLyfoyRVGKTKyEEKI8JLkSQlSqvKUUXqtp5qGpe0TWjuTzbp8DyPYzQgiLkrFuIYR53blhdFUVzL5O1TNNn2Foo6H649Eho2ng3oBxLcaZtR8hhCiOjFwJIcxr7kPFnrppZcXoWt4svXSFfXa2fODhZvbuBzYYiJ+zHykZKfzw9w+09W3L2LCxZu9HCCGKI8mVEMJ8Luwv8XS7ur4ADK9Tk6O2tmbvvuDtvqmPTCWyTiTNajQzez9CCFESSa6EEOWXngzb3oZmQ8GrASzsoj91w8qKjnV9eefaDeK0Wq5ZW+vPWSKx+nerfxc6dtA60OehPmbvRwghSiPJlRCibLLSYetb4BkIGydDRjL8Nt+g2t82WgBWODsRa29n9jA2PbmJ7iu7AzJBXQhRtUhyJYQomy86wdUjpVbLm6aebqGNlWs71mbzgM1kq9kWaV8IIcpLnhYUQpQuOwtundd9NiKxKuh07giWqcJrhhc6VhSFWo618HHyMUv7QghhLpJcCSFK99NU+CAYUq4ZVf2mVf4/LalW5vlnZkLEBP3nV5u/apY2hRDCEiS5EkKU7swW3fuf35VeVauhXV1fYpydytxNaI1Qg7KnmzwNgJ+zHwBvPfoWzwU/V+a2hRCiokhyJYQw3k//LrXK31rdbcD1To5lanpSy0l80/MbXG11W+c8E/QMoBuxOhx1GCcbJw5HHZYnAIUQVZ5MaBdClO7GX6VWyQQ2Ozow0durXF0MbzwcgO/7fc/Vu1cJ8gxifPj4crUlhBCVSZIrIUTxVBU2TCi9HtA8wL/MzTtpnUjJTOHbnt/qy7zsvfCyL1+CJoQQVYEkV0KIfP/rBY16QeQYXWI1s+Ttabbb27Hfzo5Fbi7l6u63Yb+V6zohhKjKJLkS4kGXnQXZ6ZCZBud26l6bJheqcsPaiqlensy4kUit7Px1pV6q5V3ubv/R7B/lvlYIIaoySa6EeNB9NwJO/lj8aWcnEqyt2OVgz2duLvwz8RZOqlqurqIfi6auS132Xt5Lz/o9yxuxEEJUaZJcCfGgKyGx2m1nx5teHvrjGBdnYlycAZhyI7FM3dhZ29G8ZnMASayEEPc1Sa6EeNCkJ8PhFRDQHmwMl0u4baXgkqPyrxqebCphOYW3CiRdJTn09CG2x2/nYfeHyx2yEEJUJ5JcCfGgWTceDhe9GOhuOztG1y7/PKp7hXmHYW1lTUf/jmZrUwghqjpZRFSIB809idVRGxsm1PDkiI0Nh+xszdZN6zqt+aDjB2ZrTwghqgsZuRLiQTLD1aBoiE8tADaUcUX10nzW9TOztieEENWFJFdCPCDSL/9BeIA/LVPTWHjFuA2Yy2PN42sIcAmwWPtCCFHVSXIlxP0q4y68VRtc/SDpAj85OUANL/bZ2xGcu5r64ktXzN5tfdf6Zm9TCCGqE4vPuVIUxVpRlN8VRVln6b6EELku7If1rwNwI/kifXxqc06jNaj2VJ1aJnf1Zbcv+eHxHwAY30L2AhRCiGJHrhRFOQwUu1KgqqohRvbxKnAcKN/+GEIIA8kZyWRkZ+Bp72l4Mj0FFnYB4KLGmsUuzsTZaPnMxnC+lam2Dtqq3wfwcNRhs7cvhBDVUUkjV72BPsDG3Nfw3Nf63FepFEXxBXoBX5oWphCioM4rOtPhuw5k3viL4OhglhxfAkDimS0EL4vkV3s7soHH/HxY7Gre32smtZzEV92/oq1PWzzsjFvrSgghHiTFjlypqnoOQFGUrqqqhhU4NUlRlIPAJCPa/wCYADibFKUQopDUrFTd+8ctoa4fc/bNIXrXm8y4kQC1azLWhD3/SuPv7E9ErQgiakVYrA8hhKjOjJlzpSiK0qbAQWtjrlMUpTdwTVXVA6XUe0FRlFhFUWKvX79uRDhCPBhUVeVu5l398Tv73uHlLS8XqtPZz0f/+ZJWwwu1a1oklrxbfwCP+jxqkT6EEOJ+YczTgqOA/ymKkjdh41ZuWWnaAH0VRekJ2AEuiqIsVlX1qYKVVFX9HPgcIDw8vHy7wQpxH1n/93qCvIJYf3Y9Hx/6mJ1DdpKWlcbi44sBuHD7gr5umpXl1wGe2moqbX3bciH5AmvPrEVRFIv3KYQQ1VmJyZWiKNZAe1VVm+UlV6qqJhnTsKqqk4HJue10AP51b2IlhDA0ccdEtFZaajvWBmDzuc2427rrz/dcbdlNjz/r+hlaKy2jNul+h+perztudm7UcapDq9qtLNq3EELcD0pMrlRVzVYUZSjwvrFJlRDCdJk5meSoOQDM/G1mhfVb37U+reu0BmDaI9N4c8+bOGrNu3K7EELc74y5LbhLUZT5wHLgTl6hqqoHje1EVdVtwLayBifEgyw+Jb5C++v7UF+ebfqs/nhQw0EMajioQmMQQoj7gTHJVWju+6wCZSrQyfzhCPFgi70SWyn9zm0/l8fqPVYpfQshxP2m1ORKVdWOFRGIEA+6tITTPLPpGYu0vaDzAl7a8hIvh73MCyEvEJcUx6KjiziffJ45j86hpqNlnjIUQogHkaKqpT+gpyhKLyAI3VN/AKiqOqv4K8onPDxcjY2tnN/chagsl1MuM3z9cK6nWmYpksnebRnW42OLtC2EEA8yRVEOqKoafm+5MetVfQoMBl4GFGAgUNfsEQrxgOq2sptZE6uV8ZcLHTeLeMlsbQshhCidMYvktFZVdQRwU1XVmUAk0MCyYQlx/7ly5wo9VvZg7Zm1nEw8SXZONhnZGWbtY+u5eBpkZjKySZS+rIlnE7P2ZBBI9AAAIABJREFUIYQQomTGTGhPzX2/qyhKHSABqG25kIS4/9zOuM3rv75OfEo8/975bwCsFCv9cgumqJeZxYdXr1E/sAfknAfgnxH/YngT3bJysuinEEJULGOSq3WKorgBc4GD6J4U/MKiUQlxn2mztI1BmamJ1coMVwKHrsLKyRtysgEFfv8acrIAqOVYy6T2hRBClI8xTwu+mftxpaIo6wA7WVBUCONl52Sbra3pNxLwzsqmTWoamjduQd6olJW17r3FSLP1JYQQonyMmdC+U1GU/1MU5THARhIrIcqmS0yXcl/7zaUrAAy4ncySi1cYmHyH9q4N0Ly4Kz+xEkIIUaUYc1vwaaAt8CQwV1GUdGCHqqrjLBqZEPeB0zdPcyP1Rpmvm309gYDMTELSMzjs2g7aDYboPtDiGejzgQUiFUIIYS7G3BY8qyhKGpCR++oINLZ0YELcD8q7xEK/lDv5B48v0L3PkEFjIYSoDkpNrhRFOQPcAJYAC4GXVdUMjzgJcR+7nXGbtafX8v3p701rqFFv8wQkhBCiwhhzW/C/wKPAUCAM+FVRlO2qqp6xaGRCVEMbzm7ASevEmC1jynztDxcu0cevTuHCxz8xU2RCCCEqijG3BT8EPlQUxQl4BpgB+ALWlg1NiGribiJkpcF7jZkQ4F+uJuxycqiXlZVf8OxmUHPAzsVMQQohhKgoxtwW/A+6kSsnYDcwHdhh4biEqD7eDSjzJa1S09hrr9uqs83dVJ66nQxjD8CP/elRrwf4tTR3lEIIISqIMbcFfwPeVVX1qqWDEaK6SrKyYp2Tg9H1a2Rn84+bSXS8e5fGGZm6Qq9A9g7bi421jYWiFEIIURGMSa5WAcMURQlQVfVNRVH8gVqqqu6zcGxCVG2qCiufY6mzE295eZTp0lcTb1Eru8Dioh2nAuCgNT5BE0IIUTUZk1wtAHKATsCbQDKwEoiwYFxCVHnDFj+CR0oCv/4/e/cd3mTVPnD8+yRp071pKV2UvbdsmaIIov5wIgq4cONeuFBx8urreF2AeyDIEJGhLEVA9oYWKJtSoKV7pRnP74+UtKErbZMWmvtzXVzkGefkLhdt7p7nnPtUI7F69lwGI3LzCLEUL7idkgUmA8holRBCNBiOJFe9VFXtpijKdgBVVTMURZFPAuF+TmzCcu4Quxq3JjQrmd2WfPDxrlYXt2fnlBz4hFr/1umdGKQQQoj65khyZVQURYt1w2YURWmEdSRLCPfy5TC+C/DnvdBgh5u0MxjYpy8neYrqDveucmJwQgghLhaO1rlaAIQrivIGcCPwokujEqK+7f0VWl9dZlRpv756g7bfnzpDgUYhXasl3lhcamHMz9a+hRBCNEiVJleKomiAI8AzwFBAAa5XVTWhDmITon4cWw+/jCerxwTyBj2LedMXbNs2g2uAP30cm3B+Q04uezw98QQ8LSqBluLEqv/j0PJKl4UuhBCi/lWaXKmqalEU5RNVVbsCiXUUkxD1Z2pjMBUAMDx1BbnzVhFoNpPVKJRMjYYijeJQN1PS0ksdZMEPN0J0Dxj0nCuiFkIIcRFx5LHgSkVRbgDmq6qqujogIeqNxQKmAk5rtWiAXI0GgCytdTOCnwL8HermgzPlbNZ8+1xnRSmEEOIi50hydR/wBGBSFKUQ66NBVVVV2ZdDNBz7FsKccaRqNQyLjSr3llMeVX+7zDuZQiuj0dnRCSGEuIQ4sregY7+uC3GpWnA/7JwFwJDY6Fp11SqkNZzZ44yohBBCXKI09R2AEPXp9JFVdMxexyYvPWuL9/qrrq/yS60ovO8fuOJVmLQdWgyD1iOcFKkQQohLhSOPBYVoeHbNgb2/sjl5NTQK4+7IiBp182Cn+7ms8/0MWv0Yf538CzQa6P+Y9aLMsxJCCLckyZVwT/PvBUDxrd1efv2jB4BGyweDP8CiSm1dIYQQDj4WVBSlv6Iodxa/bqQoSrxrwxLChZJWApDo6cHz4WE17qZPZB86hHUAQKvR4qH1cEp4QgghLm1VjlwpivIK0ANoDXwNeAA/AP1cG5oQzlWYn4723XjOabVYdFpuioqsdh/j240n1DuUUO9Qrm1+rQuiFEIIcalz5LHg/wFdgW0AqqqeUhRFVhCKS85lvwykVZPGHKjmFjbntQhqwVOXPeXkqIQQQjQ0jjwWLCouHnp+42Zf14YkhJNlp8Bn1oHWmiZWN7a6kfnXzndmVEIIIRooR0au5iiK8gUQpCjKvcBdwAzXhiVELa3/H/z5gv25+Ngad/dKn1dqGZAQQgh3UeXIlaqq/wHmAvOwzrt6WVXVj10dmBC1sup1AD4PCqBjfCzz/ao34Dqh/QTb6zs73OnMyIQQQjRwDpViUFV1ObDcxbEIUTOqCnvmWQt2evqwN2EuHoqZbC89nwQHAfBKo1CHuto9fjcA+87t45u93wDwSJdHXBK2EEKIhsmR1YI5FM+3KiUL2AI8qarqYVcEJoTDjq2HeXdD9zvJHDaFWze9CtGOrwRsF9qOezrew4GMA7ZzgfpAAG5oeYOUWBBCCFEtjoxcfQCcBH7CumnzrUBzrKsHvwIGuSo4IRxiyAHgiYxNLJ99ebWaLr9xOY19GwMwLG6Y7XyUXxSzr5lNi6AWzotTCCGEW3AkubpWVdXOpY6nK4qyQ1XVZxVFmeyqwISoysmck/h7+hO49WsAlqs51Wq/a9wuFEWp8Hq70Ha1ik8IIYR7ciS5ylcU5Wask9oBbgQKi19f+LhQiDpz9fyrAdCqKuYarASsLLESQgghasqROldjgTuAs8CZ4te3K4riDTzswtiEKJfZYub1f18vOa5BkuTvKXVwhRBCuEaVI1fFE9ZHVXB5rXPDEaIKmSdY8+cTzClIrFHzYXHDeH/Q+04OSgghhCjhyGpBL+BuoD3gdf68qqp3uTAuIcpIO7aWzB+u5+kmEaBxaM9xO70je0tiJYQQwuUcmXP1PZAIXAW8hvUxYYIrgxLCztkE0g/+yeCkmdUqsXAhL61X1TcJIYQQteTIr/8tVFV9CchTVfVbYCTQy7VhCbf3eiNY8gxkHodPe/Pc7tpvCvBKX9nCRgghhOs5klwZi//OVBSlAxAIhLsuJCEgUatyfNuX8EFHjnjo+Nfbu9p9LL+xZFOBr676ijDvMGeGKIQQQpTLkceC0xVFCQZeBH4D/ICXXBqVcHs3RVkf/607doJro5tUq22MfwwLr1+Ih8aDzWM346n1RKNUf46WEEIIUROVJleKomiAbFVVM4A1QLM6iUq4r4Tf4cRG2+GoaiZWAEtGL7G99tLJPCshhHAnZ997D0XvRaOHH6q3GCpNrlRVtSiK8gwwp7odF68yXAPoi99nrqqqMulFVG72WOvfxUVB07XaajUfED3A2REJIYS4iKlmMygKp199De8uXTg3YybAxZtcFVuhKMpTwGwg7/xJVVXTq2hnAIaoqpqrKIoHsFZRlKWqqm6oebiiQTMVAfCPt+OjTcObDmfZ0WUAbB67GZ3Gkf/SQgghGgLVYiGxfQfbcebs2fUYTQlHPoluKf67dAqoUsUjQlVVVSC3+NCj+I9slyMqlLPgHvpWcxubaQOncVvb2ygwFsgjQCGEcAOqqpL5889kL11G/qZNFd9nNqNU8+mHszhSoT2+pp0riqIFtgItgE9UVd1Yzj0TgYkAsbHV3x9ONBx987dXeY+fRk+uxQDAyptWAtA1vKtL4xJCCFH/TOnpFB07xrExtznWwGKBizW5UhTFB3gCiFVVdaKiKC2B1qqq/l5VW1VVzUAXRVGCgAWKonRQVXXPBfdMB6YD9OjRQ0a23FHCIlj9JvhWfevYDhNoE9KGZoHNCPeRiiBCCOEOzLl5HOzbr1ptFA8PF0VTNUfWp38NFAF9i4+TganVeRNVVTOB1cDwakUnGr7MEzD7dm7QVTWFz2po7FCuiLuCZkGycFUIIdxB7po1HOjRo1ptIt96y0XROMaROVfNVVW9RVGUMQCqquYriqJU1UhRlEaAUVXVTEVRvIFhwDu1C1c0JKb8dAbMH06OA/OsZvV8lQ4+kRDatg4iE0IIUZ9MGRlovKzzaE9MvK/a7b3a1e9nhSPJVVFxcqQCKIrSHOtKwKpEAt8Wz7vSAHMceZQoGj6zxcze6X2wZBwhp0ljh9rENBsK+kAXRyaEEKI+qUYjalERB/v0rfrmCmgCA/Fq3dqJUVWfI8nVFGAZEKMoyo9AP2BCVY1UVd0FyExjYa8wi8//ncrn3gXgXXViteXocfKePUKgJFZCCNFgmTIySJn8Arn//AMmU7Xb65pEEvPZZ+hbtcKBh2su58hqwT8VRdkK9AYU4FFVVdNcHplomN6OJTE8DHx9Kr1tQpGOK3s/jX5QPHqvkDoKTgghhKsV7t9vS4IS2hQ/vtNqwWyuVj8xM6ZjOnsW3z590AYFofGp/HOlLjmyWnAR8BPwm6qqeVXdL4SNqQgsRvD0BbMRlr/CAj9f/qoksXop5DKGb59PwPXToePNdRisEEIIV8teupTkx5/A74qh6Bo1KrlQzcSq5b/r0QUHOzk651GstT4ruUFRBmItJDoS2Az8DPyuqmqhs4Pp0aOHumXLFmd3K+qDsRDeiLC+fnI/7JkHf0ymYxWT13eP310HwQkhhKgPtpGqWoj+/DP8Bw2qfTBOoCjKVlVVyyxldOSx4N/A38UT04cA9wJfAQFOj1I0HHvnl7x+zzqx8CYHJ68LIYS49OVv207WwoVkzp5N0zmzMaU7VnKnPG0TEwAoOnkSz+hoZ4XoMg5txFa8WnAU1hGsbsC3rgxKNABn9todpms0JOo9K23SJbS9KyMSQghRB1RVJX/TZo6PH287d/TmWyppUb6W69Zy5s238Cs1SnUpJFbg2JyrOUBPrCsG/wf8raqqxdWBiUvMya2Qth+63Aa758K//7O7PDCu8m+IJaOXEO13aXzTCCGEqNjhUaMoSjpU4/Zt9u6x7QkY9d5/nBVWnXJk5OpLYEzxVjYoitJfUZQxqqo+VEU74U5mDrH+3fEmSE2sVtN1Y9YR4ClPmYUQ4lKkqiqYzSg6HcbTp2ucWDVbugTVYKi3zZadqcrtb1RV/QPopCjKu4qiHAVeB6r36SkatrxzJa8XPgxrpvFRcCDDo5uwS+9JYQU1R6adTWPesK8ksRJCiEtY2iefktihI1mLfidp0OAa9eE3ZAj6+Hi82rRxcnT1o8LVgoqitALGFP9JA2YDT6mqGueqYGS14CVk4xewaio8fwI+6w9n7Ff5VbUqEGBn7G1oBj/vqgiFEEI4ybmvv8Fv0ED08fEAJA0ZSvAdd+A3aCCHrx5Rq751TSJpsXLlRVH8s7oqWi1YWXJlAf4B7lZVNan43GFVVV22Y64kV5eQKcUV01uPhP2LbafzFYVeTWMc6kLKLgghxMXPYjCwv3MXAFr8/Re6kBASO3aqUV/BY8cS8dyzqCYTadOnE/bAA2g8K1/sdDGrSSmG0cCtwGpFUZZhrW916aWVwrVKJVYA6doqnzQLIYS4RCUNHFSjdoGjR+M3YAD+QwajeHigeHgQ/uijzg3uIlJhcqWq6q/Ar4qi+ALXAY8B4YqifAYsUFX1zzqKUVwMzEbISYHCbNi3sMLbvgmU+VNCCCHsNXnzjfoOoU45UkQ0D+v2Nz8pihIM3AQ8C0hy5S6OrIFvR1V52wmdltkB/pXes/KmlQz9ZShRflHOik4IIYSLpH/7LWfeertGbZu89x+yFi68ZGpTOZNDRUTPU1U1A5he/Ee4i6QVlV62AMc8dFwb3aTS++ZdO49G3o24t+O9jGpedbImhBCibhkOHuTwqGvxiI3FePx4jfuJeOEFAkeOJHDkSCdGd+moVnIl3FBWMqz7sMxpE2BWwIjClTFR5FQy12rJ6CXE+JdMcp/UbZIrIhVCCFELuevWceLuewCqnViFP/MM3p07ofH2xqtdO1eEd0mR5EpU7r/lf5NMbBzOZm+vKpu3DG5pl1gJIYS4eBSdTKZw9y6SH3+ixn20Sdh3SZZRcCVJrkTF9i4o9/R3Af4OJVYAA6IGODMiIYQQTmJMTubQFVfUqo+4H3+QxKocsm5elO9sAvwyoczpPEVhWmhwlc27R3QHoH2YbMYshBAXC3NuHoX791OwZy9JQ6ufWPlfPZy2iQl4RFkXJfl07+7sEBsEGbkSJYyFcC7JujfgvLvtLmVrFFSgf5xjj/i+Gf4NJ3JOyCNBIYS4CKiqivH4cQ5dNbxW/URNmwZAs98XoRqNzgitQZLkSpR4I6LMqSJgt17PhCZlr1VFEishhKgflqIisFjQeFmncBy6anitVv+FP/csAcOHo+isaYPG2xu8vZ0Sa0MkyZWwKr35crE3Q4KZFVh53aoLvdr3VTqGdXRWVEIIIarJUlDA/h6XgdlM41deJvPXX6uVWIXedx9YLPhfdRVnp00j/Kkn8e4oP9erQ5IrATOGQPLWMqerk1j1j+pPr8a9GN1ytDMjE0IIUU3H77wLzGYATr/6WrXattq8Ca1/yc/+uG+/cWZobkOSK3f360NlEiszUP523uW7q8NdPN79caeGJYQQwjFJw64k8PrrCJs4kRP33U/Bjh016qflv+vtEitRc5JcubsdP5Q5dW10JMc9PKpsemvrW7m59c20DG7pisiEEEJUoejoUYwnTpD28f9I+/h/1WrbdO5cvDu059Szz5G1cCG64KpXggvHSCkGd7btO7vD3Z6ebPbSO5RYAbzQ+wVJrIQQoo6lzZhBQpu2FO7bx6HhV9e4H33LFgBEvvUmbfbsdlZ4Ahm5cl+LHoWt39gO0zUabotqXGUzBQW1Wg8NhRBCOFPqe+8DcGT0DdVqpwsPp8Vfq1E09uMqikYDGhlrcSZJrtzRuUO2xGq9txdHdTreCgupstmOO3ag1WhZfmw5zYOauzhIIYQQ56kWC5lzfuH0lCk1ah/73bd4d+pUJrESriHJlTvZ9xvMucN2eMDDg/sah1fZbPqw6WQZstBqtAAMixvmshCFEELYy12zhhMT76tR24jJz+PVrh0+PXo4OSpRGUmu3EHSClj+CpzZYztlAWYF+FXZ9OU+L9OnSR8XBieEEOI8VVXBZELx8CBz/gJSJk+uUT+Rb0wl6IbqPTYUziPJlTv4wf4bzAT0jYumoIrh4fOPAYUQQriW8fRpjKdSSP3oI/I3bKh2e11kJC1WrURRFMw5OVJSoZ5JcuVGMjQaBsRFO3Tv0tFLJbESQog6kjRocK3aR3/0IYqiAEhidRGQ5KohW/sB7F9iO1zg7+tQs/5R/Yn2dywJE0IIUX3m7GxUsxnVYODUc89Xu33E889hSEpC36o1gdddizYgwAVRXvwKisysTDzDNZ2a1HcodiS5ashWvALAzMAAOhkMKA40mdB+Ak/2eNK1cQkhhJsxnTuH8cQJ8rdtx5yZybnp02vUT8zMmfj06G7bkLkh2nj4HN3jgsk1mPD38kCrqfjT67Xf9zFr03EiA73oHmdd9Z5VYCQzv4i4UMcGFFxBkquGymSwvfwwJKjK20e3HE2hqZC7O9ztyqiEEMLtqGYzB/v1r1UfbfbuQdE6f6qGyWzBaFbx9nT9NJBzuQYKjGaaBHrz285TtAj3I+lsLtd3jaLQaMZgsvDd+qO8t/wAvp5a8orMDGrdiLv7x9MuMoAF25NpGxmAj6eWrrHWavKzNlk3pP59VwoLd5ziu3+P2d7v6NsjXf41VUSSq4Zqajj9YqNoX1Tk0O1NfJtwX+eaLfUVQghhTzUaSZs+nZDx4ynYubNWfQVcc02NEyuT2UKhyYKfvuTjXlVVXvx1D32ah/L+8gMcTs2zS0QKjWbOZhuIDfWxnTuSlkd8mC+FRjP/XXGA67tE0Tay7KPIXSczefqXXew/k8PGyUOZu/UkpzILGNMzlms+XgvApCEt+GhVkq3NY7PL7oWYV2TdePqv/an8tT+1yq/z63VHq/7HqEOSXDVEB/4EIFur5V9v70pvHRE/giVHltDYt+rq7EII4S7MFhWDyYyPZ/U+JlVVJePHnzgzdSpAtff7Azh3/5P4fDedmIfuJ/TOO9FU8lgsv8jE47N38NI17YgO9mF9Uho/bz7B+L5xBPl48smqJOZvTwZg58tXMnPtYQK9Pfhx43F+3Hjc1s/6pDReXbSP/WdybOfG9YlDVeH7DdbRIJ1GwWSx7tDxxd+HAfi/rlEs2J7M57d35/4fttrF1uvNlbbXpd+rdGLVUCmqevFsZdKjRw91y5Yt9R3GJW/dOxHEGo2MiImq9L7NYzfjqfVkbfJaLo+63LbSRAgh3N0LC3bz48bjHH5zRKXJDUCewYTOUIAmN4ecFSs48+ZbNXrPbY1asq5JR5bE97U7v3hSf6KDfcgvMjHlt72oKtzaM4aoIB+Gf7iG8x/jn9/ejft/2Faj926I6uKxoKIoW1VVLVOhVUauGpL9y2D3HO53oOr6ousX4aWzTogcED3A1ZEJIUS9OHAmh/gwXzy01dv25fxIi0VV0ZRaDrT+UBoBXh7Eh/mSmmOgaZgv79z0GGMOrKyoK4f8Ht+HTzqXX/Rz5Edry5z7c9+ZMucksbp4SHLVkMy6xfp3fGylty0ZvYQY/5g6CEgI4S6++PsQsSE+XN0x0uE25ydT7zmVRXpeEVe2i6h0BN1ktrD5aAZ9mofanc8uNOKv1/HpX4eY9sd+9k8dzqRZ2zl2Lp/E0zn463UsfLgfgd4enMos5KYv1lNotKBRYOuLwziZUcBbSxO4rksT+rdsxIZD52x9t3hhKUffHonZotLv7VWczi4EQGcx4WUq4tqzO7mjBonVu93H0CIzmXydnlsPrOS3ZrWb8C4uLpJcNRSbZgCwQ+9Z6W29IntJYiWEsDl2Lo8v1x7hlVHtK13yfiGLReW/Kw4woW9TzKrKW0sTAeujmAXbT/LG4kQ+HtOVPs1DSc0xcC7PQJ7BREpWIX2ahaLVKHR5bbldn7deFsPbN3SyO/fBigMUFJl58srWtHpxKQAdogIwGC0cPJvLrw/14/pP1tm1+Xt/Kn/sLRnZyTGYGPLe3wR6e5BVYCz5GlTo+npJDOtLJVWlNX1usd2x1mJm0W/POfpPVcbm8Nasju7G6pjuAPzY9qoa9yUuTpJcNRAr/nqJx6sYsQrSBzHzypl1FJEQwtn2ncomr8jEZU1DbOf+OZhKr/hQPHUaDqXm0izMt1rzJx/+aTu7k7Po1yKMK9pGYFFVPLQa2yqvSUNbkpxZwKrEs7SLDODgmRzaNwlk67F0Pl6VRNLZXJbuOW3rL+lsLo/Ptq6Om/jdFu6+PJ4PVhy0e89usUGE+OrLxPLz5hPc2jOWLjHW8jH5RSZb28TTJROt9yRn215fmFgBTPx+a5lzgF1iVV1PbP2Z074h/B3VhZkr361xP7tCm/Fy33tr3L6h6hobxPbjmRVef2FEW+LDfFmblMbL17QjLddAz+IJ808Ma8X7yw/UVagOkQntl7qj6yhq0oXus3pXeMt3V39HjH8M/p7+6LVlf6AJIVxLVVVOZhQQE+JT9c3FPv/7ED6eWsb1aWo7d34EZd9rV+HjqWPXyUyu/d86JvRtyvAOjbl1+gbeGt2RrrFB+HjoiA31QVVVFEVh4Y5kHv15B/8+P4R7v9uCgsKozpF8u/4YyZkFdu99fZcm/LrjFABdYoLYcaLiDz1X+P2R/rZl+/Wq1Ofj0oVPV7v5Sd8wvuh0HSm+YaT4hqICKgo0kMVDA1o1Ys2BkjIJL45sy9C2EWw6co5n5+0G4Kd7e9E1JpiE09mM/nQ9H4/pyrxtJ23lFRQFvr+rF/1bhnHF+3+TdDYXgAcGNWfMZbEcPZfHzLVH+GbCZWUWFizdnUJMiA8dogJp+txiru3chHdv7EROoQkPrUKQT+VPcpyhogntklxdyk5ugZlDSfLw4P+iK57nsHv87joMSggBcCqzgJ0nMrm6YyQ/bjzGCwv28OtD/WyjMgD/HjpHz/gQ2+O4Y+fyaBzohV6ntSVS51c8FZkstsdi5735fx2ZvMD++/uWHjHM3nICgDA/T9JyHat1J0oEFeYw+OQ2xuxfgb+xoOoGxQwaHXqLyXY8ue9Etoe3ckWI1RIRoOdMtqHSe/z0Or6+8zJu+vzfMtfe+L8OvLBgj925x65oyWNXtCK70EinKX/SLjKAJY9ebruekVdEsG/Fyc351ZirnxpEfFjtK6lnFxrx8dCiq+bChdqS5Koh2v4jLHyQEdGRnPDwKPeWrbdvxVPr+uxdiIvBqcwC5mw5waNDW1br0diWo+mE+elpesEP+U//SuLzvw7x1YTLSM4soFtsMGdzDHSPC7a776EftzG8Q2P6NA/FYLIQ6uvJoGl/cTq7kP/e0pkv1x5hT3I2U0a149aesbR5aVmZGKaMaseURfsAWPro5Vz94T+2azPH9eCe7+Rnoyt1SDtEsl8jMvV+LFn4TI36eKb/A7y79jMAPu94HQubX17mHkcTXn+9jhnje3Dr9A22cz2bhrDpaLrt+LZesfxUvKpxSJtwks7mcjIjn1/u70tUkDc/bTzGR6uS2DXlSvIMJmasOcJX644wqHUj/tqfyhPDWjFpaEu2H8+gVYQ/vnqdXVL/7rJErmzf2PYLwfbjGWTmG1m8O4V3buhUrTl6DZUkVw2NxQKvWX/AdyxnrtVzPZ8j2i+agTED6zoyIao0a9Nxjp3L56HBzXljcQLXdYmyrQDbeiwDs0Xl5i+sv0GXV6um9AfAmgOp9GkeSlqugUHT/sJgsjChb1N6NA0mMtCb1BwDwzuULZJrMJn5dPUhRnWO5Ir31wDw0jXt6N8ijNaN/e3e50IjOjbm07HWycgJKdl2iZC4tPgV5dMs6xTvrPu82m0/6nwDk3bOA+COK19Eby5i5sp3WdukI2/0HM+G54cyb9tJpv2x39bm6NsjeWL2Dlthzy4xQYzpGcOz83bTprGCHL9SAAAgAElEQVQ/DwxqTs/4ECIDSwpAH03L4/YvNzL3/r7otApnsw0s3JHM48Na8daSBJIzC5k5vsznOxaLisFksW1to6oqablFNPLXcyg1l/hQ3zKP2i4cMRWVk+SqoUg7CH4R8HYM2/R6xjeJKPe2HXfsQKtx/V5RQtTE+R/g13SK5PddKQAkvXE1WQVGuk9dYXfv2mcH46/3IDXXwMqEM9zaM5bOr1p3Ifjw1i48+vMOHhrcnE9WH6rw/Ya3b8yyvdZJ173iQ2jfJJCv1h2p8P5PbuvG2qRUZm06UeE9syf25pZSowri0hEdqKfTjtW0P3eUy84kVOvRX2kx23awcNMRVm5KYlW6NUkZkrqPGR8+gMa3ZGHB+f/vm14YSri/F2aLyuHUXGJCfPDy0FJoNDPio3+Yen0H+jYPc84XWUPHz+WTnFlQptyFKJ8kVw3FlEBo1Ib7NedY51P+1jZtQ9oyZ9ScOg5MXOrScg30mLqCx69oxbVdmrD/dA7D2kWUO/Rvtqiczi7EQ6vg5aGl62vLeWhwC54Y1oo7vtyIRlH49q6e/LH3NLM3n+CrCZdxJruQQ6m5fP/vMbvVZeeVToCEqI07esdxMiOf1aX2pNs0eSjmjHR+/mMn1/31A4ZtNS+4+X/XvMGGJ/oRVOqX2zUHUjmdVcjNl5UtdWMwmdEoSrULmYqLX50nV4qixADfARGACkxXVfXDytpIclWFxMXw821VTmDfcNsGfD1qP0FQXFo+XHEQjQKPDG1Z7vWmzy3mhm7RvH1DR8wWFS8P+5HN/y4/wIcr7ZfMvziyLf1bhvHdv8eYel0HDCYLT8zZYZcczb2/DzeWMwm2tM7Rgew8mVXDr0xcTKbd2Imn5+5y+fu8Mqod/l4ePPWLtazD+UnVpVeo/fPMYNtjL51Gse1lV/qRVvfXl9Pcy8L7p5aRs7TsXLfqajr7Z7w7d651P6JhqI/kKhKIVFV1m6Io/sBW4HpVVfdV1EaSq0oUrwyE8udYnffFsC/o26RvhdeFcxUazagqtjkNtbFsTwphfnp6lKph5KhJs7bz207r0vmjb4/EaLawJzmL//t0PV1jg1jwYL8y84cOvzkCs6rSY+qKCuv/9IgLZsuxjOp/MaLBWvvsYKKDfSgoMpNfZGLjkXTyi8wcT8/no1LJ+fd39+SOLzcxtlcsL45sh1aj2FY7RgZ6Me+BvoT762nxwtIy77H31avw1VvLMC7aeYoBLRsR6FOyaKeieUHztp6kS2wQzRv5AaCazahGI/u7dK3x1xt8xx1kfP89EZOfJ2TcuBr3Ixqmen8sqCjKQuB/qqour+geSa4qMSXQ9rKi5GrnuJ1oFBl2rktdX/uTjHxjtSZ/Hk7NxWhWCfPzJNTPWndMVVXin18CwJ39mtIrPpS1San8sOE44/vEMeXa9pgtKomnc5i9+QQ5hUZeGNmOy99dxYCWjez2Gfvgli5sO57Bd/8ec+4XKy4JV7aLINDbg1+2ngSs/5+uat/YtuqsbWQACSnZdm2eGGYtFzBpaEu7JPzvpweh1SiczTHQyE9PZKBXpUvd1x5MY8H2ZK5qH8GV7csuIigvKfpt5ylUVUWjKGgUhZGdqt4+Z2XCGSICvOgQFVjmmjk3j8I9uzk95VWKjh6tsq/y+F5+OdH/+xjVaELr54ulqAjFw0M2txdl1GtypShKU2AN0EFV1eyK7pPkqgInt8LMIZiAd0KD+TnAv9zbpJ6VcxSZrHuOVVUvZdmeFNtGqRcmVyazBYtqLZCXmmOgkb+e6z9ZR/NGfrYRptLu6B3H9xskGWoIPr+9W4Ub6G6aPNRWVfq8zjFBjO0ZyzPzdvHIkBZ8vCoJgJbhfhwsLqj44si2ZOQX8cnqQ/RrEcr9A5vTv0UY13y8Fo2i8P7NnRn2X+uKx9L/F01mS7n/jztO+YOcwpJ6TKXbzNp0nOfn72bHy8OcXoQxq8CIokCAV/mlYxxlPH2apEGDCbrpJvQtW+B/1VXowsI4N2MmqR98UON+G7/+GpgtBN1ysyRSwiH1llwpiuIH/A28oarq/HKuTwQmAsTGxnY/dkw+YGzMRpgxGE5bk6aVPt48FtGowtvrO7kyW1R2nMigS0wwCpRZ4lvfVu8/y9LdKbx7Y2cy8oq48fP1/HJ/X0J8PTGaLWgVhV+2nuDZebvpERfMh2O6suZAKjd1j0an1bAuKY3IQC8Ons3l6V92kl3qwwkg4bXhdH7tTwK8PEjLtS/Y1y4ygH0pFf5eIZzkwUHNGdenKb3fqv5GuuW5tnMTWzK87rkhrE48S5eYIJ6Zu4tXRrUrs1rwg1u6cH3XqHJLOCx4sC9dY4PZk5zFO8sSubxlGG8uSWTVkwNpVvwYK6fQyIM/buOdGzrRJMib9Ulp9IwPsSVIhUYzOo1SbsK0OvEsp7IKGNsrrsqv62RGPgfP5DJv20m8PLT856ZLaw5RQpu2TuvLq317/IddgW//y/Hu0N5p/Qr3UC/JlaIoHsDvwB+qqr5f1f0ycnWBZZMxb/iELvGx3JeRxRfBZYfAz9ty+5Z639qm9ITonk1DmHN/n1r1Z7GoWFS1zAdJt9eXk55XxLaXhhFSqgLw9xuO8dKve3jvps4Mat3I9sjtmbk7WX/oHCczrMut/3x8AC8v3MOGw9ZifC+ObMvUxQkVxnHv5fG8MLKd7QOzkb+e1JzKqx2LuvHEsFZc2T6Cid9t5b+3dKZ7nHW+2vmtXsb0jOHK9o3p2zyU1i+WTGbWKLB/6tXkG8zM336SVxft49rOTQDo0TSYb9Yf5c5+8YztGUuzydbHteU9+j14Jsc2YvS/27oysmMkiqLw1/6zPDJrOzmFJpY/PgCdVlNuFer8IhM+nrLFqyNOTX4BfauWhIwfT2LbdrXuL+iWWwi7/z48Iqt+DClERepjQrsCfAukq6r6mCNtJLkq5fvRcGglHwcFMr2CpOrejvcyY/cMov2iWXpD2Umh5XlraQK940MZ3CbcofsLisws2Z3C6G5RZYbJ8wwmkjMLCPX1LFObCOw/jNJyDSzZncLiXSlMH9cDH09tlcuS7/pmM6sSz3L07ZGk5RrYeDidTtGBXP7uats9/zwzGL2HhlEfr61yewdRP0pXHn/12va88tveMvfce3k8M/4pqTul1Sgkvj6cid9tYfX+VLrHBfPV+Mvo/Jq1vpUjc9wOpeYSE+yDp876/ywlq4DRn64nJauQIB8Pdrx8pe3e/adzaBXhV+6joKqKKrZ7eRn5ReYy11VVJcdgqvUjMGF1frQqYMTVZC9x7OfdhbRhYWh8fDCnpdF6W/mbOwtRHRUlV678lakfcAewW1GUHcXnJququsSF79kwJC6GQ9bHGhUlVv6e/kzqNolJ3SY53K3JbOGLvw/zxd+HK/ygMJjMTFu2n0eGtiTQ24Opi/fx48bjRAZ6EeqnJzrYm/S8IrafyGTSrO0AfDWhbGVgKJm7dM3Ha+12tB84bbV1C4VJ/Xlyzk48dRp2ncxCp1F4dGhLHh7SghPpBaxKPAtUXCUbsEu0RN07+vZIEk9nM/yDf/h5Ym96xAXz+64Uru3chLwi62PTgiIzUxbt4+Vr2jG+b1NGdIzEQ6tgMFmYvy2ZTtGBdIsNxlevY1yfpnR7fTmNA7zw0Gro1yKM1ftTeWVUOwJ9PPjjsQEcOJNTRVRW51eMnRcZ6M26Z4fwyKztTOjX1O7a+YrsNbHu2SG2r7U0RVEksaomc24uptRU9PHxABQmJHDk/0bT9OdZtntqklhp/P1pvXmT0+IUoipSRPRiYzLAVOuoUqpWw5DY6HJv+2jwRwyOHUxGXpGtxktOocm2UeaRtDwa+evx05fkz4/9vN220z3AjHE9KDJZ6N8yDJ1GsdtXanyfOHz0Ov7ce5pDqXnc0z+emWsrrmhdkbv6xVdaCVvUjZVPDmToe38D1tG+zHwj4QF65m9LZtvxDJaXWm1Y2qdjuzGiYyRmi0rzUo/HCo3WoojnR4Wc6cu1RxjWNoLYUB9UVeVIWp5tTlJ9aPrcYjpGBbLokf71FoO7OHzd9Rj27yfq449IfsTxXxwr0mLN35gzM9E3a4aik8evwvnqvRSDIyS5Av58CdZ/hAXoXEk9q/OT15s+t5i2kQF0jwvihw3HSXhtOMmZ+Vzx/ho6xwShUaBN4wDG9Ymrcv+zcX3ibMv3mzfy5VBqntO+LFF9n47txoM/buORIS148srWfLX2CK/9bn289vnt3RjewTpXJKfQiKdOg16nZcpve/HT6ziRkc9tPWOJCvYmOtgHgI2HzxEZ6E1sqE+Z9yoyWfDQKhQYzXy7/hjvLEtk3gN97TYo3pOcRbCvJ1FB5e8M0FAlnc0lIkCPv4xCudSxcePJ31T70SXvHt1p9MgkzNlZBAwb5oTIhKiYJFeXiuJ6Vlkahf5xZbdRAPjt+t8YPi2RMT1j+Wb9Ubtr/VuEsTYpzdVRCgdd0TaCFQllR4Wigrx5dGhLnpm3i8euaMnortHEhvrYtqDZOHkoEQFedm0MJjNdX1vOHX3ieP5q562WEqI+qKpK0ZGjZC36DeOJk2T//nuN+/Lp05v8fzfQfPmfeMaU/3NTCFeQ5OoikWcwkV1otNvxvNBoJievgEZHF7Hyz8c5rdPydmjFVbq/6Lea22ZurItwRSUWPdyfTUfTef33ffRvEcbVHRuzeFcKU65tz47jmfRrGUZUkLdtxMlfr2P3q1fVd9hC1KvCxETyN27kzFtv16ofxcuLuG++RuPnh75FCydFJ0T1SHLlYjPWHOb33SksfKgfYE2YsguN7D2VzYp9Z/hx43Gm39Gdid9bV6isfHIghUYzfx9I5d1l+3ld9xV36FZUurUNQO6hJ1GLKq51Jco3rk8cq/ef5baecew6mcnSPadp09jfbqL9eeP7xOHtqWP6mkP89nB/rvl4LQBLH72c+DBfW42jUD89hUYzj/28g8kj2pb7uO28E+n5+Ol1tjlxQriTouPHSZs+Ha9WrTjz5ls17if2m69JfvIpms6ejWd0lBMjFKJmJLlyoTPZhXYbhiakZFc5v+k8BQv/6B8jWrE+yqsouco/dg8WYzCqMdQ5QV+khrQJt60SPG9Ex8b0aRbK7uQs5mw5aXftqvYR3D+wOacyC3noJ2tV7IcGN+f6LlHEhPig12nYeiyD7nHB5S6z/33XKf45kMaUa9sze/Nxlu09zY/39EZbqgDqifR8Ggd6yY72QlSTMSWF01NeJffvv2vdV/M/luEZV3WBVCHqkiRXTnTbjA30bxmGr6eOX3cks/14pu2ap1ZDkdnicF9HvW4DqHQCe8GpmzBlda9VzPXJU6th5ZMDueubzXhoNexLyWZEx8Ys2X0awLYSsW/zUGaM68EvW06QU2hiUOtwthxL585+8ba+zBaVlKwClu05TUJKDq9e195uRaQQon6devZZvLt3J/jmm51SST1u1k/4dK35xstCuFJ91LlqcMwWlaSzuaw/dI71h86Ve091EqtopWSE5oBn+SuR8o48hKWw7idotgz344Nbu7B092mSMwu4vXccN3y2vsL77+zXlKs7RNIzPoTNR9PpEhNEfpGZh37cxkODWxAT4sPyJwbatWn63GLu7h/PE1e2YvPRdCaPaIuvXseEUslUx2j7Ol9ajUJ0sA/3XN7MuV+wEKJGTKmpaIOCKNy3j6O33ApA1sLf0PrVvHxGk/f+AyoEjBwhe/yJS5KMXFXi/L/NJ6uT+M+fB5zadzDZbPe6H4MCS3x9eblR+Y/7chJqPulzdNco/jqQSnpeke3crHt7s2jXKcZcFouiwKj/rUVVrUnL0kcv5/O/DjF/ezIbnh9K40D71WoWi2rbCuSvpwYR7OvJhK83sf14JklvXF3lRsdCiIbFUljI/i7OG1Xyuewymrz3HzzCHdtBQoj6JiNXNfDwT9tZvDvF6f3GKaf5W/8EAFfERJGp1da4rwUP9mXjkXTeXpoIwM8Te/PTxuP8X7coBrcu/wdUn+YlidyhN0YAJZssT7upMy9e085uz77zSv8C2bR4n7T5D/TFZCm7/58QouFK/fRT0j76GL8rhtaqn0ZPPkHqe+/j07Mnvv36EXrP3Si1+HkoxMVCkqsKLNtz2iWJVQB5tsTqhE5XYWJVeOAd+jYPY+I9zfh91ykm9I3nqg/WMOve3oyZsQGAp69qTdfYYIpM1keRrSL86N0slN7NHJ/0rtHYD7lrNUq5iRVYt/P49q6eNC5Vf0lRFDy0MmwvRENiyc9H42Nd/Xp49GgM+xKI++knCvfupXDPbrIW/gZA7oqVNeo/+tNP0IVH4N2hPcFjxqDR61E8pEiraDjksWA5Sm/1URsf3NKF+duTWXMg1Xbu/AT2j4IDmRFU/r6BALvG7apwrsHRtDwmL9jNjHE98NXrbIUnP7ilC9d3leXJQoiay1m1ipMPPkTUf98n7dPPMBw86NT+W2/dgsbX16l9ClFfKnosKM9yyvHFmkPVbnN+AGhQa2sNqhdGtOX6rlFMv6M7uuKLfzzSB4BTOm2lidXfN62vdBJn0zBffrq3N77Fq+TC/PQcfXukJFZCiBozZ2aStWgR6d9+B0Dy40/UOrFqtvh32iYm4Nm0KQCeTZuieLvX9knCPcljwQu8vHCPbX89Rxx842pb/aPV+8/Sv0UY+QYzgT7WIW4vDy1Jb46A/UthRjznNBquiik/CVp43ULMqpkQH//afyFCCOEAS0EB5sxMkgYPqVU/uvBwwp95hsBrRlKYkIAuLAxdI+svm3GzfsKYfArvDu2dEbIQFz1JrkoZO3MD65LKL7FwobXPDibYx9OusOT5CeSBPuUMCM66lVM6bYWJFUCzICkvIISoG/nbt3P6lSkYDtR+JbTG15fYb79BH28to+LV1r6+lS44GF1wcHlNhWiQJLkqxdHEKsBLR3RwxVudlJFnrb5+wEO2PhFC1L2CHTvQhoaSvXgxuf+spWDr1lr3GTBqFAVbtxLx0ov4Dx7shCiFaDgkuSo24N3V5Z7fOHkoEaVWx51Iz8ffqxr/bHvmw7ZvydJoeDI8rNxbOoV1oltEt2rFK4QQVVHNZrIWLSLlueed2m+LVSvxaNLEqX0K0ZBIcoV1deDx9Pwy59+7qbNdYgUQE1KNESuAuXeSqdFweVx0uZevjLuS9wa9V70+hRCiHMYzZwEVzGaShtSuBlVp+rZtiZz6Ot7tZc6UEI5w+9WCFovKiHI2WZ48og03dC8/IXJIbipMsa4IfCO04rkGklgJIWrCdO4cBXv3YkpPJ6FNW7KXLydp4ECSBg5ySmIVMn4cLf9Zg+/AAcT/MkcSKyGqwe1HruZuO8n+Mzllzl/RNqLmne78GRbcZztc5mdf0+XT3q/y4IZXGNlsZM3fQwjh1o6MvgHTmTP49u0LQPIjk2rdp2d8PJGvv4YpLQ3/IUNQPD2J/eKLWvcrhLtx++Tqmbm77I5DfT3Z+tKw2nW66g3by47xsWUu924xir9iBxKgD6jd+wgh3IqlsJD8LVs58+abmM6cASBvfcUbqlcmYNQowh58ALWwEMXDA8ORIwQMq+XPPiEEIMlVGX8+PqB2HVgsUJABQHm178e3GYuH1oNQb8e3qBFCuJ+ENm0JGT+O0HvuIfWTT9B4+5D+9de16tO7a1cKtm8HIGrau3bX9C1a1KpvIUQJt06u0nINZc6F+ulr1+nKKVBkfcx4rpzNjJ/q9Vzt+hdCNEiqqpLy/GQUvR5LXh4A6d9+Z6uYXhvNlizGcOgQvn36oHh6ouaXXcAjhHAet06uekxdYXf811ODat/p1m9q34cQosE7N3MmHtExBAy/CktREUdvvgVDYqLT+g+5+y4UnQdhD9yPxssLfbNSRYo9peaeEK7ktslVodFc5lzTsFpuJlqUD4VZADwSHsZfvvZlGzaP3Vy7/oUQDYJqNHL2P9aVwsmAvnVrDPv317rf8KeetPUb/tRTle5RKoRwHbctxbDmQKrd8cbJTqgJs/I128sLE6u3L38bL53XhS2EEA2UMSWFohMnACg6ehTVbEY1Gjk2bjyJHTvZ3VvbxCryjam0TUwg9J57ANAGBkpiJUQ9ctuRqz/2nrE7vrBYaLWdOwQbPyv3kkbRSNkFIdyIqqq2jZB9LruM/M2b8enVi/yNG53Sf8t1a0l58SUM+/cT/9tCtH5+tmtR/30frwuSNyFE3XLb5GretpO214ffHFH7Dj8u2b7mXy/7SfFfDJM6MUI0dObMTIqOH0ffsiWnX59qO5+/2TodwBmJVdT77xEwwvrzKuazT8u9J+Dqq2v9PkKI2nHb5Ko0jaaWw+dnSyahmoCJkfYFSHtH9q5d/0KIi5YlPx9Fr+fYHeMwHDzo9P7bJiZQdPw4p6dMwW/gQKf3L4RwPrdMrtYlpdlej+sTV/sOP+1le5mic8t/UiHcgmqxcPaddzClnSPkrjvRePtweIQTRr6Ltdq8Ca2/P2BN2kxp1p9VnrGxxH71ldPeRwjhWm6ZCYydWTI8/9I17WrX2bF/7Q4/Dg60O/5g0Ae1618IUS9MGRmYMzLRN4tHLSoi/bvvMGfn2OpOZS9e7LT3ipj8PCHjxtmd0/j44BlbdocHIcTFzy2Tq9I8yin0WS1fD7e9XO/txdIL9hEcGue8nemFEHXnyOgbMKWk0DYxgeN332ObO+UMjR5/nNT//hevDh0Iuukmgm+52Wl9CyHqn9snV7Uy7167w/sah9dTIEIIZ1GLisjfvgNTSgpg3YbGmWJmzMC3X1+Cbr4Jja8vGinoKUSD49bJ1fQ7uteug91zbC9Pa7VlLn87/Nva9S+EcAmLwcDpl1+h0WOPomvcGFSVE/fcQ976f6tu7KCWa/9BNRhAoyF/0ya8u/fAo0kkisY6Wq4LDnbaewkhLi5unVz56Wvx5Wen2F7mKwrDYqPsLn8w+AO6RXS7sJUQoh6cX9GnFP8SlLtqFVkLF5K1cKHT3yvupx/x7trVrohn4HXXOf19hBAXL7dLroxmi+11x+jASu6shKrCLxNsh9ODAsrc0j+qf836FkLUWv7mzXh3747x5Ek8Y2PZ361klDrg2lGYUk477b0i33yT/I0b8WzRHLXQUCaxEkK4H7dLrp6bt9v22t/Lo/odZJ+C90vmYCz08+XLIPsk7a4Od6HX6i9sKYSoA7n/rOXEvSXzIWO++NzuevZvi2r9HtqQEFqsXoWi06FotQSN/r9a9ymEaDjcLrkqXZm9Rrb/YHf4YqNQu+Pb297O490fr917CCGqpWDnTjxiYzk29naKDh+2u3bivvud8h4Bo0aBAsG3jkHfvBkavfwCJYQon9slV7W2+g3by5RyJrE/2/PZuoxGCLeXvWwZyY85/xeaiOefw2IoIujGG9CFhDi9fyFEw+W2ydXPE2uwJY3JYHc44YJtbqL87Ce1CyFcw3jmDIbERKeNSgFEffQhyZMeBaxbzgghRE25bXLVu1lo1TddaM5428sfA/w45WH/zzd31NzahiWEqICloADDocN4RDUhaeAgp/fvP2wYPj16EDz2Nqf3LYRwL26VXKmqWvPGJgMcWGo7fDvU/jHBvR3vxc/Tr+b9CyEAMKakkLNqFUHXX0/699+TuWABPpddRs6yP7Dk5jrtfTzj4ig6dozA666lyTvvABD3w/dO618I4b7cKrlKzTFUfVNFppZUX88pZ5n1pG6Tat63EIL87dvxjIsjafAQAM68PtV2LevY8Rr12WTaNE49/bTtWNsojFb//IMpPR2NXo9qsaDx9q5d4EIIcQG3Sq7Oj1uF+lZzuwmL2e7w4+Agu+P1Y9bXIiohRNHx4xwb47zHcY0enYS+dWv8Bg2icM9ugm+/nZyVK/EfNAhAJqgLIVzKrZKrIpO1gOhzV7epXsN/P7E7nBXob3fs5yGPA4WoroOXD8CUmuq0/qI+/gjjsWN4tW+Pb58+tvMRzz8PQOiECU57LyGEqIxbJVeG4uRK71G2hEKllr9U8vrJAzD/Ctth25C2Uo1ZiAqYc/MwJOzDIzaW/E2b8O3Xj4IdOzCeOFGrxKrF33+Rs3wFWYt+wyM8AlNqKgHDhjkxciGEqDk3S66sj/c8tRrHGuSnw+Yv7U6pfuF2x19eZX9dCFHiyHXXYUxOdmqf0Z99ikdEBCG3jyXk9rFO7VsIIZzBzZKr8yNXDiZXiyZBQqmtMmL7MG7pONvhihtX4O/pX05DIdyT4fARTj39NLqICHJXrXJav/EL5gOgeHqib97caf0KIYQruFVydX7OlV7nYHJ1aqftpQVI73QjO/Z+aDsX4RtRTiMhGj5LYSHm9HQ8mjQBoOhkMicfeQRDQnHxzb17a/0e4c88g0/PnmTNn4++dWsUjYPft0IIUc9cllwpivIVcA1wVlXVDq56n+pYse8MUJJkVSmrZPn3F0EBfFoqsQrWBzs1NiEuJScfmUTeP//Q+LVX8WrThqM33+KUfgOuuQZFqwEPD0LvuhMA7w7tndK3EELUFVeOXH0D/A/4zoXvUS2rEs8CYHGkmOivD9odfnpB+YVfr//VaXEJcSlQzWYyfvyJM2++aTt3+uVXatRX0K23EP744xzoZd2Gqukvc/CMiUEbFFRFSyGEuPi5LLlSVXWNoihNXdV/TdzeO47Xft9Ht1gHRp12/Fjp5RAvqZMjGjZzbh7nPv+M4DvGUbBrJ8mPOK9QbuSUKQA0W7IYc0YG3h07Oq1vIYSob/U+50pRlInARIDY2FiXvtf5ESutppqlE278GraW/IY+vOlwZ4YlRL1TTSbSv/+B4LG3cXbaf/CICOfsf94D4NzMmq+Ibfzqq6hmE4pGw+kpr5a5rm/WrMZ9CyHExarekytVVacD0wF69OhRi83/qmayOJhcTR9kd7glzD7pe/qypxGiIbAYDFjy8jg0/Gos2dmcLd5jzxnCn3uW4Ftuth0H33orhiNHMBw46LT3EEP3EXoAABrKSURBVEKIi1G9J1d1yexIcqWqcGq73ak7/7jT9rpXZC/CfcIvbCXEJaNgz14Ktm1DGxRI5i9zyd+82Wl9+/bvT/gTj1Owe49dYnWePj4efXy8095PCCEuRu6ZXFVWUX37D/bHL5yBny6zHc68cqYrQhOiTqhGI0dvvNHp/bbZVVy2RKdD0WjwatfO6e8hhBCXCpcVjlEUZRbwL9BaUZSTiqLc7ar3cpRDjwV/e9j+2MPLhREJ4VoJbdqS0KYtOStWkNCmLYkdO9W6z7gfrb+A+PbtS+ttW2mbmIDi6Wn9I7WohBDCpasFx7iq75qyWFS0GqXivQCNBfbHj+5kT9oe1wcmRA1ZCgowHD6Md3trLShzdjaKhweFe/dizs623Xfy4Udq9T7akBB0ERGEP/Uk3l27EnrvvQTffjsaH59a9SuEEA2RWz0WNFnUyh8JLrjf/ji4KWO+LVki/sOICx4ZClHPTk2eTM7SZbT8dz264GAO9OyF4uODmp9fo/70bdpgSEwErLWoIp5/HiwWNN7edveFP/lErWMXQoiGyq2SK4uqVv5IcF+pwqBXTMFkMdld7tyos2sCE6KGCnZa5zrlrFiB34CBADVOrCJefJHgsbehFhaS9tnnhN57Dxq93mmxCiGEu3Cr5MpkriK5KsXY52FuXzLWxREJUX2WvDz2d+9hd+70Sy/XqC/vzp2J/forFJ0OxdMTAMXbm/AnHq91nEII4a7cKrmqdOTKkGt3OHP3TPad22c7Hho71JWhCVGhgt17SH70UfyGDCHj55/BZKq6kQPiZv2ET9euTulLCCFECbdKrkwWS8XJ1VtRJa9fOEPGtvftLr8zwHnFFYUoT87q1XhEReHVqhWqxcKpZ54l+/ffbdczfnDOnL82e/dgzs5GFyybjwshhCu4VXJltlRQhqEoz/64nPILeq3MPRGudfIB62bhzZct5dDwq53SZ5s9u1F0OkwZGRTu3YdPj+4oWq0kVkII4UJullxZyl8t+HOpuVWP7QZgVuIs26lNYze5OjThZk4++hheHdpTdPQovj17UrC7pOSHsxKrxlNeQdFZv8V1wcH49e/nlH6FEEJUzq2SK1WFMrmV2QiHV5ccewVRYLKvd+Wt80YIZylMSCDnjz/I+eMPALLmza91n40ee4z8LVuImDyZwr17CRx1Ta37FEIIUTPulVwBZcatjBcsW/cKoPd3UnJB1I6qqhy/Yxwhd92J/5Ah5K5bR96aNRiOHiXv7zW17t+rY0fif5lzwdn7ANA3k737hBCiPrlXcqVStjq7udTKqxZXAGBRLbZTX1/1dV2EJhoYtbCQ/C1byN+yhYBrR5H92yKn9Bt06y00njwZdG71rSuEEJcUt/oJrapq2ceCCQtLXl/zQZk2PRr3KHNOiAsVHT9Oyosv0eQ/09AGBtrNm6pNYhX304/4dOvmjBCFEELUEfdKrihnztXvpYolBsXUZTiiASg6dozMuXM5N2MmAEnFVdJrTFFoOvcXjt5wI40ee1QSKyGEuAS5V3KlqiilZ12pasnr/tYkq/Qqwbmj5tZVaOISkv3Hn3hEReEREc6hq4bXqi/PuDhCJ95Lygsv4t2lC3GzfkJRFNomJjgpWiGEEHXNvZIrLhi5Ksgoee0dTL4xnzc3vmk71TqkdZ3FJi5eqsnEwf6XY87MdFqfjV95Ga9OnfBu3x4Aja8fPr16lp0TKIQQ4pLjXsmVCprSH17vllpV5ReB0WKs+6DERUm1WFCN1v8PaZ995tTEyu+KoQSPGWN3LmD4VU7rXwghRP1yq+TKoqplSzGc1/FmHlo2znYYFxBXJzGJi0PO6tVoAwMxpqRgOpvK2Xecs91R/Px5qGYLloJ8MmfPIejGG/Dt08cpfQshhLg4uVVypUI5ha6AwS+ARsPO1J22U6/2fbWuwhL1TLVYbFvP1EbMjOkonnoM+xPRt2qNd8cOaHx9bdd9e/as9XsIIYS4+LlVcoVaKrcqPZl9wNMsOmS/XL57RPc6C0vUvdx16zhx9z217idozK1kzvqZJu++g9/llwPg20uSKCGEcGea+g6gLqmoJROGTYaSC4rC5LWTbYfTBkyr48iEq1gKC8nbtAlLXh55GzZgyc8nc968GiVWgTfeYHfceMoUIl95hbb/396dh0lRnXsc/77TPRvbMAsMCoMsCoLigmg04ZrcaBQ1jxqjF5REBMUoiWtETUyi5sZEo96gIc9V3I0aISZRr0vUJK6JKGAQkB2RZVB2cECYrd/7RxVDMwOCTPV0M/37PE8/U3XqVJ3TfZ4z887pqnPmzqHo9NOjqrKIiOzjsmrkypNHrv7+813mG9KzeY/XS/p5TQ2J6moWfv0EElVVxIqKqN+4ca+vFysqYv9f/IKS4cNZ/K2zAGhzlOagEhGRprIuuGp4WnDy7xrS9ZRg67B17lzinTqx4Y9PsXrcjrPt721gVTJyJOseeojS0cFIV0G/fhz0z7eweJxYUVGz6ywiIq1PVgVXiZ0tfzPoQuoT9Q27z5z5DJLZPJHAcoJvtBObNwOwZcYMlo4cFW1BsRilF4+mtrKSjkOHNiTHS0ujLUdERFqVrAquGm5hX7d4e+LJt1Bdv/3+q15FvVq0TrJnvL4eEgmq/vEqlVdckbJyer/0V3K7dWPuIYfS9itfJl5cTLe770pZeSIi0vpkV3DlBDe0j09ajDm3kMGP6OmuTLdoyCnULlsW6TUr7ruPZaNHk9+nD13vGofl5JB3QDC/Wa8XXyC3S5dIyxMRkeyQVcEVhJOIJup2erSssKxFayM78poaPvnFLZR9fwy55eVsevMtapYsoaB/v2YHVp2vu45Yhw50OGUIOW3aNKTvd8sttB08mNzyzjvkz+/Zs/ElRERE9khWBVfByFVSguVQW7/9ZvaXv/1yy1dKAKhduZLNb/2TDZMmUb9+PfUbN/LZu+9Gcu0ef5xE4YABOz3W8dtnRVKGiIjINtkVXNFobcExk1m2afuISG4st+UrlYVqli6l8qqrKb/hBtoMPJLN//oXS0dd2HC86pVX9vra7b/xDbbOn0fv55+nvqqKeHFxFFUWERHZY1kVXO3wtGB+EXTqS/XaOWmtU7ZxdxadFCxSvOS88yj+7ndZ//vfN+uasY4dqd+wgbzeven227sb0hVYiYhIOmRVcOUOcQ/vtzpqBABXvXYVAD899qfpqlar5e5Uz5tH/oEHUrduHQuP/2qTPM0JrCruvYdYaRkFh/Snev58crt2a051RUREIpFdwRXQjmBeJIoqAKjcVAlAYbwwTbVqPerWrOGTm2+m/UknUbdyJVWvvsaWadMiL6ff3DnUfvwxufvt15BW0Ldv5OWIiIjsjewKrtwpTmwIdgqLSXii4VjfEv1xbg53Z8HgYOHiqlf+Fvn12514Alv+PZ36tWsBdgisREREMklWBVcAd68fE2ysX8xjsx9rSO9T3CdNNdp31SxbRvXChay4ZmzDTOnN1fm66yg4pD9tjwnmHlt68cXULP6IivHjqVu7lpolSyMpR0REJFWyKrhyT9ppV8789fPTVpdMV7d+PWsn3Efnq6/CcnNZP3ESn9x4IwDdH3kEgKUjRkRWXn7fvnR/+KEmN6F3nzChYTteWqqlZ0REJONlVXCVSI6uBp7Py098KX2VyRCeSLB2wn0Unztsh4WIK6++ms/enkxB//7kVXRrCKyg+UHV/nfcQfXCBXQ45RTyKip2mNRTRERkX5dVwVVO8szsZmyp25K+yqTR5smTyevZi9zyzmx64w1WjxvH6nHjKP/xjyAnRs3SJXz29mQAVowd2+zyCg49lIr7JhDr0IG6tWvJ7dx59yeJiIjso7IquMrz7cFU8szsRflFO8veKm2ZMYOlF4wk1qmMPm++iVfXNBxb+ctfRVZO6UUX0vmaa5qkK7ASEZHWLquCq+Sbrp5d9GzD9i8H/zIdtWkx9Zs2U7t8GQUHH8xH/zU0SFu9hjkH94usjFinMupXr2nYLxszJrJri4iI7EuyKrjqW/NBsFF+KFU1VQ3px3c7Pk01Sr3kG9EtLy/Sa3e+5oesuuNOevzpKQoPOQSALe+/z5ZZs3QflYiIZK2sCq4OrJnLylgMXzOHvFgQaLSWwKp6wQK8vp7qRYtYMfZaut19F8t/cNkOebymZhdnfzHlP/kJFo9RPGwYpRddtMOxwsMPp/DwwyMpR0REZF+UVcHV4dXTOLF7VwDa/fu3AFx/9PXprNJeq121iuq5c4l36cLi089ocrxxYBWF3O7dOfDllyK/roiISGuSVcHVa4UnAC8AsKl2EwAF8YI01ujz1a5cSby4mLoNG9jy/vsUDhhATkEB8489LqXl5rRvzwGPPsLib51F+Y9/TPG5w1hywUg6X3lFSssVERFpDbIquMpNNP1arKywLA012bnPpk2j8MgjqVu5ks+mTGHFtdcR79SJutWroy0oHoe67dNSlIwcSfl11zbJ1m/unIbtHo8/1uS4iIiINJVVwVWhN53XysxarPya5cupXbGCNgMH8umLL1LQvz+xkhLixcVsfvttlo4c1eScyAMr4OD3p7N63DiKzjqLVbfeRumokZGXISIikq2yKrhqm9jYouUtG/N9vL6O0pGjWHrBBQ3pnceOZdXtt7doXXK7dqW2spIuN9+MxWJ0/uEPAai4954WrYeIiEhrl1XB1fSCVZFfM1FTA+7k5OfjtbUkqmv49Ln/45Obbm7Is/n1N3Y4J5WB1YGvv05ueWc2Pvc8W96bxvon/kCX//45xeecQ/WiReT37p2yskVERCTLgqv18WoACrc6WwqMpwaOx2tqvtD8T/VVVdSuWEFB3764O/MOC6YdKBkxgnXhgsYt5eBZM/HqauYdNQiA0ku+R255MAN60TdPo+ibp9HlZz9ryK/ASkREJPWyKrgyEvRfkuCmJxLcdXoOiV9dwidDh7LfzTc15EnU1FD18it0OO1Uts6cyZp7J1Dz4YeUjRlDrKSYNb8dz5bp0wFoM2hQw3mpDqwOeOJxqufPZ+sHs/nsvffoMfFJLB7H4nEqHrif2spKOp59dkrrICIiIrtnnrQkTOQXNxsC3AXEgPvd/dbPyz9o0CCfOnVqyupzzEOHcePDtfRaCVUlBbRftxWA9iefTNVLL7H/7b9m41/+wuZ/vU1+/35Uz56zmyumTsehQ9kwcSIAB8+cgeXmpq0uIiIi0pSZTXP3QU3SUxVcmVkMmA98A1gOTAHOdffZuzon1cHVd248hBsmJlJ2/ebY/847yOvalc2TJ1NywQXkFGTu/FsiIiKy6+AqJ4VlHgMsdPcP3b0GeBJoOpV4C8qEwKrg0EMbttsOHkz+QQcBEC8pofCIIyi75BIFViIiIvuwVN5z1RVYlrS/HPhSCsvLHEmTdHZ/8AHaHHccNYsXk9+rF55IsPWD2cTatyNeXs6q3/yG6gULiJWUprnSIiIiEoW039BuZhcDFwN07949zbX5Ynr/9UVy2rZlwX8Eiz+3/erxdL/3XgC8ro66NWvI7dIFgPxevQCwnBwKB2wfvSq/5ho6DBlCQd8+LVx7ERERSYVUfi1YCVQk7XcL03bg7hPcfZC7D+rUqVMKq/P5Ol15ZZO09kOG0G18sMBzuxNPoM/UKXQcNhSA0tEXkdejB9vuWYuVlTUEVgAWjzcEVp/H8vJoM3BgFG9BREREMkAqR66mAAeZWU+CoGoYcF4Ky9ut9Xdcxt+eHs+0A3O49eF6YqWldH/gfuLl5cSLiyn+znDmDzoa2P6EntfVUTJqFKWjLyLWrh1ll15K9dx5lIwYAUC8rIwOp55Cyfnnp/OtiYiISIZI9VQMpwLjCKZieNDdb/m8/Kl+WhBgwCMDAHhg8HiO7v7lJlMceH09iS1bibVrm9J6iIiIyL5tV08LpvSeK3d/AXghlWXsrQO7Dtjp3FEWiymwEhERkb2WynuuMlpJQUm6qyAiIiKtUNqfFmxplx95ORUdKnafUURERGQvZF1wNfqw0emugoiIiLRiWfu1oIiIiEgqKLgSERERiZCCKxEREZEIKbgSERERiZCCKxEREZEIKbgSERERiZCCKxEREZEIKbgSERERiZCCKxEREZEIKbgSERERiZCCKxEREZEIKbgSERERiZCCKxEREZEIKbgSERERiZCCKxEREZEIKbgSERERiZCCKxEREZEIKbgSERERiZC5e7rr0MDMVgNLUlxMGbAmxWXIF6M2yUxql8yjNslMapfM01JtcoC7d2qcmFHBVUsws6nuPijd9ZDt1CaZSe2SedQmmUntknnS3Sb6WlBEREQkQgquRERERCKUjcHVhHRXQJpQm2QmtUvmUZtkJrVL5klrm2TdPVciIiIiqZSNI1ciIiIiKZM1wZWZDTGzeWa20MyuT3d9WjszqzCzV81stpl9YGZXhOklZvaKmS0IfxaH6WZmd4ftM8PMBiZda0SYf4GZjUjXe2otzCxmZv82s+fC/Z5m9k742U80s7wwPT/cXxge75F0jR+F6fPM7OT0vJPWw8w6mtlTZjbXzOaY2XHqK+llZleFv7tmmdkfzKxAfaXlmdmDZrbKzGYlpUXWN8zsKDObGZ5zt5lZJBV391b/AmLAIqAXkAe8D/RPd71a8wvYDxgYbrcH5gP9gV8D14fp1wO3hdunAi8CBhwLvBOmlwAfhj+Lw+3idL+/ffkFXA08ATwX7k8ChoXb9wCXhttjgHvC7WHAxHC7f9iH8oGeYd+Kpft97csv4BHgonA7D+iovpLW9ugKLAYKw/1JwAXqK2lpi+OBgcCspLTI+gbwbpjXwnNPiaLe2TJydQyw0N0/dPca4EngjDTXqVVz94/d/b1wuwqYQ/AL6wyCPySEP88Mt88AHvXAZKCjme0HnAy84u7r3H098AowpAXfSqtiZt2A04D7w30Dvg48FWZp3Cbb2uop4IQw/xnAk+5e7e6LgYUEfUz2gpkVEfwBeQDA3WvcfQPqK+kWBwrNLA60AT5GfaXFufsbwLpGyZH0jfBYB3ef7EGk9WjStZolW4KrrsCypP3lYZq0gHCI/EjgHaDc3T8OD30ClIfbu2ojtV20xgHXAolwvxTY4O514X7y59vw2YfHN4b51SbR6gmsBh4Kv66938zaor6SNu5eCdwBLCUIqjYC01BfyRRR9Y2u4Xbj9GbLluBK0sTM2gF/Aq5090+Tj4X/Kehx1RZiZt8EVrn7tHTXRXYQJ/ja43/d/UhgM8FXHQ3UV1pWeA/PGQSB7/5AWzQKmJEytW9kS3BVCVQk7XcL0ySFzCyXILB63N3/HCavDIdiCX+uCtN31UZqu+h8BTjdzD4i+Gr868BdBEPn8TBP8ufb8NmHx4uAtahNorYcWO7u74T7TxEEW+or6XMisNjdV7t7LfBngv6jvpIZouobleF24/Rmy5bgagpwUPikRx7BDYfPprlOrVp4v8EDwBx3/5+kQ88C257UGAE8k5R+fvi0x7HAxnDY9yXgJDMrDv+bPClMky/I3X/k7t3cvQdBH/iHuw8HXgXODrM1bpNtbXV2mN/D9GHhE1I9gYMIbgqVveDunwDLzKxvmHQCMBv1lXRaChxrZm3C32Xb2kR9JTNE0jfCY5+a2bFhO5+fdK3mSfeTAC31IniKYD7B0xo3pLs+rf0FDCYYqp0BTA9fpxLch/B3YAHwN6AkzG/A78L2mQkMSrrWKIIbQRcCI9P93lrDC/ga258W7EXwC38h8EcgP0wvCPcXhsd7JZ1/Q9hW84jo6ZpsfgFHAFPD/vI0wRNN6ivpbZObgbnALOD3BE/8qa+0fDv8geC+t1qCUd4Lo+wbwKCwjRcB4wknV2/uSzO0i4iIiEQoW74WFBEREWkRCq5EREREIqTgSkRERCRCCq5EREREIqTgSkRERCRCCq5EpFnMzM3szqT9a8zspoiu/bCZnb37nM0u5xwzm2Nmr+5h/h+nuk4isu9ScCUizVUNnGVmZemuSLKkmbT3xIXAaHf/zz3Mr+BKRHZJwZWINFcdMAG4qvGBxiNPZrYp/Pk1M3vdzJ4xsw/N7FYzG25m75rZTDPrnXSZE81sqpnND9dHxMxiZna7mU0xsxlm9r2k675pZs8SzKjduD7nhtefZWa3hWk/I5j09gEzu71R/v3M7A0zmx6e8x9mditQGKY9Hub7Tlj36WZ2r5nFtr1fM/uNmX1gZn83s05h+uVmNjus+5N7/cmLSEZScCUiUfgdMNzMir7AOYcDlwD9gO8Cfdz9GOB+4LKkfD2AY4DTgHvMrIBgpGmjux8NHA2MDpcXgWBdvivcvU9yYWa2P3AbwZqKRwBHm9mZ7v5zgtnRh7v72EZ1PI9gmYwjwvpOd/frgS3ufoS7DzezfsBQ4CthvnpgeHh+W2Cqux8CvA7cGKZfDxzp7oeFn4GItCJfZNhcRGSn3P1TM3sUuBzYsoenTfFgbS/MbBHwcpg+E0j+em6SuyeABWb2IXAwwdpghyWNihURrNtWA7zr7ot3Ut7RwGvuvjos83HgeILlZnZZR+DBcBHyp919+k7ynAAcBUwJliejkO0LySaAieH2YwQLAEOwzM3jZvb0bsoXkX2QRq5EJCrjCEaU2ial1RH+njGzHCAv6Vh10nYiaT/Bjv/4NV6jywnWELssHD06wt17uvu24Gxzs95FckHubxAEYJXAw2Z2/k6yGfBIUl36uvtNu7pk+PM0gtG+gQRBmf7RFWlFFFyJSCTcfR0wiSDA2uYjglEdgNOB3L249DlmlhPeh9WLYAHcl4BLwxElzKyPmbX9vIsQLKj7VTMrC++JOpfgq7pdMrMDgJXufh/B15UDw0O128omWED2bDPrHJ5TEp4Hwe/YbaNr5wFvhUFmhbu/ClxHMOrWbvcfg4jsK/TfkohE6U7gB0n79wHPmNn7wF/Zu1GlpQSBUQfgEnffamb3E9yL9Z4F38WtBs78vIu4+8dmdj3wKsFo0/Pu/sxuyv4aMNbMaoFNwLaRqwnADDN7L7zv6ifAy2HgVAt8H1hC8H6PCY+vIrg3KwY8Ft6fZsDd7r5hzz8OEcl05t54xF1ERKJgZpvcXaNSIllGXwuKiIiIREgjVyIiIiIR0siViIiISIQUXImIiIhESMGViIiISIQUXImIiIhESMGViIiISIQUXImIiIhE6P8B7QI2KpwLWe4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,8))\n",
    "plt.plot(list(range(10000)), np.mean(R_out_list, axis=0), label='e=0.1, stat')\n",
    "plt.plot(list(range(10000)), np.mean(R_out_list_1, axis=0), label='e=0.1, al=0.1')\n",
    "plt.plot(list(range(10000)), np.mean(R_out_list_2, axis=0), label='e=0.1, al=0.1, UCB')\n",
    "plt.plot(list(range(10000)), np.mean(R_out_list_3, axis=0), label='e=0.1, al={}, Gradient Bandit'.format(alpha))\n",
    "plt.xlabel(\"Number of steps\")\n",
    "plt.ylabel(\"Average reward\")\n",
    "plt.legend(loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### UCB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 30 timesteps\n",
      "Episode finished after 59 timesteps\n",
      "Episode finished after 39 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 47 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 40 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 40 timesteps\n",
      "Episode finished after 45 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 37 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 73 timesteps\n",
      "Episode finished after 35 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 42 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 64 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 48 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 47 timesteps\n",
      "Episode finished after 36 timesteps\n",
      "Episode finished after 55 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 39 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 39 timesteps\n",
      "Episode finished after 46 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "Episode finished after 57 timesteps\n",
      "Episode finished after 39 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 39 timesteps\n",
      "Episode finished after 37 timesteps\n",
      "Episode finished after 51 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 42 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 70 timesteps\n",
      "Episode finished after 53 timesteps\n",
      "Episode finished after 38 timesteps\n",
      "Episode finished after 32 timesteps\n",
      "Episode finished after 44 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 48 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 37 timesteps\n",
      "Episode finished after 44 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 38 timesteps\n",
      "Episode finished after 67 timesteps\n",
      "Episode finished after 36 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 38 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 36 timesteps\n",
      "Episode finished after 46 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "33.07\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "ans=[]\n",
    "epoch=100\n",
    "\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset()\n",
    "    Q=np.array([0.,0.])\n",
    "    N=np.array([0.,0.])\n",
    "    for t in range(100):\n",
    "        env.render()\n",
    "        e=uniform(0,1)\n",
    "        if e>0.5:\n",
    "            action  = np.argmax(Q+100*np.sqrt(np.divide(np.log(t+1),N+1e-5)), axis=0) \n",
    "        else:\n",
    "            action = randrange(2)\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        N[action]+=1\n",
    "        Q[action]=Q[action]+0.1*(reward-Q[action])\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            ans.append(t+1)\n",
    "            break\n",
    "env.close()\n",
    "print(sum(ans)/epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 68 timesteps\n",
      "Episode finished after 108 timesteps\n",
      "Episode finished after 80 timesteps\n",
      "Episode finished after 102 timesteps\n",
      "Episode finished after 64 timesteps\n",
      "Episode finished after 62 timesteps\n",
      "Episode finished after 103 timesteps\n",
      "Episode finished after 65 timesteps\n",
      "Episode finished after 89 timesteps\n",
      "Episode finished after 78 timesteps\n",
      "Episode finished after 107 timesteps\n",
      "Episode finished after 126 timesteps\n",
      "Episode finished after 86 timesteps\n",
      "Episode finished after 81 timesteps\n",
      "Episode finished after 65 timesteps\n",
      "Episode finished after 61 timesteps\n",
      "Episode finished after 71 timesteps\n",
      "Episode finished after 80 timesteps\n",
      "Episode finished after 57 timesteps\n",
      "Episode finished after 59 timesteps\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('LunarLander-v2')\n",
    "\n",
    "epoch=20\n",
    "\n",
    "Q=np.array([0.0,0.0,0.0,0.0])\n",
    "N=np.array([0.0,0.0,0.0,0.0])\n",
    "\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset() \n",
    "    for t in range(500):\n",
    "        env.render()\n",
    "        e=uniform(0,1)\n",
    "        if e>0.5:\n",
    "            action  = np.argmax(Q+100*np.sqrt(np.divide(np.log(t+1), N+1e-5)), axis=0) \n",
    "        else:\n",
    "            action = randrange(4)\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        N[action]+=1\n",
    "        Q[action]+=0.01*(reward-Q[action])\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            break\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### My experiments - kind of optimistic initial values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 11 timesteps\n",
      "Episode finished after 36 timesteps\n",
      "Episode finished after 64 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 46 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 35 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 77 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 55 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 54 timesteps\n",
      "Episode finished after 69 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 38 timesteps\n",
      "Episode finished after 35 timesteps\n",
      "Episode finished after 35 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 35 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 12 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 15 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 49 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 61 timesteps\n",
      "Episode finished after 35 timesteps\n",
      "Episode finished after 11 timesteps\n",
      "Episode finished after 16 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 72 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 38 timesteps\n",
      "Episode finished after 10 timesteps\n",
      "Episode finished after 37 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 14 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "23.5\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "ans=[]\n",
    "epoch=100\n",
    "\n",
    "Q=np.array([0.,0.])\n",
    "\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset()\n",
    "    b = np.zeros_like(Q)\n",
    "    b[Q.argmax(0)] = abs(np.max(Q)) # zero out all the elements except for the absolute value of the maximum element\n",
    "    Q = b\n",
    "    for t in range(100):\n",
    "        env.render()\n",
    "        e=uniform(0,1)\n",
    "        if e>0.5:    \n",
    "            action = np.argmax(Q, axis=0)\n",
    "        else:\n",
    "            action = randrange(2)\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        Q[action]=Q[action]+0.1*(reward-Q[action])\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            ans.append(t+1)\n",
    "            break\n",
    "env.close()\n",
    "print(sum(ans)/epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.      , 0.468559])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 141 timesteps\n",
      "Episode finished after 101 timesteps\n",
      "Episode finished after 120 timesteps\n",
      "Episode finished after 108 timesteps\n",
      "Episode finished after 97 timesteps\n",
      "Episode finished after 102 timesteps\n",
      "Episode finished after 86 timesteps\n",
      "Episode finished after 88 timesteps\n",
      "Episode finished after 123 timesteps\n",
      "Episode finished after 73 timesteps\n",
      "Episode finished after 105 timesteps\n",
      "Episode finished after 75 timesteps\n",
      "Episode finished after 62 timesteps\n",
      "Episode finished after 107 timesteps\n",
      "Episode finished after 96 timesteps\n",
      "Episode finished after 65 timesteps\n",
      "Episode finished after 103 timesteps\n",
      "Episode finished after 107 timesteps\n",
      "Episode finished after 79 timesteps\n",
      "Episode finished after 88 timesteps\n",
      "Episode finished after 90 timesteps\n",
      "Episode finished after 135 timesteps\n",
      "Episode finished after 73 timesteps\n",
      "Episode finished after 68 timesteps\n",
      "Episode finished after 183 timesteps\n",
      "Episode finished after 89 timesteps\n",
      "Episode finished after 68 timesteps\n",
      "Episode finished after 89 timesteps\n",
      "Episode finished after 106 timesteps\n",
      "Episode finished after 107 timesteps\n",
      "Episode finished after 57 timesteps\n",
      "Episode finished after 106 timesteps\n",
      "Episode finished after 165 timesteps\n",
      "Episode finished after 75 timesteps\n",
      "Episode finished after 112 timesteps\n",
      "Episode finished after 266 timesteps\n",
      "Episode finished after 124 timesteps\n",
      "Episode finished after 165 timesteps\n",
      "Episode finished after 121 timesteps\n",
      "Episode finished after 82 timesteps\n",
      "Episode finished after 122 timesteps\n",
      "Episode finished after 82 timesteps\n",
      "Episode finished after 77 timesteps\n",
      "Episode finished after 96 timesteps\n",
      "Episode finished after 131 timesteps\n",
      "Episode finished after 74 timesteps\n",
      "Episode finished after 115 timesteps\n",
      "Episode finished after 98 timesteps\n",
      "Episode finished after 122 timesteps\n",
      "Episode finished after 75 timesteps\n",
      "Episode finished after 118 timesteps\n",
      "Episode finished after 134 timesteps\n",
      "Episode finished after 95 timesteps\n",
      "Episode finished after 102 timesteps\n",
      "Episode finished after 57 timesteps\n",
      "Episode finished after 266 timesteps\n",
      "Episode finished after 75 timesteps\n",
      "Episode finished after 115 timesteps\n",
      "Episode finished after 82 timesteps\n",
      "Episode finished after 80 timesteps\n",
      "Episode finished after 124 timesteps\n",
      "Episode finished after 131 timesteps\n",
      "Episode finished after 91 timesteps\n",
      "Episode finished after 72 timesteps\n",
      "Episode finished after 114 timesteps\n",
      "Episode finished after 97 timesteps\n",
      "Episode finished after 111 timesteps\n",
      "Episode finished after 112 timesteps\n",
      "Episode finished after 105 timesteps\n",
      "Episode finished after 233 timesteps\n",
      "Episode finished after 82 timesteps\n",
      "Episode finished after 71 timesteps\n",
      "Episode finished after 102 timesteps\n",
      "Episode finished after 94 timesteps\n",
      "Episode finished after 82 timesteps\n",
      "Episode finished after 111 timesteps\n",
      "Episode finished after 73 timesteps\n",
      "Episode finished after 113 timesteps\n",
      "Episode finished after 153 timesteps\n",
      "Episode finished after 136 timesteps\n",
      "Episode finished after 140 timesteps\n",
      "Episode finished after 183 timesteps\n",
      "Episode finished after 128 timesteps\n",
      "Episode finished after 112 timesteps\n",
      "Episode finished after 97 timesteps\n",
      "Episode finished after 96 timesteps\n",
      "Episode finished after 100 timesteps\n",
      "Episode finished after 192 timesteps\n",
      "Episode finished after 97 timesteps\n",
      "Episode finished after 99 timesteps\n",
      "Episode finished after 77 timesteps\n",
      "Episode finished after 92 timesteps\n",
      "Episode finished after 60 timesteps\n",
      "Episode finished after 102 timesteps\n",
      "Episode finished after 117 timesteps\n",
      "Episode finished after 173 timesteps\n",
      "Episode finished after 119 timesteps\n",
      "Episode finished after 100 timesteps\n",
      "Episode finished after 103 timesteps\n",
      "Episode finished after 87 timesteps\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('LunarLander-v2')\n",
    "\n",
    "epoch=100\n",
    "\n",
    "Q=np.array([0.0,0.0,0.0,0.0])\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset()\n",
    "    b = np.zeros_like(Q)\n",
    "    b[np.argmax(Q, axis=0)] = abs(np.max(Q))\n",
    "    Q = b   \n",
    "    for t in range(500):\n",
    "        env.render()\n",
    "        e=uniform(0,1)\n",
    "        if e>0.5:\n",
    "            action = np.argmax(Q, axis=0)\n",
    "        else:\n",
    "            action = randrange(4)\n",
    "        observation, reward, done, info = env.step(action) \n",
    "        Q[action]+=0.01*(reward-Q[action])\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            break\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.13008902, -0.9656906 , -0.03148007, -0.12585378])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Bandit Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 43 timesteps\n",
      "Episode finished after 49 timesteps\n",
      "Episode finished after 54 timesteps\n",
      "Episode finished after 63 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 32 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 36 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "Episode finished after 20 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 55 timesteps\n",
      "Episode finished after 42 timesteps\n",
      "Episode finished after 28 timesteps\n",
      "Episode finished after 37 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 32 timesteps\n",
      "Episode finished after 38 timesteps\n",
      "Episode finished after 18 timesteps\n",
      "Episode finished after 47 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 37 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 50 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 35 timesteps\n",
      "Episode finished after 47 timesteps\n",
      "Episode finished after 40 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 39 timesteps\n",
      "Episode finished after 43 timesteps\n",
      "Episode finished after 40 timesteps\n",
      "Episode finished after 42 timesteps\n",
      "Episode finished after 37 timesteps\n",
      "Episode finished after 64 timesteps\n",
      "Episode finished after 48 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 39 timesteps\n",
      "Episode finished after 32 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 40 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 43 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 32 timesteps\n",
      "Episode finished after 45 timesteps\n",
      "Episode finished after 57 timesteps\n",
      "Episode finished after 38 timesteps\n",
      "Episode finished after 52 timesteps\n",
      "Episode finished after 35 timesteps\n",
      "Episode finished after 47 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 56 timesteps\n",
      "Episode finished after 46 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 27 timesteps\n",
      "Episode finished after 36 timesteps\n",
      "Episode finished after 41 timesteps\n",
      "Episode finished after 32 timesteps\n",
      "Episode finished after 19 timesteps\n",
      "Episode finished after 25 timesteps\n",
      "Episode finished after 21 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 17 timesteps\n",
      "Episode finished after 13 timesteps\n",
      "Episode finished after 31 timesteps\n",
      "Episode finished after 30 timesteps\n",
      "Episode finished after 33 timesteps\n",
      "Episode finished after 55 timesteps\n",
      "Episode finished after 55 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 22 timesteps\n",
      "Episode finished after 24 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 46 timesteps\n",
      "Episode finished after 39 timesteps\n",
      "Episode finished after 51 timesteps\n",
      "Episode finished after 36 timesteps\n",
      "Episode finished after 29 timesteps\n",
      "Episode finished after 23 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "Episode finished after 34 timesteps\n",
      "Episode finished after 26 timesteps\n",
      "34.54\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "ans=[]\n",
    "epoch=100\n",
    "\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset()\n",
    "    Q=[0.,0.]\n",
    "    H=[0.,0.]   \n",
    "    for t in range(100):\n",
    "        env.render()\n",
    "        e=uniform(0,1)\n",
    "        if e>0.5:\n",
    "            action = np.argmax(H, axis=0)\n",
    "        else:\n",
    "            action = randrange(2)\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        Q[action]=Q[action]+0.1*(reward-Q[action])\n",
    "        \n",
    "        exp=np.exp(H)   \n",
    "        prob=np.divide(exp, np.sum(exp))\n",
    "        \n",
    "        for i in range(len(H)):\n",
    "            if i==action:\n",
    "                H[action]=H[action]+0.1*(reward-Q[action])*(1-prob[action])\n",
    "            else:\n",
    "                H[i]=H[i]+0.1*(reward-Q[i])*prob[i]\n",
    "                \n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            ans.append(t+1)\n",
    "            break\n",
    "env.close()\n",
    "print(sum(ans)/epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode finished after 122 timesteps\n",
      "Episode finished after 145 timesteps\n",
      "Episode finished after 144 timesteps\n",
      "Episode finished after 124 timesteps\n",
      "Episode finished after 128 timesteps\n",
      "Episode finished after 100 timesteps\n",
      "Episode finished after 110 timesteps\n",
      "Episode finished after 119 timesteps\n",
      "Episode finished after 86 timesteps\n",
      "Episode finished after 114 timesteps\n",
      "Episode finished after 101 timesteps\n",
      "Episode finished after 105 timesteps\n",
      "Episode finished after 124 timesteps\n",
      "Episode finished after 106 timesteps\n",
      "Episode finished after 97 timesteps\n",
      "Episode finished after 102 timesteps\n",
      "Episode finished after 117 timesteps\n",
      "Episode finished after 92 timesteps\n",
      "Episode finished after 120 timesteps\n",
      "Episode finished after 108 timesteps\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('LunarLander-v2')\n",
    "\n",
    "epoch=20\n",
    "\n",
    "\n",
    "for i_episode in range(epoch):\n",
    "    observation = env.reset()\n",
    "    Q=np.array([0.0,0.0,0.0,0.0])\n",
    "    H=np.array([0.,0.,0.,0.])\n",
    "    for t in range(500):\n",
    "        env.render()\n",
    "        e=uniform(0,1)\n",
    "        if e>0.5:\n",
    "            action = np.argmax(H, axis=0)\n",
    "        else:\n",
    "            action = randrange(4)\n",
    "        observation, reward, done, info = env.step(action) \n",
    "        Q[action]+=0.01*(reward-Q[action])\n",
    "               \n",
    "        exp=np.exp(H)   \n",
    "        prob=np.divide(exp,np.sum(exp))\n",
    "        \n",
    "        for i in range(len(H)):\n",
    "            if i==action:\n",
    "                H[action]=H[action]+0.1*(reward-Q[action])*(1-prob[action])\n",
    "            else:\n",
    "                H[i]=H[i]+0.1*(reward-Q[i])*prob[i]\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            break\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.32697644, -0.0405348 , -0.40871624, -0.35073234])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}