reinforcement_q_learning_simple.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solve PONG\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import namedtuple\n",
    "from itertools import count\n",
    "from PIL import Image\n",
    "import time\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as T\n",
    "\n",
    "\n",
    "env = gym.make('Pong-ram-v4')\n",
    "\n",
    "# set up matplotlib\n",
    "is_ipython = 'inline' in matplotlib.get_backend()\n",
    "if is_ipython:\n",
    "    from IPython import display\n",
    "\n",
    "plt.ion()\n",
    "\n",
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Transition = namedtuple('Transition',\n",
    "                        ('state', 'action', 'next_state', 'reward'))\n",
    "\n",
    "\n",
    "class ReplayMemory(object):\n",
    "\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.memory = []\n",
    "        self.position = 0\n",
    "\n",
    "    def push(self, *args):\n",
    "        \"\"\"Saves a transition.\"\"\"\n",
    "        if len(self.memory) < self.capacity:\n",
    "            self.memory.append(None)\n",
    "        self.memory[self.position] = Transition(*args)\n",
    "        self.position = (self.position + 1) % self.capacity\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        return random.sample(self.memory, batch_size)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0646, -0.0343]], grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class DQN(nn.Module):\n",
    "\n",
    "    def __init__(self, h, w, outputs):\n",
    "        super(DQN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 8, kernel_size=3, stride=1)\n",
    "        #self.bn1 = nn.BatchNorm2d(8)\n",
    "        self.conv2 = nn.Conv2d(8, 16, kernel_size=3, stride=1)\n",
    "        #self.bn2 = nn.BatchNorm2d(16)\n",
    "        #self.conv3 = nn.Conv2d(32, 32, kernel_size=3, stride=1)\n",
    "        #self.bn3 = nn.BatchNorm2d(32)\n",
    "        self.conv4 = nn.Conv2d(16, 4, kernel_size=3, stride=1)\n",
    "        #self.bn4 = nn.BatchNorm2d(4)\n",
    "        \n",
    "        self.pool = nn.MaxPool2d(2)\n",
    "\n",
    "        self.head = nn.Linear(204, outputs)\n",
    "\n",
    "    # Called with either one element to determine next action, or a batch\n",
    "    # during optimization. Returns tensor([[left0exp,right0exp]...]).\n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = F.relu(self.pool(self.conv1(x)))\n",
    "        x = F.relu(self.pool(self.conv2(x)))\n",
    "        x = F.relu(self.pool(self.conv4(x)))\n",
    "        return self.head(x.view(x.size(0), -1))\n",
    "    \n",
    "    def update_params(self, new_params, tau):\n",
    "        params = self.state_dict()\n",
    "        for k in params.keys():\n",
    "            params[k] = (1-tau) * params[k] + tau * new_params[k]\n",
    "        self.load_state_dict(params)\n",
    "        \n",
    "class Flatten(torch.nn.Module):\n",
    "    def forward(self, x):\n",
    "        batch_size = x.shape[0]\n",
    "        return x.view(batch_size, -1)\n",
    "\n",
    "\n",
    "class DQN_basic(nn.Module):\n",
    "    def __init__(self, h, w, outputs):\n",
    "        super(DQN_basic, self).__init__()\n",
    "        self.l1 = nn.Linear(128, 128)\n",
    "        self.l2 = nn.Linear(128, 128)\n",
    "        self.l3 = nn.Linear(128, 128)\n",
    "        self.head = nn.Linear(128, outputs)\n",
    "\n",
    "    # Called with either one element to determine next action, or a batch\n",
    "    # during optimization. Returns tensor([[left0exp,right0exp]...]).\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.l1(x))\n",
    "        x = F.relu(self.l2(x))\n",
    "        x = F.relu(self.l3(x))\n",
    "        return self.head(x)\n",
    "    \n",
    "    def update_params(self, new_params, tau):\n",
    "        params = self.state_dict()\n",
    "        for k in params.keys():\n",
    "            params[k] = (1-tau) * params[k] + tau * new_params[k]\n",
    "        self.load_state_dict(params)\n",
    "\n",
    "test = DQN(1, 1, 2)\n",
    "test(torch.zeros([1, 3, 40, 150]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAMkAAAEICAYAAAD1BdCgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAS70lEQVR4nO3df7BcZX3H8ffn3twkVxLIDyENSTRUowIdie0tpKNtMYCmVEtmilba2ujEprZ0hJEqFDsVW2cKo4J/2LFGsUbxB4hikFo1xlDrVIEAiYIREyJCTCACyYTIJcm999s/znNxd+/uffb+2N1z8fOauXP3/Nhzvnt2P/uc55w9u4oIzKyxrk4XYFZ2DolZhkNiluGQmGU4JGYZDolZhkPSBElvkfTdTtdRJpKWSgpJ0zpdS6t1PCSSHpLUL+lwxd9HOl1Xp0k6W9KeFi7/Kkk3tGr5zyVleRd4fUR8q9NFTDWSpkXEQKfraIVSPbaI6Ogf8BBwboNpHwVurhi+BtgMCJgL3Ab8AjiQbi+umPd24P3A/wGHga8C84HPAoeAu4ClFfMH8A5gN/A48AGgK017C/DdinlfBmwCngQeAN44yuM7Abge2Af8PNXUnXl8xwH9wFCq/TBwMnAVcDNwQ3oMbwPOBL4HHEzr+AgwvWKZp1fU+hhwJbAKOAocS8ve3kSt3cAH07bZDVycttm0Bo/78rSMp9I2OqdiOVcCD6ZpdwNLKp6Di4GdwE9z2xqYkWp6OD22/wB607SzgT3AZcD+9JjeOq7XaMlD8jzgJ+lF+vvpCVqcps0H/jTNMxv4IvCVmpDsAl6UnvwfpWWdS9GCfhr4z5qQbAHmAS9I876tNiTpBfwI8Na0nN9OdZ3e4DF8BfhYut9JwJ3A3zTx+M4G9tQs6yqKF/Zqil3lXuB3gBWplqXADuDSNP/s9OK4DJiZhs+qWNYNY6j17cCPgSVpG22hQUiAl6ZtdHIaXgq8KN1+F/DDNI+AM4D5Fc/BprT83ty2Bj4M3Jrmn03xRvhvFdtvAPgXoAc4H3gamDtVQ3KY4p1w+O+vK6afSfEu8jPgolGWsxw4UBOS91QMfwj474rh1wPbakKyqmL474DNdULyZ8D/1qz7Y8B769S0ADhCendL4y4CtuQeH41D8p3M9rwUuKViXfc2mO8qKkKSqxX4NvD2immvoXFIXkzx7n0u0FMz7QHgggY1BbCyYrjhtqYI2C9J4UvTfo9ftUBnU7TG0yqm7wdWjPU1WpY+yepo0CeJiDsl7aZ4Z7tpeLyk5wHXUew6zE2jZ0vqjojBNPxYxaL66wzPqlndIxW3f0axi1PrhcBZkg5WjJsGfKbBvD3APknD47oq19Po8Y2iskYkvQS4FuijaJmmUezCQPGu/2ATy2ym1pMZuX3qiohdki6lCOLpkr4BvDMi9jZRU+U6RtvWJ1I83rsr6hXF7tywJ6K6X/M0I5/zrI4f3cqRdDHFvude4N0Vky6jaLLPiojjgT8YvssEVrek4vYL0jprPQL8T0TMqfibFRF/22DeI8DzK+Y9PiJOH55hlMfX6OPZteM/SrEbtCxthyv51TZ4hGJ3s5nl5Grdx8jt01BEfC4iXkXxQg+K/lauptq6RtvWj1O80Z1eMe2EiBhzCHJKHZL0Lvl+4C+BNwPvlrQ8TZ5NsZEOSppH0QRP1LskzZW0BLgEuLHOPLcBL5H0Zkk96e93JZ1aO2NE7AO+CXxI0vGSuiS9SNIfNvH4HgPmSzohU/Nsik78YUkvAyrDehvwG5IulTRD0mxJZ1Usf6mkrmZqpWjl3iFpsaS5wBWNCpL0UkkrJc0AnqF4noZb908A/yppmQovlzS/waIabuuIGAI+Dlwn6aS03kWSXpvZXmNWlpB8teY8yS3pJNUNwDURsT0idlK8S34mbfwPU3TuHge+D3x9EurYSLGrsg34L4ojPVUi4imK/fE3Ubz7P0rxLjmjwTL/CphOceDgAMXRqYW5xxcRPwY+D+yWdFBSvV0/gH8A/pziSNHHqQh2qvU8iv7XoxRHjV6dJn8x/X9C0j2j1ZqmfRz4BrAduAf4coN6SNviaorn5lGKXckr07RrKQL3TYpwX0/xPI7QxLa+nOLgzPclHQK+RbF3MamUOjS/9iQFxS7Lrk7XYuVSlpbErLQcErMM726ZZUyoJZG0StIDknZJani0w2wqG3dLIqmb4iMV51F8RuYuijPGP2p0nxkzu+O4WT2/WkbN9O5pI09xTOSkB9Q/2dCl6qUe3zu9Znr1/EM1CznUf3TEModqtuNE655Koua9NqL2QFztOevqzy1KI09HiaFJqKx5hw4do//pgbpP20TOuJ8J7IqI3QCSvgBcQHH4sK7jZvXw2tctfnZYNe3YvPkjj6J21b5ix6jem8CMnu6q4fN+q/q82POmV2+Wp49WP6mb7nt4xDKPHBusGpaeqzEZ+eKNmF01fGzgvTXT51UNS09WDfdMe9+IZUpP1Yxpbff5xg2NPwQwkTUvovojBHvSuCqS1knaKmnrkWcGayebld5EQlLvrXLE23ZErI+IvojomzGzu85dzMptIrtbe6j+LM9i6n/W6Vk9PWLBwronVwEYqt35b5Hurur3htkzq/skP3jk8arhly95/qj3L/w6t5I9VUMRC6qGp3XfXDU8MHjhqPcvm4m0JHcByySdImk6xUcHbp2csszKY9wtSUQMSPp7is/zdAOfjIj7J60ys5KY0PUkEfE14GuTVItZKbX3oqtoX79jLAYGqw9r/vzAL6uGTzu5+hCm1ao9hvNM1VBX173Vkwdfl7l/ufizW2YZDolZhkNiluGQmGWU5dtSSmWinxez2u1X+zKbWtvXLYlZhkNiluGQmGW0vU9Sxr3R3p7qzXDe6Uuqhmd05z+9XPu4yvg4W6f6ZKyo/qqwYwP/VDN9ds3wyGtUyrT93JKYZTgkZhkOiVlGW/skQ8CRwfZe4F/PsWeqv8hh47afVg33dFfvER8brP5Q5qFnRn4RxMhL6cv3Qc7JUe9xHaoaGuq6smb6zOpBVX8A8uhQ9f3rr6e1r5vRlu6WxCzDITHLcEjMMtraJ+kfGGT7EwfzM7bYiL3d/QfGdP96n+0q03H9znt0jPN3/osg+gcaf5GHWxKzDIfELMMhMctoa58kCI4Odf48yUQNlvDLLMplrN/U2fnXRIxyXsstiVmGQ2KW4ZCYZTgkZhlt7bjP6OrmxcfPaucqzZry067GJ7ndkphlOCRmGQ6JWUZb+yQ93WLhcY1/6cqsU2ovtKvklsQswyExy8iGRNInJe2XdF/FuHmSNknamf7PbW2ZZp3TTJ/kU8BHgE9XjLsC2BwRV0u6Ig1fnl1SwNDIb0ww67xRXpbZliQivgM8WTP6AmBDur0BWD3O0sxKb7x9kgURsQ8g/T+p0YyS1knaKmlrf/+v82+d21TV8o57RKyPiL6I6OvtHet1BmadN97zJI9JWhgR+yQtBPY3c6djEfyi/8izwyfMqP4CgBldIzPrHox12nhbkluBNen2GmDj5JRjVj7NHAL+PPA94KWS9khaC1wNnCdpJ3BeGjZ7TsrubkXERQ0mnTPJtZiVUls/uzVNYk5FP2Saqhsy9z+sjPyxFLMMh8QswyExy2hrn0TA9IpzIe6D2FTglsQswyExy3BIzDIcErOMtnbcwZ11m3rckphlOCRmGQ6JWYZDYpbhkJhlOCRmGQ6JWYZDYpbhkJhlOCRmGQ6JWYZDYpbhkJhlOCRmGQ6JWYZDYpbhkJhlOCRmGQ6JWYZDYpbhkJhlOCRmGQ6JWUYzPwe3RNIWSTsk3S/pkjR+nqRNknam/3NbX65Z+zXTkgwAl0XEqcAK4GJJpwFXAJsjYhmwOQ2bPedkQxIR+yLinnT7KWAHsAi4ANiQZtsArG5VkWadNKY+iaSlwCuAO4AFEbEPiiABJzW4zzpJWyVt7e8fnFi1Zh3QdEgkzQK+BFwaEYeavV9ErI+Ivojo6+3tHk+NZh3VVEgk9VAE5LMR8eU0+jFJC9P0hcD+1pRo1lnNHN0ScD2wIyKurZh0K7Am3V4DbJz88sw6r5mfXngl8Gbgh5K2pXFXAlcDN0laCzwMvKE1JZp1VjYkEfFdit8EreecyS3HrHza/iM+ZpPjWM1wT8vW5I+lmGU4JGYZDolZhvskNkVUv1SH4iVVw13aXTP/wKSt2S2JWYZDYpbhkJhlOCRmGe64W0kNVQ1FzK4aPnas+hq/6T2XVQ1LT9RZ5vjaBLckZhkOiVmGQ2KW4T6JTVFD+VkmiVsSswyHxCzDITHLcJ/Epojai2NrL7JqdPHsxLklMctwSMwyHBKzDPdJrKRq+xhHq4a6uraNOn0y+yhuScwyHBKzDIfELMN9Eiup6j6FdLhquGfaB2rmr30pu09i1jYOiVmGQ2KW4ZCYZbjjblNU+166bknMMhwSs4xmfjNxpqQ7JW2XdL+k96Xxp0i6Q9JOSTdKmt76cs3ar5mW5AiwMiLOAJYDqyStAK4BrouIZcABYG3ryjTrnGxIojB8urMn/QWwErg5jd8ArG5JhWYd1uzvuHenX97dD2wCHgQORsTwj0DsARY1uO86SVslbe3vH5yMms3aqqmQRMRgRCwHFgNnAqfWm63BfddHRF9E9PX2do+/UrMOGdPRrYg4CNwOrADmSBo+WL0Y2Du5pZmVQzNHt06UNCfd7gXOBXYAW4AL02xrgI2tKtKsk5o5bbkQ2CCpmyJUN0XEbZJ+BHxB0vuBe4HrW1inWcdkQxIRPwBeUWf8bor+idlzms+4m2U4JGYZDolZhkNiluGQmGU4JGYZDolZhkNiluGQmGU4JGYZDolZhkNiluGQmGU4JGYZDolZhkNiluGQmGU4JGYZDolZhkNiluGQmGU4JGYZDolZhkNiluGQmGU4JGYZDolZhkNiluGQmGU4JGYZDolZhkNiluGQmGU0HZL0M9X3SrotDZ8i6Q5JOyXdKGl668o065yxtCSXUPyg6LBrgOsiYhlwAFg7mYWZlUVTIZG0GPhj4BNpWMBK4OY0ywZgdSsKNOu0ZluSDwPvBobS8HzgYEQMpOE9wKJ6d5S0TtJWSVv7+wcnVKxZJzTzO+6vA/ZHxN2Vo+vMGvXuHxHrI6IvIvp6e7vHWaZZ5zTzO+6vBP5E0vnATOB4ipZljqRpqTVZDOxtXZlmnZNtSSLiHyNicUQsBd4EfDsi/gLYAlyYZlsDbGxZlWYdNJHzJJcD75S0i6KPcv3klGRWLs3sbj0rIm4Hbk+3dwNnTn5JZuXiM+5mGQ6JWYZDYpbhkJhlOCRmGQ6JWYZDYpbhkJhlOCRmGQ6JWYZDYpbhkJhlOCRmGQ6JWYZDYpbhkJhlOCRmGWO6MnEy1PuaFbMyc0tiluGQmGU4JGYZbe2TBDAQdb/o0ayjRntVuiUxy3BIzDIcErOMtvZJnh4YZPvjB9u5SrOmPD3Q+GdB3JKYZTgkZhkOiVlGm8+TBEeGhvIzmrVZjHKmxC2JWYZDYpbR1O6WpIeAp4BBYCAi+iTNA24ElgIPAW+MiAOtKdOsc8bSkrw6IpZHRF8avgLYHBHLgM1p2Ow5ZyId9wuAs9PtDRQ/E3d57k6+6MqmmmZbkgC+KeluSevSuAURsQ8g/T+pFQWadVqzLckrI2KvpJOATZJ+3OwKUqjWAUw/zscJbOpp6lUbEXvT//3ALRS/uvuYpIUA6f/+BvddHxF9EdE3baZDYlNP9lUr6ThJs4dvA68B7gNuBdak2dYAG1tVpFknNbO7tQC4RdLw/J+LiK9Lugu4SdJa4GHgDa0r06xzsiGJiN3AGXXGPwGc04qizMrEnQSzDEUbv5hB0i+AnwHPBx5v24rHz3VOnrLX+MKIOLHehLaG5NmVSlsrztyXluucPFOhxka8u2WW4ZCYZXQqJOs7tN6xcp2TZyrUWFdH+iRmU4l3t8wyHBKzjLaGRNIqSQ9I2iWpVBdpSfqkpP2S7qsYN0/SJkk70/+5Ha5xiaQtknZIul/SJSWtc6akOyVtT3W+L40/RdIdqc4bJU3vZJ3NaltIJHUD/w78EXAacJGk09q1/iZ8ClhVM65sV18OAJdFxKnACuDitA3LVucRYGVEnAEsB1ZJWgFcA1yX6jwArO1gjU1rZ0tyJrArInZHxFHgCxRXN5ZCRHwHeLJm9AUUV12S/q9ua1E1ImJfRNyTbj8F7AAWUb46IyIOp8Ge9BfASuDmNL7jdTarnSFZBDxSMbwnjSuz0l59KWkp8ArgDkpYp6RuSdsorjPaBDwIHIyIgTTLVHj+gfaGpN7l7T7+PA6SZgFfAi6NiEOdrqeeiBiMiOXAYoq9iFPrzdbeqsannSHZAyypGF4M7G3j+sejqasv20lSD0VAPhsRX06jS1fnsIg4SPElISuAOZKGL8+YCs8/0N6Q3AUsS0c4pgNvori6scxKdfWliivfrgd2RMS1FZPKVueJkuak273AuRT9py3AhWm2jtfZtIho2x9wPvATiv3T97Rz3U3U9nlgH3CMotVbC8ynOFq0M/2f1+EaX0Wxi/IDYFv6O7+Edb4cuDfVeR/wz2n8bwJ3AruALwIzOv28N/Pnj6WYZfiMu1mGQ2KW4ZCYZTgkZhkOiVmGQ2KW4ZCYZfw/2yKMb0B3XUYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "resize = T.Compose([T.ToPILImage(),\n",
    "                    T.Resize(40, interpolation=Image.CUBIC),\n",
    "                    T.ToTensor()])\n",
    "\n",
    "\n",
    "\n",
    "def get_screen():\n",
    "    # Returned screen requested by gym is 400x600x3, but is sometimes larger\n",
    "    # such as 800x1200x3. Transpose it into torch order (CHW).\n",
    "    screen = env.render(mode='rgb_array').transpose((2, 0, 1))\n",
    "    # Cart is in the lower half, so strip off the top and bottom of the screen\n",
    "    _, screen_height, screen_width = screen.shape\n",
    "    screen = np.ascontiguousarray(screen, dtype=np.float32) / 255\n",
    "    screen = torch.from_numpy(screen)\n",
    "    # Resize, and add a batch dimension (BCHW)\n",
    "    return resize(screen).unsqueeze(0).to(device)\n",
    "\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(get_screen().cpu().squeeze(0).permute(1, 2, 0).numpy(),\n",
    "           interpolation='none')\n",
    "plt.title('Example extracted screen')\n",
    "plt.show()\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 128\n",
    "GAMMA = 0.99\n",
    "EPS_START = 1.0\n",
    "EPS_END = 0.02\n",
    "EPS_DECAY = 100000\n",
    "TARGET_UPDATE = 1000\n",
    "tau = 0.01\n",
    "PREFETCH = 10000\n",
    "MEM_SIZE = 100000\n",
    "learning_rate = 0.00025\n",
    "past = 3\n",
    "\n",
    "# Get screen size so that we can initialize layers correctly based on shape\n",
    "# returned from AI gym. Typical dimensions at this point are close to 3x40x90\n",
    "# which is the result of a clamped and down-scaled render buffer in get_screen()\n",
    "init_screen = get_screen()\n",
    "_, _, screen_height, screen_width = init_screen.shape\n",
    "\n",
    "# Get number of actions from gym action space\n",
    "n_actions = 3 #env.action_space.n\n",
    "\n",
    "policy_net = DQN_basic(screen_height, screen_width, n_actions).to(device)\n",
    "target_net = DQN_basic(screen_height, screen_width, n_actions).to(device)\n",
    "target_net.load_state_dict(policy_net.state_dict())\n",
    "target_net.eval()\n",
    "\n",
    "optimizer = optim.RMSprop(policy_net.parameters(), lr=learning_rate)\n",
    "memory = ReplayMemory(MEM_SIZE)\n",
    "\n",
    "\n",
    "steps_done = 0\n",
    "\n",
    "eps = []\n",
    "eps.append(EPS_START)\n",
    "eps_threshold = 0\n",
    "\n",
    "def get_eps1():\n",
    "    global steps_done\n",
    "    eps_threshold = EPS_END + (EPS_START - EPS_END) * \\\n",
    "        math.exp(-1. * steps_done / EPS_DECAY)\n",
    "    steps_done += 1\n",
    "    return eps_threshold\n",
    "\n",
    "def get_eps2():\n",
    "    global steps_done\n",
    "    reduction = min(steps_done / EPS_DECAY, 1)\n",
    "    steps_done += 1\n",
    "    diff = EPS_START - EPS_END\n",
    "    diff = diff*(1 - reduction)\n",
    "    return EPS_END + diff\n",
    "print(get_eps2())\n",
    "\n",
    "\n",
    "def select_action(state, eps_max=1):\n",
    "    global steps_done\n",
    "    sample = random.random()\n",
    "    eps_threshold = get_eps2()\n",
    "    eps_threshold = min(eps_max, eps_threshold)\n",
    "    eps.append(eps_threshold)\n",
    "    if sample > eps_threshold:\n",
    "        with torch.no_grad():\n",
    "            return policy_net(state).max(0)[1].view(1,1)\n",
    "    else:\n",
    "        return torch.tensor([[random.randrange(n_actions)]], device=device, dtype=torch.long)\n",
    "\n",
    "def prefetch(n_prefetch):\n",
    "    s = env.reset()\n",
    "    print(\"Prefetching:\", n_prefetch)\n",
    "    while len(memory) < n_prefetch:\n",
    "        if len(memory) % 1000 == 0:\n",
    "            print(len(memory))\n",
    "        state = (torch.from_numpy(s).float().to(device)-128)/128\n",
    "        action = torch.tensor([[random.randrange(n_actions)]], device=device, dtype=torch.long)\n",
    "        s, reward, done, _ = env.step(action.item()+1)\n",
    "        \n",
    "        reward = torch.tensor([reward], device=device, dtype=torch.float32)\n",
    "        if not done:\n",
    "            next_state = (torch.from_numpy(s).float().to(device)-128)/128\n",
    "        else:\n",
    "            next_state = None\n",
    "        \n",
    "        memory.push(state, action, next_state, reward)\n",
    "        state = next_state\n",
    "        \n",
    "        if done:\n",
    "            s = env.reset()     \n",
    "    \n",
    "episode_durations = []\n",
    "\n",
    "\n",
    "def plot_durations():\n",
    "    durations_t = torch.tensor(episode_durations, dtype=torch.float)\n",
    "    plt.title('Training...')\n",
    "    plt.xlabel('Episode')\n",
    "    plt.ylabel('Rewards')\n",
    "    plt.plot(durations_t.numpy())\n",
    "    # Take 100 episode averages and plot them too\n",
    "    if len(durations_t) >= 100:\n",
    "        means = durations_t.unfold(0, 100, 1).mean(1).view(-1)\n",
    "        means = torch.cat((torch.zeros(99), means))\n",
    "        plt.plot(means.numpy())\n",
    "        \n",
    "losses = []\n",
    "losses_avg = []\n",
    "def plot_losses():\n",
    "    plt.title('Losses')\n",
    "    plt.xlabel('steps')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.plot(losses)\n",
    "    # Take 100 episode averages and plot them too\n",
    "    avg_length = 100\n",
    "    while len(losses) > len(losses_avg) + avg_length:\n",
    "        if len(losses_avg) == 0:\n",
    "            losses_avg.append(sum(losses[:avg_length])/avg_length)\n",
    "        else:\n",
    "            i_rem = len(losses_avg) - 1\n",
    "            i_add = i_rem + avg_length\n",
    "            l_curr = losses_avg[-1]\n",
    "            l_curr += (losses[i_add] - losses[i_rem])/avg_length\n",
    "            losses_avg.append(l_curr)\n",
    "    if len(losses_avg) > 0:\n",
    "        plt.plot(losses_avg)\n",
    "    \n",
    "    \n",
    "def plot_stuff():\n",
    "    if is_ipython:\n",
    "        display.clear_output(wait=True)\n",
    "    plt.figure(figsize=(16,4))\n",
    "    plt.subplot(121)\n",
    "    plot_durations()\n",
    "    plt.subplot(122)\n",
    "    plot_losses()\n",
    "    plt.pause(0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_model():\n",
    "    if len(memory) < BATCH_SIZE:\n",
    "        return\n",
    "    transitions = memory.sample(BATCH_SIZE)\n",
    "    batch = Transition(*zip(*transitions))\n",
    "\n",
    "    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,\n",
    "                                          batch.next_state)), device=device, dtype=torch.uint8)\n",
    "    non_final_next_states = torch.stack([s for s in batch.next_state\n",
    "                                                if s is not None])\n",
    "    state_batch = torch.stack(batch.state)\n",
    "    action_batch = torch.cat(batch.action)\n",
    "    reward_batch = torch.cat(batch.reward)\n",
    "\n",
    "    state_action_values = policy_net(state_batch).gather(1, action_batch)\n",
    "\n",
    "    next_state_values = torch.zeros(BATCH_SIZE, device=device)\n",
    "    with torch.no_grad():\n",
    "        next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0]\n",
    "    expected_state_action_values = (next_state_values * GAMMA) + reward_batch\n",
    "\n",
    "    loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1))\n",
    "    losses.append(loss.cpu().detach().numpy())\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    for param in policy_net.parameters():\n",
    "        param.grad.data.clamp_(-1, 1)\n",
    "    optimizer.step()\n",
    "#do not forget 3 actions is active both in prefill and other shit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA74AAAEWCAYAAABBrkurAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdeZyd4/3/8ddnzixRhCL8rE0sbb8oiiqlaqlWUVu1tKgqVUqpqoqqpbZqKaX2fRc7IUESBAlBIntCZJns+57Mes75/P4450zOzJzlPmfOMjN5Pz1GzrmX677uM/fM3J/7uq7PZe6OiIiIiIiISHdVUe4KiIiIiIiIiBSTAl8RERERERHp1hT4ioiIiIiISLemwFdERERERES6NQW+IiIiIiIi0q0p8BUREREREZFuTYGvyHrCzEJmtsbMdijktiIiIiIinZ0CX5FOKh54Jr6iZlaf9P7UXMtz94i7b+Tuswq5rYiIiKRmZrVm9sNy10NEoLLcFRCR1Nx9o8RrM6sFznb3Iem2N7NKdw+Xom4iIiIiIl2JWnxFuigzu97MnjWzZ8xsNXCamR1gZiPMbIWZzTezO8ysKr59pZm5mfWOv38yvv4NM1ttZh+ZWZ9ct42v/4mZTTGzlWb2PzMbbma/Ke0nIiIi0jWY2e/MbKqZLTOz/ma2TXy5mdltZrYo/jd1nJntHl93lJlNiv8dnmtmf0kq7xgzGxP/+/+hme2RtO6y+ParzewLMzu89GcsUn4KfEW6thOAp4FNgGeBMHARsAVwIHAk8PsM+/8KuBLYDJgFXJfrtma2JfAccGn8uDOA/fI9IRERke7MzA4D/gn8AtgamAn0i6/+EXAw8HVgU+BkYGl83UPA7919Y2B34J14eXsDDxP7e785cB/Q38xqzOwbwAXAd+L7/RioLfIpinRKCnxFurZh7v6au0fdvd7dP3X3j9097O7TgfuBH2TY/wV3H+nuzcBTwF55bHsMMMbdX42vuw1Y0vFTExER6ZZOBR5298/cvRG4HDgg3suqGdgY+CZg7j7Z3efH92sGdjWznu6+3N0/iy//HXBf/O9/xN0fAxqB/YEIUBPfr8rda919WqlOVKQzUeAr0rXNTn5jZt80swFmtsDMVgHXEmuFTWdB0us6YKN0G2bYdpvkeri7A3MC1F1ERGR9tA2xVl4A3H0NsVbdbd39HeBO4C5goZndb2Y945v+DDgKmGlm75nZAfHlXwMuiXdzXmFmK4DtgW3cfSrwJ+AaYJGZ9Ut0qxZZ3yjwFenavM37+4AJwM7u3hO4CrAi12E+sF3ijZkZsG2RjykiItJVzSMWrAJgZhsS66I8F8Dd73D3fYDdiHV5vjS+/FN3Pw7YEniF2DAjiD18vsHdN036+oq7PxPf72l3Pyh+TAf+VYqTFOlsFPiKdC8bAyuBtWb2f2Qe31sorwN7m9lPzayS2BjjXiU4roiISFdQZWY9El/EAtYzzWwvM6sBbgQ+dvdaM/uOmX03nphyLdAARMys2sxONbNN4sOKVhHrxgzwAHBufD8zsw3N7Ggz29jMvmFmh8WP0wDUJ+0nsl5R4CvSvVwCnAGsJtb6+2yxD+juC4kl37iVWFetnYDRxMYXYWaHxLtdEX9/pZm9lvR+kJn9tdj1FBERKZOBxALOxNf3iSWLfJFYr6mdgFPi2/YkFsguJ9YdeilwS3zd6UBtfCjTucBpAO4+ktg43zvj+00FfhPfpwa4iVjujQXEWov/VpSzFOnkLDYcT0SkMMwsRKwb10nu/kG56yMiIiIiohZfEekwMzvSzDaJd6W6kti0Sp+UuVoiIiIiIoACXxEpjIOA6cS6Uh0JHB+fokFEREREpOzU1VlERERERES6NbX4ioiIiIiISLdWWe4KlNIWW2zhvXv3Lnc1RESkmxg1atQSd9f0XR2gv80iIlJI6f42r1eBb+/evRk5cmS5qyEiIt2Emc0sdx26Ov1tFhGRQkr3t1ldnUVERERERKRbU+ArIiIiIiIi3ZoCXxEREREREenWFPiKiIiIiIhIt6bAV0RERERERLo1Bb4iIiIiIiLSrSnwFRERERERkW5Nga+IiIiIBPbhtCVMW7ym3NUQEclJZbkrICIiIiJdx68e+BiA2puOLnNNRESCU4uviIiIiIiIdGtlC3zN7GYz+9zMxpnZy2a2adK6y81sqpl9YWY/TrN/HzP72My+NLNnzay6dLUXERERERGRrqKcLb6Dgd3dfQ9gCnA5gJntCpwC7AYcCdxtZqEU+/8LuM3ddwGWA2eVpNYiIiIiIiLSpZQt8HX3Qe4ejr8dAWwXf30c0M/dG919BjAV2C95XzMz4DDghfiix4Dji19rERERERER6Wo6yxjf3wJvxF9vC8xOWjcnvizZ5sCKpMA51TYAmNk5ZjbSzEYuXry4gFUWERERERGRrqCoga+ZDTGzCSm+jkva5gogDDyVWJSiKG9bdIBtYgvd73f3fd193169euVzGiIiIt2OmR0Zz6Ux1cz6plhfE8+hMTWeU6N3fPkRZjbKzMbH/z0saZ994sunmtkd8R5aIiIiZVfU6Yzc/YeZ1pvZGcAxwOHunghc5wDbJ222HTCvza5LgE3NrDLe6ptqGxEREUkhnjvjLuAIYn93PzWz/u4+KWmzs4Dl7r6zmZ1CLLfGycT+Bv/U3eeZ2e7AW6zrdXUPcA6xIUwDieXqeAMREZEyK2dW5yOBy4Bj3b0uaVV/4JT4k+Y+wC7AJ8n7xoPkd4GT4ovOAF4tfq1FRES6hf2Aqe4+3d2bgH7EcmwkO45YDg2I5dQ43MzM3Ue7e+Jh80SgR/xv9tZAT3f/KP53+nGUf0NERDqJco7xvRPYGBhsZmPM7F4Ad58IPAdMAt4Eznf3CICZDTSzbeL7Xwb82cymEhvz+1CpT0BERKSLCpJPo2WbeO+qlcT+3ib7GTDa3Rvj28/JUiag/BsiIlJ6Re3qnIm775xh3Q3ADSmWH5X0ejptsj2LiIhIIB3Op2FmuxHr/vyjHMqMLXS/H7gfYN999025jYiISCF1lqzOIiIiUjpB8mm0bGNmlcAmwLL4++2Al4Ffu/u0pO23S9pf+TdERKTTUOArIiKy/vkU2MXM+phZNXAKsRwbyfoTy6EBsZwa77i7m9mmwADgcncfntjY3ecDq81s/3g251+j/BsiItJJKPAVERFZz8TH7F5ALCPzZOA5d59oZtea2bHxzR4CNo/n0vgzkJjy6AJgZ+DKeI6OMWa2ZXzdecCDwFRgGsroLCIinUTZxviKiIhI+bj7QGJTDiUvuyrpdQPw8xT7XQ9cn6bMkcDuha2piIhIx6nFV0RERERERDrkhVFzmLJwdbmrkZZafEVERERERKRD/vL8WABqbzq6zDVJTS2+IiIiIiIi0q0p8BUREREREZFuTYGviIiIiIiIdGsKfEVERERERKRbU+ArIiIiIiIi3ZoCXxEREREREenWFPiKiIiIiIhIt6bAV0RERERERLo1Bb4iIiIiIiLSrSnwFRERERERkW5Nga+IiIiIiIh0awp8RUREREREpFtT4CsiIiIiIiUzd0V9uasg6yEFviIiIiIiUhKvjpnLgTe9w4dTl5S7KrKeUeArIiIiIiIlMXrWCgA+X7C6zDWR9Y0CXxERERERKSkvdwVkvaPAV0RERERERLo1Bb4iIiIiIlJSVu4KyHpHga+IiIiIiJSUujpLqSnwFRERERERkW5Nga+IiIiIiIh0awp8RURERESkJEyDe6VMFPiKiIiIiIhIt6bAV0REREQ6hcZwhA++XFzuaohIN6TAV0REREQ6hRsHTOb0hz5h/JyV5a6KiHQzCnxFREREpFOYtngtAMvrmlqWvTlhAWNnryhXlaRI3DWhkZRWZbkrICIiIiKSzrlPjgKg9qajy1wTEenK1OIrIiIiIiIlZUrvLCWmwFdEREREOgXFQusPdXWWUlPgKyIiIiKdikIiESk0Bb4iIiIiIiLSrSnwFRERERGRkjDUn13KoyyBr5ndbGafm9k4M3vZzDZNWne5mU01sy/M7Mdp9n/UzGaY2Zj4116lq72IiIiIFJPGf2b35oQFnPnIJ+WuRs5cHdmlTMrV4jsY2N3d9wCmAJcDmNmuwCnAbsCRwN1mFkpTxqXuvlf8a0wpKi0iIiIi0hmc++Qo3v1icbmrIdJllCXwdfdB7h6Ovx0BbBd/fRzQz90b3X0GMBXYrxx1FBERERGRwlJXZymXzjDG97fAG/HX2wKzk9bNiS9L5YZ4V+nbzKwmXeFmdo6ZjTSzkYsX66mYiIiISGeluV2lq1i+tonefQfw9uSF5a6KBFS0wNfMhpjZhBRfxyVtcwUQBp5KLEpRVKqBAJcD3wS+A2wGXJauHu5+v7vv6+779urVK+/zERER6U7M7Mh4Po2pZtY3xfoaM3s2vv5jM+sdX765mb1rZmvM7M42+wyNl5nIwbFlac5GuhuNApXObvKCVQA88MH0MtdEgqosVsHu/sNM683sDOAY4HBfl8FgDrB90mbbAfNSlD0//rLRzB4B/tLxGouIiKwf4vkz7gKOIPa391Mz6+/uk5I2OwtY7u47m9kpwL+Ak4EG4Epg9/hXW6e6+8iinoCISCehPGxdR7myOh9JrJX2WHevS1rVHzgl/pS5D7AL0C5dnZltHf/XgOOBCcWvtYiISLexHzDV3ae7exPQj1iejWTHAY/FX78AHG5m5u5r3X0YsQBYRCQn6s0u5VKuMb53AhsDg+Ndoe4FcPeJwHPAJOBN4Hx3jwCY2UAz2ya+/1NmNh4YD2wBXF/qExAREenCguTUaNkmnpByJbB5gLIfif9tv9I0YFNyVKwLZm1jmIbmSJFKl1x0txZS/ZbrOorW1TkTd985w7obgBtSLD8q6fVhRaqaiIjI+iBITo2geTeSneruc81sY+BF4HTg8XYHNzsHOAdghx12yF5bkQ7a7eq32GKjGkb+PeNIPJGcdbdAPl9dYe7tzpDVWUREREorSE6Nlm3MrBLYBFiWqVB3nxv/dzXwNGmmJFTiya7non6j+foVb2TfsFCKcA+9ZE1j4QuVnKmFVMpFga+IiMj651NgFzPrY2bVwCnE8mwk6w+cEX99EvCOZ3ikb2aVZrZF/HUVsQSWysHRTbw6Zh5NkWjRj6OgqLxGz1rOg8pSLN1UWbo6i4iISPm4e9jMLgDeAkLAw+4+0cyuBUa6e3/gIeAJM5tKrKX3lMT+ZlYL9ASqzex44EfATOCteNAbAoYAD5TwtNYraxrDVFYYPapC5a5KUbgmNCqLE+7+EICzv79jyvVTFq7m4xnLOH3/r5WyWp2SFW1EetfUBXo6K/AVERFZH7n7QGBgm2VXJb1uAH6eZt/eaYrdp1D1k8x2v/ottt10A4b3XX/Tnixc1UCowthio5pyV2W9ceR/3yfqKPCVLkldnUVERES6oLkr6stdhcDCkSjhAneV/u6Nb7Pv9UMKWmZXFCSp0KCJC6hdsrbDx4p2gVY9kXQU+IqIiIhIUe1z/RD2CRCkBuk8+vLoOaxtDHe8UuuRc54YxWH/GVruanRLehYQ0xU+B3V1FhEREZGiWlnfXJByRs1cxsXPjuWkfZYWpLzuwD1YUrDO1lrbFcaESveiFl8RERER6RLWNEaA2PheSW/qojXsfvVbzFleV+6qtKOUUFIuCnxFREREJJALnv6sJMdJ1xq4PgdNTeEo9U2RdstTfVT9PpnFmsYwb4xfUPyK5ShR38dH1PLdGzVGu7sIMta83BT4ioiIiEggr4+bX9TyTRP5pnXsncP4v6veLHc1Cmb2snoWrmosdzVkPaLAV0RERES6lC7QuFRwny9YnXJ5V2hpS9ZdHm3oGU3Xo8BXRERERDqVtF2dFWzkxLtErl3pDrrClabAV0REREQ6BcW1uUsVcBT7AUFXa2XuzMKRKJc+P5Zpi9eUuyrdngJfEREREZESWtXQzOqGwkzxlEqx41LFvfl54qNafvPIJ62WTZy3iudHzeHiZ8eUp1LrEQW+IiIiItKlFLML76JVDZxw93AWry5e4qU9rhnEt64ZVJCyunsQOmjiAl4bO6/c1SiIK1+dyNAvFqdc19W/j12h/gp8RURERCStF0fNKfkx091DW7wzdDFvsp8YMZPRs1bwzCezineQIit6V+fiFt/KOU+M4o/PjC7hEUtL49ZLR4GviIiIiKR1yfNjS3asUgYBK+ubaQynmBe3C7RcJStm6/egiQu4//1p7Y/Z1T6kYirQR6FEZMVXWe4KiIiIiIiU2p7/GMS+X/sqL5z3vXJXpVPa69pBrKiLjUM+5+CdWq1TiCZtdYXAXS2+IiIiItIlJFqEC9XgOHLm8sIUVCBL1zTSu+8AHv+oNvA+xWp8TQS9qUyev4rnR85uef/y6Dn8Z9AXxamISIEo8BURERGRTiVdV9ruOhzyw6lLeH7kbOYsrwfghTKMq87FsXcO59IXxrW8v/jZsfzvnallrFHpFepaLMW4dYlRV2cRERER6SS6R2gbicaimFBFsPP51YMfA/Dq+QcWtB6dMZhSMqfWusvn0RmvtbbU4isiIiIi0kZH4pFdrhjI4f8Zmvf+XSGIKBQlyorRx1B8gQJfM9vJzGrirw8xswvNbNPiVk1ERERECm3qojW8OmZuuasBwJrGML37DmhXn2wxQGdPpBN1qF1al/N++bT+ZQqYuktrYmfWma/F+Svr+WTGsnJXo9MI2uL7IhAxs52Bh4A+wNNFq5WIiIiIFMUPb32Pi/qNYfay3AOzQpuzPFaHu9+NTZmTNVArcHKr7k6f0/rt8P+8xy/u+6jc1eg0gga+UXcPAycA/3X3i4Gti1ctERERESmm7//73bIcd87yOm4f8qW6uObp3vemccXL41vep2pxtC7U1KvLIKYYH0NdU/t5qtdnQZNbNZvZL4EzgJ/Gl1UVp0oiIiIi0l2d++QoJsxdxdF7bF2QoOfdLxZx6De27HhBXcRNb3xe7ipIAXWhZxRdXtAW3zOBA4Ab3H2GmfUBnixetURERESkK3hixEwmz18VePv6llaodVFv25v/dAGxpUg5deYjnwY+dleRy7jRrt5i2sWrn/KaXB91heswUIuvu08CLkx6PwO4qViVEhEREZGu4cpXJgBQe9PROe133J3D2X3bTYpRJSkRd+9S3aoBhn25hIg7P/h6r4KU15mTW0lrGQNfMxtPhgcx7r5HwWskIiIiIq2saQwzYNw8frHv9l0u0EhnbVOEj/PMOFvMUKOrBTLlrG0k6lSGsl+P81bUs3RNE9/arv2DjthY79Jd06c9FJszOdcHNcWmMe/Fl63F95j4v+fH/30i/u+pQPlTAYqIiIisB656dQIvfTaXPltsxH59Nit4+X95fiyjZy3n7UsOKXjZucia1Ll7xPwlU+xQKhrwAN+76R0gFmxme3Dzj9cm8vhHM3n0zO/w/V0K0yrbmXWXrtJd4YFRxjG+7j7T3WcCB7r7X919fPyrL/Dj0lRRREREZP22eHUjAPXNuWVpHTBuPnv+YxBN4WjG7V4YNYdpi9fmXb/Cy3IT3fnvsUumGC2FQcuM5nHstmW3LeGR4bVEos7pD33CbYOn5Fx+qRSq54Ue5pRO0ORWG5rZQYk3ZvY9YMPiVElERERECuGa1yaysr6ZFfVNKddPXbSG3n0HlLhW6a3vQUChW/+K/XG6w+vj5hWt/Nvf/rJoZedq7op6JsxdWbDyFq9u5JT7P2LZ2tQ/m1J4Qacz+i3wiJltQuzBzMr4MhERERHppBKtaxVpIsqhXywqZXWkxPJtCw7akBtx54KnR+dUdtuW0q4ytPXApO7ahfDohzMYMX0Zm284qyDllVtX+D5mbfE1swpgZ3ffE9gD2Mvd93L3z4peOxEREcnIzHYys5r460PM7EIz27Tc9ZLOITEGsxQNqac/9DHzVtRn3S7I/XH66YwSZaSfCqlQytn6nEsQkWrTjlY96OHz6epcTq+OmVuQct6evJC1jWEg/4Cvbet+qnJeHzeP3a9+i8ZwbkMcJLWsga+7R4EL4q9XuXvh2vhFRESko14EIma2M/AQ0Ad4urxVks4imqXFt5A++HIJdxS5a2qqcZXu8NiHtXmXqVbv/HnmoeOdzqBJCztcxqyldZz12Ej+/NyYAtSIlqcUqZJDXfnKBNY0hvnnwM8Lc6z1XNAxvoPN7C9mtr2ZbZb4KmrNREREJIiou4eBE4D/uvvFwNZlrpMUwKqGZnr3HcDA8fPz2v+2wVNYUdcMdL9cUNPbJOK6uv/EvMv6zSOfplxe15R7K9vsZXUlHzNdjEbXYia3anesLnZ11jXHWnqXx3+2OirTI6mV9bFjPNqBBzvFtqKuiSteHk9Djon3yiFo4PtbYlMavQ+Min+NLFalREREJLBmM/slcAbwenxZVRnrIwUyIx7c3fvetLz2T04MFDRAGTdnRaDuyunMXl7HZ7OW57xfoiU30f3z7qGZz3lpDgmB8s16fPfQaTl3MR09e0Vex0roasm9IiXs6rxoVUPJjpVJd5l+qFBuGfQFT308i5c+K0w38mIKFPi6e58UXzsWu3IiIiKS1ZnAAcAN7j7DzPoAT5a5TlJAhYgt3OHhYTO4ddAXrZZfP2Byq/fH3jm8Zc7VfAyfupQT7/4w5/3aBpjj02TPzTUwnLuinhHTl7VbHjQYbswyDVS2cuubItw4cDL1ebQeBz9o8E0nzF3JS5/NKViRBWnxDVjEfje+3eFjdUZdq727vUj8R6QrtNwHzeqMme0O7Ar0SCxz98fzOaiZ3Qz8FGgCpgFnuvsKM9sceAH4DvCou1+QZv/NgGeB3kAt8At3z/3xooiISBfn7pOACwHM7KvAxu5+U3lrJZ2Nu3Pt65PKXY30inTPfOgtQ1POYdw22Frd0MzGPdp3lOhoXPfQsOnc//50evao5ILDduGtiQvYqVfpZgRtW/9j/jcMgBP33q4g5UeinT/YSVaIttpCtMoPHD+/XTmprjUz6zTpkt2dVQ1hNtkgdYeiTlLNjAK1+JrZ1cD/4l+HAv8Gju3AcQcDu7v7HsAU4PL48gbgSuAvWfbvC7zt7rsAb8ffi4iIrHfMbKiZ9Yw/FB5LbPrBW8tdL0ktHIkSLUOwUOojnv7QxyU+Ymqpgl5o31L5cYpW4UJojnirf3//xCh+eOv7WffLLatz4b+7zZFgLd0H/DP33gHdraNwPp/+H576jNqldUBypvLO7aFhM9jzH4OY22YohLUk50pt9rK6tD+HpRZ0jO9JwOHAAnc/E9gTqMn3oO4+KJ6IA2AEsF18+Vp3H0YsAM7kOOCx+OvHgOPzrYuIiEgXt4m7rwJOBB5x932AH5a5TpLGzle8wXF3DQ+0bSHHexZj2plfP/wJj39Um3LdB18uya2w+Lm2PefBHcjCmyl4y/RptPqocvzYOlurV6zBMPdK3fzWF9k3ylOpP6LaJWtZsqaxYOUV6scykQwq87VY3gvK3fly4WoAXhs7D0g/1jpVVVc3NPP9f7/L5S+NL1odcxE08K2PT2sUNrOewCKgUGN8fwu8keM+W7n7fID4v1um29DMzjGzkWY2cvHixR2opoiISKdUaWZbA79gXXIr6cTGz13JqJn5jdDK96a7GI3M709ZzFWv5p9Nua35K+sZ1iZgbjvWN9P57/S3ga1aln7/xKi02xYrnijHOMdM55Lvec5eVpffjnkodmx3yC1D2b+A44PbPpzpaCC8oi59orbkn9s38szu3hFPjJjJEbe9zyczlrE2Pk59w5rWI2VTza2dkBjb/v6XnSMGCxr4jjSzTYEHiGV0/gz4JNMOZjbEzCak+DouaZsrgDDwVJ71z8rd73f3fd193169ehXrMCIiIuVyLfAWMM3dPzWzHYHiTqYqHTayNnjX2vFzVzJx3iog/9aycrccZeXw0/8NY3VjuNXitkFF2/XJIlFvmf4F4J3P08/PGzRALVQgW98c4fYhJfyx7GA0VszLpZRdnRPXQ7igT34KewbDpy4NtN30JWuzb5Tk2tc6PqZ/3JzYg6faJWupi//s1VQGDR87X/ftQMmt3P0P8Zf3mtmbQE93H5dln4zdrMzsDOAY4HDP/bfxQjPb2t3nx59ya+ZxERFZL7n788DzSe+nAz/Ltp+ZHQncDoSAB9smxDKzGuBxYB9gKXCyu9dmSkRpZvsAjwIbAAOBi/L4Gy8pLMth+p5UOtV3IU1dlqzJfo5npplzNyFo9/CitfimKff+96cX54BkDy7yOdfCBoqZFbKV/IVRc+g/dh6P/3Y/AH5w87ttjtV5ZftVmeuv0oeHz8i4fmV9c9pEVQnJP051WebpzVS95HImzF3J1pv0YPON8h41m7egya0eN7Pfmdk33b02W9AboLwjgcuAY909n74U/YnNV0j831c7Uh8REZGuysy2M7OXzWyRmS00sxfNLGPKVjMLAXcBPyE2Y8MvzWzXNpudBSx3952B24B/xZdnSkR5D3AOsEv868h8z0tSzxeab1tTpwp8U0hXva42r22hPue2xbw+bl5hCg7ovSmtu6Y2hiPtkhp1Rn95fizvJ9V9RV1zhq1hZV0zs5bmFoq0y8ac095J5bR5n62cQj6LeH3cPPb8xyDGzQk277TjaTN45/ozesz/hnHUHR/ktlOBBG2rfhTYGvifmU2L/1G9qAPHvRPYGBhsZmPM7N7ECjOrBW4FfmNmcxJ/iM3sQTPbN77ZTcARZvYlcET8vYiIyProEWIPhLcBtgVeiy/LZD9gqrtPd/cmoB+xxJHJkhNJvgAcbmaWLhFlvAdWT3f/KN7K+zhKPllw+d77FiO5VSGla81KFfwX5nj57ReJesas3G3XFCpwv+Dp0emPma2lsADH/1O/MRzYgfmdO4u2344f3vYeB7dpFS5ZXXK8Ngr5I5wYS58YQpFOqjqmq0eq6zDdtgtXtU42Nmrm8pI83Ana1fkdM3uPWLemQ4Fzgd2IdZHKWfzpcbp1vdMsPzvp9VJiWaZFRETWd73cPTnQfdTM/pRln22B2Unv5wDfTbeNu4fNbCWwOZAuXe+28XKSy9w21YZmdg6xlmF22GGHLFWVQujcYW+wFt/efQcELm9Cm6RY7Ty8rLUAACAASURBVI/X+ohzV9TTu+8AXjzvgNbbtanYrle9ydab9GDopYcGqkexAvdUlq5ppLKigk2+krn7aj7enly8UYXlfCazeHXu2Z7L1QmhGA+vghbpTtof0sQ1nrGrc5YP7Wf3fAjAMXtsE6xCeQra1fltYDhwMvAF8B13/2YxKyYiIiKBLDGz08wsFP86jdiY3ExS3Ya0a6wKsE1e2yvxZP4tgflnde4coW+phnwf879hadetbmjmutcnt1o2bGrsec6zn85utbxtbRvD0Zb5V1Mpx5D2xBH3uX4Ie103qM0671SJzT6btZz7ijjeuRwSmYsTnhgxk959B7RMV1Qohfw+Bv390xLUJtcjzbbJy+ua0iehK6egXZ3HAU3A7sAewO5mtkHRaiUiIiJB/ZbYVEYLgPnAScCZWfaZA2yf9H47oG0/s5ZtzKwS2ATIlIp4TrycTGVKXKHuYfuPXfcRT1+8Ju04vM4S/KTrJZyuerkG+kG2v33IlzzzyaxAx8/1gUHbrSPR9HMJF0OiutMWrQm0/SPDZ9C77wDCGeY8huAJqHr3HcADGQLbV0bPTVF2x8xYsrbDyd9yYW2ixknzV/Hsp+uupzvfiWXvzja+uJ0sH0QJ8421CBIgp9rmutdjGaXLMb1XJoECX3e/2N0PBk4g9hT5ESDYaGgREREpGnef5e7Hunsvd9/S3Y8HTsyy26fALmbWx8yqgVOIjRNOlpxI8iTgnUwZmt19PrDazPa32J3hr1HyyaK78JnY2M+pi9Zw2H/e4/a3U0+ZU864N7lFLN0lVMob5MZw8GA058+tzfZ3vDM1p91vGfRFjgdsX8cnRsxkSLxrcrqu1kO/WMTh/xnKPwd+DsSmWyqUGwZOTrtuVX2OwWAAh94ylENvGVrwctNJdQ0Pmrgwj3LavM+2fQd+Rm5+6/O894XWdU37M5y0uG2G9lJ2+c8kaFfnC8zsWWAMsUQVDxPLBCkiIiKdz58zrXT3MHABsfl/JwPPuftEM7vWzI6Nb/YQsLmZTY2X1zexf7pElMB5wIPAVGAa8EbhTql7KXS24gUrY7nGRs1M3ShfjtaihL2vGxyoHhUpPpNiZHXOpRW3FAH5Xteu65489IvFGbYM5spXJrS89jRn8PdXJjBt8Vqa4i29pWqYXpNiHuZC9EZYWYSAOllyz4pUtc3nDHK9tu96d1re3afvendaXvsl1zFxjgtXNXL30Kntvm/JV1pn6WHSVqDkVsTm47sVGBX/YykiIiKdV9ZbKncfSGyu3eRlVyW9bgB+nmbf3mmWjyQ2LEpKLHHTma5lpZxdDpNbEzPVI1RhRCPFr2eqI6Sd1iVNdV4fNy9lIp58Puecu8QW4JgVbaKuSNY5ZHM+RErlfAAD7bspB3X7kCktr4sV1M1YspZ/vDaRq3+6GxAbi97WcyNn8+sDehfsmEGvneTtzntqFCvqmjni/7Zil602TvkbJ/F97mzxb9CuzjcDVcDpAGbWy8z6FLNiIiIikrdOdrsh5VbioaZppZ8KpX0wBvkHKpmMmdU+yF2UlN03SCKfdNMLdbYbfUhdp7Yfa7qx4aVQqCPnGpDmkoAp+TpM9VG983lhsl4/Mrw243EKd33l/nOV+HxXN8Q+t7b1S65b4npKLOos83EH7ep8NXAZcHl8URXwZLEqJSIiIpmZ2WozW5XiazWxOX2li8vlZjFx05lun86S1TldPdxjLb6lMGl+5rlLky1dkz5pkrtzw4BJTFscLJFUujJyWZ5647wP36IYga+7s2hVQ7tlxfLEiJktr0fWLuOo2z/IuH0iAVMQyVdmoX6WMo17HTN7BSfFp/hJ1q9N1vFiS3VdJL6Hid81iYcCyVu2/YyC/mQXu4t00KzOJwDHAmsB3H0esHGxKiUiIiKZufvG7t4zxdfG7h50KJN0YiWOfUpiwtxVTF+yNuW6UAGahQr9ORx1xwdpg8LnR83hgQ9m8NtHP837+G9MWNCB2gWTqjtrrl2d8/HQsBnsd+PbTM2SYbpQhx725bopxq96dWLWBxxtEzBl0mqsawl+2P7+yni+TPG5TZ6/iqmLVhethf70hz7m76+Mb3n/3Mh1U7Mnjpg4dLvnVEkfTNCHA8+PnN1qfu5i9zwIGvg2xTM5OoCZbVi8KomIiIhILmM3s7WU5NpKlS1Yydcv7vso7bpUcW+usXAxgpJ0n91fXxgHQDhpXHKux5+zPP2cwEHllVipzfvs0xnl7v14IDq7AOfYUo8MH3DytTJl4eqsZeVyaSW3zhaqxTfTz3em1uAf3vo+tw2eknZ9TnVoU4UPvlzCkyNmZd0uxvhs1vKWbt6tWnyjif0yf1YPDZvR6n24kwS+z5nZfcCmZvY7YAixrI0iIiIiUgS53AOuG0uXJrlVjveTlzw/NrcdOsjdCzKetxxJvJJv7nM9fpZ4s8OCft+LHXBkVKBDJweLQc4n38utM4wa+LQ205Tq6T00bAazl9UVZMytGZx494fMWtb+wUbUnQHj5re04Ab92S72dRioK5S732JmRwCrgG8AV7n74Cy7iYiIiEie8hnvlu72MudWqhLf3TsFSoBThGpn+yiS79Vz/dgiBcg65p75Wkm1qm1381Ilt+roUdw7dp2MyiFgfGP8fN6YsIA7fvntonR1ztSqm+0c82l1Xrqmketen8STI2ZywE6b57Rv1J26ptZTKWWq4sczlvHxjGX8cr/tM5bbNiCuqQzaJpufwGOA4oHuYAAzC5nZqe7+VNFqJiIiIrIeS3drmzLIyZLcKp979T8/O6YoWZVTcU99I/3WxIUlOX4m2VpxI61afHNT7BauoN++cIGnkUq+Rgt5BWWqZbZznbpoDfNWrku2lSnwBDjvqc8AmLeivk0dyt/km8+DisR1msjKDLGAtqE5Qo+qUMZ973h7artlbceJp/q1NHdFQ/uFKWy/2QZ8paqSqlBxA9+MpZtZTzO73MzuNLMfWcwFwHTgF0WtmYiIiEgXcvGzY7Jmkk0nGnWWr22dbCd9xt8Uy1rm8U0t19ZjB14aPZcXP5uTddtiWd3QzNjZaebXTaMYIUm2j64jmWijBQh8Hc84TVQmiQRF4Swtz7me48r65pT7ZLp2Oypb4Lu8rvXPV9CHAiNnLs86nVGpJT+nCEei7Hv9EF4dMzfncq56dSLfvPLNrNstWdPYblnbz+/Od9sHx0Gvm4625AeVLax+gljX5vHA2cAgYpPZH+fuxxW5biIiIiJdxsuj5+Y0VU6ye96bxrevG8z8letal1LdM94zdBrH3jWs3fJcuuN2RqmCn3x6AWf7HG4cOLlD+2fdJ8cCipFNOReJKaSytTznev30fXFdZuCgvQYmzF2ZNVDKmNwqSwtu24cMuQRaucRkQb+lmY6f7XjJn8OaxjBL1jRy1asTgx04QPnBysheSrYu2cWevqitbIHvju7+G3e/D/glsC9wjLuPKX7VRERERLqntjeNgybFuvQuSOqK+fdXJrTb76PpS5kwt31wvW4e38Iktyp1PBZr8WlT9zzuzrO1Ht7//vTcy8zyWUTcWbKmkd59BzBg/Pycyi5EV2f3zC3dmeqf+MwLPcb3zYmpp2lKdRR3+HDaEo753zAeGV6bsdyO1LLtKQYJ3Fq2bTXGN3gtFq+OXRf9PmmfKbkjkr9fifMo9VzdQR4ctH14VepAt61sgW9z4oW7R4AZ7p49P7iIiIiI5Cz5tvDzBbnfchUquVUhbqJfHh28m7TTvu75dH0sxn11tmB6RV0zU+LfqxHTc8u2GynQ2Np8A4rER1zoMb4ATeH2Tfbp6lm7JJYZ+MtFHQgzck0IlWezZ7pnBF/Er4HEdes4F/UbDcTmfM5Jlou/1YOKxKYd/BaOm5PbsIIg2n7mmbrklyKfQLbAd08zWxX/Wg3skXhtZvn15RERERGRVjp6y9f2fnLSvNa3abnGRYVoALz42eBTIrl7eafUySDIZ7emMZx9oxQK0uLbgX0TsUY4GqV33wEdrkuyxnjgm7XbLuvGGCe6XqfdNn6yK+ub2yWdynactkFYTvP4tto49Se+eHVjqzp+vmA1H05bmsNRgks+l3WBdhZZNrgsqXt6oazriZK6Colg190LmgQtnYyBr7uH3L1n/Gtjd69Met2zBPUTERERyerZT2fx0mdzeDHXlpUCWdYmMVU2jwyf0XKjnCzfFstES1riBvPPz41JuT7X8kplyZomVtY3t1r2+fzcW/86Wmuz9t+DIAm+mvNsMU3Xsp5z1/SM67KPiy3GQ4dULb6pzF9Zz/CpSwCorMjcJpc4l0NvGcr3bnqn1bpsLYbtujrn0MKY3C063UfVNmbPN3FZkJ+91l2dg+2XWGuWukG5GD/z7Vt8M1yLnSC5lYiIiEin5u5c9uJ4/vzcWC55PngrYyH99tFP066bu6KeUTOXt1o2b2UDFz4zuuX9upu+PKYpiTprmxItjhYvr/VdZK734KWa1zWT3z0+Mud9inHz/vCwGVm3yZYVOZ2gwWEm7umzOmeTuEwK1eU6WVMkdm5jsmTmPvqOYS3TVlUGbPFN9aAp1xbffKUrJvEzV4gALvu5tF8W9OwKFV8G+TjbBb7pyup4dQIJPI+viIiISGfUGeZ6rV26Nu26A+MtU7U3Hd1q+YqkFs6O3Ixe9uI4XsjS0p3rVCflD3vzCwrLlTsn3xbfRHBYTBmTW8X/LWaL762DpwTe56Pp2bsGD0xKINac9PnVNWXubv762NaJx/Lt6pwugG7pzhvwo3xjQpoEYAH2T/VgquQJ6QL8lkhUM9tn01mmMxIRERHp1FLNMVlq+U2DU5g71eSgd8jk2EOAyW2mVZq6eE1OZYZLEJBlU4qgsK3U88zGPPFRbdr9mvOsayFafCH/uXATrZT5tlhnks+5TZyXPYXQf4esC6Qve3Fcy+shkxdl3K9tl/V8pzNK92Nb0abA2qV1SfsE//4E2TI5+PYUy3LV0BwpSlboaW1+76S7Th3PKct2vtTiKyIiIl1a24Q4zZEo9c0RqkMV9KgKlaQOxe5G2VGbfaU6p+2Tb9o74vMF+edC7QzdrWHd9+TKDPOk5vugoDBdnfPfN/GTU4zPOtWDiw++XNKhMt1bB5htW3FzkVOYlXTMdMFb4tdQYtPrXp+UtE9w7p7190ny+OHEprkco22Q+c0r38xh79bHzWR1Q+tW+HT7LFzVyFY9e+Rch1ypxVdERES6tFCblpaG5gh7XDOIo+74oHSVyCNuWFXfTGM4ArQekxuJet4tiOmMnbOyoOUFdeR/S/g9oOt1dS7U9znTeQepWammM+qoWcvqWk3z1ZFeAbkktxqbPE45yxjfjl6DHqCMSKauCWm0zfTeUTMyDO9oK11rbikyOSdT4CsiIiJdWkWbFt/ENCrTFwe/MeuofO51561s4NQHPm63/Pi7hrPLFW90vFJJOkN38FL4bNby7BsVQZAuuqlkC9wmzVvFyfd9lFfZgSSSW5Uxq3MuzntyVMHKSv6tMWDcfP739peB9gua1TlZoR/ItLps4mVnayU+M0MCvnyc+Uju5SXnQjj0lqGt1pUiCFZXZxEREenS2t5wJqZGKaV8uzqPbJPt2YHxc8vTOtsd/OnZMXylurDd25cGeGgQZMqjVLK1FF/TfyKf1C7Lq+yETONLE11Rm4sxxrcIY7RX5zlfciofz1j3uZ7/9GcA/PHwXbLul76rc2GyOrtn/33Seoyvx/8NphRJpNqatayO2cvqWvUAmbFkLbtunTQ7bgkqphZfERER6dL23uGrAOzUa0MALuo3JtPmRZF8n/rcyNk575+45StE19dSz8Hb2cxaVpjxyQlrmyIFLS9Zuu9VIpjJMq1tvIyOtyh2lvHU2awtYOA7d0V9yuXRqGf8GUr3UWXKXJzTGN8AW0dSjfENeBEYVpbgd9HqhozrS1ElBb4iIiLSpfXeYkNqbzqa0/b/Wrt1f3t5PGc/lvt8sLlKvln96wvjMmyZWuJGdFXSFEd516VrxDCdUr4BQauWqxxkizfbJm7LR6AswV0k8K0r4kOIhB3/NpBznkjfpTpdgJkpK3FOWZ0DPMhI9f2KOux93WDmpQnoExasamD+ysxBaHelrs4iIiLSbT398aySHKczxQ2dqCpdTrqAI1t35802zC1rdkK2Lq1tp8hJxeP/dUQRclt1aYMnpZ8b/J6h01Iu/3zBKuauqCtIa2q2b0ekVVfndZatbWLg+Pmc/f0dM+6f6fxKaVLStGulaIVW4CsiIiLd3pzldWz31a8U7wBp7lTfnLAg0O7rWosK0MKnJt+Cu+T5sRnX5zvGO9sDkyCBL3S8lb++qXBdiLu75LHByS6N9/TYcuOaDh8j289w667OrbfNJVt1Z6KuziIiIiIFcNgt7xW1/HSBz7k5ZqFNvmetzLOba2dqfe6KUrWeZuuCnm/gmy3ACdLVOduhg1TtlkFTsm8kecvl8giybaZtCtA7viiynVcpAnYFviIiItLtFSPDbLK293SPf1TL4f8Z2nqbdImM0izfqddGedUl3yBM0vts1oqM6/N92JDue5VYHLjFN91yXQsll+pblktXdMdzmse37aaFCh8bmos/nrrUFPiKiIjIeuGZT2axbG1TUcpuG2Bc9epEprWZRzjdzWxy0GStlufbfXb9DnYWrS79nMX5BpjZZhEKBcnqnOH47jBuTuagXQorZVbnHFt8swXKia7OvfsOaDfmuFAtp3e+M7Ug5QSlrs4iIiIiAWW7ubz8pfHsfd3gohw7SItfuk0e+GA6E+etbLdNocaNbkADfwy9xE8qPqaG4gT+ncn970/Pe9/nR+U3H2+hW3wTOprVed7Kek5/6JMOlSEdN3HeqpwzO2fz6pi5ADw0bEar5YXq6ryyABnmc6HkViIiIiIFVt8UYYPqUMmPm+7G96Y3Pk+5PN9gal3iG+eflQ/yy8p3W62/rvk0HooclV/hklKx5sEN0nrnnr598JlPcp9TWjom3bcs6CXiBMvMnm6+8kK1+BY6EM12TpmmgyoUtfiKiIjIeuU/g76gd98BvDZ2XkmPG+RmNjk4zrvFN36H/XL11e2CXoArq54kRGz8nlHcsc9d0WMf1ua8T95dnbO1+HbRDL3SXtCHI+7eoeEKQceFr48U+IqIiMh65cF418A/PjMaiN2QXv7SeGYuXZtptw4Lci+bfG+cLZNw+jKcDWjg2xWxMXr/bj6ZtyPfZs+G+7kzfBwAX9ScwVWVjzO25hyqWDeVzW5Wy4bUA3BaaDC/Cr3N/hWT2M4Wcf3xu7P7tj3zqlNbe9sULqt8hs1YlXabAyomclpoXdf0XqzghsqH6Elxv08Nzbk/DBg7Z2Vex0oXC61uCPPYh7WBAqDYGN+8Dt+iijA72xzW91mgz3r00w6XkX4sf8DAt+V/+emyca+6OouIiIgUz4wla3l/ymKe+WQWE+etpP8FBxXtWEEyuybfGy+vyzfwhZur7gfgrKZLeDu6D/EGXv4b/hkXVL5KpUX5beWbAFxQ+QpPhI9gB1vISzXXMCn6Na4Pn8r1VY+0KvfjusfZYbPtmDA3fbCaTTXNXFH5JGdUxgLa8ypfY3BkH37XfEmr7bazxTxTfQMA11c9wnlNF3Fb1d30sGZOrXwbgG82PEIDHZ8ztZymLlqTcvmCVQ1c3X9ioDKiUec/g77oUD1ur7qTo0KxscCvRfbnj80Xdqi8rurtzxcVreygge+7ny/q0OOHzjqdUTZKbiUiIiJSRL9++OOWAGPcnJUsWtVQtGMFmp+zEC1udUs5JjQCgPeje7ZaFaaS3g1PtVp2UeVLjOxxHi/VXAPArhUzebr6xnbF7v3RBXl3o9zTpnJAxUSm9DijJehNOCI0im1YQgVRKuOtz/tXTAJgcGQfVvkG3FN9Oz2s9YOAz3ucyQEVwYLD7uytiQt4/KOZAbb0eGt5bBRpBVGMKPtXTOLIinUtnT8NjWArlhWrut1euh+RoF2dL+o3pkPTUBnGjCUd7xUxb0Vhfxd2hmTzZWnxNbObgZ8CTcA04Ex3X2FmmwMvAN8BHnX3C9Lsfw3wO2BxfNHf3H1g0SsuIiIinVY+MdnsZfWt3r8xYQFnfK93YSqUh0LkSArNGwnAf5pPojnlrZ7Ru+FpQkTYv2IST1X/s2XNdc2nYTh/r3qKsdEdOaPpMjaxtdxY+RDfYxLVnntW6BqaeLXmqlbLDmr8L3O8FydWfMCt1fdyVOhjLq18lhoLMz7am7HRnVjlG3BO88U8U30D+9tkAO4LH80S34Qfhj7juxWf82jVv/lm4yP4etyW0xTJfNFU08yBFRPYw6ZzcdWLabfbr+EutrGlvFJzFR/3uID3I9/ipcj3eSUa6wWxGas4tGIMU30bxvrOBT2H7iTItGVZy+hIBQz+lSZhXi6GTF7Y4TJy0Z2zOg8GLnf3sJn9C7gcuAxoAK4Edo9/ZXKbu99S3GqKiIjI+uTq/hOLFvgGavHtYLPIDraQzV69mMW+CQ9mydwcIcTw6Lfo3fA0lYTZiuXMpRcAD0aObtluhW/MPZFjOTA0kVu//DEDadvF2NnbvmS292IxXwWcs0JvsGtFLdc3n8boHue2bDnPN+PwxluopwcA/aPf41bu5e9V61qhv1VRy7cqahkW2Q2ngjvCJ/BpxTe4I3xiSyD/dORwHq6+me9WfM4tVfdxSfN5HfrcurIpC1a3en9sxYfsU/EFd4ePYyFf5c3qy9ixYkHa/WdEt+If4TNYxFdZ5F/lxcj3+VnoAw4Ojefg0HgmNvbmhNAw/lDZv2WfHRueJLoeP2zIRzSHyLcjvwb++sK4/Hcuo1JkdS5L4Ovug5LejgBOii9fCwwzMz1GEhERkbL5U7/RvDJmHrU3HZ19Y+Ar1SHqmiIZtylIN+YMpW9vizg3FAtO/tj8x5bgMogwlS1BbyrDorsTCdUQijTyeY8zmRXtxU+abmItG3B+6FUurXoOgLci+/Lj0MiW/X4WGtby+hsNj9JIdbvjXtp8DmeEBjHTt+S28EkMqfkrAHdFjgfgw+jufBht3R6ylg04uekqanv8ip+FPuC28EnM8V6A84vQUP5W+TSb2lr2abiHpWwS+HPoit6cuC6o3YQ13FF9JwB9bAEzfatWQe/j4SOY7b34Yegz+jb/jhm+dbvyLmk+jyfCR/Bs9XXUWDNPV99AL2udvOvKyif4R/jXlGZkZteSruUyNG0w/auv4OLmPzDNt81YRnF/V5RHtod6C4s4zCShMzyq+S3wRh77XWBm48zsYTP7arqNzOwcMxtpZiMXL16cbjMRERERAHr3HcArY3Kb6ijI2NdgLb5BjuYcWfEJ11Q+yjYsAaBvZT8+qLmYX1W+y+rdT2dEdNcgBeXA+PDoIS3vdqhYzKPV/+IXoXe5tOo5lvrGAK2C3gGR/QAYHd2ZPg1Ptgt6E56PHMIxTTdyfvOfmOrbcUDD//hJ4z/5KLpb1lqd2nQ5AKeHBtGTtfyoYiT/rnqATS02xvFvVU/Tg0bWj2zFzi1V9wHQ4FUcHBrP6ZWx79kxjddzQ/OvuDl8Mg9EjuHkpqtSBr0JY3xnvtH4GI+Ef9wS9L4T2YuDG28D4MzKt7i16h56sbxln91sBt+vSN3auCmr6WPzC3KWnd3CVY3tlhlRNhz8V/aomMHbNZdyVMWIjGV0hvGwhXby/ZnPuaIEWbmK1uJrZkOA/5di1RXu/mp8myuAMPBUiu0yuQe4jthvseuA/xALoNtx9/uB+wH23XffbngZiYiISLn8+Lb32X6zDQJtG2ge3wxbbclyVrIhvw4N4oqqpwH4QcVYbgv/nHMrXwNgqW9M3V6/h5GzAtUpF81f2YohW5zOD5c8AcB3KqbwnYoprPSv8OPGf/PPqgf5bsUkLms+hwnem9m+FRc1h+PnFbytZT6bM983D7Tt8Oi3GB7Zjd9XDuD3lQMAWOs17Nt4D3dU3cnPQh/ws9AH9AsfQt/wOTmecXYHVYynhiaOqBjFT0MfMTq6M79uvjxtN+CTQu8xM7oV89mMOb5lwerRkzWMrvk9IXMW+Ff5dVNfBtVcBsB3Gu5mMZsyIbJjzuU+EjmSemp4OnI4c31znAq+03AXH9T8iRNDwzgxNIxJ0a/xfORgrq6KXRcXNf2B16Lfo2/lM2xvi9jeFrN7RS0AJzZew2f+9Q6dayVherGS+WzGQRUTcGLXQWf2h1B/QqvnMDO6JV+rWMTd1XdwV7iWu8LHU5eiZ0Z3DHyzqezKga+7/zDTejM7AzgGONxzHNDi7i2jrc3sAeD1vCopIiKynjKzI4HbgRDwoLvf1GZ9DfA4sA+wFDjZ3Wvj6y4HziI2Sc6F7v5WfHktsDq+POzu+5bkZOKKdbNY1xSmOeJsskFVu3VfLFzNFwtXs2F1KGs5QW53lq5JnTzqrNAArqxq307Qp2JhS9fW7zTcxRI2YXDPPkDhA1/DGLL1OZw95ydsQAOTe8TaHO4L/5QlbNJuSiKIdWUutuHR3TkwtC678/nNF1FPD24On8whFWOpsginVA7l7ejeDI62vyTNWl87PVnLVracL327jMc9tmI4d1Tf1WrZQaGJTA+dxvDIblwfPo3J/rWWdX8MvcQlVS+0vP9ewx3MY4tA57gRdexmM9nU1vBR9P9opLpVC/p5la8RsthJnN90IVN8e05v6sv4aB9WsHGgY6Qyy7fi3+FTWi1bzFc5oela3qiJtbbvWjGTqyueaFl/e/Xd3M7dKct7qeYaxkR3Yq+KaXy/8TZm+1bxNU5P6ljFhu322dum8ET1P9nQGnkgfBS/q4zlsx0Z/Tr7VkwB1gX3nUmICJVE6G0LuKTyeeq+cSKHjj2R34de57Kqfpxf2Z/tbTEXNv+x3b4dHevfFZXilMuV1flIYsmsfuDudXnsv7W7J/pLnABMKGT9REREujMzCwF3AUcAc4BPzay/u09K2uwsYLm772xmVt0mVQAAIABJREFUpwD/Ak42s12BU4DdgG2AIWb2dXdPDHA91N2XlOxkimzZ2ib2vi42/U5ivO+0xWvYYqOaVoFwkLw1Qba5fsDkdsuqCHNB5autlv288SrmsxkvV19NL1vJr5suiyeWKt5NsxlYvEt3PT34fuNt/LnyBZ6PHFyU4wX1eOQIelode1dM4ZrmM5jkvQGY4tvzjcbH+G7FZJ6pvoEHqm/lhuZf8a2KGQyJ7E3/6IEpy7uj6k4OCY1lfLQ3xzVdH2+9dfpX/52Bke9yb+RYjGiroHeOb8Hfms/i8ep/AXBgaCJvhC7nD00XMjC6Pz8PDW0V9AJcUvUc831zKolwU/hXac/PiDKhx9mtlo2N7shxTdcBxiEVYzgv3uJ/V/hYPvNdAPggukcuH2NOJvvXOLzxZhqp5qCK8dxU9SAAlzaf0zKHNMRafxf45oz3PuxTMYVrKh9jr4ppADxVdSNHNN3MhzV/ZHOLJehq9CoeiRzJ/9lMhkd3429Vz7Q6biLoBdi3YgpfVH6db4Sn8GmPP/Bg+CfcED4Vx9iK5SxkM84L9aeeah6NHFm0zyKdB6tu4dDQ2Jb3y35wA9Gxo7kncixL6MnNVfdzdMUI/m0nt2v9v7T5Xk7oEUuJ9Gj4R1wTPoOSjqeeM5J7qm4jRJRrw7+Oj58vrlKMay5XVuc7gRpgcPwX6Ah3Pxdanhb3BKrN7HjgR+4+ycweBO5195HAv81sL2K9hmqB35f+FERERLqs/YCp7j4dwMz6AccByYHvccA18dcvAHda7I/2cUA/d28EZpjZ1Hh5H5Wo7u01rITxL4AfWvCij7j1vXbLDv/Pe+y85UYM+fMPWpZFAgSb+QSkX6GBSfGW1UuazuWQ0BjuDh/X0pJ4SOOtbEwdC1jXNThIXfJhZiT3RpztW3Fx8/lFOVYu1vAVbgr/MuW6KBV8FN2NGdGt6FOxsKWL+LGhjzg7OpArm89kPDu2dMXehiUcEg9WvlVRy+SaMwHnlchB7FExgz0qZrCUnvyr8gEAbmo+pSUQdiro3fAUt1XdzcEV49jcVnN39R3cEZ7Fb0KxIOaSpnMZ5ztyfuUrrRJ/DY3ulXJc9obU82g8mE5o9Cr2rJjOYRWjeSe6N78IDQXgwIbbMyYoK7REgqZ+kcPoFzmsZfmLkYP5tn3JKP9Gq+0/iO7BCU3Xcl/VbRwQmsQOFYv5osdvWm1TY80t3fZ/EGo9XvjB8E8IE+K96J7U0MRxoQ/5W8NZXFX5BL+sfJezK9/g7Mo3iLpRYc45TRdzWVU/AEZEd+Vz36HQH0Fah1SMaRX0PhM+lANr1rVIPx85hLHRnRhUcxnDav4EwK+a/saH0d35us3mhOi6PMC/qRzEK5GDGFOiKaQ2pg6eOomfhGLjt/evmEQdPVjgX+Wi5guY6alGsnZct23xdU//nXOPP6Zrv/zspNenF6FaIiIi64ttgdlJ7+cA3023TXz6wZXA5vHlI9rsm0hR6sAgM3PgvniejXbM7BzgHIAddijAzej452HAJWy/53+Bwo2bBFi6dl3X43FzVrRkbp66aE2r7YIEtU+OmBn4uN+1yZxZ+SZHhj4FYJVvwMvRg3gx2rp1dS0bsJbWY4yj0cxlf/P/bcznbabACaLCIFSCcXjFcEHzhVxV9QSLfRMW+6acHBrKHhUzeLXmKl6NfI+Lmi8A4OTKd4m6cUP4VK6sepIaa44vH9pSVqJFs85reCzyIyB5DLNxcfP5VNPMMRUfcWv1vVxY+UqsDk1/5PXoAQC8EdmP40MftpTZr/p6jm+8ltVs0BJQHlExkgeqbwVgfLQ35zdfxCzfkkoiDK35Mw9X38Lvm/7E4RWf8XT4sJIGvZlEqWgX9CasYkN+2fx3NmyuZ2KPs1qW79rwMHX0aOk+/nrku3zd5vBedE+eixySstv50Oi3Abg8fDa3h09kRI9Yl+GKeJfv+6tva9n2v1V3McH7cE/4p1kzKndUFWFuq4p19T6p8SoWxMdzv9vmd8QU355pFX3YKToDgKerb2Sx96SXrWrZd4ZvzQc1f+LGqoc4tum6ogwdqKGJgyrG807022zFcl6vuQLqV3Fs43VsZPU8XX0jPann/9ly3qv5Mw+Gf8J834wPo7u36srfUaXo3F2uFl8REREpn1TRS9v7jnTbZNr3QHefZ2ZbEuvV9bm7v99u40Inntz7DPj4fvb78lYquLFo84see+fwtOsiAfox3zJoSqDjXFn5BGdVtp7wYt/GewOfVzRLEJ5v8GpYoOzVndFE78PJTVe1vL8ufDpvVV/GLhVzOS70ITeHT+bnoaGcF+rP0OiePBQ5itciBzC45lI2sXWj8v7dfDJ/rXqWj6Pf5JSmv6dN2tVEFS9FD2b75sVcXPUiAB8ltei+Fd2Pi5r+wDcq5nBS6H22tBW8UhOr363NJ/F05HBuqoq1Kq/1Gk5r+hsr2Yj/3955x7lRXA/8+6RrPvdz7733brBxwQWMTQkYTO8QO0BimoPpkJAYCIRfQkJCDT20kDj0XgMB00voMdVgwAQw4LuTNL8/dqVbSbvSqp108vt+PvpoNTszOzO7q9037817YK2bPq9xb35XdTF/rroIgIcj4/M4WoXnO1oxYMt1jJP3WG+6xRw8rY3MYO0WdxN0b4RP6cTYLZdxYeUfeT4yhDr5lsPte+i60Dz2r3iQ4XzI0uBjTN5yCV8UMMTVEcE76CibOaPxINaZ4bF0t/tyZZvzmb/pepYFH6G7fBUTei9sXBor+/fwDPateIgTKm6mkQr+EtrBd4iujnxDhEDs2knG8ET1z5LCVa0P9uNlMwgMXBLamYGygaciIzmz8prYuAKc27g3l4R38dWWUkAFX0VRFEXZ+vgI6OP43RtIjN8TzfORiFQA7YFNqcoaY6LfG0XkdiwT6CTBN+8EK2HuatrfcjDzA89xX2RKwQ+ZiJ/1uwD95FM+NXWeoX3a8n1M6D2uYTmPRcYhGBpIdqzl3ZYCCb7iHaO0pREhwJKGcxgj73Fr9dk8Uf2z2L6LQnsAsJGOjKu31q5W0Uh7NvMNrXnd9OXRyDhfnqovCe/C8MAHXBlalCSs/CMyEyJwYWgpD1UdT9+AFXbzuMpbOc5eD7y68TBuDM9LqndtZFsmhN7mkIp72Wg68EBkYnYDUUQMgbya735Daw5vPBGwHEvdF57My2Ygrahn/4oHY/nW1axgmy2/5yvasIXqvB0/ys5Ba9VH1BogSsTlT6IxUM1vQ3vy29CeTJE3GBl4nxoauCK8KJbn5NBhLA0+yvIKy5fvIcF7OLrxpzySYrKjhnrCBHmhZjmbTQ0rGlfyranlRTMI59zlVHkjSei9PjSPM7ccFPt9rr2MoDNfs7LiNq4NL6CXfEEf2ZjX6645HHqp4KsoiqIoWx/PAkNEZADwMZazqkTvOmuBg7DW7i4FHjLGGBFZC9wgIhdiObcaAjwjIq2BgDHmW3t7IXB283QHGL4z39b04LDI3dzX0PyCrx9GyPsxT7jrI91Y0biSVtSzNPgYvw7ty7fUcnrFNQDs03CKrzi2bqQTwrPV2koOZUuReqpYZ4ZzU3gOy+x1sleFdrA0XQk0UBlzHhY1sfVDA5WsaDw2ZZ4QFcxquIhqGhkoG2LXyNOREdzoWDubyFmhgzgrdJDn/q2ZMEGeMSMA2EI1o7ZcQSUhXqyx3AI9ZZtFL6n/Ja+azMI8DZGPaEU9rWjg1Mpr6SVfcEzjMTwZGUM3NjEi8CH/F/oRicYxbmvvnffTs2Y4z4aHJ+UBYZ+GU1lVeRPrI93ZJfgv/lJ1HreGZ9GKLXxiOiMYbg3P5g3Tl3mB57ii6oJY6TayhWur1rjU28Su9WezX/BBLg/vxFumj2ueL2jP+PrLUtaTC2W7xldRFEVRlOJhr9k9GrgXK5zRlcaY10TkbGCdMWYtcAVwre28ahOWcIyd72YsR1gh4ChjTFhEugG3204rK4AbjDH3NFunghX8p88+THv7QsbJO7zUTI5gMiGqCQLoH/gsJuAA7FvxELvXn8meFY/xdGRE1kIvpDe7znqdbgte45uK1aEjWd14ON34ig34ix+cf4R6qviP6cfYLZfxHTWESR8iS/FHdB38nPoLeKS6KezWHdWncn94Ulworp0CT3Ncxa3s23AKG+3Jjuhqjlrqub96VVL951Zexqz6i7iu6tdWveFtkvKEwsn3pd95pOfMsJip/jXhBdxZfQpLg/HGNIdX3M21ofkcUPFAXPrNodkMC3zIEPmYWqlPqvvfkeG8ZAbxUqj0/jPzjQq+iqIoirIVYoy5C7grIe10x/YWYE+PsucA5ySkvQeMy39L/fNmr90Z+NblrKq4if0aTyn48S5//D3feXvLRg4P3sk/w9O5LrSAzdRwZ7XVxrvDU1gUfJa/VZ8JwHWh+Tm1y2kyWFsVpLaqgi82N73wZiu7BkTKxtQ5kQiBIgq90KoyyNje7fn3fze5xrJV8sN604OJW/5Ef/mUQYFPOL/yUhYEn2N90DJ4eT/SlX6BjQA8U3MU5zUu45Lwzvy3Zn8AHguPiavvjvA0vjOtWFbxCO/Zee4JT3F1xtUYTvY6l40FxWtmALvXn8m84PO8EhnIhMDbbB94kSGBj2NC79L603nODE0yxx8kHzNYPmFdZCi95AveMr0LYu6dDercSlEURVEUxSeNFW24MrSIVZU30Sf0GR+abgU9nlvMXTd2DvyL31ddDMBlocUxU9r+W66PxRv9ubmRFRX/5J7wFO6OTM2pXU6NrwCVwfiXa8nB1FmaM5ZoM+F3fXahGdXTEnz9UFMZYEtjGvfdiiubaMcm047nw0N5IjyGK6p+w8iA5XE9KvT+J9IHQ4BVlTdxQMX9sbKzgq/wSqQ/V4V2pIN8x5XhRVQQIigRlgYf47VIP05tPNT1uI0uGt9AlrNQz5uhPB8aCsA9kamsYR/asIWJgbepojHOqZaTd02vmFfrL03hHHxlQ3Os8S2M20NFURRFUZQicHt4JmEjXF15Lv3k02I3h2HyQUzoBXgtLmqj8Bl1gOVApv+WG1jeeGzOJq7LLn065f5sRddAQLLWFiup+d0+/tcND+zSmiVjexawNeXL46viY31voBNLGs5hypY/8OOGY3k2MpQF9eexqOFcljda8XV7iDUZcV7jMgCuDu/A3yKzuNJ2QBWighMal9N/yw0sbvi1p8dod41vfvplCPAttTwaGcf9kcn5qbSZaY75JxV8FUVRFEUpGzbQiZvDcxgY+JT7qlbRPK9T7lTTwL3VJwFwe3gGg7Zc2+zrNg3JTmNycW5VKqbOQ7p6hWdpeew6vicLRnbD+LxWpw2oY9fxpSv4Du/etthN8KRTm2Rv6hECfE5H7o1MYc+GM2Nmyh+YbhzfsJwHwhMYu+Uy/hxewl71p3FreFZSHX5oyJOpc7ny/pffp8+UIyr4KoqiKIpSVpwSOozHwmOolhC3VJ3FSFlfhFYYDgtaYYnuD0/k+MYVRXFWVBlMftXrW1ebVV2l9I5eItbJeSGbYd1uSJe8t2NrIFNB87bILA5vPJFvaO3wFJ3djdAYKpzGV/GHCr6KoiiKopQVEQIc2ngin5t2TAm8xV3VJ9OJr9MXzANChCOD/+Tt6gNZVXkTj4dHc0TjCUSK9MrVqXVVkibx1CUjsqxNSkZD1RzrAZuL6Jpr/+unS+MceFHKp6aYl6/bGt9s19sr2aGCr6IoiqIoZUeICubWX8iTYSss0HmVlxb0eDsH/kV/2cCy4COcXHkjlRLmY9OJi0J7FPS46Vg6Odm7bG1Vdr5NA1I6IlcJy1Zx7Diqe9o80THdd5p7/FQlfxRz4qYhHE5KU41v86KCr6IoiqIoZcGY3vFOZTZTy36Np/Cv8EjmBV9geuD1vB9zuHzAXVWr+X3VxTxUdQJrKi8HYFXjEcyo/z3PmWF5P6aTaQPqUu5fMXtQUlpAYFCXzEPmlJR2qoVIvn86YFL6TPawDu7alsVjexS2QTnQta2/sDd+1yoXg2IKvt83uAm+JXRPbQWo4KsoiqIoSlkwpX8dz5+2gKsOnhKXvqJxJV+atvy16pe0Y3Pejjc98Dr3VJ8UC4USEOuF/9LQYm4Oz01VNG9cfWjq0EduwqqIUFWR+XrjUtJOla5olTlOE+cSGuIkNn5bnz4TpW3qXMxr+LbnPkpKy0bwTXfPK96o4KsoiqIoStlQ17qKvp3inTd9TRvOD1mhSE6puCEvx6mmgUsrLwDgnvAUxmy5nItDu/JIeBwXhPbMyzES6dwmWeMW9PEmHxVErjp4Cv+39/isjy9IyQic5bXG17md/nyWupKwlM9MMa0Wnv/gf0lp2TSnTXWQ05aMzEOLtj6yW+ShKIqiKIpSori9S/41vD2HBu9mSfApzg4dwHe0AmDXwBMsr7iDEYEP2LF+DW+YvinrXhBYx/zA88wIvko7+YEDGk7i8chYAH5jC9eFwsVBsy+NUVQQ6VPXisFdu2Z9/FISuCKlLF1lSKkM61FzB/HJ/7bwzQ+NPPjGxqzrKadJiUKTjcbXGOjsEpZJSY9qfBVFURRFKXt+t88Evp13Hq2lntdqDmOAbGBG4BX+r+qPjAh8AMA91SdxRPCOpLJd+Ipx8g6/qLiSy6ouZFnFIwiGVY1H8HhkTLP1oSKQ/NqWiemmU9vlLOZHezRzcGdG9Gjn/2CKb+I0vn7yF6gdNRVBfrtsPF3b1eRUz2EzB+apRYVn4chuse1ixIbOxvQ6VE6zPs2ManwVRVEURSkrogJe66og39kOZXYZ15OvvuvMkw+MYkbwNR6uPp7PjSXIzay/iG0Cr3N+5aWcUnkDewUf5YLQnoQI8qbpzePVx8bqfiA8gcvDi3khMph6mlfr8vH/fkhKy8R00ytn/07JcX2XjO3BHS9viP3+9e5jfJlVgxVCaWi3tjz13pe+25YppexAKVP8hzGy8xdI8jUuW9mw77S+nHz7K7k2p1kY1bM9973+GQCzh3bh7Y358wHgh2yE2JBLWCTFH6rxVRRFURSlrIjKBYlCoQjs13gy5zTuC0AX+YbTGg/mI9OVW8JzGLPlcm4JzWJI4GP+VHURl1ddECf0Xti4lCMbj+fpyEhXoXe38T0L1qdciFqe5rK+sSLov+y8EV0LbhYdiaTPU0qm2amIX+NbvHZE7Avly80NxWtEM+Mc76DHNe53wicbtjQme3pO5O9HzYj73ejn4k/DUXOTvb1vDajgqyiKoihKWRF9mU18XbU0a8Jl4SXs37CaYxtWcF14fmz/t9RyYmg5+zes5uXIAO4PT2STacMD4Qn033I9vwvvTiTFq1OF2yLcEsI5HpkKWFEz61JZvhnx0ZBL9puY9+OeunhE3uvM9FxkqiH2S1T5uN2QzgWpPx0DswixlSvOkazwEHDb1hTOQDadxnd497aM79MhvkweNL4n7jA85zoSWTQ6fczqYqOmzoqiKIqilBVRwSBJoHD8fiLF2twnImN4osHaHyRMmAB+VlZWZqAVLQaZCFiJr9ZVLkL9toM68a93k82Zvd7lp/TvyLPrv/LfiBSEfZiIzh7alQUju3G/bcqaDyozmNxYOqm3z5wlEs7InkzYf3o/nv7vJu58eQMzBnfiyXcKZ7LuxO0aKzTOe8JtDT0U9pykm7+5Z+WspLRQOHeNbyH47bLxfPrN07zg4r26VCjtqUlFURRFUZQMia7/DASSTZ0zJUwQv6++hTSJzAe5aArdTJ1nDHbXDHodJZ+aykSN7wHT+/GLXUfFpQUC/jwMJ8Z9dqNVpRX3OBgQblm+DV3aJoeWSuSY7QenzQOZhzMqFBGHSfwf9p3I+jWLmdi3Y8GOl6hZLkbfjWka/+pKd7EoUeOa3+P7097ecczM2HZjiTq3CohQk0V88OZEBV9FURRFUcqSYOIa3wIfz0tjVHysF2WvtaR+5I2o4Ot0KuVVzrO+PJ6AxHf/X+w2mv2n94tLC4r4Cns0okc7BnZObWbr7NOU/nW0b1WZtt5+nfyZ7orHdnPj5jAs2/b80YeZeaLMV4wwSN/Wh2L/E14a5461hXNi57fHzrBHuWp8Hzp+dk7lc6WYE4Sl+g+tKIqiKIqSE4kCWDqN0vELhuZ0vFI1dW5ybpVJofiflS5CfeLEgtsxC4XbGt/E8xsQ8bUWWKyl36nzOPPmmYzX+PrIn2pdqtd17meS4PrDp6XPBLSuTr+a0s+5KTSb60Mxy5CqiuYXi7IZgtqq7LWqtVVBBnYpTNgmEe9r06nd9/rfOHJW4cNgqeCrKIqiKEpZ0WRSm5nG14/5aiqCBdb4nrjDsJzKewn+3zek9yybaDYO8VooP+n5lBkjPqS0QMCfxlckfdvCtoQSvbbyqZ2MqypPg7RrCg/jvTq2St+OAjBraJe4380l+NZ4mDADHLdgaEwQq24GwTdR25k4Bt19xFDeYZS3E6mTdxrO46vmeu4v1tTcL3YdnbYR2w7qVPB2qOCrKIqiKEpZkiirpdOU+dFSpaLQSxTrWudmchlvUtv0a9N3yeFr/MTJTWXq7LYvn+Pjd5mjHwFZkLTWAImedPMpsv3gCGmTr3XQqepJHJL5I7oBhTc17pEg1O0x0a/zr8LRuU117H/CU+Obx+s20Slb4rlIZwY8vHtbRITthnRx3T+pXx196pLjcuebExYmWw0I3ve4czJs2oC6pP2VQWHOsK75ap4nKvgqiqIoilKWJGoe0wkV0ew/np2dyV2htSmZeBR2En239nopndQvOwdGXppdcNce5tO5lR+vzuBPq+hH4xsNO1OIyY0fXDTuqbT7ucqniQLusO6W6WuhNbCJkylzh+dH0PGz3trJmF7t435HrRmyvb9yIfFcBAPC0G7pTZG9JsGyWT47b3hXOrfJzNpl+ezM4gA775sL9xqftL+msnmcYqngqyiKoihKWZK8xjd9mfVrFrN6UXKs1huPmJ62bLjAgoMAtyzfJmvNr5fgOapne9d0N5xd9H7J9jB1zqPQ6EcjDf6ExFQCfCKFmNxwCvHRpnRu432Ofa9b9iCxeJP5dtpqPfP8aEKv9IUTyGYs27msXU43HonX/Y1Hxt/L0fNfqPjIqUhsekVAMroeE4mWfeaUeb7L7D+9H+1a+bd2mT20i2vMcr9eud2O1VzLvVXwVRRFURSlLMn09THVy1dvj3WRTuoy9P560bJkzUc6pvSvo0NtZhquKJl6ck6H27rffNWdjohPx7Z+JiME/8JvLFseX9TdlNepTZX99cmLxEmDaJ9yiZKTzSnPRsB76YyFSWmZCk1tEpY0RJvhdzIlnyQeM1ePx9Ex7drWfa3wsQ7HZnOGdYm1IZOjei5x8FnGzbmVXwuOXFHBV1EURVGUsiTTOL6pXr3c4tg6mTusCzOHuMe19WJkz3YZ5c9VoMy3PJqpE6to9tG9Muu3G36164O6WCGF/nLIFE7eabhrnoBIswjrXjjNXZvk6vj+tXKYgvoRElJp37yKuwp+BRyYbARft37ly0S7VwdrcmuvyfFrjwupCXZb4+vWx062BUCig7BEUvnXu3X5Nhy+XdMyjuj4G1N4qwdnn5zC/Z0/teITb9MMjq0AcvPioCiKoiiKUqIkmTqneWVL5dwnnSZmUJc29O6YmVOZTJU72cog0X45Xz4n9u3Iyx99HVe3s/teQ+FM9hR8PTVC1o7eHWp59eNv0jc8BX41RGfsPIpFo3swa2gX5gzryq/uesOtYb6JmQX7L5IWp+AW0z4mHKB3x1a8vXEzANmGcT1h4VCmD+zEW59tjkuPCds+PWC7kTgR4cdRluRJ/Zar4BvtUpvqCtavWWynCTet+9A1/5KxPbjj5Q0p6wyIPw164jhVBMXVmqFbuxqePGl7uqXxPJ+NABsx+YmtGwiI539snGM9u42HzhjAqJ7teeC42bFJh0KjGl9FURRFUcqSpLWMObzbVflwfJOpkx2/a+KSytnfdxwz01d+N+dWic6TnC3pWFvpc22sV/tyM4GOCh/5oKYymF5LlpGdp/WV13BGcdWnb0y2xz56+yFM7l+XJCjW2HFh/QiQ0SyJoWeizr8OnznAs2yicJXLWlYnuVrJRu9D5/0YdpmMAGuNcT6dYCVO4AQDAc/7pFeHVq5ra52kGlMvnweZXk9+/7ei2ly3Y69fs5jTdx4JwOCubWiVQ2ziTFDBV1EURVGUrYJcXrML4fE10xf/RKEoVXxS9/JNpDLdPm5hvFDs5WE4U42v3/3NjUj6cEbbZWjGnglugltikvO3r3XLqZxb2d97TurNoyfOia15dRN805mtx9pkh3tql2LyJ/F6yVbJ+PiqufGhkFyG47IDJ8e2011v0XY48znHIp8TaIl8uTk+lFhQcqs/kzGNXvOGzCbhojmvOXSqS51N207zfEE4Y+eR3HDENP8NLAAq+CqKoiiKUlZ4x5dNZ+rsvS+d4Ovl6CllmWYydW4q31RBUqgn52/HQFyy30SOmjvYtb5M+5ythrvQBCT9pEhMK2j/zkRHtvvE1B6PjYd20SuPHw1nKs1xtK7qygD9OrVuEoB8COBedErhhRpgSv+OacOLHbdgKMGApPVa3qeull4dmpw3uQnsC0Z2S9fkuJZA/D2RKv6zL628zyPXJYzb2N4dclpT7Lwnrz50Kn85ZIpjr/vEgzEmQ4HZ+h7ft0PKfHGWDAKHzBjAtoMKN4HkBxV8FUVRFEUpK7wE2Nw0vqlLZ1N3xyzDEmWLs43JQkjmJI7JwM6tU9Y1y9aa5lv+3Xlcz5zKC/6dW6UT3o+dPzQpzcvDbhRXgTMhbZdxTcJzKqHMi0dPnJNUdyBBmM+02mjIpdfP3iFJWE2s6pbl2yaZOicO5cJR3Xn3Vztxy/JtWDop3sFUIuP6NAld6cMZpSbaDmfzwiZ5v9fvbPn7UTNYPKZH7PfpS0ZyyuLkUGqZ4LyvZ9vr2tPljZhM+2RlbldT6eplO4rzOi3DjxZWAAAcLUlEQVSVKS8VfBVFURRFKUsyfWFNFc4krbbYb6MctKtpnjXBbnJBqpoM3mPhrKsiwYXsITP6A5Z2PLH81AF1SWFk8sHVh07l9/tMyKkOydG81MnP5g9JSksXJsfVuVVCmZ/OG8x5S8cm5fcisT/9OrVOOl40S5OwlN7U2dmu+46dzX3HzqK2qoKxvS1BNJXHbr/346AubVjlYV4fZd6Ibjx24lzWr1nM0G5tU+ZNR/I4eI+xIX9C3Pg+HeJGfHzfDlQGvdf4+iGV5tZr/DN1DuasJ5Vfg5BzJqVEJF8VfBVFURRFKUuS1+Zlb+pcCuT67pgqjm/i7+hYpBqyRA3eXlP6cNjMAa7CX7q6siVTc3E3J2Ui8earPdt7a2hjImIm10qavE7BY79p/QCSNHUiQrsaa+LAVzijVM2JnduoUye7HRl6i65rXRUTOheM7MaTJ23P9sO9TYzd4rd64WeSp28ny4v69YdP44bDm9aOPnDcLN/HiT9m07ZbiKkoftbm+70+3Dyp53KbZObVucnEPRPzar85w3Ea39KQfFXwVRRFURSlrMg+7E/2x8ynl18vcg1nFF+X93rL+OzeB000da6uCHLakpG0zVCTnQuZOghr1ypZ65xo6nz3z7wFJy+NbC445djRvdqzfs1i1/AuTaapuR07pvG1+xJdv97oI05SKgEmXUiaTELmZHJaO7WpZtvBTWtHB3dN1gCn0kzGTL4dB102pa975ozNglPjvI5C0fHP4QBu6+5rPTwmN13LmR0yMe/NP96GKw6anJTPqfEtleX9KvgqiqIoiqKQW2zWYmqL/7T/JF/5MtG69OpoCTFugmKUTASZttUVedX6DLM1jd1TaGfdcFOW+nFuFcUrzq6TThmu3fY7aRIdb1/OrVJ0aNmUPuw4qnvMaVl1hSUO1IeSBd9sBZbpAzoxbUBdXFqiUJbqesi3nPSvk7bnpdPd16O69XF2ihBYeXXS5jiXDVkGaD5hYdO6crdbcoh9ryTuanJqZjIa78TzNnVAHfNGJGv6w5FIk8fsDOovJEURfEXkfBF5Q0ReFpHbRaSDnb5ARJ4TkVfs7+09yteJyP0i8rb93bF5e6AoiqIoSkvkv7/eiZN3Gu66L50A8typ8zlme3cPx82Bl6AwY3An1/QoXr06eNv+cSaisfzG8PMdh/PH/Sam9MKaSYin8/ccl9e332PmDebhE+YwqEubjMpFz/HffrJtLE18LPLNpOn3HjuLu366ne/8fp1KRTWTvkydU/SnbU0lfzpgEp3bVANQlULwTcSvprtVVZCbfrxNXFqiUJaqrsqK/IkoIkLr6gra1zZpffef3perbI/HTebrPuIYk30YJjec2vuoxj3T6gd0broHXP8jPPo1ooc9edSuJqMZDr9Z+3Ssjf1HlIpH92JpfO8HRhtjxgJvAavt9C+AnY0xY4CDgGs9yp8EPGiMGQI8aP9WFEVRFEVJiYhkrXns1Kaamkp3s8HmUPh6vTv6FkATyp+5y6iYiWhiXN+ayiA7OTzORnEKK341viN6tKOudVV262M9EIQBnVunz5hA9ND9Hc6ekjS+qRwE+bh2OrepZmRPbydPifg1XY6Od6Lgu+v4ZK/WmVzhUcE3E41jNvdQ4hrfVN1uWwBHaE5+udsY5trrqL3COW03pDMzB3cumFfnxGM2hOJN0P2y05juvvIlCp/LZw3i9p9sy7SBnTLT+PrIfNXBU+jaria2pr40xN4iCb7GmPuMMSH759NAbzv9BWPMJ3b6a0CNiFS7VLErcLW9fTWwWyHbqyiKoihKy6FXh1bsMbG3bxPgKCXu2ypGoqCbiebVi2qHhs3vOPhdX5vkGTgPA+21bjEdO4y0hIRWjgkMp0Cwy7ievjzjZtKHdFn91hWddKkPhePS1+w+NmW5dGNVbV8/DQn1guVsa3K//BhWZqL1ExF+s+e4/Bw3y3LXHjaN61wsIvJpsu889dlqfEWErm2r7e3Ux3ASCAgT+nb0LOd5vFQm6gkO06Ka+1zXpeeLUljjeyhwt0v6HsALxph6l33djDEbAOxvzyBVInKkiKwTkXWff/55XhqsKIqiKErpEggIF+w1jtG92mdWMCfnVtb3z3ccHgvrk2/G9rb68+cDJrF89qCYmW8ma2298CvEjujepMn0uzY1WnX0pTibl+ADpveLbf9it9HMGea9BjMVv/zRaJ45eR6tEoTBaBsP2rY/rauatI2dWldxyIz+BXXOk24cZ9pa+aiw/kNjvICa2BeAFXMGxbZfO2uHlPVHy7tNoHRpW82tK5rMwqMhrNyOmY7FY5MtCFLhjOU7qV9Hxjti92ZCuqst3bn9viF+vHO9FpyWCmMc/1ENPkzNcyFVs/N9eUfHPOoArzFc5oKviDwgIq+6fHZ15DkFCAHXJ5QdBZwL/DjXdhhjLjXGTDbGTO7SJbs/SUVRFEVRWi4da/15Gc7FU+9eU6yX9BVzBnHGzqOyrifKMdsPjjNhXL9mMb07WuFb+nVqzUmLhvvXoPnolt+eO81qQ34Xp9pEW5uNoP6TuZYgN21AHQdM75f1msHKYICu7VI7xHI6YXrutAWu57PKZQ3qdYclawcBdhyd2hQ11TBOHVAX0zq2qrKO+UNDsmbWydqjZ9ChtsnBVrqxGt+nAycsHMoFPjSs0wbUcdyCoazZfUzavIkM7daW9WsWu46dF1Eh8bYV2/L3o2ZkfEyAgV1Sm8Sni2f75eaG2LYxxnOSKDoxlY47jpnJs6fMB2C3Cb2YajsBi2l87fpPTBPL+Fc/GsP8EZ66vxhRb9upNP/bDclARkq5FCCe05aMpE11BR1bN5+n91QUTPA1xsw3xox2+fwDQEQOApYA+xnHVJeI9AZuBw40xrzrUf1nItLDzt8D2FiofiiKoihKOSIiO4rImyLyjogk+coQkWoRucne/28R6e/Yt9pOf1NEdvBbZzG4d+UsHjhudlxadaX1+nPU3EG8fc6i2ItnH1uwBJg+sI7dJ/TydYwlY3swvLv/NZ0Ae0zsHff74n0n8Piqudx4xHQeOG4Wxy8cxh/3y8xU24sLl41ndK92tEmxbnKfqX1i26kUkCLCSo84vU6cdUQ9MEflhZrKIDccMY3nTp2fJEDd+dOZcb/b1VRwxHYD6N6uhjd+sSPXu5ieupEutE4iv9h1NNMG1DHKXps7rnd7ls9u0pomhry56uAp9K2zrpej5w7myFkDPZ2MTezbkfVrFnse20vgevaU+Vxz6NTY76ips5vge9uKJkdSmZq+iwhHbz8k7YQAWJMCP503hE5t3FYiJlNVEaBzm3gv13tMtO4rPxMgN/14etwYpKJbO/c2/eWQ1OXX7D6WkT3a0aO9+zWzasdhsWt45fyhHLxtf/rUtaJPnZW/Y20lDx4/mxuOmM7aoy3hfEwKa5PW1RV0advU1gv3Gsc2AzuxyF5THx2VKf3rXEo3se+0vlx+kOWgK9UU1HlLx/KHfSfGvDu78bN5Q7jjmJkM6dqG4d3j80Xvueh/RKqz1s4OGxVd27tkbE9ePWsHqiuyW5qQbwq7ctwDEdkR+Dkw2xjzvSO9A3AnsNoY82SKKtZiOb9aY3//o4DNVRRFUZSyQkSCwB+ABcBHwLMistYY87oj22HAV8aYwSKyN5Yl1jIRGQnsDYwCegIPiEg0nka6OpudYd2TX/b2ntKXLzY3sGL2ICqDAW46cjqvfPw1Y3s3mVL+9UhLkPjbCx8zskeTUFvnEqomUYgFGNmjHa9v+Cb2++Bt+/OXf60H4KUzFibFFV0y1tKk9qmrjUt3lkvHoC6teffz75LSF4zsxoKRyeFGnJywcBh/eNjSN0wdkPqFe+X8oaycP5Qn3v4CsDRdf9h3YlyeaP/2m9aX05aMBIitJ9x5XI+Yt+i9p/ZldK/2LPn9E3SorWRUT0tg+NWPxvDWZ99y7IKhsbq8HIu54cer8vg+HXjxw/8BVvxcpxfifxwdL4BHhbfNWywXNf07t+bRE+fw5mff+p70eHzVXD7fXM/uf/xXXPqm7xpc8zuFI4Ce7Vux+4ReHGyb0l96wKSY2fOkfnX846gZPPifz2KCyzk/Gp2x12svLjtwcsaTCeBuZv3L3caweqcRVAYDbDOwE0+99yXbDnKfNOjatoaubf2FrPrn0TPjrv8dRnWjV4da13vWyYzBnbnrZ97Xy4S+Hbn32PjYzo+v2p5DrnqGDzf9wAV7jYuN89jeHbj9J9syuGsbxpx5H3tO6s2B2/SnW/tqpp7zYEwgdNK7Yy03Hjk99vuIWQNZd+1zDOnq/9wtm9yHix9+h7Y1yaJd25rKtGbmgYAwuld77j9uNv/7voF1679i2sA6/vvFd4zt3YH1axbzv+8bePq9TRyzvffE1y93G82YXu08z2exkeYIuJ50UJF3gGrgSzvpaWPMchE5FcvD89uO7AuNMRtF5HLgT8aYdSLSCbgZ6At8AOxpjNmU7riTJ08269aty2tfFEVRlK0XEXnOGDO52O3IFBHZBjjTGLOD/Xs1gDHm144899p5nhKRCuBToAt2JIVo3mg+u1jKOt0o9WdzQ8iKRVlhv7B+/X0j+13xNItG96AhFGHFnEGeAlkoHGFzfYjVf3uFM3YexYILH+Xb+hAvn7mQdjWWMHfbcx8RMYY9J/dxrcMvH331Pe1bVXLIVc+y7v2vALjv2FkMTaHlSeTr7xupqgj4Xr+5uT7Erhc/wQV7jU9af/nqx1+z15+f4sHjZ3tq0grBI29uZO2Ln3DBXuPSmvhurg/x6ddbGOxDwLj/9c844pp13HD4tJgn7FyJRAwHXvkMR84ayKwUcWOV0uTmZz9k1W0vc+/KWa4TbA2hCJVBiV2HX33XQEAkLqxSvjDG0BCOlIxmtdh4PZuLIvgWi1J/uCqKoigtixYs+C4FdjTGHG7/PgCYZow52pHnVTvPR/bvd4FpWELu08aY6+z0K2hyUpmyTkfdRwJHAvTt23fS+++/X5B+lhr//eI7/vHix/xs3pCCxbXc0hjmi831vPf5dypM5Zmvf2hM0tQrWzdffddAxzQaZaX58Xo2l4JXZ0VRFEVRmhc3qStxJtwrT6bpyYlbqePJAZ1bs3L+0IIJvWCZA/fuWKtCbwFQoVdJRIXeloUKvoqiKIqy9fER4LSt7Q184pXHNnVuD2xKUdZPnYqiKIpSFFTwVRRFUZStj2eBISIyQESqsJxVrU3IE3UkCbAUeMiOwrAW2Nv2+jwAGAI847NORVEURSkKRfHqrCiKoihK8TDGhETkaOBeIAhcaYx5TUTOBtYZY9YCVwDX2g4pN2EJstj5bgZeB0LAUcaYMIBbnc3dN0VRFEVxQwVfRVEURdkKMcbcBdyVkHa6Y3sLsKdH2XOAc/zUqSiKoiilgJo6K4qiKIqiKIqiKGWNCr6KoiiKoiiKoihKWaOCr6IoiqIoiqIoilLWqOCrKIqiKIqiKIqilDViRSbYOhCRz4H3i92OAtEZ+KLYjWgB6Dj5Q8fJHzpO6Sn3MepnjOlS7Ea0ZPL4bC6Ha037UBpoH0oD7UNp0BL74Pps3qoE33JGRNYZYyYXux2ljo6TP3Sc/KHjlB4dI6W5KIdrTftQGmgfSgPtQ2lQDn2IoqbOiqIoiqIoiqIoSlmjgq+iKIqiKIqiKIpS1qjgWz5cWuwGtBB0nPyh4+QPHaf06BgpzUU5XGvah9JA+1AaaB9Kg3LoA6BrfBVFURRFURRFUZQyRzW+iqIoiqIoiqIoSlmjgq+iKIqiKIqiKIpS1qjg24IQkToRuV9E3ra/O3rkO8jO87aIHOSyf62IvFr4FheHXMZJRGpF5E4ReUNEXhORNc3b+sIiIjuKyJsi8o6InOSyv1pEbrL3/1tE+jv2rbbT3xSRHZqz3c1NtuMkIgtE5DkRecX+3r65296c5HI92fv7ishmETmhudqslCfprsVSQUT6iMjDIvIf+xnzMzv9TBH5WERetD87OcqU3H+viKy3/+deFJF1dprrs1csfmf34WURmVjc1oOIDHOM9Ysi8o2IrCz18yAiV4rIRuc7XDbjnu49sQh9ON9+73pZRG4XkQ52en8R+cFxPv7kKDPJvgbfsfspRe5DxtdOMf+3PPpwk6P960XkRTu9JM9DVhhj9NNCPsB5wEn29knAuS556oD37O+O9nZHx/7dgRuAV4vdn1IcJ6AWmGvnqQIeBxYVu095Gpcg8C4w0O7bS8DIhDw/Af5kb+8N3GRvj7TzVwMD7HqCxe5TCY7TBKCnvT0a+LjY/SnFcXLsvw24BTih2P3RT8v9+LkWS+UD9AAm2tttgbfs/9cz3e6DUv3vBdYDnRPSXJ+9wE7A3YAA04F/F7v9LtfPp0C/Uj8PwCxgIo53uEzHnTTviUXqw0Kgwt4+19GH/ni8rwLPANvY/bubZnxX8+hDRtdOsf+33PqQsP8C4PRSPg/ZfFTj27LYFbja3r4a2M0lzw7A/caYTcaYr4D7gR0BRKQNcBzwy2ZoazHJepyMMd8bYx4GMMY0AM8DvZuhzc3BVOAdY8x7dt/+ijVWTpxjdyswz5692xX4qzGm3hjzX+Adu75yJOtxMsa8YIz5xE5/DagRkepmaXXzk8v1hIjshvXC9VoztVcpX/xciyWBMWaDMeZ5e/tb4D9ArxRFWtJ/r9ezd1fgGmPxNNBBRHoUo4EezAPeNca8nyJPSZwHY8xjwCaXtmUy7p7vic2BWx+MMfcZY0L2z6dJ895l96OdMeYpY0lf1+D+rlcQPM6DF17XTlH/t1L1wX5O7wXcmKqOYp+HbFDBt2XRzRizAayHJ9DVJU8v4EPH749oeqj+AmsG5/tCNrIEyHWcALBNbXYGHixQO5ubtH125rEfQl8DnXyWLRdyGScnewAvGGPqC9TOYpP1OIlIa+DnwFnN0E6l/GmR/09imf5PAP5tJx1tm3peKU1LdEq1bwa4T6wlHUfaaV7P3lLtQ5S9iX/Bb0nnATIf91LuC8ChWJrDKANE5AUReVREtrPTemG1O0qp9CGTa6eUz8N2wGfGmLcdaS3pPHiigm+JISIPiMirLh+/s0ButvVGRMYDg40xt+exuUWjUOPkqL8C60H4O2PMe/locwmQss9p8vgpWy7kMk7WTpFRWOZaP85ju0qNXMbpLOC3xpjNeW+VsjXS4v6fbAus24CVxphvgEuAQcB4YAPWJDWUbt9mGGMmAouAo0RkVoq8pdoHRKQK2AVryQW0vPOQihb3PBeRU4AQcL2dtAHoa4yZgGWxeIOItKM0+5DptVOKfYiyD/GTQS3pPKSkotgNUOIxxsz32icin4lID2PMBtu8YKNLto+AOY7fvYFHsOzvJ4nIeqzz3lVEHjHGzKEFUsBxinIp8LYx5qI8NLdU+Ajo4/jdG/jEI89HtvDfHssUxk/ZciGXcUJEegO3AwcaY94tfHOLRi7jNA1YKiLnAR2AiIhsMcZcXPhmK2VIi/p/EpFKLKH3emPM3wCMMZ859l8G3GH/LMm+RZd0GGM2isjtWGabXs/ekuyDzSLg+ej4t7TzYJPpuKd7/ykKYjnZWgLMs81msS2m6u3t50TkXWAoVh+c5tBFPx9ZXjsld03Zz+rdgUnRtJZ0HtKhGt+WxVog6n3vIOAfLnnuBRaKSEfbzGIhcK8x5hJjTE9jTH9gJvBWSxV6fZD1OAGIyC+xXtBXNkNbm5NngSEiMsCe5d4ba6ycOMduKfCQ/QBaC+wtlpfeAcAQLIcG5UjW42Sbx98JrDbGPNlsLS4OWY+TMWY7Y0x/+//oIuBXKvQqOeDnWiwJ7LVzVwD/McZc6Eh3rnn9ERD1tFpy/70i0lpE2ka3sZ6fr+L97F0LHCgW04Gvo6a5JUCcZqslnQcHmY675/tPsRCRHbGWv+xijPnekd5FRIL29kCscX/P7se3IjLdvqcOxP1dr9nI4top1f+t+cAbxpiYCXNLOg9p8fJ6pZ/S+2CtIXwQeNv+rrPTJwOXO/IdirV4/h3gEJd6+lPeXp2zHies2SqD5XDkRftzeLH7lMex2QnLi+i7wCl22tlYDxuAGiyTr3ew/pgHOsqeYpd7kxL32lescQJOBb5zXDsvAl2L3Z9SG6eEOs5EvTrrJ8eP27VYih+siWcDvOz4j9gJuBZ4xU5fC/RwlCmp/14sL7Qv2Z/XHPe+17NXgD/YfXgFmFzsPtjtqgW+BNo70kr6PGAJ6RuARixt22HZjDtp3hOL0Id3sNa7Ru+JaDSAPexr7CUsZ6M7O+qZjCVcvgtcDEiR+5DxtVPM/y23PtjpfwGWJ+QtyfOQzUfsRiuKoiiKoiiKoihKWaKmzoqiKIqiKIqiKEpZo4KvoiiKoiiKoiiKUtao4KsoiqIoiqIoiqKUNSr4KoqiKIqiKIqiKGWNCr6KoiiKoiiKoihKWaOCr6KUCSISFpEXHZ+T0uRfLiIH5uG460Wkc671KIqiKMrWjIisFJHaYrdDUcoVDWekKGWCiGw2xrQpwnHXY8UH/KK5j60oiqIo5YI+TxWlsKjGV1HKHFsje66IPGN/BtvpZ4rICfb2T0XkdRF5WUT+aqfVicjf7bSnRWSsnd5JRO4TkRdE5M+AOI61v32MF0XkzyISLEKXFUVRFKWkEZHWInKniLwkIq+KyBlAT+BhEXnYzrNQRJ4SkedF5BYRaWOnez3X97TreklEHite7xSlNFHBV1HKh1YJps7LHPu+McZMBS4GLnIpexIwwRgzFlhup50FvGCnnQxcY6efATxhjJkArAX6AojICGAZMMMYMx4IA/vlt4uKoiiKUhbsCHxijBlnjBmN9Wz+BJhrjJlrLyE6FZhvjJkIrAOOc5R3e66fDuxgjBkH7NJcHVGUlkJFsRugKEre+MEWON240fH9W5f9LwPXi8jfgb/baTOBPQCMMQ/Zmt72wCxgdzv9ThH5ys4/D5gEPCsiAK2Ajbl1SVEURVHKkleA34jIucAdxpjH7WdnlOnASOBJO70KeMqx3+25/iTwFxG5GfhbAduuKC0SFXwVZevAeGxHWYwl0O4CnCYio3CYMLuUdatDgKuNMatzaaiiKIqilDvGmLdEZBKwE/BrEbkvIYsA9xtj9vGqInHbGLNcRKZhPdNfFJHxxpgv8912RWmpqKmzomwdLHN8O2eMEZEA0McY8zCwCugAtAEewzZVFpE5wBfGmG8S0hcBHe2qHgSWikhXe1+diPQrYJ8URVEUpUUiIj2B740x1wG/ASYC3wJt7SxPAzMc63drRWSoo4qk57qIDDLG/NsYczrwBdCn8D1RlJaDanwVpXxoJSIvOn7fY4yJhjSqFpF/Y012Jc4eB4HrbDNmAX5rjPmfiJwJXCUiLwPfAwfZ+c8CbhSR54FHgQ8AjDGvi8ipwH22MN0IHAW8n++OKoqiKEoLZwxwvohEsJ6XK4BtgLtFZIO9zvdgrOdttV3mVOAte9vtuX6+iAzBepY/CLzUPF1RlJaBhjNSlDJHwyMoiqIoSvmgz3VFyQ41dVYURVEURVEURVHKGtX4KoqiKIqiKIqiKGWNanwVRVEURVEURVGUskYFX0VRFEVRFEVRFKWsUcFXURRFURRFURRFKWtU8FUURVEURVEURVHKGhV8FUVRFEVRFEVRlLLm/wHINbVgPPUwNAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eps: 0.9815269999999999\n",
      "torch.Size([128])\n",
      "selecting:\n",
      "tensor([-0.0359, -0.0386, -0.0307], device='cuda:0')\n",
      "tensor([[2]], device='cuda:0')\n",
      "selecting:\n",
      "tensor([-0.0110, -0.0018, -0.0027], device='cuda:0')\n",
      "tensor([[1]], device='cuda:0')\n",
      "selecting:\n",
      "tensor([-0.0295, -0.1004, -0.0667], device='cuda:0')\n",
      "tensor([[0]], device='cuda:0')\n",
      "selecting:\n",
      "tensor([-0.0235, -0.0386, -0.0261], device='cuda:0')\n",
      "tensor([[0]], device='cuda:0')\n",
      "selecting:\n",
      "tensor([-0.0203, -0.0374, -0.0349], device='cuda:0')\n",
      "tensor([[0]], device='cuda:0')\n",
      "selecting:\n",
      "tensor([-0.0035,  0.0116, -0.0088], device='cuda:0')\n",
      "tensor([[1]], device='cuda:0')\n",
      "selecting:\n",
      "tensor([-0.0294, -0.0092, -0.0151], device='cuda:0')\n",
      "tensor([[1]], device='cuda:0')\n",
      "selecting:\n",
      "tensor([-0.0208, -0.0104, -0.0148], device='cuda:0')\n",
      "tensor([[1]], device='cuda:0')\n",
      "selecting:\n",
      "tensor([-0.0181,  0.0116, -0.0333], device='cuda:0')\n",
      "tensor([[1]], device='cuda:0')\n",
      "selecting:\n",
      "tensor([ 0.0282, -0.0687, -0.1115], device='cuda:0')\n",
      "tensor([[0]], device='cuda:0')\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-10af8c2e2e89>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     14\u001b[0m     \u001b[0mtotal_reward\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcount\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m         \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m         \u001b[1;31m# Select and perform an action\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m         \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mselect_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorchenv\\lib\\site-packages\\gym\\core.py\u001b[0m in \u001b[0;36mrender\u001b[1;34m(self, mode, **kwargs)\u001b[0m\n\u001b[0;32m    233\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    234\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mrender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'human'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 235\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    236\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    237\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorchenv\\lib\\site-packages\\gym\\envs\\atari\\atari_env.py\u001b[0m in \u001b[0;36mrender\u001b[1;34m(self, mode)\u001b[0m\n\u001b[0;32m    153\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mviewer\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    154\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mviewer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrendering\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSimpleImageViewer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 155\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mviewer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    156\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mviewer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misopen\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    157\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorchenv\\lib\\site-packages\\gym\\envs\\classic_control\\rendering.py\u001b[0m in \u001b[0;36mimshow\u001b[1;34m(self, arr)\u001b[0m\n\u001b[0;32m    352\u001b[0m         gl.glTexParameteri(gl.GL_TEXTURE_2D, \n\u001b[0;32m    353\u001b[0m             gl.GL_TEXTURE_MAG_FILTER, gl.GL_NEAREST)\n\u001b[1;32m--> 354\u001b[1;33m         \u001b[0mtexture\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mimage\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_texture\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    355\u001b[0m         \u001b[0mtexture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwidth\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwidth\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    356\u001b[0m         \u001b[0mtexture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mheight\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mheight\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorchenv\\lib\\site-packages\\pyglet\\image\\__init__.py\u001b[0m in \u001b[0;36mget_texture\u001b[1;34m(self, rectangle, force_rectangle)\u001b[0m\n\u001b[0;32m    857\u001b[0m             self._current_texture = self.create_texture(Texture,\n\u001b[0;32m    858\u001b[0m                                                         \u001b[0mrectangle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 859\u001b[1;33m                                                         force_rectangle)\n\u001b[0m\u001b[0;32m    860\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_current_texture\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    861\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorchenv\\lib\\site-packages\\pyglet\\image\\__init__.py\u001b[0m in \u001b[0;36mcreate_texture\u001b[1;34m(self, cls, rectangle, force_rectangle)\u001b[0m\n\u001b[0;32m    842\u001b[0m         \u001b[0minternalformat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_internalformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    843\u001b[0m         texture = cls.create(self.width, self.height, internalformat,\n\u001b[1;32m--> 844\u001b[1;33m                              rectangle, force_rectangle)\n\u001b[0m\u001b[0;32m    845\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0manchor_x\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0manchor_y\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    846\u001b[0m             \u001b[0mtexture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0manchor_x\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0manchor_x\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorchenv\\lib\\site-packages\\pyglet\\image\\__init__.py\u001b[0m in \u001b[0;36mcreate\u001b[1;34m(cls, width, height, internalformat, rectangle, force_rectangle, min_filter, mag_filter)\u001b[0m\n\u001b[0;32m   1574\u001b[0m                                   0., height, 0.)\n\u001b[0;32m   1575\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1576\u001b[1;33m         \u001b[0mglFlush\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1577\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1578\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtexture_width\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mwidth\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mtexture_height\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mheight\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorchenv\\lib\\site-packages\\pyglet\\gl\\lib.py\u001b[0m in \u001b[0;36merrcheck\u001b[1;34m(result, func, arguments)\u001b[0m\n\u001b[0;32m    100\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mGLException\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'No GL context; create a Window first'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_gl_begin\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 102\u001b[1;33m         \u001b[0merror\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgl\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mglGetError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    103\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    104\u001b[0m             \u001b[0mmsg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mctypes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgl\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgluErrorString\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mctypes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mc_char_p\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "num_episodes = 50000\n",
    "prefetch(PREFETCH)   \n",
    "env.close()\n",
    "for i_episode in range(num_episodes):\n",
    "    # Initialize the environment and state\n",
    "    s = env.reset()\n",
    "    #env.render()\n",
    "    #screen = get_screen()[:,2:3,:,:] # get the blue channel\n",
    "    #screens = [screen]*past\n",
    "    state = (torch.from_numpy(s).float().to(device)-128)/128  # torch.cat(screens, dim=1)\n",
    "    print(\"eps:\", eps[-1])\n",
    "    print(state.shape)\n",
    "    #print(state)\n",
    "    total_reward = 0\n",
    "    for t in count():\n",
    "        env.render()\n",
    "        # Select and perform an action\n",
    "        action = select_action(state)\n",
    "        s, reward, done, _ = env.step(action.item() + 1)\n",
    "        total_reward += reward\n",
    "        reward = torch.tensor([reward], device=device, dtype=torch.float32)\n",
    "\n",
    "        # Observe new state\n",
    "        #screen = get_screen()[:,2:3,:,:]\n",
    "        #del screens[0]\n",
    "        #screens.append(screen)\n",
    "        if not done:\n",
    "            next_state = (torch.from_numpy(s).float().to(device)-128)/128# torch.cat(screens, dim=1)\n",
    "            #print(next_state.shape)\n",
    "        else:\n",
    "            next_state = None\n",
    "\n",
    "        # Store the transition in memory\n",
    "        memory.push(state, action, next_state, reward)\n",
    "\n",
    "        # Move to the next state\n",
    "        state = next_state\n",
    "\n",
    "        # Perform one step of the optimization (on the target network)\n",
    "        optimize_model()\n",
    "        if steps_done % TARGET_UPDATE == 0:\n",
    "            target_net.load_state_dict(policy_net.state_dict())\n",
    "        if done:\n",
    "            episode_durations.append(total_reward)\n",
    "            plot_stuff()\n",
    "            break\n",
    "    # Update the target network, copying all weights and biases in DQN\n",
    "    \n",
    "    #target_net.update_params(policy_net.state_dict(), tau)\n",
    "\n",
    "print('Complete')\n",
    "env.render()\n",
    "env.close()\n",
    "plot_durations()\n",
    "plt.ioff()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()\n",
    "env.render()\n",
    "print(len(memory))\n",
    "done = False\n",
    "screen = get_screen()[:,2:3,:,:] - 1# get the blue channel\n",
    "screens = [screen]*past\n",
    "rgb_screen = get_screen()\n",
    "rgb_screens = [rgb_screen]*past\n",
    "print(torch.max(screen))\n",
    "while not done:\n",
    "    screen = get_screen()[:,2:3,:,:] - 1\n",
    "    del screens[0]\n",
    "    screens.append(screen)\n",
    "    \n",
    "    rgb_screen = get_screen()\n",
    "    del rgb_screens[0]\n",
    "    rgb_screens.append(rgb_screen)\n",
    "    \n",
    "    state = torch.cat(screens, dim=1)\n",
    "    print(policy_net(state))\n",
    "    action = select_action(state, eps_max=0)\n",
    "    _, reward, done, _ = env.step(action.item())\n",
    "    \n",
    "env.close()   \n",
    "\n",
    "for screen in rgb_screens:\n",
    "    plt.figure()\n",
    "    plt.imshow(screen.cpu().squeeze(0).permute(1, 2, 0).numpy(),\n",
    "           interpolation='none')\n",
    "    plt.title('Screen')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l1.weight tensor(0.0002, device='cuda:0') tensor(0.0513, device='cuda:0') tensor(0.0884, device='cuda:0')\n",
      "l1.bias tensor(0.0061, device='cuda:0') tensor(0.0519, device='cuda:0') tensor(0.0864, device='cuda:0')\n",
      "l2.weight tensor(0.0004, device='cuda:0') tensor(0.0515, device='cuda:0') tensor(0.0884, device='cuda:0')\n",
      "l2.bias tensor(0.0008, device='cuda:0') tensor(0.0550, device='cuda:0') tensor(0.0882, device='cuda:0')\n",
      "l3.weight tensor(-0.0003, device='cuda:0') tensor(0.0509, device='cuda:0') tensor(0.0884, device='cuda:0')\n",
      "l3.bias tensor(0.0047, device='cuda:0') tensor(0.0498, device='cuda:0') tensor(0.0881, device='cuda:0')\n",
      "head.weight tensor(0.0037, device='cuda:0') tensor(0.0486, device='cuda:0') tensor(0.0882, device='cuda:0')\n",
      "head.bias tensor(0.0230, device='cuda:0') tensor(0.0416, device='cuda:0') tensor(0.0638, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "for name, param in policy_net.named_parameters():\n",
    "    if param.requires_grad:\n",
    "        print(name, torch.mean(param.data), torch.std(param.data), torch.max(param.data),)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy_net(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(memory))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "steps_done = 20000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(policy_net.state_dict(), \"pong_basic_256_256.torch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}