final_version_v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A - Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import random\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from matplotlib import style\n",
    "from  matplotlib import pylab\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Activation, Flatten\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"font.family\"]=\"agency fb\"\n",
    "plt.style.use('ggplot')\n",
    "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
    "load_model = False "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B - Creating Agent Class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    All classes that needs to be created are created in one class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent():\n",
    "    def __init__(self, epsilon_decay, epsilon_min, \n",
    "                       gamma, learning_rate, state_size, action_size,\n",
    "                       batch_size, training_threshold):\n",
    "        '''CREATING AND DEFINING BASIC PARAMETERS FOR TRAINING'''\n",
    "        self.movement_penalty = -1 \n",
    "        # Change epsilon if model is already trained.\n",
    "        self.epsilon = 0.99                                                                     \n",
    "        self.epsilon_decay = epsilon_decay\n",
    "        self.epsilon_min = epsilon_min                \n",
    "        self.gamma = gamma\n",
    "        self.learning_rate = learning_rate\n",
    "\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        \n",
    "        self.batch_size = batch_size\n",
    "        self.training_threshold = training_threshold\n",
    "\n",
    "         # Main Model\n",
    "        self.model = self.create_model()\n",
    "        \n",
    "         # Target Model\n",
    "        self.target_model = self.create_model()\n",
    "        self.target_model.set_weights(self.model.get_weights())\n",
    " \n",
    "        self.target_update_counter = 0\n",
    "\n",
    "        # Replay Memory\n",
    "        self.replay_memory = deque(maxlen=2000) \n",
    "\n",
    "        if load_model:\n",
    "            self.model.load_weights(\"../Deep Q-Learning - PACMAN/final_version_v2.h5\")        \n",
    "        \n",
    "    \n",
    "    def update_replay_memory(self, state, action, reward, next_state, done):\n",
    "        '''UPDATING REPLAY MEMORY and DECAYIN EPSILON'''\n",
    "        self.replay_memory.append((state, action, reward, next_state, done))\n",
    "\n",
    "        if self.epsilon > self.epsilon_min:                                                       \n",
    "            self.epsilon *= self.epsilon_decay       \n",
    "    \n",
    "\n",
    "    def create_model(self):\n",
    "        '''CREATING NEURAL NETWORK'''\n",
    "        model = Sequential()\n",
    "        model.add(Dense(128, activation=\"relu\", input_dim = self.state_size))\n",
    "        model.add(Dense(128, activation = \"relu\"))\n",
    "        model.add(Dense(128, activation = \"relu\"))\n",
    "        model.add(Dense(self.action_size, activation=\"linear\"))\n",
    "        model.summary()\n",
    "        model.compile(optimizer=Adam(lr=self.learning_rate), loss = \"mse\")          \n",
    "        return model\n",
    "    \n",
    "\n",
    "    def select_action(self, state):\n",
    "        '''DEFINING EPSILON GREEDY STRATEGY'''\n",
    "        if np.random.rand() > self.epsilon:\n",
    "            q_value = self.model.predict(state)\n",
    "            return np.argmax(q_value[0])\n",
    "        else:\n",
    "            return np.random.randint(0, self.action_size)             \n",
    " \n",
    "\n",
    "    def train_agent(self):\n",
    "        '''TRAINING AGENT'''\n",
    "        if len(self.replay_memory) < self.training_threshold:\n",
    "                  return\n",
    "        \n",
    "        batch_size = min(self.batch_size, len(self.replay_memory))\n",
    "        minibatch = random.sample(self.replay_memory, batch_size)\n",
    " \n",
    "        observations = np.zeros((batch_size, self.state_size))                    \n",
    "        next_observations = np.zeros((batch_size, self.state_size))\n",
    "        \n",
    "        action = []\n",
    "        reward = [] \n",
    "        done = []\n",
    "        \n",
    "        for sample_index in range(self.batch_size):\n",
    "            observations[sample_index] = minibatch[sample_index][0]                       \n",
    "            action.append(minibatch[sample_index][1])\n",
    "            reward.append(minibatch[sample_index][2])\n",
    "            next_observations[sample_index] = minibatch[sample_index][3]\n",
    "            done.append(minibatch[sample_index][4])\n",
    "\n",
    "            \n",
    "        current_q_values = self.model.predict(observations)\n",
    "        future_q_values = self.target_model.predict(next_observations)\n",
    "    \n",
    "        for index in range(self.batch_size):\n",
    "            if not done[index]:    \n",
    "                current_q_values[index][action[index]] = reward[index] + self.gamma * (np.amax(future_q_values[index]))\n",
    "            else:\n",
    "                current_q_values[index][action[index]] = reward[index]            \n",
    "\n",
    "        self.model.fit(observations, current_q_values, batch_size = batch_size, verbose=0)                              \n",
    "        \n",
    "        if done:\n",
    "            self.target_update_counter += 1\n",
    "        \n",
    "        if self.target_update_counter > update_target_every:\n",
    "            self.target_model.set_weights(self.model.get_weights())\n",
    "            self.target_update_counter = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## C - Setting Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_episodes= 2100                          \n",
    "epsilon_decay = 0.999998                   \n",
    "epsilon_min = 0.1                 \n",
    "gamma = 0.99\n",
    "learning_rate = 0.000001        \n",
    "batch_size = 64\n",
    "training_threshold = 1000 \n",
    "update_target_every = 10\n",
    "load_model = False                      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## D - Training Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 9)                 1161      \n",
      "=================================================================\n",
      "Total params: 50,697\n",
      "Trainable params: 50,697\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_4 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 9)                 1161      \n",
      "=================================================================\n",
      "Total params: 50,697\n",
      "Trainable params: 50,697\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Episode: 0 -----Score: 240.0 -----Epsilon: 0.988703947803517\n",
      "Saved model to disk\n",
      "Episode: 1 -----Score: 210.0 -----Epsilon: 0.9874392150872472\n",
      "Episode: 2 -----Score: 530.0 -----Epsilon: 0.9858309986109817\n",
      "Episode: 3 -----Score: 120.0 -----Epsilon: 0.9846112937681067\n",
      "Episode: 4 -----Score: 260.0 -----Epsilon: 0.983336062784356\n",
      "Episode: 5 -----Score: 250.0 -----Epsilon: 0.9820546269548255\n",
      "Episode: 6 -----Score: 220.0 -----Epsilon: 0.9807670148549225\n",
      "Episode: 7 -----Score: 240.0 -----Epsilon: 0.9795496571407752\n",
      "Episode: 8 -----Score: 480.0 -----Epsilon: 0.9779288636751856\n",
      "Episode: 9 -----Score: 170.0 -----Epsilon: 0.9768146588738256\n",
      "Episode: 10 -----Score: 210.0 -----Epsilon: 0.9755261128630653\n",
      "Episode: 11 -----Score: 190.0 -----Epsilon: 0.9742684942582864\n",
      "Episode: 12 -----Score: 240.0 -----Epsilon: 0.9731000720902979\n",
      "Episode: 13 -----Score: 230.0 -----Epsilon: 0.9717678364491212\n",
      "Episode: 14 -----Score: 100.0 -----Epsilon: 0.9704141344809316\n",
      "Episode: 15 -----Score: 210.0 -----Epsilon: 0.9691941198298837\n",
      "Episode: 16 -----Score: 180.0 -----Epsilon: 0.967787869746306\n",
      "Episode: 17 -----Score: 190.0 -----Epsilon: 0.9664783703923282\n",
      "Episode: 18 -----Score: 610.0 -----Epsilon: 0.9646650254632773\n",
      "Episode: 19 -----Score: 240.0 -----Epsilon: 0.96336360503396\n",
      "Episode: 20 -----Score: 250.0 -----Epsilon: 0.9620620162107087\n",
      "Episode: 21 -----Score: 340.0 -----Epsilon: 0.9604567118406103\n",
      "Episode: 22 -----Score: 230.0 -----Epsilon: 0.9592914237001952\n",
      "Episode: 23 -----Score: 140.0 -----Epsilon: 0.9582022863009348\n",
      "Episode: 24 -----Score: 90.0 -----Epsilon: 0.9571373565043686\n",
      "Episode: 25 -----Score: 150.0 -----Epsilon: 0.9559225624043786\n",
      "Episode: 26 -----Score: 220.0 -----Epsilon: 0.954625299308412\n",
      "Episode: 27 -----Score: 270.0 -----Epsilon: 0.9532973840073651\n",
      "Episode: 28 -----Score: 120.0 -----Epsilon: 0.9521065055406138\n",
      "Episode: 29 -----Score: 200.0 -----Epsilon: 0.9508296343055751\n",
      "Episode: 30 -----Score: 170.0 -----Epsilon: 0.9497292097574342\n",
      "Episode: 31 -----Score: 180.0 -----Epsilon: 0.9484517328984525\n",
      "Episode: 32 -----Score: 210.0 -----Epsilon: 0.9472650131837338\n",
      "Episode: 33 -----Score: 150.0 -----Epsilon: 0.9462141313155832\n",
      "Episode: 34 -----Score: 270.0 -----Epsilon: 0.9450321013920527\n",
      "Episode: 35 -----Score: 90.0 -----Epsilon: 0.9441649591350486\n",
      "Episode: 36 -----Score: 160.0 -----Epsilon: 0.943255221768591\n",
      "Episode: 37 -----Score: 370.0 -----Epsilon: 0.9419619616903807\n",
      "Episode: 38 -----Score: 190.0 -----Epsilon: 0.9407212723780763\n",
      "Episode: 39 -----Score: 160.0 -----Epsilon: 0.9397471888282375\n",
      "Episode: 40 -----Score: 320.0 -----Epsilon: 0.9385112936892454\n",
      "Episode: 41 -----Score: 230.0 -----Epsilon: 0.937158934336852\n",
      "Episode: 42 -----Score: 280.0 -----Epsilon: 0.9359339305528935\n",
      "Episode: 43 -----Score: 270.0 -----Epsilon: 0.9348918796423575\n",
      "Episode: 44 -----Score: 260.0 -----Epsilon: 0.9335820783026179\n",
      "Episode: 45 -----Score: 230.0 -----Epsilon: 0.9324904249211351\n",
      "Episode: 46 -----Score: 310.0 -----Epsilon: 0.9308935036883909\n",
      "Episode: 47 -----Score: 190.0 -----Epsilon: 0.9295855857960613\n",
      "Episode: 48 -----Score: 230.0 -----Epsilon: 0.9284781788160975\n",
      "Episode: 49 -----Score: 230.0 -----Epsilon: 0.9271347139730262\n",
      "Episode: 50 -----Score: 210.0 -----Epsilon: 0.9259061468337909\n",
      "Saved model to disk\n",
      "Episode: 51 -----Score: 180.0 -----Epsilon: 0.9246459198110267\n",
      "Episode: 52 -----Score: 290.0 -----Epsilon: 0.9232747615459757\n",
      "Episode: 53 -----Score: 170.0 -----Epsilon: 0.922182250030269\n",
      "Episode: 54 -----Score: 130.0 -----Epsilon: 0.9211186644570869\n",
      "Episode: 55 -----Score: 180.0 -----Epsilon: 0.9198723125097753\n",
      "Episode: 56 -----Score: 240.0 -----Epsilon: 0.9186956280118441\n",
      "Episode: 57 -----Score: 220.0 -----Epsilon: 0.9175387993188769\n",
      "Episode: 58 -----Score: 450.0 -----Epsilon: 0.9156066627852169\n",
      "Episode: 59 -----Score: 760.0 -----Epsilon: 0.9134556843057013\n",
      "Episode: 60 -----Score: 290.0 -----Epsilon: 0.9121430777094297\n",
      "Episode: 61 -----Score: 150.0 -----Epsilon: 0.9109434857242569\n",
      "Episode: 62 -----Score: 240.0 -----Epsilon: 0.909599923588578\n",
      "Episode: 63 -----Score: 270.0 -----Epsilon: 0.9082910410113925\n",
      "Episode: 64 -----Score: 200.0 -----Epsilon: 0.9072362187373725\n",
      "Episode: 65 -----Score: 180.0 -----Epsilon: 0.9060050268896419\n",
      "Episode: 66 -----Score: 270.0 -----Epsilon: 0.9047917919893302\n",
      "Episode: 67 -----Score: 230.0 -----Epsilon: 0.9036470492110347\n",
      "Episode: 68 -----Score: 170.0 -----Epsilon: 0.902335904505884\n",
      "Episode: 69 -----Score: 280.0 -----Epsilon: 0.900963592481313\n",
      "Episode: 70 -----Score: 170.0 -----Epsilon: 0.9000162762104561\n",
      "Episode: 71 -----Score: 180.0 -----Epsilon: 0.8989027443654388\n",
      "Episode: 72 -----Score: 210.0 -----Epsilon: 0.8977528837705956\n",
      "Episode: 73 -----Score: 220.0 -----Epsilon: 0.8962351687225968\n",
      "Episode: 74 -----Score: 130.0 -----Epsilon: 0.8950564977974946\n",
      "Episode: 75 -----Score: 190.0 -----Epsilon: 0.8939222842674502\n",
      "Episode: 76 -----Score: 120.0 -----Epsilon: 0.8928305773131993\n",
      "Episode: 77 -----Score: 220.0 -----Epsilon: 0.8917009678663954\n",
      "Episode: 78 -----Score: 240.0 -----Epsilon: 0.8905104496261022\n",
      "Episode: 79 -----Score: 220.0 -----Epsilon: 0.8894798343362683\n",
      "Episode: 80 -----Score: 220.0 -----Epsilon: 0.8882531975123552\n",
      "Episode: 81 -----Score: 200.0 -----Epsilon: 0.8873068231646029\n",
      "Episode: 82 -----Score: 250.0 -----Epsilon: 0.8860087550951283\n",
      "Episode: 83 -----Score: 260.0 -----Epsilon: 0.8849355618972378\n",
      "Episode: 84 -----Score: 160.0 -----Epsilon: 0.8838371530840521\n",
      "Episode: 85 -----Score: 300.0 -----Epsilon: 0.8826536032708735\n",
      "Episode: 86 -----Score: 190.0 -----Epsilon: 0.8814258029746154\n",
      "Episode: 87 -----Score: 320.0 -----Epsilon: 0.8802137939112141\n",
      "Episode: 88 -----Score: 200.0 -----Epsilon: 0.8791142129505475\n",
      "Episode: 89 -----Score: 220.0 -----Epsilon: 0.877979129674055\n",
      "Episode: 90 -----Score: 310.0 -----Epsilon: 0.8767736135236495\n",
      "Episode: 91 -----Score: 280.0 -----Epsilon: 0.8756678220188215\n",
      "Episode: 92 -----Score: 140.0 -----Epsilon: 0.8746998678170833\n",
      "Episode: 93 -----Score: 160.0 -----Epsilon: 0.8735984389281533\n",
      "Episode: 94 -----Score: 280.0 -----Epsilon: 0.8724181307265803\n",
      "Episode: 95 -----Score: 110.0 -----Epsilon: 0.8716629429137229\n",
      "Episode: 96 -----Score: 250.0 -----Epsilon: 0.870455653712271\n",
      "Episode: 97 -----Score: 1110.0 -----Epsilon: 0.8687877181620753\n",
      "Episode: 98 -----Score: 320.0 -----Epsilon: 0.8676434090705999\n",
      "Episode: 99 -----Score: 190.0 -----Epsilon: 0.8664486186578009\n",
      "Episode: 100 -----Score: 200.0 -----Epsilon: 0.8653748871480783\n",
      "Saved model to disk\n",
      "Episode: 101 -----Score: 140.0 -----Epsilon: 0.8643733620222269\n",
      "Episode: 102 -----Score: 260.0 -----Epsilon: 0.8629966470398281\n",
      "Episode: 103 -----Score: 490.0 -----Epsilon: 0.8615704289669356\n",
      "Episode: 104 -----Score: 250.0 -----Epsilon: 0.8604511140023557\n",
      "Episode: 105 -----Score: 210.0 -----Epsilon: 0.8593005991274295\n",
      "Episode: 106 -----Score: 190.0 -----Epsilon: 0.8582940877496456\n",
      "Episode: 107 -----Score: 230.0 -----Epsilon: 0.8572013162323716\n",
      "Episode: 108 -----Score: 510.0 -----Epsilon: 0.8554167670509545\n",
      "Episode: 109 -----Score: 230.0 -----Epsilon: 0.8543567065731055\n",
      "Episode: 110 -----Score: 190.0 -----Epsilon: 0.8533372124051266\n",
      "Episode: 111 -----Score: 320.0 -----Epsilon: 0.8518349546124315\n",
      "Episode: 112 -----Score: 300.0 -----Epsilon: 0.8506023889585136\n",
      "Episode: 113 -----Score: 120.0 -----Epsilon: 0.8496111635784886\n",
      "Episode: 114 -----Score: 180.0 -----Epsilon: 0.8487365137273778\n",
      "Episode: 115 -----Score: 260.0 -----Epsilon: 0.8477355942988889\n",
      "Episode: 116 -----Score: 230.0 -----Epsilon: 0.8467324683199045\n",
      "Episode: 117 -----Score: 160.0 -----Epsilon: 0.8458286398702362\n",
      "Episode: 118 -----Score: 180.0 -----Epsilon: 0.8448683231382291\n",
      "Episode: 119 -----Score: 240.0 -----Epsilon: 0.8436475251718504\n",
      "Episode: 120 -----Score: 200.0 -----Epsilon: 0.8425969939218647\n",
      "Episode: 121 -----Score: 210.0 -----Epsilon: 0.8415393553759348\n",
      "Episode: 122 -----Score: 210.0 -----Epsilon: 0.8402880747475514\n",
      "Episode: 123 -----Score: 170.0 -----Epsilon: 0.83928201138025\n",
      "Episode: 124 -----Score: 170.0 -----Epsilon: 0.838293918283873\n",
      "Episode: 125 -----Score: 210.0 -----Epsilon: 0.8371361951020944\n",
      "Episode: 126 -----Score: 270.0 -----Epsilon: 0.8360586566889964\n",
      "Episode: 127 -----Score: 190.0 -----Epsilon: 0.8351294752542265\n",
      "Episode: 128 -----Score: 270.0 -----Epsilon: 0.8338960643715959\n",
      "Episode: 129 -----Score: 210.0 -----Epsilon: 0.8328543444600242\n",
      "Episode: 130 -----Score: 180.0 -----Epsilon: 0.8317972897573459\n",
      "Episode: 131 -----Score: 750.0 -----Epsilon: 0.8301868260427688\n",
      "Episode: 132 -----Score: 170.0 -----Epsilon: 0.829182906473191\n",
      "Episode: 133 -----Score: 240.0 -----Epsilon: 0.827923504583897\n",
      "Episode: 134 -----Score: 210.0 -----Epsilon: 0.8267470328535822\n",
      "Episode: 135 -----Score: 290.0 -----Epsilon: 0.825625071245391\n",
      "Episode: 136 -----Score: 140.0 -----Epsilon: 0.824641511495002\n",
      "Episode: 137 -----Score: 260.0 -----Epsilon: 0.8236690074292016\n",
      "Episode: 138 -----Score: 240.0 -----Epsilon: 0.822742077159607\n",
      "Episode: 139 -----Score: 450.0 -----Epsilon: 0.8211491454705571\n",
      "Episode: 140 -----Score: 280.0 -----Epsilon: 0.8199757402038831\n",
      "Episode: 141 -----Score: 250.0 -----Epsilon: 0.8190546043261233\n",
      "Episode: 142 -----Score: 220.0 -----Epsilon: 0.818078871914111\n",
      "Episode: 143 -----Score: 180.0 -----Epsilon: 0.8169392633188205\n",
      "Episode: 144 -----Score: 230.0 -----Epsilon: 0.815899144353724\n",
      "Episode: 145 -----Score: 260.0 -----Epsilon: 0.8147756085032877\n",
      "Episode: 146 -----Score: 150.0 -----Epsilon: 0.8140068224841878\n",
      "Episode: 147 -----Score: 1360.0 -----Epsilon: 0.8123739351193702\n",
      "Episode: 148 -----Score: 200.0 -----Epsilon: 0.8113104209547465\n",
      "Episode: 149 -----Score: 190.0 -----Epsilon: 0.8102304737985939\n",
      "Episode: 150 -----Score: 260.0 -----Epsilon: 0.8090710529441838\n",
      "Saved model to disk\n",
      "Episode: 151 -----Score: 220.0 -----Epsilon: 0.8081815632055253\n",
      "Episode: 152 -----Score: 210.0 -----Epsilon: 0.8072526876847823\n",
      "Episode: 153 -----Score: 300.0 -----Epsilon: 0.8060733453853625\n",
      "Episode: 154 -----Score: 180.0 -----Epsilon: 0.8051565547543562\n",
      "Episode: 155 -----Score: 260.0 -----Epsilon: 0.804078366440329\n",
      "Episode: 156 -----Score: 230.0 -----Epsilon: 0.8030674708311337\n",
      "Episode: 157 -----Score: 280.0 -----Epsilon: 0.8021172006663735\n",
      "Episode: 158 -----Score: 190.0 -----Epsilon: 0.8011087706550019\n",
      "Episode: 159 -----Score: 150.0 -----Epsilon: 0.8001016084522319\n",
      "Episode: 160 -----Score: 270.0 -----Epsilon: 0.7991836179151489\n",
      "Episode: 161 -----Score: 290.0 -----Epsilon: 0.7980575619438811\n",
      "Episode: 162 -----Score: 160.0 -----Epsilon: 0.7970606122179186\n",
      "Episode: 163 -----Score: 160.0 -----Epsilon: 0.7960887902311667\n",
      "Episode: 164 -----Score: 770.0 -----Epsilon: 0.7945887802268812\n",
      "Episode: 165 -----Score: 290.0 -----Epsilon: 0.7935675947973172\n",
      "Episode: 166 -----Score: 260.0 -----Epsilon: 0.7922925622308002\n",
      "Episode: 167 -----Score: 120.0 -----Epsilon: 0.7912964838561215\n",
      "Episode: 168 -----Score: 290.0 -----Epsilon: 0.7902257923680308\n",
      "Episode: 169 -----Score: 160.0 -----Epsilon: 0.7892559897059297\n",
      "Episode: 170 -----Score: 290.0 -----Epsilon: 0.7881108204578084\n",
      "Episode: 171 -----Score: 260.0 -----Epsilon: 0.7868602924064922\n",
      "Episode: 172 -----Score: 200.0 -----Epsilon: 0.7857971750573388\n",
      "Episode: 173 -----Score: 270.0 -----Epsilon: 0.7846664403189537\n",
      "Episode: 174 -----Score: 160.0 -----Epsilon: 0.7837896725664302\n",
      "Episode: 175 -----Score: 180.0 -----Epsilon: 0.782802718503703\n",
      "Episode: 176 -----Score: 180.0 -----Epsilon: 0.781817007221219\n",
      "Episode: 177 -----Score: 210.0 -----Epsilon: 0.7809106243901608\n",
      "Episode: 178 -----Score: 260.0 -----Epsilon: 0.7798212326415074\n",
      "Episode: 179 -----Score: 170.0 -----Epsilon: 0.7789140479034992\n",
      "Episode: 180 -----Score: 570.0 -----Epsilon: 0.7774479538654311\n",
      "Episode: 181 -----Score: 130.0 -----Epsilon: 0.7763773673179875\n",
      "Episode: 182 -----Score: 160.0 -----Epsilon: 0.7753888913564599\n",
      "Episode: 183 -----Score: 210.0 -----Epsilon: 0.7745379807382792\n",
      "Episode: 184 -----Score: 290.0 -----Epsilon: 0.7735193581377124\n",
      "Episode: 185 -----Score: 200.0 -----Epsilon: 0.7726071426958776\n",
      "Episode: 186 -----Score: 190.0 -----Epsilon: 0.7717932429541108\n",
      "Episode: 187 -----Score: 310.0 -----Epsilon: 0.7704283758130042\n",
      "Episode: 188 -----Score: 290.0 -----Epsilon: 0.7692228279613017\n",
      "Episode: 189 -----Score: 130.0 -----Epsilon: 0.768346412712641\n",
      "Episode: 190 -----Score: 180.0 -----Epsilon: 0.7675876605834219\n",
      "Episode: 191 -----Score: 220.0 -----Epsilon: 0.7665383175287249\n",
      "Episode: 192 -----Score: 290.0 -----Epsilon: 0.7654674446045586\n",
      "Episode: 193 -----Score: 170.0 -----Epsilon: 0.764561666570803\n",
      "Episode: 194 -----Score: 240.0 -----Epsilon: 0.7635699083262059\n",
      "Episode: 195 -----Score: 190.0 -----Epsilon: 0.762561134844841\n",
      "Episode: 196 -----Score: 270.0 -----Epsilon: 0.7615034331467909\n",
      "Episode: 197 -----Score: 230.0 -----Epsilon: 0.7605354156015148\n",
      "Episode: 198 -----Score: 190.0 -----Epsilon: 0.7596810531816663\n",
      "Episode: 199 -----Score: 680.0 -----Epsilon: 0.7583436724070853\n",
      "Episode: 200 -----Score: 180.0 -----Epsilon: 0.7575038921025019\n",
      "Saved model to disk\n",
      "Episode: 201 -----Score: 330.0 -----Epsilon: 0.7565560897299042\n",
      "Episode: 202 -----Score: 430.0 -----Epsilon: 0.7550792210243733\n",
      "Episode: 203 -----Score: 220.0 -----Epsilon: 0.754211378157667\n",
      "Episode: 204 -----Score: 210.0 -----Epsilon: 0.7533249460148055\n",
      "Episode: 205 -----Score: 150.0 -----Epsilon: 0.7523598012588705\n",
      "Episode: 206 -----Score: 670.0 -----Epsilon: 0.7509211603791729\n",
      "Episode: 207 -----Score: 150.0 -----Epsilon: 0.7500430955116\n",
      "Episode: 208 -----Score: 130.0 -----Epsilon: 0.7491660573801925\n",
      "Episode: 209 -----Score: 190.0 -----Epsilon: 0.7481748168750242\n",
      "Episode: 210 -----Score: 230.0 -----Epsilon: 0.7471893710324256\n",
      "Episode: 211 -----Score: 290.0 -----Epsilon: 0.7463007436304614\n",
      "Episode: 212 -----Score: 250.0 -----Epsilon: 0.7453043505793673\n",
      "Episode: 213 -----Score: 1930.0 -----Epsilon: 0.7437973786369217\n",
      "Episode: 214 -----Score: 260.0 -----Epsilon: 0.7428221553872955\n",
      "Episode: 215 -----Score: 1100.0 -----Epsilon: 0.741253486449445\n",
      "Episode: 216 -----Score: 190.0 -----Epsilon: 0.7402386634706792\n",
      "Episode: 217 -----Score: 250.0 -----Epsilon: 0.7392178376261304\n",
      "Episode: 218 -----Score: 270.0 -----Epsilon: 0.7382072779941118\n",
      "Episode: 219 -----Score: 230.0 -----Epsilon: 0.7372732979803105\n",
      "Episode: 220 -----Score: 140.0 -----Epsilon: 0.7364509591078873\n",
      "Episode: 221 -----Score: 140.0 -----Epsilon: 0.7356692625619742\n",
      "Episode: 222 -----Score: 650.0 -----Epsilon: 0.7341788356108568\n",
      "Episode: 223 -----Score: 150.0 -----Epsilon: 0.7333878164709903\n",
      "Episode: 224 -----Score: 200.0 -----Epsilon: 0.732377903040169\n",
      "Episode: 225 -----Score: 160.0 -----Epsilon: 0.7315873610838616\n",
      "Episode: 226 -----Score: 270.0 -----Epsilon: 0.7305170997394638\n",
      "Episode: 227 -----Score: 250.0 -----Epsilon: 0.7296322484579447\n",
      "Episode: 228 -----Score: 300.0 -----Epsilon: 0.7287615865712288\n",
      "Episode: 229 -----Score: 190.0 -----Epsilon: 0.7278366458930182\n",
      "Episode: 230 -----Score: 250.0 -----Epsilon: 0.7268590895284348\n",
      "Episode: 231 -----Score: 190.0 -----Epsilon: 0.7261006438580704\n",
      "Episode: 232 -----Score: 300.0 -----Epsilon: 0.7251196181294018\n",
      "Episode: 233 -----Score: 270.0 -----Epsilon: 0.7242470985988361\n",
      "Episode: 234 -----Score: 190.0 -----Epsilon: 0.7234335013737327\n",
      "Episode: 235 -----Score: 370.0 -----Epsilon: 0.7220371757376867\n",
      "Episode: 236 -----Score: 160.0 -----Epsilon: 0.7212953021675225\n",
      "Episode: 237 -----Score: 170.0 -----Epsilon: 0.7204489975145297\n",
      "Episode: 238 -----Score: 270.0 -----Epsilon: 0.7195446806992007\n",
      "Episode: 239 -----Score: 200.0 -----Epsilon: 0.7186688079170648\n",
      "Episode: 240 -----Score: 590.0 -----Epsilon: 0.7171913069029315\n",
      "Episode: 241 -----Score: 320.0 -----Epsilon: 0.71595163732572\n",
      "Episode: 242 -----Score: 230.0 -----Epsilon: 0.7150758477384351\n",
      "Episode: 243 -----Score: 710.0 -----Epsilon: 0.713804143610238\n",
      "Episode: 244 -----Score: 330.0 -----Epsilon: 0.7127741532289309\n",
      "Episode: 245 -----Score: 190.0 -----Epsilon: 0.7119549332752778\n",
      "Episode: 246 -----Score: 230.0 -----Epsilon: 0.7109645804633034\n",
      "Episode: 247 -----Score: 220.0 -----Epsilon: 0.7100608075346455\n",
      "Episode: 248 -----Score: 450.0 -----Epsilon: 0.7090319644264218\n",
      "Episode: 249 -----Score: 230.0 -----Epsilon: 0.7080626708777714\n",
      "Episode: 250 -----Score: 130.0 -----Epsilon: 0.707371938152681\n",
      "Saved model to disk\n",
      "Episode: 251 -----Score: 210.0 -----Epsilon: 0.706588603309536\n",
      "Episode: 252 -----Score: 220.0 -----Epsilon: 0.7054279248241798\n",
      "Episode: 253 -----Score: 290.0 -----Epsilon: 0.704417065128775\n",
      "Episode: 254 -----Score: 340.0 -----Epsilon: 0.7034132812581408\n",
      "Episode: 255 -----Score: 210.0 -----Epsilon: 0.7026750841959212\n",
      "Episode: 256 -----Score: 310.0 -----Epsilon: 0.7016681692488628\n",
      "Episode: 257 -----Score: 200.0 -----Epsilon: 0.7009430186042206\n",
      "Episode: 258 -----Score: 280.0 -----Epsilon: 0.7000337838778568\n",
      "Episode: 259 -----Score: 270.0 -----Epsilon: 0.6992082303597169\n",
      "Episode: 260 -----Score: 230.0 -----Epsilon: 0.698364095933176\n",
      "Episode: 261 -----Score: 210.0 -----Epsilon: 0.6975112153712436\n",
      "Episode: 262 -----Score: 170.0 -----Epsilon: 0.6965632439370821\n",
      "Episode: 263 -----Score: 400.0 -----Epsilon: 0.6955929102844131\n",
      "Episode: 264 -----Score: 200.0 -----Epsilon: 0.6948351263217243\n",
      "Episode: 265 -----Score: 150.0 -----Epsilon: 0.6940420767319404\n",
      "Episode: 266 -----Score: 330.0 -----Epsilon: 0.6931958608487823\n",
      "Episode: 267 -----Score: 250.0 -----Epsilon: 0.6920350364677652\n",
      "Episode: 268 -----Score: 220.0 -----Epsilon: 0.6912852761819869\n",
      "Episode: 269 -----Score: 140.0 -----Epsilon: 0.6905653313606731\n",
      "Episode: 270 -----Score: 340.0 -----Epsilon: 0.6895206054613604\n",
      "Episode: 271 -----Score: 200.0 -----Epsilon: 0.6886551103601318\n",
      "Episode: 272 -----Score: 230.0 -----Epsilon: 0.6878223407710558\n",
      "Episode: 273 -----Score: 340.0 -----Epsilon: 0.6869576034354787\n",
      "Episode: 274 -----Score: 1310.0 -----Epsilon: 0.685166980079909\n",
      "Episode: 275 -----Score: 670.0 -----Epsilon: 0.684007288592007\n",
      "Episode: 276 -----Score: 300.0 -----Epsilon: 0.6831733076977532\n",
      "Episode: 277 -----Score: 140.0 -----Epsilon: 0.6824140404511752\n",
      "Episode: 278 -----Score: 220.0 -----Epsilon: 0.6815547393742927\n",
      "Episode: 279 -----Score: 200.0 -----Epsilon: 0.6808272267460428\n",
      "Episode: 280 -----Score: 250.0 -----Epsilon: 0.6798692955946649\n",
      "Episode: 281 -----Score: 330.0 -----Epsilon: 0.678924932813635\n",
      "Episode: 282 -----Score: 270.0 -----Epsilon: 0.6780198498727447\n",
      "Episode: 283 -----Score: 360.0 -----Epsilon: 0.6769223458367695\n",
      "Episode: 284 -----Score: 140.0 -----Epsilon: 0.6762308839051125\n",
      "Episode: 285 -----Score: 320.0 -----Epsilon: 0.6753091328348779\n",
      "Episode: 286 -----Score: 300.0 -----Epsilon: 0.6743468273360647\n",
      "Episode: 287 -----Score: 200.0 -----Epsilon: 0.6734101354610698\n",
      "Episode: 288 -----Score: 180.0 -----Epsilon: 0.6725433406754961\n",
      "Episode: 289 -----Score: 200.0 -----Epsilon: 0.6715607997413463\n",
      "Episode: 290 -----Score: 220.0 -----Epsilon: 0.6706816301963201\n",
      "Episode: 291 -----Score: 310.0 -----Epsilon: 0.6697446714271506\n",
      "Episode: 292 -----Score: 790.0 -----Epsilon: 0.6683076024005727\n",
      "Episode: 293 -----Score: 230.0 -----Epsilon: 0.667463394398809\n",
      "Episode: 294 -----Score: 310.0 -----Epsilon: 0.6665935884994295\n",
      "Episode: 295 -----Score: 470.0 -----Epsilon: 0.6652590712465001\n",
      "Episode: 296 -----Score: 350.0 -----Epsilon: 0.664341645979809\n",
      "Episode: 297 -----Score: 220.0 -----Epsilon: 0.6633989493728416\n",
      "Episode: 298 -----Score: 250.0 -----Epsilon: 0.6624602402847277\n",
      "Episode: 299 -----Score: 270.0 -----Epsilon: 0.6616048934719798\n",
      "Episode: 300 -----Score: 180.0 -----Epsilon: 0.6607440435747446\n",
      "Saved model to disk\n",
      "Episode: 301 -----Score: 260.0 -----Epsilon: 0.6599212683690977\n",
      "Episode: 302 -----Score: 190.0 -----Epsilon: 0.6592260770872284\n",
      "Episode: 303 -----Score: 330.0 -----Epsilon: 0.6581313522207684\n",
      "Episode: 304 -----Score: 270.0 -----Epsilon: 0.6572947405020091\n",
      "Episode: 305 -----Score: 310.0 -----Epsilon: 0.6564762604558451\n",
      "Episode: 306 -----Score: 560.0 -----Epsilon: 0.6552917330951314\n",
      "Episode: 307 -----Score: 230.0 -----Epsilon: 0.6543788919045133\n",
      "Episode: 308 -----Score: 1160.0 -----Epsilon: 0.6529264747873231\n",
      "Episode: 309 -----Score: 230.0 -----Epsilon: 0.6519947602521425\n",
      "Episode: 310 -----Score: 160.0 -----Epsilon: 0.6510956271284332\n",
      "Episode: 311 -----Score: 260.0 -----Epsilon: 0.6501795286540335\n",
      "Episode: 312 -----Score: 170.0 -----Epsilon: 0.6494361478248003\n",
      "Episode: 313 -----Score: 470.0 -----Epsilon: 0.6484082542454138\n",
      "Episode: 314 -----Score: 190.0 -----Epsilon: 0.6477057598161058\n",
      "Episode: 315 -----Score: 630.0 -----Epsilon: 0.6466767249285187\n",
      "Episode: 316 -----Score: 650.0 -----Epsilon: 0.6456196256924168\n",
      "Episode: 317 -----Score: 180.0 -----Epsilon: 0.6447963395950029\n",
      "Episode: 318 -----Score: 140.0 -----Epsilon: 0.6439496327663051\n",
      "Episode: 319 -----Score: 240.0 -----Epsilon: 0.6430333001565979\n",
      "Episode: 320 -----Score: 190.0 -----Epsilon: 0.6423122206739988\n",
      "Episode: 321 -----Score: 180.0 -----Epsilon: 0.6416484124220406\n",
      "Episode: 322 -----Score: 220.0 -----Epsilon: 0.6408468521844926\n",
      "Episode: 323 -----Score: 170.0 -----Epsilon: 0.6401794368808791\n",
      "Episode: 324 -----Score: 210.0 -----Epsilon: 0.6393630880444131\n",
      "Episode: 325 -----Score: 240.0 -----Epsilon: 0.6385299011003902\n",
      "Episode: 326 -----Score: 290.0 -----Epsilon: 0.637677393901647\n",
      "Episode: 327 -----Score: 230.0 -----Epsilon: 0.6367725337002603\n",
      "Episode: 328 -----Score: 360.0 -----Epsilon: 0.6359300039009392\n",
      "Episode: 329 -----Score: 170.0 -----Epsilon: 0.6351266953706665\n",
      "Episode: 330 -----Score: 260.0 -----Epsilon: 0.6344398592448094\n",
      "Episode: 331 -----Score: 250.0 -----Epsilon: 0.6336498386752153\n",
      "Episode: 332 -----Score: 320.0 -----Epsilon: 0.6329215594732824\n",
      "Episode: 333 -----Score: 470.0 -----Epsilon: 0.6318022780500395\n",
      "Episode: 334 -----Score: 330.0 -----Epsilon: 0.6309562291307993\n",
      "Episode: 335 -----Score: 1220.0 -----Epsilon: 0.6291655954327502\n",
      "Episode: 336 -----Score: 230.0 -----Epsilon: 0.6283105109571403\n",
      "Episode: 337 -----Score: 250.0 -----Epsilon: 0.6272106735810067\n",
      "Episode: 338 -----Score: 260.0 -----Epsilon: 0.6263895647212577\n",
      "Episode: 339 -----Score: 190.0 -----Epsilon: 0.6257209371082243\n",
      "Episode: 340 -----Score: 270.0 -----Epsilon: 0.6247355765940427\n",
      "Episode: 341 -----Score: 670.0 -----Epsilon: 0.6236095684526077\n",
      "Episode: 342 -----Score: 340.0 -----Epsilon: 0.6227558074499109\n",
      "Episode: 343 -----Score: 780.0 -----Epsilon: 0.6213449963323942\n",
      "Episode: 344 -----Score: 300.0 -----Epsilon: 0.6205266022490274\n",
      "Episode: 345 -----Score: 400.0 -----Epsilon: 0.6196733439674234\n",
      "Episode: 346 -----Score: 180.0 -----Epsilon: 0.6189388462902161\n",
      "Episode: 347 -----Score: 230.0 -----Epsilon: 0.6182348938041402\n",
      "Episode: 348 -----Score: 990.0 -----Epsilon: 0.6168639332981876\n",
      "Episode: 349 -----Score: 190.0 -----Epsilon: 0.6160649946657754\n",
      "Episode: 350 -----Score: 260.0 -----Epsilon: 0.61533231262174\n",
      "Saved model to disk\n",
      "Episode: 351 -----Score: 350.0 -----Epsilon: 0.6145501067263155\n",
      "Episode: 352 -----Score: 220.0 -----Epsilon: 0.6136780639996141\n",
      "Episode: 353 -----Score: 210.0 -----Epsilon: 0.6129420912588236\n",
      "Episode: 354 -----Score: 410.0 -----Epsilon: 0.6120772268990562\n",
      "Episode: 355 -----Score: 140.0 -----Epsilon: 0.6114434426151671\n",
      "Episode: 356 -----Score: 290.0 -----Epsilon: 0.6106503030924395\n",
      "Episode: 357 -----Score: 210.0 -----Epsilon: 0.6099606572835836\n",
      "Episode: 358 -----Score: 270.0 -----Epsilon: 0.609137765150547\n",
      "Episode: 359 -----Score: 200.0 -----Epsilon: 0.6085849185373032\n",
      "Episode: 360 -----Score: 290.0 -----Epsilon: 0.6078343871704095\n",
      "Episode: 361 -----Score: 180.0 -----Epsilon: 0.6072645052617761\n",
      "Episode: 362 -----Score: 220.0 -----Epsilon: 0.6063882472504887\n",
      "Episode: 363 -----Score: 220.0 -----Epsilon: 0.6056682850350442\n",
      "Episode: 364 -----Score: 1010.0 -----Epsilon: 0.6045923625269074\n",
      "Episode: 365 -----Score: 160.0 -----Epsilon: 0.6037670523495905\n",
      "Episode: 366 -----Score: 210.0 -----Epsilon: 0.6031816816741306\n",
      "Episode: 367 -----Score: 420.0 -----Epsilon: 0.6019331818737006\n",
      "Episode: 368 -----Score: 180.0 -----Epsilon: 0.6012185091386396\n",
      "Episode: 369 -----Score: 230.0 -----Epsilon: 0.6006199930155454\n",
      "Episode: 370 -----Score: 270.0 -----Epsilon: 0.5998552895992671\n",
      "Episode: 371 -----Score: 170.0 -----Epsilon: 0.5992006044914825\n",
      "Episode: 372 -----Score: 220.0 -----Epsilon: 0.5984580554749203\n",
      "Episode: 373 -----Score: 170.0 -----Epsilon: 0.597893377076027\n",
      "Episode: 374 -----Score: 90.0 -----Epsilon: 0.5974021101671774\n",
      "Episode: 375 -----Score: 130.0 -----Epsilon: 0.5968372344344475\n",
      "Episode: 376 -----Score: 250.0 -----Epsilon: 0.5960141662755319\n",
      "Episode: 377 -----Score: 270.0 -----Epsilon: 0.5951529517380758\n",
      "Episode: 378 -----Score: 280.0 -----Epsilon: 0.5942430630543917\n",
      "Episode: 379 -----Score: 150.0 -----Epsilon: 0.5935945031591442\n",
      "Episode: 380 -----Score: 240.0 -----Epsilon: 0.593029669530783\n",
      "Episode: 381 -----Score: 360.0 -----Epsilon: 0.5922177743468405\n",
      "Episode: 382 -----Score: 300.0 -----Epsilon: 0.5914129048042271\n",
      "Episode: 383 -----Score: 280.0 -----Epsilon: 0.5905831428853101\n",
      "Episode: 384 -----Score: 320.0 -----Epsilon: 0.5897203403299836\n",
      "Episode: 385 -----Score: 230.0 -----Epsilon: 0.5888776420734921\n",
      "Episode: 386 -----Score: 340.0 -----Epsilon: 0.5880714312781542\n",
      "Episode: 387 -----Score: 310.0 -----Epsilon: 0.587256928041053\n",
      "Episode: 388 -----Score: 350.0 -----Epsilon: 0.5863415205214361\n",
      "Episode: 389 -----Score: 230.0 -----Epsilon: 0.5854638375946761\n",
      "Episode: 390 -----Score: 240.0 -----Epsilon: 0.5846669778357131\n",
      "Episode: 391 -----Score: 180.0 -----Epsilon: 0.5839657975534597\n",
      "Episode: 392 -----Score: 190.0 -----Epsilon: 0.5833144545890407\n",
      "Episode: 393 -----Score: 250.0 -----Epsilon: 0.5824913949729585\n",
      "Episode: 394 -----Score: 300.0 -----Epsilon: 0.5816811302160195\n",
      "Episode: 395 -----Score: 290.0 -----Epsilon: 0.5808708308201149\n",
      "Episode: 396 -----Score: 250.0 -----Epsilon: 0.580130111578667\n",
      "Episode: 397 -----Score: 320.0 -----Epsilon: 0.5792385563572374\n",
      "Episode: 398 -----Score: 230.0 -----Epsilon: 0.5784490127679837\n",
      "Episode: 399 -----Score: 480.0 -----Epsilon: 0.5774133593241098\n",
      "Episode: 400 -----Score: 220.0 -----Epsilon: 0.5766482158467591\n",
      "Saved model to disk\n",
      "Episode: 401 -----Score: 570.0 -----Epsilon: 0.5756411142216367\n",
      "Episode: 402 -----Score: 260.0 -----Epsilon: 0.5748081882392861\n",
      "Episode: 403 -----Score: 210.0 -----Epsilon: 0.5740097590960356\n",
      "Episode: 404 -----Score: 300.0 -----Epsilon: 0.5733282397233889\n",
      "Episode: 405 -----Score: 390.0 -----Epsilon: 0.5723132006577993\n",
      "Episode: 406 -----Score: 300.0 -----Epsilon: 0.57160740103034\n",
      "Episode: 407 -----Score: 390.0 -----Epsilon: 0.570606820627554\n",
      "Episode: 408 -----Score: 170.0 -----Epsilon: 0.5699612585181494\n",
      "Episode: 409 -----Score: 320.0 -----Epsilon: 0.5692788530905235\n",
      "Episode: 410 -----Score: 230.0 -----Epsilon: 0.56870758337615\n",
      "Episode: 411 -----Score: 310.0 -----Epsilon: 0.5679176281913657\n",
      "Episode: 412 -----Score: 210.0 -----Epsilon: 0.5672217871211137\n",
      "Episode: 413 -----Score: 370.0 -----Epsilon: 0.5664622181916632\n",
      "Episode: 414 -----Score: 290.0 -----Epsilon: 0.565668593828829\n",
      "Episode: 415 -----Score: 280.0 -----Epsilon: 0.5649540397050379\n",
      "Episode: 416 -----Score: 190.0 -----Epsilon: 0.5642889150165138\n",
      "Episode: 417 -----Score: 300.0 -----Epsilon: 0.5635851210056062\n",
      "Episode: 418 -----Score: 290.0 -----Epsilon: 0.5628146629031292\n",
      "Episode: 419 -----Score: 100.0 -----Epsilon: 0.5623196033628735\n",
      "Episode: 420 -----Score: 270.0 -----Epsilon: 0.5615340290162312\n",
      "Episode: 421 -----Score: 160.0 -----Epsilon: 0.5609492147143635\n",
      "Episode: 422 -----Score: 230.0 -----Epsilon: 0.56015323131547\n",
      "Episode: 423 -----Score: 590.0 -----Epsilon: 0.559149216285264\n",
      "Episode: 424 -----Score: 430.0 -----Epsilon: 0.5581112805443955\n",
      "Episode: 425 -----Score: 170.0 -----Epsilon: 0.5575456419675245\n",
      "Episode: 426 -----Score: 310.0 -----Epsilon: 0.5567823266651037\n",
      "Episode: 427 -----Score: 290.0 -----Epsilon: 0.5560689883565543\n",
      "Episode: 428 -----Score: 410.0 -----Epsilon: 0.5552965886325295\n",
      "Episode: 429 -----Score: 320.0 -----Epsilon: 0.5546761133362988\n",
      "Episode: 430 -----Score: 490.0 -----Epsilon: 0.5535700828936648\n",
      "Episode: 431 -----Score: 220.0 -----Epsilon: 0.5529869268203903\n",
      "Episode: 432 -----Score: 200.0 -----Epsilon: 0.5523579850045122\n",
      "Episode: 433 -----Score: 330.0 -----Epsilon: 0.5515874304591241\n",
      "Episode: 434 -----Score: 260.0 -----Epsilon: 0.5509071906839772\n",
      "Episode: 435 -----Score: 130.0 -----Epsilon: 0.5501991786780616\n",
      "Episode: 436 -----Score: 390.0 -----Epsilon: 0.5494514156414365\n",
      "Episode: 437 -----Score: 370.0 -----Epsilon: 0.5486980844492352\n",
      "Episode: 438 -----Score: 230.0 -----Epsilon: 0.5479392108032\n",
      "Episode: 439 -----Score: 350.0 -----Epsilon: 0.5472142186138602\n",
      "Episode: 440 -----Score: 540.0 -----Epsilon: 0.5463503019584368\n",
      "Episode: 441 -----Score: 510.0 -----Epsilon: 0.5453808439862246\n",
      "Episode: 442 -----Score: 330.0 -----Epsilon: 0.5443695548951029\n",
      "Episode: 443 -----Score: 1030.0 -----Epsilon: 0.5435460021522006\n",
      "Episode: 444 -----Score: 320.0 -----Epsilon: 0.5427942540631651\n",
      "Episode: 445 -----Score: 280.0 -----Epsilon: 0.5420522184496716\n",
      "Episode: 446 -----Score: 400.0 -----Epsilon: 0.5411012090337303\n",
      "Episode: 447 -----Score: 380.0 -----Epsilon: 0.5401507878228772\n",
      "Episode: 448 -----Score: 230.0 -----Epsilon: 0.5395407615523823\n",
      "Episode: 449 -----Score: 160.0 -----Epsilon: 0.5389378914452985\n",
      "Episode: 450 -----Score: 320.0 -----Epsilon: 0.5380461479727287\n",
      "Saved model to disk\n",
      "Episode: 451 -----Score: 310.0 -----Epsilon: 0.5373750845396942\n",
      "Episode: 452 -----Score: 240.0 -----Epsilon: 0.5367735607596992\n",
      "Episode: 453 -----Score: 320.0 -----Epsilon: 0.5359754348561198\n",
      "Episode: 454 -----Score: 230.0 -----Epsilon: 0.5351335425313967\n",
      "Episode: 455 -----Score: 840.0 -----Epsilon: 0.5338945383015165\n",
      "Episode: 456 -----Score: 280.0 -----Epsilon: 0.5331412104347616\n",
      "Episode: 457 -----Score: 370.0 -----Epsilon: 0.5323825568819971\n",
      "Episode: 458 -----Score: 330.0 -----Epsilon: 0.5316940986934645\n",
      "Episode: 459 -----Score: 390.0 -----Epsilon: 0.5307676313732259\n",
      "Episode: 460 -----Score: 340.0 -----Epsilon: 0.5300844420492179\n",
      "Episode: 461 -----Score: 250.0 -----Epsilon: 0.5293947205221158\n",
      "Episode: 462 -----Score: 300.0 -----Epsilon: 0.528693207627046\n",
      "Episode: 463 -----Score: 360.0 -----Epsilon: 0.5280042402996752\n",
      "Episode: 464 -----Score: 420.0 -----Epsilon: 0.5271632711301826\n",
      "Episode: 465 -----Score: 250.0 -----Epsilon: 0.5264731386903293\n",
      "Episode: 466 -----Score: 220.0 -----Epsilon: 0.5258701454515317\n",
      "Episode: 467 -----Score: 130.0 -----Epsilon: 0.5253067141453286\n",
      "Episode: 468 -----Score: 300.0 -----Epsilon: 0.5246662300883197\n",
      "Episode: 469 -----Score: 280.0 -----Epsilon: 0.5239919423021298\n",
      "Episode: 470 -----Score: 290.0 -----Epsilon: 0.523370855614697\n",
      "Episode: 471 -----Score: 260.0 -----Epsilon: 0.5228174215143013\n",
      "Episode: 472 -----Score: 400.0 -----Epsilon: 0.5220786793553481\n",
      "Episode: 473 -----Score: 310.0 -----Epsilon: 0.5213420237247297\n",
      "Episode: 474 -----Score: 650.0 -----Epsilon: 0.5200746190288411\n",
      "Episode: 475 -----Score: 260.0 -----Epsilon: 0.5194249312448438\n",
      "Episode: 476 -----Score: 270.0 -----Epsilon: 0.5188019945412036\n",
      "Episode: 477 -----Score: 200.0 -----Epsilon: 0.5182222974426849\n",
      "Episode: 478 -----Score: 260.0 -----Epsilon: 0.5176339305800925\n",
      "Episode: 479 -----Score: 240.0 -----Epsilon: 0.5168921747479451\n",
      "Episode: 480 -----Score: 330.0 -----Epsilon: 0.5161803871536609\n",
      "Episode: 481 -----Score: 130.0 -----Epsilon: 0.5156180564952325\n",
      "Episode: 482 -----Score: 200.0 -----Epsilon: 0.5149049339865438\n",
      "Episode: 483 -----Score: 380.0 -----Epsilon: 0.513988189539357\n",
      "Episode: 484 -----Score: 130.0 -----Epsilon: 0.5134046298971293\n",
      "Episode: 485 -----Score: 350.0 -----Epsilon: 0.5126955940462168\n",
      "Episode: 486 -----Score: 320.0 -----Epsilon: 0.5119650104284654\n",
      "Episode: 487 -----Score: 270.0 -----Epsilon: 0.5113632928537799\n",
      "Episode: 488 -----Score: 290.0 -----Epsilon: 0.5106070341935691\n",
      "Episode: 489 -----Score: 170.0 -----Epsilon: 0.5100517952898778\n",
      "Episode: 490 -----Score: 200.0 -----Epsilon: 0.5094655722819241\n",
      "Episode: 491 -----Score: 170.0 -----Epsilon: 0.508886129646634\n",
      "Episode: 492 -----Score: 230.0 -----Epsilon: 0.508284980989106\n",
      "Episode: 493 -----Score: 480.0 -----Epsilon: 0.5071598674323919\n",
      "Episode: 494 -----Score: 300.0 -----Epsilon: 0.5064624946599512\n",
      "Episode: 495 -----Score: 290.0 -----Epsilon: 0.5058824205069854\n",
      "Episode: 496 -----Score: 1480.0 -----Epsilon: 0.5046384686768262\n",
      "Episode: 497 -----Score: 390.0 -----Epsilon: 0.5037057496082394\n",
      "Episode: 498 -----Score: 510.0 -----Epsilon: 0.5028712965936636\n",
      "Episode: 499 -----Score: 260.0 -----Epsilon: 0.5022862943567945\n",
      "Episode: 500 -----Score: 340.0 -----Epsilon: 0.5014501812299302\n",
      "Saved model to disk\n",
      "Episode: 501 -----Score: 210.0 -----Epsilon: 0.5008167479815879\n",
      "Episode: 502 -----Score: 290.0 -----Epsilon: 0.500183114519638\n",
      "Episode: 503 -----Score: 240.0 -----Epsilon: 0.49950632419625274\n",
      "Episode: 504 -----Score: 600.0 -----Epsilon: 0.4983089452503118\n",
      "Episode: 505 -----Score: 710.0 -----Epsilon: 0.49734216735705794\n",
      "Episode: 506 -----Score: 220.0 -----Epsilon: 0.4964944241024305\n",
      "Episode: 507 -----Score: 520.0 -----Epsilon: 0.4956292915765265\n",
      "Episode: 508 -----Score: 540.0 -----Epsilon: 0.4944975759281257\n",
      "Episode: 509 -----Score: 320.0 -----Epsilon: 0.49389761951823413\n",
      "Episode: 510 -----Score: 210.0 -----Epsilon: 0.4933990342518924\n",
      "Episode: 511 -----Score: 270.0 -----Epsilon: 0.49281223802735247\n",
      "Episode: 512 -----Score: 260.0 -----Epsilon: 0.492173966417955\n",
      "Episode: 513 -----Score: 410.0 -----Epsilon: 0.49140775564526584\n",
      "Episode: 514 -----Score: 420.0 -----Epsilon: 0.4907713030678673\n",
      "Episode: 515 -----Score: 480.0 -----Epsilon: 0.48990340530966425\n",
      "Episode: 516 -----Score: 170.0 -----Epsilon: 0.4894157039458837\n",
      "Episode: 517 -----Score: 260.0 -----Epsilon: 0.48880236068640653\n",
      "Episode: 518 -----Score: 230.0 -----Epsilon: 0.4882063848296025\n",
      "Episode: 519 -----Score: 210.0 -----Epsilon: 0.48770866769826104\n",
      "Episode: 520 -----Score: 650.0 -----Epsilon: 0.48675953501984703\n",
      "Episode: 521 -----Score: 220.0 -----Epsilon: 0.4862068895974658\n",
      "Episode: 522 -----Score: 460.0 -----Epsilon: 0.48547812524423234\n",
      "Episode: 523 -----Score: 270.0 -----Epsilon: 0.4847824478368876\n",
      "Episode: 524 -----Score: 360.0 -----Epsilon: 0.48411294056057474\n",
      "Episode: 525 -----Score: 260.0 -----Epsilon: 0.4834956057744554\n",
      "Episode: 526 -----Score: 350.0 -----Epsilon: 0.48291189513119553\n",
      "Episode: 527 -----Score: 940.0 -----Epsilon: 0.4821359960180183\n",
      "Episode: 528 -----Score: 530.0 -----Epsilon: 0.4814152590928955\n",
      "Episode: 529 -----Score: 240.0 -----Epsilon: 0.4807513635647438\n",
      "Episode: 530 -----Score: 400.0 -----Epsilon: 0.4800134955601581\n",
      "Episode: 531 -----Score: 260.0 -----Epsilon: 0.4794694682339739\n",
      "Episode: 532 -----Score: 240.0 -----Epsilon: 0.47877091042314124\n",
      "Episode: 533 -----Score: 730.0 -----Epsilon: 0.47768819809209007\n",
      "Episode: 534 -----Score: 520.0 -----Epsilon: 0.4769054307091712\n",
      "Episode: 535 -----Score: 260.0 -----Epsilon: 0.4763963670884852\n",
      "Episode: 536 -----Score: 340.0 -----Epsilon: 0.4757555682919385\n",
      "Episode: 537 -----Score: 300.0 -----Epsilon: 0.47524678157452893\n",
      "Episode: 538 -----Score: 370.0 -----Epsilon: 0.4743475151451159\n",
      "Episode: 539 -----Score: 270.0 -----Epsilon: 0.4738222286355282\n",
      "Episode: 540 -----Score: 460.0 -----Epsilon: 0.4728244623949205\n",
      "Episode: 541 -----Score: 320.0 -----Epsilon: 0.472140307337036\n",
      "Episode: 542 -----Score: 210.0 -----Epsilon: 0.47154955696841244\n",
      "Episode: 543 -----Score: 370.0 -----Epsilon: 0.4708578293793486\n",
      "Episode: 544 -----Score: 180.0 -----Epsilon: 0.4702395278863034\n",
      "Episode: 545 -----Score: 180.0 -----Epsilon: 0.4696361271952568\n",
      "Episode: 546 -----Score: 330.0 -----Epsilon: 0.4689969175514047\n",
      "Episode: 547 -----Score: 210.0 -----Epsilon: 0.46833797057467563\n",
      "Episode: 548 -----Score: 340.0 -----Epsilon: 0.4677426228022285\n",
      "Episode: 549 -----Score: 1080.0 -----Epsilon: 0.4668155404812962\n",
      "Episode: 550 -----Score: 160.0 -----Epsilon: 0.46632657356933316\n",
      "Saved model to disk\n",
      "Episode: 551 -----Score: 1080.0 -----Epsilon: 0.4654972497562411\n",
      "Episode: 552 -----Score: 210.0 -----Epsilon: 0.464913881470056\n",
      "Episode: 553 -----Score: 330.0 -----Epsilon: 0.46422260336163074\n",
      "Episode: 554 -----Score: 280.0 -----Epsilon: 0.46366865179417727\n",
      "Episode: 555 -----Score: 230.0 -----Epsilon: 0.4630597906862277\n",
      "Episode: 556 -----Score: 250.0 -----Epsilon: 0.4625562551034674\n",
      "Episode: 557 -----Score: 250.0 -----Epsilon: 0.46194331136777966\n",
      "Episode: 558 -----Score: 360.0 -----Epsilon: 0.4611005716939884\n",
      "Episode: 559 -----Score: 330.0 -----Epsilon: 0.46027501856051173\n",
      "Episode: 560 -----Score: 420.0 -----Epsilon: 0.459442673444162\n",
      "Episode: 561 -----Score: 230.0 -----Epsilon: 0.45892838519081247\n",
      "Episode: 562 -----Score: 300.0 -----Epsilon: 0.4583523323998568\n",
      "Episode: 563 -----Score: 370.0 -----Epsilon: 0.4576946101504252\n",
      "Episode: 564 -----Score: 290.0 -----Epsilon: 0.4571731350706638\n",
      "Episode: 565 -----Score: 420.0 -----Epsilon: 0.4562615266693176\n",
      "Episode: 566 -----Score: 220.0 -----Epsilon: 0.4557289237742189\n",
      "Episode: 567 -----Score: 460.0 -----Epsilon: 0.45505130273312333\n",
      "Episode: 568 -----Score: 250.0 -----Epsilon: 0.4544664822931452\n",
      "Episode: 569 -----Score: 200.0 -----Epsilon: 0.4539268961502394\n",
      "Episode: 570 -----Score: 220.0 -----Epsilon: 0.4533489609295291\n",
      "Episode: 571 -----Score: 1220.0 -----Epsilon: 0.4522613224145615\n",
      "Episode: 572 -----Score: 340.0 -----Epsilon: 0.45163492179121933\n",
      "Episode: 573 -----Score: 320.0 -----Epsilon: 0.4510761431546217\n",
      "Episode: 574 -----Score: 490.0 -----Epsilon: 0.450371210762638\n",
      "Episode: 575 -----Score: 320.0 -----Epsilon: 0.44970785195018226\n",
      "Episode: 576 -----Score: 1320.0 -----Epsilon: 0.4486110040752318\n",
      "Episode: 577 -----Score: 370.0 -----Epsilon: 0.4478015429558371\n",
      "Episode: 578 -----Score: 430.0 -----Epsilon: 0.44704003219772\n",
      "Episode: 579 -----Score: 410.0 -----Epsilon: 0.44647086526386803\n",
      "Episode: 580 -----Score: 440.0 -----Epsilon: 0.4455716857778035\n",
      "Episode: 581 -----Score: 280.0 -----Epsilon: 0.44497145918877523\n",
      "Episode: 582 -----Score: 280.0 -----Epsilon: 0.4443258288254951\n",
      "Episode: 583 -----Score: 360.0 -----Epsilon: 0.44375212997023356\n",
      "Episode: 584 -----Score: 500.0 -----Epsilon: 0.4429718123221464\n",
      "Episode: 585 -----Score: 240.0 -----Epsilon: 0.4425379121579881\n",
      "Episode: 586 -----Score: 350.0 -----Epsilon: 0.44189669653783087\n",
      "Episode: 587 -----Score: 720.0 -----Epsilon: 0.4406716911773854\n",
      "Episode: 588 -----Score: 290.0 -----Epsilon: 0.4402285979262534\n",
      "Episode: 589 -----Score: 250.0 -----Epsilon: 0.43963644824744574\n",
      "Episode: 590 -----Score: 340.0 -----Epsilon: 0.4390003147060987\n",
      "Episode: 591 -----Score: 790.0 -----Epsilon: 0.43778684084416114\n",
      "Episode: 592 -----Score: 190.0 -----Epsilon: 0.4373335280803016\n",
      "Episode: 593 -----Score: 210.0 -----Epsilon: 0.43680554761482093\n",
      "Episode: 594 -----Score: 210.0 -----Epsilon: 0.436317471409961\n",
      "Episode: 595 -----Score: 250.0 -----Epsilon: 0.43572099659440844\n",
      "Episode: 596 -----Score: 220.0 -----Epsilon: 0.43517842578190724\n",
      "Episode: 597 -----Score: 240.0 -----Epsilon: 0.434709555736417\n",
      "Episode: 598 -----Score: 240.0 -----Epsilon: 0.43427245738359443\n",
      "Episode: 599 -----Score: 320.0 -----Epsilon: 0.43366403339481124\n",
      "Episode: 600 -----Score: 300.0 -----Epsilon: 0.43300709614751587\n",
      "Saved model to disk\n",
      "Episode: 601 -----Score: 250.0 -----Epsilon: 0.4324999085936193\n",
      "Episode: 602 -----Score: 280.0 -----Epsilon: 0.43190865263870315\n",
      "Episode: 603 -----Score: 220.0 -----Epsilon: 0.43127076252914337\n",
      "Episode: 604 -----Score: 210.0 -----Epsilon: 0.4306553467762791\n",
      "Episode: 605 -----Score: 240.0 -----Epsilon: 0.4301216645639623\n",
      "Episode: 606 -----Score: 280.0 -----Epsilon: 0.4294992984731166\n",
      "Episode: 607 -----Score: 200.0 -----Epsilon: 0.4288666822272807\n",
      "Episode: 608 -----Score: 320.0 -----Epsilon: 0.4282915285790104\n",
      "Episode: 609 -----Score: 230.0 -----Epsilon: 0.4276145063695619\n",
      "Episode: 610 -----Score: 280.0 -----Epsilon: 0.42708373829709884\n",
      "Episode: 611 -----Score: 230.0 -----Epsilon: 0.4266193234129385\n",
      "Episode: 612 -----Score: 420.0 -----Epsilon: 0.4260386628334227\n",
      "Episode: 613 -----Score: 160.0 -----Epsilon: 0.4256060269356112\n",
      "Episode: 614 -----Score: 310.0 -----Epsilon: 0.4249918947287664\n",
      "Episode: 615 -----Score: 240.0 -----Epsilon: 0.42447966292936395\n",
      "Episode: 616 -----Score: 350.0 -----Epsilon: 0.4239841596178374\n",
      "Episode: 617 -----Score: 180.0 -----Epsilon: 0.42349008169706337\n",
      "Episode: 618 -----Score: 260.0 -----Epsilon: 0.4230168838794091\n",
      "Episode: 619 -----Score: 330.0 -----Epsilon: 0.4224064877008111\n",
      "Episode: 620 -----Score: 550.0 -----Epsilon: 0.4216443092761454\n",
      "Episode: 621 -----Score: 260.0 -----Epsilon: 0.4211335854237652\n",
      "Episode: 622 -----Score: 1020.0 -----Epsilon: 0.42058478457550563\n",
      "Episode: 623 -----Score: 210.0 -----Epsilon: 0.42012743671490277\n",
      "Episode: 624 -----Score: 370.0 -----Epsilon: 0.41963701384112134\n",
      "Episode: 625 -----Score: 360.0 -----Epsilon: 0.4190842960252278\n",
      "Episode: 626 -----Score: 280.0 -----Epsilon: 0.418545699473767\n",
      "Episode: 627 -----Score: 420.0 -----Epsilon: 0.41794844216089194\n",
      "Episode: 628 -----Score: 280.0 -----Epsilon: 0.4173804180554876\n",
      "Episode: 629 -----Score: 290.0 -----Epsilon: 0.4167814893100071\n",
      "Episode: 630 -----Score: 270.0 -----Epsilon: 0.4161626113382477\n",
      "Episode: 631 -----Score: 200.0 -----Epsilon: 0.4157383414648441\n",
      "Episode: 632 -----Score: 120.0 -----Epsilon: 0.4153768060369591\n",
      "Episode: 633 -----Score: 370.0 -----Epsilon: 0.414716051601097\n",
      "Episode: 634 -----Score: 410.0 -----Epsilon: 0.41413999616461\n",
      "Episode: 635 -----Score: 380.0 -----Epsilon: 0.4135771480304782\n",
      "Episode: 636 -----Score: 440.0 -----Epsilon: 0.41287631496103777\n",
      "Episode: 637 -----Score: 680.0 -----Epsilon: 0.4121898593759457\n",
      "Episode: 638 -----Score: 220.0 -----Epsilon: 0.4117103490292967\n",
      "Episode: 639 -----Score: 730.0 -----Epsilon: 0.4107530011647164\n",
      "Episode: 640 -----Score: 390.0 -----Epsilon: 0.41021444605805074\n",
      "Episode: 641 -----Score: 270.0 -----Epsilon: 0.4096536558030585\n",
      "Episode: 642 -----Score: 430.0 -----Epsilon: 0.40905926972918044\n",
      "Episode: 643 -----Score: 370.0 -----Epsilon: 0.40849760768111504\n",
      "Episode: 644 -----Score: 160.0 -----Epsilon: 0.408118697363867\n",
      "Episode: 645 -----Score: 220.0 -----Epsilon: 0.4075933783344348\n",
      "Episode: 646 -----Score: 330.0 -----Epsilon: 0.40709804551445106\n",
      "Episode: 647 -----Score: 320.0 -----Epsilon: 0.40657485338938226\n",
      "Episode: 648 -----Score: 220.0 -----Epsilon: 0.4061359891382224\n",
      "Episode: 649 -----Score: 210.0 -----Epsilon: 0.4056537855865579\n",
      "Episode: 650 -----Score: 310.0 -----Epsilon: 0.40514784492885547\n",
      "Saved model to disk\n",
      "Episode: 651 -----Score: 640.0 -----Epsilon: 0.4045276330138602\n",
      "Episode: 652 -----Score: 280.0 -----Epsilon: 0.4039212958245983\n",
      "Episode: 653 -----Score: 440.0 -----Epsilon: 0.40315537952372527\n",
      "Episode: 654 -----Score: 230.0 -----Epsilon: 0.4024569131425464\n",
      "Episode: 655 -----Score: 320.0 -----Epsilon: 0.4018793991920313\n",
      "Episode: 656 -----Score: 280.0 -----Epsilon: 0.40124011555404704\n",
      "Episode: 657 -----Score: 190.0 -----Epsilon: 0.4007901761576716\n",
      "Episode: 658 -----Score: 180.0 -----Epsilon: 0.4003639617703006\n",
      "Episode: 659 -----Score: 1990.0 -----Epsilon: 0.3995864094767898\n",
      "Episode: 660 -----Score: 710.0 -----Epsilon: 0.39883908268706136\n",
      "Episode: 661 -----Score: 230.0 -----Epsilon: 0.3982420684195261\n",
      "Episode: 662 -----Score: 770.0 -----Epsilon: 0.39744319987973054\n",
      "Episode: 663 -----Score: 300.0 -----Epsilon: 0.39705707239169014\n",
      "Episode: 664 -----Score: 330.0 -----Epsilon: 0.3965507501632896\n",
      "Episode: 665 -----Score: 320.0 -----Epsilon: 0.3959627046996707\n",
      "Episode: 666 -----Score: 250.0 -----Epsilon: 0.39541981583287156\n",
      "Episode: 667 -----Score: 340.0 -----Epsilon: 0.3948642456735998\n",
      "Episode: 668 -----Score: 1110.0 -----Epsilon: 0.3940248671349671\n",
      "Episode: 669 -----Score: 310.0 -----Epsilon: 0.39349486590466254\n",
      "Episode: 670 -----Score: 290.0 -----Epsilon: 0.39288306338732526\n",
      "Episode: 671 -----Score: 290.0 -----Epsilon: 0.3924040378359818\n",
      "Episode: 672 -----Score: 210.0 -----Epsilon: 0.3918762167765097\n",
      "Episode: 673 -----Score: 290.0 -----Epsilon: 0.39132171218096945\n",
      "Episode: 674 -----Score: 310.0 -----Epsilon: 0.39083521015152956\n",
      "Episode: 675 -----Score: 330.0 -----Epsilon: 0.3903391639945867\n",
      "Episode: 676 -----Score: 240.0 -----Epsilon: 0.38979385056092763\n",
      "Episode: 677 -----Score: 260.0 -----Epsilon: 0.3893380579906394\n",
      "Episode: 678 -----Score: 280.0 -----Epsilon: 0.3887731487888739\n",
      "Episode: 679 -----Score: 250.0 -----Epsilon: 0.3882401172389759\n",
      "Episode: 680 -----Score: 290.0 -----Epsilon: 0.3877977752458351\n",
      "Episode: 681 -----Score: 240.0 -----Epsilon: 0.38742256825368276\n",
      "Episode: 682 -----Score: 250.0 -----Epsilon: 0.38687436556794685\n",
      "Episode: 683 -----Score: 510.0 -----Epsilon: 0.3862991240226036\n",
      "Episode: 684 -----Score: 150.0 -----Epsilon: 0.38586130866638296\n",
      "Episode: 685 -----Score: 200.0 -----Epsilon: 0.38544403210047196\n",
      "Episode: 686 -----Score: 360.0 -----Epsilon: 0.38492095383234587\n",
      "Episode: 687 -----Score: 170.0 -----Epsilon: 0.3844923902154799\n",
      "Episode: 688 -----Score: 370.0 -----Epsilon: 0.3839821227036388\n",
      "Episode: 689 -----Score: 400.0 -----Epsilon: 0.3834878716013962\n",
      "Episode: 690 -----Score: 1100.0 -----Epsilon: 0.38282807307628913\n",
      "Episode: 691 -----Score: 420.0 -----Epsilon: 0.3823551894074045\n",
      "Episode: 692 -----Score: 400.0 -----Epsilon: 0.38180423001053\n",
      "Episode: 693 -----Score: 330.0 -----Epsilon: 0.3813028682177837\n",
      "Episode: 694 -----Score: 690.0 -----Epsilon: 0.38045807483350075\n",
      "Episode: 695 -----Score: 530.0 -----Epsilon: 0.37978525898592813\n",
      "Episode: 696 -----Score: 250.0 -----Epsilon: 0.37924255364968396\n",
      "Episode: 697 -----Score: 260.0 -----Epsilon: 0.3787990988900926\n",
      "Episode: 698 -----Score: 210.0 -----Epsilon: 0.37831076260882246\n",
      "Episode: 699 -----Score: 420.0 -----Epsilon: 0.3778034095680949\n",
      "Episode: 700 -----Score: 260.0 -----Epsilon: 0.37727636344509236\n",
      "Saved model to disk\n",
      "Episode: 701 -----Score: 210.0 -----Epsilon: 0.3768608334761217\n",
      "Episode: 702 -----Score: 690.0 -----Epsilon: 0.3760326501500058\n",
      "Episode: 703 -----Score: 410.0 -----Epsilon: 0.37545775952387644\n",
      "Episode: 704 -----Score: 300.0 -----Epsilon: 0.37493923476484087\n",
      "Episode: 705 -----Score: 320.0 -----Epsilon: 0.37439521750406995\n",
      "Episode: 706 -----Score: 260.0 -----Epsilon: 0.37383554044184475\n",
      "Episode: 707 -----Score: 330.0 -----Epsilon: 0.37332970913933994\n",
      "Episode: 708 -----Score: 310.0 -----Epsilon: 0.372771624887502\n",
      "Episode: 709 -----Score: 250.0 -----Epsilon: 0.3722769122368581\n",
      "Episode: 710 -----Score: 300.0 -----Epsilon: 0.37163491584662284\n",
      "Episode: 711 -----Score: 390.0 -----Epsilon: 0.3710719435920318\n",
      "Episode: 712 -----Score: 260.0 -----Epsilon: 0.3705365018520327\n",
      "Episode: 713 -----Score: 260.0 -----Epsilon: 0.37006621871741074\n",
      "Episode: 714 -----Score: 200.0 -----Epsilon: 0.3695544008185841\n",
      "Episode: 715 -----Score: 230.0 -----Epsilon: 0.3690610053072975\n",
      "Episode: 716 -----Score: 340.0 -----Epsilon: 0.3685321505763145\n",
      "Episode: 717 -----Score: 230.0 -----Epsilon: 0.3680666197499307\n",
      "Episode: 718 -----Score: 300.0 -----Epsilon: 0.36756050785989136\n",
      "Episode: 719 -----Score: 800.0 -----Epsilon: 0.36688114884185985\n",
      "Episode: 720 -----Score: 350.0 -----Epsilon: 0.3663305064416356\n",
      "Episode: 721 -----Score: 320.0 -----Epsilon: 0.3658721471819937\n",
      "Episode: 722 -----Score: 210.0 -----Epsilon: 0.3654721015178187\n",
      "Episode: 723 -----Score: 270.0 -----Epsilon: 0.365011896199948\n",
      "Episode: 724 -----Score: 210.0 -----Epsilon: 0.364652171484009\n",
      "Episode: 725 -----Score: 250.0 -----Epsilon: 0.364186443206894\n",
      "Episode: 726 -----Score: 340.0 -----Epsilon: 0.36368275729687144\n",
      "Episode: 727 -----Score: 260.0 -----Epsilon: 0.3631616094540828\n",
      "Episode: 728 -----Score: 220.0 -----Epsilon: 0.362752919208947\n",
      "Episode: 729 -----Score: 310.0 -----Epsilon: 0.3622946887235242\n",
      "Episode: 730 -----Score: 290.0 -----Epsilon: 0.3618326950533775\n",
      "Episode: 731 -----Score: 320.0 -----Epsilon: 0.36132142466466677\n",
      "Episode: 732 -----Score: 240.0 -----Epsilon: 0.36095450815545\n",
      "Episode: 733 -----Score: 210.0 -----Epsilon: 0.3603140212292374\n",
      "Episode: 734 -----Score: 300.0 -----Epsilon: 0.3597869069168808\n",
      "Episode: 735 -----Score: 300.0 -----Epsilon: 0.35927062318271896\n",
      "Episode: 736 -----Score: 210.0 -----Epsilon: 0.35884478037249434\n",
      "Episode: 737 -----Score: 130.0 -----Epsilon: 0.35846890765850464\n",
      "Episode: 738 -----Score: 240.0 -----Epsilon: 0.35797957293302896\n",
      "Episode: 739 -----Score: 160.0 -----Epsilon: 0.357584581165104\n",
      "Episode: 740 -----Score: 260.0 -----Epsilon: 0.35721002844940436\n",
      "Episode: 741 -----Score: 280.0 -----Epsilon: 0.35668246146784083\n",
      "Episode: 742 -----Score: 230.0 -----Epsilon: 0.35621622532692554\n",
      "Episode: 743 -----Score: 570.0 -----Epsilon: 0.35563322017841925\n",
      "Episode: 744 -----Score: 690.0 -----Epsilon: 0.3549496389952027\n",
      "Episode: 745 -----Score: 340.0 -----Epsilon: 0.35443533446526354\n",
      "Episode: 746 -----Score: 1010.0 -----Epsilon: 0.35375476331025574\n",
      "Episode: 747 -----Score: 230.0 -----Epsilon: 0.3533347517676976\n",
      "Episode: 748 -----Score: 360.0 -----Epsilon: 0.3528721858271703\n",
      "Episode: 749 -----Score: 390.0 -----Epsilon: 0.3524059965503381\n",
      "Episode: 750 -----Score: 180.0 -----Epsilon: 0.3519685795573979\n",
      "Saved model to disk\n",
      "Episode: 751 -----Score: 420.0 -----Epsilon: 0.35138128197223106\n",
      "Episode: 752 -----Score: 230.0 -----Epsilon: 0.3509353105499622\n",
      "Episode: 753 -----Score: 310.0 -----Epsilon: 0.35049411106076683\n",
      "Episode: 754 -----Score: 230.0 -----Epsilon: 0.35002616311928225\n",
      "Episode: 755 -----Score: 210.0 -----Epsilon: 0.349591700047653\n",
      "Episode: 756 -----Score: 300.0 -----Epsilon: 0.34909074431994497\n",
      "Episode: 757 -----Score: 230.0 -----Epsilon: 0.3485891120880095\n",
      "Episode: 758 -----Score: 320.0 -----Epsilon: 0.34816896659735114\n",
      "Episode: 759 -----Score: 200.0 -----Epsilon: 0.3478286234626053\n",
      "Episode: 760 -----Score: 210.0 -----Epsilon: 0.34745247604832896\n",
      "Episode: 761 -----Score: 370.0 -----Epsilon: 0.3470260658507894\n",
      "Episode: 762 -----Score: 500.0 -----Epsilon: 0.34635556535192624\n",
      "Episode: 763 -----Score: 180.0 -----Epsilon: 0.34591804804900317\n",
      "Episode: 764 -----Score: 300.0 -----Epsilon: 0.3454845382507033\n",
      "Episode: 765 -----Score: 270.0 -----Epsilon: 0.34505847284087166\n",
      "Episode: 766 -----Score: 380.0 -----Epsilon: 0.344557811006113\n",
      "Episode: 767 -----Score: 550.0 -----Epsilon: 0.34401383897183724\n",
      "Episode: 768 -----Score: 420.0 -----Epsilon: 0.3435524815786433\n",
      "Episode: 769 -----Score: 330.0 -----Epsilon: 0.34312811260110865\n",
      "Episode: 770 -----Score: 330.0 -----Epsilon: 0.34277075896401277\n",
      "Episode: 771 -----Score: 250.0 -----Epsilon: 0.34228231571610457\n",
      "Episode: 772 -----Score: 180.0 -----Epsilon: 0.3418287496926432\n",
      "Episode: 773 -----Score: 1310.0 -----Epsilon: 0.341098699797199\n",
      "Episode: 774 -----Score: 640.0 -----Epsilon: 0.34050093647169705\n",
      "Episode: 775 -----Score: 350.0 -----Epsilon: 0.3401197885244041\n",
      "Episode: 776 -----Score: 280.0 -----Epsilon: 0.33972411901922844\n",
      "Episode: 777 -----Score: 240.0 -----Epsilon: 0.3393261951827417\n",
      "Episode: 778 -----Score: 440.0 -----Epsilon: 0.33879996885885966\n",
      "Episode: 779 -----Score: 140.0 -----Epsilon: 0.3384931547880287\n",
      "Episode: 780 -----Score: 660.0 -----Epsilon: 0.33787157515463534\n",
      "Episode: 781 -----Score: 170.0 -----Epsilon: 0.33745692309417713\n",
      "Episode: 782 -----Score: 300.0 -----Epsilon: 0.3370710927253875\n",
      "Episode: 783 -----Score: 430.0 -----Epsilon: 0.33664193715425617\n",
      "Episode: 784 -----Score: 330.0 -----Epsilon: 0.33625367604027995\n",
      "Episode: 785 -----Score: 880.0 -----Epsilon: 0.3354207992849006\n",
      "Episode: 786 -----Score: 290.0 -----Epsilon: 0.33488187551157866\n",
      "Episode: 787 -----Score: 470.0 -----Epsilon: 0.33431105351075996\n",
      "Episode: 788 -----Score: 650.0 -----Epsilon: 0.33360906901848636\n",
      "Episode: 789 -----Score: 260.0 -----Epsilon: 0.3331996481082099\n",
      "Episode: 790 -----Score: 300.0 -----Epsilon: 0.33276943171228623\n",
      "Episode: 791 -----Score: 430.0 -----Epsilon: 0.3322686576249026\n",
      "Episode: 792 -----Score: 900.0 -----Epsilon: 0.3315882038997056\n",
      "Episode: 793 -----Score: 340.0 -----Epsilon: 0.33110774893763506\n",
      "Episode: 794 -----Score: 340.0 -----Epsilon: 0.3306491510207914\n",
      "Episode: 795 -----Score: 260.0 -----Epsilon: 0.3303431113191648\n",
      "Episode: 796 -----Score: 260.0 -----Epsilon: 0.32988557245702543\n",
      "Episode: 797 -----Score: 280.0 -----Epsilon: 0.3294121962670636\n",
      "Episode: 798 -----Score: 190.0 -----Epsilon: 0.32904148651212667\n",
      "Episode: 799 -----Score: 250.0 -----Epsilon: 0.32864490127178553\n",
      "Episode: 800 -----Score: 250.0 -----Epsilon: 0.3282612677290791\n",
      "Saved model to disk\n",
      "Episode: 801 -----Score: 240.0 -----Epsilon: 0.32778825563169994\n",
      "Episode: 802 -----Score: 250.0 -----Epsilon: 0.3274180637304695\n",
      "Episode: 803 -----Score: 270.0 -----Epsilon: 0.3268377384058597\n",
      "Episode: 804 -----Score: 270.0 -----Epsilon: 0.3264764553322769\n",
      "Episode: 805 -----Score: 220.0 -----Epsilon: 0.3260601366267859\n",
      "Episode: 806 -----Score: 250.0 -----Epsilon: 0.3256730067980038\n",
      "Episode: 807 -----Score: 460.0 -----Epsilon: 0.32521022843297076\n",
      "Episode: 808 -----Score: 170.0 -----Epsilon: 0.3248637386948225\n",
      "Episode: 809 -----Score: 640.0 -----Epsilon: 0.3243054527695273\n",
      "Episode: 810 -----Score: 220.0 -----Epsilon: 0.3239476167536692\n",
      "Episode: 811 -----Score: 240.0 -----Epsilon: 0.3236193000266847\n",
      "Episode: 812 -----Score: 440.0 -----Epsilon: 0.323198221458615\n",
      "Episode: 813 -----Score: 460.0 -----Epsilon: 0.3227970580404616\n",
      "Episode: 814 -----Score: 320.0 -----Epsilon: 0.3223983269445823\n",
      "Episode: 815 -----Score: 330.0 -----Epsilon: 0.32194342129101555\n",
      "Episode: 816 -----Score: 270.0 -----Epsilon: 0.32145508143289553\n",
      "Episode: 817 -----Score: 510.0 -----Epsilon: 0.3209225499659414\n",
      "Episode: 818 -----Score: 260.0 -----Epsilon: 0.3205113904552959\n",
      "Episode: 819 -----Score: 310.0 -----Epsilon: 0.3201820737144059\n",
      "Episode: 820 -----Score: 550.0 -----Epsilon: 0.3196503725422711\n",
      "Episode: 821 -----Score: 150.0 -----Epsilon: 0.31937303612299395\n",
      "Episode: 822 -----Score: 130.0 -----Epsilon: 0.31905765107379747\n",
      "Episode: 823 -----Score: 300.0 -----Epsilon: 0.31866353903438227\n",
      "Episode: 824 -----Score: 250.0 -----Epsilon: 0.3182724599890582\n",
      "Episode: 825 -----Score: 240.0 -----Epsilon: 0.31792509581075484\n",
      "Episode: 826 -----Score: 290.0 -----Epsilon: 0.3174993610626303\n",
      "Episode: 827 -----Score: 260.0 -----Epsilon: 0.31711795571977014\n",
      "Episode: 828 -----Score: 290.0 -----Epsilon: 0.31677882063764895\n",
      "Episode: 829 -----Score: 190.0 -----Epsilon: 0.3163539881618816\n",
      "Episode: 830 -----Score: 390.0 -----Epsilon: 0.31589181609912964\n",
      "Episode: 831 -----Score: 200.0 -----Epsilon: 0.31547322072246553\n",
      "Episode: 832 -----Score: 430.0 -----Epsilon: 0.3150444683312534\n",
      "Episode: 833 -----Score: 350.0 -----Epsilon: 0.31461566941494723\n",
      "Episode: 834 -----Score: 290.0 -----Epsilon: 0.3141861973772486\n",
      "Episode: 835 -----Score: 290.0 -----Epsilon: 0.3137880613524335\n",
      "Episode: 836 -----Score: 270.0 -----Epsilon: 0.3132024517978436\n",
      "Episode: 837 -----Score: 510.0 -----Epsilon: 0.31269359791270146\n",
      "Episode: 838 -----Score: 330.0 -----Epsilon: 0.31219930723354206\n",
      "Episode: 839 -----Score: 440.0 -----Epsilon: 0.3115119769892492\n",
      "Episode: 840 -----Score: 260.0 -----Epsilon: 0.3110481693022748\n",
      "Episode: 841 -----Score: 330.0 -----Epsilon: 0.3106850767647667\n",
      "Episode: 842 -----Score: 290.0 -----Epsilon: 0.3102888950283457\n",
      "Episode: 843 -----Score: 250.0 -----Epsilon: 0.30989197892630116\n",
      "Episode: 844 -----Score: 480.0 -----Epsilon: 0.3094726687046171\n",
      "Episode: 845 -----Score: 300.0 -----Epsilon: 0.30905021721678194\n",
      "Episode: 846 -----Score: 270.0 -----Epsilon: 0.30869377861431435\n",
      "Episode: 847 -----Score: 650.0 -----Epsilon: 0.30810658425907245\n",
      "Episode: 848 -----Score: 670.0 -----Epsilon: 0.307409204471486\n",
      "Episode: 849 -----Score: 260.0 -----Epsilon: 0.3069539609939672\n",
      "Episode: 850 -----Score: 210.0 -----Epsilon: 0.3066030060961214\n",
      "Saved model to disk\n",
      "Episode: 851 -----Score: 450.0 -----Epsilon: 0.30610303757549934\n",
      "Episode: 852 -----Score: 320.0 -----Epsilon: 0.3056784608591836\n",
      "Episode: 853 -----Score: 870.0 -----Epsilon: 0.3051561968318703\n",
      "Episode: 854 -----Score: 450.0 -----Epsilon: 0.3045775590925823\n",
      "Episode: 855 -----Score: 260.0 -----Epsilon: 0.30426765659466654\n",
      "Episode: 856 -----Score: 240.0 -----Epsilon: 0.30398542690195635\n",
      "Episode: 857 -----Score: 290.0 -----Epsilon: 0.3035832161168333\n",
      "Episode: 858 -----Score: 180.0 -----Epsilon: 0.30323429573706073\n",
      "Episode: 859 -----Score: 290.0 -----Epsilon: 0.3028276278407155\n",
      "Episode: 860 -----Score: 310.0 -----Epsilon: 0.30239307901333046\n",
      "Episode: 861 -----Score: 200.0 -----Epsilon: 0.30204975517711763\n",
      "Episode: 862 -----Score: 230.0 -----Epsilon: 0.30174061422586385\n",
      "Episode: 863 -----Score: 270.0 -----Epsilon: 0.30138055057320134\n",
      "Episode: 864 -----Score: 740.0 -----Epsilon: 0.30076275114463685\n",
      "Episode: 865 -----Score: 250.0 -----Epsilon: 0.30044891836783205\n",
      "Episode: 866 -----Score: 530.0 -----Epsilon: 0.2999919824735746\n",
      "Episode: 867 -----Score: 310.0 -----Epsilon: 0.2995650974779113\n",
      "Episode: 868 -----Score: 220.0 -----Epsilon: 0.2991579654507515\n",
      "Episode: 869 -----Score: 300.0 -----Epsilon: 0.29874839924543817\n",
      "Episode: 870 -----Score: 340.0 -----Epsilon: 0.29828808374795696\n",
      "Episode: 871 -----Score: 210.0 -----Epsilon: 0.2979553795972462\n",
      "Episode: 872 -----Score: 170.0 -----Epsilon: 0.2976379280768746\n",
      "Episode: 873 -----Score: 220.0 -----Epsilon: 0.29731427378734193\n",
      "Episode: 874 -----Score: 290.0 -----Epsilon: 0.29687279243690806\n",
      "Episode: 875 -----Score: 250.0 -----Epsilon: 0.2964450099477437\n",
      "Episode: 876 -----Score: 240.0 -----Epsilon: 0.29609244984821886\n",
      "Episode: 877 -----Score: 520.0 -----Epsilon: 0.29550262916007164\n",
      "Episode: 878 -----Score: 280.0 -----Epsilon: 0.2950898047032229\n",
      "Episode: 879 -----Score: 100.0 -----Epsilon: 0.2948101918461939\n",
      "Episode: 880 -----Score: 1050.0 -----Epsilon: 0.2941364365235231\n",
      "Episode: 881 -----Score: 770.0 -----Epsilon: 0.29365033648697525\n",
      "Episode: 882 -----Score: 320.0 -----Epsilon: 0.29315859022976226\n",
      "Episode: 883 -----Score: 650.0 -----Epsilon: 0.29259567998647573\n",
      "Episode: 884 -----Score: 340.0 -----Epsilon: 0.2923090763921925\n",
      "Episode: 885 -----Score: 170.0 -----Epsilon: 0.29204027544051486\n",
      "Episode: 886 -----Score: 260.0 -----Epsilon: 0.2916742860924149\n",
      "Episode: 887 -----Score: 470.0 -----Epsilon: 0.2912405970656886\n",
      "Episode: 888 -----Score: 300.0 -----Epsilon: 0.290879100418533\n",
      "Episode: 889 -----Score: 530.0 -----Epsilon: 0.29041174221441335\n",
      "Episode: 890 -----Score: 640.0 -----Epsilon: 0.2898651210243201\n",
      "Episode: 891 -----Score: 380.0 -----Epsilon: 0.2894063376661926\n",
      "Episode: 892 -----Score: 150.0 -----Epsilon: 0.2890823836921504\n",
      "Episode: 893 -----Score: 290.0 -----Epsilon: 0.2886918080074906\n",
      "Episode: 894 -----Score: 240.0 -----Epsilon: 0.2883421251371368\n",
      "Episode: 895 -----Score: 400.0 -----Epsilon: 0.28791108738353416\n",
      "Episode: 896 -----Score: 360.0 -----Epsilon: 0.2874461983329688\n",
      "Episode: 897 -----Score: 200.0 -----Epsilon: 0.28718186917623684\n",
      "Episode: 898 -----Score: 400.0 -----Epsilon: 0.2866367411153385\n",
      "Episode: 899 -----Score: 380.0 -----Epsilon: 0.286231150310398\n",
      "Episode: 900 -----Score: 180.0 -----Epsilon: 0.2859324806235163\n",
      "Saved model to disk\n",
      "Episode: 901 -----Score: 250.0 -----Epsilon: 0.2856512611640355\n",
      "Episode: 902 -----Score: 200.0 -----Epsilon: 0.28530811427757274\n",
      "Episode: 903 -----Score: 360.0 -----Epsilon: 0.28493460497637774\n",
      "Episode: 904 -----Score: 400.0 -----Epsilon: 0.2845911806258681\n",
      "Episode: 905 -----Score: 230.0 -----Epsilon: 0.28424077983215484\n",
      "Episode: 906 -----Score: 280.0 -----Epsilon: 0.28389421718040947\n",
      "Episode: 907 -----Score: 370.0 -----Epsilon: 0.28343751477814333\n",
      "Episode: 908 -----Score: 280.0 -----Epsilon: 0.2830392812579102\n",
      "Episode: 909 -----Score: 240.0 -----Epsilon: 0.28272132351355445\n",
      "Episode: 910 -----Score: 230.0 -----Epsilon: 0.28238226106444203\n",
      "Episode: 911 -----Score: 300.0 -----Epsilon: 0.282000737833789\n",
      "Episode: 912 -----Score: 170.0 -----Epsilon: 0.2817628293846435\n",
      "Episode: 913 -----Score: 190.0 -----Epsilon: 0.2814637557969554\n",
      "Episode: 914 -----Score: 150.0 -----Epsilon: 0.28111608115431636\n",
      "Episode: 915 -----Score: 350.0 -----Epsilon: 0.28076097454729537\n",
      "Episode: 916 -----Score: 330.0 -----Epsilon: 0.28039734364438273\n",
      "Episode: 917 -----Score: 240.0 -----Epsilon: 0.28009411749027374\n",
      "Episode: 918 -----Score: 200.0 -----Epsilon: 0.27974981317202996\n",
      "Episode: 919 -----Score: 210.0 -----Epsilon: 0.2794567886464798\n",
      "Episode: 920 -----Score: 350.0 -----Epsilon: 0.27904963712707587\n",
      "Episode: 921 -----Score: 300.0 -----Epsilon: 0.27862747521152464\n",
      "Episode: 922 -----Score: 460.0 -----Epsilon: 0.27823766928424987\n",
      "Episode: 923 -----Score: 410.0 -----Epsilon: 0.277761177912377\n",
      "Episode: 924 -----Score: 310.0 -----Epsilon: 0.27736703655220674\n",
      "Episode: 925 -----Score: 320.0 -----Epsilon: 0.2769950592697697\n",
      "Episode: 926 -----Score: 260.0 -----Epsilon: 0.27663575256403855\n",
      "Episode: 927 -----Score: 320.0 -----Epsilon: 0.27626309842225877\n",
      "Episode: 928 -----Score: 430.0 -----Epsilon: 0.27571167525538653\n",
      "Episode: 929 -----Score: 230.0 -----Epsilon: 0.2754652989179706\n",
      "Episode: 930 -----Score: 220.0 -----Epsilon: 0.2751030246510821\n",
      "Episode: 931 -----Score: 290.0 -----Epsilon: 0.27473298470262364\n",
      "Episode: 932 -----Score: 310.0 -----Epsilon: 0.2743793560504472\n",
      "Episode: 933 -----Score: 320.0 -----Epsilon: 0.2740300189764772\n",
      "Episode: 934 -----Score: 560.0 -----Epsilon: 0.27356456322825684\n",
      "Episode: 935 -----Score: 2090.0 -----Epsilon: 0.2730714971424694\n",
      "Episode: 936 -----Score: 340.0 -----Epsilon: 0.272717825357329\n",
      "Episode: 937 -----Score: 350.0 -----Epsilon: 0.2723575302394676\n",
      "Episode: 938 -----Score: 250.0 -----Epsilon: 0.2720075032211739\n",
      "Episode: 939 -----Score: 200.0 -----Epsilon: 0.2716720526423325\n",
      "Episode: 940 -----Score: 170.0 -----Epsilon: 0.27138368978185323\n",
      "Episode: 941 -----Score: 370.0 -----Epsilon: 0.2710327460459079\n",
      "Episode: 942 -----Score: 720.0 -----Epsilon: 0.2704776974806423\n",
      "Episode: 943 -----Score: 640.0 -----Epsilon: 0.2700209757537536\n",
      "Episode: 944 -----Score: 390.0 -----Epsilon: 0.2696200221651401\n",
      "Episode: 945 -----Score: 700.0 -----Epsilon: 0.26913514216329554\n",
      "Episode: 946 -----Score: 370.0 -----Epsilon: 0.2687903316370135\n",
      "Episode: 947 -----Score: 230.0 -----Epsilon: 0.26852865704854245\n",
      "Episode: 948 -----Score: 220.0 -----Epsilon: 0.2682334376310253\n",
      "Episode: 949 -----Score: 280.0 -----Epsilon: 0.2678131767512053\n",
      "Episode: 950 -----Score: 260.0 -----Epsilon: 0.26742726807146017\n",
      "Saved model to disk\n",
      "Episode: 951 -----Score: 280.0 -----Epsilon: 0.26710654749989193\n",
      "Episode: 952 -----Score: 280.0 -----Epsilon: 0.2667456630996239\n",
      "Episode: 953 -----Score: 230.0 -----Epsilon: 0.2664742539139013\n",
      "Episode: 954 -----Score: 1420.0 -----Epsilon: 0.2657711574314131\n",
      "Episode: 955 -----Score: 930.0 -----Epsilon: 0.2652900156591113\n",
      "Episode: 956 -----Score: 940.0 -----Epsilon: 0.26478379481779396\n",
      "Episode: 957 -----Score: 230.0 -----Epsilon: 0.2645450672784525\n",
      "Episode: 958 -----Score: 420.0 -----Epsilon: 0.26419345586079357\n",
      "Episode: 959 -----Score: 150.0 -----Epsilon: 0.2638570873738892\n",
      "Episode: 960 -----Score: 100.0 -----Epsilon: 0.263630267598417\n",
      "Episode: 961 -----Score: 230.0 -----Epsilon: 0.2633009353101572\n",
      "Episode: 962 -----Score: 220.0 -----Epsilon: 0.26299515700915593\n",
      "Episode: 963 -----Score: 220.0 -----Epsilon: 0.2626913099612451\n",
      "Episode: 964 -----Score: 300.0 -----Epsilon: 0.26238728918230236\n",
      "Episode: 965 -----Score: 220.0 -----Epsilon: 0.26206632329058355\n",
      "Episode: 966 -----Score: 170.0 -----Epsilon: 0.26173842123544944\n",
      "Episode: 967 -----Score: 350.0 -----Epsilon: 0.26137276620933314\n",
      "Episode: 968 -----Score: 380.0 -----Epsilon: 0.2609074142263695\n",
      "Episode: 969 -----Score: 190.0 -----Epsilon: 0.2605992035429493\n",
      "Episode: 970 -----Score: 140.0 -----Epsilon: 0.2603236351114448\n",
      "Episode: 971 -----Score: 240.0 -----Epsilon: 0.2599953135700894\n",
      "Episode: 972 -----Score: 320.0 -----Epsilon: 0.2596372864073422\n",
      "Episode: 973 -----Score: 260.0 -----Epsilon: 0.2592698998135075\n",
      "Episode: 974 -----Score: 190.0 -----Epsilon: 0.25891131810937\n",
      "Episode: 975 -----Score: 250.0 -----Epsilon: 0.2585640918142057\n",
      "Episode: 976 -----Score: 1230.0 -----Epsilon: 0.2580861899619755\n",
      "Episode: 977 -----Score: 1040.0 -----Epsilon: 0.25750923834157885\n",
      "Episode: 978 -----Score: 400.0 -----Epsilon: 0.25720607075641805\n",
      "Episode: 979 -----Score: 290.0 -----Epsilon: 0.2568503434188722\n",
      "Episode: 980 -----Score: 380.0 -----Epsilon: 0.256504855076966\n",
      "Episode: 981 -----Score: 290.0 -----Epsilon: 0.25621670530222823\n",
      "Episode: 982 -----Score: 500.0 -----Epsilon: 0.25580401627732385\n",
      "Episode: 983 -----Score: 280.0 -----Epsilon: 0.25551460967653994\n",
      "Episode: 984 -----Score: 270.0 -----Epsilon: 0.25520358202522514\n",
      "Episode: 985 -----Score: 250.0 -----Epsilon: 0.2548776398428807\n",
      "Episode: 986 -----Score: 480.0 -----Epsilon: 0.2543841648151564\n",
      "Episode: 987 -----Score: 240.0 -----Epsilon: 0.25400338223285146\n",
      "Episode: 988 -----Score: 210.0 -----Epsilon: 0.2537221020620558\n",
      "Episode: 989 -----Score: 300.0 -----Epsilon: 0.25333268367376516\n",
      "Episode: 990 -----Score: 540.0 -----Epsilon: 0.25266880943187253\n",
      "Episode: 991 -----Score: 210.0 -----Epsilon: 0.2523390390239414\n",
      "Episode: 992 -----Score: 440.0 -----Epsilon: 0.2519628295235946\n",
      "Episode: 993 -----Score: 310.0 -----Epsilon: 0.2516153602803123\n",
      "Episode: 994 -----Score: 530.0 -----Epsilon: 0.25123972722419236\n",
      "Episode: 995 -----Score: 130.0 -----Epsilon: 0.2510312845578319\n",
      "Episode: 996 -----Score: 410.0 -----Epsilon: 0.250756304711466\n",
      "Episode: 997 -----Score: 320.0 -----Epsilon: 0.25042752783570865\n",
      "Episode: 998 -----Score: 380.0 -----Epsilon: 0.25002916399479513\n",
      "Episode: 999 -----Score: 370.0 -----Epsilon: 0.24965040657409715\n",
      "Episode: 1000 -----Score: 350.0 -----Epsilon: 0.24930213740669782\n",
      "Saved model to disk\n",
      "Episode: 1001 -----Score: 930.0 -----Epsilon: 0.2488378705894152\n",
      "Episode: 1002 -----Score: 360.0 -----Epsilon: 0.2485175734074721\n",
      "Episode: 1003 -----Score: 330.0 -----Epsilon: 0.24814408353837214\n",
      "Episode: 1004 -----Score: 370.0 -----Epsilon: 0.24779147306444352\n",
      "Episode: 1005 -----Score: 350.0 -----Epsilon: 0.24747747228117326\n",
      "Episode: 1006 -----Score: 1230.0 -----Epsilon: 0.24707243559216044\n",
      "Episode: 1007 -----Score: 420.0 -----Epsilon: 0.24664783583881933\n",
      "Episode: 1008 -----Score: 770.0 -----Epsilon: 0.24616832538577188\n",
      "Episode: 1009 -----Score: 200.0 -----Epsilon: 0.2458790013017072\n",
      "Episode: 1010 -----Score: 440.0 -----Epsilon: 0.24553648442111667\n",
      "Episode: 1011 -----Score: 220.0 -----Epsilon: 0.24532492295608152\n",
      "Episode: 1012 -----Score: 240.0 -----Epsilon: 0.24505423341857346\n",
      "Episode: 1013 -----Score: 260.0 -----Epsilon: 0.24471090776752247\n",
      "Episode: 1014 -----Score: 190.0 -----Epsilon: 0.24446778573375455\n",
      "Episode: 1015 -----Score: 610.0 -----Epsilon: 0.24409501201864486\n",
      "Episode: 1016 -----Score: 360.0 -----Epsilon: 0.24383006846104757\n",
      "Episode: 1017 -----Score: 450.0 -----Epsilon: 0.2434801794133191\n",
      "Episode: 1018 -----Score: 330.0 -----Epsilon: 0.2431906100382051\n",
      "Episode: 1019 -----Score: 210.0 -----Epsilon: 0.24296551954120926\n",
      "Episode: 1020 -----Score: 110.0 -----Epsilon: 0.2427668548125168\n",
      "Episode: 1021 -----Score: 240.0 -----Epsilon: 0.2424403101132866\n",
      "Episode: 1022 -----Score: 220.0 -----Epsilon: 0.24214955593780055\n",
      "Episode: 1023 -----Score: 230.0 -----Epsilon: 0.2418988186514589\n",
      "Episode: 1024 -----Score: 880.0 -----Epsilon: 0.24155363331423704\n",
      "Episode: 1025 -----Score: 890.0 -----Epsilon: 0.241039671125377\n",
      "Episode: 1026 -----Score: 270.0 -----Epsilon: 0.24071496822293026\n",
      "Episode: 1027 -----Score: 440.0 -----Epsilon: 0.2402950461671351\n",
      "Episode: 1028 -----Score: 1260.0 -----Epsilon: 0.23978232321886334\n",
      "Episode: 1029 -----Score: 560.0 -----Epsilon: 0.23934775192531454\n",
      "Episode: 1030 -----Score: 260.0 -----Epsilon: 0.23911521870736788\n",
      "Episode: 1031 -----Score: 650.0 -----Epsilon: 0.23852247004718455\n",
      "Episode: 1032 -----Score: 270.0 -----Epsilon: 0.2382902620328169\n",
      "Episode: 1033 -----Score: 220.0 -----Epsilon: 0.23799401295573758\n",
      "Episode: 1034 -----Score: 540.0 -----Epsilon: 0.23760497268322706\n",
      "Episode: 1035 -----Score: 180.0 -----Epsilon: 0.2373371051118927\n",
      "Episode: 1036 -----Score: 720.0 -----Epsilon: 0.2369083868752143\n",
      "Episode: 1037 -----Score: 270.0 -----Epsilon: 0.2366105432096346\n",
      "Episode: 1038 -----Score: 290.0 -----Epsilon: 0.23628282782826027\n",
      "Episode: 1039 -----Score: 540.0 -----Epsilon: 0.23589281044973553\n",
      "Episode: 1040 -----Score: 260.0 -----Epsilon: 0.23558163704840396\n",
      "Episode: 1041 -----Score: 380.0 -----Epsilon: 0.2352812262819564\n",
      "Episode: 1042 -----Score: 570.0 -----Epsilon: 0.23496428054141372\n",
      "Episode: 1043 -----Score: 750.0 -----Epsilon: 0.23448122080797287\n",
      "Episode: 1044 -----Score: 300.0 -----Epsilon: 0.23420141702663036\n",
      "Episode: 1045 -----Score: 320.0 -----Epsilon: 0.23393177206989327\n",
      "Episode: 1046 -----Score: 410.0 -----Epsilon: 0.2335773998310996\n",
      "Episode: 1047 -----Score: 1550.0 -----Epsilon: 0.2329885932715997\n",
      "Episode: 1048 -----Score: 230.0 -----Epsilon: 0.23277294547892532\n",
      "Episode: 1049 -----Score: 380.0 -----Epsilon: 0.23252308128671242\n",
      "Episode: 1050 -----Score: 850.0 -----Epsilon: 0.2319944600242897\n",
      "Saved model to disk\n",
      "Episode: 1051 -----Score: 360.0 -----Epsilon: 0.23168889243369456\n",
      "Episode: 1052 -----Score: 320.0 -----Epsilon: 0.23140918095168012\n",
      "Episode: 1053 -----Score: 190.0 -----Epsilon: 0.23117788720558835\n",
      "Episode: 1054 -----Score: 1230.0 -----Epsilon: 0.23075752567555022\n",
      "Episode: 1055 -----Score: 2100.0 -----Epsilon: 0.23017444645875648\n",
      "Episode: 1056 -----Score: 260.0 -----Epsilon: 0.22996186328620163\n",
      "Episode: 1057 -----Score: 360.0 -----Epsilon: 0.2295932997559085\n",
      "Episode: 1058 -----Score: 90.0 -----Epsilon: 0.22938446467867252\n",
      "Episode: 1059 -----Score: 300.0 -----Epsilon: 0.2290731716801397\n",
      "Episode: 1060 -----Score: 250.0 -----Epsilon: 0.22873896854294842\n",
      "Episode: 1061 -----Score: 250.0 -----Epsilon: 0.22852633997718896\n",
      "Episode: 1062 -----Score: 310.0 -----Epsilon: 0.228256381150481\n",
      "Episode: 1063 -----Score: 190.0 -----Epsilon: 0.22799722886871407\n",
      "Episode: 1064 -----Score: 240.0 -----Epsilon: 0.22776752322340954\n",
      "Episode: 1065 -----Score: 380.0 -----Epsilon: 0.22741020842015935\n",
      "Episode: 1066 -----Score: 300.0 -----Epsilon: 0.22714202236693437\n",
      "Episode: 1067 -----Score: 280.0 -----Epsilon: 0.22690274055974827\n",
      "Episode: 1068 -----Score: 380.0 -----Epsilon: 0.22657578227449024\n",
      "Episode: 1069 -----Score: 270.0 -----Epsilon: 0.22633754964564512\n",
      "Episode: 1070 -----Score: 1200.0 -----Epsilon: 0.22594406434727288\n",
      "Episode: 1071 -----Score: 890.0 -----Epsilon: 0.22547323580313014\n",
      "Episode: 1072 -----Score: 1000.0 -----Epsilon: 0.22492330135462962\n",
      "Episode: 1073 -----Score: 260.0 -----Epsilon: 0.2246270474214257\n",
      "Episode: 1074 -----Score: 250.0 -----Epsilon: 0.22435316924935553\n",
      "Episode: 1075 -----Score: 280.0 -----Epsilon: 0.2240962075276299\n",
      "Episode: 1076 -----Score: 420.0 -----Epsilon: 0.22379119594801042\n",
      "Episode: 1077 -----Score: 300.0 -----Epsilon: 0.22349911512432258\n",
      "Episode: 1078 -----Score: 360.0 -----Epsilon: 0.22319804098562857\n",
      "Episode: 1079 -----Score: 290.0 -----Epsilon: 0.22287062631522261\n",
      "Episode: 1080 -----Score: 390.0 -----Epsilon: 0.2224978526126474\n",
      "Episode: 1081 -----Score: 220.0 -----Epsilon: 0.22226835304140513\n",
      "Episode: 1082 -----Score: 660.0 -----Epsilon: 0.22191966517005401\n",
      "Episode: 1083 -----Score: 200.0 -----Epsilon: 0.2216974128103345\n",
      "Episode: 1084 -----Score: 210.0 -----Epsilon: 0.22145235077099937\n",
      "Episode: 1085 -----Score: 170.0 -----Epsilon: 0.22120003868402985\n",
      "Episode: 1086 -----Score: 90.0 -----Epsilon: 0.2210518841472731\n",
      "Episode: 1087 -----Score: 150.0 -----Epsilon: 0.22088041425263308\n",
      "Episode: 1088 -----Score: 1230.0 -----Epsilon: 0.22038972157316972\n",
      "Episode: 1089 -----Score: 210.0 -----Epsilon: 0.2200637857113464\n",
      "Episode: 1090 -----Score: 840.0 -----Epsilon: 0.21969263101261544\n",
      "Episode: 1091 -----Score: 340.0 -----Epsilon: 0.21940940995225522\n",
      "Episode: 1092 -----Score: 340.0 -----Epsilon: 0.2191033278049345\n",
      "Episode: 1093 -----Score: 180.0 -----Epsilon: 0.21885982007630367\n",
      "Episode: 1094 -----Score: 330.0 -----Epsilon: 0.2185383321708149\n",
      "Episode: 1095 -----Score: 230.0 -----Epsilon: 0.21828104532993967\n",
      "Episode: 1096 -----Score: 320.0 -----Epsilon: 0.21798351265000213\n",
      "Episode: 1097 -----Score: 190.0 -----Epsilon: 0.21770293033893148\n",
      "Episode: 1098 -----Score: 460.0 -----Epsilon: 0.21738966341230406\n",
      "Episode: 1099 -----Score: 270.0 -----Epsilon: 0.2169974115979433\n",
      "Episode: 1100 -----Score: 840.0 -----Epsilon: 0.21650408661262718\n",
      "Saved model to disk\n",
      "Episode: 1101 -----Score: 290.0 -----Epsilon: 0.21626649534941486\n",
      "Episode: 1102 -----Score: 420.0 -----Epsilon: 0.21600453887279297\n",
      "Episode: 1103 -----Score: 110.0 -----Epsilon: 0.21583309916806237\n",
      "Episode: 1104 -----Score: 250.0 -----Epsilon: 0.21561263019908192\n",
      "Episode: 1105 -----Score: 140.0 -----Epsilon: 0.21543159145559787\n",
      "Episode: 1106 -----Score: 400.0 -----Epsilon: 0.2150970704181298\n",
      "Episode: 1107 -----Score: 390.0 -----Epsilon: 0.21474245253753654\n",
      "Episode: 1108 -----Score: 400.0 -----Epsilon: 0.21448019726889161\n",
      "Episode: 1109 -----Score: 400.0 -----Epsilon: 0.2141647143554272\n",
      "Episode: 1110 -----Score: 310.0 -----Epsilon: 0.21394209861014019\n",
      "Episode: 1111 -----Score: 750.0 -----Epsilon: 0.21352873403046732\n",
      "Episode: 1112 -----Score: 400.0 -----Epsilon: 0.213301233451848\n",
      "Episode: 1113 -----Score: 130.0 -----Epsilon: 0.21313790710567831\n",
      "Episode: 1114 -----Score: 230.0 -----Epsilon: 0.2128639876511881\n",
      "Episode: 1115 -----Score: 300.0 -----Epsilon: 0.2126559082306133\n",
      "Episode: 1116 -----Score: 240.0 -----Epsilon: 0.21241404321248528\n",
      "Episode: 1117 -----Score: 210.0 -----Epsilon: 0.21215760171369977\n",
      "Episode: 1118 -----Score: 310.0 -----Epsilon: 0.21186714451751182\n",
      "Episode: 1119 -----Score: 240.0 -----Epsilon: 0.21158724092960027\n",
      "Episode: 1120 -----Score: 320.0 -----Epsilon: 0.21131869542757004\n",
      "Episode: 1121 -----Score: 310.0 -----Epsilon: 0.21101967960899842\n",
      "Episode: 1122 -----Score: 450.0 -----Epsilon: 0.2107021228342297\n",
      "Episode: 1123 -----Score: 270.0 -----Epsilon: 0.21040187543410402\n",
      "Episode: 1124 -----Score: 220.0 -----Epsilon: 0.21010625796941074\n",
      "Episode: 1125 -----Score: 290.0 -----Epsilon: 0.2097749714168912\n",
      "Episode: 1126 -----Score: 240.0 -----Epsilon: 0.20953638304337002\n",
      "Episode: 1127 -----Score: 400.0 -----Epsilon: 0.20924449252848176\n",
      "Episode: 1128 -----Score: 240.0 -----Epsilon: 0.20897014349226425\n",
      "Episode: 1129 -----Score: 600.0 -----Epsilon: 0.208575979674926\n",
      "Episode: 1130 -----Score: 380.0 -----Epsilon: 0.2082958415730853\n",
      "Episode: 1131 -----Score: 270.0 -----Epsilon: 0.2079882074092636\n",
      "Episode: 1132 -----Score: 780.0 -----Epsilon: 0.2075680797259936\n",
      "Episode: 1133 -----Score: 250.0 -----Epsilon: 0.20734734452038983\n",
      "Episode: 1134 -----Score: 220.0 -----Epsilon: 0.20711565949359928\n",
      "Episode: 1135 -----Score: 250.0 -----Epsilon: 0.20691071624940746\n",
      "Episode: 1136 -----Score: 270.0 -----Epsilon: 0.20663860056610647\n",
      "Episode: 1137 -----Score: 300.0 -----Epsilon: 0.20635322297608974\n",
      "Episode: 1138 -----Score: 280.0 -----Epsilon: 0.2061032741195046\n",
      "Episode: 1139 -----Score: 790.0 -----Epsilon: 0.20564458673358188\n",
      "Episode: 1140 -----Score: 320.0 -----Epsilon: 0.2053581176085836\n",
      "Episode: 1141 -----Score: 420.0 -----Epsilon: 0.20508230141271055\n",
      "Episode: 1142 -----Score: 220.0 -----Epsilon: 0.20483430165012223\n",
      "Episode: 1143 -----Score: 690.0 -----Epsilon: 0.20440010367481465\n",
      "Episode: 1144 -----Score: 520.0 -----Epsilon: 0.20404842922322558\n",
      "Episode: 1145 -----Score: 240.0 -----Epsilon: 0.20378374591404672\n",
      "Episode: 1146 -----Score: 1050.0 -----Epsilon: 0.20337089084103194\n",
      "Episode: 1147 -----Score: 250.0 -----Epsilon: 0.20311886697146359\n",
      "Episode: 1148 -----Score: 350.0 -----Epsilon: 0.20287567602610906\n",
      "Episode: 1149 -----Score: 180.0 -----Epsilon: 0.2025910381548808\n",
      "Episode: 1150 -----Score: 570.0 -----Epsilon: 0.20223074650138417\n",
      "Saved model to disk\n",
      "Episode: 1151 -----Score: 320.0 -----Epsilon: 0.20193933962838068\n",
      "Episode: 1152 -----Score: 400.0 -----Epsilon: 0.20162254329856769\n",
      "Episode: 1153 -----Score: 260.0 -----Epsilon: 0.20136543674508472\n",
      "Episode: 1154 -----Score: 380.0 -----Epsilon: 0.20107608105430103\n",
      "Episode: 1155 -----Score: 710.0 -----Epsilon: 0.20061574196631568\n",
      "Episode: 1156 -----Score: 310.0 -----Epsilon: 0.2003346754886594\n",
      "Episode: 1157 -----Score: 370.0 -----Epsilon: 0.2000576037985557\n",
      "Episode: 1158 -----Score: 290.0 -----Epsilon: 0.1997996956182961\n",
      "Episode: 1159 -----Score: 750.0 -----Epsilon: 0.19940887016643913\n",
      "Episode: 1160 -----Score: 260.0 -----Epsilon: 0.1991888440305297\n",
      "Episode: 1161 -----Score: 300.0 -----Epsilon: 0.19891852889554718\n",
      "Episode: 1162 -----Score: 300.0 -----Epsilon: 0.19866129452845765\n",
      "Episode: 1163 -----Score: 230.0 -----Epsilon: 0.19838177597349124\n",
      "Episode: 1164 -----Score: 250.0 -----Epsilon: 0.19814029382136433\n",
      "Episode: 1165 -----Score: 160.0 -----Epsilon: 0.197925625899234\n",
      "Episode: 1166 -----Score: 210.0 -----Epsilon: 0.19766928017963667\n",
      "Episode: 1167 -----Score: 690.0 -----Epsilon: 0.197325239695795\n",
      "Episode: 1168 -----Score: 280.0 -----Epsilon: 0.19706809502085934\n",
      "Episode: 1169 -----Score: 180.0 -----Epsilon: 0.19684435255096303\n",
      "Episode: 1170 -----Score: 1110.0 -----Epsilon: 0.1965272947548427\n",
      "Episode: 1171 -----Score: 420.0 -----Epsilon: 0.19618249530298218\n",
      "Episode: 1172 -----Score: 360.0 -----Epsilon: 0.19591978655074813\n",
      "Episode: 1173 -----Score: 210.0 -----Epsilon: 0.19570439305829435\n",
      "Episode: 1174 -----Score: 260.0 -----Epsilon: 0.19549197323985748\n",
      "Episode: 1175 -----Score: 330.0 -----Epsilon: 0.19525127518857863\n",
      "Episode: 1176 -----Score: 210.0 -----Epsilon: 0.1949796742200521\n",
      "Episode: 1177 -----Score: 290.0 -----Epsilon: 0.19471623955903752\n",
      "Episode: 1178 -----Score: 210.0 -----Epsilon: 0.19448699864900787\n",
      "Episode: 1179 -----Score: 300.0 -----Epsilon: 0.19427900864696407\n",
      "Episode: 1180 -----Score: 210.0 -----Epsilon: 0.19403825173562525\n",
      "Episode: 1181 -----Score: 940.0 -----Epsilon: 0.19359905841197594\n",
      "Episode: 1182 -----Score: 330.0 -----Epsilon: 0.1932764040284632\n",
      "Episode: 1183 -----Score: 320.0 -----Epsilon: 0.1929647072076714\n",
      "Episode: 1184 -----Score: 330.0 -----Epsilon: 0.19263578973464524\n",
      "Episode: 1185 -----Score: 260.0 -----Epsilon: 0.19243170377811203\n",
      "Episode: 1186 -----Score: 330.0 -----Epsilon: 0.19217055862093763\n",
      "Episode: 1187 -----Score: 350.0 -----Epsilon: 0.1919009402034411\n",
      "Episode: 1188 -----Score: 360.0 -----Epsilon: 0.19164549805789108\n",
      "Episode: 1189 -----Score: 240.0 -----Epsilon: 0.19138733371021438\n",
      "Episode: 1190 -----Score: 320.0 -----Epsilon: 0.19113448657099302\n",
      "Episode: 1191 -----Score: 360.0 -----Epsilon: 0.1908445642371877\n",
      "Episode: 1192 -----Score: 660.0 -----Epsilon: 0.19048268442497382\n",
      "Episode: 1193 -----Score: 380.0 -----Epsilon: 0.19022608649563785\n",
      "Episode: 1194 -----Score: 370.0 -----Epsilon: 0.18994324028586246\n",
      "Episode: 1195 -----Score: 710.0 -----Epsilon: 0.18966157328422584\n",
      "Episode: 1196 -----Score: 460.0 -----Epsilon: 0.1893799452061912\n",
      "Episode: 1197 -----Score: 210.0 -----Epsilon: 0.1891532036504088\n",
      "Episode: 1198 -----Score: 290.0 -----Epsilon: 0.18889008530067394\n",
      "Episode: 1199 -----Score: 240.0 -----Epsilon: 0.18864016006483295\n",
      "Episode: 1200 -----Score: 300.0 -----Epsilon: 0.1884214641286338\n",
      "Saved model to disk\n",
      "Episode: 1201 -----Score: 260.0 -----Epsilon: 0.18815259003078746\n",
      "Episode: 1202 -----Score: 860.0 -----Epsilon: 0.18771695698523375\n",
      "Episode: 1203 -----Score: 320.0 -----Epsilon: 0.18746146026709146\n",
      "Episode: 1204 -----Score: 250.0 -----Epsilon: 0.1872242839854877\n",
      "Episode: 1205 -----Score: 440.0 -----Epsilon: 0.18694104060564048\n",
      "Episode: 1206 -----Score: 230.0 -----Epsilon: 0.18672132693913826\n",
      "Episode: 1207 -----Score: 290.0 -----Epsilon: 0.18645040403913204\n",
      "Episode: 1208 -----Score: 440.0 -----Epsilon: 0.186192907294494\n",
      "Episode: 1209 -----Score: 320.0 -----Epsilon: 0.1859469226567433\n",
      "Episode: 1210 -----Score: 270.0 -----Epsilon: 0.18573060614309503\n",
      "Episode: 1211 -----Score: 200.0 -----Epsilon: 0.1855082338821489\n",
      "Episode: 1212 -----Score: 270.0 -----Epsilon: 0.1852624127336468\n",
      "Episode: 1213 -----Score: 280.0 -----Epsilon: 0.18504319162226263\n",
      "Episode: 1214 -----Score: 230.0 -----Epsilon: 0.18480353074006226\n",
      "Episode: 1215 -----Score: 280.0 -----Epsilon: 0.1845937129184161\n",
      "Episode: 1216 -----Score: 1180.0 -----Epsilon: 0.18421347246478278\n",
      "Episode: 1217 -----Score: 1220.0 -----Epsilon: 0.18375865868748284\n",
      "Episode: 1218 -----Score: 530.0 -----Epsilon: 0.18338197215573748\n",
      "Episode: 1219 -----Score: 710.0 -----Epsilon: 0.18300166569593784\n",
      "Episode: 1220 -----Score: 430.0 -----Epsilon: 0.18274673884433473\n",
      "Episode: 1221 -----Score: 550.0 -----Epsilon: 0.18247464868887817\n",
      "Episode: 1222 -----Score: 300.0 -----Epsilon: 0.18223977239929762\n",
      "Episode: 1223 -----Score: 310.0 -----Epsilon: 0.18202449202881066\n",
      "Episode: 1224 -----Score: 330.0 -----Epsilon: 0.1817927402514486\n",
      "Episode: 1225 -----Score: 1180.0 -----Epsilon: 0.18149266627383714\n",
      "Episode: 1226 -----Score: 860.0 -----Epsilon: 0.1810923721537581\n",
      "Episode: 1227 -----Score: 270.0 -----Epsilon: 0.1808650626872571\n",
      "Episode: 1228 -----Score: 230.0 -----Epsilon: 0.18064129005972035\n",
      "Episode: 1229 -----Score: 230.0 -----Epsilon: 0.18040516547534682\n",
      "Episode: 1230 -----Score: 280.0 -----Epsilon: 0.18017331331359251\n",
      "Episode: 1231 -----Score: 270.0 -----Epsilon: 0.17993816031958704\n",
      "Episode: 1232 -----Score: 260.0 -----Epsilon: 0.1797133779126843\n",
      "Episode: 1233 -----Score: 220.0 -----Epsilon: 0.17948169688989005\n",
      "Episode: 1234 -----Score: 280.0 -----Epsilon: 0.17927684557955745\n",
      "Episode: 1235 -----Score: 420.0 -----Epsilon: 0.17901994656671935\n",
      "Episode: 1236 -----Score: 210.0 -----Epsilon: 0.17881347649805984\n",
      "Episode: 1237 -----Score: 170.0 -----Epsilon: 0.17859867160811133\n",
      "Episode: 1238 -----Score: 410.0 -----Epsilon: 0.17832633760660194\n",
      "Episode: 1239 -----Score: 260.0 -----Epsilon: 0.17812066750181438\n",
      "Episode: 1240 -----Score: 400.0 -----Epsilon: 0.177918437109918\n",
      "Episode: 1241 -----Score: 170.0 -----Epsilon: 0.17775304971508157\n",
      "Episode: 1242 -----Score: 310.0 -----Epsilon: 0.17756117985789718\n",
      "Episode: 1243 -----Score: 340.0 -----Epsilon: 0.17730106561844114\n",
      "Episode: 1244 -----Score: 1500.0 -----Epsilon: 0.17696097382496903\n",
      "Episode: 1245 -----Score: 150.0 -----Epsilon: 0.17676536298504736\n",
      "Episode: 1246 -----Score: 280.0 -----Epsilon: 0.17656926209216856\n",
      "Episode: 1247 -----Score: 330.0 -----Epsilon: 0.17635433143535675\n",
      "Episode: 1248 -----Score: 410.0 -----Epsilon: 0.1760583045946986\n",
      "Episode: 1249 -----Score: 260.0 -----Epsilon: 0.17587178148002572\n",
      "Episode: 1250 -----Score: 270.0 -----Epsilon: 0.17563310945709332\n",
      "Saved model to disk\n",
      "Episode: 1251 -----Score: 280.0 -----Epsilon: 0.17539897086011114\n",
      "Episode: 1252 -----Score: 1070.0 -----Epsilon: 0.17506182733911144\n",
      "Episode: 1253 -----Score: 530.0 -----Epsilon: 0.17476028045534114\n",
      "Episode: 1254 -----Score: 1020.0 -----Epsilon: 0.17439471494318184\n",
      "Episode: 1255 -----Score: 650.0 -----Epsilon: 0.17408282731425853\n",
      "Episode: 1256 -----Score: 560.0 -----Epsilon: 0.17377983870579478\n",
      "Episode: 1257 -----Score: 1140.0 -----Epsilon: 0.17336222658510564\n",
      "Episode: 1258 -----Score: 480.0 -----Epsilon: 0.17310999460041693\n",
      "Episode: 1259 -----Score: 2130.0 -----Epsilon: 0.1728159573180708\n",
      "Episode: 1260 -----Score: 500.0 -----Epsilon: 0.17256762631590628\n",
      "Episode: 1261 -----Score: 2040.0 -----Epsilon: 0.17228588078155704\n",
      "Episode: 1262 -----Score: 770.0 -----Epsilon: 0.17194543578004062\n",
      "Episode: 1263 -----Score: 270.0 -----Epsilon: 0.17176395705601372\n",
      "Episode: 1264 -----Score: 800.0 -----Epsilon: 0.17150101587524172\n",
      "Episode: 1265 -----Score: 1010.0 -----Epsilon: 0.1712127933414267\n",
      "Episode: 1266 -----Score: 1200.0 -----Epsilon: 0.17081911439580144\n",
      "Episode: 1267 -----Score: 320.0 -----Epsilon: 0.17062585636973265\n",
      "Episode: 1268 -----Score: 140.0 -----Epsilon: 0.1704791810660038\n",
      "Episode: 1269 -----Score: 440.0 -----Epsilon: 0.17022603688251295\n",
      "Episode: 1270 -----Score: 250.0 -----Epsilon: 0.1699974065343279\n",
      "Episode: 1271 -----Score: 520.0 -----Epsilon: 0.1696536794089772\n",
      "Episode: 1272 -----Score: 170.0 -----Epsilon: 0.16950546672334313\n",
      "Episode: 1273 -----Score: 470.0 -----Epsilon: 0.16921213705847446\n",
      "Episode: 1274 -----Score: 390.0 -----Epsilon: 0.1688933034038008\n",
      "Episode: 1275 -----Score: 400.0 -----Epsilon: 0.1686603911586102\n",
      "Episode: 1276 -----Score: 250.0 -----Epsilon: 0.16843790609225037\n",
      "Episode: 1277 -----Score: 280.0 -----Epsilon: 0.16823152754890563\n",
      "Episode: 1278 -----Score: 290.0 -----Epsilon: 0.16801700080240392\n",
      "Episode: 1279 -----Score: 400.0 -----Epsilon: 0.1677561048908945\n",
      "Episode: 1280 -----Score: 160.0 -----Epsilon: 0.16760184009282192\n",
      "Episode: 1281 -----Score: 470.0 -----Epsilon: 0.16738008187112396\n",
      "Episode: 1282 -----Score: 400.0 -----Epsilon: 0.16715326806781142\n",
      "Episode: 1283 -----Score: 390.0 -----Epsilon: 0.16688737150897748\n",
      "Episode: 1284 -----Score: 340.0 -----Epsilon: 0.16662556364685704\n",
      "Episode: 1285 -----Score: 260.0 -----Epsilon: 0.16633987908142328\n",
      "Episode: 1286 -----Score: 340.0 -----Epsilon: 0.16611646822625747\n",
      "Episode: 1287 -----Score: 200.0 -----Epsilon: 0.1659255438994076\n",
      "Episode: 1288 -----Score: 300.0 -----Epsilon: 0.16571926065130166\n",
      "Episode: 1289 -----Score: 420.0 -----Epsilon: 0.16548377509708637\n",
      "Episode: 1290 -----Score: 270.0 -----Epsilon: 0.16529027210804753\n",
      "Episode: 1291 -----Score: 250.0 -----Epsilon: 0.16513662345903857\n",
      "Episode: 1292 -----Score: 320.0 -----Epsilon: 0.16485085408915826\n",
      "Episode: 1293 -----Score: 190.0 -----Epsilon: 0.16467949815008692\n",
      "Episode: 1294 -----Score: 330.0 -----Epsilon: 0.16446325117865077\n",
      "Episode: 1295 -----Score: 510.0 -----Epsilon: 0.16420557461435364\n",
      "Episode: 1296 -----Score: 300.0 -----Epsilon: 0.16400405372636087\n",
      "Episode: 1297 -----Score: 160.0 -----Epsilon: 0.16385684404594783\n",
      "Episode: 1298 -----Score: 300.0 -----Epsilon: 0.16357656083954789\n",
      "Episode: 1299 -----Score: 320.0 -----Epsilon: 0.16335392099320326\n",
      "Episode: 1300 -----Score: 300.0 -----Epsilon: 0.16315344529502482\n",
      "Saved model to disk\n",
      "Episode: 1301 -----Score: 590.0 -----Epsilon: 0.1628854411227539\n",
      "Episode: 1302 -----Score: 310.0 -----Epsilon: 0.16267708092873434\n",
      "Episode: 1303 -----Score: 390.0 -----Epsilon: 0.16249205952276838\n",
      "Episode: 1304 -----Score: 400.0 -----Epsilon: 0.16227771129081234\n",
      "Episode: 1305 -----Score: 390.0 -----Epsilon: 0.1620594322079191\n",
      "Episode: 1306 -----Score: 240.0 -----Epsilon: 0.16185730798615366\n",
      "Episode: 1307 -----Score: 330.0 -----Epsilon: 0.16165220277871012\n",
      "Episode: 1308 -----Score: 350.0 -----Epsilon: 0.1614812650192569\n",
      "Episode: 1309 -----Score: 210.0 -----Epsilon: 0.16128437780467286\n",
      "Episode: 1310 -----Score: 260.0 -----Epsilon: 0.16110931787007152\n",
      "Episode: 1311 -----Score: 340.0 -----Epsilon: 0.1608864965792242\n",
      "Episode: 1312 -----Score: 1980.0 -----Epsilon: 0.16058335030033577\n",
      "Episode: 1313 -----Score: 180.0 -----Epsilon: 0.1604096929072497\n",
      "Episode: 1314 -----Score: 270.0 -----Epsilon: 0.160251286238035\n",
      "Episode: 1315 -----Score: 750.0 -----Epsilon: 0.15994101968045135\n",
      "Episode: 1316 -----Score: 460.0 -----Epsilon: 0.1597581515915642\n",
      "Episode: 1317 -----Score: 170.0 -----Epsilon: 0.159543580645235\n",
      "Episode: 1318 -----Score: 480.0 -----Epsilon: 0.15931973840819316\n",
      "Episode: 1319 -----Score: 190.0 -----Epsilon: 0.15916336306156648\n",
      "Episode: 1320 -----Score: 300.0 -----Epsilon: 0.15894005427597557\n",
      "Episode: 1321 -----Score: 210.0 -----Epsilon: 0.1587497579109437\n",
      "Episode: 1322 -----Score: 660.0 -----Epsilon: 0.15848074623890265\n",
      "Episode: 1323 -----Score: 150.0 -----Epsilon: 0.1582555465474858\n",
      "Episode: 1324 -----Score: 120.0 -----Epsilon: 0.15809547278529518\n",
      "Episode: 1325 -----Score: 280.0 -----Epsilon: 0.15790713506498374\n",
      "Episode: 1326 -----Score: 150.0 -----Epsilon: 0.15773321707464197\n",
      "Episode: 1327 -----Score: 410.0 -----Epsilon: 0.15746088916229667\n",
      "Episode: 1328 -----Score: 310.0 -----Epsilon: 0.1572711056026773\n",
      "Episode: 1329 -----Score: 260.0 -----Epsilon: 0.15710825694459873\n",
      "Episode: 1330 -----Score: 340.0 -----Epsilon: 0.15686775112895443\n",
      "Episode: 1331 -----Score: 360.0 -----Epsilon: 0.15665204932457977\n",
      "Episode: 1332 -----Score: 290.0 -----Epsilon: 0.1564454048282047\n",
      "Episode: 1333 -----Score: 200.0 -----Epsilon: 0.15625434511371117\n",
      "Episode: 1334 -----Score: 410.0 -----Epsilon: 0.15600110574535206\n",
      "Episode: 1335 -----Score: 380.0 -----Epsilon: 0.15580373311134993\n",
      "Episode: 1336 -----Score: 1510.0 -----Epsilon: 0.15547719920659597\n",
      "Episode: 1337 -----Score: 1350.0 -----Epsilon: 0.15521373309569736\n",
      "Episode: 1338 -----Score: 300.0 -----Epsilon: 0.15503006860288795\n",
      "Episode: 1339 -----Score: 280.0 -----Epsilon: 0.15485591252567768\n",
      "Episode: 1340 -----Score: 780.0 -----Epsilon: 0.15454836446809575\n",
      "Episode: 1341 -----Score: 210.0 -----Epsilon: 0.15435560822140465\n",
      "Episode: 1342 -----Score: 240.0 -----Epsilon: 0.1542022548227493\n",
      "Episode: 1343 -----Score: 1020.0 -----Epsilon: 0.15399699394816077\n",
      "Episode: 1344 -----Score: 360.0 -----Epsilon: 0.15381138533234706\n",
      "Episode: 1345 -----Score: 310.0 -----Epsilon: 0.15363030201835945\n",
      "Episode: 1346 -----Score: 220.0 -----Epsilon: 0.15348534329957916\n",
      "Episode: 1347 -----Score: 670.0 -----Epsilon: 0.15325344858520445\n",
      "Episode: 1348 -----Score: 150.0 -----Epsilon: 0.15308404379978363\n",
      "Episode: 1349 -----Score: 400.0 -----Epsilon: 0.15273907512299012\n",
      "Episode: 1350 -----Score: 300.0 -----Epsilon: 0.15252966059372938\n",
      "Saved model to disk\n",
      "Episode: 1351 -----Score: 570.0 -----Epsilon: 0.15225687644313965\n",
      "Episode: 1352 -----Score: 770.0 -----Epsilon: 0.1519812365125876\n",
      "Episode: 1353 -----Score: 370.0 -----Epsilon: 0.1517828783754066\n",
      "Episode: 1354 -----Score: 1260.0 -----Epsilon: 0.15152446032694955\n",
      "Episode: 1355 -----Score: 180.0 -----Epsilon: 0.15138754397341742\n",
      "Episode: 1356 -----Score: 1170.0 -----Epsilon: 0.15112677643305372\n",
      "Episode: 1357 -----Score: 540.0 -----Epsilon: 0.15090780118367866\n",
      "Episode: 1358 -----Score: 220.0 -----Epsilon: 0.1507060213647223\n",
      "Episode: 1359 -----Score: 320.0 -----Epsilon: 0.1505024042963799\n",
      "Episode: 1360 -----Score: 330.0 -----Epsilon: 0.15031770059049132\n",
      "Episode: 1361 -----Score: 2000.0 -----Epsilon: 0.15004767192332538\n",
      "Episode: 1362 -----Score: 630.0 -----Epsilon: 0.1497919085674423\n",
      "Episode: 1363 -----Score: 380.0 -----Epsilon: 0.14958025227334248\n",
      "Episode: 1364 -----Score: 350.0 -----Epsilon: 0.1493772599507514\n",
      "Episode: 1365 -----Score: 140.0 -----Epsilon: 0.14924526866970886\n",
      "Episode: 1366 -----Score: 1420.0 -----Epsilon: 0.14894260777639962\n",
      "Episode: 1367 -----Score: 380.0 -----Epsilon: 0.1487270947339307\n",
      "Episode: 1368 -----Score: 1010.0 -----Epsilon: 0.14849318219037788\n",
      "Episode: 1369 -----Score: 610.0 -----Epsilon: 0.14823651081328124\n",
      "Episode: 1370 -----Score: 710.0 -----Epsilon: 0.14793441627033563\n",
      "Episode: 1371 -----Score: 320.0 -----Epsilon: 0.1477558192112308\n",
      "Episode: 1372 -----Score: 260.0 -----Epsilon: 0.14759721448877888\n",
      "Episode: 1373 -----Score: 820.0 -----Epsilon: 0.14727992649025543\n",
      "Episode: 1374 -----Score: 620.0 -----Epsilon: 0.14703299775782838\n",
      "Episode: 1375 -----Score: 1280.0 -----Epsilon: 0.1467826666220244\n",
      "Episode: 1376 -----Score: 340.0 -----Epsilon: 0.14658053920307382\n",
      "Episode: 1377 -----Score: 540.0 -----Epsilon: 0.14635644223190047\n",
      "Episode: 1378 -----Score: 290.0 -----Epsilon: 0.14618676701988995\n",
      "Episode: 1379 -----Score: 180.0 -----Epsilon: 0.1460134921132643\n",
      "Episode: 1380 -----Score: 390.0 -----Epsilon: 0.14582233947975962\n",
      "Episode: 1381 -----Score: 1600.0 -----Epsilon: 0.1454960626795005\n",
      "Episode: 1382 -----Score: 850.0 -----Epsilon: 0.1452451528219095\n",
      "Episode: 1383 -----Score: 250.0 -----Epsilon: 0.1450526851910455\n",
      "Episode: 1384 -----Score: 690.0 -----Epsilon: 0.1448112283654501\n",
      "Episode: 1385 -----Score: 630.0 -----Epsilon: 0.14453894965602637\n",
      "Episode: 1386 -----Score: 550.0 -----Epsilon: 0.14431451033834397\n",
      "Episode: 1387 -----Score: 310.0 -----Epsilon: 0.14412558191601238\n",
      "Episode: 1388 -----Score: 230.0 -----Epsilon: 0.14397317753407946\n",
      "Episode: 1389 -----Score: 200.0 -----Epsilon: 0.14384394752438606\n",
      "Episode: 1390 -----Score: 140.0 -----Epsilon: 0.1436932778813983\n",
      "Episode: 1391 -----Score: 300.0 -----Epsilon: 0.14351406034639255\n",
      "Episode: 1392 -----Score: 310.0 -----Epsilon: 0.14332417327359273\n",
      "Episode: 1393 -----Score: 1130.0 -----Epsilon: 0.1429663099980367\n",
      "Episode: 1394 -----Score: 320.0 -----Epsilon: 0.14277971768189565\n",
      "Episode: 1395 -----Score: 460.0 -----Epsilon: 0.14258709492002647\n",
      "Episode: 1396 -----Score: 340.0 -----Epsilon: 0.14243717201952413\n",
      "Episode: 1397 -----Score: 530.0 -----Epsilon: 0.14224102857421467\n",
      "Episode: 1398 -----Score: 590.0 -----Epsilon: 0.142008228252131\n",
      "Episode: 1399 -----Score: 420.0 -----Epsilon: 0.1417653179130398\n",
      "Episode: 1400 -----Score: 420.0 -----Epsilon: 0.14158850497369918\n",
      "Saved model to disk\n",
      "Episode: 1401 -----Score: 470.0 -----Epsilon: 0.14139494413114181\n",
      "Episode: 1402 -----Score: 380.0 -----Epsilon: 0.14117764563515242\n",
      "Episode: 1403 -----Score: 1180.0 -----Epsilon: 0.14094940467301104\n",
      "Episode: 1404 -----Score: 1310.0 -----Epsilon: 0.14065315858950594\n",
      "Episode: 1405 -----Score: 560.0 -----Epsilon: 0.1404260464141813\n",
      "Episode: 1406 -----Score: 270.0 -----Epsilon: 0.1402640880626077\n",
      "Episode: 1407 -----Score: 240.0 -----Epsilon: 0.14009727290570667\n",
      "Episode: 1408 -----Score: 220.0 -----Epsilon: 0.13993821260798434\n",
      "Episode: 1409 -----Score: 600.0 -----Epsilon: 0.13969269648737748\n",
      "Episode: 1410 -----Score: 230.0 -----Epsilon: 0.1395282352131675\n",
      "Episode: 1411 -----Score: 380.0 -----Epsilon: 0.13938821902488888\n",
      "Episode: 1412 -----Score: 360.0 -----Epsilon: 0.1391823552021521\n",
      "Episode: 1413 -----Score: 870.0 -----Epsilon: 0.13894066611578182\n",
      "Episode: 1414 -----Score: 290.0 -----Epsilon: 0.13878624981202553\n",
      "Episode: 1415 -----Score: 330.0 -----Epsilon: 0.1386170336608839\n",
      "Episode: 1416 -----Score: 120.0 -----Epsilon: 0.13849648916551646\n",
      "Episode: 1417 -----Score: 260.0 -----Epsilon: 0.13829415503831455\n",
      "Episode: 1418 -----Score: 350.0 -----Epsilon: 0.13809156413985474\n",
      "Episode: 1419 -----Score: 170.0 -----Epsilon: 0.13796099116036603\n",
      "Episode: 1420 -----Score: 300.0 -----Epsilon: 0.13775916384113066\n",
      "Episode: 1421 -----Score: 420.0 -----Epsilon: 0.13753287361102767\n",
      "Episode: 1422 -----Score: 350.0 -----Epsilon: 0.1373360672861755\n",
      "Episode: 1423 -----Score: 190.0 -----Epsilon: 0.13719056803758217\n",
      "Episode: 1424 -----Score: 300.0 -----Epsilon: 0.13693727373058653\n",
      "Episode: 1425 -----Score: 500.0 -----Epsilon: 0.13669702263986372\n",
      "Episode: 1426 -----Score: 130.0 -----Epsilon: 0.13656640260192973\n",
      "Episode: 1427 -----Score: 660.0 -----Epsilon: 0.13631180705160809\n",
      "Episode: 1428 -----Score: 380.0 -----Epsilon: 0.1361113034779639\n",
      "Episode: 1429 -----Score: 320.0 -----Epsilon: 0.13594670823636854\n",
      "Episode: 1430 -----Score: 570.0 -----Epsilon: 0.1357228523452637\n",
      "Episode: 1431 -----Score: 710.0 -----Epsilon: 0.13546224328798392\n",
      "Episode: 1432 -----Score: 380.0 -----Epsilon: 0.13528219808097333\n",
      "Episode: 1433 -----Score: 430.0 -----Epsilon: 0.1350332373863471\n",
      "Episode: 1434 -----Score: 260.0 -----Epsilon: 0.13481277302443548\n",
      "Episode: 1435 -----Score: 300.0 -----Epsilon: 0.13463763011192467\n",
      "Episode: 1436 -----Score: 410.0 -----Epsilon: 0.1344640593727111\n",
      "Episode: 1437 -----Score: 840.0 -----Epsilon: 0.13414146397169158\n",
      "Episode: 1438 -----Score: 640.0 -----Epsilon: 0.13391254567419314\n",
      "Episode: 1439 -----Score: 650.0 -----Epsilon: 0.1336559473114911\n",
      "Episode: 1440 -----Score: 200.0 -----Epsilon: 0.13348898158570974\n",
      "Episode: 1441 -----Score: 750.0 -----Epsilon: 0.13328196716402366\n",
      "Episode: 1442 -----Score: 340.0 -----Epsilon: 0.13311014410426456\n",
      "Episode: 1443 -----Score: 440.0 -----Epsilon: 0.13288936434217832\n",
      "Episode: 1444 -----Score: 580.0 -----Epsilon: 0.13258937319646258\n",
      "Episode: 1445 -----Score: 240.0 -----Epsilon: 0.1324303612158888\n",
      "Episode: 1446 -----Score: 290.0 -----Epsilon: 0.1322628102931552\n",
      "Episode: 1447 -----Score: 770.0 -----Epsilon: 0.1320180860084109\n",
      "Episode: 1448 -----Score: 240.0 -----Epsilon: 0.13184789230581107\n",
      "Episode: 1449 -----Score: 340.0 -----Epsilon: 0.13168213177676746\n",
      "Episode: 1450 -----Score: 400.0 -----Epsilon: 0.1315171057114661\n",
      "Saved model to disk\n",
      "Episode: 1451 -----Score: 220.0 -----Epsilon: 0.13137566927073907\n",
      "Episode: 1452 -----Score: 720.0 -----Epsilon: 0.1311834758186726\n",
      "Episode: 1453 -----Score: 240.0 -----Epsilon: 0.13099758928877703\n",
      "Episode: 1454 -----Score: 290.0 -----Epsilon: 0.13080856508914204\n",
      "Episode: 1455 -----Score: 470.0 -----Epsilon: 0.1305586978120685\n",
      "Episode: 1456 -----Score: 550.0 -----Epsilon: 0.13035831338648615\n",
      "Episode: 1457 -----Score: 330.0 -----Epsilon: 0.13020145626725005\n",
      "Episode: 1458 -----Score: 330.0 -----Epsilon: 0.13002424241663207\n",
      "Episode: 1459 -----Score: 300.0 -----Epsilon: 0.1298693457015617\n",
      "Episode: 1460 -----Score: 360.0 -----Epsilon: 0.12970399733906526\n",
      "Episode: 1461 -----Score: 390.0 -----Epsilon: 0.12948964403217103\n",
      "Episode: 1462 -----Score: 380.0 -----Epsilon: 0.1292735765771168\n",
      "Episode: 1463 -----Score: 270.0 -----Epsilon: 0.1290940109531342\n",
      "Episode: 1464 -----Score: 160.0 -----Epsilon: 0.1289727194774744\n",
      "Episode: 1465 -----Score: 260.0 -----Epsilon: 0.12881933307092688\n",
      "Episode: 1466 -----Score: 500.0 -----Epsilon: 0.12857532276628725\n",
      "Episode: 1467 -----Score: 370.0 -----Epsilon: 0.12840211982301103\n",
      "Episode: 1468 -----Score: 430.0 -----Epsilon: 0.12813352683052312\n",
      "Episode: 1469 -----Score: 230.0 -----Epsilon: 0.12798830562186791\n",
      "Episode: 1470 -----Score: 310.0 -----Epsilon: 0.1278023456651889\n",
      "Episode: 1471 -----Score: 150.0 -----Epsilon: 0.12768379998357285\n",
      "Episode: 1472 -----Score: 810.0 -----Epsilon: 0.12745417563789366\n",
      "Episode: 1473 -----Score: 430.0 -----Epsilon: 0.12727891912110012\n",
      "Episode: 1474 -----Score: 290.0 -----Epsilon: 0.12712373336714866\n",
      "Episode: 1475 -----Score: 250.0 -----Epsilon: 0.1269448689218266\n",
      "Episode: 1476 -----Score: 370.0 -----Epsilon: 0.1268020093016248\n",
      "Episode: 1477 -----Score: 370.0 -----Epsilon: 0.12662688978511283\n",
      "Episode: 1478 -----Score: 210.0 -----Epsilon: 0.12648792962426156\n",
      "Episode: 1479 -----Score: 300.0 -----Epsilon: 0.1263412885475281\n",
      "Episode: 1480 -----Score: 290.0 -----Epsilon: 0.12618497465426073\n",
      "Episode: 1481 -----Score: 710.0 -----Epsilon: 0.12597366555192824\n",
      "Episode: 1482 -----Score: 600.0 -----Epsilon: 0.12572599292026243\n",
      "Episode: 1483 -----Score: 200.0 -----Epsilon: 0.12558249564539326\n",
      "Episode: 1484 -----Score: 520.0 -----Epsilon: 0.12528847470874627\n",
      "Episode: 1485 -----Score: 900.0 -----Epsilon: 0.12506615962957982\n",
      "Episode: 1486 -----Score: 190.0 -----Epsilon: 0.12491816877646499\n",
      "Episode: 1487 -----Score: 690.0 -----Epsilon: 0.12473242858056727\n",
      "Episode: 1488 -----Score: 140.0 -----Epsilon: 0.12462246297055928\n",
      "Episode: 1489 -----Score: 260.0 -----Epsilon: 0.12446653312344705\n",
      "Episode: 1490 -----Score: 370.0 -----Epsilon: 0.12431974908719438\n",
      "Episode: 1491 -----Score: 790.0 -----Epsilon: 0.12408798450724189\n",
      "Episode: 1492 -----Score: 340.0 -----Epsilon: 0.12395404169217429\n",
      "Episode: 1493 -----Score: 1300.0 -----Epsilon: 0.12365616645759883\n",
      "Episode: 1494 -----Score: 200.0 -----Epsilon: 0.12351898412593129\n",
      "Episode: 1495 -----Score: 240.0 -----Epsilon: 0.12336320133180441\n",
      "Episode: 1496 -----Score: 410.0 -----Epsilon: 0.1232039188350556\n",
      "Episode: 1497 -----Score: 480.0 -----Epsilon: 0.12305714711036293\n",
      "Episode: 1498 -----Score: 540.0 -----Epsilon: 0.12287933488705384\n",
      "Episode: 1499 -----Score: 350.0 -----Epsilon: 0.12272975881790706\n",
      "Episode: 1500 -----Score: 250.0 -----Epsilon: 0.12260905201201473\n",
      "Saved model to disk\n",
      "Episode: 1501 -----Score: 800.0 -----Epsilon: 0.1223914913691226\n",
      "Episode: 1502 -----Score: 290.0 -----Epsilon: 0.12224739893530757\n",
      "Episode: 1503 -----Score: 290.0 -----Epsilon: 0.12209834789917884\n",
      "Episode: 1504 -----Score: 390.0 -----Epsilon: 0.12193240684559277\n",
      "Episode: 1505 -----Score: 1590.0 -----Epsilon: 0.12163987638156307\n",
      "Episode: 1506 -----Score: 600.0 -----Epsilon: 0.12140825135433334\n",
      "Episode: 1507 -----Score: 350.0 -----Epsilon: 0.12121512299601687\n",
      "Episode: 1508 -----Score: 520.0 -----Epsilon: 0.12101407260878773\n",
      "Episode: 1509 -----Score: 710.0 -----Epsilon: 0.12080344938977196\n",
      "Episode: 1510 -----Score: 270.0 -----Epsilon: 0.12068970600825013\n",
      "Episode: 1511 -----Score: 670.0 -----Epsilon: 0.12049265986213503\n",
      "Episode: 1512 -----Score: 390.0 -----Epsilon: 0.12031566559780488\n",
      "Episode: 1513 -----Score: 560.0 -----Epsilon: 0.12010457646842894\n",
      "Episode: 1514 -----Score: 590.0 -----Epsilon: 0.11985741545699592\n",
      "Episode: 1515 -----Score: 560.0 -----Epsilon: 0.11958851751841226\n",
      "Episode: 1516 -----Score: 280.0 -----Epsilon: 0.11941237371732455\n",
      "Episode: 1517 -----Score: 1540.0 -----Epsilon: 0.11914876350438298\n",
      "Episode: 1518 -----Score: 490.0 -----Epsilon: 0.11895232995042031\n",
      "Episode: 1519 -----Score: 610.0 -----Epsilon: 0.11876025802906504\n",
      "Episode: 1520 -----Score: 350.0 -----Epsilon: 0.11858272533309226\n",
      "Episode: 1521 -----Score: 230.0 -----Epsilon: 0.11844051126598822\n",
      "Episode: 1522 -----Score: 250.0 -----Epsilon: 0.11830532927055516\n",
      "Episode: 1523 -----Score: 500.0 -----Epsilon: 0.11811949920781425\n",
      "Episode: 1524 -----Score: 260.0 -----Epsilon: 0.11795141657610077\n",
      "Episode: 1525 -----Score: 360.0 -----Epsilon: 0.11776378712595795\n",
      "Episode: 1526 -----Score: 240.0 -----Epsilon: 0.11760726524268766\n",
      "Episode: 1527 -----Score: 250.0 -----Epsilon: 0.11747773329207513\n",
      "Episode: 1528 -----Score: 220.0 -----Epsilon: 0.11731478714610934\n",
      "Episode: 1529 -----Score: 340.0 -----Epsilon: 0.11715628456727668\n",
      "Episode: 1530 -----Score: 380.0 -----Epsilon: 0.11698021377764471\n",
      "Episode: 1531 -----Score: 160.0 -----Epsilon: 0.11686749907606629\n",
      "Episode: 1532 -----Score: 260.0 -----Epsilon: 0.11675255790236905\n",
      "Episode: 1533 -----Score: 230.0 -----Epsilon: 0.11660181085038124\n",
      "Episode: 1534 -----Score: 220.0 -----Epsilon: 0.11644310712707259\n",
      "Episode: 1535 -----Score: 300.0 -----Epsilon: 0.11628182861114975\n",
      "Episode: 1536 -----Score: 170.0 -----Epsilon: 0.11616769578971081\n",
      "Episode: 1537 -----Score: 210.0 -----Epsilon: 0.11603441166139113\n",
      "Episode: 1538 -----Score: 300.0 -----Epsilon: 0.11592028984334027\n",
      "Episode: 1539 -----Score: 500.0 -----Epsilon: 0.1157111265220557\n",
      "Episode: 1540 -----Score: 250.0 -----Epsilon: 0.1155591817994101\n",
      "Episode: 1541 -----Score: 250.0 -----Epsilon: 0.11542890440233187\n",
      "Episode: 1542 -----Score: 310.0 -----Epsilon: 0.11527087492018764\n",
      "Episode: 1543 -----Score: 390.0 -----Epsilon: 0.11504723588125151\n",
      "Episode: 1544 -----Score: 430.0 -----Epsilon: 0.11490926186119253\n",
      "Episode: 1545 -----Score: 580.0 -----Epsilon: 0.11470398746571286\n",
      "Episode: 1546 -----Score: 280.0 -----Epsilon: 0.1145584057200296\n",
      "Episode: 1547 -----Score: 1390.0 -----Epsilon: 0.114291337923376\n",
      "Episode: 1548 -----Score: 300.0 -----Epsilon: 0.11417687524369625\n",
      "Episode: 1549 -----Score: 660.0 -----Epsilon: 0.11400915837147824\n",
      "Episode: 1550 -----Score: 610.0 -----Epsilon: 0.11382165346816853\n",
      "Saved model to disk\n",
      "Episode: 1551 -----Score: 430.0 -----Epsilon: 0.11368173875967706\n",
      "Episode: 1552 -----Score: 120.0 -----Epsilon: 0.11358446873154285\n",
      "Episode: 1553 -----Score: 250.0 -----Epsilon: 0.11343259418723743\n",
      "Episode: 1554 -----Score: 1070.0 -----Epsilon: 0.11324694350173344\n",
      "Episode: 1555 -----Score: 330.0 -----Epsilon: 0.11309732980925491\n",
      "Episode: 1556 -----Score: 450.0 -----Epsilon: 0.11294723608946713\n",
      "Episode: 1557 -----Score: 400.0 -----Epsilon: 0.11277568652976053\n",
      "Episode: 1558 -----Score: 300.0 -----Epsilon: 0.11264021145591088\n",
      "Episode: 1559 -----Score: 260.0 -----Epsilon: 0.11251187465239464\n",
      "Episode: 1560 -----Score: 340.0 -----Epsilon: 0.1123630073533902\n",
      "Episode: 1561 -----Score: 770.0 -----Epsilon: 0.11214142133265105\n",
      "Episode: 1562 -----Score: 200.0 -----Epsilon: 0.11202149406443161\n",
      "Episode: 1563 -----Score: 250.0 -----Epsilon: 0.11188491103768093\n",
      "Episode: 1564 -----Score: 1220.0 -----Epsilon: 0.11161001214590498\n",
      "Episode: 1565 -----Score: 260.0 -----Epsilon: 0.11147794395925442\n",
      "Episode: 1566 -----Score: 330.0 -----Epsilon: 0.11134335977358528\n",
      "Episode: 1567 -----Score: 90.0 -----Epsilon: 0.11126455649571663\n",
      "Episode: 1568 -----Score: 230.0 -----Epsilon: 0.11111756178994729\n",
      "Episode: 1569 -----Score: 580.0 -----Epsilon: 0.11084166633558984\n",
      "Episode: 1570 -----Score: 410.0 -----Epsilon: 0.11065848556855014\n",
      "Episode: 1571 -----Score: 210.0 -----Epsilon: 0.1105204697803136\n",
      "Episode: 1572 -----Score: 340.0 -----Epsilon: 0.11037225063894199\n",
      "Episode: 1573 -----Score: 220.0 -----Epsilon: 0.11024164709614\n",
      "Episode: 1574 -----Score: 250.0 -----Epsilon: 0.11010172893125432\n",
      "Episode: 1575 -----Score: 280.0 -----Epsilon: 0.10997694421217281\n",
      "Episode: 1576 -----Score: 460.0 -----Epsilon: 0.10981254154309086\n",
      "Episode: 1577 -----Score: 350.0 -----Epsilon: 0.10967163258175676\n",
      "Episode: 1578 -----Score: 740.0 -----Epsilon: 0.10946827060661883\n",
      "Episode: 1579 -----Score: 1050.0 -----Epsilon: 0.10927774268669832\n",
      "Episode: 1580 -----Score: 350.0 -----Epsilon: 0.10913075364387459\n",
      "Episode: 1581 -----Score: 610.0 -----Epsilon: 0.10893384117137052\n",
      "Episode: 1582 -----Score: 220.0 -----Epsilon: 0.10878231060267711\n",
      "Episode: 1583 -----Score: 340.0 -----Epsilon: 0.10863598796351434\n",
      "Episode: 1584 -----Score: 340.0 -----Epsilon: 0.10847879672967474\n",
      "Episode: 1585 -----Score: 730.0 -----Epsilon: 0.10826357141362959\n",
      "Episode: 1586 -----Score: 340.0 -----Epsilon: 0.10812616381282558\n",
      "Episode: 1587 -----Score: 700.0 -----Epsilon: 0.10794617541589521\n",
      "Episode: 1588 -----Score: 460.0 -----Epsilon: 0.1077563570465133\n",
      "Episode: 1589 -----Score: 210.0 -----Epsilon: 0.10765059213720132\n",
      "Episode: 1590 -----Score: 380.0 -----Epsilon: 0.10750041660731477\n",
      "Episode: 1591 -----Score: 370.0 -----Epsilon: 0.10736548070660795\n",
      "Episode: 1592 -----Score: 380.0 -----Epsilon: 0.10721849056229285\n",
      "Episode: 1593 -----Score: 370.0 -----Epsilon: 0.1070689178259598\n",
      "Episode: 1594 -----Score: 260.0 -----Epsilon: 0.10693259874202865\n",
      "Episode: 1595 -----Score: 660.0 -----Epsilon: 0.10677530959261039\n",
      "Episode: 1596 -----Score: 240.0 -----Epsilon: 0.10666218761656238\n",
      "Episode: 1597 -----Score: 320.0 -----Epsilon: 0.10648037687134389\n",
      "Episode: 1598 -----Score: 250.0 -----Epsilon: 0.10637607711294202\n",
      "Episode: 1599 -----Score: 710.0 -----Epsilon: 0.10618668355391798\n",
      "Episode: 1600 -----Score: 290.0 -----Epsilon: 0.10606294191255224\n",
      "Saved model to disk\n",
      "Episode: 1601 -----Score: 740.0 -----Epsilon: 0.10589422387550537\n",
      "Episode: 1602 -----Score: 340.0 -----Epsilon: 0.10576384239279732\n",
      "Episode: 1603 -----Score: 360.0 -----Epsilon: 0.10563172005135431\n",
      "Episode: 1604 -----Score: 410.0 -----Epsilon: 0.10549069016128307\n",
      "Episode: 1605 -----Score: 580.0 -----Epsilon: 0.10530160933027535\n",
      "Episode: 1606 -----Score: 180.0 -----Epsilon: 0.10518794486839479\n",
      "Episode: 1607 -----Score: 500.0 -----Epsilon: 0.10503490250950226\n",
      "Episode: 1608 -----Score: 190.0 -----Epsilon: 0.1049049496276776\n",
      "Episode: 1609 -----Score: 410.0 -----Epsilon: 0.10478605472206066\n",
      "Episode: 1610 -----Score: 310.0 -----Epsilon: 0.10466122403387537\n",
      "Episode: 1611 -----Score: 290.0 -----Epsilon: 0.10448783932668836\n",
      "Episode: 1612 -----Score: 260.0 -----Epsilon: 0.10433331154872333\n",
      "Episode: 1613 -----Score: 420.0 -----Epsilon: 0.1041921396986285\n",
      "Episode: 1614 -----Score: 940.0 -----Epsilon: 0.10398188379614279\n",
      "Episode: 1615 -----Score: 130.0 -----Epsilon: 0.103892705513624\n",
      "Episode: 1616 -----Score: 290.0 -----Epsilon: 0.10377620315158542\n",
      "Episode: 1617 -----Score: 600.0 -----Epsilon: 0.10360552785483101\n",
      "Episode: 1618 -----Score: 860.0 -----Epsilon: 0.10342913418906222\n",
      "Episode: 1619 -----Score: 980.0 -----Epsilon: 0.10326770382996539\n",
      "Episode: 1620 -----Score: 120.0 -----Epsilon: 0.10316366232619342\n",
      "Episode: 1621 -----Score: 860.0 -----Epsilon: 0.10301438929363338\n",
      "Episode: 1622 -----Score: 510.0 -----Epsilon: 0.1028206983396545\n",
      "Episode: 1623 -----Score: 450.0 -----Epsilon: 0.1026591910183756\n",
      "Episode: 1624 -----Score: 540.0 -----Epsilon: 0.10249014783238271\n",
      "Episode: 1625 -----Score: 790.0 -----Epsilon: 0.10223587798479714\n",
      "Episode: 1626 -----Score: 560.0 -----Epsilon: 0.10206283684168305\n",
      "Episode: 1627 -----Score: 760.0 -----Epsilon: 0.10183650843505646\n",
      "Episode: 1628 -----Score: 380.0 -----Epsilon: 0.10171661553901068\n",
      "Episode: 1629 -----Score: 1400.0 -----Epsilon: 0.10155359270393363\n",
      "Episode: 1630 -----Score: 970.0 -----Epsilon: 0.10137116321508435\n",
      "Episode: 1631 -----Score: 180.0 -----Epsilon: 0.10125121066381193\n",
      "Episode: 1632 -----Score: 650.0 -----Epsilon: 0.1010790275005206\n",
      "Episode: 1633 -----Score: 300.0 -----Epsilon: 0.10095457469286154\n",
      "Episode: 1634 -----Score: 410.0 -----Epsilon: 0.1008129337835411\n",
      "Episode: 1635 -----Score: 220.0 -----Epsilon: 0.10069122516463146\n",
      "Episode: 1636 -----Score: 710.0 -----Epsilon: 0.10051778286922716\n",
      "Episode: 1637 -----Score: 340.0 -----Epsilon: 0.10037715629415885\n",
      "Episode: 1638 -----Score: 1020.0 -----Epsilon: 0.10020144930489634\n",
      "Episode: 1639 -----Score: 220.0 -----Epsilon: 0.10004865834961918\n",
      "Episode: 1640 -----Score: 420.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1641 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1642 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1643 -----Score: 740.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1644 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1645 -----Score: 350.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1646 -----Score: 260.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1647 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1648 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1649 -----Score: 370.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1650 -----Score: 220.0 -----Epsilon: 0.0999998464666021\n",
      "Saved model to disk\n",
      "Episode: 1651 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1652 -----Score: 280.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1653 -----Score: 220.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1654 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1655 -----Score: 460.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1656 -----Score: 440.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1657 -----Score: 170.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1658 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1659 -----Score: 250.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1660 -----Score: 280.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1661 -----Score: 240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1662 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1663 -----Score: 1560.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1664 -----Score: 390.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1665 -----Score: 440.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1666 -----Score: 740.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1667 -----Score: 440.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1668 -----Score: 350.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1669 -----Score: 450.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1670 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1671 -----Score: 480.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1672 -----Score: 490.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1673 -----Score: 790.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1674 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1675 -----Score: 570.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1676 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1677 -----Score: 670.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1678 -----Score: 800.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1679 -----Score: 850.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1680 -----Score: 360.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1681 -----Score: 450.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1682 -----Score: 610.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1683 -----Score: 530.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1684 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1685 -----Score: 370.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1686 -----Score: 280.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1687 -----Score: 270.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1688 -----Score: 360.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1689 -----Score: 1160.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1690 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1691 -----Score: 1680.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1692 -----Score: 100.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1693 -----Score: 230.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1694 -----Score: 400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1695 -----Score: 800.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1696 -----Score: 470.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1697 -----Score: 230.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1698 -----Score: 400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1699 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1700 -----Score: 180.0 -----Epsilon: 0.0999998464666021\n",
      "Saved model to disk\n",
      "Episode: 1701 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1702 -----Score: 410.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1703 -----Score: 760.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1704 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1705 -----Score: 1430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1706 -----Score: 240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1707 -----Score: 1290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1708 -----Score: 240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1709 -----Score: 250.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1710 -----Score: 520.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1711 -----Score: 280.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1712 -----Score: 410.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1713 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1714 -----Score: 370.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1715 -----Score: 440.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1716 -----Score: 670.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1717 -----Score: 370.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1718 -----Score: 210.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1719 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1720 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1721 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1722 -----Score: 1200.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1723 -----Score: 240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1724 -----Score: 360.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1725 -----Score: 470.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1726 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1727 -----Score: 250.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1728 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1729 -----Score: 760.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1730 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1731 -----Score: 510.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1732 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1733 -----Score: 270.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1734 -----Score: 370.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1735 -----Score: 390.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1736 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1737 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1738 -----Score: 420.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1739 -----Score: 480.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1740 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1741 -----Score: 360.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1742 -----Score: 360.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1743 -----Score: 680.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1744 -----Score: 1040.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1745 -----Score: 990.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1746 -----Score: 540.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1747 -----Score: 1080.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1748 -----Score: 270.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1749 -----Score: 720.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1750 -----Score: 360.0 -----Epsilon: 0.0999998464666021\n",
      "Saved model to disk\n",
      "Episode: 1751 -----Score: 820.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1752 -----Score: 510.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1753 -----Score: 420.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1754 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1755 -----Score: 240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1756 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1757 -----Score: 530.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1758 -----Score: 590.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1759 -----Score: 590.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1760 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1761 -----Score: 600.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1762 -----Score: 460.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1763 -----Score: 700.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1764 -----Score: 610.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1765 -----Score: 920.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1766 -----Score: 810.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1767 -----Score: 240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1768 -----Score: 460.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1769 -----Score: 770.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1770 -----Score: 350.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1771 -----Score: 480.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1772 -----Score: 740.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1773 -----Score: 200.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1774 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1775 -----Score: 880.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1776 -----Score: 400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1777 -----Score: 260.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1778 -----Score: 110.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1779 -----Score: 190.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1780 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1781 -----Score: 270.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1782 -----Score: 1400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1783 -----Score: 1300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1784 -----Score: 450.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1785 -----Score: 1180.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1786 -----Score: 220.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1787 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1788 -----Score: 790.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1789 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1790 -----Score: 540.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1791 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1792 -----Score: 190.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1793 -----Score: 810.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1794 -----Score: 550.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1795 -----Score: 710.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1796 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1797 -----Score: 250.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1798 -----Score: 730.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1799 -----Score: 230.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1800 -----Score: 2160.0 -----Epsilon: 0.0999998464666021\n",
      "Saved model to disk\n",
      "Episode: 1801 -----Score: 860.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1802 -----Score: 390.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1803 -----Score: 800.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1804 -----Score: 250.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1805 -----Score: 420.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1806 -----Score: 870.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1807 -----Score: 420.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1808 -----Score: 120.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1809 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1810 -----Score: 650.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1811 -----Score: 190.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1812 -----Score: 270.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1813 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1814 -----Score: 580.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1815 -----Score: 280.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1816 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1817 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1818 -----Score: 920.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1819 -----Score: 390.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1820 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1821 -----Score: 640.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1822 -----Score: 400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1823 -----Score: 390.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1824 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1825 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1826 -----Score: 460.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1827 -----Score: 360.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1828 -----Score: 2120.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1829 -----Score: 500.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1830 -----Score: 710.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1831 -----Score: 470.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1832 -----Score: 510.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1833 -----Score: 1240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1834 -----Score: 750.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1835 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1836 -----Score: 350.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1837 -----Score: 230.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1838 -----Score: 450.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1839 -----Score: 400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1840 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1841 -----Score: 260.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1842 -----Score: 580.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1843 -----Score: 750.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1844 -----Score: 1340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1845 -----Score: 230.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1846 -----Score: 190.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1847 -----Score: 200.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1848 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1849 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1850 -----Score: 250.0 -----Epsilon: 0.0999998464666021\n",
      "Saved model to disk\n",
      "Episode: 1851 -----Score: 760.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1852 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1853 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1854 -----Score: 190.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1855 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1856 -----Score: 500.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1857 -----Score: 270.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1858 -----Score: 570.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1859 -----Score: 1080.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1860 -----Score: 200.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1861 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1862 -----Score: 480.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1863 -----Score: 1420.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1864 -----Score: 170.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1865 -----Score: 400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1866 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1867 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1868 -----Score: 540.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1869 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1870 -----Score: 200.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1871 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1872 -----Score: 150.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1873 -----Score: 520.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1874 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1875 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1876 -----Score: 390.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1877 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1878 -----Score: 460.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1879 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1880 -----Score: 260.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1881 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1882 -----Score: 400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1883 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1884 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1885 -----Score: 410.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1886 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1887 -----Score: 250.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1888 -----Score: 350.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1889 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1890 -----Score: 220.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1891 -----Score: 220.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1892 -----Score: 470.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1893 -----Score: 500.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1894 -----Score: 600.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1895 -----Score: 260.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1896 -----Score: 210.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1897 -----Score: 650.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1898 -----Score: 900.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1899 -----Score: 770.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1900 -----Score: 560.0 -----Epsilon: 0.0999998464666021\n",
      "Saved model to disk\n",
      "Episode: 1901 -----Score: 790.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1902 -----Score: 210.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1903 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1904 -----Score: 360.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1905 -----Score: 430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1906 -----Score: 480.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1907 -----Score: 490.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1908 -----Score: 1090.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1909 -----Score: 280.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1910 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1911 -----Score: 330.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1912 -----Score: 260.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1913 -----Score: 280.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1914 -----Score: 350.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1915 -----Score: 440.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1916 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1917 -----Score: 270.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1918 -----Score: 400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1919 -----Score: 540.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1920 -----Score: 1230.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1921 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1922 -----Score: 280.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1923 -----Score: 490.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1924 -----Score: 810.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1925 -----Score: 360.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1926 -----Score: 230.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1927 -----Score: 860.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1928 -----Score: 190.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1929 -----Score: 480.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1930 -----Score: 670.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1931 -----Score: 100.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1932 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1933 -----Score: 270.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1934 -----Score: 930.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1935 -----Score: 170.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1936 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1937 -----Score: 520.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1938 -----Score: 1430.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1939 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1940 -----Score: 550.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1941 -----Score: 540.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1942 -----Score: 680.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1943 -----Score: 440.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1944 -----Score: 870.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1945 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1946 -----Score: 1190.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1947 -----Score: 580.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1948 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1949 -----Score: 350.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1950 -----Score: 410.0 -----Epsilon: 0.0999998464666021\n",
      "Saved model to disk\n",
      "Episode: 1951 -----Score: 1030.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1952 -----Score: 630.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1953 -----Score: 240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1954 -----Score: 510.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1955 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1956 -----Score: 770.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1957 -----Score: 900.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1958 -----Score: 400.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1959 -----Score: 370.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1960 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1961 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1962 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1963 -----Score: 800.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1964 -----Score: 570.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1965 -----Score: 640.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1966 -----Score: 530.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1967 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1968 -----Score: 260.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1969 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1970 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1971 -----Score: 320.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1972 -----Score: 210.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1973 -----Score: 170.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1974 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1975 -----Score: 470.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1976 -----Score: 570.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1977 -----Score: 910.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1978 -----Score: 370.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1979 -----Score: 1000.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1980 -----Score: 860.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1981 -----Score: 250.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1982 -----Score: 420.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1983 -----Score: 280.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1984 -----Score: 630.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1985 -----Score: 450.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1986 -----Score: 410.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1987 -----Score: 480.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1988 -----Score: 580.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1989 -----Score: 780.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1990 -----Score: 790.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1991 -----Score: 1200.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1992 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1993 -----Score: 480.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1994 -----Score: 1100.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1995 -----Score: 240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1996 -----Score: 630.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1997 -----Score: 340.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1998 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 1999 -----Score: 300.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2000 -----Score: 290.0 -----Epsilon: 0.0999998464666021\n",
      "Saved model to disk\n",
      "Episode: 2001 -----Score: 2020.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2002 -----Score: 150.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2003 -----Score: 1590.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2004 -----Score: 240.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2005 -----Score: 160.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2006 -----Score: 980.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2007 -----Score: 250.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2008 -----Score: 260.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2009 -----Score: 310.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2010 -----Score: 910.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2011 -----Score: 640.0 -----Epsilon: 0.0999998464666021\n",
      "Episode: 2012 -----Score: 380.0 -----Epsilon: 0.0999998464666021\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-97a201ef670f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     30\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     31\u001b[0m                 \u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate_replay_memory\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maction\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 32\u001b[1;33m                 \u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_agent\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     33\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     34\u001b[0m                 \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-3-771b19144c31>\u001b[0m in \u001b[0;36mtrain_agent\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     95\u001b[0m                 \u001b[0mcurrent_q_values\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     96\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 97\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobservations\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcurrent_q_values\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     98\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     99\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    106\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    107\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 108\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    109\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m     \u001b[1;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1047\u001b[0m          \u001b[0mtraining_utils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mRespectCompiledTrainableState\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1048\u001b[0m       \u001b[1;31m# Creates a `tf.data.Dataset` and handles batch and epoch iteration.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1049\u001b[1;33m       data_handler = data_adapter.DataHandler(\n\u001b[0m\u001b[0;32m   1050\u001b[0m           \u001b[0mx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1051\u001b[0m           \u001b[0my\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\keras\\engine\\data_adapter.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, x, y, sample_weight, batch_size, steps_per_epoch, initial_epoch, epochs, shuffle, class_weight, max_queue_size, workers, use_multiprocessing, model, steps_per_execution)\u001b[0m\n\u001b[0;32m   1103\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1104\u001b[0m     \u001b[0madapter_cls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mselect_data_adapter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1105\u001b[1;33m     self._adapter = adapter_cls(\n\u001b[0m\u001b[0;32m   1106\u001b[0m         \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1107\u001b[0m         \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\keras\\engine\\data_adapter.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, x, y, sample_weights, sample_weight_modes, batch_size, epochs, steps, shuffle, **kwargs)\u001b[0m\n\u001b[0;32m    362\u001b[0m     \u001b[0mindices_dataset\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mindices_dataset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflat_map\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mslice_batch_indices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    363\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 364\u001b[1;33m     \u001b[0mdataset\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mslice_inputs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindices_dataset\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    365\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    366\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mshuffle\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"batch\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\keras\\engine\\data_adapter.py\u001b[0m in \u001b[0;36mslice_inputs\u001b[1;34m(self, indices_dataset, inputs)\u001b[0m\n\u001b[0;32m    394\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mnest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmap_structure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0md\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0md\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    395\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 396\u001b[1;33m     dataset = dataset.map(\n\u001b[0m\u001b[0;32m    397\u001b[0m         grab_batch, num_parallel_calls=dataset_ops.AUTOTUNE)\n\u001b[0;32m    398\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\data\\ops\\dataset_ops.py\u001b[0m in \u001b[0;36mmap\u001b[1;34m(self, map_func, num_parallel_calls, deterministic)\u001b[0m\n\u001b[0;32m   1695\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mMapDataset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmap_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpreserve_cardinality\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1696\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1697\u001b[1;33m       return ParallelMapDataset(\n\u001b[0m\u001b[0;32m   1698\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1699\u001b[0m           \u001b[0mmap_func\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\data\\ops\\dataset_ops.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, input_dataset, map_func, num_parallel_calls, deterministic, use_inter_op_parallelism, preserve_cardinality, use_legacy_function)\u001b[0m\n\u001b[0;32m   4078\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_input_dataset\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minput_dataset\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4079\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_use_inter_op_parallelism\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0muse_inter_op_parallelism\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4080\u001b[1;33m     self._map_func = StructuredFunctionWrapper(\n\u001b[0m\u001b[0;32m   4081\u001b[0m         \u001b[0mmap_func\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4082\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transformation_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\data\\ops\\dataset_ops.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, func, transformation_name, dataset, input_classes, input_shapes, input_types, input_structure, add_to_graph, use_legacy_function, defun_kwargs)\u001b[0m\n\u001b[0;32m   3369\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0mtracking\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresource_tracker_scope\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresource_tracker\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3370\u001b[0m         \u001b[1;31m# TODO(b/141462134): Switch to using garbage collection.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3371\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mwrapper_fn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_concrete_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3372\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0madd_to_graph\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3373\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_to_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mget_concrete_function\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2936\u001b[0m       \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0minputs\u001b[0m \u001b[0mto\u001b[0m \u001b[0mspecialize\u001b[0m \u001b[0mon\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2937\u001b[0m     \"\"\"\n\u001b[1;32m-> 2938\u001b[1;33m     graph_function = self._get_concrete_function_garbage_collected(\n\u001b[0m\u001b[0;32m   2939\u001b[0m         *args, **kwargs)\n\u001b[0;32m   2940\u001b[0m     \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_garbage_collector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_get_concrete_function_garbage_collected\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2904\u001b[0m       \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2905\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2906\u001b[1;33m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2907\u001b[0m       \u001b[0mseen_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2908\u001b[0m       captured = object_identity.ObjectIdentitySet(\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_maybe_define_function\u001b[1;34m(self, args, kwargs)\u001b[0m\n\u001b[0;32m   3211\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3212\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_function_cache\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmissed\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcall_context_key\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3213\u001b[1;33m       \u001b[0mgraph_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_create_graph_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3214\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_function_cache\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprimary\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcache_key\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3215\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_create_graph_function\u001b[1;34m(self, args, kwargs, override_flat_arg_shapes)\u001b[0m\n\u001b[0;32m   3063\u001b[0m     \u001b[0marg_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbase_arg_names\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mmissing_arg_names\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3064\u001b[0m     graph_function = ConcreteFunction(\n\u001b[1;32m-> 3065\u001b[1;33m         func_graph_module.func_graph_from_py_func(\n\u001b[0m\u001b[0;32m   3066\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3067\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_python_function\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\func_graph.py\u001b[0m in \u001b[0;36mfunc_graph_from_py_func\u001b[1;34m(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)\u001b[0m\n\u001b[0;32m    900\u001b[0m       \u001b[0marg_shapes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    901\u001b[0m       \u001b[0mkwarg_shapes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 902\u001b[1;33m     func_args = _get_defun_inputs_from_args(\n\u001b[0m\u001b[0;32m    903\u001b[0m         args, arg_names, flat_shapes=arg_shapes)\n\u001b[0;32m    904\u001b[0m     func_kwargs = _get_defun_inputs_from_kwargs(\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\func_graph.py\u001b[0m in \u001b[0;36m_get_defun_inputs_from_args\u001b[1;34m(args, names, flat_shapes)\u001b[0m\n\u001b[0;32m   1136\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_get_defun_inputs_from_args\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mflat_shapes\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1137\u001b[0m   \u001b[1;34m\"\"\"Maps Python function positional args to graph-construction inputs.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1138\u001b[1;33m   return _get_defun_inputs(\n\u001b[0m\u001b[0;32m   1139\u001b[0m       args, names, structure=args, flat_shapes=flat_shapes)\n\u001b[0;32m   1140\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\func_graph.py\u001b[0m in \u001b[0;36m_get_defun_inputs\u001b[1;34m(args, names, structure, flat_shapes)\u001b[0m\n\u001b[0;32m   1208\u001b[0m         \u001b[0mplaceholder_shape\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mshape\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mshape\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0marg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1209\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1210\u001b[1;33m           placeholder = graph_placeholder(\n\u001b[0m\u001b[0;32m   1211\u001b[0m               \u001b[0marg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mplaceholder_shape\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1212\u001b[0m               name=requested_name)\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\graph_only_ops.py\u001b[0m in \u001b[0;36mgraph_placeholder\u001b[1;34m(dtype, shape, name)\u001b[0m\n\u001b[0;32m     36\u001b[0m   \u001b[0mg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m   \u001b[0mattrs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m\"dtype\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mdtype_value\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"shape\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 38\u001b[1;33m   op = g._create_op_internal(  # pylint: disable=protected-access\n\u001b[0m\u001b[0;32m     39\u001b[0m       \u001b[1;34m\"Placeholder\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_types\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m       attrs=attrs, name=name)\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\func_graph.py\u001b[0m in \u001b[0;36m_create_op_internal\u001b[1;34m(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_device)\u001b[0m\n\u001b[0;32m    589\u001b[0m       \u001b[0minp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcapture\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    590\u001b[0m       \u001b[0minputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 591\u001b[1;33m     return super(FuncGraph, self)._create_op_internal(  # pylint: disable=protected-access\n\u001b[0m\u001b[0;32m    592\u001b[0m         \u001b[0mop_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtypes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_types\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop_def\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    593\u001b[0m         compute_device)\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36m_create_op_internal\u001b[1;34m(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_device)\u001b[0m\n\u001b[0;32m   3475\u001b[0m     \u001b[1;31m# Session.run call cannot occur between creating and mutating the op.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3476\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_mutation_lock\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3477\u001b[1;33m       ret = Operation(\n\u001b[0m\u001b[0;32m   3478\u001b[0m           \u001b[0mnode_def\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3479\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, node_def, g, inputs, output_types, control_inputs, input_types, original_op, op_def)\u001b[0m\n\u001b[0;32m   1972\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mop_def\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1973\u001b[0m         \u001b[0mop_def\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_op_def\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mop\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1974\u001b[1;33m       self._c_op = _create_c_op(self._graph, node_def, inputs,\n\u001b[0m\u001b[0;32m   1975\u001b[0m                                 control_input_ops, op_def)\n\u001b[0;32m   1976\u001b[0m       \u001b[0mname\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_str\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36m_create_c_op\u001b[1;34m(graph, node_def, inputs, control_inputs, op_def)\u001b[0m\n\u001b[0;32m   1810\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1811\u001b[0m   \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1812\u001b[1;33m     \u001b[0mc_op\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpywrap_tf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_FinishOperation\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mop_desc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1813\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mInvalidArgumentError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1814\u001b[0m     \u001b[1;31m# Convert to ValueError for backwards compatibility.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAGnCAYAAADRxWOoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABqZElEQVR4nO3de5gU1Z038G/3XJgb49xvxXUEHS6CijEb4z3Gu1HfNXkTnkQhye6qoMnqRk1MBNfkFVxfs5uFiGueVfNu1jdCMl7I8sZgZE00Md5QBERQEGwY5sbIMDAMM13vHz1VVPd0dVdV1+Wc6u/neXjo6a7LqVNVp3516tQ5EVVVVRARERERkRSiQSeAiIiIiIisYwBPRERERCQRBvBERERERBJhAE9EREREJBEG8EREREREEmEAT0REREQkEQbwRCSV888/H5FIxPRfQUEB5s6dix07dniell/+8peoqalJWv+3v/1tz9dLx/36179GY2NjxmOiubkZzzzzTNBJJSJyDQN4IgqVeDyOd955B4888ojn6/rJT36CAwcOeL4eMvejH/0InZ2dGafp6OjAfffd51OKiIi8Vxh0AoiInJo7dy5OPfVU/e8//vGP+OCDDwAA3d3dnq9/YGBA/3zJJZegqakJZ555pufrpeN6enr0z5/97Gcxbdo0/e8dO3bg5ZdfHjMdEZHsGMATkbSuueYaLF26VP97wYIFegDv9yDTd911F84//3xf10nJvvnNb2LBggX6348//rgewMs66PiGDRtwwQUXAAAmT56MXbt2BZsgIhICm9AQEREREUmEATwRERERkUTYhIaIiEhQ559/vrTNf4jIO6yBJyIiIiKSCAN4Igq14eFhXH755SgqKkrbR7iiKNiwYYM+vfG3xYsX45JLLsGUKVMwbtw4NDc344UXXsCGDRsQiUTw9ttv6/NdcMEFiEQi2LBhQ9Z1nnDCCUndXE6ZMkX/bcGCBbjqqqswbdo0lJaWorq6Gr/4xS8AJF7K1KZra2vD3/7t3+K8887D5MmTUVJSgurqatx222349re/jcsvvxxtbW2oqKhARUUFli9frq9v7dq1pn2nl5SU4Otf/7o+7YIFC/TfPv/5z+P666/HZz/7WbS0tKC4uBgTJ07En/70J336r3/96ygpKUm77JqaGvznf/6nPu2SJUtQXl6edtrx48fjoYcecvVYAIA1a9bo6zjxxBOTfrv66qv137T8+rd/+zdUVVWlTWNZWRluu+22tHl18cUX4ytf+QrmzJmDyspKlJWV4ZZbbgEAXHnllfp0xu4te3t7k5bf0dGBjRs36n9PmTJFn9Zquv70pz/p35933nn6/P/6r/+qf79z504AwMGDB/Xv6uvr3c14InKXSkQkkfPOO08FoAJQlyxZkvTbDTfcoP92ww03qKqqqq+++qr+ndm/+fPn68vINu1XvvIV9cUXXzT9/cUXX7S0zpkzZ+rrnDx5csZpP/OZz6iqqqqPPfZY1uWa/WtubtbXd9lll2WdvrOzc0yemv276aabVFVV1f3792ed9qyzztLTUVpamnHalpaWrMeDMe8ee+yxpN+M+TV58mRVVVW1q6tLjUQi+vcHDhzQp1cURf/+z3/+s6qqqjpjxoyMaYxGo+rg4KClvIpEIurg4KD64IMP6t9dc801+vp/97vf6d+3tbWpqqqqb7311phtsJOuI0eOqEVFRSoAtba2Vp//7/7u7/Rp165dq6qqqr7yyiv6d5dddlnWvCei4LANPBGF2uHDh/XP1dXV+MIXvgAguY9wY3/uRp/97GdRWFiIjo4ObNu2TV9eU1MTbrjhBjz77LP6QE5aP/BNTU3o6OhIu87e3l4899xzGdc5b948VFRUoK+vT6/hN26DpqmpCZ/+9KcBJPq817alsrJS73bw8OHD+N3vfjdmGcbPf/3Xf42KigoAidppLV0DAwNjamFnz56t11q/9957SXmSutyioiLMnz8fQGIgpd/+9rdjtvvIkSP656985SsoLi7GoUOH8Ktf/SpjHuWirq4Os2bNwrvvvgsAePPNN3HhhRdi//79iMViAIDx48dj3rx5Y7bpqquuQk1NDQDgiSeeAJAYOOzIkSMYN25c0npOPPFETJgwAQDw3//93wAAVVVx5MiRpO5G33jjjbSfs3VJajVdVVVVOOWUU/Dmm2+ip6cH+/fvR2NjIzZv3qzPv3nzZlxxxRVJ33E8AyKxMYAnImk9/fTTSf1i//GPf9Q/RyKRMdNPmjQJjz/+OIDkPsLNrF27FlVVVXj88cexcOFC/fu2tjY8/vjjOPXUU/UA3tgPvDGAN65z48aNegBv5mc/+xlOPfXUpP6/0/n0pz+Np59+GkByX+FTp07Vv9+1axemTp2acX0PPvig3jRjw4YNGYPmb3zjG/j2t78NAFi6dCnuvfde02nLysr07d6wYYMewJv56U9/iqqqKuzatUsP4L1ywQUXjAngjcHz2WefjcLCsZfHf/zHf9QHDtMCZTOLFy/W8yr1WDzttNNQVVWFvr4+7NmzB93d3airq0tKQ6Z9bzddn/70p/Hmm28CSATrjY2N2LJli/679tkYwGs3h0QkJgbwRCStt99+O6kdulFdXZ3PqZHHvffei66uLgBAQ0NDwKnx3/nnn49//dd/BXC81ttO7XeuotEozjnnHP1m7o033sAll1ziWRrOPPNMPPzwwwASQfrMmTPR29ur/64F7qyBJ5IHA3giCpVIJIJZs2bhxhtvHPPb7t279ZE6d+zY4XPKxNHS0oL29nb09fVh7dq1+vfd3d2erO+cc85Bf38/AKCgoCDtNDfffLPehMZr5513HiKRCFRV1WumrQTP99xzj95UJVcXXHCBHsC/+eabOPPMM/Hhhx8CAGbMmGHrxipbuozB+ObNmzFr1qyk37du3QpVVfUAvrW1FbW1tZbXT0T+YwBPRNJasmQJli5dann6AwcOpG1iEI1m7pBrwYIFeuDvFy/7//7mN7+Jl156yZNlp1NQUKC3szfz5JNPjvku235xqra2FnPmzMHbb7+N7du3o7+/Xw/gje3fU5k1f8qWznT70dhE5o033tBvJAD7te/Z0tXW1obKykocPHgQmzdv1gP1lpYW7N27FwMDA9i4cSP27t0LgLXvRDJgN5JElNeKiorwD//wD0Enw1e7d+8OOgmWfO973/Ns2VqQrKoq/t//+3/4+OOPASSeFpg9JUhn0aJFqKystL3+OXPmoLq6GkCiBt5p+3cr6YpGozjjjDMAICmAv+KKK/SmZqtXr9bnZft3IvExgCeivDF37lyoqpr0b2hoCGeddVbQSfOVsUZ4586del5MnjzZk/V1d3fj6aefxtNPP40XX3wx7TQHDhwYs2+8vLEyBsmPPvqo/jlT7fdbb701Jo0rVqxwtP5oNKr3y75z5069t6BsaXCaLq1W/cCBA3jhhRcAAKeffjpOO+00AMkBPGvgicTHJjRElJc2btyo99bS1taGL3/5y8EmKMTeffddXHvttQASN1EbN240nbavrw///M//DAAoKSnBXXfd5Umazj33XESjUcTjcaxfv17/3mrwbGy6ddddd6GkpMR2Gs4//3z9GNSC6pkzZ+Y0iJJZuoxBufb+x2mnnabfPGjfFRYW4vTTT3e8fiLyBwN4IspLGzdu1LtBvPrqqxnAC6Kvr0/fLyeccIJnAXx1dTXmzp2r114DiT70rQavxi40v/3tbzsK4I1PAbQ05Np8xixdqc1iCgoKMGfOHH0UVs2cOXMcbQsR+YsBPBHljU8++USv8TS+NOglY883xq77RPH000/r7aDdHDjp8OHD+nYb+8U385vf/Abl5eXo7Ox0vM6f/exn2LBhg/53tp6Gzj//fLz11lv639nav7/44otJ4w7k6pRTTkFtbS16enqS0mSXlXS1tLRAURR9sKoZM2agtLRUb0KjYfMZIjkwgCeiUCsrK9M/79q1S2/KYTaN2+s06/nG7XXaYVz33//933uy3GPHjmXd7tLSUn001q9+9asZpzVTU1ODjz76CADw8ssvmw7Ola5bxAsuuAA//vGP9b/TBc/GNNx2221jfo9EIo5rrCORCM477zz8+te/TvrbCifpOvPMM9He3g4AeuA+ffp0lJeX6zdvDOCJ5MCXWIko1E477TRceumlaUfWBICmpiZ8/etf93Wd48ePx6233urqOu249dZbMX78eJSWlmLq1Km44IIL8PWvfx3l5eU5LbehoQELFizAuHHj0v5eVVWFRYsW6X//wz/8A0pLS9NOW15ejm9961tZ13n33XdnbTPe2NiIH/zgB2O+P/fcc5Nq3NMF8N/61rdMe5kpKSnBokWLcmpyYmwyY6f9u5N0GZvRaE2FotEo5s6dq3/PAJ5IDhHVq46GiYiIiIjIdayBJyIiIiKSCAN4IiIiIiKJMIAnIiIiIpIIA3giIiIiIokwgCciIiIikggDeCIiIiIiiXAgJwf27t3r+zrb29vTDkBD6TG/7GF+Wce8sof5ZR3zyh7mlz3ML+tEyauWlhbT31gDT0REREQkEQbwREREREQSYQBPRERERCQRBvBERERERBJhAE9EREREJBEG8EREREREEmEAT0REREQkEQbwREREREQSYQBPRERERCQRBvBERERERBJhAE9EREREJBEG8EREREQ0RrS7G02KggZFCToplKIw6AQQERERkXhq585FFKztFRH3CRERERGNwSBRXNw3REREREQSYQBPREQkkCJFQbWiIHL4cNBJISJBMYAnIiISSB2AUgDV06cHnRQiEhRfYiUiIhJIZPT/okBTQWRNo6IgCqAfwKFYLOjk5A3WwBMRERGRIwVI3HSODzoheYYBPBERERGRRNiEhoiIhKQ9mh8G0MVH80REOtbAExERAKBeUdCkKIj09gadFADHH82zpomIKBkDeCIiApB4aTIKoO6UU4JOChERZcAAnoiIkvDCQEQkNpbTREREREQSYQBPRERERCQRBvBERERERBJhAE9EREREJBEhe+datWoVfv/73yd9d+GFF+LGG2/Efffdh02bNiX91tzcjOXLl6OkpAS7d+/GkiVLMDAwoP8eiURw9dVXY/78+QCANWvWYPXq1VBVVZ+mtLQUS5YsQWtrq4dbRkREZE0k6AQQkbCEDOA3bdqEa665BmeddRYA4JVXXsHLL7+s/7Zw4ULMmDEDANDb24tly5ahq6sLEydOxIcffggAeOCBB/TlPf/880lB/+bNm3HOOefgyiuv1L+7//778f777zOAJyIiIfARORGZETKAB4CamhpMmTIFAPDee+8l/dbQ0KD/VlZWBgBJtenRaFT/HQCqqqqSfldVFZWVlUnTFBYKmxVERERERDre4BMRERERASj89a9x0+LFqP7854NOSkZCBvAtLS2oqKhI+i4aFTKpRERERBQSdbfcgiiAki1bgk5KRkK2G7n77ruT/t6xY0dScxciIiIiIrdFUv4XlfDV2j09PXj55ZfxecEfZRAREcmuSVHQpCjAsWNBJ4WIMhCyBt7oN7/5DSZOnIg5c+YElob169dj/fr1AIBly5ahvb3d9zT09fUFsl5ZMb/skTW/xh0+jC/fcQd+DaB/xQpf1ilrXlmxyPDZrW3MJb+8SI/ItLwKcru1dUemTMGvfTqnnArzuegFJ/l1o+Gz2bxhO09vNnwOensWLVpk+pvQAfzAwABeeOEFfOMb3wg0HRdddBEuuugi/e9rr73W9zS0t7cHsl5ZMb/skTW/GhUFBQC+BmCfT+mXNa8sWbxY/+jWNuaUXx6kR2R6XgW53aPrLgli3TaF+lz0gKP8snIshu08lWR7hA7gf/e736GsrEzvD56IyEhrAyh6W0UiPzUqCqIAugAMx2KOliF8+1qiPCfsOXrs2DGsW7cOl112GftoJyIisqgAiZvauqATQkSeETaA/8Mf/oDBwcGkpitERETkvXjQCSCijIQM4FVVxXPPPYfPfe5z+kirRERE5A8hgwMi0gl5jr7++uvo6OjAFVdcEXRSiIiIiIiEImTj8meffRZtbW3o7+9Hf38/AKCwsBATJkwAAHR2dmLXrl0AgN7eXgBAJHL8NbZ4PK7/DiS6TjL+HolEcPDgwaRphoeHPdoaIiIiubAJDZHYhAvge3t7sW3bNgDAHXfcoX9fX1+PlStXYvbs2XjssceS5mlqakJ9fT0AoLW1FaqqJs0biUTwhS98Qf975syZWLNmDV566SX9u9LSUkyfPt2TbSIiIpKJkI/niUgnXABfU1ODp556yvT3e+65J+P8kyZNwuOPP55xmi9+8Yv44he/6CR5FIBCRUEdgBEAXQ67RCMiIiIKC+ECeKJUdUjUBrFGiIiIiIgxEUmAg/QQERERHccAnoiIiIhIIgzgiYiIiIgkwgCeiIiIiEgiDOCJiIiIiCTCAJ6IiIiISCIM4ImIiIiIJMIAnoiIiIhIIgzgiYiIiIgkwgCeiIiIiEgiDOCJiIiIiCTCAJ6IiIiISCIM4ImIiIiIJMIAnoiIiABFCToFRGQRA3giIiJKEgk6AUSUEQN4IiIiIiKJMIAnIiIiIpIIA3giIiJKwiY0RGJjAE9EREREJBEG8ERERJSENfBEYmMAT0REREQkEQbwREREREQSYQBPRERERCQRBvBERERERBJhAE9EREREJBEG8ERERJSEvdAQiY0BPBERERGRRBjAExERERFJhAE8EREREZFEGMATEREREUmEATwREREl4UusRGJjAE9EREREJBEG8EREREREEmEAT0REREQkkcKgE0BEREREyaKKgkYAKoCOWCzo5JBgGMATERFREr7EGrw6JPYD9wWlwyY0RERERIJh4E6ZMIAnIiIiIpIIA3giIiIiIokwgCciIiIikggDeCIiImKbayKJMIAnIiIiIpIIA3giIiJKwtp4IrExgCciIiIikggDeCIiImJAQCQRnq9ERESUUbWioE5RgKNHg04KEYEBPBEREWVRCqAYQHlra9BJISIwgCciIiJYCwhKPU+FdX+zeDGaFQWNihJ0UiikSn/wg6CTYIoBPBEREUmnCInecgqCTgiFVtW//3vQSTDFAJ6IiIiISCIM4ImIiCgpIDDrB579wxOJoTDoBBARERHlm1pFQRGAPgCDsVjAqSHZMIAnIiKhFPKlRMoDxUg80agGsC/gtJB8GMATEZFQ6sGmGkFgnvuL+U25kCKAf+aZZ7Bjxw7cfvvtWLp0KbZs2ZL0e01NDVatWgUA2LBhAx555BGMjIzovxcVFeG2227DvHnzAAD33XcfNm3alLSM5uZmLF++HCUlJR5vDRERZcLAhpxoVBREAXQBGGaTFAo5KQL43bt3J/197rnn4sorr9T/Liw8vhlbtmzBzJkz8bWvfU3/btWqVdiyZYsewG/atAkLFy7EjBkzAAC9vb1YtmwZurq6MHHiRC83hYiIiDygdSdZB6AjyIQQ+UCKAH7Pnj1obGzU/66srMSUKVPSTquqKsrKypJ+LykpgaqqSdM1NDTo05SVlenzEhER5SM++SCSh/DdSMbjccT4KIyIiIiICIAENfBdXV1BJ4GIiCgUxikKqgD0AjjGyjEiaQlfA9/Y2Iizzjor6GQQERFJrwaJtuJ1aX4TPiAgIp3wNfDprF27FmvXrtX/nj17Nu65554AU0RERGRfnaKgGEAcAFas8Hx9bOdOFA7SBfA33ngjBgcH9b+3bt2Kxx57zNN1rl+/HuvXrwcALFu2DO3t7Z6uL52+vr5A1iuCmw2freZBPueXE7Lml5NjI1ey5pUViwyf3dpGJ/m1KM13Yc1z7RiO4nhe5bofMs2f6bevpizH+LsXx0auUs9/EdOYSbb03pjld7ucnItJaVCUtDeZsuV7Njen/B3kNi1alK40TIioEnS9snLlSgwODuL2228f89sbb7yB5cuX46mnnjKddunSpWhtbcX1118PAPjSl76EO++8U+9WsrOzE4sXL8aDDz6ISZMmZU3P3r173dgsW9rb23Httdf6vl4RNCuKXmu012KbzXzOL6uiXV2InHoq4tu349e//a2U+eXk2MhVmI+tltERUEcA7HcpP53kV0vKSKwqgH0hba9tPIZXrliBa6+9Nmn7nRzX2vxxAB0p82u/pcvTckXBCaOfjwLoMfyuzXcMQJcg+yL1/M+03SLKtC+ARL/2WteYbpRvTs7FpDRUVwPvvjtmmmzbIRvjcRX0NrW0tJj+xiZvRHmq8dRT0Qigfvr0oJNCRIJhUxsisUkXwO/evRsHDx4MOhlE0tMu0AUZpyIiIgJw4EDQKSAD6QL45cuX45VXXgk6GUREREREgZAugCciIqLcpGsiw2YzRPJgAE9ERJQnhO+1gogska4bSQDYt28fNm/eDCDRJt4oEong8OHD2LVrl/7d4OAgIpHkuoXOzk59mt7eXn1eIiIiIiKRSRfAz5o1C+vWrcO6dev072bPnq1/bmtrw6OPPoo77rhD/66oqAjXXXdd0vSpfcc3NTWhvr7ew5QTEVFW4vdsHFqswiKShxQBvLEj+5tvvhk335zazf5xF154IS688MKMy+OorUREgpowIegUEAmrebTP9f0A4iHoc90t9YqCQiTGAHBrDAvRsQ08ERGZihw8iCZFQUPK4EoUbkUATuA+F05k9F9d0AlxIHLokGdlSSES+ZJP3SIzgCciIlM1M2YgCkke11JOIimfy4JKCIVS1cknsyxxEQN4IiIylU81WpSMbeLJTQzc3cX8JCIiInJJo6IgAqBz0ybEa2qCTo5lWjvyQwD686QducxYA09ERER5oVBRUKsoiIx2H+2FAiSCq9pTTvFsHV7Q2pFXBJ0QsoQBPBEREVkie7OaegDjADT4EFzLlleypTffMYAnIiKivKAFqQx+SHY8homIiIg1sEQSYQBPREQcAZWISCLshYZcN15RUAFABdDBN9mJ5MARUImIpMEAnlxXgeOjxRGR3FgvT0QkHjahISIiIiLXRC65BFFFAYaHg05KaDGAJyIiIqKM7DxVb3r3XTQBKJ082avk5D0G8ERERMRmjy7L5/zUtr0y0FSEGwN4IvJEtKMD1YqSeIxKRERSMLvxYMAoFu4PIvJEw7x5KAXQGHRCiIgoK76wLhf2QkN5pUlREAHQCWCEXVx6KpLyf9hpx9Z+AHEeW5RHtGO/Y8cOqKWlQSeHKC+wBp7yShSJgLIu6IRQ6PDYcke+3PCFiXbsl0yb5u2KRkZQpyioZbO8QDBgFAtr4ImIXMQAlMgjkyahOOg0EAmCN1RERETk7ObTUBvudY/f4z1evkwKAlgnA0axsAaeiIiEwSBBLLWKgqKgE0Fj8Ekf5RTAd3Z24v3338fIyAjGjx+P008/Pe1v0WgUJ598MhoaGnJOMBEREfljnI1pefMVbrxpEEtOAfySJUvQ29sLAGhtbdUD+G3btuGHP/whhoaG9Glramrw8MMP57I6IiIKOQaBcjHWzjPAI/JPTgF8b28vrrrqKpx99tkYNy5xn75//3780z/9E2pqanDLLbegsLAQf/7zn9He3u5KgomIKBzqFQUFALoBDLPrzcBE+vvR2NbmaF6+VEoUjJzbwNfU1GDKlCkAgIGBASxbtgyqquK73/0umpqaAADbt2/PdTVERBQALwd30Wpv6wB0jH5mDbz/atraHOe78WVK1sCHG/evWFwrK+PxOB588EF0dnbiO9/5jh68ExERZcLAIFi5BAJRk89e4HFCdFxONfDRaBQvvvgitmzZggMHDmDHjh245ZZb0NbWhkOHDmHVqlUAgK6uLkSjrFchIqLMGKTJhVf2/JFtX0eQGJW3g83hfJHTuXfZZZdhcHAQO3fuxJEjR/DNb34TZ599NgBgeHgYO3fuxM6dO3H48GFcdtllriSYiIgCkjICZoGioEFRUJhhZMzxixejQVFQ5OHomYWj6SjgCJ2+4w0XGflxQ1epKKhXFEQOHfJhbeLKqQb++uuvx/XXX5/2t6qqKqxcuTKXxRMRkcAakAjg6gHsM5nmq0hc1OsyTGPkJCCsN6SjI8u05K6IyWcir1SM/l9z8snoyePafg7kRORAraKgGEAcwP48LkDMNCsKL+Z5JNO+jliYxq31i3jMNYz2tDMMoCsEZYWVfS2bsO2jfJHvAWxO2//qq6/irbfeQjwe178rKSnBl7/8ZZSVleGPf/wj3nnnHUSjUZx66qn4q7/6q5wTTCSCYiQuVkEMZy0DWS/kFLywtakuTPk/bMJwrod9H4kiDMeKSHI6XlesWAFVVVFUlOgMLB6PY3BwEHPmzMEZZ5yB3bt347XXXsPw8DBefvllBvBEROScqgITJiQ+h7WmNLUdv6IAe/YAEnQEwQAtIENDwNSpnh8nod6/IyPAvn2ARD0o5hTADw0N4YYbbsDll18OAOjs7MQtt9yi/z5//nzMnz8fL730EtvDk1BCXRARuUioc2XCBLQg0Te9lfb0YdACYHDiRPQKesMi1PGRp5qnTkUEwMDEifhE0ONEdPWTJqEIiWaxsnD1Vi0SSX8qm31PRERkV75dUcYFnQCL8m2/iELL91Kf1iO6RkVBk6IAw8OW59Fqs8V/znVczmllcE5EJD+W5N4bx64uSRIyd9FYgNHg9rzzAk6Jt3JqQlNYWIhnn30WGzZsAJDo+x0AnnjiCaxevVqf7tChQygs5OshREQyiyDRfMXrdViiKNK1g69F+Jr/8MYvnOpPPhmdkp1fqSIff+x5eRWknKLq+fPn480330zqhWbmzJljpquoqNDbyRMRkZyiAEaCToTkwhzwhnnb8k26Hta4f8WSUwB/xRVX4IorrnArLUTCKFQUVAPoBTAieS2E79hEQEpW2lP6EcAzSCAiyi6nAH5gYAB79uxBPB5HUVERpk+f7la6iALFkR2JxpLpBS8KuTyoKOA4I7mJDg+H+olhTgH8XXfdhc7OTgBATU0NHn74YVcSRRQ0kUd2JAqKH+cDzzl5cd8R+SenAL67uxsXXHAB5s2bh+LiYrfSREREPrNSuy5SgObHC7VBYHcP5kQ6/oiCllNZUVlZiUmTJuFTn/qUW+khIiJBidSEJqwv1NYHnQAiE7yBEktOAXxDQ4Nb6SAiIsGJdAEvRDgD+HREe9JQrSgoRWLUStHSRqQJ602+JqcAfs+ePeju7sZ///d/W5p++fLluayOiIg8IkoTmkzrMP4m0tOAfFMy+n9qgOT18SHSDSSJL+xlRE4B/JQpUzgSa0hF+vsRbWuD+s47iNfWBp0coqyi3d2IzJ2L+LZtUCsqgk5OKIlU2vt9cY4qCiIARnbsAEq9HrSewkCk8yUfhT3/cwrgly5d6lIySDQNbW0oAKDOmYN97AedJNA4d24iwDr5ZOznMWubDBc7Y9Dud3obR9d5dNo09PD4ojwkQxlhJFt67Qr7EwZySDswRDsBivKg719yRjtW0xVqTYqCJkUBhof9TFLoBF0eRE0++0Hb9iKf10tEzoQ9wM2pBn7Dhg148803EY/HM04XjUZx+umn4/zzz89ldUSoA7Av6ESQdPSCfPJkgLWnQrN6kxD0zQSNxTbw5IWC730P1U88gQOf+QxG1qzxfH2yHGc5BfA/+9nPUFxcjNIs7QGPHDmCt956iwE8EZGgrFy0gr6wBdmEhoiCUf/EE4gCaPjTn2xV4IW9jMgpgD927Bjmz5+Pyy+/PON0v/nNb/Dzn/88l1URkWAaFSXxngTA9ySIiARUqCiox2h3nytWBJaO8tHrhRMcGT09XwZ9Y081ROGjFcY8u/NH0Ps6YvI5LP5m8eJQbpeoGhUFPUEnwmN1SJwrbhxXuSyj0qU02BH2cynsbfyJiMgCUS4G+dwGvjjoBOSRCBKVEBz5NrzCWEYY5VQDH41G8eKLL2LLli0Zp9u/fz+iUVEuD0REZJVItd5Br5+Cxf1PdFxOAfyll16K119/HTt37sw4XTQaxWWXXZbLqoiInDF0PRpBeId+r1AUVAAYANAf0ncSPH2JdWQEjZMmAQDHEch38TgaJ04EwGNBZmG/4cspgL/hhhtwww03uJUWU8888wx27NiB22+/HWvWrMHq1auhqscvw6WlpViyZAlaW1sxPDyMO++8E3v27Elaxsknn4z77rsPAPDOO+/ggQcewNDQkP57QUEBFi5ciIsvvtjz7SEKM5ELzdSh38NkPBJ5XwGgP+C0SGnSJMcv2dmmKKHszlTkc9+WiRP9OxbssDsOykhYSzsCfHqJNVe7d+/WP2/evBnnnHMOrrzySv27+++/H++//z5aW1tx4MAB7NmzB7fddhuampoAAB988AEeeeQRDA4OoqSkBNu2bUNDQwNuvfVWfRm//OUv8e677zKAJyKpeRlEhSZAS0PIgI0CEZbjPDppUuDvtgT5xDMs+9FMTgH8O++8g7fffhvxeBz19fVJ3UkePnwYq1evBpBoQjN37lzMmTPH0Xr27NmDxsZGAICqqqisrMSUKVP03wsLj2+GVjPf0tKCSaOPQwcGBpKWp6oqiouLk5ZRUVGBwcFBR+kjCkLDaLdc3QCOCVSbF/QFI5Mw18Dnm7BfnEWVD/kuchlmR23QCchEVQGPeygsBVCgKOgR6ProppwC+P/9v/+3HvROnz49KYAfHBzEf/3Xf+l/v/DCC3j88cdtryMejyMWi+kBPBElaCdvLYCOIBPisgJFQRWAAwDiLhe8Ybkwe0GGwEyGNMqkVlHQF3QibPDrhWo3y4lqRcEhBFPJEnd5eZbz/H/+z+zTLF4MrFyZS3KyiiDcPTvlFMAPDg7ihhtuSDuQU01NDX75y18CyG0gp66urlySSBR6ogU1uV78GpDYpga4f2MiWl7RWJn2Efefu8YhcZ4NB50QweRUhqW8NF8KoASwNYKoW8L6wr4dYS4zfKmQymUgp8bGRpx11lkupobIHw2KgmZFQbXdF48kl2uh4uWoe46XOTiIZkVBU57tS5HJfGEWKe0ipcUOmZ6myZrHjr3xhv7RdNsN05AzOZ8DHGWVKL1CJAqvkqATQjmrOPFERCBX0GCXlZI86NJepD7pcyFF7xF5LOnYkvimPbDRV48ccWHN3qtTFNQrSqI9voRyKkcKCwvx7LPPYsOGDRmnO3jwYNKLpkQUXiIHVk4D8DAH7rKSudcYHk+58zIPZT62pJHS1XcQ9PbxWcYyElVOUfX8+fPx5ptvIh7P/KpERUVFUrePslm/fj3Wr18PAFi2bBna29t9T0NfX5+v673Z8NnuenOZN9vyrC7TLL8WWVxOtulSt/FvFy9GIRIvDa1ascLWupzKdfnG+Z0cX2br/0qaae0s2852pR4b6eZLLXmc5NUCw+dcjy2v5HrepY7ooS3j70y+t7rOTNOky7NvmKSvvb09KY0Rwzxu5P11WZbhdB2L0nwXNSzD7BhO5dY5ZHV9VqXesHt57H8zw2/t7e2W9lG6/ZG6nK8a/058aSl9QOKdgi+aLDdTWqz8/lcAWtLMY7ac1PzKVs6nu9belOa7TOkEEu39U9OZuuyXAXRazNfUMiTd9mY6rtNu84cf6svZuGwZmu3O75NFi8yP2Iiqiv/sYOXKlRgcHMTtt9+OpUuXorW1Fddff73++6JFi3DVVVfh0ksvRWdnJxYvXowHH3xQ70Zy8+bNuPfee/Hzn/8cJSUleOqpp/Dmm29i2bJladeRzd69e93fyCza29tx7bXX+ra+ZkXRC+a9Nt+ez2XebMtTAeyzsEyz/GoZfRwaB9CRYTnadGbrS93GdNucbRm5ynX5xvl/umKF7eOrxfBo2bifSxUF1SnT2jkO7GyXMd81Y+ZTFP1ichDAIQd5dYKioHz080qTvNLSPYJgRm/M9byrHB3JFUjehgZF0Wt6hgB0G5ZtZZ2Zpkm3r2sUZUyzM+13YxqNy3PjXCtSFNRrywXGDLRktexIZUyblg+fABgYXUa6YziV03Wmmy/d+o4BKLK47DgS2xEx/K3Vhh8D0OXhsV+rKBg3+rkHQI0hHXtjMUv7yFhuAcn7RTt+xisKxmvLBSwNutUwmq+dgB4MGped7vzIdtwmlbEffwxMmJAUGKce/6nbXacoSb2wmJVdmnTX2iZF0ffvIIDeLOkEEnmmpVMrS1KPO6v5mpou4342llOZzqO0ZdMLL6BlNI7ce/HFaH7+edP5OzZuRLy+3uRXb7W0tJj+lrUGfsuWLXjjjTeSatlbWlrw+c9/fsy08XgcL730Enbv3p00Umo0GsW8efMwc+ZMu2knInKNyM17KCEfeqEJy3aElZP9owVTVS6mI8mECV4t2bJQHbeGGLVo166Mk1Z/9avo+e1vPU6QfVkD+AceeABHUl5IOOWUU8YE8AcPHsRDDz2ErVu3pl3O73//ezz22GM5JJWIZCBa+143LjrCP6Z0mVmeheoCnsLP4zaM54jfRB1xmG/7SXI8dXfrH4sPHMg4aeHu3V6nxpGsx9qRI0dM+3rXfPjhh3jwwQfR39+PW265BWeffXbS7//3//7fQNsQEZHHFMXy41C/GYMlpxcWKS5IlBM/9zGPJ2dkyLd8u9lPR4qbGMMYQwUDAwEmxLmc8/mll17Co48+isrKSvzjP/4jpk6d6ka6iIiIkogQwAXWNR8lEbUGXiRBbYcUvfj09ekfI8NZhjITtLt0xwF8PB7Hz3/+c6xbtw4zZ87E3//936OystLNtKUViURw8OBB7DK0WRo2ZL7WL/3evXv1dvsdHR1jljE0NJS0jEOHDqGoyOprPEQkCzdq4PNBvudNvm8/HZfLscAaeEkC+IMH9Y/RY8cyTytoXy+WAvjUwZr6+/vx4x//GJs3b8Yll1yCBQsWIBr1p1XfzJkzsWbNGrz00kv6d6WlpZg+fToAoLq6Goqi4KGHHkqa76STTkJJSYn++emnn8Ydd9yh/15QUIAFCxZ4vwFEEhmnKKhBoncDqz2riBYIidbeWGZSXJgdsnOc1I727jECoNNB0zHRzhEZiXJe1xh6xqEEKY7vQ4f0jxFBA/RssgbwJSUl+M///E889dRT+nfDw8OIx+O46aabcP755ydNv2vXLtx7771J3w0NDenBsxPGfjC/+MUv4otfTNfLakJhYSF+/OMfZ1ze3Llz8Ytf/MJxeojyhdZNm8yBmxQXE0nYzctqiUexzKQYibxw+gibx6TY7OyfcTanzwd+31wVWeiKdYzDh/WPsu6/rOXPrbfeitdeey2pG8mysjJ87nOfw8SJE8dMX15ejjPOOCOp1j4ajeJTn/qUS0kmIpGJXBiKnLYwKnV5edx/ROLz+zytczKTodZd1nIlawA/b948zJs3z/IC6+vrM44cRUThJlphKFp6SEzshYY0kZTPcjawyKxptNb6II4PKuZk/nT9t/h9fDtan6F7dFnPR1GakREReYIvsRKFj1/nskhBkt1tzjR9dPT38RmmyUSbvzzNbzKUs7WvvGJ52ujBgyh89VUPU+OMSMcmEZH0ki5e8TgiioKIBP0Mi3LRDSodomw/pXH0KCKSvE8h23EkbHoHBxG57jrg6NGcFxX5538GUrqKLLYzP4Dq227LOR1uYwBPRKHm+gVq8WLLkxZMnIhmAE0nneR2KshlbEKTO6+2q7G1Fc0AjB09+9UPvEhBkkhpycSNfdN44olo/tOf0NjamvOymv/pn1A/eXLSd6nNpLIp3Lcv53S4TZbjgYgkEcbgxOk21eY4f5jVOahR9Ssfub/EogUqQewXBknHWc1/N/ZTQcr/uZJidFibwrhNREQ6uzUtrqxzYADjTzqJgWAGdh5hp+Pli4VRJPp4J3Hl482cVzcTBYqSti27l7K9HKxCrLwXcTAnBvBEFGpuXASMy2iwMH3tSSflHKCKQqiLqIfyZTvJnnw4Lhrg7nZ6mmf/5/+4vkhZ9zGfDhERucyLmpEGRUGzoqDco5f5ZL2IecHrC6OneR3wy57NioJGSV44NZPL/vFy33p1XFpNcxES5RCJgQE8EeUsiGYqQbFSA++FQiTytjKg9ecTxxfGoSHUKUpoR6DNpgDyj9ycyu6xIFL553Zachl9OJXjc+yNN5L+rPfrXBOwCQ0DeCIKNbdvLipcWAY541fPI06VTp2KYrg/Ai2NJVKgbORlu2RRtzmVp+n86CP9YwGSeybKN2wDT0Q5C/tLf7JcOMkdTve33y8Ckvf8Pve1EU67fV6v3xxfMzo7XU6JvFgDTxQGR46gUVH8e5yYQuSCxO0LMGtXxRUBgGPHUK+1wx4cdL4ccl2DoiTaUI+E+XY/N9oIp7VpfuNxCeDgwaBTIAyRr7tEZFHltGnCPE4M+0UmLL3LhFVkyhQUIfF4vfzEE63P51mKSFM4+g+TJgWcEut4XJCoGMAThUDQQWU+FSQi3CSROeM7CiUOl5FPx3MQxrmwDL/ehxApgA/Tcel4Ww4dcjMZUgvT8UBEARHpIpfK7X7giXKV78dTvm9/UG5avBgYHvZ8PZ7u36EhL5dujr3QEBEFx80hvpPkabeBXsh1HzE4FF+2fRTp60O5oiC6f78v6cmYloDW60W4GAWAyZMtT+/VmBMAz1M3MIAnopzlU2Ecpj6ujfJpH2bDvPBWtsCjYdYsnACg8fTT/UiOOHyoCLDTBPAEiBEk8nxMT4R9Q+QPAR+BhZGMhW3B6Cinso8gSc6J2u45NAznVrbAQ/s9034Yl+V3GWXanuYsv7vN63X5ve8iSIwSXPDcc4GnxS0M4Cl/TJiQ9uuC0e4XCxm8OWa1IDlBUVCrKL62Y7RSONfD+giSLDSD48WFNnL4MOoUJbRPVqzwO4Bx4xzya3/5lTeFipJxlOcI3OnCVvbyK5f9EQFQd+ONbiUlcLLvS6Kc1SPxWLE+6ITkgXIkas6Kpk4NOimOsdAMl6rp01GM5KYFstbIyYL5O1Y9st+UyJJvXqYzsPJXwCf4HImV8l4k5X+7mkdHziPrKgH0BJ0IG7h/w2ecoqAm6ETkKdHPpyDS59c6Rcl7p4F4rgG82faLki92sDKJfBHp70ejoqAuhM1UZDzx3ZZPecBC07loR0fQSdBVI3Hc5tOxayaI9sj12qisQXULaEhLLr/LJrWWP2zbl094LconAT4CqmlrQwGCH3CIvCHyRcDttIm8raKrnzfP83VY3T+Zpsu3fex3IBBBoslSIYACiZvTiS7dcSz7sc2g9Tg2ockjzRMmIAKgA0A8FvN13fn8gli+EfkCIXLa8gHzn1JVAPgk6ERIItt1NAqgRVEwBKDb5BrvdwBcgESa3Ko+ZBlyHG9m8oh24Ne5ucxPPkETu9/Le3YLVfFeB7IurBcQEbYr2tHheq1SraKgSVFQZLOMihw6JH3ZFkFiUCQRGI8vmZ7EinBeaKymxU5f735JTbvTfBVpfwSNATzlpGbmTETBGnbKHyIUmpGBARQoCqJdXUEnxVV18+a5PhJrMRL7zE7FRQRA1cknS1+2RQDUzpoVdDIAJJ83MgdhkUOHEuded7et+QoUBZGBAY9SZZ3MeQ+wFxojEa5FJDGZL25Esqo/6SQ0Amg89dSgk+IqLy5ITgOWsLQvFeUiL0o6clV/8smJc2/uXFvzNSJx3qYTpgGayD9hOaeIyGPN2iBMaXh9UThhdKTUeombM7hJu3HmxXgsGfJEhjS6TdbKnnFAUjOqXM49EfLAj2PPyjry8RxwGwN4IrIkAudtVyNHjqAmh+C7bHT9TmpF3bhQREw+h1mdoiDS2+vNwtMcC1a6mM23vrLDxCxPS5BoXiIqqyM0kz94bh7HAJ6ILHNaeFZNm4aSANfvJhHS4LUoEjdr9aec4ts6tbbqoSFwUBoEs31bAKDBz4TkOVHKL77EmrtQlZeUvwpHm1g08aLpyM2LFydGlD10yNH82QpVmXqdoONs1Tymnns2zkU3L8puLKsQ9p72aOVPpt4/8jnwaFIUVGb4PVPeNFoc6dqvst/OfhRxn4uYJinwJVYib9QhUTDxgHZGG5Gy4uSTg06K8HgBDL9C2NvPWvnDYyO9KJznjdWbSK/L/sIQVQ6J3GQpG55jxzHeISJXsYANDxmDUqHSK3GgRMncHD8laPVBJ8Ah8erAgxWWnrKIshLqwi6oeNAJ8FgRgHpFQSESF4MOiyMS53NBKfN541ba2TTvuCjyM5Dy+zzwan0y3pQbyZx2t+XzdSlviV74nqAoKAXQD2DAYoBFBooC+Jxvnhaq8biry9faKTtty5puPl5UxOHFvpD5UTWPTRKRSMelSGmxQ+ZyiUKqDIkDsxJAlDVfNHFi0CkI1rFjUrdZJX/JGoyIRJY8jHzyCa+ReYwBPAknYvi/Ce4VpjzYsxOxn+1c+2B2e5vSLc/SseXwQnvClClodDQnEYVZ48yZaAo6ET6T5ebKD2xCQ3kjzAG8W4WaiIWjbPvN7TampS4uK2zK03RP6Htb5b4+Ic8bTRSJUZS7AAz73SRRVdE0YYK/6/RQs2C13U7KRlFGYqXcyXZtJKcEK3hyUasovj42rFYUFEmaf9GeHtQqiuUu0LSCN9rRgdqA+vGWkbEg9aPGP5toR4cv6wlaJYJPd+2sWUK9F5F6UdduKAPpRWXChNAEGbK//OkmN66H0g3kxH7giXITATAOzkbuc1pbUQp5uxCrnzMH42C/27CGefMwDmJcsEQbxlyEPMmmYd68oJOQkYhNtZwyK1d4caWwkvV6GDZsQkNSCkuXXl7Tggir6Y+k/C+CdAF8k8XRGf1iN1jTHsXv27kTKHZ/nFqt2cQwgC6fmk1EkNzDVab9kw/BbRTASNCJIPJAtl65RBamsidM20KUkWwFTZjlOhx5LiM7ikB7HB+ZOtXTdQRZQ5Pp4uLlhUeU4yKoiysv6kT5gec6EelECX6MMhVSoqTXaTq8DrD9zJ/U/ZTvFxdRjk0iI9mPSy/Sb2mZHjwpzVW+l7F5Q7R2xEGQveCyKpftdOPFIhFe4pRFmEa+tRPA58OFJ8zHLVGupDs/IuKlmG3gJVWgKKgHcAxAj4U2rsYdLd5h6A8vewoRSdja3oq2r0TqcUQk6Xo/SRVB4v2Fww7XIVM+swlNcGQ6TsyEYRtChb3QkFvqkdh54yxOzx2dP8K2r8N8IQvTvkrdT2YBfBRAuffJyWx4OOgUkIfCUGaEYRucivT1BZ0EKYTp+pFX7J7c3NH5I5d9LeJFQ8Q0pbKS5/p2GPpQDtN5mbqfRN620smTPT+uZDhu3WblJo6CJ/p+aTIZWwFgP/BGbEKTJ0S+mOYq2tMjfIGUiwZFQQGQcSRFmbv1yiYs25OuaZPX2ybeJcd7VvI0dfRWL/g58E+0pwcNc+ZABfCJT+vU5Ht3obKR4X04Ict8AQN4nl95QsgTwiW1c+ZY2j5Z86AQAY6k6BE/94UoL9WmK2xlPSbTsdIG3m9BpsHPddfOmYMo/AvOrFYYiBZgZGuaEQEwzsYoo1ZHuA4K39cJN9HOL/JImE9aHsTJguiFhrJLd5yG+djlseTe/s2Wl6IeR6Klq37WrKzT1NpZnvOkkGwErIFnE5o8IdTFVFGAWAxFioI6JLrS2+/DaJFC5UGKGkXBOCSaWHTmmBeiXTRzJfJ+syPstWFhav9sJ+1ml3XthV2/iZTvIqUFcP8JRerow2SdaMeGjMJ2rScTIp4stUikS7Q2eZH+ft/XOQ7ZR84UcR9qRE6bH5w24QpTvoVpW7yWL3mVL9tJeUDAGngG8HlCxILU7zRZXV9tW5un6fCaiPs6F2HZnrC3gQ9TDbwdmXrLyPcLbL5vP7mPvdAcxyY0RClEeyJgl8xt4AsEfyksnSZFyWkE20ZFQRTAUQC9PjQly5QWkp+sQXOToqAjwOOf8sjbbwedAldIGcAvXboUW7ZsSfqupqYGq1atwoYNG/DII49gZOR4h21FRUW47bbbMG/ePADAfffdh02bNiXN39zcjOXLl6OkpMT7DQhAmLsZFO++2HtVGQLdXPZvIRDoIDcy9rRjNWCKpPyv0W4YrQ7KJrKooqABwDASo0TnMz+7kRSVne2X9caDnBkHoEhRcCyIm7Zly2zPEmENvHvOPfdcXHnllfrfhYWJTdmyZQtmzpyJr33ta/pvq1atwpYtW/QAftOmTVi4cCFmzJgBAOjt7cWyZcvQ1dWFiRMn+rgVFsXjKFq8GJFLLoFaVhZ0asyNvpwqqjBdTEs9Wm4UAApkfwbhrVxq24HkQteVY3J4GCgMviiP4PgI0cVIBPH5LkxljhP5vv1kTusaeV8A6y7YvDmAtbov+FLfocrKSkyZMmXM96qqoqysLOm3kpISqCl3Tw0NDfo0ZaNBceo0oiiYOBF/C0CdPh37HAbILEjDJdP+zHlfR8Q6WsRKjXPadrhd01gzeXKgTW+MjPtK4KoG8oms566s6SZrynt6gk6CK/jUSgJaswIWKv5gPpOX3G7ONg5AeQ7t8N1KC8+bZG6+xBpBYkRmojDR3gGyO0+upK25TsEAnrIbGUGJouQ06lyY2+CLxm7+BrE/MvXc4Zpjx1wJbN1i1gY+nej+/ThBURDt7LS07ErHqQofUfa328yCDpFHAw3rviD3BNFgMyyBb1huRCiLnHommTQJNUi8LOpGe7UoEgMW+Y0Xk/wybsoUnBB0ItIwXjzMXplvOP10RAGUnnaaDymiXARdrtSDFSSULMzHgBvbFpb8kTaAX7t2LdauXav/PXv2bNxzzz0Bpii8akb/d+ugDyqADxMV5hftnPdTjjV66dYfRI2Ha8G7SX4UjPa4YpWdgZyiWX4XkVcvVuciyPzzqxcakY8Ru326aeeUG9vUrCjYt2MHUFqKqKKg0aXlphI5/6W0fLnnqwjLPpMygL/xxhsxODio/71161Y89thjnq1v/fr1WL9+PQBg2bJlaG9v92xd6dxo+Kyt++Y032Xytyl/Z5vHuPyiDPP9neHzYQC/bW9PmzbjNkRHvzfbhkUZUzZ2nr9L831fX9+YtC40mT81HTdlWV/qPGbTLUrznZlM+9O4nAZFwS8AXJ9xadaOE7N8fhnAdRaWbZw/AvP9of1uthyj1Lxvb29Pu3+zHSPp5jObLnW9qelqAPDFNPPdBPsXgvb2dnwty+/A8bx1Y6h2q+WVlfxKXZ7x+PL6omh1n2ZbBpBcHgUl3TlqdR4jq+WlNr+d9eXCynmaKvVpQi4iAMqnTcN/rFiBv3Vxublycq1zuh6/9nWujHnyP37ykwBTkpnfsR8ALFpkfpREVFG7XrHhjTfewPLly/HUU09h5cqVGBwcxO23367/vnTpUrS2tuL66xNhz5e+9CXceeedereSnZ2dWLx4MR588EFMmjQp6/r27t3rzYaYaFQUvZ3Y3tHeJpoNbXv3WuiBol5R9EB8BMD+LPM0m7QdTl1Xg6Lod4F7ASAWS5s24zb0Ahg0mQ4AWizUAKuA3iOPMQ3astrb23HttdcmzVOtKHotoXH+1HQ0jQ6sk8qYRrP8MU6nbYdxXWYy7c/U/IgjuXZPHf1Oy98jAA5YOE7M8nkvgGaYX/BStw9I9PndZbI/0hkG0JkmT4x5r+Wb8djR1p/pGNHms5IObVuM69WOY52ioMUwzxCA7gzHiRltv1QqCioypBtIztvUpy1agR0x+dtsmdmk5nO69acuL9N5kEnqNlmZPnWfmm13HMefYIzg+HlhTLvZtlpNi3G9xnUYHUXiWBmf5rc4oA9aZCcP05X1qedCf5Z1Ot1ndmU7T9Mx7js3HAJw0GJZYGT3+LSz3H02r3VOZSvHvWQ3/4zHtfE6nW65xmudU2bnbDZWYi23tbS0mP4mZQ18PmtWFKEGLnJSOIhSEyKM4WFbeZJtWivLyvTmf1CFPqXHfUHkTD6fO141GfIag1LrpMyr3bt3o6qqCpWV+df3glbzajeI96stppV0ifwGeInFGhE387No8mQXl2ZNptoHGQt9P7G7xeDIlnduP50oVhRUI1GD2B2L5fy+CoWXrMPxiRwfiEbKvFq+fDleeeWVoJNBKewOKS+i6gDWKV1BqyjAiMlryPG4v2mRiJDH/eBgYn/K35JSOLns7w6T72uRKC+KPVpv2Ah5VKtq4pw7dizolIjj29/29doRlnNEygCexBSGAN5S2gSv9fI6f1sANJm8K1I6caKcj/UycPsGS6TeWppPPBEtAMonTAg6Ka4RuXyxKm7S1tbKtoVh+91iZ+yFdPN5YsIEtABoTjOSfL5qWb0aDRMnBp0M6YTtWksBqgYQtRDcen2BaVIU/BUApLzESu4xu1kTsd/1XLldy2FleV69RJdKW8d4+Ftb6eW2ZVp2oaLoI1v7wet9mO7mkgH8cSLnhchpc0ort5xsm3Ysu/GOV76QNoDft28fNm/eDCDRJl4TiURw+PBh7Nq1S/9ucHAQkUjybu/s7NSn6e3t1eelLBQluYcOg9TeIYISBXAaxj6GFnXvSteEJs+4tX9EPf7ySR38e+zsx3qK0nzH4+w4EfOC5T25RcoAftasWVi3bh3WrVunfzd79mwAQFtbGx599FHccccd+m9FRUW47rrrkqZN7Te+qakJ9fX1Hqc8P4hwB+3lOtJdNHMh4kWGjuMFNzz8PNcK4f2xk+4CzvLkOBHzwu3rRz7xa3A0WUgZwN988824+eb0QxRceOGFuPDCCzPOn48jtob5oPdt20afPmR6gcwJvoiSWer+rfX4HQQ3Bk9ybGQETZMm+X6+RhH8k7Mw8no/pis7wlzWyy4CoCroRFBoSBnAk9xkv8DYDbgjAKJdXYibPOFxOz8iSAzcdcDl5WZTbyOwTt3mwtHu8VKnqUqzTLdvoFJFkeimz/i3UQESQ757YtKkwG7oLL8geeQIUBr8q7iB3mgZOD1/axUFufRDUpJhwBsRlAvwsr+I1xpW2ARPxOPCCR5L5DvZTx4nJ039qaea/uZ2fhSN/vOzQVjB6DqdFij1o/On5kVZmmm9Pn5SazVS1xcF0OBxGkQ2ftq0oJOgy3YsiNr8KQJgHJB2RF6rqiH2S6xORmkR4YaMSBasgc9DuRTwotR6OeHWhS1bkNqsKBi0MY9XF1w/95VbgxulW47fAUnq+txav5XliBpwGpUA6A86ESHi9PiyO18UiR66/DqfRLiRECENZE0EiWsnWccAnmwpBHJ67Ou3SG8vGk45BcNuLtPC7yUuLs/r+UUnagDvRbrcLpAjSARtI0gEcMcA9Jr0IkXhkOkmkI/cSWRhv5a5jecz2eJGDaGfJ2ndKaegAInH1W7xPP0ORsU0O5FZIGYxMjImv4MsFL2oUYki0TypAPZuLElOMjzF8QvLP7JKxjHEWQNPtsh2cfAivV68dGrUMmECjgHocjgaIy9a1jWNvjSa6ZbJz/wc7+O6iEQj2/WFKEgM4MkWBof+8OLErFUUFAHo8WDZstJq2yNpvguCLI9EG31sS62pUhRp8ofEUIBEEzJZ39siyoTlYchEe3tRpCiIHDzoyfLduGi7eeGPdnb6/lKW1zXwmgJFAQZTX4d1rhiJE97PoeSD5NbLgWG7aY12d+e8jAL4f/EoRfj2BXkravhHFDY8rkOm4ZRTUA+gccYMT5Yv2gW08bTTQjU0uiYCoBFAzYknurpMosa5c4NOgiM8fsXFGu7jVDA/wmjgf/yPoJMwBpvQhEy6JgFuEu0imkt6mm0+Wi1E4jF+LidNjaJgHBK9gXRb6A3E60GL3CLacZGLILuR9IMo6bBD5DRbubFXIfY25Crs20d05AtfCDoJYzCAzxNhC0KsyvZyop2+0iNIBNS55MG40fmLANQpCvpyXJ4Tsu1Dcl/56I2kE3xsS0R5JypeyccAnmwRrQ28n+vVAn43aDcD9YCrfdRT7sQrpt1XCf8GECIie2R8ohGHfGm2pUC8PpLy4VoVeo0+jl4my0VfG9XNuN6iHNPjxckS6gJPUqVI9FyhcWsfeVX8O0kfjzsicaS2m5exDb2MNx22RMTbOgbwISDefaFBgEMjp9aYWz39xDtNya5cuzmU7WaNx6x/mNfJZAw2RbMPx5/Eup2ffu0fHgf+YwBPtuTDxctsG4N4MTgf8tttESRuanPOOwcj4lJ+4Hl5XO6dkuY3FQBiMXS//z4+AdDp8vL3u7y8vCVgExq2gQ+pCBIvSVrp6URTP9rDSqYhhVN7RQnDhSzomnk3igXZ9kMUiSZOI/CuFiE1T+w+4m2eMMH0/QTZ8tsrVvIhDnFqirjf3Hcs6ASEhFpejoHR63WXoqDRpeXGYzHfnoSPQPAWAaOclAOqgC+xipciGsPpRacYiba80Y4OS9MX4njtpdtpybaMJkVBQ4DNbazIl645/aA1bwpiQCCrIhCzhsPNl6n98PCKFUI8Xpcpzyi/jcRinp8zXiw/U+Wf9FwcVNEtol47ySVRAHXz5mWfMKW5QMaLnY1AuwBApL8/63RR2A+WCi2kw82LNgMAe1QAe7NMwzwVXy77SITA3S/5tK2aOAB8/HHQySCbVCRqy900DDHOAc/SwJdYKQiWdvKECZYv1NU21t0IoLGtzfoMFm8OIgAabKRDVuIVGfJwoyD3sxZeln3NiwZp4oCQgU0qR00mXE+F//zYhmNI3Awc2LpV+jzLlP7hSZN8S4dVIj4hJpdFkf1EtjOoi902bqnrLwJMm8s021y2n+xcBMS/pPnEx/aXqfJ18DJK5lf3dtnWEcZu9tj+PXdeHhNmtexuBdoqgC4b79n5oR+JcS5SWTn/4jCPb9TJk3NJlidYmZInUg/cIkVBtaIAR46k/d3OspzMY3bnKPIFzunIlZpoT48r6SB5RXt7g04CBWSfoV2zyOWcHW62eXa7SQcBnT4H1yK0gT8Ui2Hv6D/NCJLPP0cKxavvZgCfp+qQGLDmhGnTAMjx5njQcr3oNsyZE5oLt4yCfrwbAVB/yikBp8IbQeet6MJ63osQdHMka3GIEMDnEwbwIdGsKJZHZI3i+AWlZPR/JwF8o6LwALLIq55DZAicWKgfF9YbZRmOQ3KfmwG802PI2IzH6jJ4vLpzPTqc8reXZb3X+6z7W9/yeA3uY/wVEtm6f8wmtX/3TAoA4OjRnIORsNZKmfFie2UIjjsEayNJ7mNA5D4Z8tTN2m+nNwMRHM+rPmTPNxVylJtW0phpGtXks1tUAJ+klO1evhORug2u96Jzxx0uL9F7DOAJQOLFUquiAMa1tnqVFCIiUzIEX2asBFLa9nVC/G3VAjY3AkSnNwMRAF0APgFwZOdOF1Iihs6tW8fUcFt1GMkjsGr547WjPqwDAA7B2fEiw02xHQzgCYD92uHxnqRCbCI+Mci1QLI7f9gKQLeY5Usc2YMws3mHnCfHFjf2qdNlpJsvW82aX0FtHP63r1aRCLYAfwbzyUaFvZuOTEayLCuXGvjhWCwximmxnWfJCX7lse2ytrISfQ6fXvbFYokRWEfp+eMxp3k5guzHUHdBAVQkzsmDDs+NnpaWwM8pNzGAl4DXgWOTg27+7NTYU3CyXYDzKYBPTbsf29IRi6Ejw8Um0/7pjsV8CVatBmlWlpPrfCqA/VkCDRX20uy0/OyIxbL24uH2/jkGJAVeQVOR6L0jGys1r/tjsYxNLJw2v7Dbr1PqPvPzhtBt6c6BQMvoM87QP9pJx/4s5SQADO/ejX2Gc9LJdh577bWkG0VtGdn2zXBtrYO1eY8BvATSXYDc7OPayUEgYm2012TZ5n0uL89YUOZ6EfLy4hLPsvx0v7mdV27bv3Wr8GkEEnnr1+NzANj/l79IkS+prLTP9nL5uRpz4zQaTGUfazs7u8ePCuDfAMRTRoLNVE7HAezfvt3mmsby6+lYNqlNbFQAHTt2ZJ3Pi5FYAQDPPJPT7B3vvWd5WjvXonRBu/Z537/9Gzo++CCxzIL0b/Z1vvaajbX5hwF8npMlKCXrVJdr8I7i+ItfTnuyPwr/2/NauUC5nVduUysrhU8jkDg+jjiYz/gColVxAHFF0YPHoI3AelMbrwNsJ/vADWpKEG2FsTxw3APNihX2R4ItK3O4tsR+HgHQs3mz42WkcnIzoAXg/SnfHQKglpZmXFccQDcS25CtWVO2NLhNHW+9ca6V8l0bJbbn3XfTrw+AesUVQEmiP76ejRsxkm7Z43IdBcYbDODzHAN4eaWrcfaiVqU3FsO+0UecIxmCpkwFes/o/F5KfUkrW1MMco/TAN5Mpps9t9qluxWA7E9papOueY92XvZmWW+uN7lO20znzG4QjTS9U7mU9kxBaa5NTrqQ2N9qVZXdZJnqdrDd+2KxRPlmmPcIgP4sy3p0xQq9HFerqiyXkXZq7N1qkpfNoeXLs07TNZpPanW1/l3G86+mBvtjMRxauNCFFHqPAbwEZG53HGa5XGy1Wm2Nk32c2mZwGMEFrVpbWWN6/Lw5jAM44vG27/VgmX6f2+mCX+MF1+yY7ELmtA4DjgIwsy79OjK0/zc2G8gl/8y2O1Wm89zsGE89F7Tzcnj0ZthOgCkLK3lpNo+bT+f2Z8njVGb7MF2actk/x3KcPxuvlq2V7VavLfts5n+69Vny1a86WofM51gqBvBESJzUXvZhmyr14nDQpeXY5VabXG26rjTfmbHT5lXrfSDdOr3g17K7TKeyxu7+NwtWOpGoYetOmfYoEufFcJZ2tU7bvw8C6DJ5vG3mmMlnq7TeZbqQqBHXPg/Dfn46PU7MzncRRjZ1uk0DsP90pH90nq633nK4VnO5nludb7yBY0gco7kYRuI4PbBtG7rg3T5288lUkKPbet3c0tLxbdIWXjQM4CUgygszYXfIx3WlFlADDnscMXs5x6puHK8FTX0qAKTv4SHderS28cNZammMAWRPlprI1Atdao8g6fJrCJkvQMZ+qwfSrFPThzTb6UENf6YmSUbDSF9b7UYwEEei55P9sRiOpeyTnlgMXbFYxna1wPEgx+4xPAIkPd62xNg9ns31AYljvnO0GcHR0eYvw6P/2z2HjO+EaPvIys2MWZd+bgROdpo6uFnLfNBCrz2pDo3OE29oSPt7Ls0x0pVFVpc1DEBtakJXLIbe0bJZBYCPPrKdjk7tHKqowLDWpMP2UsxpaRuw8PKqVXb3o5t6YzHsX7AgqbcpN296hqMWwt646CMwJBQGnQDKricWQ4uDrh7JniMAbIYSjo25UMfj6IjF0KwotpqeZLvgR5Ao4M2WORKLYR+A9vZ2XHvttQCgH2sqgKNpCvJhjO1G9FhKG+BsLydqv/UCSNdB1z4AyHDcx5EYqKQl5fueLBeeLpOLegSJi4TxwnVEUcYs37h+L2s/UvNOS1ejIT/MArBMjgJIfYUv15pKAHpQrbVrTt1vWh6nynRhthLkHMTY7clkGJlvMq2sc6/J/OluMAssLlPjRmXNfgvliLF7SD+vLT0Amm3O02Eyj5OgLt2+SD0nVIzdlyKPJp1r2rTjVCTqj36EfT/6kSfLPnjuuajYsCGxHk/W4B/WwIeY7AenXanb63WTmFzyd8xjWYsvgY2pubeQns433/S1B5hOJLav8+23x/yWmsaju3Ylek5IndBCX+Bu1Yh3IXHz1rVli600pNbgZ2L3WLHy2L4L9o/xT1Jq6T4BgD17bC7Fvs7XX0+7TZkCVit5Ft+1S9hyrmvTpsR5YGMeO90xDsL+caWOzmcnTW5SP/oI/bB306h+9NGYYP0IgK6tW22vP1052Ldjh61zOWw633037QvouTavTPe7EPXav/iF/tH0qmulll4AcqSSHHHzsVPQF0krj1KHU6ax/TjaELBZybsujB1p02o+jSnI0gTwuTzeNk4Xb2x0df9lW9aI9ti5rm7Mb9o+OqB9UVSEgy4E4rm8MDsci+FALAb1hBNszfeJjXT3wFrbUu146rPQH/JwLGb73YnUpjADsVjOFysrF+V4c3NyU4RRmW5AutMse8x5WZR+SDkRXhKN19SgN0vPTalSW4JnSm9vLOaoxt5umlxVWIj+0aZaduZJDfgPxGJQKyttrz7t8VZaautcThX0tTFXanU1DqTZfu2lW7Obm+4slU5a80ytW8Y4gK6ZM3NJ6hie3XixCQ0FTevz1f7g0mPtc9C8w+yxuRNaAF+Q5jst/OjasQMN06bpB7XWnZiVNGiBgTZ9uuYZqeKxGDoARBQFzbDf+4L26NKsqBiEheYBsRgw+gjcr15fDgBodDhvuiYsqezmYwT+vXSlra8/5e8jSOwrs+Pt2GhTJQBoztBkYT+Qsd/3TwDUwJBHhv2fLc1eXZLsPMLviMXQqChjHtlrTZKMAbp2fhmbdFnpCUPF8f2R7rdMugE0pUxn3J9eB2t2a8Z7TMrlXJpFpCt3tP3jddMxM3HDcZ7LPnDaWUA6xqZ3qYHUMSSuu25XnHh5HhvXA2Tu3lIFMJKl7/8RQ5mXxMUmWwdjMVTksDzZu9FmDbzkMhUQcQDdO3b42ltH6t8dDpfpZL7UN8et9owRB9A5Whjth/3ROdU9e9AxOm/S95lm+vhjdL7/PjqQPHKecZ6ganay1rB/9BE6LEznlJV9r617/+uvYx+s3Ri48USqA8AbAAZGX2bbP/pd34cfogPJTUKc1AJny9PB0by3c37ER6e3OgKlmvK/H/a/9x46AHQ6eBHPmM4hIOOxmW2b4rt3Z8xfR+WSnbSsWOHKsju3b7cd6A1j9FjetWvMb/u3bUscQx5fTzzn4pMH7ThJlx/do+VBR2qzvBR289LOeUzZyVHPbo4BvGTsnPBRADDpPcLJgZuulkfrIstM3EGBedTGfElpKk5+1nAUid5ErC0osaR4LJaoATVZv/YoMGmbo1HEYzF72xqJQC0vT8xj2Edas4F0PcLk0qY/dd+pGZY3iOOj9aVVWOhov1plZ+TReHOzpelVAL1ZLqZWqLEY/rxiBVCYqHPT9/u4cWPyxOzxrtb0ytELi6N5b9xmbXRIPY0Ye7MSj8Wgjo5AaaUWWjsGPTe6Her48WPOBSdUJPaR4yY0BQVj8jdpfheP+4NIPxibGTujZqplZfpxZqfZXTwWS9ssSa2ocLR/RAr23e6+Uc1U5o+WB2bN8rRzzM7gZ9r+UXMYSVYEx2DvuLTDapmlpaH7tdc8SIV/2IRGMtpBb+XRT6bHhak9nhhr3azc1cWR/OjcSU8GZs0Nes0unil/Hx2d3/ioMnWaI7EYqh0+YkvNa6uP8DWDAEpgPU+BRLtmrZav3JDuOBK1zI1ptkVLZ8+YX8zty7Ad6dpDpnMUwDi4UxAPACi3sCz98a7N5auA7TbuuTJr2z9i2Mdu0HrMSG1qYnZOZmtWNmwhfW42j/OC2XHkJIizemzaXm4shgEAtYoCKwO174/F0DDa/MhYAWN2HmrHWaYmW6LResxy69gawfGmWV51jegkrcdcLgNkoj0tdfu4VJG9F7LUNFhdrqgYwEvOrPA4AvNa7HQHpArzLuDSydTuNfXirv1t/H6vybrs1JyZBfo6rXYPzgrZXLtZO5DjDY4x2MhUW5spGPdS1vy34aDFFzLtbqtb7Uatzp9zYe/Bvkx9P0BrxyzyhSlXZvvLybsSVo9Npw4DlgJ4IH0Q6tZ5KMJ4Ix2G4C5bV7RWBDUyNWVnjAtEc/gzn8EJ//7vAIK7vlrBJjQSshKMml2oRpD7m9tWHsGP4HhXYX2w9/hXczjlb6u9V6RLTy7tjt0uYOw85vMqDfniCBLnQvc77/iyPic3Cl7t20Ektr03pUebrk2bMAx7j+9TiXo8ZqthTy1TRHDkww9xDOkDaLfzOV33jUeROE4OujgQUK76kXwNAdzNC5GfHuULbVTavoDTkY568cU4VFaGoaamoJOSEWvgQ8qsH2mrvTgYm41YKTiHkej14CgSjweNL3QeicVwBECTzVrovlgMZYYeVnoBaOP1mTUXMesB5LCioAzJj2i17czW9GQYiRPFyk2D9rjdLJCz85gvqUcOS3NQqj6fa08+2b4dpdOn2wo2vBpIxbQpWk1Nzs0JegDUQbzjUrs5Np47Iziev0LeeIwbd/yRfnt70k89SPT4lEu6jeV5ukGs0pVHWjmZrmzsQqIc9jIvD8ViY0bG1vaj1+N7kD9GUuIEoRQW4okHHtAHNxQVA/iQSq29Tfeoyixw2IfjXSh2wtpw77kGBGZp1L4/hgzdUhkMADghzXI+icUSg9Y4YKe9nFeP27XtGYT1tuLG/BzG8e5EhQxiBGXsKs7S9GVlth+5dsHQZWk8LsUgIseynIt+dXmnrUujPVUw7q+kLmEFfhyeTtxCmeeFTE0krZTDGiej0Zrxqg17rgYBVASdCMpL4l8pKEm2bht1hsKuE4maqdQ+hrs2bsy6vtTeLdzWD+AQEmkbwtg0diNRQPZa7DprYOdOHEaitj5MtP3wyQcf4Aiyv8TZheT87HnvPQyOfmdnFMSgBX2zoR1/3Q5GfbRs9GVGpzeYIurCaJmzaVPQSQGQKGP6AUAN+ojyX5Bb3LVpE45CvDLHzTw5OFom55P8O4vExBp4SWhv03fDfBAds0E24rFY2keh8fr6pOVmcwxAEZy3ndUevRpr5foNNxrp0nAsFrMXjBcXoy8WQ3t7O8R++GWPfiNVUmKpl5iRWCwpP9Xx41196dQvQ0g8OTBrEuY128efQ7mMBCmiYZMyx3Oj+ViQ8p2XL6GKrguJa4ZfzZ26kWheNYLEaLRWmwz6qRtjr0WOjZbJhYqCQmBM058w0ZrK2u0FjLzBAF4Sq1asON4ey2REOid3xWnbxJuM7phpZDYr7Dx6JSTth3yt8RDx4k+5GTOarAe0G758PW+M/G6Gk615lQi8uBbZaWopIq3ZW6YKOlGbMeUrNqGRkB8XJW2ESWQZLpn84cugOhQKXpUPbi138KOPsB9jRy52Km2PIh9/nHZ0ZCJZef0ERbvmf5JmJF6R5fNNOmvgJTQAoGz0f69enrHy4ip5T286I1mhSv47hsTjba/aG3cBqIezvtSTFBa6Wr6kvYBHIp6OFkzkl2GMNnN96y1P1yPr+aLFQ2FuumSGAbyEjD2dlBpG5tPa9ZF4tHcN7AY/HIiErPL6Eb7bI8hapXVpaNZ9YA8SNxZEudCOMxEGtDJis5XMvB5oTWQM4CXX+fHHQOT4Q2TR2x7mKwbiRM5k6tIQSLwwy3KPcpXtOCMSDdvAyy7CMeWIiIiI8gkDeCIiIiIiiTCAJyIiIiKSSF62gV+zZg1Wr14N1TAqX2lpKZYsWYLW1tYAU0ZERERElFleBvCbN2/GOeecgyuvvFL/7v7778f777/PAJ6IiIiIhJaXAbyqqqisrMSUKVP07woL8zIriIiIiEgybANPRERERCQRBvBERERERBJhAE9EREREJBEG8EREREREEmEAT0REREQkkYhq7Aw9TyxduhStra24/vrr9e8WLVqEq666CpdeeumY6devX4/169cDAJYtW4aVK1f6llZNX18fqqqqfF+vrJhf9jC/rGNe2cP8so55ZQ/zyx7ml3Wi5NWiRYtMf2PfiRZcdNFFuOiii/S/r732Wt/T0N7eHsh6ZcX8sof5ZR3zyh7ml3XMK3uYX/Ywv6yTIa9YAz8qUw08EREREZEo2AZeEnfddVfQSZAK88se5pd1zCt7mF/WMa/sYX7Zw/yyToa8YgBPRERERCQRBvBERERERBLJy5dYI5EIDh48iF27dunfDQ8PB5cgC4wv0VJ2zC97mF/WMa/sYX5Zx7yyh/llD/PLOhnyKi9fYl29ejXWrFkD46aXlpbinnvuwYknnhhgyoiIiIiIMsvLAJ6IiIiISFZ52YRGFmvWrMHq1avHPClYsmQJWltbA0xZcFatWoXf//73Sd9deOGFuPHGG/GlL31pzPRnn302br31Vn3eF198MSk/q6qqcP/996O2ttbbhPts6dKl2LJlS9J3NTU1WLVqFTZs2IBHHnkEIyMj+m9FRUW47bbbMG/ePADAfffdh02bNiXN39zcjOXLl6OkpMT7DfDZ5s2bce+99475ftasWZgxYwbWrFkz5ref/OQnaGpqwu7du7FkyRIMDAzov0UiEVx99dWYP3++p+n22zPPPIMdO3bg9ttvz1o+DQ8P484778SePXuSlnHyySfjvvvuAwC88847eOCBBzA0NKT/XlBQgIULF+Liiy/2Z6M8ZMyvTGUXgLwvv4x5lan8AsAybJSWZ5deeqlp+bVkyRI89dRTeVmGZTrnsh0fVvIk6BiNAbzANm/ejHPOOQdXXnml/t3999+P999/P28D+E2bNuGaa67BWWedBQB45ZVX8PLLL+u/L1y4EDNmzND/rqioSJr36quv1ucFgDvuuAMfffRRKC6Aqc4999ykY6ewMHG6b9myBTNnzsTXvvY1/bdVq1Zhy5Yt+sVv06ZNSXnZ29uLZcuWoaurCxMnTvRxK/z1wx/+EMXFxQCAlStXQlEUAMCECRP0QEpTV1cHAPjwww8BAA888ID+2/PPPz/m4hAGu3fv1j9nK58OHDiAPXv24LbbbkNTUxMA4IMPPsAjjzyCwcFBlJSUYNu2bWhoaEjK21/+8pd49913QxHAG/MrW9kF5Hf5ZcwrwLz8AliGaVLzzKz8AvKzDMt0zmU7PqzkSdAxGgN4gamqisrKSkyZMkX/zliI5auamho9T957772k3xoaGpLyK9O8mrC2Iks9djSqqqKsrCzpt5KSkjH5YMzLsrIyfd4wmzRpEkpKShCPx9HR0YErr7wS+/fvR3FxccbjKhqNJv1eVVUVyrzas2cPGhsbAWQvn7Ttb2lpwaRJkwAgqTZLmyY1bysqKjA4OOjRFvjLmF9A5rILyO/yKzWvzMovgGWYJjXP0pVfmnwtw6zGC+mOj2x5EnSMxm4kiSjv1dbW4sorr9QL3927d+Po0aOYPn16wCkTRzweRywWCzoZ0mB+Wce8ss+YZyy/8hOrc0kqLS0tSY+VgcRdMlEumpqacP311+t/b9++HeXl5Whubg4wVWLp6uoKOglSSc0vll3meGzZZ8wzll/phf2cYwBPUrn77ruT/t6xY0fGx4JETmzfvh3Tpk1DJBIJOinCaGxsxFlnnRWa5i1eS80vll3meGzZlynPWH4lhP2cYwBP0urp6cHLL7+M7373u/p3y5cv1z9HIhH89V//ddreHfLB2rVrsXbtWv3v2bNn45577gkwRfLYsWMHPv3pT+t/f/jhh0nHUXV1NZYvX46qqqoAUkeyS1d2ASy/jFh+OZdafgEsw8zOOZkxgCdp/eY3v8HEiRMxZ84cAMlviwOJC8DWrVuDSFrgbrzxxqSama1bt+Kxxx4LMEXyOHz4MGKxmN5+9OKLL8aZZ56p/z40NITvf//72Lt3b95c/MhdqWUXwPLLiOWXc6nlF8AyDEh/zsmOATxJaWBgAC+88AK+8Y1v6N+lPhqrrKxEd3e3zykTg9Ztn6anpyeglMjngw8+gKqqmDZtGoBEzwPGi5wWWIShhwbyX7qyC2D5ZcTyy7nU8gtgGWZ2zskuPK35Ka/87ne/Q1lZWVKfyERu2L59OxobG1FZWRl0UiiEWHaRl1h+jRXWc44BPEnn2LFjWLduHS677DK926zh4WF88MEHSSM65rPdu3fj4MGDQSdDSjt27Eiqvers7MT+/fsDTBGFRbqyC2D5lYrll3Op5ReQ32WY2TkXBgzgSTp/+MMfMDg4iIsuukj/rre3F9/97nfR0dERYMrEsXz5crzyyitBJ0NK27dvT2o/unr1avzHf/xHgCmisEhXdgEsv1Kx/HIutfwC8rsMMzvnwoABPElFVVU899xz+NznPqePnEbklq6uLnzyySdjarCIcsWyi7zG8itZ2M85BvAklddffx0dHR244oorgk4KhdD27dtRWFiIqVOnBp0UChmWXeQ1ll/Jwn7OhatBUMhEIhEcPHgQu3bt0r8bHh4OLkECePbZZ9HW1ob+/n709/cDAAoLCzFu3DgAiTfwte97enrGDGTR29ublJ8AQjvYxb59+7B582YAiTalmkgkgsOHDyflw+Dg4Jh86Ozs1Kfp7e3V5w2zHTt2YPLkySgqKtK/085DLS+1dsrGvIjH40n52dfXF/q8ylY+adu/d+9exONxABjTRCQSiWBoaChpGYcOHUrK/7AwK7smTJig5xXLr+PMyi+AZZiZdOUXkL9lWKZzDsh+fGTLk6BjNAbwAps5cybWrFmDl156Sf+utLR0TPu2fNHb24tt27YBAO644w79+/r6evzLv/wLFEXBww8/rH8fiURw3XXX6X/PmjULTz/9NJ5++mn9uxNOOAGTJk3yPvE+mzVrFtatW4d169bp382ePRsA0NbWhkcffTQpD4uKipLyavbs2WP6XW5qakJ9fb3HKQ9WuvajbW1t+MMf/oB7771X/666uhqKogAAWltboapqUn5GIhF84Qtf8CfRAclWPml59NBDDyXNd9JJJ6GkpET//PTTTyflXUFBARYsWOD9BvgoU9m1cuVKPa9YfiVkKr8AlmFm0pVfQH6WYdnOuWzHh5U8CTpGi6j50hEoEREREVEIsA08EREREZFEGMATEREREUmEATwRERERkUQYwBMRERERSYQBPBERERGRRBjAExERERFJhAE8EREREZFEOJATEVEeeOqpp7BmzZox399www1CDjX+pS99CZdffnnoBnUiInIDA3giojzyla98BcXFxfrfM2fODDA1RETkBAN4IqI8cvHFF6O8vNzRvKwVJyISA9vAExERERFJhDXwRERkyWOPPYaioqKgk0FElPcYwBMRkSVOm94QEZG72ISGiIiIiEgirIEnIspznZ2dWLx4MRRFQUlJCQ4cOID+/n6UlJTg+9//PqZOnQpg7EusP//5z7Fu3TqMjIzoy6qpqcG9996LxsZGxONx/OAHP8D27duT1ldZWYkf/OAHmDx5MgCgo6MDP/3pT7F9+/akZRk98cQTWLduHeLxuP5deXk57rzzTrS1tQEAHnroIbz66qtQVVWfpqWlBf/rf/0vlJWV5Z5RRESCYABPRJRHuru7MTAwoP9dV1enf25padG7lTx06BB+9atfYevWrXoAn+q1115DS0sLLrzwQgBAb28vnnvuOWzbtg2NjY3o7e3F9u3b8alPfWrMcrds2aIH8I888gh27tyJq6++GuPHjweQCNiNXn/9dSiKoq9raGgITz75JN599109gH/11VfR1taGM888EwCwe/duvPjii9i9e7c+DRFRGDCAJyLKI9/5zneS/n7sscf0zzNnztQHders7MSvfvWrpNrsVPF4HI2Njfo8u3btwnPPPafXkmv/p1uusSZ927ZtOOecc/DlL39Z/y41gE9d18DAAJ588smk5aiqiqlTp+rT/OUvf8GLL75oWqtPRCQrBvBERHnklltuwbhx4/S/x40bl1Qjb0dxcTEOHDiA4eFhFBYWori4GK2trXotulXDw8MoLS11lIbU9HR1dUFVVUQiEVRUVKC1tdWVZRMRiYQvsRIR5ZHTTz8dZ555pv6vsNB5Pc5ll12GDz74AD/60Y9w6NAhtLS0YNmyZZg3b56LKT5uaGgInZ2d6OzsRHd395jfL7/8crz22mv4l3/5FwwNDWHmzJlYtmwZWltbPUkPEVFQWANPRESOXHzxxSgpKcGqVavw/e9/H3fddReampo8W98777yDxYsXJ31XUlKif54/fz7Kysrw5JNPoru7G3fccQcqKys9Sw8RUVAYwBMRkWPnnnsu6urq8OCDD+Luu+/Gd77zHc9eGJ02bRquueYa/e9IJIJTTz01aZprrrkGjY2NWLFiBb73ve/hrrvuwoQJEzxJDxFRUNiEhoiIHOnr60N/fz9mzpyJH/7whygvL8cPf/hDdHR0eLK+qqqqpOY/n/rUp5JGhu3p6cHhw4fxmc98BkuWLMHRo0dx7733Om7jT0QkKgbwRETkyN13342HH34YQKILyltvvRVDQ0N4//33PV/30NAQHn/8cWzcuFH/7qabbsJTTz0FADjppJOwcOFCfPLJJ9izZ4/n6SEi8hMDeCIiGuPIkSMAkFTDnY32Qqyxa0crCgoK9PVZdezYMfzXf/1XxpsFLT3sRpKIwoZt4ImI8sjzzz+P4uJi/e/p06ejqqoKALBlyxYAiQD81VdfRSQSwUknnWS6rGg0iq6uLjz//PMAEgM5ad8b/9eWCyQGcjL+BgBTpkzBK6+8gpqaGlRUVJiuq7e3Fy+99BKARA186nIikQh2796tp+ejjz4aMw0RURgwgCciyiNPPvlk0t/XXXcdzj//fACJkVVfe+01AIn25gsXLsSUKVNMl3XGGWdg3bp1+NnPfqZ/V11drQf9NTU1mD59etJyAaCyslIfmRUA/uZv/gYrVqzAs88+i5qaGtTX15uua8WKFfp35eXlmD17tv73mWeeib/85S9499139e+am5sxadKkTFlCRCSdiJppmD0iIgq9zs5OLF68GDfccIM+iikREYmLzxWJiIiIiCTCAJ6IiIiISCIM4ImIiIiIJMIAnoiIiIhIInyJlYiIiIhIIqyBJyIiIiKSCAN4IiIiIiKJMIAnIiIiIpIIA3giIiIiIokwgCciIiIikggDeCIiIiIiifx/pJnhRn69fHwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    \n",
    "    env = gym.make('MsPacman-ram-v0')\n",
    "    env.reset() \n",
    "    action_size = env.action_space.n\n",
    "    state_size = env.observation_space.shape[0]\n",
    "    agent = Agent(epsilon_decay, epsilon_min, \n",
    "                  gamma, learning_rate, state_size, \n",
    "                  action_size, batch_size, training_threshold)\n",
    "\n",
    "    scores = []\n",
    "    eps_plot = []\n",
    "    episodes = []\n",
    "    rewards_lst = []\n",
    "\n",
    "    for episode in range(num_episodes):\n",
    "        done = False\n",
    "        score = 0\n",
    "        state = env.reset()\n",
    "        state = np.reshape(state, [1, state_size])                                      \n",
    "        lives = 3\n",
    "        while not done:\n",
    "            dead = False\n",
    "            while not dead:\n",
    "                #env.render()\n",
    "                action = agent.select_action(state)\n",
    "                \n",
    "                next_state, reward, done, info = env.step(action)                       \n",
    "                next_state = np.reshape(next_state, [1, state_size])                    \n",
    "                \n",
    "                agent.update_replay_memory(state, action, reward, next_state, done)\n",
    "                agent.train_agent()\n",
    "                \n",
    "                state = next_state\n",
    "                score += reward\n",
    "                dead = info['ale.lives']<lives\n",
    "                lives = info['ale.lives']\n",
    "                \n",
    "                if dead == True:\n",
    "                    reward = -100\n",
    "                else:\n",
    "                    if reward ==0:\n",
    "                        reward = agent.movement_penalty\n",
    "                    else:\n",
    "                        reward = reward\n",
    "                        \n",
    "            if done:\n",
    "                scores.append(score)\n",
    "                episodes.append(episode)\n",
    "                pylab.plot(episodes, scores, 'red', linewidth=2)\n",
    "                plt.xlabel(\"Episodes\",size=24)\n",
    "                plt.ylabel(\"Scores\",size=24)\n",
    "                plt.xticks(size=20)\n",
    "                plt.yticks(size=20)\n",
    "                pylab.title(\"Performance Overview\",size=28,fontweight=\"bold\")\n",
    "                plt.grid(True,color=\"gray\")\n",
    "                pylab.savefig(\"final_version_v2.png\")\n",
    "                \n",
    "                print(\"Episode:\", episode, \"-----Score:\", score,\"-----Epsilon:\", agent.epsilon)\n",
    "\n",
    "        if (episode % 50 == 0) & (load_model==False):\n",
    "            agent.model.save_weights(\"../Deep Q-Learning - PACMAN/final_version_v2.h5\")\n",
    "            print(\"Saved model to disk\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}