td3withtau.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "td3withtau.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPy/vODWhRXFnIkwHUNOvhl",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abhisheksuran/Reinforcement_Learning/blob/master/td3withtau.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iffzAWwBELNL",
        "outputId": "07816cb3-31f4-4ed0-8817-09e8c7b8bd5e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import gym\n",
        "from tensorflow.keras.models import load_model\n",
        "!pip3 install box2d-py\n",
        "\n",
        "print(tf.config.list_physical_devices('GPU'))\n",
        "\n",
        "env= gym.make(\"LunarLanderContinuous-v2\")\n",
        "state_low = env.observation_space.low\n",
        "state_high = env.observation_space.high\n",
        "action_low = env.action_space.low \n",
        "action_high = env.action_space.high\n",
        "print(state_low)\n",
        "print(state_high)\n",
        "print(action_low)\n",
        "print(action_high)\n",
        "\n",
        "\n",
        "class RBuffer():\n",
        "  def __init__(self, maxsize, statedim, naction):\n",
        "    self.cnt = 0\n",
        "    self.maxsize = maxsize\n",
        "    self.state_memory = np.zeros((maxsize, *statedim), dtype=np.float32)\n",
        "    self.action_memory = np.zeros((maxsize, naction), dtype=np.float32)\n",
        "    self.reward_memory = np.zeros((maxsize,), dtype=np.float32)\n",
        "    self.next_state_memory = np.zeros((maxsize, *statedim), dtype=np.float32)\n",
        "    self.done_memory = np.zeros((maxsize,), dtype= np.bool)\n",
        "\n",
        "  def storexp(self, state, next_state, action, done, reward):\n",
        "    index = self.cnt % self.maxsize\n",
        "    self.state_memory[index] = state\n",
        "    self.action_memory[index] = action\n",
        "    self.reward_memory[index] = reward\n",
        "    self.next_state_memory[index] = next_state\n",
        "    self.done_memory[index] = 1- int(done)\n",
        "    self.cnt += 1\n",
        "\n",
        "  def sample(self, batch_size):\n",
        "    max_mem = min(self.cnt, self.maxsize)\n",
        "    batch = np.random.choice(max_mem, batch_size, replace= False)  \n",
        "    states = self.state_memory[batch]\n",
        "    next_states = self.next_state_memory[batch]\n",
        "    rewards = self.reward_memory[batch]\n",
        "    actions = self.action_memory[batch]\n",
        "    dones = self.done_memory[batch]\n",
        "    return states, next_states, rewards, actions, dones\n",
        "\n",
        "\n",
        "class Critic(tf.keras.Model):\n",
        "  def __init__(self):\n",
        "    super(Critic, self).__init__()\n",
        "    self.f1 = tf.keras.layers.Dense(512, activation='relu')\n",
        "    self.f2 = tf.keras.layers.Dense(512, activation='relu')\n",
        "    self.v =  tf.keras.layers.Dense(1, activation=None)\n",
        "\n",
        "  def call(self, inputstate, action):\n",
        "    x = self.f1(tf.concat([inputstate, action], axis=1))\n",
        "    x = self.f2(x)\n",
        "    x = self.v(x)\n",
        "    return x\n",
        "\n",
        "\n",
        "class Actor(tf.keras.Model):\n",
        "  def __init__(self, no_action):\n",
        "    super(Actor, self).__init__()    \n",
        "    self.f1 = tf.keras.layers.Dense(512, activation='relu')\n",
        "    self.f2 = tf.keras.layers.Dense(512, activation='relu')\n",
        "    self.mu =  tf.keras.layers.Dense(no_action, activation='tanh')\n",
        "\n",
        "  def call(self, state):\n",
        "    x = self.f1(state)\n",
        "    x = self.f2(x)\n",
        "    x = self.mu(x)  \n",
        "    return x\n",
        "\n",
        " \n",
        "\n",
        "class Agent():\n",
        "  def __init__(self, n_action= len(env.action_space.high)):\n",
        "    self.actor_main = Actor(n_action)\n",
        "    self.actor_target = Actor(n_action)\n",
        "    self.critic_main = Critic()\n",
        "    self.critic_main2 = Critic()\n",
        "    self.critic_target = Critic()\n",
        "    self.critic_target2 = Critic()\n",
        "    self.batch_size = 64\n",
        "    self.n_actions = len(env.action_space.high)\n",
        "    self.a_opt = tf.keras.optimizers.Adam(0.001)\n",
        "    # self.actor_target = tf.keras.optimizers.Adam(.001)\n",
        "    self.c_opt1 = tf.keras.optimizers.Adam(0.002)\n",
        "    self.c_opt2 = tf.keras.optimizers.Adam(0.002)\n",
        "    # self.critic_target = tf.keras.optimizers.Adam(.002)\n",
        "    self.memory = RBuffer(1_00_000, env.observation_space.shape, len(env.action_space.high))\n",
        "    self.trainstep = 0\n",
        "    #self.replace = 5\n",
        "    self.gamma = 0.99\n",
        "    self.min_action = env.action_space.low[0]\n",
        "    self.max_action = env.action_space.high[0]\n",
        "    self.actor_update_steps = 2\n",
        "    self.warmup = 200\n",
        "    self.actor_target.compile(optimizer=self.a_opt)\n",
        "    self.critic_target.compile(optimizer=self.c_opt1)\n",
        "    self.critic_target2.compile(optimizer=self.c_opt2)\n",
        "    self.tau = 0.005\n",
        "    \n",
        "\n",
        "  def act(self, state, evaluate=False):\n",
        "      if self.trainstep > self.warmup:\n",
        "            evaluate = True\n",
        "      state = tf.convert_to_tensor([state], dtype=tf.float32)\n",
        "      actions = self.actor_main(state)\n",
        "      if not evaluate:\n",
        "          actions += tf.random.normal(shape=[self.n_actions], mean=0.0, stddev=0.1)\n",
        "\n",
        "      actions = self.max_action * (tf.clip_by_value(actions, self.min_action, self.max_action))\n",
        "      #print(actions)\n",
        "      return actions[0]\n",
        "\n",
        "\n",
        "  def savexp(self,state, next_state, action, done, reward):\n",
        "        self.memory.storexp(state, next_state, action, done, reward)\n",
        "\n",
        "  def update_target(self, tau=None):\n",
        "  \n",
        "    if tau is None:\n",
        "        tau = self.tau\n",
        "\n",
        "    weights1 = []\n",
        "    targets1 = self.actor_target.weights\n",
        "    for i, weight in enumerate(self.actor_main.weights):\n",
        "        weights1.append(weight * tau + targets1[i]*(1-tau))\n",
        "    self.actor_target.set_weights(weights1)\n",
        "\n",
        "    weights2 = []\n",
        "    targets2 = self.critic_target.weights\n",
        "    for i, weight in enumerate(self.critic_main.weights):\n",
        "        weights2.append(weight * tau + targets2[i]*(1-tau))\n",
        "    self.critic_target.set_weights(weights2)\n",
        "\n",
        "\n",
        "    weights3 = []\n",
        "    targets3 = self.critic_target2.weights\n",
        "    for i, weight in enumerate(self.critic_main2.weights):\n",
        "        weights3.append(weight * tau + targets3[i]*(1-tau))\n",
        "    self.critic_target2.set_weights(weights3)\n",
        "\n",
        "  \n",
        "  def train(self):\n",
        "      if self.memory.cnt < self.batch_size:\n",
        "        return \n",
        "\n",
        "\n",
        "      states, next_states, rewards, actions, dones = self.memory.sample(self.batch_size)\n",
        "  \n",
        "      states = tf.convert_to_tensor(states, dtype= tf.float32)\n",
        "      next_states = tf.convert_to_tensor(next_states, dtype= tf.float32)\n",
        "      rewards = tf.convert_to_tensor(rewards, dtype= tf.float32)\n",
        "      actions = tf.convert_to_tensor(actions, dtype= tf.float32)\n",
        "      #dones = tf.convert_to_tensor(dones, dtype= tf.bool)\n",
        "\n",
        "      with tf.GradientTape() as tape1, tf.GradientTape() as tape2:\n",
        "            \n",
        "          target_actions = self.actor_target(next_states)\n",
        "          target_actions += tf.clip_by_value(tf.random.normal(shape=[*np.shape(target_actions)], mean=0.0, stddev=0.2), -0.5, 0.5)\n",
        "          target_actions = self.max_action * (tf.clip_by_value(target_actions, self.min_action, self.max_action))\n",
        "          \n",
        "          \n",
        "          target_next_state_values = tf.squeeze(self.critic_target(next_states, target_actions), 1)\n",
        "          target_next_state_values2 = tf.squeeze(self.critic_target2(next_states, target_actions), 1)\n",
        "          \n",
        "          critic_value = tf.squeeze(self.critic_main(states, actions), 1)\n",
        "          critic_value2 = tf.squeeze(self.critic_main2(states, actions), 1)\n",
        "          \n",
        "          next_state_target_value = tf.math.minimum(target_next_state_values, target_next_state_values2)\n",
        "          \n",
        "          target_values = rewards + self.gamma * next_state_target_value * dones\n",
        "          critic_loss1 = tf.keras.losses.MSE(target_values, critic_value)\n",
        "          critic_loss2 = tf.keras.losses.MSE(target_values, critic_value2)\n",
        "          \n",
        "\n",
        "\n",
        "      \n",
        "      grads1 = tape1.gradient(critic_loss1, self.critic_main.trainable_variables)\n",
        "      grads2 = tape2.gradient(critic_loss2, self.critic_main2.trainable_variables)\n",
        "      \n",
        "      self.c_opt1.apply_gradients(zip(grads1, self.critic_main.trainable_variables))\n",
        "      self.c_opt2.apply_gradients(zip(grads2, self.critic_main2.trainable_variables))\n",
        "      \n",
        "      \n",
        "      self.trainstep +=1\n",
        "      \n",
        "      if self.trainstep % self.actor_update_steps == 0:\n",
        "                \n",
        "          with tf.GradientTape() as tape3:\n",
        "            \n",
        "              new_policy_actions = self.actor_main(states)\n",
        "              actor_loss = -self.critic_main(states, new_policy_actions)\n",
        "              actor_loss = tf.math.reduce_mean(actor_loss)\n",
        "          \n",
        "          grads3 = tape3.gradient(actor_loss, self.actor_main.trainable_variables)\n",
        "          self.a_opt.apply_gradients(zip(grads3, self.actor_main.trainable_variables))\n",
        "\n",
        "      #if self.trainstep % self.replace == 0:\n",
        "      self.update_target()\n",
        "           \n",
        "      \n",
        " \n",
        "\n",
        "\n",
        "with tf.device('GPU:0'):\n",
        "    tf.random.set_seed(336699)\n",
        "    agent = Agent(2)\n",
        "    episods = 20000\n",
        "    ep_reward = []\n",
        "    total_avgr = []\n",
        "    target = False\n",
        "\n",
        "    for s in range(episods):\n",
        "      if target == True:\n",
        "        break\n",
        "      total_reward = 0 \n",
        "      state = env.reset()\n",
        "      done = False\n",
        "\n",
        "      while not done:\n",
        "        #env.render()\n",
        "        action = agent.act(state)\n",
        "        next_state, reward, done, _ = env.step(action)\n",
        "        agent.savexp(state, next_state, action, done, reward)\n",
        "        agent.train()\n",
        "        state = next_state\n",
        "        total_reward += reward\n",
        "        if done:\n",
        "            ep_reward.append(total_reward)\n",
        "            avg_reward = np.mean(ep_reward[-100:])\n",
        "            total_avgr.append(avg_reward)\n",
        "            print(\"total reward after {} steps is {} and avg reward is {}\".format(s, total_reward, avg_reward))\n",
        "            if int(avg_reward) == 200:\n",
        "              target = True\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: box2d-py in /usr/local/lib/python3.6/dist-packages (2.3.8)\n",
            "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
            "[-inf -inf -inf -inf -inf -inf -inf -inf]\n",
            "[inf inf inf inf inf inf inf inf]\n",
            "[-1. -1.]\n",
            "[1. 1.]\n",
            "total reward after 0 steps is -457.8452693760505 and avg reward is -457.8452693760505\n",
            "total reward after 1 steps is -748.9619461646998 and avg reward is -603.4036077703752\n",
            "total reward after 2 steps is -717.0862373053681 and avg reward is -641.2978176153728\n",
            "total reward after 3 steps is -686.8149388203329 and avg reward is -652.6770979166129\n",
            "total reward after 4 steps is -348.5881866239932 and avg reward is -591.8593156580889\n",
            "total reward after 5 steps is -495.7190764910088 and avg reward is -575.8359424635756\n",
            "total reward after 6 steps is -434.3451819857483 and avg reward is -555.6229766810288\n",
            "total reward after 7 steps is -674.9840926730252 and avg reward is -570.5431161800284\n",
            "total reward after 8 steps is -625.8411111498443 and avg reward is -576.6873378433413\n",
            "total reward after 9 steps is -562.7597267354882 and avg reward is -575.294576732556\n",
            "total reward after 10 steps is -277.6982889153845 and avg reward is -548.2403687491768\n",
            "total reward after 11 steps is -564.784014464268 and avg reward is -549.619005892101\n",
            "total reward after 12 steps is -244.2710002980735 and avg reward is -526.1306977694835\n",
            "total reward after 13 steps is -1097.0983902272503 and avg reward is -566.9141043736097\n",
            "total reward after 14 steps is -204.27761397727213 and avg reward is -542.7383383471872\n",
            "total reward after 15 steps is -665.837635938512 and avg reward is -550.432044446645\n",
            "total reward after 16 steps is -368.91751537569274 and avg reward is -539.7547192071772\n",
            "total reward after 17 steps is -287.7708120693405 and avg reward is -525.7556132550752\n",
            "total reward after 18 steps is -696.971683954101 and avg reward is -534.7669853971292\n",
            "total reward after 19 steps is -216.41440098313282 and avg reward is -518.8493561764294\n",
            "total reward after 20 steps is -604.0353930167769 and avg reward is -522.9058341212078\n",
            "total reward after 21 steps is -311.9866209852066 and avg reward is -513.3185971604805\n",
            "total reward after 22 steps is -489.0267669840842 and avg reward is -512.262430631072\n",
            "total reward after 23 steps is -391.71938744147195 and avg reward is -507.2398038315053\n",
            "total reward after 24 steps is -584.1269800375801 and avg reward is -510.31529087974826\n",
            "total reward after 25 steps is -791.56327234517 and avg reward is -521.1325209361107\n",
            "total reward after 26 steps is -141.74148436581402 and avg reward is -507.08100106313674\n",
            "total reward after 27 steps is -609.0992753573685 and avg reward is -510.72451085935927\n",
            "total reward after 28 steps is -1109.7211602938332 and avg reward is -531.3795677364101\n",
            "total reward after 29 steps is -381.0266608546244 and avg reward is -526.3678041736839\n",
            "total reward after 30 steps is -1322.7715471509687 and avg reward is -552.0582474955319\n",
            "total reward after 31 steps is -145.25801706811677 and avg reward is -539.3457402946751\n",
            "total reward after 32 steps is -100.4266963514733 and avg reward is -526.0451632054873\n",
            "total reward after 33 steps is -149.30505740367138 and avg reward is -514.964571858375\n",
            "total reward after 34 steps is -580.1402377315494 and avg reward is -516.8267337404657\n",
            "total reward after 35 steps is -427.7240724041027 and avg reward is -514.3516598144556\n",
            "total reward after 36 steps is -278.55399376735454 and avg reward is -507.9787499212907\n",
            "total reward after 37 steps is -284.6576408378269 and avg reward is -502.10187862962067\n",
            "total reward after 38 steps is -380.6157513005276 and avg reward is -498.98684972374645\n",
            "total reward after 39 steps is -94.97708571095085 and avg reward is -488.8866056234265\n",
            "total reward after 40 steps is -142.25681378935818 and avg reward is -480.43222045674185\n",
            "total reward after 41 steps is -66.44712380304887 and avg reward is -470.5754324411778\n",
            "total reward after 42 steps is -72.53532805419299 and avg reward is -461.31868582752696\n",
            "total reward after 43 steps is -61.788593403016954 and avg reward is -452.2384564542426\n",
            "total reward after 44 steps is -99.81700266112708 and avg reward is -444.40686859217334\n",
            "total reward after 45 steps is -109.99963573993276 and avg reward is -437.1371461388638\n",
            "total reward after 46 steps is -96.77354525863001 and avg reward is -429.89536739673116\n",
            "total reward after 47 steps is -154.22334334083845 and avg reward is -424.1522002289001\n",
            "total reward after 48 steps is -3.0636430400409296 and avg reward is -415.5585562046377\n",
            "total reward after 49 steps is -70.00172795718106 and avg reward is -408.6474196396885\n",
            "total reward after 50 steps is -69.52841008915632 and avg reward is -401.9980272955604\n",
            "total reward after 51 steps is -438.5356825395592 and avg reward is -402.70067451179114\n",
            "total reward after 52 steps is -279.8840281425619 and avg reward is -400.3833792972774\n",
            "total reward after 53 steps is -31.990568570719063 and avg reward is -393.5612902097485\n",
            "total reward after 54 steps is -132.1524595735823 and avg reward is -388.8084023800001\n",
            "total reward after 55 steps is -78.93623363839589 and avg reward is -383.27497079532856\n",
            "total reward after 56 steps is -144.54665367513445 and avg reward is -379.0867547055006\n",
            "total reward after 57 steps is -193.90972983660225 and avg reward is -375.89404738017475\n",
            "total reward after 58 steps is -63.02285114743178 and avg reward is -370.5911457491113\n",
            "total reward after 59 steps is -428.9117476953915 and avg reward is -371.5631557815493\n",
            "total reward after 60 steps is -97.27164760967293 and avg reward is -367.066573680371\n",
            "total reward after 61 steps is 12.445719907153759 and avg reward is -360.94540765476575\n",
            "total reward after 62 steps is -119.59545764726006 and avg reward is -357.11445606734503\n",
            "total reward after 63 steps is -43.27640314558667 and avg reward is -352.21073649044257\n",
            "total reward after 64 steps is -83.83512892045641 and avg reward is -348.08188098936586\n",
            "total reward after 65 steps is -73.2087429278835 and avg reward is -343.91713647328277\n",
            "total reward after 66 steps is 18.327012680899912 and avg reward is -338.5105073814293\n",
            "total reward after 67 steps is -103.13821347738512 and avg reward is -335.04915011813455\n",
            "total reward after 68 steps is -148.53110244573145 and avg reward is -332.34599000694027\n",
            "total reward after 69 steps is -185.5087548406479 and avg reward is -330.24831521885034\n",
            "total reward after 70 steps is -85.12321104800641 and avg reward is -326.79584896292295\n",
            "total reward after 71 steps is -272.3020468174842 and avg reward is -326.038990599792\n",
            "total reward after 72 steps is -305.2192504270015 and avg reward is -325.7537886796168\n",
            "total reward after 73 steps is -297.7655422560215 and avg reward is -325.3755691333519\n",
            "total reward after 74 steps is -183.04725210474544 and avg reward is -323.47785823963716\n",
            "total reward after 75 steps is -90.77668114528402 and avg reward is -320.41600064629046\n",
            "total reward after 76 steps is -179.39061857448968 and avg reward is -318.5845021778255\n",
            "total reward after 77 steps is -200.55551154307904 and avg reward is -317.0713099902006\n",
            "total reward after 78 steps is -34.29810229836443 and avg reward is -313.49190229789883\n",
            "total reward after 79 steps is -144.67235384944166 and avg reward is -311.38165794229315\n",
            "total reward after 80 steps is -283.6546503784764 and avg reward is -311.0393492069374\n",
            "total reward after 81 steps is -58.06032489022577 and avg reward is -307.95423915429456\n",
            "total reward after 82 steps is -70.71328387837934 and avg reward is -305.0959143919341\n",
            "total reward after 83 steps is 5.615335796707797 and avg reward is -301.39697093730746\n",
            "total reward after 84 steps is -444.70467980095117 and avg reward is -303.08294398276206\n",
            "total reward after 85 steps is -125.2490545726133 and avg reward is -301.01510805938824\n",
            "total reward after 86 steps is -230.05658013578548 and avg reward is -300.19949279589855\n",
            "total reward after 87 steps is -205.19945786793627 and avg reward is -299.1199469444444\n",
            "total reward after 88 steps is -64.54359036381139 and avg reward is -296.484257544662\n",
            "total reward after 89 steps is -52.56181465747643 and avg reward is -293.7740081792488\n",
            "total reward after 90 steps is -215.67252086554635 and avg reward is -292.91575007690045\n",
            "total reward after 91 steps is -210.04405512091313 and avg reward is -292.0149707839006\n",
            "total reward after 92 steps is -22.856380719369177 and avg reward is -289.12079239610995\n",
            "total reward after 93 steps is -48.77616235550396 and avg reward is -286.5639346297205\n",
            "total reward after 94 steps is -349.3796606262847 and avg reward is -287.2251527981054\n",
            "total reward after 95 steps is -148.28766198092723 and avg reward is -285.7778872687598\n",
            "total reward after 96 steps is -257.56326403246067 and avg reward is -285.4870148642619\n",
            "total reward after 97 steps is -61.489982229407204 and avg reward is -283.2013308577838\n",
            "total reward after 98 steps is -51.953769079245525 and avg reward is -280.86549690042483\n",
            "total reward after 99 steps is -32.26205497014557 and avg reward is -278.379462481122\n",
            "total reward after 100 steps is -176.62472570712015 and avg reward is -275.56725704443267\n",
            "total reward after 101 steps is -17.733164366077048 and avg reward is -268.25496922644646\n",
            "total reward after 102 steps is -153.6309103658368 and avg reward is -262.62041595705114\n",
            "total reward after 103 steps is -314.1519348167718 and avg reward is -258.89378591701546\n",
            "total reward after 104 steps is -121.03835959712082 and avg reward is -256.6182876467468\n",
            "total reward after 105 steps is -55.28214781403572 and avg reward is -252.21391835997707\n",
            "total reward after 106 steps is -348.52150611548996 and avg reward is -251.3556816012745\n",
            "total reward after 107 steps is -206.8636764700291 and avg reward is -246.67447743924455\n",
            "total reward after 108 steps is -83.37265887546977 and avg reward is -241.24979291650072\n",
            "total reward after 109 steps is -93.69120424483738 and avg reward is -236.55910769159425\n",
            "total reward after 110 steps is -123.80511857778839 and avg reward is -235.02017598821828\n",
            "total reward after 111 steps is -93.47115580826578 and avg reward is -230.3070474016583\n",
            "total reward after 112 steps is -169.40581119873906 and avg reward is -229.55839551066495\n",
            "total reward after 113 steps is -107.00681087476985 and avg reward is -219.65747971714015\n",
            "total reward after 114 steps is -124.24496553527561 and avg reward is -218.85715323272018\n",
            "total reward after 115 steps is -94.51724416450426 and avg reward is -213.1439493149801\n",
            "total reward after 116 steps is -125.0843827084723 and avg reward is -210.7056179883079\n",
            "total reward after 117 steps is -122.89314450498318 and avg reward is -209.05684131266432\n",
            "total reward after 118 steps is -72.81167123320778 and avg reward is -202.8152411854554\n",
            "total reward after 119 steps is -237.43891757039486 and avg reward is -203.02548635132803\n",
            "total reward after 120 steps is -125.61728570875982 and avg reward is -198.2413052782478\n",
            "total reward after 121 steps is -236.27501931704853 and avg reward is -197.48418926156626\n",
            "total reward after 122 steps is -136.55650294265078 and avg reward is -193.95948662115188\n",
            "total reward after 123 steps is -131.84134558374143 and avg reward is -191.3607062025746\n",
            "total reward after 124 steps is -63.79932444879246 and avg reward is -186.1574296466867\n",
            "total reward after 125 steps is 45.608671491152904 and avg reward is -177.78571020832348\n",
            "total reward after 126 steps is 4.994404014639798 and avg reward is -176.31835132451894\n",
            "total reward after 127 steps is -147.54527426030126 and avg reward is -171.70281131354827\n",
            "total reward after 128 steps is -36.38948521134981 and avg reward is -160.9694945627235\n",
            "total reward after 129 steps is -153.01771451291074 and avg reward is -158.68940509930636\n",
            "total reward after 130 steps is -78.90774373082998 and avg reward is -146.25076706510498\n",
            "total reward after 131 steps is -96.488574156676 and avg reward is -145.76307263599057\n",
            "total reward after 132 steps is 117.2594794170623 and avg reward is -143.58621087830522\n",
            "total reward after 133 steps is -8.904414542854921 and avg reward is -142.18220444969702\n",
            "total reward after 134 steps is -38.877246844568816 and avg reward is -136.76957454082722\n",
            "total reward after 135 steps is -76.11782153411399 and avg reward is -133.25351203212733\n",
            "total reward after 136 steps is -7.423479470138787 and avg reward is -130.54220688915518\n",
            "total reward after 137 steps is -92.48495874342639 and avg reward is -128.62048006821118\n",
            "total reward after 138 steps is -31.765966167743244 and avg reward is -125.13198221688332\n",
            "total reward after 139 steps is -30.99967770703354 and avg reward is -124.49220813684414\n",
            "total reward after 140 steps is 12.564481062330287 and avg reward is -122.94399518832728\n",
            "total reward after 141 steps is -135.82066253263014 and avg reward is -123.63773057562308\n",
            "total reward after 142 steps is 16.768498803583903 and avg reward is -122.74469230704531\n",
            "total reward after 143 steps is -36.451572718986284 and avg reward is -122.49132210020501\n",
            "total reward after 144 steps is -172.37962035874097 and avg reward is -123.21694827718115\n",
            "total reward after 145 steps is -23.146364230037673 and avg reward is -122.34841556208221\n",
            "total reward after 146 steps is -24.439258983368973 and avg reward is -121.62507269932958\n",
            "total reward after 147 steps is -228.1962001070744 and avg reward is -122.36480126699196\n",
            "total reward after 148 steps is -73.63087458949747 and avg reward is -123.07047358248651\n",
            "total reward after 149 steps is -330.9484015410586 and avg reward is -125.6799403183253\n",
            "total reward after 150 steps is -6.1556846809986165 and avg reward is -125.04621306424369\n",
            "total reward after 151 steps is -53.74462011111931 and avg reward is -121.19830243995932\n",
            "total reward after 152 steps is -164.66398652241847 and avg reward is -120.04610202375788\n",
            "total reward after 153 steps is -59.143225561634786 and avg reward is -120.31762859366705\n",
            "total reward after 154 steps is -41.04960040627566 and avg reward is -119.40660000199398\n",
            "total reward after 155 steps is -57.868836635759955 and avg reward is -119.19592603196762\n",
            "total reward after 156 steps is -132.66088925762412 and avg reward is -119.07706838779251\n",
            "total reward after 157 steps is -469.77881022786744 and avg reward is -121.83575919170518\n",
            "total reward after 158 steps is -164.97102933191843 and avg reward is -122.85524097355004\n",
            "total reward after 159 steps is -87.22856930483024 and avg reward is -119.43840918964445\n",
            "total reward after 160 steps is -248.69483404575962 and avg reward is -120.95264105400531\n",
            "total reward after 161 steps is -295.6547247590115 and avg reward is -124.03364550066695\n",
            "total reward after 162 steps is -738.0660978636702 and avg reward is -130.21835190283105\n",
            "total reward after 163 steps is -55.81189691746771 and avg reward is -130.34370684054986\n",
            "total reward after 164 steps is 193.43126333774723 and avg reward is -127.57104291796783\n",
            "total reward after 165 steps is -60.3922007007303 and avg reward is -127.44287749569627\n",
            "total reward after 166 steps is -18.327529662239744 and avg reward is -127.80942291912768\n",
            "total reward after 167 steps is -191.0575924976235 and avg reward is -128.68861670933006\n",
            "total reward after 168 steps is 234.05120612978186 and avg reward is -124.86279362357493\n",
            "total reward after 169 steps is -80.192011840948 and avg reward is -123.80962619357793\n",
            "total reward after 170 steps is -87.83316541676102 and avg reward is -123.8367257372655\n",
            "total reward after 171 steps is 167.45427460598341 and avg reward is -119.43916252303079\n",
            "total reward after 172 steps is 200.81001109968696 and avg reward is -114.37886990776389\n",
            "total reward after 173 steps is -68.63630012713985 and avg reward is -112.0875774864751\n",
            "total reward after 174 steps is 215.41765081727453 and avg reward is -108.10292845725489\n",
            "total reward after 175 steps is 108.98021618275979 and avg reward is -106.10535948397444\n",
            "total reward after 176 steps is -49.02295243199489 and avg reward is -104.80168282254952\n",
            "total reward after 177 steps is 207.46667807954975 and avg reward is -100.7214609263232\n",
            "total reward after 178 steps is 28.22822420444009 and avg reward is -100.09619766129518\n",
            "total reward after 179 steps is 181.73199871041507 and avg reward is -96.8321541356966\n",
            "total reward after 180 steps is -21.774530320089127 and avg reward is -94.21335293511274\n",
            "total reward after 181 steps is -75.57987576517561 and avg reward is -94.38854844386223\n",
            "total reward after 182 steps is 90.30499703464702 and avg reward is -92.77836563473198\n",
            "total reward after 183 steps is -48.98542567065678 and avg reward is -93.32437324940561\n",
            "total reward after 184 steps is -30.2689714572046 and avg reward is -89.18001616596815\n",
            "total reward after 185 steps is -99.96798989021916 and avg reward is -88.92720551914421\n",
            "total reward after 186 steps is -186.2571313576066 and avg reward is -88.48921103136243\n",
            "total reward after 187 steps is -189.3018905657628 and avg reward is -88.33023535834069\n",
            "total reward after 188 steps is -376.20914225829017 and avg reward is -91.44689087728548\n",
            "total reward after 189 steps is 208.3394769918907 and avg reward is -88.83787796079181\n",
            "total reward after 190 steps is -47.55936658156672 and avg reward is -87.156746417952\n",
            "total reward after 191 steps is -248.24826804369948 and avg reward is -87.53878854717988\n",
            "total reward after 192 steps is -21.28338417073691 and avg reward is -87.52305858169353\n",
            "total reward after 193 steps is -92.42881040013883 and avg reward is -87.95958506213988\n",
            "total reward after 194 steps is 71.4328268054564 and avg reward is -83.75146018782247\n",
            "total reward after 195 steps is -40.633414177017755 and avg reward is -82.6749177097834\n",
            "total reward after 196 steps is -184.04520153971407 and avg reward is -81.93973708485593\n",
            "total reward after 197 steps is 157.52903876040997 and avg reward is -79.74954687495777\n",
            "total reward after 198 steps is -74.61833838643886 and avg reward is -79.9761925680297\n",
            "total reward after 199 steps is -99.87666112315512 and avg reward is -80.65233862955978\n",
            "total reward after 200 steps is 175.60054071602957 and avg reward is -77.13008596532829\n",
            "total reward after 201 steps is 209.5174357485679 and avg reward is -74.85757996418185\n",
            "total reward after 202 steps is 245.22389488555618 and avg reward is -70.86903191166792\n",
            "total reward after 203 steps is 147.29977906301912 and avg reward is -66.25451477287001\n",
            "total reward after 204 steps is 220.33097823963243 and avg reward is -62.84082139450247\n",
            "total reward after 205 steps is -55.29214106702976 and avg reward is -62.840921327032405\n",
            "total reward after 206 steps is 106.63333282124282 and avg reward is -58.28937293766508\n",
            "total reward after 207 steps is 282.5116305057678 and avg reward is -53.39561986790711\n",
            "total reward after 208 steps is -186.54047787695913 and avg reward is -54.427298057922\n",
            "total reward after 209 steps is -7.53119078939018 and avg reward is -53.56569792336752\n",
            "total reward after 210 steps is -460.43166225960965 and avg reward is -56.93196336018575\n",
            "total reward after 211 steps is -327.3632884083923 and avg reward is -59.27088468618701\n",
            "total reward after 212 steps is 249.0851997027325 and avg reward is -55.08597457717231\n",
            "total reward after 213 steps is -93.07097227246078 and avg reward is -54.946616191149204\n",
            "total reward after 214 steps is -8.540917862656695 and avg reward is -53.78957571442302\n",
            "total reward after 215 steps is -50.10569735124188 and avg reward is -53.34546024629039\n",
            "total reward after 216 steps is 251.0086309954139 and avg reward is -49.58453010925154\n",
            "total reward after 217 steps is 244.04493958315143 and avg reward is -45.915149268370186\n",
            "total reward after 218 steps is 156.4464599654957 and avg reward is -43.62256795638316\n",
            "total reward after 219 steps is 271.4743036790069 and avg reward is -38.53343574388913\n",
            "total reward after 220 steps is -384.21861440185694 and avg reward is -41.11944903082011\n",
            "total reward after 221 steps is -24.90079867856462 and avg reward is -39.00570682443526\n",
            "total reward after 222 steps is 160.15555356427228 and avg reward is -36.03858625936603\n",
            "total reward after 223 steps is 215.83674098018045 and avg reward is -32.561805393726814\n",
            "total reward after 224 steps is 1.420604531910965 and avg reward is -31.909606103919778\n",
            "total reward after 225 steps is -14.151598207826128 and avg reward is -32.50720880090957\n",
            "total reward after 226 steps is 191.4395038861798 and avg reward is -30.642757802194165\n",
            "total reward after 227 steps is 259.77142203905805 and avg reward is -26.569590839200572\n",
            "total reward after 228 steps is 251.2284583527977 and avg reward is -23.693411403559104\n",
            "total reward after 229 steps is 229.17623462978648 and avg reward is -19.87147191213213\n",
            "total reward after 230 steps is 217.54330456175327 and avg reward is -16.906961429206294\n",
            "total reward after 231 steps is 264.6241675614231 and avg reward is -13.295834012025304\n",
            "total reward after 232 steps is -4.124916782688786 and avg reward is -14.509677974022816\n",
            "total reward after 233 steps is 25.027980584836257 and avg reward is -14.170354022745904\n",
            "total reward after 234 steps is 261.2405993211596 and avg reward is -11.169175561088624\n",
            "total reward after 235 steps is 250.5957705470401 and avg reward is -7.902039640277078\n",
            "total reward after 236 steps is 4.638964287538357 and avg reward is -7.781415202700309\n",
            "total reward after 237 steps is 191.11870036917873 and avg reward is -4.945378611574258\n",
            "total reward after 238 steps is 256.8027508647055 and avg reward is -2.05969144124977\n",
            "total reward after 239 steps is 236.50489259923208 and avg reward is 0.6153542618128856\n",
            "total reward after 240 steps is 248.19086720393724 and avg reward is 2.9716181232289562\n",
            "total reward after 241 steps is 206.52428198623528 and avg reward is 6.395067568417613\n",
            "total reward after 242 steps is 212.80696234693357 and avg reward is 8.355452203851108\n",
            "total reward after 243 steps is 264.3047174559978 and avg reward is 11.363015105600947\n",
            "total reward after 244 steps is 247.13771618564738 and avg reward is 15.558188471044835\n",
            "total reward after 245 steps is 225.74462439273748 and avg reward is 18.04709835727259\n",
            "total reward after 246 steps is 247.31490310703765 and avg reward is 20.76463997817665\n",
            "total reward after 247 steps is 261.4574118218353 and avg reward is 25.661176097465745\n",
            "total reward after 248 steps is 213.3224699417978 and avg reward is 28.5307095427787\n",
            "total reward after 249 steps is 203.612129304478 and avg reward is 33.87631485123406\n",
            "total reward after 250 steps is 253.4375420265872 and avg reward is 36.47224711830992\n",
            "total reward after 251 steps is 221.24910679650415 and avg reward is 39.222184387386164\n",
            "total reward after 252 steps is 193.21464583019812 and avg reward is 42.800970710912324\n",
            "total reward after 253 steps is 258.8159163726323 and avg reward is 45.980562130254995\n",
            "total reward after 254 steps is 169.83344050869817 and avg reward is 48.08939253940473\n",
            "total reward after 255 steps is 242.6741425187861 and avg reward is 51.094822330950194\n",
            "total reward after 256 steps is 187.15943580044404 and avg reward is 54.293025581530884\n",
            "total reward after 257 steps is 250.25096787127137 and avg reward is 61.49332336252226\n",
            "total reward after 258 steps is 236.91834107051997 and avg reward is 65.51221706654665\n",
            "total reward after 259 steps is 237.56558170759655 and avg reward is 68.7601585766709\n",
            "total reward after 260 steps is 267.3365650374173 and avg reward is 73.92047256750269\n",
            "total reward after 261 steps is 271.1674376390249 and avg reward is 79.58869419148304\n",
            "total reward after 262 steps is 262.2893547397871 and avg reward is 89.59224871751763\n",
            "total reward after 263 steps is 195.583656970326 and avg reward is 92.10620425639557\n",
            "total reward after 264 steps is 217.7318121492644 and avg reward is 92.34920974451073\n",
            "total reward after 265 steps is 276.59415790118805 and avg reward is 95.71907333052992\n",
            "total reward after 266 steps is 206.43977510334463 and avg reward is 97.96674637818576\n",
            "total reward after 267 steps is 244.44680950972986 and avg reward is 102.3217903982593\n",
            "total reward after 268 steps is 233.80233932645305 and avg reward is 102.31930173022602\n",
            "total reward after 269 steps is 228.65739071431517 and avg reward is 105.40779575577866\n",
            "total reward after 270 steps is -23.72831052374255 and avg reward is 106.04884430470884\n",
            "total reward after 271 steps is 113.07793905396625 and avg reward is 105.50508094918867\n",
            "total reward after 272 steps is -17.153614818813356 and avg reward is 103.32544469000366\n",
            "total reward after 273 steps is -96.72457396645987 and avg reward is 103.04456195161046\n",
            "total reward after 274 steps is -6.08087942365934 and avg reward is 100.8295766492011\n",
            "total reward after 275 steps is -103.51350714825584 and avg reward is 98.70463941589094\n",
            "total reward after 276 steps is 90.96320049817811 and avg reward is 100.10450094519267\n",
            "total reward after 277 steps is 247.78204712407225 and avg reward is 100.50765463563788\n",
            "total reward after 278 steps is 246.07770769220423 and avg reward is 102.68614947051552\n",
            "total reward after 279 steps is 225.06300474969265 and avg reward is 103.1194595309083\n",
            "total reward after 280 steps is 259.73383310365193 and avg reward is 105.93454316514573\n",
            "total reward after 281 steps is 282.6469994990282 and avg reward is 109.51681191778775\n",
            "total reward after 282 steps is 264.25506911643254 and avg reward is 111.2563126386056\n",
            "total reward after 283 steps is 272.85211000523054 and avg reward is 114.47468799536449\n",
            "total reward after 284 steps is 263.5695367874472 and avg reward is 117.413073077811\n",
            "total reward after 285 steps is -95.72827482880822 and avg reward is 117.45547022842513\n",
            "total reward after 286 steps is 9.617473653512263 and avg reward is 119.41421627853632\n",
            "total reward after 287 steps is 254.2175120997102 and avg reward is 123.84941030519104\n",
            "total reward after 288 steps is -158.72578412595374 and avg reward is 126.0242438865144\n",
            "total reward after 289 steps is -15.488164508000906 and avg reward is 123.7859674715155\n",
            "total reward after 290 steps is 204.99976095996686 and avg reward is 126.31155874693084\n",
            "total reward after 291 steps is 241.28632649007065 and avg reward is 131.2069046922685\n",
            "total reward after 292 steps is 227.83948752280048 and avg reward is 133.6981334092039\n",
            "total reward after 293 steps is 235.66584090261938 and avg reward is 136.97907992223148\n",
            "total reward after 294 steps is -12.584567296862332 and avg reward is 136.13890598120827\n",
            "total reward after 295 steps is 253.7462460810772 and avg reward is 139.08270258378923\n",
            "total reward after 296 steps is 268.3166551214208 and avg reward is 143.6063211504006\n",
            "total reward after 297 steps is 232.98479603242092 and avg reward is 144.36087872312066\n",
            "total reward after 298 steps is 258.3439066335969 and avg reward is 147.69050117332105\n",
            "total reward after 299 steps is 226.10354323437423 and avg reward is 150.95030321689634\n",
            "total reward after 300 steps is -237.58578303743874 and avg reward is 146.81843997936167\n",
            "total reward after 301 steps is 233.47503074543278 and avg reward is 147.0580159293303\n",
            "total reward after 302 steps is 12.419497332538466 and avg reward is 144.72997195380015\n",
            "total reward after 303 steps is 192.51320197428592 and avg reward is 145.1821061829128\n",
            "total reward after 304 steps is 213.0311896724065 and avg reward is 145.10910829724057\n",
            "total reward after 305 steps is -25.546893031963606 and avg reward is 145.4065607775912\n",
            "total reward after 306 steps is 158.90216159523465 and avg reward is 145.92924906533113\n",
            "total reward after 307 steps is 232.9187061543406 and avg reward is 145.43331982181687\n",
            "total reward after 308 steps is 280.31332515911583 and avg reward is 150.1018578521776\n",
            "total reward after 309 steps is 223.53068263610476 and avg reward is 152.41247658643255\n",
            "total reward after 310 steps is 246.15655067621046 and avg reward is 159.47835871579076\n",
            "total reward after 311 steps is -2.550550910184441 and avg reward is 162.72648609077285\n",
            "total reward after 312 steps is -41.86612270804891 and avg reward is 159.81697286666505\n",
            "total reward after 313 steps is 255.77490470376807 and avg reward is 163.30543163642733\n",
            "total reward after 314 steps is 256.58382975580406 and avg reward is 165.95667911261197\n",
            "total reward after 315 steps is 240.8881743045921 and avg reward is 168.86661782917028\n",
            "total reward after 316 steps is 206.7356309218842 and avg reward is 168.423887828435\n",
            "total reward after 317 steps is -4.99675950264627 and avg reward is 165.93347083757698\n",
            "total reward after 318 steps is 258.33626404982647 and avg reward is 166.9523688784203\n",
            "total reward after 319 steps is 205.1096551321939 and avg reward is 166.28872239295214\n",
            "total reward after 320 steps is 239.39331724285557 and avg reward is 172.5248417093993\n",
            "total reward after 321 steps is 261.53187251983354 and avg reward is 175.3891684213833\n",
            "total reward after 322 steps is 273.2273528988114 and avg reward is 176.5198864147287\n",
            "total reward after 323 steps is 238.63366123906758 and avg reward is 176.7478556173175\n",
            "total reward after 324 steps is 253.05230449045055 and avg reward is 179.2641726169029\n",
            "total reward after 325 steps is 251.06996973328037 and avg reward is 181.91638829631404\n",
            "total reward after 326 steps is -14.04892822044738 and avg reward is 179.86150397524773\n",
            "total reward after 327 steps is 257.0422611469953 and avg reward is 179.8342123663271\n",
            "total reward after 328 steps is 210.49462847728827 and avg reward is 179.426874067572\n",
            "total reward after 329 steps is 286.5732081529377 and avg reward is 180.00084380280353\n",
            "total reward after 330 steps is 265.99505381501064 and avg reward is 180.48536129533608\n",
            "total reward after 331 steps is 189.65849586944324 and avg reward is 179.73570457841635\n",
            "total reward after 332 steps is 283.6521909327636 and avg reward is 182.61347565557082\n",
            "total reward after 333 steps is 212.23891888312465 and avg reward is 184.48558503855372\n",
            "total reward after 334 steps is 229.9721776812442 and avg reward is 184.1729008221546\n",
            "total reward after 335 steps is 219.63292399892435 and avg reward is 183.8632723566734\n",
            "total reward after 336 steps is 273.67692684290193 and avg reward is 186.55365198222705\n",
            "total reward after 337 steps is 243.68184014354534 and avg reward is 187.0792833799707\n",
            "total reward after 338 steps is 277.0631516589976 and avg reward is 187.2818873879136\n",
            "total reward after 339 steps is 249.77996978447138 and avg reward is 187.414638159766\n",
            "total reward after 340 steps is 271.3420691850518 and avg reward is 187.64615017957712\n",
            "total reward after 341 steps is 274.0622807093292 and avg reward is 188.3215301668081\n",
            "total reward after 342 steps is 267.50297410150296 and avg reward is 188.86849028435375\n",
            "total reward after 343 steps is 252.50450437207263 and avg reward is 188.75048815351448\n",
            "total reward after 344 steps is 237.37381073368869 and avg reward is 188.65284909899492\n",
            "total reward after 345 steps is 270.54637122457945 and avg reward is 189.10086656731335\n",
            "total reward after 346 steps is 231.66750771366517 and avg reward is 188.94439261337965\n",
            "total reward after 347 steps is 208.1016512599519 and avg reward is 188.41083500776082\n",
            "total reward after 348 steps is 266.75651886842303 and avg reward is 188.9451754970271\n",
            "total reward after 349 steps is 283.18769974772283 and avg reward is 189.7409312014595\n",
            "total reward after 350 steps is 271.81506365058533 and avg reward is 189.92470641769953\n",
            "total reward after 351 steps is 238.24810301785857 and avg reward is 190.09469637991305\n",
            "total reward after 352 steps is 292.1770142979532 and avg reward is 191.08432006459057\n",
            "total reward after 353 steps is -5.8805457580647555 and avg reward is 188.4373554432836\n",
            "total reward after 354 steps is 225.58696615586592 and avg reward is 188.99489069975527\n",
            "total reward after 355 steps is 244.8454531761298 and avg reward is 189.01660380632876\n",
            "total reward after 356 steps is 271.8903052749717 and avg reward is 189.86391250107397\n",
            "total reward after 357 steps is -51.71795096397011 and avg reward is 186.84422331272154\n",
            "total reward after 358 steps is 240.297050188864 and avg reward is 186.878010403905\n",
            "total reward after 359 steps is 187.43358930018582 and avg reward is 186.37669047983087\n",
            "total reward after 360 steps is 264.7814741974956 and avg reward is 186.35113957143164\n",
            "total reward after 361 steps is 247.7335914748266 and avg reward is 186.11680110978966\n",
            "total reward after 362 steps is 216.7436389345899 and avg reward is 185.66134395173773\n",
            "total reward after 363 steps is -283.2448933247547 and avg reward is 180.87305844878694\n",
            "total reward after 364 steps is -10.888763493468085 and avg reward is 178.5868526923596\n",
            "total reward after 365 steps is 238.34528763730765 and avg reward is 178.2043639897208\n",
            "total reward after 366 steps is 160.73974986076115 and avg reward is 177.747363737295\n",
            "total reward after 367 steps is -225.90154867293398 and avg reward is 173.04388015546837\n",
            "total reward after 368 steps is 276.85066815535015 and avg reward is 173.47436344375737\n",
            "total reward after 369 steps is 245.40106987849936 and avg reward is 173.64180023539916\n",
            "total reward after 370 steps is -201.10280256027187 and avg reward is 171.8680553150339\n",
            "total reward after 371 steps is -152.36217384412478 and avg reward is 169.21365418605296\n",
            "total reward after 372 steps is -229.34667799175858 and avg reward is 167.0917235543235\n",
            "total reward after 373 steps is 244.93275983690708 and avg reward is 170.50829689235715\n",
            "total reward after 374 steps is 253.3799007802324 and avg reward is 173.1029046943961\n",
            "total reward after 375 steps is 241.00177795066975 and avg reward is 176.54805754538535\n",
            "total reward after 376 steps is 229.7696978266924 and avg reward is 177.9361225186705\n",
            "total reward after 377 steps is 276.7275611255777 and avg reward is 178.22557765868558\n",
            "total reward after 378 steps is 285.087475818351 and avg reward is 178.61567533994705\n",
            "total reward after 379 steps is 256.1931948162585 and avg reward is 178.92697724061273\n",
            "total reward after 380 steps is 301.3795571679706 and avg reward is 179.34343448125585\n",
            "total reward after 381 steps is 303.8259359807272 and avg reward is 179.55522384607283\n",
            "total reward after 382 steps is 299.68541584002696 and avg reward is 179.90952731330879\n",
            "total reward after 383 steps is 271.1255831132458 and avg reward is 179.89226204438896\n",
            "total reward after 384 steps is 282.79793577960834 and avg reward is 180.08454603431056\n",
            "total reward after 385 steps is 272.11328300768434 and avg reward is 183.76296161267547\n",
            "total reward after 386 steps is 253.7417621329329 and avg reward is 186.20420449746973\n",
            "total reward after 387 steps is 258.63130110719214 and avg reward is 186.24834238754448\n",
            "total reward after 388 steps is 240.69112470696297 and avg reward is 190.24251147587364\n",
            "total reward after 389 steps is 236.3233473062035 and avg reward is 192.76062659401566\n",
            "total reward after 390 steps is 256.1059742320401 and avg reward is 193.27168872673644\n",
            "total reward after 391 steps is 262.11847903349553 and avg reward is 193.4800102521707\n",
            "total reward after 392 steps is 236.8258446602745 and avg reward is 193.56987382354544\n",
            "total reward after 393 steps is 261.5890720108686 and avg reward is 193.82910613462792\n",
            "total reward after 394 steps is 275.873083514785 and avg reward is 196.71368264274443\n",
            "total reward after 395 steps is 242.7453978581536 and avg reward is 196.6036741605152\n",
            "total reward after 396 steps is 294.375947578196 and avg reward is 196.8642670850829\n",
            "total reward after 397 steps is 244.59430141337026 and avg reward is 196.9803621388924\n",
            "total reward after 398 steps is 231.62049354861648 and avg reward is 196.71312800804262\n",
            "total reward after 399 steps is 53.47505166958456 and avg reward is 194.9868430923947\n",
            "total reward after 400 steps is 253.98963041532616 and avg reward is 199.90259722692238\n",
            "total reward after 401 steps is 281.4862928973938 and avg reward is 200.38270984844195\n",
            "total reward after 402 steps is 297.3475065707404 and avg reward is 203.23198994082398\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cFoOf-MYf-Ep"
      },
      "source": [
        "ep = [i  for i in range(len(avg_rewards_list))]\n",
        "plt.plot( range(len(avg_rewards_list)),avg_rewards_list,'b')\n",
        "plt.title(\"Avg Test Aeward Vs Test Episods\")\n",
        "plt.xlabel(\"Test Episods\")\n",
        "plt.ylabel(\"Average Test Reward\")\n",
        "plt.grid(True)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}