atari_DDDQN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "atari_DDDQN.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyO46O8xFvBLIHrAVqqInHCo",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abhisheksuran/Atari_DQN/blob/master/atari_DDDQN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XMCXZFsn2ldc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# importing some lib...:p\n",
        "\n",
        "import tensorflow as tf \n",
        "from tensorflow.keras.layers import  Dense, Dropout, Conv2D, MaxPooling2D, Activation, Flatten, Input\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.models import Model\n",
        "from collections import deque\n",
        "import numpy as np\n",
        "import gym\n",
        "import random\n",
        "import os\n",
        "from tensorflow.keras.losses import Huber\n",
        "import tqdm\n",
        "from skimage import transform \n",
        "from skimage.color import rgb2gray"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "abCSx6kL2uKf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "9d8a9e7a-1b66-49aa-ad15-a5c47ad2e116"
      },
      "source": [
        "! pip install gym[atari]"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: gym[atari] in /usr/local/lib/python3.6/dist-packages (0.17.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym[atari]) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.6/dist-packages (from gym[atari]) (1.18.5)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym[atari]) (1.5.0)\n",
            "Requirement already satisfied: cloudpickle<1.4.0,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym[atari]) (1.3.0)\n",
            "Requirement already satisfied: opencv-python; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari]) (4.1.2.30)\n",
            "Requirement already satisfied: atari-py~=0.2.0; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari]) (0.2.6)\n",
            "Requirement already satisfied: Pillow; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari]) (7.0.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[atari]) (0.16.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from atari-py~=0.2.0; extra == \"atari\"->gym[atari]) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E3goXW5K2xO9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# loading environment\n",
        "env = gym.make('Breakout-v0')"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y1ypVbxGeMVN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "7fcb7d3d-6708-4350-deb0-76de60d085aa"
      },
      "source": [
        "# getting idea about environment\n",
        "print(\"The size of our frame is: \", env.observation_space)\n",
        "print(\"The action size is : \", env.action_space.n)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The size of our frame is:  Box(210, 160, 3)\n",
            "The action size is :  4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x905pF9e20y6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class model(tf.keras.Model):\n",
        "    def __init__(self):\n",
        "      super().__init__()\n",
        "      self.c1 = Conv2D(filters = 32,kernel_size = [8,8],strides = [4,4],padding = \"VALID\", activation='relu')\n",
        "      self.c2 = Conv2D(filters = 64,kernel_size = [4,4],strides = [2,2],padding = \"VALID\", activation = 'relu')\n",
        "      self.c3 = Conv2D(filters = 64,kernel_size = [3,3],strides = [2,2],padding = \"VALID\", activation = 'relu')\n",
        "      self.d1 = Dense(512, activation = 'relu')\n",
        "      self.f = Flatten()\n",
        "      self.d2 = tf.keras.layers.Dense(128, activation='relu')\n",
        "      self.d3 = tf.keras.layers.Dense(128, activation='relu')  \n",
        "      self.v = tf.keras.layers.Dense(1, activation=None)\n",
        "      self.a = tf.keras.layers.Dense(env.action_space.n, activation=None)\n",
        "\n",
        "    def call(self, input_data):\n",
        "      x = self.c1(input_data)\n",
        "      x = self.c2(x)\n",
        "      x = self.c3(x)\n",
        "      x = self.d1(x)\n",
        "      x = self.f(x)\n",
        "      v = self.d2(x)\n",
        "      v = self.v(v)\n",
        "      a = self.d3(x)\n",
        "      a = self.a(a)\n",
        "      Q = v +(a -tf.math.reduce_mean(a, axis=1, keepdims=True))\n",
        "      return Q\n",
        "\n",
        "    def advantage(self, state):\n",
        "      x = self.c1(state)\n",
        "      x = self.c2(x)\n",
        "      x = self.c3(x)\n",
        "      x = self.d1(x)\n",
        "      x = self.f(x)\n",
        "      a = self.d3(x)\n",
        "      a = self.a(a)\n",
        "      return a\n"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZF7gFMGU8Zk8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# processing images from environment\n",
        "\n",
        "def image_process(frame):\n",
        "    \n",
        "    gray = rgb2gray(frame)\n",
        "    cropped_frame = gray[8:-12,4:-12]\n",
        "    normalized_frame = cropped_frame/255.0\n",
        "    preprocessed_frame = transform.resize(normalized_frame, [110,80])\n",
        "    #preprocessed_frame =  np.array(preprocessed_frame).reshape(-1, 105, 80, 1)    \n",
        "    return preprocessed_frame \n",
        "\n",
        "\n",
        "# 4 images represents a single state\n",
        "\n",
        "\n",
        "stack_size = 4 \n",
        "\n",
        "\n",
        "stacked_frames  =  deque([np.zeros((110,80), dtype=np.int) for i in range(stack_size)], maxlen=4)\n",
        "\n",
        "\n",
        "def stack_state(stacked_frames, state, is_new_episode):\n",
        "    \n",
        "    frame = image_process(state)\n",
        "    \n",
        "    #print(frame.shape)\n",
        "\n",
        "    if is_new_episode:\n",
        "\n",
        "        stacked_frames = deque([np.zeros((110,80), dtype=np.int) for i in range(stack_size)], maxlen=4)\n",
        "\n",
        "\n",
        "        stacked_frames.append(frame)\n",
        "        stacked_frames.append(frame)\n",
        "        stacked_frames.append(frame)\n",
        "        stacked_frames.append(frame)\n",
        "\n",
        "\n",
        "        stacked_state = np.stack(stacked_frames, axis=0)\n",
        "\n",
        "    else:\n",
        "\n",
        "        stacked_frames.append(frame)\n",
        "\n",
        "\n",
        "        stacked_state = np.stack(stacked_frames, axis=0) \n",
        "    stacked_state =  np.array(stacked_state).reshape(-1,110, 80, 4)   \n",
        "    stac = stacked_state\n",
        "    #print(stac.shape)\n",
        "    return stac, stacked_frames"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mIo_4GpjDukC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#our agent 007 :P\n",
        "\n",
        "class my_agent:\n",
        "    def __init__(self,env):\n",
        "        self.env = env\n",
        "        #self.state_input = Input((105,80,4), name='state_input')\n",
        "        #self.action_input = action\n",
        "        self.gamma = 0.99\n",
        "        self.epsilon = 1.0\n",
        "        self.epsilon_decay = 1e-3 \n",
        "        self.memory = deque(maxlen=100000)\n",
        "        self.q_net = model()\n",
        "        self.target_net = model()\n",
        "        optimizer = Adam(learning_rate=0.01)\n",
        "        huber = Huber()\n",
        "        self.q_net.compile(optimizer, loss=huber)\n",
        "        self.target_net.compile(optimizer, loss=huber)\n",
        "        self.alighn_target_model()\n",
        "        self.min_epsilon = 0.01\n",
        "        self.trainstep = 0\n",
        "        self.replace = 100\n",
        "        self.mem_pointer = 0\n",
        "  \n",
        "        \n",
        "    def store(self, state, action, reward, next_state, terminated):\n",
        "        self.memory.append((state, action, reward, next_state, terminated))\n",
        "        \n",
        "    \n",
        "    def _update_epsilon(self):\n",
        "        self.epsilon = self.epsilon - self.epsilon_decay if self.epsilon > self.min_epsilon else self.min_epsilon\n",
        "        #return self.epsilon\n",
        "        \n",
        "        \n",
        "   \n",
        "    def alighn_target_model(self):\n",
        "        self.target_net.set_weights(self.q_net.get_weights())     \n",
        "        \n",
        "     \n",
        "    def act(self, state):\n",
        "        \n",
        "        if np.random.rand() <= self.epsilon:\n",
        "            action = np.random.choice([i for i in range(env.action_space.n)])\n",
        "            \n",
        "            return action\n",
        "\n",
        "        state= state        \n",
        "        q_values = self.q_net.advantage(state)\n",
        "        a = np.argmax(q_values[0])\n",
        "        #print(a)\n",
        "        return a\n",
        "        \n",
        "   \n",
        "    def retrain(self, batch_size):\n",
        "        \n",
        "        if self.trainstep % self.replace == 0:\n",
        "            self.alighn_target_model()\n",
        "\n",
        "\n",
        "        minibatch = random.sample(self.memory, batch_size)\n",
        "        \n",
        "        for state, action, reward, next_state, terminated in minibatch:\n",
        "            \n",
        "            state = state  \n",
        "      \n",
        "            next_state = next_state\n",
        "           \n",
        "            \n",
        "            target = self.q_net.predict(state)\n",
        "            max_action = np.argmax(self.q_net.predict(next_state)[0])\n",
        "           \n",
        "            if terminated:\n",
        "                target[0][action] = reward\n",
        "                \n",
        "            else:\n",
        "                t = self.target_net.predict(next_state)\n",
        "                \n",
        "                target[0][action] = reward + self.gamma * t[0][max_action]\n",
        "              \n",
        "            \n",
        "            self.q_net.fit(state, target, epochs=1, verbose=0)\n",
        "            self._update_epsilon()\n",
        " \n",
        "     \n",
        "    def load(self, path):\n",
        "        self.q_net.load_weights(path)\n",
        "\n",
        "    def save(self, path):\n",
        "        self.q_net.save_weights(path)\n",
        "        \n",
        "\n"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9sqqQIF1QH86",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "b4885571-c1e2-41e1-d787-ea2980615a48"
      },
      "source": [
        "state = env.reset()\n",
        "\n",
        "agent = my_agent(env)\n",
        "\n",
        "# some hyperparameter...\n",
        "\n",
        "batch_size = 64                             \n",
        "num_of_episodes = 1000\n",
        "# our agent will play and learn\n",
        "\n",
        "\n",
        "for e in tqdm.tqdm(range(0, num_of_episodes)):\n",
        "    \n",
        "    state = env.reset()\n",
        "    state , stacked_frames = stack_state(stacked_frames, state, True)\n",
        "    #print(state.shape)\n",
        "\n",
        "    t_reward = 0\n",
        "    terminated = False\n",
        "    done = False\n",
        "    #for timestep in range(timesteps_per_episode):\n",
        "    while not done:\n",
        "       # env.render()\n",
        "        state = state \n",
        "        #print(state.shape)\n",
        "        action = agent.act(state)\n",
        "        #action = np.array([1])\n",
        "        print(action)\n",
        "        \n",
        "        next_state, reward, terminated, info = env.step(action)\n",
        "        next_state , stacked_frames = stack_state(stacked_frames, next_state , False)\n",
        "\n",
        "        agent.store(state, action, reward, next_state, terminated)\n",
        "        agent.mem_pointer += 1\n",
        "\n",
        "        state  = next_state\n",
        "        t_reward += reward\n",
        "\n",
        "\n",
        "        if terminated:\n",
        "            done = True\n",
        "         \n",
        "            print(\"Total reward is {} and  epsilon is {}\".format(t_reward, agent.epsilon))\n",
        "            break\n",
        "\n",
        "        if len(agent.memory) > batch_size:\n",
        "            agent.retrain(batch_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "  0%|          | 0/1000 [00:00<?, ?it/s]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "3\n",
            "2\n",
            "1\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "3\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "3\n",
            "2\n",
            "2\n",
            "2\n",
            "2\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "2\n",
            "0\n",
            "2\n",
            "WARNING:tensorflow:Layer conv2d_25 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
            "\n",
            "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
            "\n",
            "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
            "\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "2\n",
            "3\n",
            "2\n",
            "3\n",
            "2\n",
            "3\n",
            "3\n",
            "1\n",
            "1\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "2\n",
            "0\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "1\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "3\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "1\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "  0%|          | 1/1000 [20:31<341:42:21, 1231.37s/it]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "3\n",
            "Total reward is 1.0 and  epsilon is 0.01\n",
            "3\n",
            "2\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "3\n",
            "3\n",
            "2\n",
            "2\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "3\n",
            "2\n",
            "2\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "3\n",
            "2\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "0\n",
            "1\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "3\n",
            "1\n",
            "1\n",
            "2\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "2\n",
            "2\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "3\n",
            "2\n",
            "2\n",
            "0\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "0\n",
            "2\n",
            "2\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "1\n",
            "2\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "0\n",
            "1\n",
            "3\n",
            "3\n",
            "0\n",
            "2\n",
            "1\n",
            "0\n",
            "3\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "3\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "2\n",
            "3\n",
            "3\n",
            "2\n",
            "0\n",
            "3\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "0\n",
            "2\n",
            "2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "  0%|          | 2/1000 [1:02:35<448:51:11, 1619.11s/it]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1\n",
            "Total reward is 3.0 and  epsilon is 0.01\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "1\n",
            "2\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "2\n",
            "3\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "2\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "2\n",
            "2\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "3\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "2\n",
            "2\n",
            "2\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "3\n",
            "1\n",
            "1\n",
            "2\n",
            "0\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "0\n",
            "0\n",
            "3\n",
            "2\n",
            "0\n",
            "2\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "1\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "  0%|          | 3/1000 [1:36:23<482:26:11, 1742.00s/it]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1\n",
            "Total reward is 2.0 and  epsilon is 0.01\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "2\n",
            "1\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "1\n",
            "1\n",
            "2\n",
            "2\n",
            "2\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "2\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "2\n",
            "0\n",
            "0\n",
            "3\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "3\n",
            "3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "  0%|          | 4/1000 [1:57:19<441:32:43, 1595.95s/it]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2\n",
            "Total reward is 0.0 and  epsilon is 0.01\n",
            "2\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "2\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "2\n",
            "3\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "1\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "2\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "2\n",
            "3\n",
            "1\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "  0%|          | 5/1000 [2:20:11<422:34:48, 1528.93s/it]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "Total reward is 0.0 and  epsilon is 0.01\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "0\n",
            "3\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "2\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "3\n",
            "3\n",
            "2\n",
            "2\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "3\n",
            "3\n",
            "2\n",
            "2\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "3\n",
            "3\n",
            "0\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "3\n",
            "3\n",
            "2\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "3\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "2\n",
            "2\n",
            "2\n",
            "2\n",
            "3\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "3\n",
            "3\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "2\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "2\n",
            "2\n",
            "2\n",
            "3\n",
            "3\n",
            "3\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n",
            "0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "  1%|          | 6/1000 [2:56:50<477:38:51, 1729.91s/it]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1\n",
            "Total reward is 2.0 and  epsilon is 0.01\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n",
            "3\n",
            "0\n",
            "3\n",
            "3\n",
            "2\n",
            "3\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "2\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "2\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "2\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "2\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "3\n",
            "3\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "1\n",
            "3\n",
            "3\n",
            "2\n",
            "0\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "3\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "0\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "1\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "2\n",
            "3\n",
            "2\n",
            "2\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "3\n",
            "3\n",
            "2\n",
            "3\n",
            "1\n",
            "3\n",
            "3\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "2\n",
            "2\n",
            "2\n",
            "3\n",
            "3\n",
            "3\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "3\n",
            "3\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "2\n",
            "2\n",
            "0\n",
            "1\n",
            "3\n",
            "3\n",
            "3\n",
            "2\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "2\n",
            "0\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "3\n",
            "3\n",
            "0\n",
            "2\n",
            "2\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "3\n",
            "3\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "2\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n",
            "3\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "0\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "3\n",
            "3\n",
            "3\n",
            "3\n",
            "2\n",
            "2\n",
            "0\n",
            "2\n",
            "2\n",
            "2\n",
            "0\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "2\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "  1%|          | 7/1000 [3:46:50<582:14:42, 2110.86s/it]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1\n",
            "Total reward is 4.0 and  epsilon is 0.01\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "3\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "2\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "3\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "2\n",
            "2\n",
            "2\n",
            "1\n",
            "3\n",
            "3\n",
            "2\n",
            "0\n",
            "2\n",
            "2\n",
            "0\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "2\n",
            "0\n",
            "3\n",
            "3\n",
            "0\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "3\n",
            "3\n",
            "0\n",
            "0\n",
            "2\n",
            "1\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "1\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "0\n",
            "3\n",
            "3\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "1\n",
            "0\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "0\n",
            "3\n",
            "3\n",
            "3\n",
            "1\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "1\n",
            "3\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "1\n",
            "3\n",
            "3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "  1%|          | 8/1000 [4:15:23<548:48:28, 1991.64s/it]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "Total reward is 1.0 and  epsilon is 0.01\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "3\n",
            "3\n",
            "1\n",
            "1\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "0\n",
            "3\n",
            "1\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "1\n",
            "1\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "3\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "2\n",
            "1\n",
            "3\n",
            "0\n",
            "2\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "2\n",
            "2\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "1\n",
            "2\n",
            "3\n",
            "1\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "3\n",
            "1\n",
            "2\n",
            "0\n",
            "0\n",
            "3\n",
            "3\n",
            "0\n",
            "1\n",
            "1\n",
            "3\n",
            "3\n",
            "3\n",
            "0\n",
            "0\n",
            "3\n",
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "2\n",
            "3\n",
            "0\n",
            "3\n",
            "2\n",
            "0\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "3\n",
            "0\n",
            "0\n",
            "1\n",
            "3\n",
            "2\n",
            "3\n",
            "3\n",
            "2\n",
            "0\n",
            "3\n",
            "2\n",
            "1\n",
            "1\n",
            "2\n",
            "3\n",
            "1\n",
            "1\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "0\n",
            "2\n",
            "2\n",
            "2\n",
            "1\n",
            "0\n",
            "1\n",
            "2\n",
            "2\n",
            "0\n",
            "3\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "2\n",
            "1\n",
            "0\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "2\n",
            "3\n",
            "1\n",
            "0\n",
            "3\n",
            "1\n",
            "3\n",
            "0\n",
            "1\n",
            "0\n",
            "2\n",
            "0\n",
            "3\n",
            "2\n",
            "2\n",
            "3\n",
            "2\n",
            "0\n",
            "2\n",
            "1\n",
            "3\n",
            "3\n",
            "2\n",
            "3\n",
            "0\n",
            "0\n",
            "0\n",
            "3\n",
            "3\n",
            "1\n",
            "2\n",
            "0\n",
            "3\n",
            "1\n",
            "0\n",
            "0\n",
            "2\n",
            "2\n",
            "1\n",
            "1\n",
            "3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9BdhcyI0QjZK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "  "
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}