alphago_clone_project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "T4",
      "authorship_tag": "ABX9TyNwzmAwMx+QC4d0mnElj+QX",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gitHubAndyLee2020/Monte_Carlo_Go_Agent/blob/main/alphago_clone_project.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### AlphaGo Clone Project"
      ],
      "metadata": {
        "id": "CSzTQxDbRwLc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "###### Installations"
      ],
      "metadata": {
        "id": "jDKVQv-7Xk8z"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ktU18IIuRoUw",
        "outputId": "9902a987-ff44-4d12-c0d6-c499e1987b01"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'GymGo'...\n",
            "remote: Enumerating objects: 917, done.\u001b[K\n",
            "remote: Counting objects: 100% (208/208), done.\u001b[K\n",
            "remote: Compressing objects: 100% (78/78), done.\u001b[K\n",
            "remote: Total 917 (delta 130), reused 189 (delta 130), pack-reused 709\u001b[K\n",
            "Receiving objects: 100% (917/917), 457.37 KiB | 13.07 MiB/s, done.\n",
            "Resolving deltas: 100% (555/555), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/aigagror/GymGo.git"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pyglet"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GRoapLN_R4rj",
        "outputId": "5329cab6-fb0b-4616-993e-b3161c079bdb"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pyglet\n",
            "  Downloading pyglet-2.0.9-py3-none-any.whl (854 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/854.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.6/854.7 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━\u001b[0m \u001b[32m727.0/854.7 kB\u001b[0m \u001b[31m10.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m854.7/854.7 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pyglet\n",
            "Successfully installed pyglet-2.0.9\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###### Environment Demo"
      ],
      "metadata": {
        "id": "yz6wfo_gXqAz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append('/content/GymGo')"
      ],
      "metadata": {
        "id": "RXpcWT6DSGO-"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import gym\n",
        "\n",
        "DIM = 7\n",
        "\n",
        "go_env = gym.make('gym_go:go-v0', size=DIM, komi=0, reward_method='real')\n",
        "\n",
        "go_env.reset()\n",
        "go_env.render('terminal')\n",
        "\n",
        "first_action = (2,5)\n",
        "state, reward, done, info = go_env.step(first_action)\n",
        "go_env.render('terminal')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pkFFH419SCZG",
        "outputId": "e50a158f-c13b-4b3d-abbd-a85b43795c87"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═╤═╗\n",
            "1\t╟─┼─┼─┼─┼─┼─╢\n",
            "2\t╟─┼─┼─┼─┼─┼─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 0, White Area: 0\n",
            "\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═╤═╗\n",
            "1\t╟─┼─┼─┼─┼─┼─╢\n",
            "2\t╟─┼─┼─┼─┼─○─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 49, White Area: 0\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:20: UserWarning: \u001b[33mWARN: It seems a Box observation space is an image but the `dtype` is not `np.uint8`, actual type: float32. If the Box observation space is not an image, we recommend flattening the observation to have only a 1D vector.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:25: UserWarning: \u001b[33mWARN: It seems a Box observation space is an image but the upper and lower bounds are not in [0, 255]. Generally, CNN policies assume observations are within that range, so you may encounter an issue if the observation values are not.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:190: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `return_info` to return information from the environment resetting.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method is not within the observation space.\u001b[0m\n",
            "  logger.warn(f\"{pre} is not within the observation space.\")\n",
            "/usr/local/lib/python3.10/dist-packages/gym/core.py:43: DeprecationWarning: \u001b[33mWARN: The argument mode in render method is deprecated; use render_mode during environment initialization instead.\n",
            "See here for more information: https://www.gymlibrary.ml/content/api/\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:280: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n",
            "  logger.warn(\n",
            "/content/GymGo/gym_go/gogame.py:283: DeprecationWarning: Please use `label` from the `scipy.ndimage` namespace, the `scipy.ndimage.measurements` namespace is deprecated.\n",
            "  empty_labels, num_empty_areas = ndimage.measurements.label(empties)\n",
            "/content/GymGo/gym_go/state_utils.py:166: DeprecationWarning: Please use `label` from the `scipy.ndimage` namespace, the `scipy.ndimage.measurements` namespace is deprecated.\n",
            "  all_opp_groups, _ = ndimage.measurements.label(state[opponent])\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:227: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
            "  logger.deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:234: UserWarning: \u001b[33mWARN: Expects `done` signal to be a boolean, actual type: <class 'int'>\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
            "  logger.warn(f\"{pre} is not within the observation space.\")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get turn; 0 for white, 1 for black\n",
        "turn = go_env.turn()\n",
        "# Check if game has ended; 0 for not ended, 1 for ended\n",
        "game_ended = go_env.game_ended()\n",
        "# Get the winner of the game; 1 if BLACK is winner, -1 if WHITE is winner\n",
        "winner = go_env.winner()\n",
        "# Get the list of valid moves; given as DIMxDIM numpy array, 0. for not valid move, 1. for valid move\n",
        "valid_moves = go_env.valid_moves()\n",
        "# Get a random action as an index of the valid moves above\n",
        "random_action = go_env.uniform_random_action()\n",
        "# More methods at https://github.com/aigagror/GymGo/blob/master/gym_go/envs/go_env.py\n",
        "\n",
        "print(f\"turn: {turn}\")\n",
        "print(f\"game_ended: {game_ended}\")\n",
        "print(f\"winner: {winner}\")\n",
        "print(f\"valid_moves: {valid_moves}\")\n",
        "print(f\"random_action: {random_action}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4D2FqMFEVQsL",
        "outputId": "fa66d68a-d203-41cf-d966-037efa6c6748"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "turn: 1\n",
            "game_ended: 0\n",
            "winner: 0\n",
            "valid_moves: [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 0. 1. 1. 1. 1.\n",
            " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n",
            " 1. 1.]\n",
            "random_action: 13\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "# Get a numpy array of all valid actions\n",
        "valid_moves = go_env.valid_moves()\n",
        "valid_move_idcs = np.argwhere(valid_moves).flatten()\n",
        "valid_move_idcs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AdzwteLcbfAw",
        "outputId": "fb941ca9-bcde-460f-ac4e-160067c66834"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16,\n",
              "       17, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,\n",
              "       35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49])"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import copy\n",
        "\n",
        "# Deep-copying the environment\n",
        "copied_env = copy.deepcopy(go_env)\n",
        "print('=== Copied Env ===')\n",
        "copied_env.render('terminal')\n",
        "\n",
        "second_move = (5, 2)\n",
        "copied_env.step(second_move)\n",
        "print('=== Copied Env (After Move) ===')\n",
        "copied_env.render('terminal')\n",
        "print('=== Original Env ===')\n",
        "go_env.render('terminal')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ICX6EJ0pgB55",
        "outputId": "4aaee3ea-90c1-4504-b07b-bbf5c7ea5de9"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "=== Copied Env ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═╤═╗\n",
            "1\t╟─┼─┼─┼─┼─┼─╢\n",
            "2\t╟─┼─┼─┼─┼─○─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 49, White Area: 0\n",
            "\n",
            "=== Copied Env (After Move) ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═╤═╗\n",
            "1\t╟─┼─┼─┼─┼─┼─╢\n",
            "2\t╟─┼─┼─┼─┼─○─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─●─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 1, White Area: 1\n",
            "\n",
            "=== Original Env ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═╤═╗\n",
            "1\t╟─┼─┼─┼─┼─┼─╢\n",
            "2\t╟─┼─┼─┼─┼─○─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 49, White Area: 0\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/gym/core.py:43: DeprecationWarning: \u001b[33mWARN: The argument mode in render method is deprecated; use render_mode during environment initialization instead.\n",
            "See here for more information: https://www.gymlibrary.ml/content/api/\u001b[0m\n",
            "  deprecation(\n",
            "/content/GymGo/gym_go/gogame.py:283: DeprecationWarning: Please use `label` from the `scipy.ndimage` namespace, the `scipy.ndimage.measurements` namespace is deprecated.\n",
            "  empty_labels, num_empty_areas = ndimage.measurements.label(empties)\n",
            "/content/GymGo/gym_go/state_utils.py:166: DeprecationWarning: Please use `label` from the `scipy.ndimage` namespace, the `scipy.ndimage.measurements` namespace is deprecated.\n",
            "  all_opp_groups, _ = ndimage.measurements.label(state[opponent])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###### Model"
      ],
      "metadata": {
        "id": "azu01cK_XusL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def make_go_env():\n",
        "  go_env = gym.make('gym_go:go-v0', size=DIM, komi=0, reward_method='real')\n",
        "  go_env.reset()\n",
        "  return go_env"
      ],
      "metadata": {
        "id": "uK_RitL2hf2K"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_all_moves(go_env):\n",
        "  valid_moves = go_env.valid_moves()\n",
        "  valid_move_idcs = np.argwhere(valid_moves).flatten()\n",
        "  return valid_move_idcs"
      ],
      "metadata": {
        "id": "nEoCkw4CqSbh"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from math import log, sqrt, e, inf\n",
        "import random\n",
        "import copy"
      ],
      "metadata": {
        "id": "-bhMlcCDlC9E"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class node():\n",
        "  def __init__(self):\n",
        "    self.state = make_go_env()\n",
        "    self.action = ''\n",
        "    self.children = set()\n",
        "    self.parent = None\n",
        "    self.N = 0\n",
        "    self.n = 0\n",
        "    self.v = 0\n",
        "  def copy_state(self):\n",
        "    return copy.deepcopy(self.state)\n",
        "  def get_all_moves(self):\n",
        "    valid_moves = self.state.valid_moves()\n",
        "    valid_move_idcs = np.argwhere(valid_moves).flatten()\n",
        "    return valid_move_idcs"
      ],
      "metadata": {
        "id": "wLx7UjPiXxif"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def ucb1(curr_node):\n",
        "    ans = curr_node.v+2*(sqrt(log(curr_node.N+e+(10**-6))/(curr_node.n+(10**-10))))\n",
        "    return ans"
      ],
      "metadata": {
        "id": "WqRo7VvSYRHS"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def rollout(curr_node):\n",
        "  if (curr_node.state.game_ended()):\n",
        "    return (curr_node.state.reward(), curr_node)\n",
        "\n",
        "  # If the game is not over, get list of all possible moves\n",
        "  all_moves = curr_node.get_all_moves()\n",
        "\n",
        "  # Get child node for each possible move\n",
        "  for move in all_moves:\n",
        "    tmp_state = curr_node.copy_state()\n",
        "    tmp_state.step(move)\n",
        "    child = node()\n",
        "    child.state = tmp_state\n",
        "    child.parent = curr_node\n",
        "    curr_node.children.add(child)\n",
        "\n",
        "  # Get a random move\n",
        "  rnd_node = random.choice(list(curr_node.children))\n",
        "\n",
        "  # Uncomment to see the random action tree\n",
        "  # rnd_node.state.render('terminal')\n",
        "\n",
        "  return rollout(rnd_node)"
      ],
      "metadata": {
        "id": "dquOWQ4xa70C"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Test rollout function\n",
        "test_node = node()\n",
        "rollout(test_node)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9lHn0fS8d2Ej",
        "outputId": "049a9708-37e5-4a32-9e58-4dca3f3ab06b"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:20: UserWarning: \u001b[33mWARN: It seems a Box observation space is an image but the `dtype` is not `np.uint8`, actual type: float32. If the Box observation space is not an image, we recommend flattening the observation to have only a 1D vector.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:25: UserWarning: \u001b[33mWARN: It seems a Box observation space is an image but the upper and lower bounds are not in [0, 255]. Generally, CNN policies assume observations are within that range, so you may encounter an issue if the observation values are not.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:190: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `return_info` to return information from the environment resetting.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method is not within the observation space.\u001b[0m\n",
            "  logger.warn(f\"{pre} is not within the observation space.\")\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:227: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
            "  logger.deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:234: UserWarning: \u001b[33mWARN: Expects `done` signal to be a boolean, actual type: <class 'int'>\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
            "  logger.warn(f\"{pre} is not within the observation space.\")\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(-1.0, <__main__.node at 0x7e4a64b57f40>)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def expand(curr_node, white):\n",
        "  if (len(curr_node.children) == 0):\n",
        "    return curr_node\n",
        "  max_ucb = -inf\n",
        "  if (white):\n",
        "    idx = -1\n",
        "    max_ucb = -inf\n",
        "    sel_child = None\n",
        "    for i in curr_node.children:\n",
        "      tmp = ucb1(i)\n",
        "      if (tmp > max_ucb):\n",
        "        idx = i\n",
        "        max_ucb = tmp\n",
        "        sel_child = i\n",
        "\n",
        "    return expand(sel_child, 0)\n",
        "\n",
        "  else:\n",
        "    idx = -1\n",
        "    min_ucb = inf\n",
        "    sel_child = None\n",
        "    for i in curr_node.children:\n",
        "      tmp = ucb1(i)\n",
        "      if (tmp < min_ucb):\n",
        "        idx = i\n",
        "        min_ucb = tmp\n",
        "        sel_child = i\n",
        "\n",
        "    return expand(sel_child, 1)"
      ],
      "metadata": {
        "id": "3t4gXw0tjV-I"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Test expand function\n",
        "expand(test_node, 1).state.render('terminal')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U6QAbma_jjvi",
        "outputId": "6a0f63d6-6582-4c40-9b3a-d2e5e0cd6709"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═○═╤═╗\n",
            "1\t╟─┼─┼─┼─┼─┼─╢\n",
            "2\t╟─┼─┼─┼─┼─┼─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 49, White Area: 0\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:280: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n",
            "  logger.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def rollback(curr_node, reward):\n",
        "  curr_node.n += 1\n",
        "  curr_node.v += reward\n",
        "  while (curr_node.parent != None):\n",
        "    curr_node.N += 1\n",
        "    curr_node = curr_node.parent\n",
        "  return curr_node"
      ],
      "metadata": {
        "id": "9PVvMEMdj2Q0"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def mcts_pred(curr_node, over, white, iterations=10):\n",
        "  if (over):\n",
        "    return -1\n",
        "\n",
        "  all_moves = curr_node.get_all_moves()\n",
        "  map_state_move = dict()\n",
        "\n",
        "  for move in all_moves:\n",
        "    tmp_state = curr_node.copy_state()\n",
        "    tmp_state.step(move)\n",
        "    child = node()\n",
        "    child.state = tmp_state\n",
        "    child.parent = curr_node\n",
        "    curr_node.children.add(child)\n",
        "    map_state_move[child] = move\n",
        "\n",
        "  # Expands the tree\n",
        "  while (iterations > 0):\n",
        "    if (white):\n",
        "      idx = -1\n",
        "      max_ucb = -inf\n",
        "      sel_child = None\n",
        "      # Select best child node\n",
        "      for i in curr_node.children:\n",
        "        tmp = ucb1(i)\n",
        "        if (tmp > max_ucb):\n",
        "          idx = i\n",
        "          max_ucb = tmp\n",
        "          sel_child = i\n",
        "      # Go down the tree until leaf node and play until the end of the game\n",
        "      ex_child = expand(sel_child, 0)\n",
        "      reward, state = rollout(ex_child)\n",
        "      curr_node = rollback(state, reward)\n",
        "      iterations -= 1\n",
        "\n",
        "    else:\n",
        "      idx = -1\n",
        "      min_ucb = inf\n",
        "      sel_child = None\n",
        "      for i in curr_node.children:\n",
        "        tmp = ucb1(i)\n",
        "        if (tmp < min_ucb):\n",
        "          idx = i\n",
        "          min_ucb = tmp\n",
        "          sel_child = i\n",
        "\n",
        "      ex_child = expand(sel_child, 1)\n",
        "      reward, state = rollout(ex_child)\n",
        "      curr_node = rollback(state, reward)\n",
        "      iterations -= 1\n",
        "\n",
        "  # Selects the most promising child for the next move\n",
        "  if (white):\n",
        "    mx = -inf\n",
        "    idx = -1\n",
        "    selected_move = ''\n",
        "    for i in (curr_node.children):\n",
        "      tmp = ucb1(i)\n",
        "      if (tmp > mx):\n",
        "        mx = tmp\n",
        "        selected_move = map_state_move[i]\n",
        "    return selected_move\n",
        "\n",
        "  else:\n",
        "    mn = inf\n",
        "    idx = -1\n",
        "    selected_move = ''\n",
        "    for i in (curr_node.children):\n",
        "      tmp = ucb1(i)\n",
        "      if (tmp < mn):\n",
        "        mn = tmp\n",
        "        selected_move = map_state_move[i]\n",
        "    return selected_move"
      ],
      "metadata": {
        "id": "whBK1kDRj4kr"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_node2 = node()\n",
        "mcts_pred(test_node2, 0, 0, 10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t7AK7R1hmAJT",
        "outputId": "58781f44-3774-4089-8a16-e046cf1fc6ff"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:20: UserWarning: \u001b[33mWARN: It seems a Box observation space is an image but the `dtype` is not `np.uint8`, actual type: float32. If the Box observation space is not an image, we recommend flattening the observation to have only a 1D vector.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:25: UserWarning: \u001b[33mWARN: It seems a Box observation space is an image but the upper and lower bounds are not in [0, 255]. Generally, CNN policies assume observations are within that range, so you may encounter an issue if the observation values are not.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:190: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `return_info` to return information from the environment resetting.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method is not within the observation space.\u001b[0m\n",
            "  logger.warn(f\"{pre} is not within the observation space.\")\n",
            "/content/GymGo/gym_go/state_utils.py:166: DeprecationWarning: Please use `label` from the `scipy.ndimage` namespace, the `scipy.ndimage.measurements` namespace is deprecated.\n",
            "  all_opp_groups, _ = ndimage.measurements.label(state[opponent])\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:227: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
            "  logger.deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:234: UserWarning: \u001b[33mWARN: Expects `done` signal to be a boolean, actual type: <class 'int'>\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
            "  logger.warn(f\"{pre} is not within the observation space.\")\n",
            "/content/GymGo/gym_go/gogame.py:283: DeprecationWarning: Please use `label` from the `scipy.ndimage` namespace, the `scipy.ndimage.measurements` namespace is deprecated.\n",
            "  empty_labels, num_empty_areas = ndimage.measurements.label(empties)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "42"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get Human Move\n",
        "def get_human_move(go_env):\n",
        "  action = -1\n",
        "  all_moves = get_all_moves(go_env)\n",
        "\n",
        "  while True:\n",
        "    print(\"Enter y: -1, x: -1 for pass\")\n",
        "    y = input('y: ')\n",
        "    x = input('x: ')\n",
        "    try:\n",
        "      x = int(x)\n",
        "      y = int(y)\n",
        "      if (x == -1 and y == -1):\n",
        "        return DIM * DIM\n",
        "      action = (y * 7) + x\n",
        "      if (action not in all_moves):\n",
        "        raise ValueError\n",
        "      break\n",
        "    except ValueError:\n",
        "      print('Invalid move')\n",
        "  return action"
      ],
      "metadata": {
        "id": "STyzPEgXmWZt"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_game = make_go_env()\n",
        "\n",
        "get_human_move(test_game)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OUYEmHc8mvej",
        "outputId": "e987e9b7-5bdc-4a2a-b0ea-b76bd690c698"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:20: UserWarning: \u001b[33mWARN: It seems a Box observation space is an image but the `dtype` is not `np.uint8`, actual type: float32. If the Box observation space is not an image, we recommend flattening the observation to have only a 1D vector.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:25: UserWarning: \u001b[33mWARN: It seems a Box observation space is an image but the upper and lower bounds are not in [0, 255]. Generally, CNN policies assume observations are within that range, so you may encounter an issue if the observation values are not.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:190: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `return_info` to return information from the environment resetting.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
            "  logger.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method is not within the observation space.\u001b[0m\n",
            "  logger.warn(f\"{pre} is not within the observation space.\")\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter y: -1, x: -1 for pass\n",
            "Enter y: -2, x: -2 for resign\n",
            "y: 2\n",
            "x: 1\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "15"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "game = make_go_env()\n",
        "white = 1\n",
        "\n",
        "while (not game.game_ended()):\n",
        "\n",
        "    if white:  # if it's white's turn, let the agent play\n",
        "        print(\"=== Agent's Turn ===\")\n",
        "        root = node()\n",
        "        root.state = game\n",
        "        result = mcts_pred(root, game.game_ended(), white)\n",
        "        game.step(result)\n",
        "        game.render()\n",
        "    else:  # if it's black's turn, let the human play\n",
        "        print(\"=== Player's Turn ===\")\n",
        "        result = get_human_move(game)\n",
        "        game.step(result)\n",
        "        game.render()\n",
        "\n",
        "    white ^= 1\n",
        "\n",
        "winner = game.winner()\n",
        "if winner == 1:\n",
        "    print(\"Black wins!\")\n",
        "elif winner == -1:\n",
        "    print(\"White wins!\")\n",
        "else:\n",
        "    print(\"Draw!\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "bji-t0iLoop_",
        "outputId": "c3fa5b23-7cb9-4b94-d1aa-2ce8cd6ccd0d"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "=== Agent's Turn ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═╤═╗\n",
            "1\t╟─┼─┼─┼─┼─┼─╢\n",
            "2\t╟─○─┼─┼─┼─┼─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 49, White Area: 0\n",
            "\n",
            "=== Player's Turn ===\n",
            "Enter y: -1, x: -1 for pass\n",
            "y: 2\n",
            "x: 5\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═╤═╗\n",
            "1\t╟─┼─┼─┼─┼─┼─╢\n",
            "2\t╟─○─┼─┼─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 1, White Area: 1\n",
            "\n",
            "=== Agent's Turn ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═╤═╗\n",
            "1\t╟─┼─┼─┼─┼─┼─○\n",
            "2\t╟─○─┼─┼─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 2, White Area: 1\n",
            "\n",
            "=== Player's Turn ===\n",
            "Enter y: -1, x: -1 for pass\n",
            "y: 1\n",
            "x: 4\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═╤═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─┼─┼─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 2, White Area: 2\n",
            "\n",
            "=== Agent's Turn ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─┼─┼─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─╢\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 4, White Area: 2\n",
            "\n",
            "=== Player's Turn ===\n",
            "Enter y: -1, x: -1 for pass\n",
            "y: 3\n",
            "x: 6\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─┼─┼─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 4, White Area: 3\n",
            "\n",
            "=== Agent's Turn ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═╤═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─┼─○─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 5, White Area: 3\n",
            "\n",
            "=== Player's Turn ===\n",
            "Enter y: -1, x: -1 for pass\n",
            "y: 0\n",
            "x: 3\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═●═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─┼─○─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─┼─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 5, White Area: 4\n",
            "\n",
            "=== Agent's Turn ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═●═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─┼─○─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─○─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 6, White Area: 4\n",
            "\n",
            "=== Player's Turn ===\n",
            "Enter y: -1, x: -1 for pass\n",
            "y: 2\n",
            "x: 2\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═●═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─●─○─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─○─┼─┼─┼─┼─╢\n",
            "6\t╚═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 6, White Area: 5\n",
            "\n",
            "=== Agent's Turn ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═●═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─●─○─┼─●─╢\n",
            "3\t╟─┼─┼─┼─┼─┼─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─○─┼─┼─┼─┼─╢\n",
            "6\t○═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 7, White Area: 5\n",
            "\n",
            "=== Player's Turn ===\n",
            "Enter y: -1, x: -1 for pass\n",
            "y: 3\n",
            "x: 3\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═●═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─●─○─┼─●─╢\n",
            "3\t╟─┼─┼─●─┼─┼─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─○─┼─┼─┼─┼─╢\n",
            "6\t○═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 7, White Area: 6\n",
            "\n",
            "=== Agent's Turn ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═●═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─●─○─┼─●─○\n",
            "3\t╟─┼─┼─●─┼─┼─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─○─┼─┼─┼─┼─╢\n",
            "6\t○═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 8, White Area: 6\n",
            "\n",
            "=== Player's Turn ===\n",
            "Enter y: -1, x: -1 for pass\n",
            "y: 2\n",
            "x: 4\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═●═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─●─○─●─●─○\n",
            "3\t╟─┼─┼─●─┼─┼─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─○─┼─┼─┼─┼─╢\n",
            "6\t○═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 8, White Area: 7\n",
            "\n",
            "=== Agent's Turn ===\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═●═╤═○═╗\n",
            "1\t╟─┼─┼─┼─●─┼─○\n",
            "2\t╟─○─●─○─●─●─○\n",
            "3\t╟─┼─┼─●─┼─○─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─○─┼─┼─┼─┼─╢\n",
            "6\t○═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: WHITE, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 9, White Area: 7\n",
            "\n",
            "=== Player's Turn ===\n",
            "Enter y: -1, x: -1 for pass\n",
            "y: 1\n",
            "x: 3\n",
            "\t0 1 2 3 4 5 6 \n",
            "0\t╔═╤═╤═●═╤═○═╗\n",
            "1\t╟─┼─┼─●─●─┼─○\n",
            "2\t╟─○─●─┼─●─●─○\n",
            "3\t╟─┼─┼─●─┼─○─●\n",
            "4\t╟─┼─┼─┼─┼─┼─╢\n",
            "5\t╟─○─┼─┼─┼─┼─╢\n",
            "6\t○═╧═╧═╧═╧═╧═╝\n",
            "\tTurn: BLACK, Game State (ONGOING|PASSED|END): ONGOING\n",
            "\tBlack Area: 8, White Area: 9\n",
            "\n",
            "=== Agent's Turn ===\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-28-6e4dfc7a3fe0>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mroot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mroot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgame\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmcts_pred\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgame_ended\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwhite\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m         \u001b[0mgame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mgame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-19-2d5c6b8397d1>\u001b[0m in \u001b[0;36mmcts_pred\u001b[0;34m(curr_node, over, white, iterations)\u001b[0m\n\u001b[1;32m     30\u001b[0m       \u001b[0;31m# Go down the tree until leaf node and play until the end of the game\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m       \u001b[0mex_child\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msel_child\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m       \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mex_child\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m       \u001b[0mcurr_node\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrollback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m       \u001b[0miterations\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     21\u001b[0m   \u001b[0;31m# rnd_node.state.render('terminal')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnd_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-b0506d5c2c65>\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(curr_node)\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mtmp_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcurr_node\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mtmp_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmove\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m     \u001b[0mchild\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m     \u001b[0mchild\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mchild\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcurr_node\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-12-d0b99982878f>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_go_env\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-9-93fe9879d2e9>\u001b[0m in \u001b[0;36mmake_go_env\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmake_go_env\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m   \u001b[0mgo_env\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgym\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'gym_go:go-v0'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDIM\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkomi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward_method\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'real'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m   \u001b[0mgo_env\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mgo_env\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(id, max_episode_steps, autoreset, new_step_api, disable_env_checker, **kwargs)\u001b[0m\n\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    676\u001b[0m     \u001b[0;31m# Copies the environment creation specification and kwargs to add to the environment specification details\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m     \u001b[0mspec_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    678\u001b[0m     \u001b[0mspec_\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    679\u001b[0m     \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munwrapped\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspec_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/copy.py\u001b[0m in \u001b[0;36mdeepcopy\u001b[0;34m(x, memo, _nil)\u001b[0m\n\u001b[1;32m    149\u001b[0m             \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_deepcopy_atomic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m             \u001b[0mcopier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"__deepcopy__\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    152\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mcopier\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m                 \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmemo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    }
  ]
}