nb_ex10_2_rl_qlearn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ea120714-3fd5-4255-acdc-ff6b9f56f670",
   "metadata": {},
   "source": [
    "## 10.3 Q-Learning을 이용하는 강화학습\n",
    "강화학습의 대표적인 학습 방법인 Q-learning의 사례를 알아봅니다. \n",
    "\n",
    "### 10.3.1 패키지 부르기 및 초기화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "71f733b0-0f3c-44f2-8885-9795739cd4a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c82deb68-fc6c-4b07-b813-9abd8b5d8abb",
   "metadata": {},
   "outputs": [],
   "source": [
    "flake = gym.make(\"FrozenLake-v1\", is_slippery=False)\n",
    "Q = np.zeros((flake.observation_space.n, flake.action_space.n))\n",
    "PI = np.argmax(Q,axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d79c8682-88c6-4596-8ab3-b0a374d79159",
   "metadata": {},
   "source": [
    "### 10.3.2 주어진 정책에 따른 시도 함수 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "afad72c6-b145-4295-b0fc-3d6cc1653d06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>S</th>\n",
       "      <th>S:(x,y)</th>\n",
       "      <th>R</th>\n",
       "      <th>done</th>\n",
       "      <th>A</th>\n",
       "      <th>A:name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>Left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>Left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>(0, 1)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>Left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>(0, 1)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>Left</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>4</td>\n",
       "      <td>(0, 1)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>0</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>4</td>\n",
       "      <td>(0, 1)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>8</td>\n",
       "      <td>(0, 2)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>12</td>\n",
       "      <td>(0, 3)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>75 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     S S:(x,y)    R   done  A A:name\n",
       "k                                   \n",
       "0    0  (0, 0)  0.0  False  0   Left\n",
       "1    0  (0, 0)  0.0  False  0   Left\n",
       "2    0  (0, 0)  0.0  False  1   Down\n",
       "3    4  (0, 1)  0.0  False  0   Left\n",
       "4    4  (0, 1)  0.0  False  0   Left\n",
       "..  ..     ...  ...    ... ..    ...\n",
       "70   4  (0, 1)  0.0  False  3     Up\n",
       "71   0  (0, 0)  0.0  False  1   Down\n",
       "72   4  (0, 1)  0.0  False  1   Down\n",
       "73   8  (0, 2)  0.0  False  1   Down\n",
       "74  12  (0, 3)  0.0   True  0       \n",
       "\n",
       "[75 rows x 6 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# %% Q-learning\n",
    "def run_with_PI_exploration(PI=None, exploration=0.2, N_Iter = 100, render_flag=False):\n",
    "    \"\"\"\n",
    "    Return buff_df if done, otherwise return None \n",
    "    \"\"\"\n",
    "    if PI is None:\n",
    "        # No PI, action will be determined fully randomly\n",
    "        exploration = 1.0\n",
    "\n",
    "    s = flake.reset()\n",
    "    if render_flag: flake.render()\n",
    "    buff_df = pd.DataFrame({\"S\":[s],\"S:(x,y)\":[(0,0)], \n",
    "                \"R\":[0.0], \"done\":[False], \n",
    "                \"A\":[0], \"A:name\": [\"\"]})\n",
    "    buff_df.index.name = 'k'\n",
    "\n",
    "    Actions = [\"Left\", \"Down\", \"Right\", \"Up\"]\n",
    "    rand_buff =  np.random.rand(N_Iter)\n",
    "    for iter in range(N_Iter):\n",
    "        # if np.random.rand() <= exploration:\n",
    "        if rand_buff[iter] <= exploration:\n",
    "            a_k = flake.action_space.sample()\n",
    "        else:\n",
    "            a_k = PI[s]\n",
    "        buff_df.loc[iter,'A':\"A:name\"] = (a_k, Actions[a_k])\n",
    "        s, r, done, info = flake.step(a_k)\n",
    "        if render_flag: flake.render()\n",
    "        new_df = pd.DataFrame({\"S\":[s], \"S:(x,y)\":[(s%4,s//4)],\n",
    "                                \"R\":[r], \"done\":[done], \n",
    "                                \"A\":[0], \"A:name\": [\"\"]})\n",
    "        buff_df = buff_df.append(new_df, ignore_index=True)\n",
    "        buff_df.index.name = 'k'\n",
    "        if done:\n",
    "            return buff_df\n",
    "    return None\n",
    "\n",
    "buff_df = run_with_PI_exploration(PI, exploration=0.2, N_Iter=100)\n",
    "buff_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe91713d-1dd8-4857-8fb4-1a62d366f941",
   "metadata": {},
   "source": [
    "### 10.3.3 Q-learning 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f8c8b35b-b317-4485-815f-2696a79b0613",
   "metadata": {},
   "outputs": [],
   "source": [
    "def q_learning(N_epoch=1000, N_Iter=100, learning_rate=0.01, exploration=0.2, trace_flag=False):\n",
    "    \"\"\"\n",
    "    return PI 1-D integer array if trace_flag is False \n",
    "    else return PI_array 2-D integer array \n",
    "    \"\"\"\n",
    "    PI_list = []\n",
    "    Q = np.zeros((flake.observation_space.n, flake.action_space.n))\n",
    "    for e in range(N_epoch):\n",
    "        PI = np.argmax(Q,axis=1)\n",
    "        buff_df = run_with_PI_exploration(PI, exploration=exploration, N_Iter=N_Iter)\n",
    "        #buff_df = get_g(N_Iter=100)\n",
    "        #print(np.sum(buff_df.R))\n",
    "        for k in range(len(buff_df)-1):\n",
    "            S, A = buff_df.S[k], buff_df.A[k]\n",
    "            S_next, R_next = buff_df.S[k+1], buff_df.R[k+1]\n",
    "            # print(k, S, A, S_next, R_next)\n",
    "            Q_new = R_next + np.max(Q[S_next])\n",
    "            Q[S,A] += learning_rate * (Q_new - Q[S,A])\n",
    "\n",
    "        PI = np.argmax(Q,axis=1)\n",
    "        PI_list.append(PI)\n",
    "    if trace_flag:\n",
    "        return np.array(PI_list)\n",
    "    else:\n",
    "        return PI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8bd26b0-8896-43ed-96ae-47f263d25321",
   "metadata": {},
   "source": [
    "### 10.3.4 Q-learning 실행하기"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bade2517-73d0-4c32-8129-2091b4b7f481",
   "metadata": {},
   "source": [
    "- exploration=0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ae29882-1616-46f7-b066-6c619a94aac1",
   "metadata": {},
   "outputs": [],
   "source": [
    "PI_array = q_learning(N_epoch=1000, N_Iter=100, learning_rate=0.01, exploration=0.8, trace_flag=True)\n",
    "print(PI_array)\n",
    "plt.plot(PI_array)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de092eef-1ea1-4dd5-8e8b-b08dc3f5dff7",
   "metadata": {},
   "source": [
    "- exploration=0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1fc07a0-2ba4-49d0-8e5a-eb470b736c34",
   "metadata": {},
   "outputs": [],
   "source": [
    "PI_array = q_learning(N_epoch=1000, N_Iter=100, learning_rate=0.01, exploration=0.8, trace_flag=True)\n",
    "print(PI_array[-1])\n",
    "plt.plot(PI_array)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35d2f251-4cd9-41b6-a2bb-5acfeccf28b6",
   "metadata": {},
   "source": [
    "---\n",
    "### 10.3.5 전체 코드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "483b3939-00cf-4842-a947-8e983263ecd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " ...\n",
      " [1 2 1 ... 2 2 0]\n",
      " [1 2 1 ... 2 2 0]\n",
      " [1 2 1 ... 2 2 0]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fe64d615c70>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d615d60>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d615e20>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d615ee0>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d615fa0>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d6210a0>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d621160>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d621220>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d6212e0>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d6213a0>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d68b850>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d6214f0>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d6215b0>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d621670>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d621730>,\n",
       " <matplotlib.lines.Line2D at 0x7fe64d6217f0>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAfb0lEQVR4nO3dbZAdV33n8e9PTwaMgwEJcCQLmSothUgh8E4JG1N+gEBknpSnFzIECAWrImXvhuwWW2azZdjlxWaXrewGbFBUoPWywTapgEGVCGw2y2ISAvGYNX5ERIgHK3KiwQYMtmN5pP++uD2jnjt9b/eduX275/TvU6Wae/vpnh7D75455/Q5igjMzCxdq5ougJmZ1ctBb2aWOAe9mVniHPRmZolz0JuZJW5N0wUosn79+tiyZUvTxTAzWzHuuOOOH0XEhqJ9rQz6LVu2MD093XQxzMxWDEk/GLTPTTdmZolz0JuZJc5Bb2aWOAe9mVniHPRmZokrDXpJ50r6sqT7Jd0r6XcLjpGkD0s6LOkuSefn9u2UdCjbd/W4b8DMzIarUqOfBf5NRLwIuAC4UtK2vmMuB7Zm//YAHwOQtBq4Ltu/Dbii4FwzM6tR6Tj6iHgQeDB7/TNJ9wMbgftyh+0CPhm9OY+/LulsSecAW4DDEXEEQNJN2bH5c2tx5M4ZZn74s7FeM2Znefzuu+DJWQBOPPljTp58rO8g+NHJdcy2rVXs5AmIU02XwsyGWLX2FL/939479uuO9MCUpC3Ay4Bv9O3aCDyQe38021a0/eUDrr2H3l8DbN68eZRiFfrKDYd47JEToGVf6rQA4uzchvVjvLiZdd2ax39ez3WrHijp6cBngPdExCP9uwtOiSHbF2+M2AfsA5iamlr2aigRwS9dvJFL3vzC5V5q3s+/+lUe+Bd72PLpm3jq9u185bbzed7zfpUX/rNr5o858tgTvOIb93PdizbzG8971tg+e9k+uAEuvBJ++QNNl8TMJqxS0EtaSy/kPxURny045Chwbu79JuAYsG7A9trVsnDWoosu/pAo/h5rnlcSM+usKqNuBHwCuD8i/nDAYQeAt2Wjby4Afpq17d8ObJV0nqR1wO7s2OT1fm1mZs2rUqO/CHgrcLekO7Nt/w7YDBARe4GDwOuAw8BjwDuyfbOSrgJuAVYD+yPi3nHewFB1ZW0uxNX3ITG/vY3aWSozq1eVUTd/RUlCZKNtrhyw7yC9L4LJaqrpJtvUvkh1041ZV7VsDODK5zg1s7ZJNuiDGHutOuar6zr9XsVNN61TUFYz64Zkg75pzlQza4u0g762tNWA16eHV/Z30rZDG8tkZnVLN+gn0hk7+JD2RWprG5XMrGbpBr2ZmQGpB/3Ye2Pnrqv5DStmHL07Y806K+2gb5Az1czaIumgry1rh1y4tTV6oK2lMrN6JRv09czhVeXJ2LZ2era1XGZWt2SDvintrtGbWRelG/QRNXTGZmPkKzwZ28px9O44MOukdIO+Yc5UM2uLpIO+tlq1hj0ZW7S1LdpZKjOrV7JBX09fbPVpilullYUys0lJNuib4kg1s7YpXXhE0n7gDcDxiPilgv3vBd6Su96LgA0R8bCk7wM/A04CsxExNa6Clxq0NPlyLtk3TXHxk7Fzk5q1yKJym1mXVKnRXw/sHLQzIj4UES+NiJcC7wO+EhEP5w65LNs/uZBvAa8Za2ZtURr0EXEb8HDZcZkrgBuXVaJxaiJrWzt7JbS1VGZWr7G10Ut6Gr2a/2dymwO4VdIdkvaUnL9H0rSk6ZmZmWWXp9bO2FzTzaBRN+3SzlKZ2WSMszP2jcBf9zXbXBQR5wOXA1dKunjQyRGxLyKmImJqw4YNYyzWZLV7eKWZddE4g343fc02EXEs+3kcuBnYMcbPK1Vf2OauPPDJ2BZyv4FZJ40l6CU9A7gE+Hxu25mSzpp7DbwWuGccn1dJHWPH+y457CNalakeR2/WaVWGV94IXAqsl3QUeD+wFiAi9maH/Rpwa0Q8mjv1ucDN2eiTNcANEfHF8RW9ggbStr2zV0JL/84ws5qVBn1EXFHhmOvpDcPMbzsCbF9qwZatzjVj5/NycGdsuyY1a/OXj5nVzU/Gjlmr2+jNrJOSDvq6wjb/MFT/Z7Q66FtZKDOrW7JBX09jRfmkZnPcGWtmbZFs0AON1GDbnalt+vYxs0lJN+hr7Ywd9mRsG5O+jWUys0lJN+gb0uo2ejPrpKSDvrZ2cg1+MnZ+cxujvlUdB2Y2KckGfS1NKH0N8MMejmpVpra748DMapZs0Pc08WTsxD9yBG369jGzSUk36Gup0Fefprhdkdrqbx8zq1m6Qd+Qdga9mXVZ2kFfW9rmn4xdAWvGzmlVx4GZTUq6QV/LOPpJfEgN2t1xYGY1SzfoaWjJ2Plm/DbWnttYJjOrW7JBX++asfmN7ow1s3ZLNuib0s6gN7MuKw16SfslHZdUuAygpEsl/VTSndm/a3L7dko6JOmwpKvHWfBKGkzbVgZ9K5uTzKxuVWr01wM7S475akS8NPv3HwEkrQauAy4HtgFXSNq2nMKOpJYOyGxEjVQwpj5/RMu4M9as00qDPiJuAx5ewrV3AIcj4khEnABuAnYt4TpL1kSHaBS247dFKwtlZjUbVxv9hZK+JekLkl6cbdsIPJA75mi2rZCkPZKmJU3PzMwsu0D1dsaKQa3xXjPWzNpmHEH/TeD5EbEd+AjwuWx7UdINTJyI2BcRUxExtWHDhjEUq1ltinkz67ZlB31EPBIRP89eHwTWSlpPrwZ/bu7QTcCx5X7eStHKfs9WFsrM6rbsoJf0PGWN4ZJ2ZNd8CLgd2CrpPEnrgN3AgeV+XmW1rzBVPNVBKxtJ3Blr1mlryg6QdCNwKbBe0lHg/cBagIjYC/wm8DuSZoHHgd3R65GclXQVcAuwGtgfEffWchct0uq+WDPrpNKgj4grSvZfC1w7YN9B4ODSijYGY07b/JDK04uOrIRJzfwYl1mX+cnYMXOkmlnbJB30Dts+7ow166Qkg37YWq7Lu3D2Mz+OfsCTsa2avdKdsWadlmTQN8mdsWbWNmkG/YKa9zivm2+Bn+t0XUnTFLerVGY2GWkGfYPcSGJmbZN00LepmbwV/Asx66Qkg762WnXMTVOc/5QB4+jblKnujDXrtCSDvkmnO2PblPRm1mVpBn1tNdj8k7Fz24oDvZ0x385SmVm90gz6Bnl8i5m1TdJB36p28jbwL8Ssk5IM+ro7Y6s9GVtXIZbAnbFmnZZk0DdpbvqFNuW8mXVbmkFf0lG65MsuqBkX15LbvWZsm8pkZpNSGvSS9ks6LumeAfvfIumu7N/XJG3P7fu+pLsl3SlpepwFb1SuXWZFrDBlZp1WpUZ/PbBzyP7vAZdExEuADwL7+vZfFhEvjYippRVxGRqswLay7tyqjgMzm5QqK0zdJmnLkP1fy739Or1FwJtV2zD6gs7YQZOatSlT3Rlr1mnjbqN/J/CF3PsAbpV0h6Q9w06UtEfStKTpmZmZMRdrcjxNsZm1TWmNvipJl9EL+lfmNl8UEcckPQf4kqRvR8RtRedHxD6yZp+pqallVUFrm28mV6pBi5tEK1vp21gmM5uUsdToJb0E+DiwKyIemtseEceyn8eBm4Ed4/i85uW+QQaMozcza4tlB72kzcBngbdGxHdy28+UdNbca+C1QOHInRS1summVR0HZjYppU03km4ELgXWSzoKvB9YCxARe4FrgGcDH83WSZ3NRtg8F7g527YGuCEivljDPSxWe2ds/kO8ZqyZtVuVUTdXlOx/F/Cugu1HgO2Lz0icO2PNrGXSfDI200Stut3PoLazVGZWrySDvr6GirnRPOWLg7dLO0tlZpORZNC3QSvrzm3qNzCziUkz6CfyZOwcrxlrZu2WZtA3yGvGmlnbpB30DWRtu+vO/vIx66Ikg37Q9ARju6604HWRdkVqu79+zKxeSQb9nCaaT1o9vLJVHQdmNilJB/3YFU5T3HcIuUPawp2xZp3moB+zupqNzMyWKu2gd2dsnzb9mWFmk5Jm0Nc2jj77OeTJ2DntGl7Z7q8fM6tXmkGfaaKd3J2xZtY2SQZ9bfXXBe3vwz+lVZnqfgOzTksy6CerbwoET1NsZi3joB+zdq4ZO8dfP2ZdVBr0kvZLOi6pcBlA9XxY0mFJd0k6P7dvp6RD2b6rx1nwoepqqojT0xT7yVgzWymq1OivB3YO2X85sDX7twf4GICk1cB12f5twBWSti2nsKPywiN9WtVxYGaTUmUpwdskbRlyyC7gk9Gr4n5d0tmSzgG2AIezJQWRdFN27H3LLnWJqhX6E7OneP+Be/nJYycqHb99+odcArx6/9/w6BlP5Ykn/oBTdzyN2dn/PX/Mk6th7Vrxplv/klU1VaSfCaxetPXk0GajVfEnxKdFfPov6imUmS3bGQSf+oM3jP26pUFfwUbggdz7o9m2ou0vH3QRSXvo/UXA5s2bx1Cscj946FFu/Nsfcs4znsJZTyn/VWz6+RMAPPSTJ/inM1Zz6tRTiVhFxOyC4854DB45eaqWMgtglTgVRQ0yw2rsq92CY9ZyZ9bUFjCOoC8qWQzZXigi9gH7AKampiYSSXMf8u9fv43Xv+Sc0uMf/uQP+Mevwdt/Yxv/6kXP5qt/tYN/+Oab+fELdnPVr7+43sJmTj76JA9+8Ouc/aYX8PSLNs5vf9WfvoqLN13MB17xgcUn/fgH8EcvgV0fhZe9ZSLlNLP2GEfQHwXOzb3fBBwD1g3Y3hoj99kWTWo26VrygEIPH+3jqrxZl41jeOUB4G3Z6JsLgJ9GxIPA7cBWSedJWgfszo6dnJK/gsa17F8Tnb79hY6I8nK4M9ask0pr9JJuBC4F1ks6CrwfWAsQEXuBg8DrgMPAY8A7sn2zkq4CbqHXd7g/Iu6t4R4WG7ECWzX+5oZULpqkuIn8LPjMgfPr+MlYs06rMurmipL9AVw5YN9Bel8ErVQyFH4w9b2ZZE15QGYH0bKJ1MysLZJ+MrYsf09XdEcLyP7rNhKwRTX60i8cfxGYdVGSQV91GoKRpyvIfzE01RwyqEY/tDxuujHrsiSDvqrRm25Ot9HnaaK/xeJnb9s9x46ZNSnNoB+xSWb0ztiF3bGNNIgspTPWo27MOinNoB/RyMMj88eHJpufwzpjHeRmViDpoK/eGVtNGxpHBv2xUm1Rcn8RmHVRkkFfNcBjQJv7QKey41edbrrpDWr08Eoza68kg76qUZuuB01BPNnO2LkyLC60m27MrEing35O5Xxc8GRsUw05A9YqDHfGmlmxTgf96Rp6tQCcP37RE1PtaLoxMyuSdNCXd8auwHAc0H5ULehdozfroiSDvmqAjzoDwul1YlnQ49uGFpHhs1euwC80MxubJIO+qhjQ3F1mQVPPpMfR50qxeEsLvnHMrHXSDvqKCVx5tMqCvxQaejJ2wLfT0OGV7ow167S0g77UaOPoB3XGqonxlQU8vNLMirQjoRqyEvtiB3bGrsibMbNJqBT0knZKOiTpsKSrC/a/V9Kd2b97JJ2U9Kxs3/cl3Z3tmx73DRSq/GRsT/WWm7mlB/tq9C35uhzcRu8vAbMuq7KU4GrgOuA19BYCv13SgYi4b+6YiPgQ8KHs+DcCvxcRD+cuc1lE/GisJR+D083dy1l4ZLJdoKdHChVMU+yWGzMrUKUuugM4HBFHIuIEcBOwa8jxVwA3jqNwSzVq3+OSnozNN5VM9IGp4n6FCHfGmlmxKkG/EXgg9/5otm0RSU8DdgKfyW0O4FZJd0jaM+hDJO2RNC1pemZmpkKxlm/Udu2Y/7lweGUjlvTAlJl1UZWgL0qyQanyRuCv+5ptLoqI84HLgSslXVx0YkTsi4ipiJjasGFDhWIt36BJysosmtSsLfPRl96Ja/RmXVQl6I8C5+bebwKODTh2N33NNhFxLPt5HLiZXlNQzSo+GTti0p/ujF34GW1pEfGTsWZWpErQ3w5slXSepHX0wvxA/0GSngFcAnw+t+1MSWfNvQZeC9wzjoKP0+idsX2jbpqoKY8ye6WZdVrpqJuImJV0FXALsBrYHxH3Snp3tn9vduivAbdGxKO5058L3JwF4xrghoj44jhvYJiyB4jmFx4ZdSXB/jVjJzm8csCfIZXa6Nvyp4eZTVRp0ANExEHgYN+2vX3vrweu79t2BNi+rBIuQeU+1lGXElwweiVbYSoaqkeP0hnrh6nMOq0lj/rUpGya4mqHLT6+fzr6EYq0bIM6Y4fOXjnHNXqzLko76EucrqAvc1KzljSJ+MlYMyvS6aCfM3JOL1pgqvnO2KBKjd7MuqjTQT/fGVv5eIqPn/yDsYUfWtpb4C8Cs05KMugrrzA1aotGbsRLjDoIf1xi8Uih0vt1Z6xZpyUZ9HNK14yteNzi6/aPo28HPxlrZkXSDPqq0xSPWCuPU/mmnvw4+mYDdH5o5cBiuEZv1mVpBv2IllWjn/Q4+oLvpvmpGVxjN7MCiQd92ZOxK9npe6s8c6U7Y806KfGgL1HW4rHo8EFNN+MsVFkhFhf6dLlK5qM3s05KOujLO2OLlwYsvW5fm3xbKsp+MtbMiiQZ9FUrsAWV4+HHnyquOavpX2Ppkoiu0Zt1WZJBP6rlDa/UhJtu5j82t2lpf5mYWTekHfRlTTejzl659JKMn9wZa2bVJBr0FZ+MzX5WHpY4aHHwCSr6XD8Za2bDVAp6STslHZJ0WNLVBfsvlfRTSXdm/66pem6dyuI7CqYTGHp8wYUj2tNk4idjzaxI6cIjklYD1wGvobd+7O2SDkTEfX2HfjUi3rDEc8eqcmfsEi+8aHhlE/k5Uhu9a/RmXValRr8DOBwRRyLiBHATsKvi9Zdz7sSMEtSnVDDmZpJBXzBSyE/GmtkwVYJ+I/BA7v3RbFu/CyV9S9IXJL14xHORtEfStKTpmZmZCsWqoGzN2EQ6Y2s9x8xWvCpBX5QO/Zn3TeD5EbEd+AjwuRHO7W2M2BcRUxExtWHDhgrFGqJyIo9YE15Qc15BK0y5M9as06oE/VHg3Nz7TcCx/AER8UhE/Dx7fRBYK2l9lXObtGCt76rnLDpekw36olE3lcfRt+MLycwmq0rQ3w5slXSepHXAbuBA/gBJz1OWMpJ2ZNd9qMq5TRp1PvrID6/M1+jHW6xqRll4pF2NTmY2YaWjbiJiVtJVwC3AamB/RNwr6d3Z/r3AbwK/I2kWeBzYHb30KTy3pnuZjP5vhUlnaMHnVX5gysw6qTToYb455mDftr2519cC11Y9d1JKJzUrnSNmwHX737dlCgSvGWtmBZJ8MrbymrHzbdujXVe9N7k9DQ+kn9sy6EZc4TfrtCSDfl7lGn11sajzdcKTmhWkdvVx9K7Rm3VR2kFfYjmdsTTUGTtsmVs/GWtmRTod9EsRi7K0+VpyUxOsmdnKkHTQlzVlxLDq8dDrLrRq1QR/jQVj/z1NsZkNk2TQj1rBHbXpZtH5o33cmBR0xvrJWDMrkGTQV7WkzlhpYW06YsIV5eU8GWtmXZRm0FdskRk5IHOjW+bOXTwlQs2KxtGXjrpZWhOVmaUhzaDvgqW00ZtZJ3U66JfSdLO43qzWzANf/mTsZMphZu2SZNBXreGOOntl0ZOxMb9hQoYM/h/8ZKxr/GZdlmTQVzXy4uD0xtEvmqZ4nIVaAo+jN7Nh0gz6+Zp61XH0FS+74PiGwnXIfPRDTsp+Nv2VZGZNSDPoRzTqqJkF67Uy2WGN85FeNOrGwyvNrECng34pdfLob6qJ9tSTPU2xmRXpdNAXTScw9PCBa8aOuVxDCzH3mcptKhlH7zZ8s06rFPSSdko6JOmwpKsL9r9F0l3Zv69J2p7b931Jd0u6U9L0OAu/XEt5onTRk7FtGl7pGruZFShdYUrSauA64DX0Fvu+XdKBiLgvd9j3gEsi4seSLgf2AS/P7b8sIn40xnIPtZRFv0e6MA0+pFTUGVt5zVh/EZh1UZUa/Q7gcEQciYgTwE3ArvwBEfG1iPhx9vbrwKbxFrMeoz4wNSguG58CoepSgmbWSVWCfiPwQO790WzbIO8EvpB7H8Ctku6QtGfQSZL2SJqWND0zM1OhWBWUznUz99nLu+aKiVc37Zh1UpXFwYvSobCtQNJl9IL+lbnNF0XEMUnPAb4k6dsRcduiC0bso9fkw9TU1DLbRUZ8MrZiVA9cM7bh/Czta3BnrFmnVanRHwXOzb3fBBzrP0jSS4CPA7si4qG57RFxLPt5HLiZXlNQK4y6OHjvnIVrxkZMuDO24M+Q6mvGmlkXVQn624Gtks6TtA7YDRzIHyBpM/BZ4K0R8Z3c9jMlnTX3GngtcM+4Cj/IqDX1qtSGJ2OHzEdffo6/CMy6qLTpJiJmJV0F3AKsBvZHxL2S3p3t3wtcAzwb+GhW252NiCngucDN2bY1wA0R8cVa7mQJVmRn7PyHFmxyG7yZFajSRk9EHAQO9m3bm3v9LuBdBecdAbb3b5+Yip2xo1R0Qwv/Tlj0pGzdiirvVf+C8ReBWSel+WRs1VaVEdu2F3TGzlHzNenSpht3xpp1WppBX9FKjL8o6oxdkXdiZpPS6aCfM9qom4XHR0y6RWTxSKHqs1e66casizod9CN3xhYsDj7K+XUrXxzczLoo6aAvq+AuaR73os7YlTIFgjtjzTopyaCvunLUyKPLBz0Z23Cd3k/GmtkwSQZ9VaPmXyvicsjwSjOzImkHfcWmitGnQMi/b6YztmjUTfkwUTfdmHVR2kFf4nTTTeUlpgrOblF8DiyIq/xmXZZ00JcFcIw47CaAUN/hock+MFXUGVv1wS93xpp1UpJBX1vfYysmNVvMT8aa2TBJBn1VS11yMF9zDibcdFOQ2V5hysyGSTvoK+beKPHYvzg4kx5HP/+xiz/UT8aaWZE0g75iS0Xp+PMB11Xu3F6NfnIBWjT2v7yN3k03Zl2WZtBXtCLnox9SaHfGmlmRJIO+6myOI9dz41TudUO15CFt9IPPcY3erMsqBb2knZIOSTos6eqC/ZL04Wz/XZLOr3puk0btjC1sNhlngZZo1GGiZtYtpUEvaTVwHXA5sA24QtK2vsMuB7Zm//YAHxvh3NpUbXsfpY291xm78NHY1nTG+slYMyugsgnAJF0IfCAifiV7/z6AiPhPuWP+GPi/EXFj9v4QcCmwpezcIlNTUzE9PT3yzXzkze/hZDwCiFOr1vKUf/p7Vs8+Vn5ixfxbc7JXgz/6i4Ci1ykbqn6BsWrD3xJmNk6rnnKKa/7oy0s6V9Id2Vrdi1RZM3Yj8EDu/VHg5RWO2Vjx3LlC7qH31wCbN2+uUKwij0M8AcCqk49zctUTnFq7xEsVeHINPLEuexOaH71uZtZmVYK+KMn6q5ODjqlybm9jxD5gH/Rq9BXKtci/vOGPl3KamVnSqgT9UeDc3PtNwLGKx6yrcK6ZmdWoyqib24Gtks6TtA7YDRzoO+YA8LZs9M0FwE8j4sGK55qZWY1Ka/QRMSvpKuAWYDWwPyLulfTubP9e4CDwOuAw8BjwjmHn1nInZmZWqHTUTROWOurGzKyrho26SfLJWDMzO81Bb2aWOAe9mVniHPRmZolrZWespBngB0s8fT3wozEWZyXwPXeD7zl9y7nf50fEhqIdrQz65ZA0PajnOVW+527wPaevrvt1042ZWeIc9GZmiUsx6Pc1XYAG+J67wfecvlruN7k2ejMzWyjFGr2ZmeU46M3MEpdM0Ld5EfLlkHSupC9Lul/SvZJ+N9v+LElfkvR32c9n5s55X/Z7OCTpV5or/fJIWi3p/0n68+x90vcs6WxJfybp29l/7ws7cM+/l/3v+h5JN0p6Smr3LGm/pOOS7sltG/keJf1zSXdn+z6sqotiA0TEiv9Hbwrk7wIvoLfYybeAbU2Xa0z3dg5wfvb6LOA79BZa/y/A1dn2q4H/nL3elt3/GcB52e9lddP3scR7/9fADcCfZ++TvmfgfwLvyl6vA85O+Z7pLTX6PeCp2fs/BX47tXsGLgbOB+7JbRv5HoG/BS6kt3LfF4DLq5YhlRr9DuBwRByJiBPATcCuhss0FhHxYER8M3v9M+B+ev8H2UUvGMh+/mr2ehdwU0Q8ERHfo7dGwI6JFnoMJG0CXg98PLc52XuW9Av0AuETABFxIiJ+QsL3nFkDPFXSGuBp9FagS+qeI+I24OG+zSPdo6RzgF+IiL+JXup/MndOqVSCftDi5EmRtAV4GfAN4LnRW8WL7OdzssNS+V38d+DfAqdy21K+5xcAM8D/yJqrPi7pTBK+54j4e+C/Aj8EHqS3Mt2tJHzPOaPe48bsdf/2SlIJ+sqLkK9Ukp4OfAZ4T0Q8MuzQgm0r6nch6Q3A8Yi4o+opBdtW1D3Tq9meD3wsIl4GPErvT/pBVvw9Z+3Su+g1UfwicKak3xp2SsG2FXXPFQy6x2XdeypBX2UB8xVL0lp6If+piPhstvkfsz/nyH4ez7an8Lu4CHiTpO/Ta4Z7laQ/Ie17PgocjYhvZO//jF7wp3zPvwx8LyJmIuJJ4LPAK0j7nueMeo9Hs9f92ytJJeiTXYQ861n/BHB/RPxhbtcB4O3Z67cDn89t3y3pDEnnAVvpdeKsGBHxvojYFBFb6P23/D8R8Vukfc//ADwg6YXZplcD95HwPdNrsrlA0tOy/52/ml4fVMr3PGeke8yad34m6YLsd/W23Dnlmu6RHmPP9uvojUj5LvD7TZdnjPf1Snp/ot0F3Jn9ex3wbOAvgb/Lfj4rd87vZ7+HQ4zQM9/Gf8ClnB51k/Q9Ay8FprP/1p8DntmBe/4PwLeBe4D/RW+0SVL3DNxIrw/iSXo183cu5R6Bqez39F3gWrKZDar88xQIZmaJS6XpxszMBnDQm5klzkFvZpY4B72ZWeIc9GZmiXPQm5klzkFvZpa4/w8Yo44pO/aCeQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# File: ex10_2_rl_qlearn.py\n",
    "\n",
    "# 1. 패키지 부르기 및 초기화\n",
    "import gym\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "flake = gym.make(\"FrozenLake-v1\", is_slippery=False)\n",
    "Q = np.zeros((flake.observation_space.n, flake.action_space.n))\n",
    "PI = np.argmax(Q,axis=1)\n",
    "\n",
    "# 2. 주어진 정책에 따른 시도 함수 만들기\n",
    "def run_with_PI_exploration(PI=None, exploration=0.2, N_Iter = 100, render_flag=False):\n",
    "    \"\"\"\n",
    "    Return buff_df if done, otherwise return None \n",
    "    \"\"\"\n",
    "    if PI is None:\n",
    "        # No PI, action will be determined fully randomly\n",
    "        exploration = 1.0\n",
    "\n",
    "    s = flake.reset()\n",
    "    if render_flag: flake.render()\n",
    "    buff_df = pd.DataFrame({\"S\":[s],\"S:(x,y)\":[(0,0)], \n",
    "                \"R\":[0.0], \"done\":[False], \n",
    "                \"A\":[0], \"A:name\": [\"\"]})\n",
    "    buff_df.index.name = 'k'\n",
    "\n",
    "    Actions = [\"Left\", \"Down\", \"Right\", \"Up\"]\n",
    "    rand_buff =  np.random.rand(N_Iter)\n",
    "    for iter in range(N_Iter):\n",
    "        # if np.random.rand() <= exploration:\n",
    "        if rand_buff[iter] <= exploration:\n",
    "            a_k = flake.action_space.sample()\n",
    "        else:\n",
    "            a_k = PI[s]\n",
    "        buff_df.loc[iter,'A':\"A:name\"] = (a_k, Actions[a_k])\n",
    "        s, r, done, info = flake.step(a_k)\n",
    "        if render_flag: flake.render()\n",
    "        new_df = pd.DataFrame({\"S\":[s], \"S:(x,y)\":[(s%4,s//4)],\n",
    "                                \"R\":[r], \"done\":[done], \n",
    "                                \"A\":[0], \"A:name\": [\"\"]})\n",
    "        buff_df = buff_df.append(new_df, ignore_index=True)\n",
    "        buff_df.index.name = 'k'\n",
    "        if done:\n",
    "            return buff_df\n",
    "    return None\n",
    "\n",
    "buff_df = run_with_PI_exploration(PI, exploration=0.2, N_Iter=100)\n",
    "buff_df\n",
    "\n",
    "# 3. Q-learning 만들기\n",
    "def q_learning(N_epoch=1000, N_Iter=100, learning_rate=0.01, exploration=0.2, trace_flag=False):\n",
    "    \"\"\"\n",
    "    return PI 1-D integer array if trace_flag is False \n",
    "    else return PI_array 2-D integer array \n",
    "    \"\"\"\n",
    "    PI_list = []\n",
    "    Q = np.zeros((flake.observation_space.n, flake.action_space.n))\n",
    "    for e in range(N_epoch):\n",
    "        PI = np.argmax(Q,axis=1)\n",
    "        buff_df = run_with_PI_exploration(PI, exploration=exploration, N_Iter=N_Iter)\n",
    "        #buff_df = get_g(N_Iter=100)\n",
    "        #print(np.sum(buff_df.R))\n",
    "        for k in range(len(buff_df)-1):\n",
    "            S, A = buff_df.S[k], buff_df.A[k]\n",
    "            S_next, R_next = buff_df.S[k+1], buff_df.R[k+1]\n",
    "            # print(k, S, A, S_next, R_next)\n",
    "            Q_new = R_next + np.max(Q[S_next])\n",
    "            Q[S,A] += learning_rate * (Q_new - Q[S,A])\n",
    "\n",
    "        PI = np.argmax(Q,axis=1)\n",
    "        PI_list.append(PI)\n",
    "    if trace_flag:\n",
    "        return np.array(PI_list)\n",
    "    else:\n",
    "        return PI\n",
    "\n",
    "# 4. Q-learning 실행하기\n",
    "PI_array = q_learning(N_epoch=1000, N_Iter=100, learning_rate=0.01, exploration=0.8, trace_flag=True)\n",
    "print(PI_array)\n",
    "plt.plot(PI_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3b1c83d-04c1-4b5c-a5ff-fc1099a8c975",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "keras-gpu",
   "language": "python",
   "name": "keras-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}