Dueling-DQN/Dueling_Deep_Q_Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dueling Deep Q-Network (Dueling-DQN)\n",
    "---\n",
    "In this notebook, you will implement a Dueling-DQN agent with OpenAI Gym's Breakout environment.\n",
    "The Agent uses game pixel frames as input state and predicts the action q-values.\n",
    "\n",
    "### 1. Import the Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "# !pip3 install box2d\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import cv2\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# !python -m pip install pyvirtualdisplay\n",
    "from pyvirtualdisplay import Display\n",
    "display = Display(visible=0, size=(1400, 900))\n",
    "display.start()\n",
    "\n",
    "is_ipython = 'inline' in plt.get_backend()\n",
    "if is_ipython:\n",
    "    from IPython import display\n",
    "\n",
    "plt.ion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from wrappers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('./models'):\n",
    "    os.makedirs('./models')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Instantiate the Environment and Agent\n",
    "\n",
    "Initialize the environment in the code cell below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BreakoutDeterministic-v4 -> Selects action every 4th frame and \n",
    "# repeats the action for skipped frames\n",
    "env = gym.make('BreakoutDeterministic-v4')\n",
    "env = make_env(env)\n",
    "env.seed(0)\n",
    "print('State shape: ', env.observation_space.shape)\n",
    "print('Number of actions: ', env.action_space.n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_process(state, crop_h=(31, 16), crop_w=(6,6)):\n",
    "    \"\"\"\n",
    "    Given the environment state (pixel image) as input. Do the following\n",
    "    1. Convert to grayscale\n",
    "    2. Crops the image as specified by the params\n",
    "    3. Resize image to size 84 x 84\n",
    "    \n",
    "    Params\n",
    "    ========================\n",
    "    state   (h, w, c): a 3 channel image representing the pixel state of the environment\n",
    "    crop_h (int, int): the number of pixels to be cropped from the (top, bottom) of the image\n",
    "    crop_w (int, int): the number of pixels to be cropped from the (left, right) of the image\n",
    "    \n",
    "    Returns\n",
    "    ========================\n",
    "    the processed state of shape (1, 84, 84)\n",
    "    \"\"\"\n",
    "    \n",
    "    state = np.mean(state, axis=2).astype(np.uint8)                          # grayscale\n",
    "    state = state[crop_h[0]:-crop_h[1],crop_w[0]:-crop_w[1]]                   # crop frame to remove useless pixels\n",
    "    state = cv2.resize(state, dsize=(84, 84), interpolation=cv2.INTER_CUBIC) # resize\n",
    "    \n",
    "    return state.reshape((1, 84, 84))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of frames stacked together to form the context for the Agent\n",
    "FRAME_HISTORY = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dueling_dqn_agent import Agent\n",
    "\n",
    "agent = Agent(action_size=4, frame_history=FRAME_HISTORY, seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = []\n",
    "# watch an untrained agent\n",
    "state = env.reset()\n",
    "state = pre_process(state)\n",
    "\n",
    "stacked_state = deque(maxlen=FRAME_HISTORY)\n",
    "for _ in range(FRAME_HISTORY-1):\n",
    "    stacked_state.append(np.zeros_like(state))\n",
    "stacked_state.append(state)\n",
    "\n",
    "t = env.render(mode='rgb_array')\n",
    "frames.append(t)\n",
    "img = plt.imshow(t)\n",
    "\n",
    "for j in range(200):\n",
    "    action = agent.act(np.concatenate(stacked_state, axis=0)) \n",
    "    t = env.render(mode='rgb_array')\n",
    "    frames.append(t)\n",
    "    img.set_data(t)\n",
    "    plt.axis('off')\n",
    "    display.display(plt.gcf())\n",
    "    display.clear_output(wait=True)\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    state = pre_process(state)\n",
    "    stacked_state.append(state)\n",
    "    if done:\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the Agent with DQN\n",
    "\n",
    "Run the code cell below to train the agent from scratch.  You are welcome to amend the supplied values of the parameters in the function, to try to see if you can get better performance!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def dqn(n_episodes=50000, max_t=20000, eps_start=1.0, eps_end=0.1):\n",
    "    \"\"\"Deep Q-Learning.\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        eps_start (float): starting value of epsilon, for epsilon-greedy action selection\n",
    "        eps_end (float): minimum value of epsilon\n",
    "        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon\n",
    "    \"\"\"\n",
    "    scores = []                        # list containing scores from each episode\n",
    "    scores_window = deque(maxlen=100)  # last 100 scores\n",
    "    moving_avgs = []                   # list containing scores window averages\n",
    "    eps = eps_start                    # initialize epsilon\n",
    "    eps_decay = (eps_start - eps_end) / 1e5\n",
    "    \n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        state = env.reset()\n",
    "        state = pre_process(state)\n",
    "        \n",
    "        stacked_state = deque(maxlen=FRAME_HISTORY)\n",
    "        for _ in range(FRAME_HISTORY-1):\n",
    "            stacked_state.append(np.zeros_like(state))\n",
    "        stacked_state.append(state)\n",
    "\n",
    "        score = 0\n",
    "        for t in range(max_t):\n",
    "            action = agent.act(np.concatenate(stacked_state, axis=0), eps)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            next_state = pre_process(next_state)\n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            state = next_state\n",
    "            stacked_state.append(state)\n",
    "            score += reward\n",
    "            if done:\n",
    "                break \n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        moving_avgs.append(np.mean(scores_window)) # save the moving average\n",
    "        eps = max(eps_end, eps-eps_decay) # decrease epsilon\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "        if np.mean(scores_window)>=200.0:\n",
    "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_window)))\n",
    "            torch.save(agent.qnetwork_local.state_dict(), 'checkpoint_best.pth')\n",
    "            break\n",
    "        if i_episode % 1000 == 0:\n",
    "            torch.save(agent.qnetwork_local.state_dict(), f'./models/checkpoint_episode_{i_episode}.pth')\n",
    "    return scores, moving_avgs\n",
    "\n",
    "scores, moving_avgs = dqn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('scores.np', scores)\n",
    "np.save('moving_avgs.np', moving_avgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10,6))\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores, c='k', label='DQN agent score', alpha=0.3)\n",
    "plt.plot(np.arange(len(scores)), moving_avgs, c='r', label='Moving average score')\n",
    "# plt.plot(np.arange(len(scores)), [200.0]*len(scores), c='b', label='baseline', alpha=0.5)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.legend();\n",
    "plt.grid()\n",
    "# plt.savefig('scores_plot.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Watch a Smart Agent!\n",
    "\n",
    "In the next code cell, you will load the trained weights from file to watch a smart agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the weights from file\n",
    "if torch.cuda.is_available():\n",
    "    map_location=lambda storage, loc: storage.cuda()\n",
    "else:\n",
    "    map_location='cpu'\n",
    "\n",
    "agent.qnetwork_local.load_state_dict(torch.load('./models/checkpoint_episode_30000.pth', map_location=map_location))\n",
    "\n",
    "frames = []\n",
    "for i in range(3):\n",
    "    state = env.reset()\n",
    "    state = pre_process(state)\n",
    "\n",
    "    stacked_state = deque(maxlen=FRAME_HISTORY)\n",
    "    for _ in range(FRAME_HISTORY-1):\n",
    "        stacked_state.append(np.zeros_like(state))\n",
    "    stacked_state.append(state)\n",
    "\n",
    "    t = env.render(mode='rgb_array')\n",
    "    frames.append(t)\n",
    "    img = plt.imshow(t)\n",
    "\n",
    "    for j in range(200):\n",
    "        action = agent.act(np.concatenate(stacked_state, axis=0)) \n",
    "        t = env.render(mode='rgb_array')\n",
    "        frames.append(t)\n",
    "        img.set_data(t)\n",
    "        plt.axis('off')\n",
    "        display.display(plt.gcf())\n",
    "        display.clear_output(wait=True)\n",
    "        state, reward, done, _ = env.step(action)\n",
    "        state = pre_process(state)\n",
    "        stacked_state.append(state)\n",
    "        if done:\n",
    "            break \n",
    "            \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import animation\n",
    "\n",
    "# NOTE: for imagemagick to work, you may need to run `conda install imagemagick`\n",
    "# and then re run the trained agent\n",
    "\n",
    "def display_frames_as_gif(frames):\n",
    "    patch = plt.imshow(frames[0])\n",
    "    plt.axis('off')\n",
    "    def animate(i):\n",
    "        patch.set_data(frames[i])\n",
    "        \n",
    "    anim = animation.FuncAnimation(plt.gcf(), animate, frames = len(frames), interval=5)\n",
    "    anim.save('./breakout_result.gif', writer='imagemagick', fps=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display_frames_as_gif(frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](./breakout_result.gif)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "drlnd",
   "language": "python",
   "name": "drlnd"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}