dqn_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2016802f",
   "metadata": {},
   "source": [
    "# Deep Q-Learning Network From Scratch\n",
    "\n",
    "Algorithm: \n",
    "![dqn](https://i.imgur.com/uevfmj2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fb4fcfcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fd7b381b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pygame\n",
    "\n",
    "def get_surface(rgb_array):\n",
    "    surface = pygame.surfarray.make_surface(np.transpose(rgb_array, (1, 0, 2)))\n",
    "    return surface\n",
    "# utility function to view how our agent plays the cartpole, using pygame\n",
    "# After done; this function will print the score (total reward)\n",
    "def play(net,env):\n",
    "    pygame.init()\n",
    "    screen = pygame.display.set_mode((600,400))\n",
    "    pygame.display.set_caption('CartPole')   \n",
    "\n",
    "    state,_ = env.reset()\n",
    "    done = False\n",
    "    rewards = 0\n",
    "    while not done:\n",
    "        action = np.argmax(net.predict_single(state))\n",
    "        state, r, done, _,_ = env.step(action)\n",
    "        rewards += r\n",
    "        surface = get_surface(env.render())\n",
    "\n",
    "        screen.blit(surface, (0, 0))\n",
    "        pygame.display.flip()\n",
    "        \n",
    "        for event in pygame.event.get():\n",
    "            if event.type == pygame.QUIT: done = True\n",
    "    \n",
    "    print(rewards)\n",
    "    pygame.quit()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66dbe087",
   "metadata": {},
   "source": [
    "## Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "41097631",
   "metadata": {},
   "outputs": [],
   "source": [
    "# If enough data in replay memory, learn with that\n",
    "def learn(data,batch_size):\n",
    "    D.append(data)\n",
    "    if len(D)<batch_size: return\n",
    "    \n",
    "    minibatch = random.sample(D,batch_size)\n",
    "    X = np.zeros((batch_size, state_shape))\n",
    "    y = np.zeros((batch_size, action_size))\n",
    "    for i, (state, action, reward, nxt_state, done) in enumerate(minibatch):\n",
    "        X[i] = state\n",
    "        y_i = reward + (1-done) * gamma * np.max(Q_target.predict_single(nxt_state))\n",
    "        y[i] = Q.predict_single(state)\n",
    "        y[i][action] = y_i \n",
    "        # for Q(s_i,a_i) - y_i we let our network to compute Q(s_i,a_i), \n",
    "        # so every index except the action became zero:- [0,si_ai-y_i,0,0])**2\n",
    "        \n",
    "    Q.train_on_batch(X, y,epoch=1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e48dfb2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Update Q' with Q weights\n",
    "def update_target(target):\n",
    "    for i in range(Q.L):\n",
    "        W1,b1 = Q.NN[i]\n",
    "        W2,b2 = target[i]\n",
    "        W2[:] = W1[:]\n",
    "        b2[:] = b1[:]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b830385c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train for N episodes\n",
    "def train(num_episode=100,batch_size=32,C=10,ep=10):\n",
    "    global epsilon,best_score\n",
    "    steps = 0\n",
    "    for i in range(1,num_episode+1):\n",
    "        episode_reward = 0\n",
    "        episode_loss = 0\n",
    "\n",
    "        # Sample Phase\n",
    "        done = False\n",
    "        nxt_state,_ = env.reset()\n",
    "        while not done:\n",
    "            state = nxt_state\n",
    "            epsilon = min(epsilon_min,epsilon*epsilon_decay) # e decay\n",
    "\n",
    "            # e-greedy(Q)\n",
    "            if np.random.randn() < epsilon: action = np.random.randint(action_size)\n",
    "            else:\n",
    "                q_vals = Q.predict_single(state)\n",
    "                action = np.argmax(q_vals)\n",
    "\n",
    "            nxt_state,reward,done,_,_ = env.step(action)\n",
    "            episode_reward += reward\n",
    "            \n",
    "            # Learining Phase\n",
    "            learn((state,action,reward,nxt_state,done),batch_size)\n",
    "            steps+=1\n",
    "            \n",
    "            if steps%C ==0: update_target(Q_target.NN)\n",
    "        if episode_reward > best_score:\n",
    "            best_score = episode_reward\n",
    "        if i%ep==0: print(f\"Episode: {i} Reward: {episode_reward}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95253448",
   "metadata": {},
   "source": [
    "# Lets train our agent to play Cartpole"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8449a125",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nn import NeuralNetwork\n",
    "import copy\n",
    "from collections import deque"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c4224ae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v1',render_mode= \"rgb_array\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ee1afda9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4, Discrete(2))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state, _ = env.reset()\n",
    "len(state),env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ec63e8d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "arch = [4,4,3,2]\n",
    "af = [\"sigmoid\",\"relu\",\"linear\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "9c685419",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Q\n",
    "Q = NeuralNetwork(arch,af,eta=5e-4,momentum=0,seed=8)\n",
    "\n",
    "# Q'\n",
    "Q_target = copy.deepcopy(Q) #Q' NeuralNetwork(same parms as above) then update_target(Q_target.NN) will also work\n",
    "\n",
    "# Replay Memory\n",
    "D = deque(maxlen=10000) # if D==maxlen and we append new data oldest one will get removed\n",
    "\n",
    "action_size = 2 # Action Space\n",
    "state_shape = 4 # State Size\n",
    "\n",
    "# Epsilon\n",
    "epsilon = 0.1\n",
    "epsilon_min = 0.01\n",
    "epsilon_decay = 0.995\n",
    "\n",
    "# Gamma\n",
    "gamma = 0.95\n",
    "# Just to check the highest score obtained during training\n",
    "best_score = -np.inf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "a2b20d89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 100 Reward: 46.0\n",
      "Episode: 200 Reward: 139.0\n",
      "Episode: 300 Reward: 348.0\n",
      "Episode: 400 Reward: 134.0\n",
      "Episode: 500 Reward: 133.0\n",
      "Episode: 600 Reward: 139.0\n",
      "Episode: 700 Reward: 173.0\n",
      "Episode: 800 Reward: 167.0\n",
      "Episode: 900 Reward: 55.0\n",
      "Episode: 1000 Reward: 59.0\n"
     ]
    }
   ],
   "source": [
    "train(1000,42,ep=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "c95d0298",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "558.0"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "dbc558be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "776.0\n"
     ]
    }
   ],
   "source": [
    "play(Q,env)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b316a6ee",
   "metadata": {},
   "source": [
    "## Saving The Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "95c13611",
   "metadata": {},
   "outputs": [],
   "source": [
    "from saveload import save_network,load_network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "c39385e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save_network(Q,\"CartPoleScratchNet500\") \n",
    "# save_network(Q,\"CartPoleScratchNetBetter\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5f4585d",
   "metadata": {},
   "source": [
    "## Loading Pretrained Network\n",
    "\n",
    "when loading a network, please move that network out of the `networks` folder.\n",
    "\n",
    "#### CartPoleScratchNet500\n",
    "- Avg score 500 \n",
    "- HyperParameters `(eta=5e-3,momentum=0.3,num_episodes=300,batch=42,C=10)`\n",
    "#### CartPoleScratchNetBetter\n",
    "- Avg score range `500<score<5000` (one time it got 6283)\n",
    "- HyperParameters `(eta=5e-4,momentum=0,num_episodes=2000,batch=42,C=10)`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "98d9448d",
   "metadata": {},
   "outputs": [],
   "source": [
    "cpsn500 = load_network(\"CartPoleScratchNet500\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "9540e9e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "522.0\n"
     ]
    }
   ],
   "source": [
    "play(cpsn500,env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "08e91163",
   "metadata": {},
   "outputs": [],
   "source": [
    "cpsnb = load_network(\"CartPoleScratchNetBetter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "048430a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3674.0\n"
     ]
    }
   ],
   "source": [
    "play(cpsnb,env)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf3d526c",
   "metadata": {},
   "source": [
    
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}