REINFORCE/VanillaPolicyGradient.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implementation of the Vanilla Policy Gradient (REINFORCE) \n",
    "### using the Cartpole environment provided by gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import gym\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a policy network that input (obs) --> output (logits of actions, logits due to use of softmax function)\n",
    "\n",
    "class policy_network():\n",
    "    def __init__(self, env):\n",
    "        self.n_inputs = env.observation_space.shape[0]\n",
    "        self.n_outputs = env.action_space.n\n",
    "        \n",
    "        # Define network\n",
    "        self.network = nn.Sequential(\n",
    "            nn.Linear(self.n_inputs, 40), #fully connected layer\n",
    "            nn.ReLU(), \n",
    "            nn.Linear(40, 40),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(40, self.n_outputs),\n",
    "            nn.Softmax(dim=-1)) # turns logits into probs\n",
    "    \n",
    "    def predict(self, obs):\n",
    "        action_probs = self.network(torch.FloatTensor(obs))\n",
    "        return action_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create data loop --> save current obs \n",
    "#                  --> run obs through policy network get logits of actions\n",
    "#                  --> sample action using prob_actions from policy network \n",
    "#                  --> save action \n",
    "#                  --> act using gym env, returns new obs and reward\n",
    "#                  --> save reward \n",
    "#             --> after 1 episode ends save discounted rewards used for optimization\n",
    "#             --> reset env\n",
    "\n",
    "def create_episode(policy_network, env): # done\n",
    "    action_space = np.arange(env.action_space.n)\n",
    "    done = False\n",
    "    obs = env.reset()\n",
    "    observations, rewards, actions = [], [], []\n",
    "    while not done:\n",
    "        observations.append(obs)\n",
    "        act_probs = policy_network.predict(obs).detach().numpy() \n",
    "        act = np.random.choice(action_space, p=act_probs) \n",
    "        actions.append(act)\n",
    "        obs, r, done, _ = env.step(act)\n",
    "        rewards.append(r)\n",
    "    return observations, actions, rewards\n",
    "\n",
    "def normalized_discounted_rewards(rewards, gamma): # done\n",
    "    r = np.array([gamma**i * rewards[i] for i in range(len(rewards))])\n",
    "    r = r[::-1].cumsum()[::-1]\n",
    "    return r - r.mean()\n",
    "\n",
    "def create_batch(policy_network, batch_size, env, gamma): # done\n",
    "    batch_discrew, batch_obs, batch_acts, total_rewards = [],[],[],[]\n",
    "    for ep in range(batch_size):\n",
    "        observations, actions, rewards = create_episode(policy_network, env)\n",
    "        batch_discrew.extend(normalized_discounted_rewards(rewards, gamma))\n",
    "        batch_obs.extend(observations)\n",
    "        batch_acts.extend(actions)\n",
    "        total_rewards.append(sum(rewards))\n",
    "    avg_return = np.mean(total_rewards)\n",
    "    #print(\"Created a batch\")\n",
    "    return batch_discrew, batch_obs, batch_acts, avg_return\n",
    "\n",
    "\n",
    "# optimize policy parameters --> calculate loss:\n",
    "#                                       --> sum of gradients of policy gradient for all actions in an episode\n",
    "#                                           --> arguments should contain all episodes (in a tensor for example)\n",
    "#                                           --> sum(actions * softmax(logits)), use softmax to transform logits to probs\n",
    "#                                       --> weight the sum by discounted rewards (discounted rewards is acting as A)\n",
    "#                                       --> loss =  -average of previous result\n",
    "#                            --> calculate gradients of each layer: use the loss to backpropogate and obtain the gradients\n",
    "#                            --> update weights of each layer\n",
    "\n",
    "\n",
    "def VPG_loss(policy_net, obs_tensor, reward_tensor, action_tensor):\n",
    "    # calculate loss\n",
    "    logprobs = torch.log(policy_net.predict(obs_tensor))\n",
    "    logprobs_a = torch.gather(logprobs, 1, action_tensor.view(-1,1)).squeeze()\n",
    "    logprobs_b = reward_tensor * logprobs_a \n",
    "    loss = -logprobs_b.mean()\n",
    "    return loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rl(env, policy_net, num_epochs, batch_size, gamma, loss_type, optimizer_type):\n",
    "    avg_returns = []\n",
    "    \n",
    "    epoch = 0\n",
    "    # Define optimizer\n",
    "    if optimizer_type == \"Adam\":\n",
    "        optimizer = optim.Adam(policy_net.network.parameters(), lr=0.01) \n",
    "    \n",
    "    action_space = np.arange(env.action_space.n)\n",
    "    \n",
    "    while epoch < num_epochs:\n",
    "        #create a batch\n",
    "        batch_discrew, batch_obs, batch_acts, avg_return = create_batch(policy_net, batch_size, env, gamma)\n",
    "        avg_returns.append(avg_return)\n",
    "        print(\"\\rEpoch:\"+\" {:.2f}\".format(epoch+1)+\" Average reward of batch:\" + \"{:.2f}\".format(avg_return), end=\"\")\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        #convert to tensors\n",
    "        obs_tensor = torch.FloatTensor(batch_obs)\n",
    "        reward_tensor = torch.FloatTensor(batch_discrew)\n",
    "        action_tensor = torch.LongTensor(batch_acts)\n",
    "        \n",
    "        #calculate loss and update parameters of policy network\n",
    "        if loss_type == \"VPG\":\n",
    "            logprobs = torch.log(policy_net.predict(obs_tensor))\n",
    "            logprobs1= reward_tensor * torch.gather(logprobs, 1, action_tensor.view(-1,1)).squeeze() # does the sum over grads\n",
    "            loss = -logprobs1.mean()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        \n",
    "        \n",
    "        epoch += 1\n",
    "    \n",
    "    \n",
    "    return avg_returns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    env = gym.make('CartPole-v0')\n",
    "    avg_returns = rl(env, policy_network(env), num_epochs=40, batch_size=20, \n",
    "                     gamma=0.99, loss_type = \"VPG\", optimizer_type = \"Adam\")\n",
    "    return avg_returns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40.00 Average reward of batch:200.00"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xl8VPW9//HXB0gCWVhCEnYIu4ALsqpUa12q1ap11966W2pX297b1i73V9t7e29ve7t5e+tWvS611r3a9l4VUUFlUcKiIAghEAhLNgLZ98/vjzmxAYdkEjKZJPN+Ph7zmJnvnDnnkwMzn/ku5/s1d0dERORI/WIdgIiI9ExKECIiEpYShIiIhKUEISIiYSlBiIhIWEoQIiISlhKEiIiEpQQhIiJhKUGIiEhYA2IdwLHIyMjw7OzsWIchItKr5OTklLh7Znvb9eoEkZ2dzZo1a2IdhohIr2Jm+ZFspyYmEREJSwlCRETCUoIQEZGwlCBERCQsJQgREQkragnCzMaZ2WtmttnMNpnZ7UF5upktMbNtwf2woNzM7C4zyzWzd81sTrRiExGR9kWzBtEI/KO7zwBOAb5sZjOBO4Cl7j4VWBo8B/gUMDW4LQbujmJsIiLSjqhdB+Hu+4B9weMKM9sMjAEuAc4MNnsYeB34TlD+iIfWQF1lZkPNbFSwH5G4lVdcyVvbS7n4xNEMSU7o9H5qG5ooq67nQFU9ZVUNHKiup6yqnur6Ji6ZPZrRQwd1eJ/biyv5y4a9NDcffenipIT+DE9JZHhqEukpiWSkhh6nJPbHzABobnYO1TRQWlVHaWU9pVX1lFbWcbC6gRGDBzJlRCpTslIZPLDzf38kDtU0sPtANfml1ewuq6ahsZlBif1JThxAcmJ/Bib0JzkxdEtJGsDUrFQG9O/47+y1u8rYur+Cq+aNo18/6/D7H125k0mZqSyaktHh93ZEt1woZ2bZwMnAamBEy5e+u+8zs6xgszHA7lZvKwjKDksQZraYUA2D8ePHRzVukVjbVVrNNfetoqiijn/722aumDuWGxdlMzkztd337j1Yw/Pr9/LChr3sLKmipqHpqNs+8OYOHrhhHieNGxpxbG9sK+ZLf1hLRV0j1sZ33NGWvU8a0I/hKYk0NDsHquppaiPJtBgxOImpWWlMyQoljMmZqYwaMpCswUkkJ7b/dVbf2MzegzUUlNWwu6w6lAwOhO53HajmYHVDu/tobdSQgXx2wXiuXjCOrLSBbW7b3Oy8uqWI+5bn8fbOAwBs2V/BDy+a+WGijMTTOQX88/ObuOik0VFPEOZH+9frqgOYpQLLgJ+4+7NmdtDdh7Z6vczdh5nZ34B/d/c3g/KlwLfdPedo+543b57rSmrpq4oqarnynpUcqmngPy4/kVfeL+T59Xupb2rmrOOyuHnRRBZNGX7Yl0t5bQMvvref59btYdWOUtxh7oRhzB43lPSURNJTEhmWnBg8TmBYciLFlXXc+vAaSirr+OVVs7nghFHtxvaHVfn88IVNTM1K5YEb5zOmjdpHbUMTpVX1HKisp6SlhlBZx4Gqekoq60kcYKSnJDI8JYnhqa3uUxMZMiiB/Ydq2VZYybaiSnKLKsktqiC3qJKq+sMTXmrSALIGJ5GVlkRW2kBGDE5iUEJ/Cg7WUHCghoKyavaX19I6Dw3oZ4wdNohx6cmMT09mwvDQ/fj0FMalD2JgQn+q65uoqW+iur4x9Lihier6Jkor63hu3R7e2FZCQn/j/ONHcf2pE5g3Ydhh/yZ1jU08v34v9y3PI7eokjFDB3Hzxyay+0A1D63YydfOnso3z50W0f+JFzfu40uPreW0yRk8cOM8kgb0j+h9RzKzHHef1+520UwQZpYA/BV4yd1/GZR9AJwZ1B5GAa+7+3Qzuzd4/PiR2x1t/0oQ0lcdqmng6ntXsutANY/dupCTxw8DoLiijj+u3sWjq/Ipqaxj2ohUbl40kYzUJJ5bv4dX3i+krrGZiRkpfGb2GD5z8mgmDE9p93gllXUsfmQNa3cd5FvnTedLZ04O+6u2qdn5yd828+BbOzjruCzuuvZkUpO6f8Yed2ffoVryiqsoLK+lsKKWovI6iivqKKqopbA8dF/X2MzIwQMZNyyZscMGMTY9mXHDBjF2WDLj0gcxcvDATjURtba9uJI/rMrn6ZwCKmobOW5kGtefms3ZM7J4bt0eHnxzB0UVdcwYNZgvnDGJC08cRUL/frg73376XZ7KKeAHF87g1tMntXmcN7eVcPND7zBrzGD+cMtCUo7hvMc8QVjof9fDwAF3/3qr8p8Dpe7+UzO7A0h392+b2YXAV4ALgIXAXe6+oK1jKEFIX1RT38R1D6xmQ8FBHrxxPqdP/eicanWNTfxlwz4efHMH7+8rByA9JZGLThzFpXPGctLYIR1qtoDQL/1vP/0uL2zYy+VzxvLvl51A4oC/f3lW1jVy++PrWLqliJsWZfODC2fSvxPt593F3Wlq9mNOAJGqrm/k+fV7eWRlPpuDfxOARVOG84UzJnP61IyP/Js0NjXz1cfX8X8b9/Ozy0/kqvnjwu47J7+M6x5Yzfj0ZJ5YfOox9UVBz0gQHwPeAN4DmoPi7xHqh3gSGA/sAq509wNBQvktcD5QDdzk7m1++ytBSFcpKq9ly/4KzpjW7gSXUdXQ1MziR9bw+tZifnvtHC48se3mHndnTX4ZVXWNLJqSQcIxfhm6O79Zuo1fv7KNBRPTufdzcxmWksjegzXc8vAathZWcOdFM7nu1OxjOk5f5u6s3VXG8q0lnDtzBMePGdLm9nWNTdz68Breyi3ht5+d85Emvs37yrn63pWkpyTy5G2nttvXEYmYJ4juoAQhXeXOFzbx0Iqd/PSyE7hmQWwGPzQ3O994cj3Pr9/Lv116Ap9dGLtBGM+v38O3nnqX0UMH8q3zjuNHf9lEdX0Tv/3syZw5Pav9HUiHVNc3ct0Db/NuwUF+f8N8Ph78UNlRUsWV96wkob/x1G2nMnZYcpccL9IEoSupRYCthRUAfO+591jyfmG3H9/d+dFfNvH8+r1867zpMU0OAJfMHsMfP7+Q8tpGvvzHtST078czXzxNySFKkhMH8OCN85mSlcZtj+aQk3+AfYdq+NzvV9PszqO3LOyy5NARqkGIAPN/8goLstMpOFjDln3l/OHWhczPTo/ovfsP1fKjv2yiock5ZVI6p0wazoxRgyNun2/drHPrxyby/QtndLj/IFp2lVbz+Du7uHnRRDLTkmIdTp9XXFHHVfeupKSyjozUJEoq6nh88SntNlN1lJqYRCJ0qLqBk378Mnd86jiumjeOK+5ZQUlFHU/ddhrTR6a1+d4VuSV87U/rqK5vIistiZ2l1QCkDRzAgux0Fk5KZ+HE4cwaPZi6xmZ2lFQddssrqWJHcSXltY1cMXcsP7/ixB6THCQ29hys4cq7V1BaVc+jtyxkwcTIfqh0RKQJolevKCfSFXKLQ81LUzJTSU9J5JGbF3D53Su4/sHVPPPF08JW7ZubnbuXbecXL3/ApMxU/rR4DlOy0th/qJbVO0pZlVfK6rwDLN1SBIQuCqtrbD5sH2OGDmJiRgqXzB7DjFGDuWreWCUHYczQQTz/lY9RXtsQ0QWR0aQEIXEvt6gSgKkjQh/GscOSeeTmhVx5zwquf/Btnr7tNNJTEj/c/lB1A998cj1LtxRx8Umj+ffLTvhwTPrIIQO5ZPYYLpk9BgiNjlq14wAbdh8kPSWRSRkpTMxMYUJ6CoMSO3eRk/R9mWlJPaJJTwlC4l5uUSWJA/odVlOYPjKN398wn+seWM3ND73DHz+/kOTEAWzcc4gvPpYT6ne4eBbXnzqhzV/9WYMHcvFJo7n4pNHd8aeIdCmNYpK4t62okkkZKR/pVF4wMZ3/uvZk3i04yJceW8sfV+/isrtX0NjkPPGFU7nhtGw1CUmfphqExL3cosoPp7I40idnjeTfLj2BO559j9c/KOb0qRn8+urZDE+NffVfJNqUICSuVdc3hkaNzA0/xQHANQvG40B5TQO3nj6pR08vIdKVlCAkruUVV+H+9w7qo7k2RldXi8SS+iAkrrWMYJqSFdvhhCI9kRKExLVtRRX072dkRzAltki8UYKQuJZbVMmE4cmHTWstIiH6VEhcyy2qZEqMr1YV6amUICRu1Tc2s7O0Wv0PIkehBCFxK7+0iqZmb3cEk0i8UoKQuPXhCKbMtmdsFYlXShASt7YFCWJylkYwiYQTtQRhZg+aWZGZbWxV9oSZrQ9uO81sfVCebWY1rV67J1pxibTILapkzNBBJCfqelGRcKL5yXgI+C3wSEuBu1/d8tjMfgEcarX9dnefHcV4RA6TW1SpDmqRNkStBuHuy4ED4V6z0BSYVwGPR+v4Im1pana2FytBiLQlVn0QpwOF7r6tVdlEM1tnZsvM7PQYxSVxYk9ZDXWNzUxVghA5qlg1vl7L4bWHfcB4dy81s7nAn81slruXH/lGM1sMLAYYP14TqEnnbCsKlhlVghA5qm6vQZjZAOAy4ImWMnevc/fS4HEOsB2YFu797n6fu89z93mZmZndEbL0QZqkT6R9sWhiOgfY4u4FLQVmlmlm/YPHk4CpQF4MYpM4kVtUSUZqEkOTE9vfWCRORXOY6+PASmC6mRWY2S3BS9fw0c7pM4B3zWwD8DRwm7uH7eAW6QrbiiqZousfRNoUtT4Id7/2KOU3hil7BngmWrGItObubC+q5DMnj4l1KCI9mq6klrhTVFFHRV2j+h9E2qEEIXFnW6E6qEUioQQhcSc3GOKqayBE2qYEIXEnt7iStIEDyExLinUoIj2aEoTEnW2FoSk2QjO+iMjRKEFI3NleXKnmJZEIKEFIXCmrqqeksl4d1CIRUIKQuJJbHBrBNDVLq8iJtEcJQuKK5mASiZwShMSVbYWVDEzox5ihg2IdikiPpwQhcSW3uJLJman066cRTCLtUYKQuLJdy4yKREwJQuJGVV0jew7WMCVTCUIkEkoQEje2t4xgGqEEIRIJJQiJG5qkT6RjlCAkbuQWVzKgnzFhuBYKEomEEoTEjdyiSrIzUkjor//2IpHQJ0XiRm5RpTqoRTpACULiQl1jE/mlVep/EOmAqCUIM3vQzIrMbGOrsjvNbI+ZrQ9uF7R67btmlmtmH5jZedGKS+LTzpJqml0jmEQ6Ipo1iIeA88OU/8rdZwe3/wUws5nANcCs4D2/M7P+UYxN4sx7ew4BGsEk0hFRSxDuvhw4EOHmlwB/cvc6d98B5AILohWbxJ/n1+9hzNBBzBg5ONahiPQaseiD+IqZvRs0QQ0LysYAu1ttUxCUfYSZLTazNWa2pri4ONqxSh+w/1Atb+WWcNmcMZqDSaQDujtB3A1MBmYD+4BfBOXhPrUebgfufp+7z3P3eZmZmdGJUvqUP6/fQ7PDZXPGxjoUkV6lWxOEuxe6e5O7NwP38/dmpAJgXKtNxwJ7uzM26ZvcnWdyCpg7YRgTM3SBnEhHdGuCMLNRrZ5eCrSMcHoBuMbMksxsIjAVeLs7Y5O+6b09h9hWVMnlqj2IdNiAaO3YzB4HzgQyzKwA+CFwppnNJtR8tBP4AoC7bzKzJ4H3gUbgy+7eFK3YJH48k1NA4oB+XHjiqPY3FpHDRC1BuPu1YYofaGP7nwA/iVY8En/qG5t5YcNezp05giGDEmIdjkivoyuppc96dUsRZdUNXKHmJZFOUYKQPuuZtQVkpiVx+tSMWIci0itF1MQUXNU8ovX27r4rWkGJHKvSyjpe21LETYuyGaDZW0U6pd0EYWZfJdTBXAg0B8UOnBjFuESOyQsb9tLY7Fw+V81LIp0VSQ3idmC6u5dGOxiRrvLM2gJmjR7McZpaQ6TTIql77wYORTsQka7ywf4KNu4p17UPIscokhpEHvC6mf0NqGspdPdfRi0qkWPwzNoCBvQzLpk9OtahiPRqkSSIXcEtMbiJ9FiNTc08t24PZ07PYnhqUqzDEenV2kwQweilVHf/VjfFI3JM3sgtobiijivmhp0MWEQ6oM0+iGC6izndFIvIMXsmp4ChyQl84risWIci0utF0sS03sxeAJ4CqloK3f3ZqEUl0gmHahp4+f1Crpk/jqQBWpBQ5FhFkiDSgVLgrFZlDihBSI/yt3f3Ud/YrNFLIl2k3QTh7jd1RyAix+qZtQVMyUrlxLFDYh2KSJ8QyZXU/0OY1d3c/eaoRCTSCbtKq8nJL+M75x+HmZYVFekKkTQx/bXV44GEFvrRam/So7y4aR8An9a6DyJdJpImpmdaPw8WAnolahGJdMJLmwqZNXow49KTYx2KSJ/RmWkupwLjuzoQkc4qKq8lJ7+M82aNjHUoIn1KJH0QFRzeB7Ef+E7UIhLpoJffLwTg/OOVIES6UiRNTGmd2bGZPQh8Gihy9+ODsp8DFwH1wHbgJnc/aGbZwGbgg+Dtq9z9ts4cV+LPS5v2MzEjhalZqbEORaRPabeJycyWRlIWxkPA+UeULQGOd/cTga3Ad1u9tt3dZwc3JQeJyKHqBlZuL+W8WSM1ekmkix21BmFmA4FkIMPMhgEtn77BQLvTZLr78qBm0Lrs5VZPVwFXdDBekcMs3VJIY7OreUkkCtpqYvoC8HVCyWBtq/Jy4L+74Ng3A0+0ej7RzNYF+/+Bu7/RBceQPu6lTfsZOXggJ47RxXEiXe2oCcLdfwP8xsy+6u7/1ZUHNbPvA43AY0HRPmC8u5ea2Vzgz2Y2y93Lw7x3MbAYYPx4DaaKZzX1TSzbWszV88bRr5+al0S6WiTDXB80sx+Y2X0AZjbVzD7d2QOa2Q2EOq//wd0dwN3rWpY0dfccQh3Y08K9393vc/d57j4vMzOzs2FIH7BsazG1Dc0a3ioSJRElCEKjjk4LnhcA/9qZg5nZ+YSGyF7s7tWtyjODtScws0mErrXI68wxJH68tGk/Q5MTWDAxPdahiPRJkSSIye7+M6ABwN1r+HuH9VEFV1yvBKabWYGZ3QL8FkgDlpjZejO7J9j8DOBdM9sAPA3c5u4HOv7nSLyob2xm6eZCzpkxggH9O3O9p4i0J5K5mOrNbBDBxXJmNplWa1MfjbtfG6b4gaNs+wzwTLjXRMJZlVdKeW0j56t5SSRqIkkQPwReBMaZ2WPAIuDGaAYl0p4XN+0nObE/H5uaEetQRPqs9takNmALcBlwCqGmpdvdvaQbYhMJq6nZeXlTIZ+YnsXABK0cJxItbSYId3cz+7O7zwX+1k0xibRp3a4ySirr+OSsEbEORaRPi6R3b5WZzY96JCIRemnTfhL79+Os47JiHYpInxZJH8QngC+YWT5QRaiZyYP5lES6lbvz4qb9nDZlOGkDE2IdjkifFkmC+FTUoxCJ0OZ9Few+UMOXz5wS61BE+rxIpvvO745ARCLx4qb99DM4Z6b6H0SiTVcYSa/y8qb9zMtOJyM1KdahiPR5ShDSa+wsqWLL/gpdHCfSTSJKEGY2wczOCR4PMrNOrTIncixe2rQfQMNbRbpJJCvKfZ7Q/Ej3BkVjgT9HMyiRIxWV1/Lcuj2cMGYIY4clxzockbgQySimLwMLgNUA7r7NzDQAXbrF9uJK7l+ex7Nr99DY3MzPrjgp1iGJxI1IEkSdu9e3rPdrZgMIJu4TiZac/APcsyyPVzYXkti/H1fNH8utH5tEdkZKrEMTiRuRJIhlZvY9YJCZnQt8CfhLdMOSeNTc7LyyuZD7luexJr+MockJfPWsqdxw6gSGa9SSSLeLJEHcAdwCvEdoner/BX4fzaAkPv3T0xt4du0exg4bxJ0XzeSq+eNITozkv6iIREMkF8o1A/cHN5GoqKpr5K8b9nHF3LH89LITtAiQSA/QboIws/f4aJ/DIWAN8K8ta0mLHIs3thVT39TMFXPHKjmI9BCR1N//D2gC/hg8vya4LwceAi7q+rAk3ix5v4ghgxKYN2FYrEMRkUAkCWKRuy9q9fw9M3vL3ReZ2eeiFZjEj6Zm59UthZx1XJZqDyI9SCSfxlQzW9jyxMwWAKnB08a23mhmD5pZkZltbFWWbmZLzGxbcD8sKDczu8vMcs3sXTOb04m/R3qhtbvKKKtu4JwZukJapCeJJEHcCvzezHaY2U5CI5g+b2YpwL+3896HgPOPKLsDWOruU4GlwXMITSs+NbgtBu6O5A+Q3u+V9wtJ6G+cMU3rS4v0JJGMYnoHOMHMhgDm7gdbvfxkO+9dbmbZRxRfApwZPH4YeB34TlD+iLs7oVXshprZKHffF8HfIb3Yks2FnDJJCwCJ9DQRDTI3swuBWcDAliuq3f3HnTzmiJYvfXff12rajjHA7lbbFQRlShB92PbiSvKKq7jxtOxYhyIiR4hksr57gKuBrxJabvRKYEIUYrEwZR+Z0sPMFpvZGjNbU1xcHIUwpDst3VwIwNnqfxDpcSLpgzjN3a8Hytz9R8CpwLhjOGahmY0CCO6LgvKCI/Y7Fth75Jvd/T53n+fu8zIzM48hDOkJXnm/iJmjBjNm6KBYhyIiR4gkQdQG99VmNhpoACYewzFfAG4IHt8APN+q/PpgNNMpwCH1P/RtB6rqWZN/QMuHivRQkfRB/MXMhgI/B9YSavaJaNoNM3ucUId0hpkVAD8Efgo8aWa3ALsINVlBaI6nC4BcoBq4KfI/Q3qj17YU0exwrpqXRHqkNhOEmfUjNCT1IPCMmf0VGOjuhyLZubtfe5SXzg6zrRNae0LixCubCxkxOInjxwyOdSgiEkabTUzBRH2/aPW8LtLkINKW2oYmlm0t5pwZI2gZGSciPUskfRAvm9nlpk+xdKFVeaVU1zep/0GkB4ukD+KbQArQZGY1hIajururXUA67ZXNhSQn9ufUScNjHYqIHEUkV1KndUcgEj/cnVfeL+KMqZkMTOgf63BE5CgiuVDOzOxzZvbPwfNxwYR9Ip2yaW85+8tr1bwk0sNF0gfxO0IXx302eF4J/HfUIpI+b8n7hfQz+MR0Xego0pNF0gex0N3nmNk6AHcvM7PEKMclfdgrmwuZO2EYw1OTYh2KiLQhkhpEg5n1J5gXycwygeaoRiV91t6DNWzaW661H0R6gUgSxF3Ac0CWmf0EeBP4t6hGJX1Wy+R86n8Q6fkiGcX0mJnlELr62YDPuPvmqEcmfdKSzUVMykhhcmZq+xuLSEy1myDM7DfAE+6ujmk5JhW1DazcXsJNi45lrkcR6S6RNDGtBX4QrBX9czObF+2gpG96Y1sJDU2u/geRXqLdBOHuD7v7BcACYCvwH2a2LeqRSZ/zp3d2k5GayJzxQ2MdiohEIJIaRIspwHFANrAlKtFIn5WTX8byrcXcevokBvTvyH87EYmVSK6kbqkx/BjYBMx194uiHpn0Kb9+ZSvDUxK5/tRorFYrItEQyYVyO4BT3b0k2sFI35STf4A3tpXwvQuOIzkxkv9yItITRDLM9R4zGxbMvzSwVfnyqEYmfcavlmwjIzWRz52i2oNIbxLJMNdbgduBscB64BRgJXBWdEOTvuCdnQd4M7eE718wQ7UHkV4mkt7C24H5QL67fwI4GSiOalTSZ/xqyVYyUpNUexDphSJJELXuXgtgZknuvgWY3tkDmtl0M1vf6lZuZl83szvNbE+r8gs6ewzpGVbnlbJieym3fXwSgxK17oNIbxNJnb/AzIYCfwaWmFkZsLezB3T3D4DZAMEkgHsIzfV0E/Ard//Pzu5bepZfv7KNzDTVHkR6q0g6qS8NHt5pZq8BQ4AXu+j4ZwPb3T1fS173LavySlmZV8r/+/RMrRon0kt16Iold1/m7i+4e30XHf8a4PFWz79iZu+a2YNmNizcG8xssZmtMbM1xcXqCuku7k5xRR2b9h6iqdnb3f5XS7aSlZbEZxeO74boRCQazL39D3tUDhxadGgvMMvdC81sBFBCaN2JfwFGufvNbe1j3rx5vmbNmugHG0cO1TSwZV85+aXV7CitIr+0ip0l1eSXVlFV3wTA9BFpfO/CGXx8WvgV4VZsL+Gz96/mhxfN1MR8Ij2QmeW4e7vz6sVy3OGngLXuXgjQcg9gZvcDf41VYPHqUHUDZ/9yGSWVdQAM6GeMT09mwvBkFkxMJ3t4MkkJ/bn79e3c8ODbnDEtk+9fMIPpI9M+3Ie78+sl2xgxOIlrF6j2INKbxTJBXEur5iUzG+Xu+4KnlwIbYxJVHHvgzTxKKuu469qTOXncUEYNGRh23qTL5ozh0ZX53LV0G5/6zXKunj+Ob5w7jay0gazcXsrbOw/wo4tnqe9BpJeLSYIws2TgXOALrYp/ZmazCTUx7TziNYmyg9X1PPjWTj51/EguPml0m9smDejPradP4vI5Y7nr1W08ujKfF9bv5baPT2bZ1mJGDh7I1fPHdVPkIhItMUkQ7l4NDD+i7LpYxCIhD7y5g8q6Rm4/Z2rE7xmWksgPL5rF9adm89P/28wvlmwF4F8uUe1BpC/Q3AfCwep6/uetnVx4wiiOGzm4w++fmJHCvdfNY3VeKW/mlnCVag8ifYIShHD/G3lU1TfytbMjrz2Es3DScBZOGt7+hiLSK2jlljhXVlXPQ2/t5IITRh02GklERAkizt3/Rh7VDU18/RhrDyLS9yhBxLEDVfU8vGInnz5xNFNHqPYgIodTgohj9y0P1R6+dtaUWIciIj2QEkScKq2s45GVO7lItQcROQoliDh13xt51DY0HfPIJRHpu5Qg4lBJZR2PrMjn4pNGMyUrNdbhiEgPpQQRh+5bnkddYxNfVe1BRNqgBBFnSoK+h0tmj2FypmoPInJ0ShBx5r7ledQ3NvNVjVwSkXYoQcSZv727j3NmjGCSag8i0g4liDiy71ANew7WcIrmSxKRCChBxJG1+QcBmDsh7HLfIiKHUYKIIzn5ZQxM6MfM0R2f0ltE4o8SRBzJ2VXGiWOHkhBmGVERkSPpmyJO1DY0sWnPITUviUjEYrZgkJntBCqAJqDR3eeZWTrwBJBNaF3qq9y9LFYx9iXvFhyisdmZO14JQkQiE+saxCfcfba7zwue3wEsdfepwNLguXSBnPxQnp2jGoSIRCjWCeJIlwAPB48fBj4Tw1j6lJz8MiZlpJCekhjrUESkl4hlgnDgZTPLMbPFQdkId99mj5ObAAANHUlEQVQHENxnxSy6PsTdWburTLUHEemQmPVBAIvcfa+ZZQFLzGxLJG8KksligPHjx0czvj5jZ2k1B6rq1UEtIh0SsxqEu+8N7ouA54AFQKGZjQII7ovCvO8+d5/n7vMyMzO7M+Req6X/QQlCRDoiJgnCzFLMLK3lMfBJYCPwAnBDsNkNwPOxiK+vyckvI23gAKZo/iUR6YBYNTGNAJ4zs5YY/ujuL5rZO8CTZnYLsAu4Mkbx9Slr88uYM34Y/fpZrEMRkV4kJgnC3fOAk8KUlwJnd39Efdehmga2FlVw4YmjYh2KiPQyPW2Yq3Sx9bsP4q7+BxHpOCWIPi4nv4x+BieNGxrrUESkl1GC6OPW5pdx3MjBpCbFckSziPRGShC9TFF5Lfcs205jU3O72zY1O+t2lal5SUQ6RT8re5mfv/QBT+UUMDwlkSvnjWtz2w/2V1BV36QEISKdohpEL7L3YA1/Xr8HgLte3UZDO7WInF26QE5EOk8Johf5/Rs7aHb48SWz2H2ghmfXFrS5/dr8MjLTkhg7bFA3RSgifYkSRC9xoKqex9/exSUnjea6UyZw0tgh3LU0l/rGo9cicvLLmDt+GMEFiSIiHaIE0Us8vGInNQ1N3HbmZMyMb5w7jT0Ha3gqZ3fY7Ysqatl1oFrNSyLSaUoQvUBVXSMPrdjJuTNHMG1EGgAfn5bJnPFD+e9Xc6lrbPrIe9bmHwS0QJCIdJ4SRC/w+Nu7OFTTwBfPnPxhWUstYu+hWp5856O1iLW7ykjs34/jxwzuzlBFpA9Rgujh6hqbuP+NPE6ZlM6cI9aT/tiUDOZnD+O3r+VS23B4LSInv4wTxg4haUD/7gxXRPoQJYge7s/r9lBYXseXzpzykddaahGF5XU8/vauD8vrGpt4r+CQ+h9E5JgoQfRgTc3OvcvymDV6MKdPzQi7zWmTM1g4MZ3fvb79w1rExj3l1Dc1f6TGISLSEUoQPdhLm/aTV1LFl86c0uZQ1W+cO43iijr+sCofCF3/ADBngiboE5HOU4Loodyd372ey8SMFM4/fmSb254yaTiLpgznnmXbqa5vJCe/jPHpyWSlDeymaEWkL9JcTF3stQ+K+NmLH1Be08DCieksnJTOKZOGMz49uUMXrL2ZW8LGPeX89LIT6B/BSnDfOGcaV9yzkkdX5pOzq4yPTQnfJCUiEikliC6ytbCCf/3bZpZvLSZ7eDInjRvCsq3FPLsuNHfSyMEDP0wWCyemMzEjpc2E8bvXtjNicBKXzhkT0fHnZadz+tQM7lq6jar6Jl3/ICLHTAniGJVW1vGrV7by+Nu7SUnszw8unMH1p2aTOKAf7s724kpW5h1gdV4pK7aX8vz6vUAoYZw2eTinTclg0ZThjBry9/mS1u0qY2VeKT+4cEaHhql+49xpXPa7FQDMVQe1iByjbk8QZjYOeAQYCTQD97n7b8zsTuDzQHGw6ffc/X+7O75I1TU28fCKnfzXq7lU1zfxuYXjuf2caaSnJH64jZkxJSuNKVlpXHfKBNydvJIqVgXJ4vVWNYxJGSmcNmU4iyZn8OSa3QwZlMC1C8Z3KKY544fxiemZrN11kOkj07r07xWR+GPu3r0HNBsFjHL3tWaWBuQAnwGuAird/T8j3de8efN8zZo1UYo0vKLyWl56v5Dfv5FHfmk1n5ieyfcvnMGUrI5/ITc3O1v2V7Biewlv5ZawescBqutDQ1W/dvZUvnnutA7v81B1A8WVtZ2KR0Tig5nluPu89rbr9hqEu+8D9gWPK8xsMxBZQ3uM5JdW8dKm/by4cT/rdh/EHY4bmcYjNy/gjGmZnd5vv37GzNGDmTl6MLeePomGpmY27D7I+/vKuXzO2E7tc0hyAkOSEzodk4hIi26vQRx2cLNsYDlwPPBN4EagHFgD/KO7l4V5z2JgMcD48ePn5ufnd3lc7qFf9i9u3M9Lm/azZX8FALNGD+a8WSM5//iRTM1K1TTaItIrRVqDiFmCMLNUYBnwE3d/1sxGACWAA/9CqBnq5rb20dVNTDX1TbywYQ+PrMxn095yzGD+hHQ+OWsE580aybj05C47lohIrPTYJiYAM0sAngEec/dnAdy9sNXr9wN/7a54dpZU8eiqfJ5as5vy2kamj0jjXy6ZxfnHjyIzLam7whAR6VFiMYrJgAeAze7+y1blo4L+CYBLgY3RjKOp2XltSxGPrMpn+dZiBvQzzjt+JNefMoEFE9PVfCQicS8WNYhFwHXAe2a2Pij7HnCtmc0m1MS0E/hCtALYsPsgX3psLXsO1jBicBLfOGca1y4YR9ZgTU0hItIiFqOY3gTC/TzvtmsesoenMCkzhe9fOINzZ44gob+mpBIROVJcXkk9JDmBR29ZGOswRER6NP10FhGRsJQgREQkLCUIEREJSwlCRETCUoIQEZGwlCBERCQsJQgREQlLCUJERMKK6XTfx8rMioFjme87g9AMsj2RYuscxdY5iq1zemtsE9y93cVsenWCOFZmtiaSKW9jQbF1jmLrHMXWOX09NjUxiYhIWEoQIiISVrwniPtiHUAbFFvnKLbOUWyd06dji+s+CBERObp4r0GIiMhRxGWCMLPzzewDM8s1sztiHU9rZrbTzN4zs/VmtibGsTxoZkVmtrFVWbqZLTGzbcH9sB4U251mtic4d+vN7IIYxTbOzF4zs81mtsnMbg/KY37u2ogt5ufOzAaa2dtmtiGI7UdB+UQzWx2ctyfMLLEHxfaQme1odd5md3dsrWLsb2brzOyvwfNjP2/uHlc3oD+wHZgEJAIbgJmxjqtVfDuBjFjHEcRyBjAH2Niq7GfAHcHjO4D/6EGx3Qn8Uw84b6OAOcHjNGArMLMnnLs2Yov5uSO00mRq8DgBWA2cAjwJXBOU3wN8sQfF9hBwRaz/zwVxfRP4I/DX4Pkxn7d4rEEsAHLdPc/d64E/AZfEOKYeyd2XAweOKL4EeDh4/DDwmW4NKnCU2HoEd9/n7muDxxXAZmAMPeDctRFbzHlIZfA0Ibg5cBbwdFAeq/N2tNh6BDMbC1wI/D54bnTBeYvHBDEG2N3qeQE95AMScOBlM8sxs8WxDiaMEe6+D0JfNkBWjOM50lfM7N2gCSomzV+tmVk2cDKhX5w96twdERv0gHMXNJOsB4qAJYRq+wfdvTHYJGaf1yNjc/eW8/aT4Lz9ysySYhEb8Gvg20Bz8Hw4XXDe4jFBWJiyHvNLAFjk7nOATwFfNrMzYh1QL3I3MBmYDewDfhHLYMwsFXgG+Lq7l8cyliOFia1HnDt3b3L32cBYQrX9GeE2696ogoMeEZuZHQ98FzgOmA+kA9/p7rjM7NNAkbvntC4Os2mHz1s8JogCYFyr52OBvTGK5SPcfW9wXwQ8R+hD0pMUmtkogOC+KMbxfMjdC4MPcTNwPzE8d2aWQOgL+DF3fzYo7hHnLlxsPencBfEcBF4n1M4/1MwGBC/F/PPaKrbzgyY7d/c64H+IzXlbBFxsZjsJNZmfRahGccznLR4TxDvA1KCHPxG4BnghxjEBYGYpZpbW8hj4JLCx7Xd1uxeAG4LHNwDPxzCWw7R8+QYuJUbnLmj/fQDY7O6/bPVSzM/d0WLrCefOzDLNbGjweBBwDqE+kteAK4LNYnXewsW2pVXCN0Jt/N1+3tz9u+4+1t2zCX2fveru/0BXnLdY97zH4gZcQGj0xnbg+7GOp1VckwiNqtoAbIp1bMDjhJobGgjVvG4h1La5FNgW3Kf3oNgeBd4D3iX0ZTwqRrF9jFB1/l1gfXC7oCecuzZii/m5A04E1gUxbAT+X1A+CXgbyAWeApJ6UGyvBudtI/AHgpFOsboBZ/L3UUzHfN50JbWIiIQVj01MIiISASUIEREJSwlCRETCUoIQEZGwlCBERCQsJQiRGDGzM1tm3hTpiZQgREQkLCUIkXaY2eeCtQDWm9m9waRtlWb2CzNba2ZLzSwz2Ha2ma0KJm97rmXSOzObYmavBOsJrDWzycHuU83saTPbYmaPBVfkivQIShAibTCzGcDVhCZRnA00Af8ApABrPTSx4jLgh8FbHgG+4+4nErrCtqX8MeC/3f0k4DRCV4FDaDbVrxNak2ESoXl1RHqEAe1vIhLXzgbmAu8EP+4HEZpkrxl4ItjmD8CzZjYEGOruy4Lyh4Gngvm1xrj7cwDuXgsQ7O9tdy8Inq8HsoE3o/9nibRPCUKkbQY87O7fPazQ7J+P2K6tOWvaajaqa/W4CX0mpQdRE5NI25YCV5hZFny4rvQEQp+dlpkyPwu86e6HgDIzOz0ovw5Y5qH1FgrM7DPBPpLMLLlb/wqRTtCvFZE2uPv7ZvYDQqv89SM0e+yXgSpglpnlAIcI9VNAaFrle4IEkAfcFJRfB9xrZj8O9nFlN/4ZIp2i2VxFOsHMKt09NdZxiESTmphERCQs1SBERCQs1SBERCQsJQgREQlLCUJERMJSghARkbCUIEREJCwlCBERCev/A4c4PLaqeTy/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(main())\n",
    "plt.ylabel(\"average return\")\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}