Train_DQN_portpy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "66b95a50",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import gym\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "import rl_utils\n",
    "from tqdm import tqdm\n",
    "import env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0870a67b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeatureExtractor(torch.nn.Module):\n",
    "    # input_dim = 72 + 8 * 128 = 1096\n",
    "    # output_dim = 128 + 8 * 64 = 640\n",
    "    def __init__(self, input_dim = 1096, batch_norm = True):\n",
    "        \"Three FC layers, two hidden layers, len of hidden_sizes: 2\"\n",
    "        super(FeatureExtractor, self).__init__()\n",
    "        self.theta_dim = 72\n",
    "        self.dvh_dim = 128\n",
    "        hidden_sizes = [512, 256]\n",
    "        self.theta_out_dim = 128\n",
    "        self.fc_theta_1 = torch.nn.Linear(self.theta_dim, hidden_sizes[0])\n",
    "        self.fc_theta_2 = torch.nn.Linear(hidden_sizes[0], hidden_sizes[1])\n",
    "        self.fc_theta_3 = torch.nn.Linear(hidden_sizes[1], self.theta_out_dim)\n",
    "        \n",
    "        input_channels = (input_dim - self.theta_dim) // self.dvh_dim\n",
    "        self.input_channels = input_channels\n",
    "        dvh_output_size = input_channels * self.dvh_dim // 2\n",
    "        Conv_dim = [input_channels * 4, input_channels * 8, input_channels * 16, input_channels * 32]\n",
    "        self.fc1_dim =  input_channels * self.dvh_dim * 2\n",
    "        self.batch_norm = batch_norm\n",
    "        \n",
    "        if self.batch_norm:\n",
    "            self.bn1 = torch.nn.BatchNorm1d(Conv_dim[0])\n",
    "            self.bn2 = torch.nn.BatchNorm1d(Conv_dim[1])\n",
    "            self.bn3 = torch.nn.BatchNorm1d(Conv_dim[2])\n",
    "            self.bn4 = torch.nn.BatchNorm1d(Conv_dim[3])\n",
    "            self.bn5 = torch.nn.BatchNorm1d(dvh_output_size * 2)\n",
    "            \n",
    "        self.conv_1 = torch.nn.Conv1d(in_channels = input_channels, out_channels = Conv_dim[0],\n",
    "                                      kernel_size = 3, stride = 1, padding = 1)\n",
    "        self.conv_2 = torch.nn.Conv1d(in_channels = Conv_dim[0], out_channels = Conv_dim[1],\n",
    "                                      kernel_size = 3, stride = 1, padding = 1)\n",
    "        self.conv_3 = torch.nn.Conv1d(in_channels = Conv_dim[1], out_channels = Conv_dim[2],\n",
    "                                      kernel_size = 3, stride = 1, padding = 1)\n",
    "        self.conv_4 = torch.nn.Conv1d(in_channels = Conv_dim[2], out_channels = Conv_dim[3],\n",
    "                                      kernel_size = 3, stride = 1, padding = 1)\n",
    "        \n",
    "        self.maxpool = torch.nn.MaxPool1d(3, stride=2, padding=1)\n",
    "        self.relu = torch.nn.ReLU()\n",
    "        \n",
    "        self.fc_dvh_1 = torch.nn.Linear(self.fc1_dim, self.fc1_dim // 2)\n",
    "        self.fc_dvh_2 = torch.nn.Linear(self.fc1_dim // 2, dvh_output_size)\n",
    "        \n",
    "    def forward(self, state):\n",
    "        theta_state = state[:,:self.theta_dim]\n",
    "        dvh_state = torch.reshape(state[:, self.theta_dim:], (-1, self.input_channels, self.dvh_dim))\n",
    "        \n",
    "        x = self.relu(self.fc_theta_1(theta_state))\n",
    "        x = self.relu(self.fc_theta_2(x))\n",
    "        x = self.fc_theta_3(x)\n",
    "        \n",
    "        if self.batch_norm:\n",
    "            y = self.maxpool(self.relu(self.bn1(self.conv_1(dvh_state))))\n",
    "            y = self.maxpool(self.relu(self.bn2(self.conv_2(y))))\n",
    "            y = self.maxpool(self.relu(self.bn3(self.conv_3(y))))\n",
    "            y = self.maxpool(self.relu(self.bn4(self.conv_4(y))))\n",
    "            y = torch.reshape(y, (-1, self.fc1_dim))\n",
    "            y = self.relu(self.bn5(self.fc_dvh_1(y)))\n",
    "            y = self.fc_dvh_2(y)\n",
    "        else:\n",
    "            y = self.maxpool(self.relu(self.conv_1(dvh_state)))\n",
    "            y = self.maxpool(self.relu(self.conv_2(y)))\n",
    "            y = self.maxpool(self.relu(self.conv_3(y)))\n",
    "            y = self.maxpool(self.relu(self.conv_4(y)))\n",
    "            y = torch.reshape(y, (-1, self.fc1_dim))\n",
    "            y = self.relu(self.bn5(self.fc_dvh_1(y)))\n",
    "            y = self.relu(self.fc_dvh_1(y))\n",
    "            y = self.fc_dvh_2(y)\n",
    "        \n",
    "        out = torch.cat((x, y), dim = 1)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "46fdbc5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# methon in Dong.\n",
    "class Conv_Block(torch.nn.Module):\n",
    "    def __init__(self, input_channels, output_channels):\n",
    "        super(Conv_Block, self).__init__()\n",
    "        self.conv_1 = torch.nn.Conv1d(in_channels = input_channels, out_channels = output_channels,\n",
    "                                      kernel_size = 3, stride = 1, padding = 1)\n",
    "        self.conv_2 = torch.nn.Conv1d(in_channels = output_channels, out_channels = output_channels,\n",
    "                                      kernel_size = 3, stride = 1, padding = 1)\n",
    "        self.maxpool = torch.nn.MaxPool1d(3, stride=2, padding=1)\n",
    "        self.relu = torch.nn.ReLU()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.relu(self.conv_1(x))\n",
    "        x = self.relu(self.conv_2(x))\n",
    "        x = self.maxpool(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6ce81e49",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Qnet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, action_dim = 72):\n",
    "        super(Qnet, self).__init__()\n",
    "        self.theta_dim = 72\n",
    "        self.theta_out_dim = 128\n",
    "        self.dvh_dim = 128\n",
    "        self.dvh_channels = (state_dim - self.theta_dim) // self.dvh_dim\n",
    "        self.feature = FeatureExtractor(input_dim = state_dim)\n",
    "        self.feature_dim = self.theta_out_dim + self.dvh_channels * self.dvh_dim // 2\n",
    "        # hidden layers: fit Q function, num : feature_num(72 + 128 * n) - 512 - 256 - 72\n",
    "        hidden_dims = [512, 256]\n",
    "        self.fc1 = torch.nn.Linear(self.feature_dim, hidden_dims[0])\n",
    "        self.fc2 = torch.nn.Linear(hidden_dims[0], hidden_dims[1])\n",
    "        self.fc3 = torch.nn.Linear(hidden_dims[1], action_dim)\n",
    "\n",
    "    def forward(self, state):\n",
    "        # Splitting and restructuring of state to theta and dvh\n",
    "        # the feature extract of theta and dvh is done separately \n",
    "        # then merge two features into one\n",
    "        x = self.feature(state)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ed5f5b74",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN:\n",
    "    def __init__(self,\n",
    "                 state_dim,\n",
    "                 action_dim,\n",
    "                 learning_rate,\n",
    "                 gamma,\n",
    "                 epsilon,\n",
    "                 target_update,\n",
    "                 device,\n",
    "                 dqn_type='VanillaDQN'):\n",
    "        self.action_dim = action_dim\n",
    "        self.q_net = Qnet(state_dim, self.action_dim).to(device)\n",
    "        self.target_q_net = Qnet(state_dim, self.action_dim).to(device)\n",
    "        self.optimizer = torch.optim.Adam(self.q_net.parameters(), lr=learning_rate)\n",
    "        self.gamma = gamma\n",
    "        self.epsilon = epsilon\n",
    "        self.target_update = target_update\n",
    "        self.count = 0\n",
    "        self.dqn_type = dqn_type\n",
    "        self.device = device\n",
    "\n",
    "    def optional_action(self, state):\n",
    "        # return a int list, which represents the optional action id\n",
    "        # like [0, 1, 36, 71]\n",
    "        theta_dim = 72\n",
    "        theta = state[0,:theta_dim]\n",
    "        optional_action_list = np.where(theta == 0)\n",
    "        return optional_action_list[0]\n",
    "    \n",
    "    def take_action(self, state):\n",
    "        optional_action_list = self.optional_action(state)\n",
    "        if np.random.random() < self.epsilon:\n",
    "            index = np.random.randint(len(optional_action_list))\n",
    "            action = optional_action_list[index]\n",
    "        else:\n",
    "            state = torch.tensor(state, dtype=torch.float).to(self.device)\n",
    "            action = self.q_net(state).argmax().item()\n",
    "        return action\n",
    "    \n",
    "    def max_q_value(self, state):\n",
    "        state = torch.tensor(state, dtype=torch.float).to(self.device)\n",
    "        return self.q_net(state).max().item()\n",
    "\n",
    "    def update(self, transition_dict):\n",
    "        states = torch.tensor(transition_dict['states'],\n",
    "                              dtype=torch.float).to(self.device)\n",
    "        actions = torch.tensor(transition_dict['actions']).view(-1, 1).to(\n",
    "            self.device)\n",
    "        rewards = torch.tensor(transition_dict['rewards'],\n",
    "                               dtype=torch.float).view(-1, 1).to(self.device)\n",
    "        next_states = torch.tensor(transition_dict['next_states'],\n",
    "                                   dtype=torch.float).to(self.device)\n",
    "        dones = torch.tensor(transition_dict['dones'],\n",
    "                             dtype=torch.float).view(-1, 1).to(self.device)\n",
    "\n",
    "        # x.gather(1, index): 按列索引x的值（长度为len(index），第i个元素为x第i行，第index[i]列的值）\n",
    "        # q_values即为该批次(batch_size)的value向量\n",
    "        q_values = self.q_net(states).gather(1, actions)\n",
    "        if self.dqn_type == 'DoubleDQN':\n",
    "            max_action = self.q_net(next_states).max(1)[1].view(-1, 1)\n",
    "            max_next_q_values = self.target_q_net(next_states).gather(1, max_action)\n",
    "        else: # DQN\n",
    "            max_next_q_values = self.target_q_net(next_states).max(1)[0].view(-1, 1)\n",
    "        q_targets = rewards + self.gamma * max_next_q_values * (1 - dones)  # TD误差目标\n",
    "        dqn_loss = torch.mean(F.mse_loss(q_values, q_targets))  #  MSE loss fucntion\n",
    "        self.optimizer.zero_grad()  # PyTorch中默认梯度会累积,这里需要显式将梯度置为0\n",
    "        dqn_loss.backward()  # 反向传播更新参数\n",
    "        self.optimizer.step()\n",
    "\n",
    "        if self.count % self.target_update == 0:\n",
    "            self.target_q_net.load_state_dict(\n",
    "                self.q_net.state_dict())  # 更新目标网络\n",
    "        self.count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "efddce67",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-2\n",
    "num_episodes = 200\n",
    "gamma = 0.98\n",
    "epsilon = 0.01\n",
    "target_update = 50\n",
    "buffer_size = 6\n",
    "minimal_size = 2\n",
    "batch_size = 2\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\n",
    "    \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fb41d8ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "creating rinds.. This step may take some time due to dilation\n",
      "Optimization structures created!!\n"
     ]
    }
   ],
   "source": [
    "data_path = r'D:\\MyJupyter\\RL_BAO\\Data'\n",
    "portpy_env = env.PortPyEnv(data_path = data_path, step_max_num = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "af7466da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_DQN(agent, env, num_episodes, replay_buffer, minimal_size,\n",
    "              batch_size):\n",
    "    return_list = []\n",
    "    max_q_value_list = []\n",
    "    max_q_value = 0\n",
    "    for i in range(1):\n",
    "        with tqdm(total=int(num_episodes / 10),\n",
    "                  desc='Iteration %d' % i) as pbar:\n",
    "            for i_episode in range(int(num_episodes / 10)):\n",
    "                episode_return = 0\n",
    "                state = env.reset()\n",
    "                done = False\n",
    "                while not done:\n",
    "                    # batch_normalization 不允许输入的批量数为1\n",
    "                    state = np.array([state, state])\n",
    "                    action = agent.take_action(state)\n",
    "                    max_q_value = agent.max_q_value(\n",
    "                        state) * 0.005 + max_q_value * 0.995  # 平滑处理\n",
    "                    max_q_value_list.append(max_q_value)  # 保存每个状态的最大Q值\n",
    "                    next_state, reward, done, _ = env.step(action)\n",
    "                    replay_buffer.add(state[0], action, reward, next_state, done)\n",
    "                    state = next_state\n",
    "                    episode_return += reward\n",
    "                    if replay_buffer.size() > minimal_size:\n",
    "                        b_s, b_a, b_r, b_ns, b_d = replay_buffer.sample(batch_size)\n",
    "                        transition_dict = {\n",
    "                            'states': b_s,\n",
    "                            'actions': b_a,\n",
    "                            'next_states': b_ns,\n",
    "                            'rewards': b_r,\n",
    "                            'dones': b_d\n",
    "                        }\n",
    "                        agent.update(transition_dict)\n",
    "                return_list.append(episode_return)\n",
    "                if (i_episode + 1) % 10 == 0:\n",
    "                    pbar.set_postfix({\n",
    "                        'episode':\n",
    "                        '%d' % (num_episodes / 10 * i + i_episode + 1),\n",
    "                        'return':\n",
    "                        '%.3f' % np.mean(return_list[-10:])\n",
    "                    })\n",
    "                pbar.update(1)\n",
    "    return return_list, max_q_value_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0589f7a9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:   0%|                                                                              | 0/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.25501755641\n",
      "Elapsed time: 12.915423393249512 seconds\n",
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.2660412982\n",
      "Elapsed time: 15.670580387115479 seconds\n",
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.26535015475\n",
      "Elapsed time: 13.186717510223389 seconds\n",
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 3595.029462701458\n",
      "Elapsed time: 23.526299715042114 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:   5%|███▌                                                                  | 1/20 [01:15<23:55, 75.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 100358.30120635209\n",
      "Elapsed time: 18.960392713546753 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  10%|███████                                                               | 2/20 [01:35<12:51, 42.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 100358.31860740624\n",
      "Elapsed time: 20.763087511062622 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  15%|██████████▌                                                           | 3/20 [01:57<09:24, 33.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 100358.32112367342\n",
      "Elapsed time: 23.48955535888672 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  20%|██████████████                                                        | 4/20 [02:21<07:54, 29.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 100358.30411548409\n",
      "Elapsed time: 27.209455728530884 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  25%|█████████████████▌                                                    | 5/20 [02:49<07:16, 29.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 100358.29000367378\n",
      "Elapsed time: 25.046062707901 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  30%|█████████████████████                                                 | 6/20 [03:15<06:31, 27.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.26728477314\n",
      "Elapsed time: 28.759214639663696 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  35%|████████████████████████▌                                             | 7/20 [03:44<06:10, 28.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.26208743571\n",
      "Elapsed time: 27.022502660751343 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  40%|████████████████████████████                                          | 8/20 [04:13<05:40, 28.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.2553736168\n",
      "Elapsed time: 28.114092588424683 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  45%|███████████████████████████████▌                                      | 9/20 [04:42<05:14, 28.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.25901539277\n",
      "Elapsed time: 30.862940073013306 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration 0:  50%|█████████████████████                     | 10/20 [05:13<04:55, 29.55s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.26207904387\n",
      "Elapsed time: 33.46454191207886 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  55%|███████████████████████                   | 11/20 [05:48<04:39, 31.01s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.26666318532\n",
      "Elapsed time: 34.72940921783447 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  60%|█████████████████████████▏                | 12/20 [06:23<04:19, 32.41s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.26828784104\n",
      "Elapsed time: 38.53583526611328 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  65%|███████████████████████████▎              | 13/20 [07:03<04:01, 34.52s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.26578627377\n",
      "Elapsed time: 40.86039137840271 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  70%|█████████████████████████████▍            | 14/20 [07:44<03:40, 36.71s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.25377648891\n",
      "Elapsed time: 43.567569971084595 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  75%|███████████████████████████████▌          | 15/20 [08:29<03:15, 39.05s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.25593127216\n",
      "Elapsed time: 44.27509045600891 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  80%|█████████████████████████████████▌        | 16/20 [09:14<02:43, 40.91s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.259990453\n",
      "Elapsed time: 45.66170334815979 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  85%|███████████████████████████████████▋      | 17/20 [10:01<02:07, 42.61s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.2629195493\n",
      "Elapsed time: 48.751964807510376 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  90%|█████████████████████████████████████▊    | 18/20 [10:50<01:29, 44.76s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.259826963\n",
      "Elapsed time: 52.09355354309082 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Iteration 0:  95%|███████████████████████████████████████▉  | 19/20 [11:44<00:47, 47.25s/it, episode=10, return=64.800]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating BEV..\n",
      "Loading sparse influence matrix...\n",
      "Done\n",
      "Objective Start\n",
      "Objective done\n",
      "Constraints Start\n",
      "Constraints done\n",
      "Running Optimization..\n",
      "Optimal value: 66085.2684308791\n",
      "Elapsed time: 55.44804501533508 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration 0: 100%|██████████████████████████████████████████| 20/20 [12:40<00:00, 38.02s/it, episode=20, return=66.000]\n"
     ]
    }
   ],
   "source": [
    "random.seed(0)\n",
    "np.random.seed(0)\n",
    "portpy_env.seed(0)\n",
    "torch.manual_seed(0)\n",
    "# state_dim = 72 + 8 * 128 = 1096\n",
    "state_dim = 1096\n",
    "action_dim = 72\n",
    "replay_buffer = rl_utils.ReplayBuffer(buffer_size)\n",
    "agent = DQN(state_dim, action_dim, lr, gamma, epsilon,\n",
    "            target_update, device)\n",
    "return_list, max_q_value_list = train_DQN(agent, portpy_env, num_episodes,\n",
    "                                          replay_buffer, minimal_size,\n",
    "                                          batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "33ab5846",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[119, 53, 53, 53, 53, 53, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66] [0.0004086413234472275, 0.0008098867941647768, 0.0012091260375287384, 0.0026914045130806883, 0.003968596517147594, 0.038938260732895594, 0.22431594568045182, 0.2661122565007556, 0.2712830563499199, 0.27140441783399366, 0.27330499551584647, 0.27929092341245093, 0.29336746264609664, 0.3209611822214465, 0.3714052740535643, 0.4622562154414752, 0.6224996371230569, 0.8979650579437892, 1.346610799365496, 1.8613607089306803, 2.316272960561808, 2.6729189440866357, 2.942394830170646]\n"
     ]
    }
   ],
   "source": [
    "print(return_list, max_q_value_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "502e5927",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'env_name' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[17], line 6\u001b[0m\n\u001b[0;32m      4\u001b[0m plt\u001b[38;5;241m.\u001b[39mxlabel(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mEpisodes\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      5\u001b[0m plt\u001b[38;5;241m.\u001b[39mylabel(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mReturns\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m----> 6\u001b[0m plt\u001b[38;5;241m.\u001b[39mtitle(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDQN on \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\u001b[43menv_name\u001b[49m))\n\u001b[0;32m      7\u001b[0m plt\u001b[38;5;241m.\u001b[39mshow()\n\u001b[0;32m      9\u001b[0m frames_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(max_q_value_list)))\n",
      "\u001b[1;31mNameError\u001b[0m: name 'env_name' is not defined"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABFeklEQVR4nO3deXgUVd728buzdUJIOiSQ7gQDhDWICAgaA7hBBCKDYWR08EEHBWUexRkBV2YEdVxQ3Bh8HNB5FcV1dFRcQBBQYVBAVofNEGQnCwgknYXs9f6BaY0QIKG7q7vz/VxXXRddVV39KypNbs45dcpiGIYhAACAABVkdgEAAACeRNgBAAABjbADAAACGmEHAAAENMIOAAAIaIQdAAAQ0Ag7AAAgoIWYXYAvqKmpUU5OjqKiomSxWMwuBwAAnAHDMFRUVKTExEQFBdXffkPYkZSTk6OkpCSzywAAAI2wb98+nXPOOfVuJ+xIioqKknT8Lys6OtrkagAAwJlwOp1KSkpy/R6vD2FHcnVdRUdHE3YAAPAzpxuCwgBlAAAQ0Ag7AAAgoBF2AABAQCPsAACAgEbYAQAAAc3UsLN8+XINGzZMiYmJslgsmjdvnmtbZWWl7rvvPnXv3l2RkZFKTEzUH/7wB+Xk5NQ5xpEjRzRq1ChFR0crJiZGY8eOVXFxsZfPBAAA+CpTw05JSYl69OihF1544YRtpaWlWr9+vaZMmaL169frgw8+UFZWlq6++uo6+40aNUpbtmzR4sWL9emnn2r58uUaN26ct04BAAD4OIthGIbZRUjH75H/8MMPNXz48Hr3WbNmjS666CLt2bNHbdq00bZt23TuuedqzZo16tOnjyRp4cKFuuqqq7R//34lJiae0Wc7nU7ZbDYVFhYyzw4AAH7iTH9/+9WYncLCQlksFsXExEiSVq5cqZiYGFfQkaT09HQFBQVp9erV9R6nvLxcTqezzgIAAAKT34SdsrIy3Xfffbr++utd6S0vL0/x8fF19gsJCVFsbKzy8vLqPda0adNks9lcC8/FAgAgcPlF2KmsrNR1110nwzA0a9assz7e5MmTVVhY6Fr27dvnhioBAIAv8vlnY9UGnT179uiLL76o0yfncDh08ODBOvtXVVXpyJEjcjgc9R7TarXKarV6rGYAAOA7fLplpzboZGdna8mSJYqLi6uzPS0tTQUFBVq3bp1r3RdffKGamhqlpqZ6u9wTlFZUaeehYjnLKs0uBQCAJsvUlp3i4mLt2LHD9XrXrl3auHGjYmNjlZCQoN/97ndav369Pv30U1VXV7vG4cTGxiosLExdu3bVkCFDdOutt2r27NmqrKzUHXfcoZEjR57xnViedMP/W631ewv0j1EX6KruCWaXAwBAk2Rq2Fm7dq2uuOIK1+tJkyZJkkaPHq2HHnpIH3/8sSSpZ8+edd735Zdf6vLLL5ckvfnmm7rjjjs0cOBABQUFacSIEZo5c6ZX6j8dhy1ckpRXWGZyJQAANF2mhp3LL79cp5rm50ymAIqNjdVbb73lzrLcxh59POzkOwk7AACYxafH7Pg7x09hJ4+wAwCAaQg7HkQ3FgAA5iPseBDdWAAAmI+w40G/7MbykUeQAQDQ5BB2PKi2G6usskbOY1UmVwMAQNNE2PGg8NBgxTQLlSTlOo+ZXA0AAE0TYcfDXF1ZDFIGAMAUhB0PY5AyAADmIux42M8tO+UmVwIAQNNE2PEwu42JBQEAMBNhx8McdGMBAGAqwo6HOWxWSQxQBgDALIQdD3NER0iiZQcAALMQdjysdmLBwyUVKq+qNrkaAACaHsKOh7VoFqqwkON/zQed3JEFAIC3EXY8zGKxyB7907gdurIAAPA6wo4XMIsyAADmIex4AbMoAwBgHsKOF9CyAwCAeQg7XuBgFmUAAExD2PGC2rBDNxYAAN5H2PECVzcWYQcAAK8j7HiBa4ByYbkMwzC5GgAAmhbCjhfUhp2K6hodKakwuRoAAJoWwo4XhIUEKS4yTBJdWQAAeBthx0uYawcAAHMQdrzEdft5Ic/HAgDAmwg7XmLnjiwAAExB2PGShNq5dphFGQAAryLseAlz7QAAYA7CjpfYbTwfCwAAMxB2vISWHQAAzEHY8ZLasFN4rFJlldUmVwMAQNNB2PGS6IgQhYce/+umKwsAAO8h7HiJxWKhKwsAABMQdryIWZQBAPA+wo4XJXBHFgAAXkfY8SLX7ee07AAA4DWEHS9yjdmhZQcAAK8h7HgRA5QBAPA+U8PO8uXLNWzYMCUmJspisWjevHl1tn/wwQcaNGiQ4uLiZLFYtHHjxhOOUVZWpvHjxysuLk7NmzfXiBEjlJ+f750TaCA7z8cCAMDrTA07JSUl6tGjh1544YV6t/fv319PPvlkvceYOHGiPvnkE7333ntatmyZcnJydM0113iq5LNS27JzsKhcNTWGydUAANA0hJj54RkZGcrIyKh3+4033ihJ2r1790m3FxYW6uWXX9Zbb72lAQMGSJLmzJmjrl27atWqVbr44ovdXvPZaBVllcUiVdUY+rGkXPFR4WaXBABAwPPrMTvr1q1TZWWl0tPTXetSUlLUpk0brVy5st73lZeXy+l01lm8ITQ4SC2bWyVJ+YXlXvlMAACaOr8OO3l5eQoLC1NMTEyd9Xa7XXl5efW+b9q0abLZbK4lKSnJw5X+LIHbzwEA8Cq/DjuNNXnyZBUWFrqWffv2ee2z7dyRBQCAV5k6ZudsORwOVVRUqKCgoE7rTn5+vhwOR73vs1qtslqtXqjwRD/PtXPMlM8HAKCp8euWnd69eys0NFRLly51rcvKytLevXuVlpZmYmX1c7geGcGYHQAAvMHUlp3i4mLt2LHD9XrXrl3auHGjYmNj1aZNGx05ckR79+5VTk6OpONBRjreouNwOGSz2TR27FhNmjRJsbGxio6O1p/+9CelpaX53J1YtXgYKAAA3mVqy87atWvVq1cv9erVS5I0adIk9erVS1OnTpUkffzxx+rVq5eGDh0qSRo5cqR69eql2bNnu47x3HPP6Te/+Y1GjBihSy+9VA6HQx988IH3T+YMMYsyAADeZTEMo8nPbud0OmWz2VRYWKjo6GiPftaOg0VKf3a5oqwh2vTwYI9+FgAAgexMf3/79Zgdf1TbjVVUXqWS8iqTqwEAIPARdrwsKjxUza3Hh0rRlQUAgOcRdkxgj66dRZmwAwCApxF2TOBgFmUAALyGsGOC2nE7ubTsAADgcYQdEziYawcAAK8h7Jjg51mUCTsAAHgaYccEzKIMAID3EHZMwCzKAAB4D2HHBAk/dWMdKipXVXWNydUAABDYCDsmiGtuVXCQRTWG9GNxhdnlAAAQ0Ag7JggOsig+6vjEgnRlAQDgWYQdk9QOUs4rPGZyJQAABDbCjklcg5S5/RwAAI8i7Jjk50dGlJtcCQAAgY2wYxLm2gEAwDsIOyZx2H4aoEw3FgAAHkXYMYkjOkISLTsAAHgaYcckP4/ZKZNhGCZXAwBA4CLsmKT2bqzSimoVlVeZXA0AAIGLsGOSiLBgRYeHSGLcDgAAnkTYMZGrK4uwAwCAxxB2TGTn6ecAAHgcYcdEteN28mnZAQDAYwg7JvrlHVkAAMAzCDsmqg07zLUDAIDnEHZM5GDMDgAAHkfYMZFrgHIhDwMFAMBTCDsmqu3G+rG4XBVVNSZXAwBAYCLsmCi2WZhCgy2SpINFdGUBAOAJhB0TBQVZFB/FIGUAADyJsGOyn2dRZtwOAACeQNgxGXdkAQDgWYQdkzHXDgAAnkXYMZmrZYdHRgAA4BGEHZPZeWQEAAAeRdgxGS07AAB4FmHHZL8coGwYhsnVAAAQeAg7JouPtkqSKqpqVFBaaXI1AAAEHsKOycJDg9WiWagkxu0AAOAJpoad5cuXa9iwYUpMTJTFYtG8efPqbDcMQ1OnTlVCQoIiIiKUnp6u7OzsOvscOXJEo0aNUnR0tGJiYjR27FgVFxd78SzOnp25dgAA8BhTw05JSYl69OihF1544aTbp0+frpkzZ2r27NlavXq1IiMjNXjwYJWV/RwKRo0apS1btmjx4sX69NNPtXz5co0bN85bp+AWCbVz7TBIGQAAtwsx88MzMjKUkZFx0m2GYWjGjBl64IEHlJmZKUmaO3eu7Ha75s2bp5EjR2rbtm1auHCh1qxZoz59+kiSnn/+eV111VV6+umnlZiY6LVzORsObj8HAMBjfHbMzq5du5SXl6f09HTXOpvNptTUVK1cuVKStHLlSsXExLiCjiSlp6crKChIq1evrvfY5eXlcjqddRYz1XZjMYsyAADu57NhJy8vT5Jkt9vrrLfb7a5teXl5io+Pr7M9JCREsbGxrn1OZtq0abLZbK4lKSnJzdU3DHPtAADgOT4bdjxp8uTJKiwsdC379u0ztZ7aWZRzCTsAALidz4Ydh8MhScrPz6+zPj8/37XN4XDo4MGDdbZXVVXpyJEjrn1Oxmq1Kjo6us5iJgfdWAAAeIzPhp3k5GQ5HA4tXbrUtc7pdGr16tVKS0uTJKWlpamgoEDr1q1z7fPFF1+opqZGqampXq+5sWrDztHSSpVVVptcDQAAgcXUu7GKi4u1Y8cO1+tdu3Zp48aNio2NVZs2bTRhwgQ9+uij6tSpk5KTkzVlyhQlJiZq+PDhkqSuXbtqyJAhuvXWWzV79mxVVlbqjjvu0MiRI/3mTixJimkWqrCQIFVU1eigs1xt4pqZXRIAAAHD1LCzdu1aXXHFFa7XkyZNkiSNHj1ar776qu69916VlJRo3LhxKigoUP/+/bVw4UKFh4e73vPmm2/qjjvu0MCBAxUUFKQRI0Zo5syZXj+Xs2GxWJRgC9eew6XKc5YRdgAAcCOLwdMn5XQ6ZbPZVFhYaNr4neteXKlvdx3RzOt76eoe/tMqBQCAWc7097fPjtlpalyDlLkjCwAAtyLs+AhmUQYAwDMIOz7CzsSCAAB4BGHHRzh48jkAAB5B2PERDptVEi07AAC4G2HHR9R2Yx0sKlNNTZO/QQ4AALch7PiI+KhwWSxSZbWhI6UVZpcDAEDAIOz4iLCQIMVF0pUFAIC7EXZ8SO24HR4ICgCA+xB2fAh3ZAEA4H6EHR/CXDsAALgfYceHOAg7AAC4HWHHh9h5ZAQAAG5H2PEhroeBEnYAAHAbwo4PSbDRjQUAgLsRdnxIbTeWs6xKxyqqTa4GAIDAQNjxIVHWEDULC5bEuB0AANyFsONDLBYLd2QBAOBmhB0f45prx3nM5EoAAAgMhB0f43ANUi43uRIAAAIDYcfH2Ln9HAAAtyLs+BhHNE8+BwDAnQg7PsZhi5DE3VgAALgLYcfH1I7ZoRsLAAD3IOz4mNpbzw8Wlau6xjC5GgAA/B9hx8e0bB6mIItUXWPocDF3ZAEAcLYIOz4mJDhIraKOD1LOZZAyAABnjbDjg1yzKDNuBwCAs0bY8UHMtQMAgPsQdnzQz7MoE3YAADhbhB0f5Ao7tOwAAHDWCDs+yEE3FgAAbkPY8UGuAcp0YwEAcNYIOz7I7ppFmXl2AAA4W4QdH1TbslNcXqWiskqTqwEAwL8RdnxQpDVEUdYQSYzbAQDgbBF2fJTddfs5XVkAAJwNwo6PYhZlAADcg7Djoxw2bj8HAMAdCDs+itvPAQBwD58PO0VFRZowYYLatm2riIgI9e3bV2vWrHFtNwxDU6dOVUJCgiIiIpSenq7s7GwTK3YPO7MoAwDgFo0KO8eOHVNpaanr9Z49ezRjxgx9/vnnbius1i233KLFixfr9ddf16ZNmzRo0CClp6frwIEDkqTp06dr5syZmj17tlavXq3IyEgNHjxYZWX+HRKYRRkAAPdoVNjJzMzU3LlzJUkFBQVKTU3VM888o8zMTM2aNcttxR07dkzvv/++pk+frksvvVQdO3bUQw89pI4dO2rWrFkyDEMzZszQAw88oMzMTJ1//vmaO3eucnJyNG/ePLfVYQa6sQAAcI9GhZ3169frkksukST9+9//lt1u1549ezR37lzNnDnTbcVVVVWpurpa4eHhddZHRERoxYoV2rVrl/Ly8pSenu7aZrPZlJqaqpUrV9Z73PLycjmdzjqLr7HbrJKkQ8XlqqyuMbkaAAD8V6PCTmlpqaKioiRJn3/+ua655hoFBQXp4osv1p49e9xWXFRUlNLS0vTII48oJydH1dXVeuONN7Ry5Url5uYqLy9PkmS32+u8z263u7adzLRp02Sz2VxLUlKS22p2l5aRVoUEWWQY0qEi5toBAKCxGhV2OnbsqHnz5mnfvn1atGiRBg0aJEk6ePCgoqOj3Vrg66+/LsMw1Lp1a1mtVs2cOVPXX3+9goIaP7Z68uTJKiwsdC379u1zY8XuERRkUXzU8dYdBikDANB4jUoMU6dO1d1336127dopNTVVaWlpko638vTq1cutBXbo0EHLli1TcXGx9u3bp2+//VaVlZVq3769HA6HJCk/P7/Oe/Lz813bTsZqtSo6OrrO4otcc+0wbgcAgEZrVNj53e9+p71792rt2rVauHCha/3AgQP13HPPua24X4qMjFRCQoKOHj2qRYsWKTMzU8nJyXI4HFq6dKlrP6fTqdWrV7sCmD9zcPs5AABnLaSxb3Q4HCe0nlx00UVnXdCvLVq0SIZhqEuXLtqxY4fuuecepaSk6Oabb5bFYtGECRP06KOPqlOnTkpOTtaUKVOUmJio4cOHu70Wb7PzyAgAAM5ao8JOSUmJnnjiCS1dulQHDx5UTU3du4V27tzpluIkqbCwUJMnT9b+/fsVGxurESNG6LHHHlNoaKgk6d5771VJSYnGjRungoIC9e/fXwsXLjzhDi5/5Jprh24sAAAarVFh55ZbbtGyZct04403KiEhQRaLxd11uVx33XW67rrr6t1usVj0t7/9TX/72988VoNZ6MYCAODsNSrsfPbZZ5o/f7769evn7nrwC3YmFgQA4Kw1aoByixYtFBsb6+5a8CuOX4zZMQzD5GoAAPBPjQo7jzzyiKZOnVrn+Vhwv9purLLKGjmPVZlcDQAA/qlR3VjPPPOMfvjhB9ntdrVr1841WLjW+vXr3VJcUxceGqyYZqEqKK1UnrNMtmahp38TAACoo1FhJxBu6/YXjuhwV9jp4ogyuxwAAPxOg8NOVVWVLBaLxowZo3POOccTNeEX7NHh+j6viNvPAQBopAaP2QkJCdFTTz2lqirGkHiDg4kFAQA4K40aoDxgwAAtW7bM3bXgJOzMtQMAwFlp1JidjIwM3X///dq0aZN69+6tyMjIOtuvvvpqtxSHX7Ts0I0FAECjNCrs3H777ZKkZ5999oRtFotF1dXVZ1cVXBw2qyTCDgAAjdWosPPrZ2HBc2pnUc6nGwsAgEZp1JgdeE+CLUKSdLikQuVVtJgBANBQjWrZOd1DN6dOndqoYnCiFs1CFRYSpIqqGh10lisptpnZJQEA4FcaFXY+/PDDOq8rKyu1a9cuhYSEqEOHDoQdN7JYLLJHW7XvyDHlO8sIOwAANFCjws6GDRtOWOd0OnXTTTfpt7/97VkXhboc0eHad+QYt58DANAIbhuzEx0drYcfflhTpkxx1yHxEzu3nwMA0GhuHaBcWFiowsJCdx4SYq4dAADORqO6sWbOnFnntWEYys3N1euvv66MjAy3FIafOZhFGQCARmtU2HnuuefqvA4KClKrVq00evRoTZ482S2F4WfMtQMAQOM1Kuzs2rXL3XXgFBJo2QEAoNEaNWZnzJgxKioqOmF9SUmJxowZc9ZFoa6fW3bKZRiGydUAAOBfGhV2XnvtNR07duyE9ceOHdPcuXPPuijUVRt2KqpqdLS00uRqAADwLw3qxnI6nTIMQ4ZhqKioSOHh4a5t1dXVWrBggeLj491eZFMXFhKkuMgwHS6pUF5hmWIjw8wuCQAAv9GgsBMTEyOLxSKLxaLOnTufsN1isejhhx92W3H4mT06XIdLKpTvLNO5idFmlwMAgN9oUNj58ssvZRiGBgwYoPfff1+xsbGubWFhYWrbtq0SExPdXiSO336+NdfJIGUAABqoQWHnsssuk3T8bqw2bdrIYrF4pCicqHbcTi4TCwIA0CCNGqDctm1brVixQjfccIP69u2rAwcOSJJef/11rVixwq0F4rjaWZTzCTsAADRIo8LO+++/r8GDBysiIkLr169XeXm5pOOPi3j88cfdWiCOY64dAAAap1Fh59FHH9Xs2bP1z3/+U6Ghoa71/fr10/r1691WHH5mtzGLMgAAjdGosJOVlaVLL730hPU2m00FBQVnWxNOwvUwUMIOAAAN0qiw43A4tGPHjhPWr1ixQu3btz/ronCi2rBTUFqpsspqk6sBAMB/NCrs3Hrrrbrzzju1evVqWSwW5eTk6M0339Rdd92l2267zd01QlJ0RIjCQ49fLrqyAAA4c416EOj999+vmpoaDRw4UKWlpbr00ktltVp1zz336JZbbnF3jdDxCRsd0eHafbhUeYVlahsXaXZJAAD4hUa17FgsFv31r3/VkSNHtHnzZq1atUqHDh2SzWZTcnKyu2vET+yM2wEAoMEaFHbKy8s1efJk9enTR/369dOCBQt07rnnasuWLerSpYv+/ve/a+LEiZ6qtclz1N5+zlw7AACcsQZ1Y02dOlUvvvii0tPT9c033+jaa6/VzTffrFWrVumZZ57Rtddeq+DgYE/V2uQ5mGsHAIAGa1DYee+99zR37lxdffXV2rx5s84//3xVVVXpu+++49ERXuCaRZmwAwDAGWtQN9b+/fvVu3dvSdJ5550nq9WqiRMnEnS8xDXXDt1YAACcsQaFnerqaoWFhbleh4SEqHnz5m4vCif38yzK5SZXAgCA/2hQN5ZhGLrppptktVolSWVlZfrf//1fRUbWvQ36gw8+cF+FcPllN1ZNjaGgIFrUAAA4nQa17IwePVrx8fGy2Wyy2Wy64YYblJiY6Hpdu7hLdXW1pkyZouTkZEVERKhDhw565JFHZBiGax/DMDR16lQlJCQoIiJC6enpys7OdlsNvqRVlFUWi1RVY+hwSYXZ5QAA4Bca1LIzZ84cT9VxUk8++aRmzZql1157Td26ddPatWt18803y2az6c9//rMkafr06Zo5c6Zee+01JScna8qUKRo8eLC2bt2q8PBwr9braaHBQWrZ3KpDReXKKyxTqyir2SUBAODzGjWDsrd88803yszM1NChQyVJ7dq109tvv61vv/1W0vFWnRkzZuiBBx5QZmamJGnu3Lmy2+2aN2+eRo4cedLjlpeXq7z853EvTqfTw2fiPo7o8ONhx1mm7nJfKxoAAIGqUTMoe0vfvn21dOlSbd++XZL03XffacWKFcrIyJAk7dq1S3l5eUpPT3e9x2azKTU1VStXrqz3uNOmTavT7ZaUlOTZE3Ej5toBAKBhfLpl5/7775fT6VRKSoqCg4NVXV2txx57TKNGjZIk5eXlSZLsdnud99ntdte2k5k8ebImTZrkeu10Ov0m8LgGKXP7OQAAZ8Snw867776rN998U2+99Za6deumjRs3asKECUpMTNTo0aMbfVyr1eq6o8zf0LIDAEDD+HTYueeee3T//fe7xt50795de/bs0bRp0zR69Gg5HA5JUn5+vhISElzvy8/PV8+ePc0o2ePszKIMAECD+PSYndLSUgUF1S0xODhYNTU1kqTk5GQ5HA4tXbrUtd3pdGr16tVKS0vzaq3ewizKAAA0jE+37AwbNkyPPfaY2rRpo27dumnDhg169tlnNWbMGEmSxWLRhAkT9Oijj6pTp06uW88TExM1fPhwc4v3EIftePcb3VgAAJwZnw47zz//vKZMmaLbb79dBw8eVGJiov74xz9q6tSprn3uvfdelZSUaNy4cSooKFD//v21cOHCgJtjp1ZtN1ZRWZVKyqsUafXpSwgAgOksxi+nI26inE6nbDabCgsLFR0dbXY5p9Vt6kKVVFRr6V2XqUMrnk0GAGiazvT3t0+P2cHJ1d6Rxe3nAACcHmHHD3H7OQAAZ46w44dqx+0QdgAAOD3Cjh9iFmUAAM4cYccP0Y0FAMCZI+z4oZ+7scpPsycAACDs+KGfZ1E+ZnIlAAD4PsKOH6rtxjpUVK6q6hqTqwEAwLcRdvxQy+ZWBQdZVGNIPxZXmF0OAAA+jbDjh4KDLIqP4hlZAACcCcKOn7Lz9HMAAM4IYcdPuebaoWUHAIBTIuz4KebaAQDgzBB2/JSdWZQBADgjhB0/5bAdH6CcS9gBAOCUCDt+ys6YHQAAzghhx08l2CIkHR+zYxiGydUAAOC7CDt+qvZurNKKaiYWBADgFAg7fioiLFjntY6WJP2//+w0uRoAAHwXYceP3XVlF0nSq9/sZnJBAADqQdjxY5d3aaWL2sWqvKpGf1+abXY5AAD4JMKOH7NYLLp3yPHWnXfX7tPOQ8UmVwQAgO8h7Pi5Pu1iNTAlXtU1hp5dvN3scgAA8DmEnQBw9+AuslikT/+bq80HCs0uBwAAn0LYCQBdE6KV2SNRkjR9UZbJ1QAA4FsIOwFi0pVdFBJk0fLth7Tyh8NmlwMAgM8g7ASINnHNdP1FbSRJ0xd9z6zKAAD8hLATQP40oKMiQoO1YW+Blmw7aHY5AAD4BMJOAImPDteY/u0kSU8t+l7VNbTuAABA2Akw4y7tIFtEqLbnF2vehgNmlwMAgOkIOwHGFhGq2y7vIEl6bsl2lVdVm1wRAADmIuwEoNFp7RQfZdX+o8f09uq9ZpcDAICpCDsBKCIsWHemd5Ik/d+XO1RSXmVyRQAAmIewE6Cu65OkdnHN9GNxhV5ZscvscgAAMA1hJ0CFBgdp0qDjDwl9aflOHS2pMLkiAADMQdgJYL/pnqBzE6JVVF6lWct+MLscAABMQdgJYEFBFt075Hjrzqvf7FZu4TGTKwIAwPsIOwHuss6tdFFyrCqqavT3JdlmlwMAgNcRdgKcxWLRfT+17ry3br9+OFRsckUAAHgXYacJ6N02Vuld41VdY+jZz7ebXQ4AAF7l82GnXbt2slgsJyzjx4+XJJWVlWn8+PGKi4tT8+bNNWLECOXn55tcte+5e3AXWSzS/E252rS/0OxyAADwGp8PO2vWrFFubq5rWbx4sSTp2muvlSRNnDhRn3zyid577z0tW7ZMOTk5uuaaa8ws2SelOKL1256tJUnTF31vcjUAAHiPxTAMv3o09oQJE/Tpp58qOztbTqdTrVq10ltvvaXf/e53kqTvv/9eXbt21cqVK3XxxRef9Bjl5eUqLy93vXY6nUpKSlJhYaGio6O9ch5m2HekVAOe+UqV1YbeujVVfTu0NLskAAAazel0ymaznfb3t8+37PxSRUWF3njjDY0ZM0YWi0Xr1q1TZWWl0tPTXfukpKSoTZs2WrlyZb3HmTZtmmw2m2tJSkryRvmmS4ptpv+5qI0kafrCLPlZzgUAoFH8KuzMmzdPBQUFuummmyRJeXl5CgsLU0xMTJ397Ha78vLy6j3O5MmTVVhY6Fr27dvnwap9yx0DOikiNFgb9xXo862MbQIABD6/Cjsvv/yyMjIylJiYeFbHsVqtio6OrrM0Fa2irBrbP1mS9PSiLFXX0LoDAAhsfhN29uzZoyVLluiWW25xrXM4HKqoqFBBQUGdffPz8+VwOLxcof8Yd1l7xTQLVfbBYn244YDZ5QAA4FF+E3bmzJmj+Ph4DR061LWud+/eCg0N1dKlS13rsrKytHfvXqWlpZlRpl+IDg/VbZd1kCQ9t3i7yquqTa4IAADP8YuwU1NTozlz5mj06NEKCQlxrbfZbBo7dqwmTZqkL7/8UuvWrdPNN9+stLS0eu/EwnGj+7aTPdqqAwXH9NbqvWaXAwCAx/hF2FmyZIn27t2rMWPGnLDtueee029+8xuNGDFCl156qRwOhz744AMTqvQv4aHBunNgZ0nS/32xQ8XlVSZXBACAZ/jdPDuecKb36QeayuoaDXpuuXb9WKJJV3bWnwd2MrskAADOWEDOswP3Cg0O0l2DjrfuvLR8p46UVJhcEQAA7kfYaeKuOi9B3RKjVVxepX98ucPscgAAcDvCThMXFGTRvUNSJElzV+1RTsExkysCAMC9CDvQpZ1a6uL2saqoqtHfl2SbXQ4AAG5F2IEslp9bd95bt087DhabXBEAAO5D2IEk6YI2LXTluXbVGNKzi7PMLgcAALch7MDl7kFdZLFICzbl6b/7C8wuBwAAtyDswKWLI0q/7dVakvTUIlp3AACBgbCDOiamd1ZosEX/yf5R3+z40exyAAA4a4Qd1JEU20yjUttKkp5clCUm2AYA+DvCDk4w/oqOahYWrO/2FWjRlnyzywEA4KyEnH4XNDWtoqwa2z9Zz3+xQ89/ka0h5znMLgl+5KCzTJsOFIpGQQC/1LNNjFo2t5ry2YQdnNTY/smavewHbclxant+kTrbo8wuCT6uuLxKs77aof/3n10qr6oxuxwAPmbumIt0aedWpnw2YQcnFdMsTJd1jteSbfn6aOMB3TM4xeyS4KOqqmv0zpp9mrFku34sPv4w2Q6tIhUVHmpyZQB8SVS4eZGDsIN6De+V+FPYyflpDh6L2SXBhxiGoa+yDunxBduU/dOs2+1bRur+jBRdea6dnxcAPoOwg3oNTLErMixY+48e0/q9R9W7bazZJcFHbM1x6vEF27Tip+kJWjQL1Z0DO2nUxW0VGsx9DwB8C2EH9YoIC9bgbg59sOGAPtqYQ9iB8p1lenpRlv69fr8MQwoLDtLN/drp9is6yhZBtxUA30TYwSll9mqtDzYc0Pz/5mrKb87lf+1NVGlFlV5ctlMvLd+pY5XVkqTfnJ+g+4akKCm2mcnVAcCpEXZwSv06xKll8zD9WFyhFTt+1BVd4s0uCV5UXWPo/XX79fTnWTpYVC5J6t22hf46tKsuaNPC5OoA4MwQdnBKIcFB+s35iXr1m936eGMOYacJ+U/2IT02f5u+zyuSJLWJbab7M1KUcZ6DwccA/AphB6d1dc/jYWfRljyVVlSpWRg/NoFse36RHl+wTV9lHZIkRYeH6M8DO+nGtLayhgSbXB0ANBy/tXBavZJi1Ca2mfYeKdWSbQd1dY9Es0uCBxwqKtezi7frX2v2qsaQQoMtuvHidvrzwI6KaRZmdnkA0GiEHZyWxWJRZs9EPf/FDn288QBhJ8Acq6jWyyt2atZXP6ik4vjg4yHdHLo/I0XtWkaaXB0AnD3CDs5Ibdj5KuuQjpZUqEUk/9P3dzU1huZtPKCnFmUpt7BMktTjHJv+OvRcXZTMNAMAAgdhB2ekY3yUuiVGa0uOUws252pUaluzS8JZWPnDYT22YKs2H3BKklrHROjeIV007PxEBQUx+BhAYCHs4Ixl9kzUlhynPtqQQ9jxUz8cKta0Bd9rybZ8SVKUNUS3X9FRN/drp/BQBh8DCEyEHZyxYT0SNe2z7/Xt7iM6UHBMrWMizC4JZ+hwcbn+vjRbb67eq+oaQ8FBFv3PRW00Ib2T4ppbzS4PADyKsIMzlmCLUGpyrFbtPKKPN+botss7mF0STqOsslqvfrNbL3yxQ0XlVZKk9K7xuj+jqzrGNze5OgDwDsIOGiSzZ2ut2nlEH208QNjxYYZh6JP/5urJz77XgYJjkqRuidH669Cu6tuhpcnVAYB3EXbQIFedl6CpH23W93lFysorUhdHlNkl4VfW7j6iR+Zv03f7CiRJjuhw3TO4i37bqzWDjwE0SYQdNIitWagu7xKvxVvz9dHGA7p3SIrZJeEnew6X6InPvtdnm/MkSc3CgnXbZR10yyXtFRHG4GMATRdhBw02vGfrn8JOju4Z3IXnJJmsoLRCM5fu0Ourdquy2lCQRfr9hW008cpOio8KN7s8ADAdYQcNNrBrvCLDgnWg4JjW7TmqPu2YgM4MFVU1mrtyt57/YocKj1VKki7r3Ep/uaor3YsA8AuEHTRYeGiwBp/n0AfrD+ijjTmEHS8zDEMLN+fpiYXfa8/hUklSiiNKf7mqqy7t3Mrk6gDA9xB20CjDe7bWB+sPaP6mXE0ddq5Cg4PMLqlJ2LD3qB6bv01r9xyVJLWKsuruQZ31u95JCmbwMQCcFGEHjdK3Q5xaNg/Tj8UVWpH9o65IiTe7pIC270ippi/K0iff5UiSwkODNO7SDvrjpe0VaeVrDACnwr+SaJSQ4CD95vxEvfrNbn208QBhx0MKj1XqH1/u0Jyvd6uiukYWi/S7C87RXYO6yGFj8DEAnAnCDhots+fxsPP51nyVVlSpWRg/Tu5SWV2jt1bv1Ywl23W09Pjg434d4/SXq7qqW6LN5OoAwL/w2wmN1jMpRm3jmmnP4VIt3pqvzJ6tzS7J7xmGoSXbDmraZ9u081CJJKljfHP95aoUXdElntv8AaARCDtoNIvFosweiZr5xQ59vDGHsHOWNu0v1GMLtmrVziOSpLjIME28srNGXpikEAaAA0Cj+fy/oAcOHNANN9yguLg4RUREqHv37lq7dq1ru2EYmjp1qhISEhQREaH09HRlZ2ebWHHTcvVPAWfZ9kM6UlJhcjX+KafgmCb9a6OG/d8Krdp5RNaQIN1+eQd9dc/luuHitgQdADhLPv2v6NGjR9WvXz+Fhobqs88+09atW/XMM8+oRYsWrn2mT5+umTNnavbs2Vq9erUiIyM1ePBglZWVmVh509ExvrnOax2tqhpDCzblml2OXykur9JTi77XFU9/pQ82HJAk/bZXa31x9+W6d0iKosJDTa4QAAKDT3djPfnkk0pKStKcOXNc65KTk11/NgxDM2bM0AMPPKDMzExJ0ty5c2W32zVv3jyNHDnypMctLy9XeXm567XT6fTQGTQNmT1aa/MBpz7aeEA3XNzW7HJ8XlV1jf61dp+eW7xdPxYfbw27KDlWDwztqvPPiTG3OAAIQD7dsvPxxx+rT58+uvbaaxUfH69evXrpn//8p2v7rl27lJeXp/T0dNc6m82m1NRUrVy5st7jTps2TTabzbUkJSV59DwC3bAeibJYpDW7j2r/0VKzy/FZhmHoy6yDyvj7f/TXDzfrx+IKJbeM1Is39ta/xl1M0AEAD/HpsLNz507NmjVLnTp10qJFi3Tbbbfpz3/+s1577TVJUl7e8ac72+32Ou+z2+2ubSczefJkFRYWupZ9+/Z57iSaAIctXBcnx0mSPvmOrqyT2Zrj1I0vf6ub56xR9sFixTQL1YPDztWiCZdqcDcHd1kBgAf5dDdWTU2N+vTpo8cff1yS1KtXL23evFmzZ8/W6NGjG31cq9Uqq9XqrjKh43PurNx5WB9tPKDbLu9gdjk+I99Zpmc+z9J76/bLMKSw4CDd1K+dxl/eUbZmjMkBAG/w6ZadhIQEnXvuuXXWde3aVXv37pUkORwOSVJ+fn6dffLz813b4B0Z5yUoLDhI3+cV6fs8xkCVVlRpxpLtuvypr/Tu2uNBZ+j5CVoy6TL95aquBB0A8CKfDjv9+vVTVlZWnXXbt29X27bHB8EmJyfL4XBo6dKlru1Op1OrV69WWlqaV2tt6mzNQnV5l+NP3P5oY47J1ZinusbQu2v26fKnvtKMJdk6VlmtC9rE6P3b+uqF/7lAbeKamV0iADQ5Pt2NNXHiRPXt21ePP/64rrvuOn377bd66aWX9NJLL0k6PqndhAkT9Oijj6pTp05KTk7WlClTlJiYqOHDh5tbfBOU2bO1Pt+ar4835uieQV0U1MSewr0i+0c9On+rvs8rkiQlxUbo/iFddVV3xuQAgJl8OuxceOGF+vDDDzV58mT97W9/U3JysmbMmKFRo0a59rn33ntVUlKicePGqaCgQP3799fChQsVHs5DEr1tYNd4NbeG6EDBMa3fe1R92sWaXZJXbM8v0uMLtumrrEOSpOjwEP1pQCf9oW9bWUOCTa4OAGAxDMMwuwizOZ1O2Ww2FRYWKjo62uxy/Npd736n99fv1w0Xt9Gjw7ubXY5HHSoq13NLtuudb/eqxpBCgiy6Ma2t/jygk1pEhpldHgAEvDP9/e3TLTvwP8N7Jer99fs1/7+5enBYN4UG4KMOyiqr9fKKXfrHlztUUlEtSRrcza77M7oquWWkydUBAH6NsAO3Smsfp5bNrfqxuFz/yT6kASn207/JT9TUGJq38YCeWpSl3MLjjyM5/xyb/npVV6W2jzO5OgBAfQg7cKuQ4CAN65GgOV/v1kcbcwIm7KzaeViPzd+mTQcKJUmtYyJ0z+AuurpHYpMbiA0A/oawA7fL7Nlac77erc+35Ku0okrNwvz3x+yHQ8WatuB7Ldl2fC6n5tYQ3X5FB43pl6zwUAYfA4A/8N/fQvBZPc6xqW1cM+05XKrFW/OV2bO12SU12JGSCv19yXa9uXqvqmoMBQdZ9D8XtdGd6Z3UsjmzbwOAPyHswO0sFosye7bWzKXZ+mhjjl+FnbLKar32zW7935c7VFRWJUkamBKvyVelqGN8lMnVAQAag7ADj8jsmaiZS7O1fPshHSmpUKyP34ptGIY++W+upi/8XvuPHpMknZsQrQeGdlXfji1Nrg4AcDYIO/CIDq2aq3trmzYdKNT8Tbm68eK2ZpdUr7W7j+jR+du0cV+BJMkRHa67B3fRNb1aM/gYAAIAYQcek9kzUZsOFOqjDQd8MuzsOVyiJz77Xp9tzpMkNQsL1v9e1kG3XtJeEWEMPgaAQEHYgcf85vxEPbZgm9buOap9R0qVFOsbD8EsKK3Q81/s0NyVu1VZbSjIIv3+wiRNTO+s+GgeMwIAgYawA49x2MKV1j5O3/xwWJ/8N0e3X97R1Hoqqmr0+qo9mrk0W4XHKiVJl3Zupb9claIUB48JAYBARdiBR2X2TNQ3PxzWRxvMCzuGYWjh5jw9sfB77TlcKklKcURp8lVddVnnVqbUBADwHsIOPGrIeQmaMm+LsvKL9H2e0+stKBv3Feix+Vu1ZvdRSVKrKKvuurKzru2TpGAGHwNAk0DYgUfZIkJ1RUorLdqSr3kbcnR/hnfCzv6jpZq+MEsff5cjSQoPDdK4Szvoj5e2V6SVH3sAaEr4Vx8el9mztRZtydcn3+Xo3sFdPHo7t7OsUi98uUNzvt6tiqoaWSzSiAvO0d2DushhY/AxADRFhB143ICUeEVZQ3Sg4JjW7T2qC9vFuv0zKqtr9Pa3ezVjSbaOlFRIkvp2iNNfh3ZVt0Sb2z8PAOA/CDvwuPDQYA0+z6F/r9uveRsOuDXsGIahJdsOatpn27TzUIkkqWN8c/3lqhRd0SVeFgvjcgCgqSPswCuG92ytf6/br/mbcvXgsG4KCwk662NuPlCoR+dv1aqdRyRJcZFhmnBlZ11/YZJCgs/++ACAwEDYgVekdYhTy+ZW/Vhcrv9kH9LArvZGHyun4JieXpSlDzYckCSFhQTplv7Juu3yDooKD3VXyQCAAEHYgVcEB1k0rEeC5ny9Wx9tzGlU2Ckur9Lsr37QP/+zU+VVNZKk4T0Tdc+QFLWOiXB3yQCAAEHYgdcM79lac77ercVb81VSXnXGt4BXVdfo3bX79ezi7fqxuFySdFG7WP11aFf1SIrxYMUAgEBA2IHXnH+OTe3immn34VIt3pqv4b1an3J/wzD01fZDmrZgm7bnF0uS2sU10/0ZXTW4m53BxwCAM0LYgddYLBZl9mytvy/N1kcbD5wy7GzLderxBdv0n+wfJUkxzUJ158BOGpXa1i2DmwEATQdhB151dc9E/X1ptpZn/6jDxeWKa26tsz3fWaZnP9+ud9ftk2FIYcFBGt23re64opNszRh8DABoOMIOvKpDq+bq3tqmTQcKtWBTrm5MaydJKq2o0kvLd+rFZTt1rLJakjT0/ATdNzhFbeKamVgxAMDfEXbgdZk9E7XpQKE+2pij/0ltq/fX79czn2cp33l88HGvNjF6YOi56t22hcmVAgACgcUwDMPsIszmdDpls9lUWFio6GjvPpW7Kcp3luniaUtlGFIXe5Sy8oskSUmxEbpvSIqGdk9g8DEA4LTO9Pc3LTvwOnt0uPp2iNPXOw4rK79IUeEh+vOATvpD37ayhgSbXR4AIMAQdmCKO67opP1Hj+mKLvG6c2AntYgMM7skAECAohtLdGMBAOCPzvT3NxOWAACAgEbYAQAAAY2wAwAAAhphBwAABDTCDgAACGiEHQAAENAIOwAAIKARdgAAQEAj7AAAgIBG2AEAAAHN58POQw89JIvFUmdJSUlxbS8rK9P48eMVFxen5s2ba8SIEcrPzzexYgAA4Et8PuxIUrdu3ZSbm+taVqxY4do2ceJEffLJJ3rvvfe0bNky5eTk6JprrjGxWgAA4Ev84qnnISEhcjgcJ6wvLCzUyy+/rLfeeksDBgyQJM2ZM0ddu3bVqlWrdPHFF5/0eOXl5SovL3e9djqdnikcAACYzi9adrKzs5WYmKj27dtr1KhR2rt3ryRp3bp1qqysVHp6umvflJQUtWnTRitXrqz3eNOmTZPNZnMtSUlJHj8HAABgDp9v2UlNTdWrr76qLl26KDc3Vw8//LAuueQSbd68WXl5eQoLC1NMTEyd99jtduXl5dV7zMmTJ2vSpEmu14WFhWrTpg0tPAAA+JHa39uGYZxyP58POxkZGa4/n3/++UpNTVXbtm317rvvKiIiolHHtFqtslqtrte1f1m08AAA4H+Kiopks9nq3e7zYefXYmJi1LlzZ+3YsUNXXnmlKioqVFBQUKd1Jz8//6RjfOqTmJioffv2KSoqShaLxW21Op1OJSUlad++fYqOjnbbcX1VUzpfzjVwNaXz5VwDV1M5X8MwVFRUpMTExFPu53dhp7i4WD/88INuvPFG9e7dW6GhoVq6dKlGjBghScrKytLevXuVlpZ2xscMCgrSOeec46mSFR0dHdA/bL/WlM6Xcw1cTel8OdfA1RTO91QtOrV8PuzcfffdGjZsmNq2baucnBw9+OCDCg4O1vXXXy+bzaaxY8dq0qRJio2NVXR0tP70pz8pLS2t3juxAABA0+LzYWf//v26/vrrdfjwYbVq1Ur9+/fXqlWr1KpVK0nSc889p6CgII0YMULl5eUaPHiw/vGPf5hcNQAA8BU+H3beeeedU24PDw/XCy+8oBdeeMFLFZ05q9WqBx98sM5g6EDWlM6Xcw1cTel8OdfA1dTO93Qsxunu1wIAAPBjfjGpIAAAQGMRdgAAQEAj7AAAgIBG2AEAAAGNsHOWXnjhBbVr107h4eFKTU3Vt99+e8r933vvPaWkpCg8PFzdu3fXggULvFTp2Zk2bZouvPBCRUVFKT4+XsOHD1dWVtYp3/Pqq6/KYrHUWcLDw71UceM99NBDJ9SdkpJyyvf463WVpHbt2p1wvhaLRePHjz/p/v50XZcvX65hw4YpMTFRFotF8+bNq7PdMAxNnTpVCQkJioiIUHp6urKzs0973IZ+773hVOdaWVmp++67T927d1dkZKQSExP1hz/8QTk5Oac8ZmO+C95wuut60003nVD3kCFDTntcX7yu0unP92TfX4vFoqeeeqreY/rqtfUUws5Z+Ne//qVJkybpwQcf1Pr169WjRw8NHjxYBw8ePOn+33zzja6//nqNHTtWGzZs0PDhwzV8+HBt3rzZy5U33LJlyzR+/HitWrVKixcvVmVlpQYNGqSSkpJTvi86Olq5ubmuZc+ePV6q+Ox069atTt0rVqyod19/vq6StGbNmjrnunjxYknStddeW+97/OW6lpSUqEePHvVOTTF9+nTNnDlTs2fP1urVqxUZGanBgwerrKys3mM29HvvLac619LSUq1fv15TpkzR+vXr9cEHHygrK0tXX331aY/bkO+Ct5zuukrSkCFD6tT99ttvn/KYvnpdpdOf7y/PMzc3V6+88oosFovryQL18cVr6zEGGu2iiy4yxo8f73pdXV1tJCYmGtOmTTvp/tddd50xdOjQOutSU1ONP/7xjx6t0xMOHjxoSDKWLVtW7z5z5swxbDab94pykwcffNDo0aPHGe8fSNfVMAzjzjvvNDp06GDU1NScdLu/XldJxocffuh6XVNTYzgcDuOpp55yrSsoKDCsVqvx9ttv13uchn7vzfDrcz2Zb7/91pBk7Nmzp959GvpdMMPJznX06NFGZmZmg47jD9fVMM7s2mZmZhoDBgw45T7+cG3diZadRqqoqNC6deuUnp7uWhcUFKT09HStXLnypO9ZuXJlnf0lafDgwfXu78sKCwslSbGxsafcr7i4WG3btlVSUpIyMzO1ZcsWb5R31rKzs5WYmKj27dtr1KhR2rt3b737BtJ1raio0BtvvKExY8ac8qG4/npdf2nXrl3Ky8urc+1sNptSU1PrvXaN+d77qsLCQlksljoPUT6ZhnwXfMlXX32l+Ph4denSRbfddpsOHz5c776BdF3z8/M1f/58jR079rT7+uu1bQzCTiP9+OOPqq6ult1ur7PebrcrLy/vpO/Jy8tr0P6+qqamRhMmTFC/fv103nnn1btfly5d9Morr+ijjz7SG2+8oZqaGvXt21f79+/3YrUNl5qaqldffVULFy7UrFmztGvXLl1yySUqKio66f6Bcl0lad68eSooKNBNN91U7z7+el1/rfb6NOTaNeZ774vKysp033336frrrz/lQyIb+l3wFUOGDNHcuXO1dOlSPfnkk1q2bJkyMjJUXV190v0D5bpK0muvvaaoqChdc801p9zPX69tY/n84yLge8aPH6/Nmzeftn83LS2tztPn+/btq65du+rFF1/UI4884ukyGy0jI8P15/PPP1+pqalq27at3n333TP635I/e/nll5WRkaHExMR69/HX64rjKisrdd1118kwDM2aNeuU+/rrd2HkyJGuP3fv3l3nn3++OnTooK+++koDBw40sTLPe+WVVzRq1KjT3jTgr9e2sWjZaaSWLVsqODhY+fn5ddbn5+fL4XCc9D0Oh6NB+/uiO+64Q59++qm+/PJLnXPOOQ16b2hoqHr16qUdO3Z4qDrPiImJUefOneutOxCuqyTt2bNHS5Ys0S233NKg9/nrda29Pg25do353vuS2qCzZ88eLV68+JStOidzuu+Cr2rfvr1atmxZb93+fl1r/ec//1FWVlaDv8OS/17bM0XYaaSwsDD17t1bS5cuda2rqanR0qVL6/yv95fS0tLq7C9Jixcvrnd/X2IYhu644w59+OGH+uKLL5ScnNzgY1RXV2vTpk1KSEjwQIWeU1xcrB9++KHeuv35uv7SnDlzFB8fr6FDhzboff56XZOTk+VwOOpcO6fTqdWrV9d77RrzvfcVtUEnOztbS5YsUVxcXIOPcbrvgq/av3+/Dh8+XG/d/nxdf+nll19W79691aNHjwa/11+v7Rkze4S0P3vnnXcMq9VqvPrqq8bWrVuNcePGGTExMUZeXp5hGIZx4403Gvfff79r/6+//toICQkxnn76aWPbtm3Ggw8+aISGhhqbNm0y6xTO2G233WbYbDbjq6++MnJzc11LaWmpa59fn+/DDz9sLFq0yPjhhx+MdevWGSNHjjTCw8ONLVu2mHEKZ+yuu+4yvvrqK2PXrl3G119/baSnpxstW7Y0Dh48aBhGYF3XWtXV1UabNm2M++6774Rt/nxdi4qKjA0bNhgbNmwwJBnPPvussWHDBtcdSE888YQRExNjfPTRR8Z///tfIzMz00hOTjaOHTvmOsaAAQOM559/3vX6dN97s5zqXCsqKoyrr77aOOecc4yNGzfW+Q6Xl5e7jvHrcz3dd8EspzrXoqIi4+677zZWrlxp7Nq1y1iyZIlxwQUXGJ06dTLKyspcx/CX62oYp/85NgzDKCwsNJo1a2bMmjXrpMfwl2vrKYSds/T8888bbdq0McLCwoyLLrrIWLVqlWvbZZddZowePbrO/u+++67RuXNnIywszOjWrZsxf/58L1fcOJJOusyZM8e1z6/Pd8KECa6/G7vdblx11VXG+vXrvV98A/3+9783EhISjLCwMKN169bG73//e2PHjh2u7YF0XWstWrTIkGRkZWWdsM2fr+uXX3550p/b2vOpqakxpkyZYtjtdsNqtRoDBw484e+gbdu2xoMPPlhn3am+92Y51bnu2rWr3u/wl19+6TrGr8/1dN8Fs5zqXEtLS41BgwYZrVq1MkJDQ422bdsat9566wmhxV+uq2Gc/ufYMAzjxRdfNCIiIoyCgoKTHsNfrq2nWAzDMDzadAQAAGAixuwAAICARtgBAAABjbADAAACGmEHAAAENMIOAAAIaIQdAAAQ0Ag7AAAgoBF2AABAQCPsAPAbu3fvlsVi0caNGz32GTfddJOGDx/useMD8D7CDgCvuemmm2SxWE5YhgwZckbvT0pKUm5urs477zwPVwogkISYXQCApmXIkCGaM2dOnXVWq/WM3hscHCyHw+GJsgAEMFp2AHiV1WqVw+Gos7Ro0UKSZLFYNGvWLGVkZCgiIkLt27fXv//9b9d7f92NdfToUY0aNUqtWrVSRESEOnXqVCdIbdq0SQMGDFBERITi4uI0btw4FRcXu7ZXV1dr0qRJiomJUVxcnO699179+nGBNTU1mjZtmpKTkxUREaEePXrUqel0NQAwH2EHgE+ZMmWKRowYoe+++06jRo3SyJEjtW3btnr33bp1qz777DNt27ZNs2bNUsuWLSVJJSUlGjx4sFq0aKE1a9bovffe05IlS3THHXe43v/MM8/o1Vdf1SuvvKIVK1boyJEj+vDDD+t8xrRp0zR37lzNnj1bW7Zs0cSJE3XDDTdo2bJlp60BgI8w+anrAJqQ0aNHG8HBwUZkZGSd5bHHHjMMwzAkGf/7v/9b5z2pqanGbbfdZhiGYezatcuQZGzYsMEwDMMYNmyYcfPNN5/0s1566SWjRYsWRnFxsWvd/PnzjaCgICMvL88wDMNISEgwpk+f7tpeWVlpnHPOOUZmZqZhGIZRVlZmNGvWzPjmm2/qHHvs2LHG9ddff9oaAPgGxuwA8KorrrhCs2bNqrMuNjbW9ee0tLQ629LS0uq9++q2227TiBEjtH79eg0aNEjDhw9X3759JUnbtm1Tjx49FBkZ6dq/X79+qqmpUVZWlsLDw5Wbm6vU1FTX9pCQEPXp08fVlbVjxw6VlpbqyiuvrPO5FRUV6tWr12lrAOAbCDsAvCoyMlIdO3Z0y7EyMjK0Z88eLViwQIsXL9bAgQM1fvx4Pf300245fu34nvnz56t169Z1ttUOqvZ0DQDOHmN2APiUVatWnfC6a9eu9e7fqlUrjR49Wm+88YZmzJihl156SZLUtWtXfffddyopKXHt+/XXXysoKEhdunSRzWZTQkKCVq9e7dpeVVWldevWuV6fe+65slqt2rt3rzp27FhnSUpKOm0NAHwDLTsAvKq8vFx5eXl11oWEhLgG9b733nvq06eP+vfvrzfffFPffvutXn755ZMea+rUqerdu7e6deum8vJyffrpp65gNGrUKD344IMaPXq0HnroIR06dEh/+tOfdOONN8put0uS7rzzTj3xxBPq1KmTUlJS9Oyzz6qgoMB1/KioKN19992aOHGiampq1L9/fxUWFurrr79WdHS0Ro8efcoaAPgGwg4Ar1q4cKESEhLqrOvSpYu+//57SdLDDz+sd955R7fffrsSEhL09ttv69xzzz3pscLCwjR58mTt3r1bERERuuSSS/TOO+9Ikpo1a6ZFixbpzjvv1IUXXqhmzZppxIgRevbZZ13vv+uuu5Sbm6vRo0crKChIY8aM0W9/+1sVFha69nnkkUfUqlUrTZs2TTt37lRMTIwuuOAC/eUvfzltDQB8g8UwfjWpBACYxGKx6MMPP+RxDQDcijE7AAAgoBF2AABAQGPMDgCfQa86AE+gZQcAAAQ0wg4AAAhohB0AABDQCDsAACCgEXYAAEBAI+wAAICARtgBAAABjbADAAAC2v8HZqcfswUxwakAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "episodes_list = list(range(len(return_list)))\n",
    "mv_return = rl_utils.moving_average(return_list, 5)\n",
    "plt.plot(episodes_list, mv_return)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('DQN on {}'.format(env_name))\n",
    "plt.show()\n",
    "\n",
    "frames_list = list(range(len(max_q_value_list)))\n",
    "plt.plot(frames_list, max_q_value_list)\n",
    "plt.axhline(0, c='orange', ls='--')\n",
    "plt.axhline(10, c='red', ls='--')\n",
    "plt.xlabel('Frames')\n",
    "plt.ylabel('Q value')\n",
    "plt.title('DQN on {}'.format(env_name))\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "211a6245",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(0)\n",
    "np.random.seed(0)\n",
    "env.seed(0)\n",
    "torch.manual_seed(0)\n",
    "replay_buffer = rl_utils.ReplayBuffer(buffer_size)\n",
    "agent = DQN(state_dim, hidden_dim, action_dim, lr, gamma, epsilon,\n",
    "            target_update, device, 'DoubleDQN')\n",
    "return_list, max_q_value_list = train_DQN(agent, env, num_episodes,\n",
    "                                          replay_buffer, minimal_size,\n",
    "                                          batch_size)\n",
    "\n",
    "episodes_list = list(range(len(return_list)))\n",
    "mv_return = rl_utils.moving_average(return_list, 5)\n",
    "plt.plot(episodes_list, mv_return)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('Double DQN on {}'.format(env_name))\n",
    "plt.show()\n",
    "\n",
    "frames_list = list(range(len(max_q_value_list)))\n",
    "plt.plot(frames_list, max_q_value_list)\n",
    "plt.axhline(0, c='orange', ls='--')\n",
    "plt.axhline(10, c='red', ls='--')\n",
    "plt.xlabel('Frames')\n",
    "plt.ylabel('Q value')\n",
    "plt.title('Double DQN on {}'.format(env_name))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "917790dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40988978",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}