Optimization_A2C.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1zhicPEYwYND"
      },
      "outputs": [],
      "source": [
        "import gym\n",
        "import tensorflow as tf\n",
        "import tensorflow.compat.v1 as tfc\n",
        "import numpy as np\n",
        "import random\n",
        "from collections import deque\n",
        "from sklearn.ensemble import GradientBoostingRegressor\n",
        "from sklearn.model_selection import train_test_split\n",
        "import random\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1B9-e2sjw6TC"
      },
      "outputs": [],
      "source": [
        "# Hyper Parameters\n",
        "GAMMA = 0.95 # discount factor\n",
        "LEARNING_RATE=0.01\n",
        "bound = np.vstack((np.zeros((30)),np.ones((30))))\n",
        "tfc.disable_eager_execution()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WseLyo0ijxwu",
        "outputId": "121e6e90-e641-422b-f770-cf10454d08c9"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "GradientBoostingRegressor(max_depth=16, min_samples_split=18)"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# data=pd.read_csv('/content/drive/MyDrive/Colab Notebooks/NOx-216.csv',index_col='Time') # 11 features\n",
        "data=pd.read_csv('/content/drive/MyDrive/Colab Notebooks/NOx-Data/MIC_0.15_in_regression.csv',index_col='Time') # 30 features\n",
        "df= data.values\n",
        "df[:,1:] = (df[:,1:]-df[:,1:].min(axis=0))/(df[:,1:].max(axis=0)-df[:,1:].min(axis=0))\n",
        "X_train, X_test, y_train, y_test = train_test_split(df[:,1:],df[:,0], test_size=0.2)\n",
        "\n",
        "model = GradientBoostingRegressor(min_samples_split=18, max_depth=16, learning_rate=0.1)\n",
        "model.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UENQLHp4K84i"
      },
      "outputs": [],
      "source": [
        "class NOXEnv(gym.Env):\n",
        "  def __init__(self):\n",
        "    self.action_space = gym.spaces.Discrete(60)\n",
        "    self.observation_space = gym.spaces.Box(low=0, high=1, shape=(30,1), dtype=np.float32)\n",
        "\n",
        "  def reset(self):\n",
        "    temp = random.randint(0,X_test.shape[0]-1)\n",
        "    self.state = X_test[temp,:]\n",
        "    print(y_test[temp])\n",
        "    return self.state\n",
        "\n",
        "  def step(self, state, action):\n",
        "    reward = 0.0\n",
        "    done = False\n",
        "\n",
        "    if action%2 == 0: # 偶数为增加，奇数为减少，action的范围是[0,21]\n",
        "      state[int(action/2)] = state[int(action/2)] + 0.1*(bound[1, int(action/2)] - state[int(action/2)]) # increase action\n",
        "    else:\n",
        "      state[int(action/2)] = state[int(action/2)] - 0.1*(state[int(action/2)] - bound[0, int(action/2)]) # decrease action\n",
        "\n",
        "    self.state = state\n",
        "\n",
        "    output = model.predict(state[np.newaxis,:])\n",
        "    # reward = 3*state[5] + 2*state[2] - state[0] - state[1] - output[0]/10 # consumed NOx + actual feed - ammonia - furnace temperature - boiler pressure\n",
        "    reward = -output[0]\n",
        "\n",
        "    if output[0] <= 150:\n",
        "      done = True\n",
        "\n",
        "    return self.state, reward, done, output[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W1nuKeS2w3k-"
      },
      "outputs": [],
      "source": [
        "class Actor():\n",
        "    def __init__(self, env, sess):\n",
        "        # init some parameters\n",
        "        self.time_step = 0\n",
        "        self.state_dim = env.observation_space.shape[0]\n",
        "        self.action_dim = env.action_space.n\n",
        "        self.create_softmax_network()\n",
        "\n",
        "        # Init session\n",
        "        self.session = sess\n",
        "        self.session.run(tfc.global_variables_initializer())\n",
        "\n",
        "    def create_softmax_network(self):\n",
        "        # network weights\n",
        "        W1 = self.weight_variable([self.state_dim, 20])\n",
        "        b1 = self.bias_variable([20])\n",
        "        W2 = self.weight_variable([20, self.action_dim])\n",
        "        b2 = self.bias_variable([self.action_dim])\n",
        "        # input layer\n",
        "        self.state_input = tfc.placeholder(\"float\", [None, self.state_dim])\n",
        "        self.tf_acts = tfc.placeholder(tf.int32, [None,60], name=\"actions_num\")\n",
        "        self.td_error = tfc.placeholder(tf.float32, None, \"td_error\")  # TD_error\n",
        "        # hidden layers\n",
        "        h_layer = tf.nn.relu(tf.matmul(self.state_input, W1) + b1)\n",
        "        # softmax layer\n",
        "        self.softmax_input = tf.matmul(h_layer, W2) + b2\n",
        "        # softmax output\n",
        "        self.all_act_prob = tf.nn.softmax(self.softmax_input, name='act_prob')\n",
        "\n",
        "        self.neg_log_prob = tf.nn.softmax_cross_entropy_with_logits(logits=self.softmax_input,\n",
        "                                                                           labels=self.tf_acts)\n",
        "        self.exp = tf.reduce_mean(self.neg_log_prob * self.td_error)\n",
        "\n",
        "        #这里需要最大化当前策略的价值，因此需要最大化self.exp,即最小化-self.exp\n",
        "        self.train_op = tfc.train.AdamOptimizer(LEARNING_RATE).minimize(-self.exp)\n",
        "\n",
        "    def weight_variable(self, shape):\n",
        "        initial = tfc.truncated_normal(shape)\n",
        "        return tf.Variable(initial)\n",
        "\n",
        "    def bias_variable(self, shape):\n",
        "        initial = tf.constant(0.01, shape=shape)\n",
        "        return tf.Variable(initial)\n",
        "\n",
        "    def choose_action(self, observation):\n",
        "        prob_weights = self.session.run(self.all_act_prob, feed_dict={self.state_input: observation[np.newaxis, :]})\n",
        "        action = np.random.choice(range(prob_weights.shape[1]), p=prob_weights.ravel())  # select action w.r.t the actions prob\n",
        "        return action\n",
        "\n",
        "    def learn(self, state, action, td_error):\n",
        "        s = state[np.newaxis, :]\n",
        "        one_hot_action = np.zeros(self.action_dim)\n",
        "        one_hot_action[action] = 1\n",
        "        a = one_hot_action[np.newaxis, :]\n",
        "        # train on episode\n",
        "        self.session.run(self.train_op, feed_dict={\n",
        "             self.state_input: s,\n",
        "             self.tf_acts: a,\n",
        "             self.td_error: td_error,\n",
        "        })"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mzdLzYBVw_GD"
      },
      "outputs": [],
      "source": [
        "EPSILON = 0.01 # final value of epsilon\n",
        "REPLAY_SIZE = 10000 # experience replay buffer size\n",
        "BATCH_SIZE = 32 # size of minibatch\n",
        "REPLACE_TARGET_FREQ = 10 # frequency to update target Q network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QTAmqjKOwuY7"
      },
      "outputs": [],
      "source": [
        "class Critic():\n",
        "    def __init__(self, env, sess):\n",
        "        # init some parameters\n",
        "        self.time_step = 0\n",
        "        self.epsilon = EPSILON\n",
        "        self.state_dim = env.observation_space.shape[0]\n",
        "        self.action_dim = env.action_space.n\n",
        "\n",
        "        self.create_Q_network()\n",
        "        self.create_training_method()\n",
        "\n",
        "        # Init session\n",
        "        self.session = sess\n",
        "        self.session.run(tfc.global_variables_initializer())\n",
        "\n",
        "    def create_Q_network(self):\n",
        "        #network weights\n",
        "        W1q = self.weight_variable([self.state_dim, 20])\n",
        "        b1q = self.bias_variable([20])\n",
        "        W2q = self.weight_variable([20, 1])\n",
        "        b2q = self.bias_variable([1])\n",
        "        self.state_input = tfc.placeholder(tf.float32, [1, self.state_dim], \"state\")\n",
        "        # hidden layers\n",
        "        h_layerq = tfc.nn.relu(tf.matmul(self.state_input, W1q) + b1q)\n",
        "        # Q Value layer\n",
        "        self.Q_value = tfc.matmul(h_layerq, W2q) + b2q\n",
        "\n",
        "      # def create_Q_network(self, scope, trainable):\n",
        "      #   with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):\n",
        "\n",
        "      #     self.state_input = tfc.placeholder(tf.float32, [1, self.state_dim], \"state\")\n",
        "\n",
        "      #     # 卷积层，32个8×8卷积，步长为4，全零填充，激活函数为ReLU\n",
        "      #     h_conv1 = tf.keras.layers.Conv2D(filters=32, kernel_size=[8,8],strides=4,padding=\"same\",activation=tf.nn.relu,kernel_initializer=tf.random_normal_initializer(mean=0, stddev=0.01),\\\n",
        "      #                                       bias_initializer=tf.constant_initializer(0.01),trainable=trainable)(self.state_input)\n",
        "          \n",
        "      #     # 池化层，最大池化，窗口大小为2×2，步长为2\n",
        "      #     h_pool1 = tf.keras.layers.MaxPooling2D(pool_size=[2,2],strides=2, padding=\"SAME\")(h_conv1)\n",
        "          \n",
        "      #     # 卷积层，64个4×4卷积，步长为2，全零填充，激活函数为ReLU\n",
        "      #     h_conv2 = tf.keras.layers.Conv2D(filters=64, kernel_size=[4,4],strides=2, padding=\"same\", activation=tf.nn.relu, kernel_initializer=tf.random_normal_initializer(mean=0, stddev=0.01),\\\n",
        "      #                                     bias_initializer=tf.constant_initializer(0.01),trainable=trainable)(h_pool1)\n",
        "          \n",
        "      #     # 卷积层，64个3×3卷积，步长为1，全零填充，激活函数为ReLU\n",
        "      #     h_conv3 = tf.keras.layers.Conv2D(filters=64, kernel_size=[3,3],strides=1,padding=\"same\",activation=tf.nn.relu, kernel_initializer=tf.random_normal_initializer(mean=0,stddev=0.01),\\\n",
        "      #                                     bias_initializer=tf.constant_initializer(0.01),trainable=trainable)(h_conv2)\n",
        "          \n",
        "      #     # 扁平化\n",
        "      #     h_conv3_flat = tf.keras.layers.Flatten()(h_conv3)\n",
        "          \n",
        "      #     # 全连接层，512个神经元，激活函数为ReLU\n",
        "      #     h_fc1 = tf.keras.layers.Dense(units=512,activation=tf.nn.relu, kernel_initializer=tf.random_normal_initializer(0,stddev=0.01),\\\n",
        "      #                                   bias_initializer=tf.constant_initializer(0.01),trainable=trainable)(h_conv3_flat)\n",
        "          \n",
        "      #     # 输出层，2个神经元，表示两个动作分别对应的预测Q值，没有激活函数\n",
        "      #     self.Q_value = tf.keras.layers.Dense(units=22,kernel_initializer=tf.random_normal_initializer(0,stddev=0.01),\\\n",
        "      #                                   bias_initializer=tf.constant_initializer(0.01),trainable=trainable)(h_fc1)\n",
        "\n",
        "\n",
        "\n",
        "    def create_training_method(self):\n",
        "        self.next_value = tfc.placeholder(tf.float32, [1,1], \"v_next\")\n",
        "        self.reward = tfc.placeholder(tf.float32, None, 'reward')\n",
        "\n",
        "        with tfc.variable_scope('squared_TD_error'):\n",
        "            self.td_error = self.reward + GAMMA * self.next_value - self.Q_value\n",
        "            self.loss = tfc.square(self.td_error)\n",
        "        with tfc.variable_scope('train'):\n",
        "            self.train_op = tfc.train.AdamOptimizer(self.epsilon).minimize(self.loss)\n",
        "\n",
        "    def train_Q_network(self, state, reward, next_state):\n",
        "        s, s_ = state[np.newaxis, :], next_state[np.newaxis, :]\n",
        "        v_ = self.session.run(self.Q_value, {self.state_input: s_})\n",
        "        td_error, _ = self.session.run([self.td_error, self.train_op],\n",
        "                                          {self.state_input: s, self.next_value: v_, self.reward: reward})\n",
        "        return td_error\n",
        "\n",
        "    def weight_variable(self,shape):\n",
        "        initial = tfc.truncated_normal(shape)\n",
        "        return tf.Variable(initial)\n",
        "\n",
        "    def bias_variable(self,shape):\n",
        "        initial = tfc.constant(0.01, shape = shape)\n",
        "        return tf.Variable(initial)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "3kThlRp4wx4I",
        "outputId": "7f866875-fa8f-46df-8c1a-6185b272354a"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/client/session.py:1768: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
            "  warnings.warn('An interactive session is already active. This can '\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "198.43518\n",
            "-199.6618211996385 4999 199.6618211996385\n",
            "182.72223\n",
            "-185.7021194590272 4999 185.7021194590272\n",
            "192.73148\n",
            "-189.7658059243592 4999 189.7658059243592\n",
            "225.92592\n",
            "-228.47693847035814 4999 228.47693847035814\n",
            "189.46297\n",
            "-189.37111867475357 4999 189.37111867475357\n",
            "225.42593\n",
            "-225.72611636672647 4999 225.72611636672647\n",
            "199.65741\n",
            "-205.77637723057325 4999 205.77637723057325\n",
            "202.69444\n",
            "-199.5573596531857 4999 199.5573596531857\n",
            "185.7037\n",
            "-184.91548025715954 4999 184.91548025715954\n",
            "191.40741\n",
            "-190.99387011921593 4999 190.99387011921593\n",
            "219.18518\n",
            "-216.16503606402256 4999 216.16503606402256\n",
            "212.66667\n",
            "-209.53431231659997 4999 209.53431231659997\n",
            "198.18518\n",
            "-193.61457402662307 4999 193.61457402662307\n",
            "212.44444\n",
            "-213.47023230140522 4999 213.47023230140522\n",
            "153.19444\n",
            "-153.55423436411712 4999 153.55423436411712\n",
            "224.70369\n",
            "-217.93660165380078 4999 217.93660165380078\n",
            "179.97221\n",
            "-178.40772425850804 4999 178.40772425850804\n",
            "208.90741\n",
            "-203.35992469627888 4999 203.35992469627888\n",
            "181.0\n",
            "-180.23715623455738 4999 180.23715623455738\n",
            "212.41667\n",
            "-210.6571173029328 4999 210.6571173029328\n",
            "211.41667\n",
            "-209.40433492909 4999 209.40433492909\n",
            "181.22223\n",
            "-179.16653576350777 4999 179.16653576350777\n",
            "238.34259\n",
            "-238.70521067578147 4999 238.70521067578147\n",
            "213.4537\n",
            "-212.3738369528203 4999 212.3738369528203\n",
            "207.1852\n",
            "-208.33071265872337 4999 208.33071265872337\n",
            "202.69444\n",
            "-201.3686281964162 4999 201.3686281964162\n",
            "213.92593\n",
            "-213.0664792956827 4999 213.0664792956827\n",
            "180.18518\n",
            "-180.9117076657594 4999 180.9117076657594\n",
            "214.17592\n",
            "-205.14604393300004 4999 205.14604393300004\n",
            "187.96297\n",
            "-189.80021502434064 4999 189.80021502434064\n",
            "187.96297\n",
            "-190.7108291300981 4999 190.7108291300981\n",
            "179.71297\n",
            "-183.7191701596047 4999 183.7191701596047\n",
            "191.69446\n",
            "-196.23422389268504 4999 196.23422389268504\n",
            "220.69444\n",
            "-219.5077940385235 4999 219.5077940385235\n",
            "202.98148\n",
            "-199.39572809542028 4999 199.39572809542028\n",
            "216.46297\n",
            "-211.73142791330724 4999 211.73142791330724\n",
            "177.46295\n",
            "-176.48985969052245 4999 176.48985969052245\n",
            "203.2037\n",
            "-209.86970297682242 4999 209.86970297682242\n",
            "153.87962\n",
            "-160.84334351547668 4999 160.84334351547668\n",
            "180.46297\n",
            "-183.86100057070635 4999 183.86100057070635\n",
            "247.93518\n",
            "-248.10347222620638 4999 248.10347222620638\n",
            "202.47223\n",
            "-208.37970295677326 4999 208.37970295677326\n",
            "251.91667\n",
            "-263.2783749611738 4999 263.2783749611738\n",
            "228.1574\n",
            "-229.40607042176532 4999 229.40607042176532\n",
            "215.92593\n",
            "-216.10044078656995 4999 216.10044078656995\n",
            "215.42592\n",
            "-210.0070808830462 4999 210.0070808830462\n",
            "202.66666\n",
            "-201.64168726459624 4999 201.64168726459624\n",
            "191.18518\n",
            "-191.26750353335873 4999 191.26750353335873\n",
            "196.2037\n",
            "-199.54818029267668 4999 199.54818029267668\n",
            "186.4537\n",
            "-190.03096483689168 4999 190.03096483689168\n",
            "189.93518\n",
            "-191.101019225811 4999 191.101019225811\n",
            "201.97223\n",
            "-200.94468564223692 4999 200.94468564223692\n",
            "211.94444\n",
            "-213.5398600604351 4999 213.5398600604351\n",
            "200.65741\n",
            "-200.73984179647698 4999 200.73984179647698\n",
            "212.23149\n",
            "-209.84255403797874 4999 209.84255403797874\n",
            "184.23149\n",
            "-184.20516744495342 4999 184.20516744495342\n",
            "185.23148\n",
            "-185.86470048938565 4999 185.86470048938565\n",
            "162.47223\n",
            "-162.8330495137233 4999 162.8330495137233\n",
            "205.67593\n",
            "-206.01071516000746 4999 206.01071516000746\n",
            "195.99074\n",
            "-193.51032528780252 4999 193.51032528780252\n",
            "206.42593\n",
            "-205.0984420711761 4999 205.0984420711761\n",
            "190.1852\n",
            "-191.5281081293512 4999 191.5281081293512\n",
            "230.63889\n",
            "-223.6287519120401 4999 223.6287519120401\n",
            "148.42592\n",
            "-154.6074586306562 4999 154.6074586306562\n",
            "204.89815\n",
            "-204.44596331959966 4999 204.44596331959966\n",
            "217.68518\n",
            "-214.03755379108054 4999 214.03755379108054\n",
            "209.15742\n",
            "-210.6524788502123 4999 210.6524788502123\n",
            "221.19443\n",
            "-221.61818058840694 4999 221.61818058840694\n",
            "207.43518\n",
            "-204.8399121773355 4999 204.8399121773355\n",
            "241.91666\n",
            "-234.82969129607034 4999 234.82969129607034\n",
            "155.16666\n",
            "-155.45988435044896 4999 155.45988435044896\n",
            "188.93518\n",
            "-187.65016545646407 4999 187.65016545646407\n",
            "233.4537\n",
            "-232.51964002983473 4999 232.51964002983473\n",
            "200.47221\n",
            "-201.4629306211818 4999 201.4629306211818\n",
            "181.22223\n",
            "-186.38401859263803 4999 186.38401859263803\n",
            "200.72221\n",
            "-202.75758054113754 4999 202.75758054113754\n",
            "216.21297\n",
            "-215.1664559104936 4999 215.1664559104936\n",
            "198.40741\n",
            "-197.34595472380644 4999 197.34595472380644\n",
            "220.91667\n",
            "-214.7756489730662 4999 214.7756489730662\n",
            "213.4537\n",
            "-218.16224438810312 4999 218.16224438810312\n",
            "217.90741\n",
            "-213.77608675908135 4999 213.77608675908135\n",
            "219.94446\n",
            "-228.5972561621184 4999 228.5972561621184\n",
            "221.73148\n",
            "-221.79067264198514 4999 221.79067264198514\n",
            "189.75\n",
            "-191.80187673323337 4999 191.80187673323337\n",
            "232.66666\n",
            "-231.718713274342 4999 231.718713274342\n",
            "188.93518\n",
            "-191.7058602747974 4999 191.7058602747974\n",
            "216.08334\n",
            "-210.75860438311804 4999 210.75860438311804\n",
            "183.9537\n",
            "-184.57678959647762 4999 184.57678959647762\n",
            "212.66667\n",
            "-209.53431231659997 4999 209.53431231659997\n",
            "201.69444\n",
            "-205.71677316178187 4999 205.71677316178187\n",
            "180.46297\n",
            "-181.44891820533644 4999 181.44891820533644\n",
            "218.15741\n",
            "-214.97183966366904 4999 214.97183966366904\n",
            "180.97221\n",
            "-181.19404236705412 4999 181.19404236705412\n",
            "258.1296\n",
            "-253.22859947702486 4999 253.22859947702486\n",
            "198.18518\n",
            "-197.9855908162384 4999 197.9855908162384\n",
            "187.99074\n",
            "-177.28432220231483 4999 177.28432220231483\n",
            "1.3518518\n",
            "-43.34428042405043 0 43.34428042405043\n",
            "202.98148\n",
            "-212.48174619475165 4999 212.48174619475165\n",
            "194.2037\n",
            "-196.03775436833416 4999 196.03775436833416\n",
            "205.7037\n",
            "-204.82145389932728 4999 204.82145389932728\n",
            "209.47223\n",
            "-207.41425657440968 4999 207.41425657440968\n",
            "165.9537\n",
            "-171.20624700408857 4999 171.20624700408857\n",
            "180.22223\n",
            "-183.7490078444903 4999 183.7490078444903\n",
            "179.71297\n",
            "-179.72384317581395 4999 179.72384317581395\n",
            "194.42593\n",
            "-193.6956457412109 4999 193.6956457412109\n",
            "178.71297\n",
            "-188.8518222590925 4999 188.8518222590925\n",
            "199.68518\n",
            "-199.03121274338366 4999 199.03121274338366\n",
            "139.6852\n",
            "-139.94835702082088 0 139.94835702082088\n",
            "184.98148\n",
            "-185.9961942666983 4999 185.9961942666983\n",
            "204.89815\n",
            "-204.44596331959966 4999 204.44596331959966\n",
            "170.72223\n",
            "-171.40728962189934 4999 171.40728962189934\n",
            "222.9537\n",
            "-221.7075705023592 4999 221.7075705023592\n",
            "179.46295\n",
            "-184.82514490851563 4999 184.82514490851563\n",
            "191.69446\n",
            "-193.18070686013255 4999 193.18070686013255\n",
            "227.9352\n",
            "-224.79645629937096 4999 224.79645629937096\n",
            "210.19444\n",
            "-211.03232747837328 4999 211.03232747837328\n",
            "221.70369\n",
            "-220.42931150979857 4999 220.42931150979857\n",
            "160.72221\n",
            "-158.42998648673586 4999 158.42998648673586\n",
            "222.91667\n",
            "-220.12168438166702 4999 220.12168438166702\n",
            "214.67593\n",
            "-213.28528044385675 4999 213.28528044385675\n",
            "194.2037\n",
            "-194.672940119981 4999 194.672940119981\n",
            "182.91667\n",
            "-181.0985665874538 4999 181.0985665874538\n",
            "171.50925\n",
            "-178.05418712752893 4999 178.05418712752893\n",
            "189.21297\n",
            "-188.90254862916328 4999 188.90254862916328\n",
            "221.94446\n",
            "-212.77906543980296 4999 212.77906543980296\n",
            "195.71297\n",
            "-199.17121654927016 4999 199.17121654927016\n",
            "216.93518\n",
            "-211.7850770660926 4999 211.7850770660926\n",
            "241.41667\n",
            "-242.74912532277432 4999 242.74912532277432\n",
            "164.98149\n",
            "-162.86368021438452 4999 162.86368021438452\n",
            "187.71297\n",
            "-203.9274679165904 4999 203.9274679165904\n",
            "191.94444\n",
            "-196.62712608869398 4999 196.62712608869398\n",
            "178.46295\n",
            "-188.36089953873832 4999 188.36089953873832\n",
            "206.42593\n",
            "-206.58966875622352 4999 206.58966875622352\n",
            "195.92593\n",
            "-204.50050699025343 4999 204.50050699025343\n",
            "271.6389\n",
            "-273.28012541999595 4999 273.28012541999595\n",
            "177.71297\n",
            "-180.6743015479269 4999 180.6743015479269\n",
            "205.99074\n",
            "-214.61873173133023 4999 214.61873173133023\n",
            "158.24074\n",
            "-236.5253933093869 4999 236.5253933093869\n",
            "195.9537\n",
            "-200.0328836273495 4999 200.0328836273495\n",
            "197.21295\n",
            "-195.3436601711827 4999 195.3436601711827\n",
            "132.94444\n",
            "-143.15281027544296 0 143.15281027544296\n",
            "206.42593\n",
            "-206.58966875622352 4999 206.58966875622352\n",
            "182.15741\n",
            "-186.4925041500703 4999 186.4925041500703\n",
            "183.69444\n",
            "-188.40460473374594 4999 188.40460473374594\n",
            "158.21297\n",
            "-165.12690291871368 4999 165.12690291871368\n",
            "184.73148\n",
            "-184.6245750638752 4999 184.6245750638752\n",
            "186.83333\n",
            "-184.86766194718044 4999 184.86766194718044\n",
            "200.94444\n",
            "-202.82731419167987 4999 202.82731419167987\n",
            "179.97221\n",
            "-182.6766711518092 4999 182.6766711518092\n",
            "176.99074\n",
            "-179.45042467822185 4999 179.45042467822185\n",
            "198.43518\n",
            "-200.7587595469767 4999 200.7587595469767\n",
            "191.97221\n",
            "-195.11240305921046 4999 195.11240305921046\n",
            "201.69444\n",
            "-184.19601128791794 4999 184.19601128791794\n",
            "200.94444\n",
            "-199.08082182753023 4999 199.08082182753023\n",
            "197.46297\n",
            "-195.2192841280959 4999 195.2192841280959\n",
            "226.43518\n",
            "-224.562869502881 4999 224.562869502881\n",
            "216.14813\n",
            "-215.79276696756554 4999 215.79276696756554\n",
            "177.71297\n",
            "-177.56351034707637 4999 177.56351034707637\n",
            "194.42593\n",
            "-193.6956457412109 4999 193.6956457412109\n",
            "209.44446\n",
            "-206.80284951800093 4999 206.80284951800093\n",
            "179.93518\n",
            "-185.8363904431684 4999 185.8363904431684\n",
            "203.48148\n",
            "-201.3306450192723 4999 201.3306450192723\n",
            "205.39815\n",
            "-205.13633674333022 4999 205.13633674333022\n",
            "201.44443\n",
            "-202.03600967669556 4999 202.03600967669556\n",
            "204.17592\n",
            "-199.96681704247882 4999 199.96681704247882\n",
            "202.44443\n",
            "-204.08174605882675 4999 204.08174605882675\n",
            "161.72221\n",
            "-160.06332485516802 4999 160.06332485516802\n",
            "193.16666\n",
            "-200.82305945041801 4999 200.82305945041801\n",
            "202.69444\n",
            "-200.07272857155425 4999 200.07272857155425\n",
            "179.46295\n",
            "-184.56979902583888 4999 184.56979902583888\n",
            "205.4537\n",
            "-200.72306553555492 4999 200.72306553555492\n",
            "184.98148\n",
            "-185.3047965021096 4999 185.3047965021096\n",
            "217.46297\n",
            "-217.22930468036637 4999 217.22930468036637\n",
            "170.72223\n",
            "-171.81120084646378 4999 171.81120084646378\n",
            "212.3889\n",
            "-215.69204804843048 4999 215.69204804843048\n",
            "227.9352\n",
            "-224.79645629937096 4999 224.79645629937096\n",
            "228.43518\n",
            "-227.1994796172226 4999 227.1994796172226\n",
            "201.41666\n",
            "-202.27470994424368 4999 202.27470994424368\n",
            "222.70369\n",
            "-220.39935976326302 4999 220.39935976326302\n",
            "188.71297\n",
            "-193.36241627170733 4999 193.36241627170733\n",
            "192.98148\n",
            "-190.4798542769859 4999 190.4798542769859\n",
            "168.18518\n",
            "-168.19576320937398 4999 168.19576320937398\n",
            "159.49074\n",
            "-160.62947294400368 4999 160.62947294400368\n",
            "172.72221\n",
            "-175.67385312569942 4999 175.67385312569942\n",
            "212.7037\n",
            "-214.7491419942982 4999 214.7491419942982\n",
            "179.71297\n",
            "-179.20049561568402 4999 179.20049561568402\n",
            "198.96297\n",
            "-155.64679208807112 4999 155.64679208807112\n",
            "168.21297\n",
            "-185.83494217513893 4999 185.83494217513893\n",
            "219.72223\n",
            "-218.2052666408691 4999 218.2052666408691\n",
            "187.96297\n"
          ]
        }
      ],
      "source": [
        "# Hyper Parameters\n",
        "EPISODE = 1000 # Episode limitation\n",
        "STEP = 5000 # Step limitation in an episode\n",
        "TEST = 10 # The number of experiment test every 100 episode\n",
        "\n",
        "def main():\n",
        "  # initialize OpenAI Gym env and dqn agent\n",
        "  sess = tfc.InteractiveSession()\n",
        "  env = NOXEnv()\n",
        "  actor = Actor(env, sess)\n",
        "  critic = Critic(env, sess)\n",
        "  result_reward = []\n",
        "  result_step = []\n",
        "\n",
        "  for episode in range(EPISODE):\n",
        "    # initialize task\n",
        "    state = env.reset()\n",
        "    total_reward = 0\n",
        "    # Train\n",
        "    for step in range(STEP):\n",
        "      action = actor.choose_action(state) # e-greedy action for train\n",
        "      next_state,reward,done,output = env.step(state, action)\n",
        "      td_error = critic.train_Q_network(state, reward, next_state)  # gradient = grad[r + gamma * V(s_) - V(s)]\n",
        "      actor.learn(state, action, td_error)  # true_gradient = grad[logPi(s,a) * td_error]\n",
        "      state = next_state\n",
        "      total_reward += reward\n",
        "      ave_reward = total_reward/(step+1)\n",
        "      if done or step == STEP-1:\n",
        "        result_reward.append(ave_reward)\n",
        "        result_step.append(step+1)\n",
        "        break\n",
        "    print(reward,step,output)\n",
        "\n",
        "\n",
        "    # Test every 100 episodes\n",
        "    # if episode % 10 == 0:\n",
        "    #   total_reward = 0\n",
        "    #   for i in range(TEST):\n",
        "    #     state = env.reset()\n",
        "    #     reward1 = []\n",
        "    #     for j in range(STEP):\n",
        "    #       action = actor.choose_action(state) # direct action for test\n",
        "    #       state,reward,done,_ = env.step(state, action)\n",
        "    #       total_reward += reward\n",
        "    #       reward1.append(reward)\n",
        "    #       if done:\n",
        "    #         break\n",
        "      \n",
        "    #   ave_reward = total_reward/TEST\n",
        "    #   print ('episode: ',episode,'Evaluation Average Reward:',ave_reward)\n",
        "      # print('reward1: ',reward1)\n",
        "      # plt.plot(reward1)\n",
        "      # plt.show()\n",
        "\n",
        "  visualization1(result_reward, result_step)\n",
        "\n",
        "def visualization1(reward, step):\n",
        "  fig = plt.figure(figsize=(10,6))\n",
        "  ax1 = fig.add_subplot(111)\n",
        "  ax1.plot(range(1,len(reward)+1),reward,linestyle='dashed',label='average reward')\n",
        "  ax1.set_xticks([0,20,40,60,80,100])\n",
        "  ax2 = ax1.twinx()\n",
        "  ax2.plot(range(1,len(step)+1),step,color='r',label='time step')\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "  main()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}