Main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Importing DataLoaders for each model. These models include rule-based, vanilla DQN and encoder-decoder DQN.\n",
    "from DataLoader.DataLoader import YahooFinanceDataLoader\n",
    "from DataLoader.DataForPatternBasedAgent import DataForPatternBasedAgent\n",
    "from DataLoader.DataAutoPatternExtractionAgent import DataAutoPatternExtractionAgent\n",
    "from DataLoader.DataSequential import DataSequential \n",
    "\n",
    "from DeepRLAgent.MLPEncoder.Train import Train as SimpleMLP\n",
    "from DeepRLAgent.SimpleCNNEncoder.Train import Train as SimpleCNN\n",
    "from EncoderDecoderAgent.GRU.Train import Train as gru\n",
    "from EncoderDecoderAgent.CNN.Train import Train as cnn\n",
    "from EncoderDecoderAgent.CNN2D.Train import Train as cnn2d\n",
    "from EncoderDecoderAgent.CNNAttn.Train import Train as cnn_attn\n",
    "from EncoderDecoderAgent.CNN_GRU.Train import Train as cnn_gru\n",
    "\n",
    "\n",
    "# Imports for Deep RL Agent\n",
    "from DeepRLAgent.VanillaInput.Train import Train as DeepRL\n",
    "\n",
    "\n",
    "\n",
    "# Imports for RL Agent with n-step SARSA\n",
    "from RLAgent.Train import Train as RLTrain\n",
    "\n",
    "# Imports for Rule-Based\n",
    "from PatternDetectionInCandleStick.LabelPatterns import label_candles\n",
    "from PatternDetectionInCandleStick.Evaluation import Evaluation\n",
    "\n",
    "\n",
    "import torch\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from kaleido.scopes.plotly import PlotlyScope\n",
    "import plotly.graph_objs as go\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import random\n",
    "\n",
    "CURRENT_PATH = os.getcwd()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`BATCH_SIZE` is set to 10 based on some analysis on the experiments. You can change it to whatever you want. `n_step` is the cumulative reward of n-steps in the future. `initial_investment` is the amount invested at the beginning of the process of trading. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 10\n",
    "GAMMA=0.7\n",
    "n_step = 10\n",
    "\n",
    "initial_investment = 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`train_portfolios` and `test_portfolios` are dictionaries for saving the portfolio result of different models. The `key` of the dictionary is the model name and the `value` is a list of values showing the value of portfolio at each time-step. The `window_size_experiment` is the dictionary for storing the results of the experiments done for evaluating the effect of different window-sizes on portfolio values.   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_portfolios = {}\n",
    "test_portfolios = {}\n",
    "window_size_experiment = {}\n",
    "window_sizes = [3, 5, 8, 10, 12, 15, 20, 25, 30, 40, 50, 75]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following two helper functions are for storing the portfolio result of diffrent experiments given the `model_name` and a list of portfolio values. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_train_portfo(model_name, portfo):\n",
    "    counter = 0\n",
    "    key = f'{model_name}'\n",
    "    while key in train_portfolios.keys():\n",
    "        counter += 1\n",
    "        key = f'{model_name}{counter}'\n",
    "        \n",
    "    train_portfolios[key] = portfo\n",
    "\n",
    "def add_test_portfo(model_name, portfo):\n",
    "    counter = 0\n",
    "    key = f'{model_name}'\n",
    "    while key in test_portfolios.keys():\n",
    "        counter += 1\n",
    "        key = f'{model_name}{counter}'\n",
    "    \n",
    "    test_portfolios[key] = portfo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following blocks are for choosing a specific data to run the experiments. For example if you want to run the experiments on `BTC-USD` you should only run the first block, for `GOOGL` run the second block, blah blah blah. As also discussed in the documentation of the YahooFinanceDataLoader, you can set the `begin_date` and `end_date` to select a specific period from the original data. Also you can set the `split_point` which is a date of splitting train and test data. If your origianl file has changed or you are running the data for the first time, set the `load_from_file` to `false`. Otherwise set it to `True` to load from the preprocessed file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BTC-USD\n",
    "\n",
    "DATASET_NAME = 'BTC-USD'\n",
    "DATASET_FOLDER = r'BTC-USD'\n",
    "FILE = r'BTC-USD.csv'\n",
    "data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, '2018-01-01', load_from_file=True)\n",
    "transaction_cost = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GOOGL\n",
    "\n",
    "DATASET_NAME = 'GOOGL'\n",
    "DATASET_FOLDER = 'GOOGL'\n",
    "FILE = 'GOOGL.csv'\n",
    "data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, '2018-01-01', load_from_file=True)\n",
    "transaction_cost = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AAPL\n",
    "\n",
    "DATASET_NAME = 'AAPL'\n",
    "DATASET_FOLDER = r'AAPL'\n",
    "FILE = r'AAPL.csv'\n",
    "data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, split_point='2018-01-01', \n",
    "                                     begin_date='2010-01-01', end_date='2020-08-24', load_from_file=True)\n",
    "transaction_cost = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# # DJI\n",
    "#\n",
    "# DATASET_NAME = 'DJI'\n",
    "# DATASET_FOLDER = r'DJI'\n",
    "# FILE = r'DJI.csv'\n",
    "# data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, '2016-01-01', begin_date='2009-01-01', end_date='2018-09-30',\n",
    "#                                      load_from_file=True)\n",
    "# transaction_cost = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# S&P\n",
    "\n",
    "# DATASET_NAME = 'S&P'\n",
    "# DATASET_FOLDER = 'S&P'\n",
    "# FILE = 'S&P.csv'\n",
    "# data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, split_point=2000, end_date='2018-09-25', load_from_file=True)\n",
    "# transaction_cost = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# AMD\n",
    "\n",
    "# DATASET_NAME = 'AMD'\n",
    "# DATASET_FOLDER = 'AMD'\n",
    "# FILE = 'AMD.csv'\n",
    "# data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, split_point=2000, end_date='2018-09-25', load_from_file=True)\n",
    "# transaction_cost = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GE\n",
    "\n",
    "DATASET_NAME = 'GE'\n",
    "DATASET_FOLDER = r'GE'\n",
    "FILE = r'GE.csv'\n",
    "data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, '2015-01-01', load_from_file=True)\n",
    "transaction_cost = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KSS\n",
    "\n",
    "DATASET_NAME = 'KSS'\n",
    "DATASET_FOLDER = 'KSS'\n",
    "FILE = 'KSS.csv'\n",
    "data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, split_point='2018-01-01', load_from_file=True)\n",
    "transaction_cost = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# HSI\n",
    "\n",
    "DATASET_NAME = 'HSI'\n",
    "DATASET_FOLDER = 'HSI'\n",
    "FILE = 'HSI.csv'\n",
    "data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, split_point='2015-01-01', load_from_file=True)\n",
    "transaction_cost = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# American Airlines\n",
    "\n",
    "DATASET_NAME = 'AAL'\n",
    "DATASET_FOLDER = r'AAL'\n",
    "FILE = r'AAL.csv'\n",
    "data_loader = YahooFinanceDataLoader(DATASET_FOLDER, FILE, '2018-01-01', load_from_file=True)\n",
    "transaction_cost = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block is for choosing the observation-space mode. `state_mode = 1` is when the observation space contains only the Open, High, Low, and Close data. `state_mode = 2` also has a `trend` showing the trend of the stock before the candle stick at a specific time-step. `state_mode = 3` has OHLC and trend plus the size of the body, upper and lower shadow of the candle. `state_mode = 4` contains only the trend of the stock before that time-step plus the size of the body of the candle, upper and lower shadows. `state_mode = 5` is a window of `window-size` OHLCs plus the trend of the stock in that window. Pay attention that the window_size here is only for `DataAutoPatternExtractionAgent` not for the `DataSequential` which is the data used for time-series models like CNN and GRU. `DataForPatternBasedAgent` uses the extracted patterns as observation space. This kind of observation space is mainly used by RL agent with SARSA-$\\lambda$ and one kind of DQN model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Agent with Auto pattern extraction\n",
    "# State Mode\n",
    "state_mode = 1  # OHLC\n",
    "# state_mode = 2  # OHLC + trend\n",
    "# state_mode = 3  # OHLC + trend + %body + %upper-shadow + %lower-shadow\n",
    "window_size = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataTrain_autoPatternExtractionAgent = DataAutoPatternExtractionAgent(data_loader.data_train, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "dataTest_autoPatternExtractionAgent = DataAutoPatternExtractionAgent(data_loader.data_test, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "dataTrain_patternBased = DataForPatternBasedAgent(data_loader.data_train, data_loader.patterns, 'action_deepRL', device, GAMMA, n_step, BATCH_SIZE, transaction_cost)\n",
    "dataTest_patternBased = DataForPatternBasedAgent(data_loader.data_test, data_loader.patterns, 'action_deepRL', device, GAMMA, n_step, BATCH_SIZE, transaction_cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As discussed before, when `state_mode` is set to 4, the observation space has the trend, length of the body, upper and lower shadow of the candle. Therefore, we call this `state_mode` as candle representation because it contains a representation of the candle stick, not its original data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_mode = 4  # trend + %body + %upper-shadow + %lower-shadow\n",
    "\n",
    "dataTrain_autoPatternExtractionAgent_candle_rep = DataAutoPatternExtractionAgent(data_loader.data_train, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "dataTest_autoPatternExtractionAgent_candle_rep = DataAutoPatternExtractionAgent(data_loader.data_test, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following blocks are for loading windowed input for non-timeseries models. As we discussed above, the `state_mode` is 5 in this case and you should provide the window-size in this case. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_mode = 5  # window with k candles inside + the trend of those candles\n",
    "window_size = 20\n",
    "dataTrain_autoPatternExtractionAgent_windowed = DataAutoPatternExtractionAgent(data_loader.data_train, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "dataTest_autoPatternExtractionAgent_windowed = DataAutoPatternExtractionAgent(data_loader.data_test, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This block is for loading the data for encoder-decoder models with time-series encoder. The `window_size` here is the number of candles inside one single time-step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "window_size = 15\n",
    "\n",
    "dataTrain_sequential = DataSequential(data_loader.data_train,\n",
    "                           'action_encoder_decoder', device, GAMMA,\n",
    "                           n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "dataTest_sequential = DataSequential(data_loader.data_test,\n",
    "                          'action_encoder_decoder', device, GAMMA,\n",
    "                          n_step, BATCH_SIZE, window_size, transaction_cost)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reinforcement Learning Agent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section is for running the experiments for RL agent with n-step SARSA algorithm. The following block is for setting the HPs for this algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_iteration = 1000\n",
    "gamma = 0.9\n",
    "alpha = 0.3\n",
    "epsilon = 0.1\n",
    "n = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the following block, if you want the agent to learn a new policy, uncomment the two lines:\n",
    "```\n",
    "rlAgent.training()\n",
    "rlAgent.write_to_file()\n",
    "```\n",
    "but if you want to load a trained model, comment out the above two lines and instead use the function:\n",
    "```\n",
    "rlAgent.read_from_file(<file name inside the ./Objects/RLAgent directory>)\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read RL Agent experiment\n",
    "data_test_rlagent = dataTest_patternBased.data if dataTest_patternBased is not None else None\n",
    "\n",
    "rlAgent = RLTrain(dataTrain_patternBased.data, data_test_rlagent, data_loader.patterns, DATASET_NAME, n=n, num_iteration=num_iteration, gamma=gamma, alpha=alpha, epsilon=epsilon)\n",
    "\n",
    "rlAgent.training()\n",
    "rlAgent.write_to_file()\n",
    "\n",
    "# rlAgent.read_from_file('GOOGL-TRAIN_TEST_SPLIT(True)-NUM_ITERATIONS10000-N_STEP10-GAMMA1-ALPHA0.1-EPSILON0.1-EXPERIMENT(1).pkl')\n",
    "# rlAgent.read_from_file('AAPL-TRAIN_TEST_SPLIT(True)-NUM_ITERATIONS1000-N_STEP10-GAMMA1-ALPHA0.1-EPSILON0.1-EXPERIMENT.pkl')\n",
    "# rlAgent.read_from_file('BTC-USD-TRAIN_TEST_SPLIT(True)-NUM_ITERATIONS10000-N_STEP10-GAMMA0.9-ALPHA0.1-EPSILON0.1-EXPERIMENT.pkl')\n",
    "# rlAgent.read_from_file('KSS-TRAIN_TEST_SPLIT(True)-NUM_ITERATIONS10000-N_STEP10-GAMMA0.9-ALPHA0.1-EPSILON0.1-EXPERIMENT(1).pkl')\n",
    "\n",
    "ev_rlAgent = rlAgent.test(test_type= 'train')\n",
    "print('train')\n",
    "ev_rlAgent.evaluate()\n",
    "rlAgent_portfolio_train = ev_rlAgent.get_daily_portfolio_value()\n",
    "ev_rlAgent = rlAgent.test(test_type= 'test')\n",
    "print('test')\n",
    "ev_rlAgent.evaluate()\n",
    "rlAgent_portfolio_test = ev_rlAgent.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = f'RL'\n",
    "# model_kind = f'RL-{n}'\n",
    "\n",
    "add_train_portfo(model_kind, rlAgent_portfolio_train)\n",
    "add_test_portfo(model_kind, rlAgent_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Reinforcement Learning Agent "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section dedicates to the Deep Q-Learning agent without encoder part. `TARGET_UPDATE` is how frequent the policy network is hard-copied to the target network in terms of number of episodes. `n_actions` is set to 3 because we have `buy`, `sell` and `none` actions. `n_episodes` is the number of episodes to run the algorithm. `EPS` is the $\\epsilon$ in the $\\epsilon$-greedy method. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPS=0.1\n",
    "ReplayMemorySize=20\n",
    "TARGET_UPDATE=5\n",
    "n_actions=3\n",
    "n_episodes = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the following bocks, each block is for running the DQN algorithm in a specific condition. For each block, if you want to train from scratch, use the following two lines:\n",
    "```\n",
    "deepRLAgent.train(n_episodes)\n",
    "file_name = None\n",
    "```\n",
    "pay attention that the `file_name` should be set to `None` so that for the evaluation process, it does not load from file and instead use the recently trained agent. If you want to load from a specific model, set the `file_name` to the name of the file inside the `./Objects/DeepRL` directory."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Patterns as input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model uses the extracted patterns as observation space. These patterns are extracted in the preprocessing phase and used here instead of directly applying OHLC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deepRLAgent = DeepRL(data_loader, dataTrain_patternBased, dataTest_patternBased, DATASET_NAME, \n",
    "                     state_mode, window_size, transaction_cost,\n",
    "                    BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS, ReplayMemorySize=ReplayMemorySize,\n",
    "                    TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "deepRLAgent.train(n_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'GOOGL; DATA_KIND(PatternBased); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TransactionCost(0.0) TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(PatternBased); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TransactionCost(0.0) TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT(1).pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(PatternBased); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TransactionCost(0.0) TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT(1).pkl'\n",
    "# file_name = 'KSS; DATA_KIND(PatternBased); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0.0); StateMode(1); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "ev_deepRLAgent = deepRLAgent.test(file_name=file_name, action_name=dataTrain_patternBased.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "deepRLAgent_portfolio_train = ev_deepRLAgent.get_daily_portfolio_value()\n",
    "ev_deepRLAgent = deepRLAgent.test(file_name=file_name, action_name=dataTrain_patternBased.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "deepRLAgent_portfolio_test = ev_deepRLAgent.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'DQN-pattern'\n",
    "\n",
    "add_train_portfo(model_kind, deepRLAgent_portfolio_train)\n",
    "add_test_portfo(model_kind, deepRLAgent_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### OHLC as input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model uses the OHLC representation of candles. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deepRLAgent = DeepRL(data_loader, dataTrain_autoPatternExtractionAgent, dataTest_autoPatternExtractionAgent, \n",
    "                     DATASET_NAME,  state_mode, window_size, transaction_cost,\n",
    "                     BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS, ReplayMemorySize=ReplayMemorySize,\n",
    "                     TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "deepRLAgent.train(n_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'GOOGL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TransactionCost(0.0) TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TransactionCost(0.0) TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT(1).pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TransactionCost(0.0) TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT(1).pkl'\n",
    "# file_name = 'KSS; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0.0); StateMode(1); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'GE; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2015-01-01); DeepRL; TC(0); StateMode(1); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'HSI; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2015-01-01); DeepRL; TC(0); StateMode(1); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'AAL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0); StateMode(1); WindowSize(10); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "\n",
    "ev_deepRLAgentAutoExtraction = deepRLAgent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "deepRLAgentAutoExtraction_portfolio_train = ev_deepRLAgentAutoExtraction.get_daily_portfolio_value()\n",
    "ev_deepRLAgentAutoExtraction = deepRLAgent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "deepRLAgentAutoExtraction_portfolio_test = ev_deepRLAgentAutoExtraction.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'DQN-vanilla'\n",
    "# model_kind = 'DQN'\n",
    "\n",
    "add_train_portfo(model_kind, deepRLAgentAutoExtraction_portfolio_train)\n",
    "add_test_portfo(model_kind, deepRLAgentAutoExtraction_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With candle representation as input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model uses the `state_mode = 4` as input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deepRLAgent = DeepRL(data_loader, dataTrain_autoPatternExtractionAgent_candle_rep, \n",
    "                     dataTest_autoPatternExtractionAgent_candle_rep, \n",
    "                     DATASET_NAME,  state_mode, window_size, transaction_cost,\n",
    "                     BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS, ReplayMemorySize=ReplayMemorySize,\n",
    "                     TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "deepRLAgent.train(n_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'GOOGL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0); StateMode(4); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0.0); StateMode(4); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(2010-01-01); END_DATE(2020-08-24); SPLIT_POINT(2018-01-01); DeepRL; TC(0); StateMode(4); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'KSS; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0.0); StateMode(4); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "ev_deepRLAgentAutoExtraction = deepRLAgent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_candle_rep.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "deepRLAgentAutoExtraction_portfolio_train = ev_deepRLAgentAutoExtraction.get_daily_portfolio_value()\n",
    "ev_deepRLAgentAutoExtraction = deepRLAgent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_candle_rep.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "deepRLAgentAutoExtraction_portfolio_test = ev_deepRLAgentAutoExtraction.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'DQN-candle-rep'\n",
    "\n",
    "add_train_portfo(model_kind, deepRLAgentAutoExtraction_portfolio_train)\n",
    "add_test_portfo(model_kind, deepRLAgentAutoExtraction_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Windowed Input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Input is a window of OHLC candles."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deepRLAgent = DeepRL(data_loader, dataTrain_autoPatternExtractionAgent_windowed, \n",
    "                     dataTest_autoPatternExtractionAgent_windowed, \n",
    "                     DATASET_NAME,  state_mode, window_size, transaction_cost,\n",
    "                     BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS, ReplayMemorySize=ReplayMemorySize,\n",
    "                     TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "deepRLAgent.train(n_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'AAPL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(2010-01-01); END_DATE(2020-08-24); SPLIT_POINT(2018-01-01); DeepRL; TC(0); StateMode(5); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0.0); StateMode(5); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'GOOGL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0); StateMode(5); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'KSS; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0.0); StateMode(5); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "# file_name = 'BTC-USD; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0.0); StateMode(5); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'GOOGL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0); StateMode(5); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(2010-01-01); END_DATE(2020-08-24); SPLIT_POINT(2018-01-01); DeepRL; TC(0); StateMode(5); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'KSS; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0.0); StateMode(5); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'GE; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2015-01-01); DeepRL; TC(0); StateMode(5); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'HSI; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2015-01-01); DeepRL; TC(0); StateMode(5); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'AAL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0); StateMode(5); WindowSize(10); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "\n",
    "ev_deepRLAgentAutoExtraction = deepRLAgent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_windowed.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "deepRLAgentAutoExtraction_portfolio_train = ev_deepRLAgentAutoExtraction.get_daily_portfolio_value()\n",
    "ev_deepRLAgentAutoExtraction = deepRLAgent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_windowed.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "deepRLAgentAutoExtraction_portfolio_test = ev_deepRLAgentAutoExtraction.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'DQN-windowed'\n",
    "\n",
    "add_train_portfo(model_kind, deepRLAgentAutoExtraction_portfolio_train)\n",
    "add_test_portfo(model_kind, deepRLAgentAutoExtraction_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test Window Size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block is for testing the effect of window-size in the input representation on the performance of the model. Here we assume that we have a set of pretrained models in the `./Objects/DeepRL` directory. If you do not have these pretrained models, you should train from scratch and set `file_name = None`. The set of window sizes that are being tested in the experiments are in `window_sizes` list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_classes = 64\n",
    "state_mode = 5\n",
    "\n",
    "for window_size in tqdm(window_sizes):\n",
    "    \n",
    "    dataTrain_autoPatternExtractionAgent_windowed = DataAutoPatternExtractionAgent(data_loader.data_train, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "    dataTest_autoPatternExtractionAgent_windowed = DataAutoPatternExtractionAgent(data_loader.data_test, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "    \n",
    "    deepRLAgent = DeepRL(data_loader, dataTrain_autoPatternExtractionAgent_windowed, \n",
    "                         dataTest_autoPatternExtractionAgent_windowed, \n",
    "                         DATASET_NAME,  state_mode, window_size, transaction_cost,\n",
    "                         BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS, ReplayMemorySize=ReplayMemorySize,\n",
    "                         TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "  \n",
    "    file_name = None\n",
    "    deepRLAgent.train(n_episodes)\n",
    "    \n",
    "#     file_name = f'GOOGL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); DeepRL; TC(0.0); StateMode(4); WindowSize({window_size}); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "\n",
    "    ev_deepRLAgentAutoExtraction = deepRLAgent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_windowed.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='train')\n",
    "    deepRLAgentAutoExtraction_portfolio_train = ev_deepRLAgentAutoExtraction.get_daily_portfolio_value()\n",
    "    ev_deepRLAgentAutoExtraction = deepRLAgent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_windowed.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='test')\n",
    "    deepRLAgentAutoExtraction_portfolio_test = ev_deepRLAgentAutoExtraction.get_daily_portfolio_value()\n",
    "\n",
    "    model_kind = 'DQN-windowed'\n",
    "\n",
    "    add_train_portfo(model_kind, deepRLAgentAutoExtraction_portfolio_train)\n",
    "    add_test_portfo(model_kind, deepRLAgentAutoExtraction_portfolio_test)\n",
    "    \n",
    "    if model_kind not in window_size_experiment.keys():\n",
    "        window_size_experiment[model_kind] = {}\n",
    "\n",
    "    window_size_experiment[model_kind][window_size] = \\\n",
    "        ((test_portfolios[model_kind][-1] - test_portfolios[model_kind][0])/test_portfolios[model_kind][0] * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder Decoder models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section dedicates to the Deep Q-Learning agent with encoder. `TARGET_UPDATE` is how frequent the policy network is hard-copied to the target network in terms of number of episodes. `n_actions` is set to 3 because we have `buy`, `sell` and `none` actions. `n_episodes` is the number of episodes to run the algorithm. `EPS` is the $\\epsilon$ in the $\\epsilon$-greedy method. Decoder models include: MLP, 1-layerd 1d CNN, 1-layerd 2d CNN, two layered 1d CNN, GRU, CNN-GRU, CNN-Attn (CNN with an attention layer). For details about these models, please refer to the paper and README file.   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 10\n",
    "EPS = 0.1\n",
    "# EPS_START = 0.9\n",
    "# EPS_END = 0.05\n",
    "# EPS_DECAY = 200\n",
    "\n",
    "ReplayMemorySize = 20\n",
    "\n",
    "TARGET_UPDATE = 5\n",
    "n_actions = 3\n",
    "# window_size = 20\n",
    "\n",
    "num_episodes = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the following bocks, each block is for running the DQN algorithm in a specific condition. For each block, if you want to train from scratch, use the following two lines:\n",
    "```\n",
    "<name of the agent>.train(n_episodes)\n",
    "file_name = None\n",
    "```\n",
    "pay attention that the `file_name` should be set to `None` so that for the evaluation process, it does not load from file and instead use the recently trained agent. If you want to load from a specific model, set the `file_name` to the name of the file inside the `./Objects/<agent name>` directory."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple MLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model uses MLP as encoder with alternative input representation as below: "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pattern Input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model uses the extracted patterns as observation space. These patterns are extracted in the preprocessing phase and used here instead of directly applying OHLC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "n_classes = 64\n",
    "\n",
    "simpleMLP = SimpleMLP(data_loader, dataTrain_patternBased, dataTest_patternBased, DATASET_NAME, \n",
    "                    state_mode, window_size, transaction_cost, n_classes, BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                    ReplayMemorySize=ReplayMemorySize, TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "simpleMLP.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'BTC-USD; DATA_KIND(PatternBased); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TransactionCost(0.0); StateMode(1); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT(1).pkl'\n",
    "# file_name = 'GOOGL; DATA_KIND(PatternBased); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(1); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(PatternBased); BEGIN_DATE(2010-01-01); END_DATE(2020-08-24); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(1); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'KSS; DATA_KIND(PatternBased); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(1); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_patternBased.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "simpleMLP_portfolio_train = ev_simpleMLP.get_daily_portfolio_value()\n",
    "ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_patternBased.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "simpleMLP_portfolio_test = ev_simpleMLP.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'MLP-pattern'\n",
    "\n",
    "add_train_portfo(model_kind, simpleMLP_portfolio_train)\n",
    "add_test_portfo(model_kind, simpleMLP_portfolio_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### OHLC input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model uses the OHLC representation of candles. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "n_classes = 64\n",
    "\n",
    "simpleMLP = SimpleMLP(data_loader, dataTrain_autoPatternExtractionAgent, dataTest_autoPatternExtractionAgent, DATASET_NAME, \n",
    "                    state_mode, window_size, transaction_cost, n_classes, BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                    ReplayMemorySize=ReplayMemorySize, TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "simpleMLP.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'GOOGL; MLP; StateMode(1); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'AAPL; MLP; StateMode(1); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'KSS; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(1); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'BTC-USD; MLP; StateMode(1); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'GE; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2015-01-01); MLP; TC(0); StateMode(1); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'HSI; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2015-01-01); MLP; TC(0); StateMode(1); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'AAL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0); StateMode(1); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "\n",
    "ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "simpleMLP_portfolio_train = ev_simpleMLP.get_daily_portfolio_value()\n",
    "ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "simpleMLP_portfolio_test = ev_simpleMLP.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'MLP-vanilla'\n",
    "# model_kind = 'MLP'\n",
    "\n",
    "add_train_portfo(model_kind, simpleMLP_portfolio_train)\n",
    "add_test_portfo(model_kind, simpleMLP_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Candle Representation Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "n_classes = 64\n",
    "\n",
    "simpleMLP = SimpleMLP(data_loader, dataTrain_autoPatternExtractionAgent_candle_rep, \n",
    "                      dataTest_autoPatternExtractionAgent_candle_rep, DATASET_NAME, \n",
    "                    state_mode, window_size, transaction_cost, n_classes, BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                    ReplayMemorySize=ReplayMemorySize, TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "simpleMLP.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "\n",
    "# file_name = 'BTC-USD; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TransactionCost(0.0); StateMode(4); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT(1).pkl'\n",
    "# file_name = 'GOOGL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(4); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(2010-01-01); END_DATE(2020-08-24); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(4); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'KSS; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(4); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_candle_rep.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "simpleMLP_portfolio_train = ev_simpleMLP.get_daily_portfolio_value()\n",
    "ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_candle_rep.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "simpleMLP_portfolio_test = ev_simpleMLP.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'MLP-candle-rep'\n",
    "\n",
    "add_train_portfo(model_kind, simpleMLP_portfolio_train)\n",
    "add_test_portfo(model_kind, simpleMLP_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Windowed Input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Input is a window of OHLC candles."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "n_classes = 64\n",
    "\n",
    "simpleMLP = SimpleMLP(data_loader, dataTrain_autoPatternExtractionAgent_windowed, \n",
    "                      dataTest_autoPatternExtractionAgent_windowed, DATASET_NAME, \n",
    "                    state_mode, window_size, transaction_cost, n_classes, BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                    ReplayMemorySize=ReplayMemorySize, TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "simpleMLP.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "\n",
    "# file_name = 'BTC-USD; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TransactionCost(0.0); StateMode(5); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'GOOGL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(5); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(2010-01-01); END_DATE(2020-08-24); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(5); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'KSS; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(5); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "# file_name = 'BTC-USD; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(5); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'GOOGL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(5); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(2010-01-01); END_DATE(2020-08-24); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(5); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'KSS; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0.0); StateMode(5); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'GE; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2015-01-01); MLP; TC(0); StateMode(5); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'HSI; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2015-01-01); MLP; TC(0); StateMode(5); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'AAL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); MLP; TC(0); StateMode(5); WindowSize(10); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "\n",
    "ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_windowed.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "simpleMLP_portfolio_train = ev_simpleMLP.get_daily_portfolio_value()\n",
    "ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_windowed.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "simpleMLP_portfolio_test = ev_simpleMLP.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'MLP-windowed'\n",
    "\n",
    "add_train_portfo(model_kind, simpleMLP_portfolio_train)\n",
    "add_test_portfo(model_kind, simpleMLP_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test Window Size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block is for testing the effect of window-size in the input representation on the performance of the model. Here we assume that we have a set of pretrained models in the `./Objects/<model name>` directory. If you do not have these pretrained models, you should train from scratch and set `file_name = None`. The set of window sizes that are being tested in the experiments are in `window_sizes` list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_classes = 64\n",
    "state_mode = 5\n",
    "\n",
    "for window_size in tqdm(window_sizes):\n",
    "    \n",
    "    dataTrain_autoPatternExtractionAgent_windowed = DataAutoPatternExtractionAgent(data_loader.data_train, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "    dataTest_autoPatternExtractionAgent_windowed = DataAutoPatternExtractionAgent(data_loader.data_test, state_mode, 'action_encoder_decoder', device, GAMMA, n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "    \n",
    "    simpleMLP = SimpleMLP(data_loader, dataTrain_autoPatternExtractionAgent_windowed, \n",
    "                          dataTest_autoPatternExtractionAgent_windowed, DATASET_NAME, \n",
    "                        state_mode, window_size, transaction_cost, n_classes, BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                        ReplayMemorySize=ReplayMemorySize, TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "    \n",
    "    simpleMLP.train(num_episodes)\n",
    "    file_name = None\n",
    "    \n",
    "#     file_name = f'GE; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2015-01-01); MLP; TC(0); StateMode(5); WindowSize({window_size}); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "    ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_windowed.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='train')\n",
    "    simpleMLP_portfolio_train = ev_simpleMLP.get_daily_portfolio_value()\n",
    "    ev_simpleMLP = simpleMLP.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent_windowed.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='test')\n",
    "    simpleMLP_portfolio_test = ev_simpleMLP.get_daily_portfolio_value()\n",
    "\n",
    "    model_kind = 'MLP-windowed'\n",
    "\n",
    "    add_train_portfo(model_kind, simpleMLP_portfolio_train)\n",
    "    add_test_portfo(model_kind, simpleMLP_portfolio_test)\n",
    "    \n",
    "    if model_kind not in window_size_experiment.keys():\n",
    "        window_size_experiment[model_kind] = {}\n",
    "\n",
    "    window_size_experiment[model_kind][window_size] = \\\n",
    "        ((test_portfolios[model_kind][-1] - test_portfolios[model_kind][0])/test_portfolios[model_kind][0] * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple CNN "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1-layerd 1d CNN as encoder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "n_classes = 128\n",
    "\n",
    "simpleCNN = SimpleCNN(data_loader, dataTrain_autoPatternExtractionAgent, dataTest_autoPatternExtractionAgent, DATASET_NAME, \n",
    "                    state_mode, window_size, transaction_cost, n_classes, BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                    ReplayMemorySize=ReplayMemorySize, TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step)\n",
    "\n",
    "simpleCNN.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'KSS; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); SimpleCNNEncoder; TC(0.0); StateMode(1); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); SimpleCNNEncoder; TC(0); StateMode(1); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'GOOGL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(None); END_DATE(None); SPLIT_POINT(2018-01-01); SimpleCNNEncoder; TC(0.0); StateMode(1); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(AutoPatternExtraction); BEGIN_DATE(2010-01-01); END_DATE(2020-08-24); SPLIT_POINT(2018-01-01); SimpleCNNEncoder; TC(0.0); StateMode(1); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "ev_simpleCNN = simpleCNN.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "simpleCNN_portfolio_train = ev_simpleCNN.get_daily_portfolio_value()\n",
    "ev_simpleCNN = simpleCNN.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "simpleCNN_portfolio_test = ev_simpleCNN.get_daily_portfolio_value()\n",
    "\n",
    "# model_kind = 'CNN1d-vanilla'\n",
    "model_kind = 'CNN1D'\n",
    "\n",
    "add_train_portfo(model_kind, simpleCNN_portfolio_train)\n",
    "add_test_portfo(model_kind, simpleCNN_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN2D"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1-layerd 2d CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "decoder_input_size = 128\n",
    "\n",
    "cnn2d_agent = cnn2d(data_loader, dataTrain_sequential, dataTest_sequential, DATASET_NAME, decoder_input_size, \n",
    "                    transaction_cost, BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS, ReplayMemorySize=ReplayMemorySize,\n",
    "                    TARGET_UPDATE=TARGET_UPDATE, n_actions=n_actions, n_step=n_step, window_size=window_size)\n",
    "\n",
    "cnn2d_agent.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'KSS; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN2D; TC(0.0); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN2D; TC(0.0); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'GOOGL; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN2D; TC(0.0); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(LSTMSequential); Dates(2010-01-01, 2018-01-01, 2020-08-24); CNN2D; TC(0.0); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "\n",
    "ev_cnn2d_agent = cnn2d_agent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "cnn2d_agent_portfolio_train = ev_cnn2d_agent.get_daily_portfolio_value()\n",
    "ev_cnn2d_agent = cnn2d_agent.test(file_name=file_name, action_name=dataTrain_autoPatternExtractionAgent.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "cnn2d_agent_portfolio_test = ev_cnn2d_agent.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'CNN2D'\n",
    "\n",
    "add_train_portfo(model_kind, cnn2d_agent_portfolio_train)\n",
    "add_test_portfo(model_kind, cnn2d_agent_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GRU as Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size = 64\n",
    "\n",
    "gru_agent = gru(data_loader, dataTrain_sequential, dataTest_sequential, DATASET_NAME, transaction_cost, hidden_size,\n",
    "                    BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                    ReplayMemorySize=ReplayMemorySize,\n",
    "                    TARGET_UPDATE=TARGET_UPDATE,\n",
    "                    n_actions=n_actions,\n",
    "                    n_step=n_step,\n",
    "                    window_size=window_size)\n",
    "\n",
    "gru_agent.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'GOOGL; DATA_KIND(LSTMSequential); GRU; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(LSTMSequential); GRU; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.7; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(LSTMSequential); GRU; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'KSS; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); GRU; TC(0.0); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'GE; DATA_KIND(LSTMSequential); Dates(None, 2015-01-01, None); GRU; TC(0); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'HSI; DATA_KIND(LSTMSequential); Dates(None, 2015-01-01, None); GRU; TC(0); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'AAL; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); GRU; TC(0); WindowSize(10); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "\n",
    "# file_name = 'BTC-USD; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); GRU; TC(0); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'GOOGL; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); GRU; TC(0); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(LSTMSequential); Dates(2010-01-01, 2018-01-01, 2020-08-24); GRU; TC(0); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "# file_name = 'KSS; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); GRU; TC(0.0); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "model_kind = 'GRU'\n",
    "\n",
    "ev_gru_agent = gru_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "gru_agent_portfolio_train = ev_gru_agent.get_daily_portfolio_value()\n",
    "ev_gru_agent = gru_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "gru_agent_portfolio_test = ev_gru_agent.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'GRU'\n",
    "\n",
    "add_train_portfo('GRU', gru_agent_portfolio_train)\n",
    "add_test_portfo('GRU', gru_agent_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### window size test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block is for testing the effect of window-size in the input representation on the performance of the model. Here we assume that we have a set of pretrained models in the `./Objects/GRU` directory. If you do not have these pretrained models, you should train from scratch and set `file_name = None`. The set of window sizes that are being tested in the experiments are in `window_sizes` list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size = 64\n",
    "\n",
    "for window_size in tqdm(window_sizes):\n",
    "    dataTrain_sequential = DataSequential(data_loader.data_train,\n",
    "                               'action_encoder_decoder', device, GAMMA,\n",
    "                               n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "    dataTest_sequential = DataSequential(data_loader.data_test,\n",
    "                              'action_encoder_decoder', device, GAMMA,\n",
    "                              n_step, BATCH_SIZE, window_size, transaction_cost)  \n",
    "    \n",
    "    gru_agent = gru(data_loader, dataTrain_sequential, dataTest_sequential, DATASET_NAME, transaction_cost, hidden_size,\n",
    "                        BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                        ReplayMemorySize=ReplayMemorySize,\n",
    "                        TARGET_UPDATE=TARGET_UPDATE,\n",
    "                        n_actions=n_actions,\n",
    "                        n_step=n_step,\n",
    "                        window_size=window_size)\n",
    "\n",
    "    gru_agent.train(num_episodes)\n",
    "    file_name = None\n",
    "    \n",
    "#     file_name = f'GOOGL; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); GRU; TC(0); WindowSize({window_size}); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "\n",
    "\n",
    "    ev_gru_agent = gru_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='train')\n",
    "    gru_agent_portfolio_train = ev_gru_agent.get_daily_portfolio_value()\n",
    "    ev_gru_agent = gru_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='test')\n",
    "    gru_agent_portfolio_test = ev_gru_agent.get_daily_portfolio_value()\n",
    "\n",
    "    add_train_portfo('GRU', gru_agent_portfolio_train)\n",
    "    add_test_portfo('GRU', gru_agent_portfolio_test)\n",
    "\n",
    "    if 'GRU' not in window_size_experiment.keys():\n",
    "        window_size_experiment['GRU'] = {}\n",
    "\n",
    "    window_size_experiment['GRU'][window_size] = \\\n",
    "        ((test_portfolios['GRU'][-1] - test_portfolios['GRU'][0])/test_portfolios['GRU'][0] * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convolutional"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Two-layered 1d CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn_agent = cnn(data_loader, dataTrain_sequential, dataTest_sequential, DATASET_NAME, transaction_cost,\n",
    "                    BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                    ReplayMemorySize=ReplayMemorySize,\n",
    "                    TARGET_UPDATE=TARGET_UPDATE,\n",
    "                    n_actions=n_actions,\n",
    "                    n_step=n_step,\n",
    "                    window_size=window_size)\n",
    "\n",
    "cnn_agent.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'GOOGL; DATA_KIND(LSTMSequential); Convolutional; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(LSTMSequential); Convolutional; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(LSTMSequential); Convolutional; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'KSS; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN; TC(0.0); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'GE; DATA_KIND(LSTMSequential); Dates(None, 2015-01-01, None); CNN; TC(0); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'HSI; DATA_KIND(LSTMSequential); Dates(None, 2015-01-01, None); CNN; TC(0); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'AAL; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN; TC(0); WindowSize(10); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "\n",
    "# file_name = 'BTC-USD; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN; TC(0.0); WindowSize(3); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "\n",
    "model_kind = 'CNN'\n",
    "\n",
    "ev_cnn = cnn_agent.test(file_name=file_name,action_name=dataTrain_sequential.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "cnn_portfolio_train = ev_cnn.get_daily_portfolio_value()\n",
    "ev_cnn = cnn_agent.test(file_name=file_name,action_name=dataTrain_sequential.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "cnn_portfolio_test = ev_cnn.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'CNN'\n",
    "\n",
    "add_train_portfo('CNN', cnn_portfolio_train)\n",
    "add_test_portfo('CNN', cnn_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test Window Size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block is for testing the effect of window-size in the input representation on the performance of the model. Here we assume that we have a set of pretrained models in the `./Objects/CNN` directory. If you do not have these pretrained models, you should train from scratch and set `file_name = None`. The set of window sizes that are being tested in the experiments are in `window_sizes` list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for window_size in tqdm(window_sizes):\n",
    "    dataTrain_sequential = DataSequential(data_loader.data_train,\n",
    "                               'action_encoder_decoder', device, GAMMA,\n",
    "                               n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "    dataTest_sequential = DataSequential(data_loader.data_test,\n",
    "                              'action_encoder_decoder', device, GAMMA,\n",
    "                              n_step, BATCH_SIZE, window_size, transaction_cost) \n",
    "    \n",
    "    cnn_agent = cnn(data_loader, dataTrain_sequential, dataTest_sequential, DATASET_NAME, transaction_cost,\n",
    "                        BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                        ReplayMemorySize=ReplayMemorySize,\n",
    "                        TARGET_UPDATE=TARGET_UPDATE,\n",
    "                        n_actions=n_actions,\n",
    "                        n_step=n_step,\n",
    "                        window_size=window_size)\n",
    "\n",
    "    cnn_agent.train(num_episodes)\n",
    "    file_name = None\n",
    "    \n",
    "#     file_name = f'GE; DATA_KIND(LSTMSequential); Dates(None, 2015-01-01, None); CNN; TC(0); WindowSize({window_size}); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "    \n",
    "    ev_cnn = cnn_agent.test(file_name=file_name,action_name=dataTrain_sequential.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='train')\n",
    "    cnn_portfolio_train = ev_cnn.get_daily_portfolio_value()\n",
    "    ev_cnn = cnn_agent.test(file_name=file_name,action_name=dataTrain_sequential.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='test')\n",
    "    cnn_portfolio_test = ev_cnn.get_daily_portfolio_value()\n",
    "\n",
    "    add_train_portfo('CNN', cnn_portfolio_train)\n",
    "    add_test_portfo('CNN', cnn_portfolio_test)\n",
    "    \n",
    "    if 'CNN' not in window_size_experiment.keys():\n",
    "        window_size_experiment['CNN'] = {}\n",
    "\n",
    "    window_size_experiment['CNN'][window_size] = \\\n",
    "        ((test_portfolios['CNN'][-1] - test_portfolios['CNN'][0])/test_portfolios['CNN'][0] * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN-GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn_gru_agent = cnn_gru(data_loader, dataTrain_sequential, dataTest_sequential, DATASET_NAME, transaction_cost, hidden_size,\n",
    "                    BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                    ReplayMemorySize=ReplayMemorySize,\n",
    "                    TARGET_UPDATE=TARGET_UPDATE,\n",
    "                    n_actions=n_actions,\n",
    "                    n_step=n_step,\n",
    "                    window_size=window_size)\n",
    "\n",
    "cnn_gru_agent.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'GOOGL; DATA_KIND(LSTMSequential); CNN_GRU; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(LSTMSequential); CNN_GRU; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(LSTMSequential); CNN_GRU; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'KSS; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN-GRU; TC(0.0); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'GE; DATA_KIND(LSTMSequential); Dates(None, 2015-01-01, None); CNN-GRU; TC(0); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'HSI; DATA_KIND(LSTMSequential); Dates(None, 2015-01-01, None); CNN-GRU; TC(0); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'AAL; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN-GRU; TC(0); WindowSize(10); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "\n",
    "ev_cnn_gru = cnn_gru_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "cnn_gru_portfolio_train = ev_cnn_gru.get_daily_portfolio_value()\n",
    "ev_cnn_gru = cnn_gru_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "cnn_gru_portfolio_test = ev_cnn_gru.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'CNN-GRU'\n",
    "\n",
    "add_train_portfo('CNN-GRU', cnn_gru_portfolio_train)\n",
    "add_test_portfo('CNN-GRU', cnn_gru_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test Window Size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block is for testing the effect of window-size in the input representation on the performance of the model. Here we assume that we have a set of pretrained models in the `./Objects/CNN-GRU` directory. If you do not have these pretrained models, you should train from scratch and set `file_name = None`. The set of window sizes that are being tested in the experiments are in `window_sizes` list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size = 64\n",
    "\n",
    "for window_size in tqdm(window_sizes):\n",
    "    dataTrain_sequential = DataSequential(data_loader.data_train,\n",
    "                               'action_encoder_decoder', device, GAMMA,\n",
    "                               n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "    dataTest_sequential = DataSequential(data_loader.data_test,\n",
    "                              'action_encoder_decoder', device, GAMMA,\n",
    "                              n_step, BATCH_SIZE, window_size, transaction_cost) \n",
    "    \n",
    "    cnn_gru_agent = cnn_gru(data_loader, dataTrain_sequential, dataTest_sequential, DATASET_NAME, transaction_cost, hidden_size,\n",
    "                        BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                        ReplayMemorySize=ReplayMemorySize,\n",
    "                        TARGET_UPDATE=TARGET_UPDATE,\n",
    "                        n_actions=n_actions,\n",
    "                        n_step=n_step,\n",
    "                        window_size=window_size)\n",
    "    \n",
    "    cnn_gru_agent.train(num_episodes)\n",
    "    file_name = None\n",
    "\n",
    "\n",
    "#     file_name = f'GOOGL; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN-GRU; TC(0); WindowSize({window_size}); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "    \n",
    "    ev_cnn_gru = cnn_gru_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='train')\n",
    "    cnn_gru_portfolio_train = ev_cnn_gru.get_daily_portfolio_value()\n",
    "    ev_cnn_gru = cnn_gru_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='test')\n",
    "    cnn_gru_portfolio_test = ev_cnn_gru.get_daily_portfolio_value()\n",
    "\n",
    "    add_train_portfo('CNN-GRU', cnn_gru_portfolio_train)\n",
    "    add_test_portfo('CNN-GRU', cnn_gru_portfolio_test)\n",
    "    \n",
    "    if 'CNN-GRU' not in window_size_experiment.keys():\n",
    "        window_size_experiment['CNN-GRU'] = {}\n",
    "\n",
    "    window_size_experiment['CNN-GRU'][window_size] = \\\n",
    "        ((test_portfolios['CNN-GRU'][-1] - test_portfolios['CNN-GRU'][0])/test_portfolios['CNN-GRU'][0] * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN with Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attn_output_size = hidden_size\n",
    "\n",
    "cnn_attn_agent = cnn_attn(data_loader, dataTrain_sequential, dataTest_sequential, DATASET_NAME, transaction_cost, attn_output_size, \n",
    "                    BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                    ReplayMemorySize=ReplayMemorySize,\n",
    "                    TARGET_UPDATE=TARGET_UPDATE,\n",
    "                    n_actions=n_actions,\n",
    "                    n_step=n_step,\n",
    "                    window_size=window_size)\n",
    "\n",
    "cnn_attn_agent.train(num_episodes)\n",
    "file_name = None\n",
    "\n",
    "# file_name = 'GOOGL; DATA_KIND(LSTMSequential); CNNAttn; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'AAPL; DATA_KIND(LSTMSequential); Dates(2010-01-01, 2018-01-01, 2020-08-24); CNN-ATTN; TC(0); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'BTC-USD; DATA_KIND(LSTMSequential); CNNAttn; PredictionStep(None); WindowSize(20); TRAIN_TEST_SPLIT(True); BATCH_SIZE10; GAMMA0.7; EPSILON0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10; EXPERIMENT.pkl'\n",
    "# file_name = 'KSS; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN-ATTN; TC(0.0); WindowSize(20); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'GE; DATA_KIND(LSTMSequential); Dates(None, 2015-01-01, None); CNN-ATTN; TC(0); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'HSI; DATA_KIND(LSTMSequential); Dates(None, 2015-01-01, None); CNN-ATTN; TC(0); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "# file_name = 'AAL; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN-ATTN; TC(0); WindowSize(15); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10.pkl'\n",
    "\n",
    "ev_cnn_attn = cnn_attn_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='train')\n",
    "cnn_attn_portfolio_train = ev_cnn_attn.get_daily_portfolio_value()\n",
    "ev_cnn_attn = cnn_attn_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                  initial_investment=initial_investment, test_type='test')\n",
    "cnn_attn_portfolio_test = ev_cnn_attn.get_daily_portfolio_value()\n",
    "\n",
    "model_kind = 'CNN-ATTN'\n",
    "\n",
    "add_train_portfo('CNN-ATTN', cnn_attn_portfolio_train)\n",
    "add_test_portfo('CNN-ATTN', cnn_attn_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test Window Size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block is for testing the effect of window-size in the input representation on the performance of the model. Here we assume that we have a set of pretrained models in the `./Objects/CNNAttn` directory. If you do not have these pretrained models, you should train from scratch and set `file_name = None`. The set of window sizes that are being tested in the experiments are in `window_sizes` list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attn_output_size = hidden_size\n",
    "\n",
    "for window_size in tqdm(window_sizes):\n",
    "    dataTrain_sequential = DataSequential(data_loader.data_train,\n",
    "                               'action_encoder_decoder', device, GAMMA,\n",
    "                               n_step, BATCH_SIZE, window_size, transaction_cost)\n",
    "    dataTest_sequential = DataSequential(data_loader.data_test,\n",
    "                              'action_encoder_decoder', device, GAMMA,\n",
    "                              n_step, BATCH_SIZE, window_size, transaction_cost) \n",
    "    \n",
    "    cnn_attn_agent = cnn_attn(data_loader, dataTrain_sequential, dataTest_sequential, DATASET_NAME, transaction_cost, attn_output_size, \n",
    "                        BATCH_SIZE=BATCH_SIZE, GAMMA=GAMMA, EPS=EPS,\n",
    "                        ReplayMemorySize=ReplayMemorySize,\n",
    "                        TARGET_UPDATE=TARGET_UPDATE,\n",
    "                        n_actions=n_actions,\n",
    "                        n_step=n_step,\n",
    "                        window_size=window_size)\n",
    "    \n",
    "    cnn_attn_agent.train(num_episodes)\n",
    "    file_name = None\n",
    "\n",
    "#     file_name = f'BTC-USD; DATA_KIND(LSTMSequential); Dates(None, 2018-01-01, None); CNN-ATTN; TC(0); WindowSize({window_size}); BATCH_SIZE10; GAMMA0.7; EPS0.1; REPLAY_MEMORY_SIZE20; C5; N_SARSA10(1).pkl'\n",
    "    \n",
    "    ev_cnn_attn = cnn_attn_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='train')\n",
    "    cnn_attn_portfolio_train = ev_cnn_attn.get_daily_portfolio_value()\n",
    "    ev_cnn_attn = cnn_attn_agent.test(file_name=file_name, action_name=dataTrain_sequential.action_name,\n",
    "                                      initial_investment=initial_investment, test_type='test')\n",
    "    cnn_attn_portfolio_test = ev_cnn_attn.get_daily_portfolio_value()\n",
    "\n",
    "    add_train_portfo('CNN-ATTN', cnn_attn_portfolio_train)\n",
    "    add_test_portfo('CNN-ATTN', cnn_attn_portfolio_test)\n",
    "\n",
    "    if 'CNN-ATTN' not in window_size_experiment.keys():\n",
    "        window_size_experiment['CNN-ATTN'] = {}\n",
    "\n",
    "    window_size_experiment['CNN-ATTN'][window_size] = \\\n",
    "        ((test_portfolios['CNN-ATTN'][-1] - test_portfolios['CNN-ATTN'][0])/test_portfolios['CNN-ATTN'][0] * 100)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rule Based"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The rule-base agent that uses a set of rules for generating trading strategies. You can find the explanation about this agent in both the paper and the README file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ev_RuleBased = Evaluation(dataTrain_patternBased.data, 'action', 1000)\n",
    "print('train')\n",
    "ev_RuleBased.evaluate()\n",
    "ruleBased_portfolio_train = ev_RuleBased.get_daily_portfolio_value()\n",
    "ev_RuleBased = Evaluation(dataTest_patternBased.data, 'action', 1000)\n",
    "print('test')\n",
    "ev_RuleBased.evaluate()\n",
    "ruleBased_portfolio_test = ev_RuleBased.get_daily_portfolio_value()\n",
    "\n",
    "add_train_portfo('Rule-Based', ruleBased_portfolio_train)\n",
    "add_test_portfo('Rule-Based', ruleBased_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Buy and Hold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Buy the stock in the beginning of the process and hold it until the end without selling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "dataTrain_patternBased.data[dataTrain_patternBased.action_name] = 'buy'\n",
    "ev_BandH = Evaluation(dataTrain_patternBased.data, dataTrain_patternBased.action_name, initial_investment)\n",
    "print('train')\n",
    "ev_BandH.evaluate()\n",
    "BandH_portfolio_train = ev_BandH.get_daily_portfolio_value()\n",
    "\n",
    "dataTest_patternBased.data[dataTest_patternBased.action_name] = 'buy'\n",
    "ev_BandH = Evaluation(dataTest_patternBased.data, dataTest_patternBased.action_name, initial_investment)\n",
    "print('test')\n",
    "ev_BandH.evaluate()\n",
    "BandH_portfolio_test = ev_BandH.get_daily_portfolio_value()\n",
    "\n",
    "add_train_portfo('B&H', BandH_portfolio_train)\n",
    "add_test_portfo('B&H', BandH_portfolio_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Diagrams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the blocks related to `Action List`, you can represent the strategies generated by each model on a diagram. In order to do so, you should first run the block related to that model in the above, then run one of the blocks for related to `Action List` for plotting the strategy devised by that specific model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Action List on candlestick chart"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, in order to plot a better representation, we limit the number of canldlestick in each plot to 100. You can increase or decrease this amount by changing the values of `begin` and `end` in the following block. The `begin` and `end` are indices to the original data to select the interval of data you want to see it's trading strategy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "begin = 0\n",
    "end =100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_test = dataTest_autoPatternExtractionAgent\n",
    "# data_test = dataTest_autoPatternExtractionAgent_windowed\n",
    "data_test = dataTest_sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_test.data[data_test.data[data_test.action_name] == 'None']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_num = 1\n",
    "RESULTS_PATH = f'TestResults/ActionList/{model_kind}/'\n",
    "\n",
    "import os\n",
    "if not os.path.exists(RESULTS_PATH):\n",
    "    os.mkdir(RESULTS_PATH)\n",
    "\n",
    "while os.path.exists(f'{RESULTS_PATH}{DATASET_NAME};{model_kind};actions({experiment_num}).svg'):\n",
    "    experiment_num += 1\n",
    "\n",
    "fig_file = f'{RESULTS_PATH}{DATASET_NAME};{model_kind};actions({experiment_num}).svg'\n",
    "\n",
    "scope = PlotlyScope()\n",
    "\n",
    "df1 = data_loader.data_test_with_date[begin:end]\n",
    "actionlist = list(data_test.data[data_test.action_name][begin:end])\n",
    "df1[data_test.action_name] = actionlist\n",
    "\n",
    "buy = df1.copy()\n",
    "sell = df1.copy()\n",
    "none = df1.copy()\n",
    "\n",
    "# buy['action'] = [(c + o)/2 if a == 0 else None for a, o, c in zip(df2.action, df1.open, df1.close)]\n",
    "# sell['action'] = [(c + o)/2 if a == 2 else None for a, o, c in zip(df2.action, df1.open, df1.close)]\n",
    "# none['action'] = [(c + o)/2 if a == 1 else None for a, o, c in zip(df2.action, df1.open, df1.close)]\n",
    "\n",
    "buy['action'] = [(c + o)/2 if a == 'buy' else None for a, o, c in zip(df1[data_test.action_name], df1.open, df1.close)]\n",
    "sell['action'] = [(c + o)/2 if a == 'sell' else None for a, o, c in zip(df1[data_test.action_name], df1.open, df1.close)]\n",
    "none['action'] = [(c + o)/2 if a == 'None' else None for a, o, c in zip(df1[data_test.action_name], df1.open, df1.close)]\n",
    "\n",
    "\n",
    "data=[go.Candlestick(x=df1.index,\n",
    "                open=df1['open'],\n",
    "                high=df1['high'],\n",
    "                low=df1['low'],\n",
    "                close=df1['close'], increasing_line_color= 'lightgreen', decreasing_line_color= '#ff6961'),\n",
    "     go.Scatter(x=df1.index, y=buy.action, mode = 'markers', \n",
    "                marker=dict(color='green', colorscale='Viridis'), name=\"buy\"), \n",
    "     go.Scatter(x=df1.index, y=none.action, mode = 'markers', \n",
    "                marker=dict(color='blue', colorscale='Viridis'), name=\"none\"), \n",
    "     go.Scatter(x=df1.index, y=sell.action, mode = 'markers', \n",
    "                marker=dict(color='red', colorscale='Viridis'), name=\"sell\")]\n",
    "\n",
    "layout = go.Layout(\n",
    "    autosize=False,\n",
    "    width=900,\n",
    "    height=600)\n",
    "\n",
    "\n",
    "figSignal = go.Figure(data=data, layout=layout)\n",
    "figSignal.show()\n",
    "# with open(fig_file, \"wb\") as f:\n",
    "#     f.write(scope.transform(figSignal, format=\"svg\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Data (Plotted using seaborn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block plots the result portfolio of different models stored in the `train_portfolios` dictionary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_num = 1\n",
    "RESULTS_PATH = 'TestResults/Train/'\n",
    "\n",
    "import os\n",
    "\n",
    "if not os.path.exists(RESULTS_PATH):\n",
    "    os.mkdir(RESULTS_PATH)\n",
    "\n",
    "while os.path.exists(f'{RESULTS_PATH}{DATASET_NAME};train;EXPERIMENT({experiment_num}).jpg'):\n",
    "    experiment_num += 1\n",
    "\n",
    "fig_file = f'{RESULTS_PATH}{DATASET_NAME};train;EXPERIMENT({experiment_num}).jpg'\n",
    "\n",
    "sns.set(rc={'figure.figsize': (15, 7)})\n",
    "\n",
    "items = list(test_portfolios.keys())\n",
    "random.shuffle(items)\n",
    "\n",
    "first = True\n",
    "for k in items:\n",
    "    profit_percentage = [(train_portfolios[k][i] - train_portfolios[k][0])/train_portfolios[k][0] * 100 \n",
    "              for i in range(len(train_portfolios[k]))]\n",
    "    difference = len(train_portfolios[k]) - len(data_loader.data_train_with_date)\n",
    "    df = pd.DataFrame({'date': data_loader.data_train_with_date.index, \n",
    "                       'portfolio':profit_percentage[difference:]})\n",
    "    if not first:\n",
    "        df.plot(ax=ax, x='date', y='portfolio', label=k)\n",
    "    else:\n",
    "        ax = df.plot(x='date', y='portfolio', label=k)\n",
    "        first = False\n",
    "\n",
    "ax.set(xlabel='Time', ylabel='%Rate of Return')\n",
    "ax.set_title(f'%Rate of Return at each point of time for training data of {DATASET_NAME}')\n",
    "        \n",
    "plt.legend()\n",
    "# plt.savefig(fig_file, dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Data (Plotted using Seaborn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle = False\n",
    "shuffle = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block plots the results of test portfolios."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "experiment_num = 1\n",
    "RESULTS_PATH = 'TestResults/Test/'\n",
    "\n",
    "import os\n",
    "\n",
    "if not os.path.exists(RESULTS_PATH):\n",
    "    os.mkdir(RESULTS_PATH)\n",
    "\n",
    "while os.path.exists(f'{RESULTS_PATH}{DATASET_NAME};test;EXPERIMENT({experiment_num}).jpg'):\n",
    "    experiment_num += 1\n",
    "\n",
    "fig_file = f'{RESULTS_PATH}{DATASET_NAME};test;EXPERIMENT({experiment_num}).jpg'\n",
    "\n",
    "sns.set(rc={'figure.figsize': (15, 7)})\n",
    "sns.set_palette(sns.color_palette(\"Paired\", 15))\n",
    "\n",
    "items = list(test_portfolios.keys())\n",
    "\n",
    "if shuffle:\n",
    "    random.shuffle(items)\n",
    "\n",
    "first = True\n",
    "for k in items:\n",
    "    profit_percentage = [(test_portfolios[k][i] - test_portfolios[k][0])/test_portfolios[k][0] * 100 \n",
    "                  for i in range(len(test_portfolios[k]))]\n",
    "    difference = len(test_portfolios[k]) - len(data_loader.data_test_with_date)\n",
    "    df = pd.DataFrame({'date': data_loader.data_test_with_date.index, \n",
    "                       'portfolio':profit_percentage[difference:]})\n",
    "    if not first:\n",
    "        df.plot(ax=ax, x='date', y='portfolio', label=k)\n",
    "    else:\n",
    "        ax = df.plot(x='date', y='portfolio', label=k)\n",
    "        first = False\n",
    "        \n",
    "ax.set(xlabel='Time', ylabel='%Rate of Return')\n",
    "ax.set_title(f'Comparing the %Rate of Return for different models '\n",
    "             f'at each point of time for test data of {DATASET_NAME}')\n",
    "plt.legend()\n",
    "plt.savefig(fig_file, dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test window size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This block plots a line showing the relation between the window-size and the profit in each model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.interpolate import interp1d\n",
    "import numpy as np\n",
    "\n",
    "# 300 represents number of points to make between T.min and T.max\n",
    "# xnew = np.linspace(3, 75, 300)\n",
    "\n",
    "experiment_num = 1\n",
    "RESULTS_PATH = 'TestResults/WindowSize/'\n",
    "\n",
    "import os\n",
    "\n",
    "if not os.path.exists(RESULTS_PATH):\n",
    "    os.mkdir(RESULTS_PATH)\n",
    "\n",
    "while os.path.exists(f'{RESULTS_PATH}{DATASET_NAME};WindowSize;test({experiment_num}).jpg'):\n",
    "    experiment_num += 1\n",
    "\n",
    "fig_file = f'{RESULTS_PATH}{DATASET_NAME};WindowSize;test({experiment_num}).jpg'\n",
    "\n",
    "sns.set(rc={'figure.figsize': (9, 5)})\n",
    "sns.set_palette(sns.color_palette(\"Paired\", 15))\n",
    "\n",
    "first = True\n",
    "for key, val in window_size_experiment.items():\n",
    "    total_returns = list(val.values())\n",
    "    \n",
    "    # Normalize returns:\n",
    "    total_returns = [(r - min(total_returns))/(max(total_returns) - min(total_returns)) for r in total_returns]\n",
    "#     f1 = interp1d(window_sizes, total_returns, kind='previous')\n",
    "    if first:\n",
    "#         ax = sns.lineplot(xnew, f1(xnew), label=key)\n",
    "        ax = sns.lineplot(window_sizes, total_returns, label=key)\n",
    "        first = False\n",
    "    else:\n",
    "        sns.lineplot(ax=ax, x=window_sizes, y=total_returns, label=key)\n",
    "#         sns.lineplot(ax=ax, x=xnew, y=f1(xnew), label=key)\n",
    "        \n",
    "ax.set(xlabel='WindowSize', ylabel='%Total Return')\n",
    "ax.set_title(f'The impact of window size on different models')\n",
    "plt.legend()\n",
    "# plt.savefig(fig_file, dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following block plots the relation between the window-size and the profit in each model in a heatmap diagram."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_num = 1\n",
    "RESULTS_PATH = 'TestResults/WindowSize/'\n",
    "\n",
    "import os\n",
    "\n",
    "if not os.path.exists(RESULTS_PATH):\n",
    "    os.mkdir(RESULTS_PATH)\n",
    "\n",
    "while os.path.exists(f'{RESULTS_PATH}{DATASET_NAME};WindowSize;heatmap;test({experiment_num}).jpg'):\n",
    "    experiment_num += 1\n",
    "\n",
    "fig_file = f'{RESULTS_PATH}{DATASET_NAME};WindowSize;heatmap;test({experiment_num}).jpg'\n",
    "\n",
    "sns.set(rc={'figure.figsize': (10, 5)})\n",
    "\n",
    "normalized = {}\n",
    "for k1 in window_size_experiment.keys():\n",
    "    normalized[k1] = {}\n",
    "    total_returns = list(window_size_experiment[k1].values())\n",
    "    max_return = max(total_returns)\n",
    "    min_return = min(total_returns)\n",
    "    for k2 in window_size_experiment[k1].keys():\n",
    "        return_val = window_size_experiment[k1][k2]\n",
    "        normalized[k1][k2] = (return_val - min_return)/(max_return - min_return)\n",
    "        \n",
    "\n",
    "df = pd.DataFrame.transpose(pd.DataFrame.from_dict(normalized))\n",
    "sns.heatmap(df)\n",
    "\n",
    "plt.savefig(fig_file, dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}