examples/unreal_stacked_lstm_strat_4_11.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\") # suppress h5py deprecation warning\n",
    "\n",
    "import os\n",
    "import backtrader as bt\n",
    "import numpy as np\n",
    "\n",
    "from btgym import BTgymEnv, BTgymDataset\n",
    "from btgym.strategy.observers import Reward, Position, NormPnL\n",
    "from btgym.algorithms import Launcher, Unreal, AacStackedRL2Policy\n",
    "from btgym.research import DevStrat_4_11"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stacked LSTM Agent usage example.\n",
    "\n",
    "Based on NAV_A3C+D from [\"LEARNING TO NAVIGATE IN COMPLEX ENVIRONMENTS\"](https://arxiv.org/pdf/1611.03673.pdf) paper by Mirowski at al.;\n",
    "\n",
    "Modifications to original paper arhcitecture:\n",
    "- splitted Policy/Value outputs: Policy is taken off first LSTM layer, Value - off the second;\n",
    "- LSTM state initialisation: first RNN layer context (policy) is initialised on every episode start, while second   (Value) is reset either on begining of every Trial (future work) or or every N-constant episodes (60 for this     example), motivated by RL^2 approach by Duan et al., \n",
    "  [\"FAST REINFORCEMENT LEARNING VIA SLOW REINFORCEMENT LEARNING\"](https://arxiv.org/pdf/1611.02779.pdf);\n",
    "- inner/external observation state state split: external (market) is encoded via conolution layers and fed to       first LSTM layer, inner (broker) state is fed into second LSTM layer, can optionally be encoded via separate       convolution block (doesnt seem to improve much though);\n",
    "- optional Value Replay losss (`Unreal` feature) improves sample efficiency, but is computationally expensive;\n",
    "\n",
    "Other details:\n",
    "- All convolution and LSTM layers are layer-normalized, see \n",
    "  [\"Layer Normalisation\"](https://arxiv.org/abs/1607.06450) paper by Jimmy Ba at al.;\n",
    "  \n",
    "- Upd 2.02.18: linear layers aer Noisy-Net ones, see: [Noisy Networks for Exploration] (https://arxiv.org/abs/1706.10295) paper by Fortunato at al.; policy output is centered using layer normalisation;\n",
    " added linearly decayed state scaling;\n",
    "\n",
    "- A3C option `time_flat` is ON by default, improves training stability, reduces computation costs, see \n",
    "  [Base_AAC class Note](https://kismuz.github.io/btgym/btgym.algorithms.html#module-btgym.algorithms.aac) for       details;\n",
    "  \n",
    "Diagram: https://kismuz.github.io/btgym/_images/a3c_stacked_lstm_agent.png\n",
    "\n",
    "**NOTE:**\n",
    "Currently it takes ~20M env.steps to fit 6-month 1min bars data set. Much faster on smaller ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set backtesting engine parameters:\n",
    "\n",
    "MyCerebro = bt.Cerebro()\n",
    "\n",
    "# Define strategy and broker account parameters:\n",
    "MyCerebro.addstrategy(\n",
    "    DevStrat_4_11,\n",
    "    start_cash=2000,  # initial broker cash\n",
    "    commission=0.0001,  # commisssion to imitate spread\n",
    "    leverage=10.0,\n",
    "    order_size=2000,  # fixed stake, mind leverage\n",
    "    drawdown_call=10, # max % to loose, in percent of initial cash\n",
    "    target_call=10,  # max % to win, same\n",
    "    skip_frame=10,\n",
    "    gamma=0.99,\n",
    "    reward_scale=7, # gardient`s nitrox, touch with care!\n",
    "    state_ext_scale = np.linspace(3e3, 1e3, num=5)\n",
    ")\n",
    "# Visualisations for reward, position and PnL dynamics:\n",
    "MyCerebro.addobserver(Reward)\n",
    "MyCerebro.addobserver(Position)\n",
    "MyCerebro.addobserver(NormPnL)\n",
    "\n",
    "# Data: uncomment to get up to six month of 1 minute bars:\n",
    "data_m1_6_month = [\n",
    "    './data/DAT_ASCII_EURUSD_M1_201701.csv',\n",
    "    './data/DAT_ASCII_EURUSD_M1_201702.csv',\n",
    "    './data/DAT_ASCII_EURUSD_M1_201703.csv',\n",
    "    './data/DAT_ASCII_EURUSD_M1_201704.csv',\n",
    "    './data/DAT_ASCII_EURUSD_M1_201705.csv',\n",
    "    './data/DAT_ASCII_EURUSD_M1_201706.csv',\n",
    "]\n",
    "\n",
    "# Uncomment single choice:\n",
    "MyDataset = BTgymDataset(\n",
    "    filename=data_m1_6_month,\n",
    "    #filename='./data/test_sine_1min_period256_delta0002.csv',  # simple sine \n",
    "    start_weekdays={0, 1, 2, 3, 4, 5, 6},\n",
    "    episode_duration={'days': 1, 'hours': 23, 'minutes': 40}, # note: 2day-long episode\n",
    "    start_00=False,\n",
    "    time_gap={'hours': 10},\n",
    ")\n",
    "\n",
    "env_config = dict(\n",
    "    class_ref=BTgymEnv, \n",
    "    kwargs=dict(\n",
    "        dataset=MyDataset,\n",
    "        engine=MyCerebro,\n",
    "        render_modes=['episode', 'human', 'internal', ], #'external'],\n",
    "        render_state_as_image=True,\n",
    "        render_ylabel='OHL_diff. / Internals',\n",
    "        render_size_episode=(12,8),\n",
    "        render_size_human=(9, 4),\n",
    "        render_size_state=(11, 3),\n",
    "        render_dpi=75,\n",
    "        port=5000,\n",
    "        data_port=4999,\n",
    "        connect_timeout=90,\n",
    "        verbose=0,\n",
    "    )\n",
    ")\n",
    "\n",
    "cluster_config = dict(\n",
    "    host='127.0.0.1',\n",
    "    port=12230,\n",
    "    num_workers=4,  # set according CPU's available or so\n",
    "    num_ps=1,\n",
    "    num_envs=1,\n",
    "    log_dir=os.path.expanduser('~/tmp/test_4_11'),\n",
    ")\n",
    "\n",
    "policy_config = dict(\n",
    "    class_ref=AacStackedRL2Policy,\n",
    "    kwargs={\n",
    "        'lstm_layers': (256, 256),\n",
    "        'lstm_2_init_period': 60,\n",
    "    }\n",
    ")\n",
    "\n",
    "trainer_config = dict(\n",
    "    class_ref=Unreal,\n",
    "    kwargs=dict(\n",
    "        opt_learn_rate=[1e-4, 1e-4], # random log-uniform \n",
    "        opt_end_learn_rate=1e-5,\n",
    "        opt_decay_steps=50*10**6,\n",
    "        model_gamma=0.99,\n",
    "        model_gae_lambda=1.0,\n",
    "        model_beta=0.05, # entropy reg\n",
    "        rollout_length=20,\n",
    "        time_flat=True, \n",
    "        use_value_replay=False, \n",
    "        model_summary_freq=10,\n",
    "        episode_summary_freq=1,\n",
    "        env_render_freq=5,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "launcher = Launcher(\n",
    "    cluster_config=cluster_config,\n",
    "    env_config=env_config,\n",
    "    trainer_config=trainer_config,\n",
    "    policy_config=policy_config,\n",
    "    test_mode=False,\n",
    "    max_env_steps=100*10**6,\n",
    "    root_random_seed=0,\n",
    "    purge_previous=1,  # ask to override previously saved model and logs\n",
    "    verbose=0\n",
    ")\n",
    "\n",
    "# Train it:\n",
    "launcher.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}