stockdownloads.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stocks Download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "stocks=\"XLE XLF XLU XLI XLK XLV XLY XLP XLB XOP XHB XME IYR IBB GLD\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*********************100%***********************]  15 of 15 completed\n"
     ]
    }
   ],
   "source": [
    "import yfinance as yf\n",
    "data = yf.download(stocks, start=\"1995-01-01\", end=\"2021-09-01\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"10\" halign=\"left\">Adj Close</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">Volume</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>GLD</th>\n",
       "      <th>IBB</th>\n",
       "      <th>IYR</th>\n",
       "      <th>XHB</th>\n",
       "      <th>XLB</th>\n",
       "      <th>XLE</th>\n",
       "      <th>XLF</th>\n",
       "      <th>XLI</th>\n",
       "      <th>XLK</th>\n",
       "      <th>XLP</th>\n",
       "      <th>...</th>\n",
       "      <th>XLE</th>\n",
       "      <th>XLF</th>\n",
       "      <th>XLI</th>\n",
       "      <th>XLK</th>\n",
       "      <th>XLP</th>\n",
       "      <th>XLU</th>\n",
       "      <th>XLV</th>\n",
       "      <th>XLY</th>\n",
       "      <th>XME</th>\n",
       "      <th>XOP</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1998-12-22</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.525131</td>\n",
       "      <td>13.500203</td>\n",
       "      <td>12.079448</td>\n",
       "      <td>15.476096</td>\n",
       "      <td>24.669796</td>\n",
       "      <td>15.752134</td>\n",
       "      <td>...</td>\n",
       "      <td>15200</td>\n",
       "      <td>55887</td>\n",
       "      <td>600</td>\n",
       "      <td>300500</td>\n",
       "      <td>150300</td>\n",
       "      <td>7900</td>\n",
       "      <td>5700</td>\n",
       "      <td>3700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998-12-23</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.656684</td>\n",
       "      <td>13.781269</td>\n",
       "      <td>12.257560</td>\n",
       "      <td>15.746142</td>\n",
       "      <td>25.259169</td>\n",
       "      <td>16.132940</td>\n",
       "      <td>...</td>\n",
       "      <td>67800</td>\n",
       "      <td>78784</td>\n",
       "      <td>900</td>\n",
       "      <td>338300</td>\n",
       "      <td>44000</td>\n",
       "      <td>24100</td>\n",
       "      <td>18100</td>\n",
       "      <td>12000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998-12-24</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.947957</td>\n",
       "      <td>13.708741</td>\n",
       "      <td>12.338526</td>\n",
       "      <td>15.953880</td>\n",
       "      <td>25.162939</td>\n",
       "      <td>16.105070</td>\n",
       "      <td>...</td>\n",
       "      <td>12300</td>\n",
       "      <td>43824</td>\n",
       "      <td>1400</td>\n",
       "      <td>243700</td>\n",
       "      <td>45500</td>\n",
       "      <td>23800</td>\n",
       "      <td>4900</td>\n",
       "      <td>1600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998-12-28</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.835207</td>\n",
       "      <td>13.636205</td>\n",
       "      <td>12.176604</td>\n",
       "      <td>16.036970</td>\n",
       "      <td>25.235113</td>\n",
       "      <td>16.012196</td>\n",
       "      <td>...</td>\n",
       "      <td>13500</td>\n",
       "      <td>51948</td>\n",
       "      <td>7400</td>\n",
       "      <td>309300</td>\n",
       "      <td>25400</td>\n",
       "      <td>3100</td>\n",
       "      <td>15500</td>\n",
       "      <td>8600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998-12-29</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13.070112</td>\n",
       "      <td>13.772202</td>\n",
       "      <td>12.306145</td>\n",
       "      <td>16.265474</td>\n",
       "      <td>25.307283</td>\n",
       "      <td>16.365131</td>\n",
       "      <td>...</td>\n",
       "      <td>22000</td>\n",
       "      <td>100819</td>\n",
       "      <td>1400</td>\n",
       "      <td>123800</td>\n",
       "      <td>123500</td>\n",
       "      <td>12000</td>\n",
       "      <td>5300</td>\n",
       "      <td>1100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 90 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Adj Close                                                          \\\n",
       "                 GLD IBB IYR XHB        XLB        XLE        XLF        XLI   \n",
       "Date                                                                           \n",
       "1998-12-22       NaN NaN NaN NaN  12.525131  13.500203  12.079448  15.476096   \n",
       "1998-12-23       NaN NaN NaN NaN  12.656684  13.781269  12.257560  15.746142   \n",
       "1998-12-24       NaN NaN NaN NaN  12.947957  13.708741  12.338526  15.953880   \n",
       "1998-12-28       NaN NaN NaN NaN  12.835207  13.636205  12.176604  16.036970   \n",
       "1998-12-29       NaN NaN NaN NaN  13.070112  13.772202  12.306145  16.265474   \n",
       "\n",
       "                                  ... Volume                                \\\n",
       "                  XLK        XLP  ...    XLE     XLF   XLI     XLK     XLP   \n",
       "Date                              ...                                        \n",
       "1998-12-22  24.669796  15.752134  ...  15200   55887   600  300500  150300   \n",
       "1998-12-23  25.259169  16.132940  ...  67800   78784   900  338300   44000   \n",
       "1998-12-24  25.162939  16.105070  ...  12300   43824  1400  243700   45500   \n",
       "1998-12-28  25.235113  16.012196  ...  13500   51948  7400  309300   25400   \n",
       "1998-12-29  25.307283  16.365131  ...  22000  100819  1400  123800  123500   \n",
       "\n",
       "                                         \n",
       "              XLU    XLV    XLY XME XOP  \n",
       "Date                                     \n",
       "1998-12-22   7900   5700   3700 NaN NaN  \n",
       "1998-12-23  24100  18100  12000 NaN NaN  \n",
       "1998-12-24  23800   4900   1600 NaN NaN  \n",
       "1998-12-28   3100  15500   8600 NaN NaN  \n",
       "1998-12-29  12000   5300   1100 NaN NaN  \n",
       "\n",
       "[5 rows x 90 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "adj_close=data['Adj Close']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up dataframe and train/validate/test splits\n",
    "We reserve data afer 2020 as our test dataset, and data from 2016/10/1 to 2019/12/31 for validation and rest as train dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "etfs=adj_close[['XLB','XLE','XLF','XLI','XLK','XLP','XLU','XLV','XLY']]\n",
    "\n",
    "etfs.loc[:, 'split']='val'\n",
    "etfs.loc[etfs.index > '2020-01-01', 'split']='test'\n",
    "etfs.loc[etfs.index < '2016-10-01', 'split']='train'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "etfs_log_chg = np.log1p(etfs[etfs.columns[:-2]].pct_change())\n",
    "etfs_log_chg['split'] =  etfs.split\n",
    "etfs_log_chg = etfs_log_chg.iloc[1:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save csv files\n",
    "etfs.to_csv('sector_etf.csv')\n",
    "etfs_log_chg.to_csv('sector_etf_logchg.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}