3. parameter_optimization_strategy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "36ee7fa9",
   "metadata": {},
   "source": [
    "# Finding optimal parameters\n",
    "With the setup of all required function, I explore optimal parameter values in the test set. Note that for robust parameter values, more comprenhesive testing should be done."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "59d84367",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pandas_datareader as pdr \n",
    "import yfinance as yf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats as st\n",
    "import seaborn as sns\n",
    "import networkx as nx\n",
    "import datetime\n",
    "from scipy.stats import pearsonr\n",
    "from trading_functions import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "f001308e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load ticker symbols from a CSV file into a DataFrame.\n",
    "ticker_df = pd.read_csv('nasdaq_screener_1686671592339.csv')\n",
    "\n",
    "# Extract the 'Symbol' column from the DataFrame and convert it to a list of ticker symbols.\n",
    "tickers = list(ticker_df['Symbol'].values)\n",
    "\n",
    "# Limit the list of tickers to the first 500 for manageability or computational efficiency.\n",
    "tickers = tickers[0:500]\n",
    "\n",
    "# Define the start and end dates for the training period of the trading strategy.\n",
    "training_start_date = \"2010-01-03\"\n",
    "training_end_date = \"2021-01-01\"\n",
    "\n",
    "# Define the start and end dates for the first testing period of the trading strategy.\n",
    "testing_start_date = '2021-01-03'\n",
    "testing_end_date = '2023-04-01'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7e81c5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the threshold for determining if a link (correlation) between assets is strong enough to consider.\n",
    "link_threshold = 0.6\n",
    "\n",
    "# Set the threshold for how frequently a link must occur to be considered significant.\n",
    "frequency_threshold = 0.5\n",
    "\n",
    "# Set the starting capital for the investment strategy.\n",
    "initial_capital = 1000000\n",
    "\n",
    "# Set the parameter determining how much to invest based on the strategy's signals.\n",
    "investment_parameter = 0.2\n",
    "\n",
    "# Set the threshold for determining if a loss in link (correlation) strength between assets is significant.\n",
    "link_loss_threshold = 0\n",
    "\n",
    "# Set the threshold for determining when to drop an asset pair based on decreased correlation.\n",
    "drop_out_threshold = 0.8\n",
    "\n",
    "# A boolean flag to control whether to draw plots for visual analysis. False means no plots will be drawn.\n",
    "draw = False\n",
    "\n",
    "# Set the step size for iterating through ranges of parameters when optimizing the strategy.\n",
    "step = 0.5\n",
    "\n",
    "# Set the annual risk-free rate, used for calculating metrics like the Sharpe Ratio.\n",
    "annual_risk_free_rate = 0.005\n",
    "\n",
    "# Retrieve price and daily returns data for training period using a predefined list of tickers and date range.\n",
    "price_df_train, daily_returns_df_train = get_data(tickers, training_start_date, training_end_date)\n",
    "\n",
    "# Retrieve price and daily returns data for testing period using the same list of tickers and a different date range.\n",
    "price_df_test, daily_returns_df_test = get_data(tickers, testing_start_date, testing_end_date)\n",
    "\n",
    "# Set the number of time windows to split the data into for analyzing correlations and performance.\n",
    "n_time_windows = 100\n",
    "\n",
    "# A boolean flag to indicate whether the size of the time windows is fixed (True) or should be calculated dynamically (False).\n",
    "size_boolean = True\n",
    "\n",
    "# If time window sizes are fixed, this sets the size of each time window.\n",
    "chosen_time_window_size = 30\n",
    "\n",
    "# Calculate the total number of days in the training dataset.\n",
    "tot_days = len(price_df_train)\n",
    "\n",
    "# Split the total days into time windows based on the number of windows and size settings defined above.\n",
    "time_window_splits, time_window_size = split_time_windows(tot_days, n_time_windows, chosen_time_window_size, size_boolean)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cac9c67c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the step size for iterating through parameter values.\n",
    "step = 0.1\n",
    "\n",
    "# Initialize an empty dictionary to store results of metric calculations.\n",
    "metric_results_dict = {}\n",
    "\n",
    "# Nested loops to iterate through different thresholds and parameters in increments of 'step'.\n",
    "for link_threshold in np.arange(0, 1 + step, step):\n",
    "    for frequency_threshold in np.arange(0, 1 + step, step):\n",
    "        for drop_out_threshold in np.arange(0, 1 + step, step):\n",
    "            for investment_parameter in np.arange(0, 1 + step, step):\n",
    "                # Print the current set of parameters for debugging or tracking progress.\n",
    "                print((link_threshold, frequency_threshold, drop_out_threshold, investment_parameter))\n",
    "\n",
    "                # Find groups of correlated assets based on the current link_threshold and frequency_threshold.\n",
    "                separate_groups = find_correlation_groups(time_window_splits, price_df_train, link_threshold, frequency_threshold, draw)\n",
    "\n",
    "                # Identify pairs of assets to drop based on the drop_out_threshold in the test dataset.\n",
    "                correlation_pairs_test_dict, pairs_to check = find_dropout_candidates(price_df_test, time_window_splits_test, drop_out_threshold, separate_groups)\n",
    "\n",
    "                # Choose which of the identified dropout candidates to actually drop.\n",
    "                drop_out_chosen_dict = choose_dropout_candidates(price_df_test, correlation_pairs_test_dict, separate_groups, drop_out_threshold, pairs_to_check)\n",
    "\n",
    "                # Generate trading signals based on the current set of parameters.\n",
    "                signals_dictionary, current_capital = create_signals(price_df_test, time_window_splits_test, drop_out_threshold, link_loss_threshold, initial_capital, investment_parameter, separate_groups)\n",
    "\n",
    "                # Create a portfolio based on the generated signals and calculate its performance over the test period.\n",
    "                balance_df = create_portfolio(signals_dictionary, testing_start_date, testing_end_date, initial_capital)\n",
    "\n",
    "                # Calculate performance metrics for the strategy with the current set of parameters and store the results.\n",
    "                metric_results_dict[(link_threshold, frequency_threshold, drop_out_threshold, investment_parameter)] = strategy_metrics(balance_df, annual_risk_free_rate)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "id": "66c95fff",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rank_2</th>\n",
       "      <th>rank_1</th>\n",
       "      <th>parameters</th>\n",
       "      <th>total_return</th>\n",
       "      <th>total_return_vs_risk_free</th>\n",
       "      <th>sharpe_ratio</th>\n",
       "      <th>average_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(0.0, 0.0, 0.0, 1.0)</td>\n",
       "      <td>0.001250</td>\n",
       "      <td>-0.009893</td>\n",
       "      <td>-0.033569</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.9)</td>\n",
       "      <td>0.001124</td>\n",
       "      <td>-0.010017</td>\n",
       "      <td>-0.037838</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.8)</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>-0.010140</td>\n",
       "      <td>-0.043143</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.7000000000000001)</td>\n",
       "      <td>0.000875</td>\n",
       "      <td>-0.010264</td>\n",
       "      <td>-0.049991</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.6000000000000001)</td>\n",
       "      <td>0.000750</td>\n",
       "      <td>-0.010388</td>\n",
       "      <td>-0.059120</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.5)</td>\n",
       "      <td>0.000624</td>\n",
       "      <td>-0.010511</td>\n",
       "      <td>-0.071902</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.4)</td>\n",
       "      <td>0.000500</td>\n",
       "      <td>-0.010634</td>\n",
       "      <td>-0.090967</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.30000000000000004)</td>\n",
       "      <td>0.000375</td>\n",
       "      <td>-0.010758</td>\n",
       "      <td>-0.122876</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.2)</td>\n",
       "      <td>0.000250</td>\n",
       "      <td>-0.010882</td>\n",
       "      <td>-0.186756</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.1)</td>\n",
       "      <td>0.000125</td>\n",
       "      <td>-0.011006</td>\n",
       "      <td>-0.378977</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0)</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.011129</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>(0.0, 0.0, 0.1, 0.0)</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.011129</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    rank_2  rank_1                            parameters  total_return  \\\n",
       "0        0       0                  (0.0, 0.0, 0.0, 1.0)      0.001250   \n",
       "1        1       1                  (0.0, 0.0, 0.0, 0.9)      0.001124   \n",
       "2        2       2                  (0.0, 0.0, 0.0, 0.8)      0.001000   \n",
       "3        3       3   (0.0, 0.0, 0.0, 0.7000000000000001)      0.000875   \n",
       "4        4       4   (0.0, 0.0, 0.0, 0.6000000000000001)      0.000750   \n",
       "5        5       5                  (0.0, 0.0, 0.0, 0.5)      0.000624   \n",
       "6        6       6                  (0.0, 0.0, 0.0, 0.4)      0.000500   \n",
       "7        7       7  (0.0, 0.0, 0.0, 0.30000000000000004)      0.000375   \n",
       "8        8       8                  (0.0, 0.0, 0.0, 0.2)      0.000250   \n",
       "9        9       9                  (0.0, 0.0, 0.0, 0.1)      0.000125   \n",
       "10      10      10                  (0.0, 0.0, 0.0, 0.0)      0.000000   \n",
       "11      11      11                  (0.0, 0.0, 0.1, 0.0)      0.000000   \n",
       "\n",
       "    total_return_vs_risk_free  sharpe_ratio  average_rank  \n",
       "0                   -0.009893     -0.033569           0.0  \n",
       "1                   -0.010017     -0.037838           1.0  \n",
       "2                   -0.010140     -0.043143           2.0  \n",
       "3                   -0.010264     -0.049991           3.0  \n",
       "4                   -0.010388     -0.059120           4.0  \n",
       "5                   -0.010511     -0.071902           5.0  \n",
       "6                   -0.010634     -0.090967           6.0  \n",
       "7                   -0.010758     -0.122876           7.0  \n",
       "8                   -0.010882     -0.186756           8.0  \n",
       "9                   -0.011006     -0.378977           9.0  \n",
       "10                  -0.011129           NaN          10.0  \n",
       "11                  -0.011129           NaN          11.0  "
      ]
     },
     "execution_count": 300,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize a dictionary to store aggregated results.\n",
    "results_dict = {'parameters':[], 'total_return':[], 'total_return_vs_risk_free':[], 'sharpe_ratio':[]}\n",
    "\n",
    "# Iterate over each set of parameters and corresponding results in the metric_results_dict.\n",
    "for parameters, one_result_dict in metric_results_dict.items():\n",
    "    # Append the parameters and calculated metrics (total return, total return vs. risk-free rate, Sharpe ratio) to the results_dict.\n",
    "    results_dict['parameters'].append(parameters)\n",
    "    results_dict['total_return'].append(one_result_dict['total_return'])\n",
    "    results_dict['total_return_vs_risk_free'].append(one_result_dict['total_return_vs_risk_free'])\n",
    "    results_dict['sharpe_ratio'].append(one_result_dict['sharpe_ratio'])\n",
    "\n",
    "# Convert the results_dict to a DataFrame for easier data manipulation.\n",
    "results_df = pd.DataFrame(results_dict)\n",
    "\n",
    "# Sort the DataFrame by total return in descending order and reset the index.\n",
    "results_df = results_df.sort_values(by='total_return', ascending=False).reset_index(drop=True)\n",
    "\n",
    "# Reset the index again to use it as a ranking column for total returns (rank_1).\n",
    "results_df = results_df.reset_index()\n",
    "results_df = results_df.rename(columns={\"index\": \"rank_1\"})\n",
    "\n",
    "# Sort the DataFrame by Sharpe ratio in descending order and reset the index.\n",
    "results_df = results_df.sort_values(by='sharpe_ratio', ascending=False).reset_index(drop=True)\n",
    "\n",
    "# Reset the index again to use it as a ranking column for Sharpe ratios (rank_2).\n",
    "results_df = results_df.reset_index()\n",
    "results_df = results_df.rename(columns={\"index\": \"rank_2\"})\n",
    "\n",
    "# Calculate the average rank based on the two ranking columns created above.\n",
    "results_df['average_rank'] = (results_df['rank_1'] + results_df['rank_2']) / 2\n",
    "\n",
    "# Sort the DataFrame by the average rank to determine the overall performance taking into account both total return and Sharpe ratio.\n",
    "results_df = results_df.sort_values(by='average_rank')\n",
    "\n",
    "# Return the sorted DataFrame with all calculated columns and rankings.\n",
    "results_df\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}