[Top 100 Stocks] FMP AI + DCF Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from urllib.request import urlopen\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "from datetime import timedelta\n",
    "import holidays\n",
    "\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.inspection import permutation_importance\n",
    "\n",
    "\n",
    "from pandas.tseries.holiday import AbstractHolidayCalendar, Holiday, nearest_workday, \\\n",
    "    USMartinLutherKingJr, USPresidentsDay, GoodFriday, USMemorialDay, \\\n",
    "    USLaborDay, USThanksgivingDay\n",
    "\n",
    "import warnings; warnings.simplefilter('ignore')\n",
    "\n",
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_progress(progress):\n",
    "    bar_length = 20\n",
    "    if isinstance(progress, int):\n",
    "        progress = float(progress)\n",
    "    if not isinstance(progress, float):\n",
    "        progress = 0\n",
    "    if progress < 0:\n",
    "        progress = 0\n",
    "    if progress >= 1:\n",
    "        progress = 1\n",
    "\n",
    "    block = int(round(bar_length * progress))\n",
    "\n",
    "    text = \"Progress: [{0}] {1:.3f}%\".format( \"#\" * block + \"-\" * (bar_length - block), progress * 100)\n",
    "    print(text, end=\"\\r\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Util Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class USTradingCalendar(AbstractHolidayCalendar):\n",
    "    rules = [\n",
    "        Holiday('NewYearsDay', month=1, day=1, observance=nearest_workday),\n",
    "        USMartinLutherKingJr,\n",
    "        USPresidentsDay,\n",
    "        GoodFriday,\n",
    "        USMemorialDay,\n",
    "        Holiday('USIndependenceDay', month=7, day=4, observance=nearest_workday),\n",
    "        USLaborDay,\n",
    "        USThanksgivingDay,\n",
    "        Holiday('Christmas', month=12, day=25, observance=nearest_workday)\n",
    "    ]\n",
    "\n",
    "\n",
    "def get_trading_close_holidays(strt, end):\n",
    "    inst = USTradingCalendar()\n",
    "\n",
    "    return inst.holidays(datetime(strt, 12, 31), datetime(end, 12, 31))\n",
    "\n",
    "\n",
    "def get_days_between(date1, date2):\n",
    "    date_format = \"%Y-%m-%d\"\n",
    "    a = datetime.strptime(date1, date_format)\n",
    "    b = datetime.strptime(date2, date_format)\n",
    "    delta = b - a\n",
    "    return delta.days\n",
    "\n",
    "\n",
    "def is_holiday(d):\n",
    "    return (d in get_trading_close_holidays(d.year-1, d.year))\n",
    "\n",
    "\n",
    "def next_weekday(date1):\n",
    "    \n",
    "    date_format = \"%Y-%m-%d\"\n",
    "    d = datetime.strptime(date1, date_format)\n",
    "    \n",
    "    while is_holiday(d) or (d.weekday() >=5):\n",
    "            d = d - timedelta(1)\n",
    "    \n",
    "    return d.strftime(date_format)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# API Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_built_url(query):\n",
    "    url = \"https://financialmodelingprep.com/api/v3/\"\n",
    "    api_key = \"apikey=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n",
    "    \n",
    "    return url + query + api_key\n",
    "\n",
    "\n",
    "def get_jsonparsed_data(url):\n",
    "    \"\"\"\n",
    "    Receive the content of ``url``, parse it as JSON and return the object.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    url : str\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    dict\n",
    "    \"\"\"\n",
    "    response = urlopen(url)\n",
    "    data = response.read().decode(\"utf-8\")\n",
    "    return json.loads(data)\n",
    "\n",
    "\n",
    "def get_df_from_url(url):\n",
    "    return pd.DataFrame.from_dict(get_jsonparsed_data(url))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reduced FMP API Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_symbols_df():\n",
    "    \"\"\"\n",
    "    Fetches all symbols on FMP, parses them in pandas and return a DataFrame object.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    None\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "    \"\"\"\n",
    "    query = 'available-traded/list?'\n",
    "    url = get_built_url(query)\n",
    "    data = get_df_from_url(url)['symbol']\n",
    "    data.to_pickle(\"./all_symbols.pkl\")\n",
    "    \n",
    "    return data\n",
    "\n",
    "\n",
    "def get_ratios_df(symbol='AAPL'):\n",
    "    \"\"\"\n",
    "    Fetches ratios for ``symbol``, parse it in pandas and return a DataFrame object.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    symbol : str\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "    \"\"\"\n",
    "    query = \"ratios/\"+symbol+\"?period=quarter&\"\n",
    "    url = get_built_url(query)\n",
    "    ret_df = get_df_from_url(url)\n",
    "#     ret_df.set_index('date', inplace=True)\n",
    "    return ret_df\n",
    "\n",
    "\n",
    "def get_key_metrics_df(symbol='AAPL'):\n",
    "    \"\"\"\n",
    "    Fetches key-metrics for ``symbol``, parse it in pandas and return a DataFrame object.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    symbol : str\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "    \"\"\"\n",
    "    query = \"key-metrics/\"+symbol+\"?period=quarter&\"\n",
    "    url = get_built_url(query)\n",
    "    ret_df = get_df_from_url(url)\n",
    "#     ret_df.set_index('date', inplace=True)\n",
    "    return ret_df\n",
    "\n",
    "\n",
    "def get_company_profile_df(symbol='AAPL'):\n",
    "    \"\"\"\n",
    "    Fetches profile for ``symbol``, parse it in pandas and return a DataFrame object.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    symbol : str\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "    \"\"\"\n",
    "    query = \"profile/\"+symbol+\"?\"\n",
    "    url = get_built_url(query)\n",
    "    return get_df_from_url(url)\n",
    "\n",
    "\n",
    "def fetch_save_exchange_profiles(symbol='NYSE'):\n",
    "    \"\"\"\n",
    "    Fetches all profiles for exchange =>``symbol``, parse it in pandas and return a DataFrame object.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    symbol : str\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "    \"\"\"\n",
    "    query = \"stock-screener?exchange=\"+symbol+\"&isActivelyTrading=true&isEtf=false&\"\n",
    "    url = get_built_url(query)\n",
    "    data = get_df_from_url(url)\n",
    "    \n",
    "    if not os.path.exists('exchanges'):\n",
    "        os.makedirs('exchanges')\n",
    "    data.to_csv('exchanges/'+symbol+'.csv')\n",
    "    \n",
    "    return data\n",
    "\n",
    "        \n",
    "def get_latest_earnings_update_symbols():\n",
    "    \"\"\"\n",
    "    Fetches ...\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    symbol : str\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "    \"\"\"\n",
    "    \n",
    "    start = (datetime.today() - timedelta(45)).strftime('%Y-%m-%d')\n",
    "    end = (datetime.today() - timedelta(0)).strftime('%Y-%m-%d')\n",
    "\n",
    "    query = \"earning_calendar?from=\"+start+\"&to=\"+end+\"&\"\n",
    "    url = get_built_url(query)\n",
    "    return get_df_from_url(url)\n",
    "\n",
    "\n",
    "def get_full_historic_price(symbol='AAPL'):\n",
    "    \"\"\"\n",
    "    Fetches all historic prices for ``symbol``, parse it in pandas and return a DataFrame object.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    symbol : str\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pd.DataFrame\n",
    "    \"\"\"\n",
    "    start = \"1989-06-20\"\n",
    "    end = datetime.today().strftime('%Y-%m-%d')\n",
    "    \n",
    "    query = \"historical-discounted-cash-flow-statement/\"+symbol+\"?period=quarter&\"\n",
    "    url = get_built_url(query)\n",
    "    ret_df = get_df_from_url(url)\n",
    "    ret_df['date'] = pd.to_datetime(ret_df['date'])\n",
    "    ret_df = ret_df.set_index('date').resample('Q').nearest()\n",
    "    ret_df['open'] = ret_df['price']\n",
    "    ret_df = ret_df.iloc[::-1]\n",
    "    ret_df = ret_df.reset_index()\n",
    "    \n",
    "    return ret_df\n",
    "\n",
    "def get_latest_price(symbol='AAPL'):\n",
    "    query = \"quote-short/\"+symbol+\"?\"\n",
    "    url = get_built_url(query)\n",
    "    ret_df = get_df_from_url(url)\n",
    "    \n",
    "    return ret_df['price'].iloc[0]\n",
    "\n",
    "\n",
    "def get_latest_bookValue(symbol='AAPL'):\n",
    "    query = \"key-metrics/\"+symbol+\"?period=quarter&\"\n",
    "    url = get_built_url(query)\n",
    "    ret_df = get_df_from_url(url)\n",
    "    \n",
    "    return ret_df['bookValuePerShare'].iloc[0]\n",
    "\n",
    "\n",
    "def get_latest_tangibleBookValue(symbol='AAPL'):\n",
    "    query = \"key-metrics/\"+symbol+\"?period=quarter&\"\n",
    "    url = get_built_url(query)\n",
    "    ret_df = get_df_from_url(url)\n",
    "    \n",
    "    return ret_df['tangibleBookValuePerShare'].iloc[0]\n",
    "\n",
    "\n",
    "def get_latest_dividendYield(symbol='AAPL'):\n",
    "    query = \"key-metrics/\"+symbol+\"?period=quarter&\"\n",
    "    url = get_built_url(query)\n",
    "    ret_df = get_df_from_url(url)\n",
    "    \n",
    "    return ret_df['dividendYield'].iloc[0]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Wrangling Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratios = ['date', 'operatingProfitMargin',\n",
    "       'netProfitMargin',\n",
    "       'assetTurnover', 'freeCashFlowOperatingCashFlowRatio',\n",
    "       'capitalExpenditureCoverageRatio']\n",
    "\n",
    "key_metrics = ['revenuePerShare', 'netIncomePerShare',\n",
    "       'operatingCashFlowPerShare', 'freeCashFlowPerShare',\n",
    "       'tangibleBookValuePerShare',\n",
    "       'shareholdersEquityPerShare', 'interestDebtPerShare',\n",
    "       'enterpriseValue', 'evToSales',\n",
    "       'evToFreeCashFlow',\n",
    "       'debtToEquity', 'netDebtToEBITDA', 'currentRatio',\n",
    "       'incomeQuality',\n",
    "       'researchAndDdevelopementToRevenue',\n",
    "       'capexToOperatingCashFlow', 'capexToRevenue',\n",
    "       'roic', 'workingCapital',\n",
    "       'netCurrentAssetValue',\n",
    "       'averageReceivables', 'averagePayables', 'averageInventory',\n",
    "       'receivablesTurnover', 'payablesTurnover',\n",
    "       'inventoryTurnover']\n",
    "\n",
    "price = ['date', 'open', 'dcf']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_single_dataset(symbol='AAPL'):\n",
    "    my_df = pd.concat([get_key_metrics_df(symbol)[key_metrics], get_ratios_df(symbol)[ratios]], axis=1)\n",
    "\n",
    "    my_df.interpolate(method='cubic', inplace=True)\n",
    "    \n",
    "    prices = get_full_historic_price(symbol)[price]\n",
    "    \n",
    "    my_df['open'] = prices['open']\n",
    "    my_df['dcf'] = prices['dcf']\n",
    "    \n",
    "    my_df = my_df.iloc[::-1]\n",
    "    \n",
    "    my_df = my_df.set_index('date')\n",
    "    \n",
    "    return my_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Top 100 Companies List by Market Cap."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exchange_codes = ['SIX', 'LSE', 'EURONEXT', 'NYSE', 'NASDAQ']\n",
    "\n",
    "for ex in exchange_codes:\n",
    "    fetch_save_exchange_profiles(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exchanges_lst = ['exchanges/'+f for f in os.listdir('exchanges/') if os.path.isfile(os.path.join('exchanges/', f))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for choice in exchange_codes:\n",
    "\n",
    "    top_x_companies = 100\n",
    "\n",
    "    index = [idx for idx, s in enumerate(exchanges_lst) if choice in s][0]\n",
    "    all_stks = pd.read_csv(exchanges_lst[index], index_col=0).nlargest(top_x_companies, 'marketCap')['symbol'].to_list()\n",
    "\n",
    "    results = []\n",
    "\n",
    "    for sym in all_stks:\n",
    "        update_progress(all_stks.index(sym)/len(all_stks))\n",
    "\n",
    "        try:\n",
    "            dataset = get_single_dataset(sym).fillna(0)\n",
    "\n",
    "            ltp = get_latest_price(sym)\n",
    "            my_X = dataset.drop(['open'], 1).iloc[-1]\n",
    "\n",
    "            if my_X.name >= '2021-09-30':\n",
    "\n",
    "                print(sym)\n",
    "                print()\n",
    "                dataset['LABELS'] = dataset['open'].shift(-1)\n",
    "                dataset = dataset.dropna()\n",
    "\n",
    "                if len(dataset) > 8:\n",
    "\n",
    "                    X = dataset.drop(['open', 'LABELS'], 1)\n",
    "                    y = dataset['LABELS']\n",
    "\n",
    "                    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)\n",
    "\n",
    "                    regr = RandomForestRegressor(n_estimators=1000, max_depth=1000, random_state=0)\n",
    "                    cv_results = cross_validate(regr, X_train, y_train, verbose=0, return_estimator=True, cv=10)\n",
    "\n",
    "                    best_estimators = []\n",
    "                    test_scores = []\n",
    "\n",
    "                    for i in range(len(cv_results['estimator'])):\n",
    "                        y_pred = cv_results['estimator'][i].predict(X_test)\n",
    "                        conf_mat = r2_score(y_test, y_pred)\n",
    "                        my_score = (conf_mat - 0.8) / abs(cv_results['test_score'][i] - conf_mat)\n",
    "                        best_estimators.append(my_score)\n",
    "                        test_scores.append(conf_mat)\n",
    "\n",
    "                    best_estimator = best_estimators.index(max(best_estimators))\n",
    "\n",
    "                    print(sym,\n",
    "                          my_X.name,\n",
    "                          round(test_scores[best_estimator]*100, 2),\n",
    "                          round(cv_results['test_score'][best_estimator]*100, 2),\n",
    "                          ltp,\n",
    "                          my_X.dcf,\n",
    "                          cv_results['estimator'][best_estimator].predict([my_X.fillna(0)])[0]\n",
    "                          )\n",
    "\n",
    "                    results.append([sym,\n",
    "                          my_X.name,\n",
    "                          round(test_scores[best_estimator]*100, 2),\n",
    "                          round(cv_results['test_score'][best_estimator]*100, 2),\n",
    "                          ltp,\n",
    "                          my_X.dcf,\n",
    "                          cv_results['estimator'][best_estimator].predict([my_X.fillna(0)])[0]       \n",
    "                         ])\n",
    "\n",
    "        except Exception as e:\n",
    "            with open('errors.txt', 'w') as f:\n",
    "                print(e, file=f)\n",
    "\n",
    "    df = pd.DataFrame.from_records(results, columns=['sym', 'date', 'train', 'test', 'ltp', 'dcf', 'pred'])\n",
    "    df['train'] = df['train'].where(df['train'] > 75, np.nan)\n",
    "    df['test'] = df['test'].where(df['test'] > 75, np.nan)\n",
    "    df = df.drop('date', axis=1)\n",
    "    df = df.dropna()\n",
    "    \n",
    "    df['DCF Growth (%)'] = ((df['dcf'] - df['ltp'])/df['ltp'])*100\n",
    "    df['AI Growth (%)'] = ((df['pred'] - df['ltp'])/df['ltp'])*100\n",
    "    \n",
    "    df['H.Mean Growth (%)'] = stats.hmean(df[['DCF Growth (%)', 'AI Growth (%)']].clip(lower=0), axis=1)\n",
    "\n",
    "    df.to_csv('FMP_AI_results_'+choice+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}