notebooks/1_data_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-31T05:53:12.252482Z",
     "start_time": "2018-03-31T05:53:12.248119Z"
    }
   },
   "source": [
    "### Data Scraping\n",
    "Scrape data for each season for boxscores and DraftKings fantasy salary information from Basketball-Reference and RotoGuru. Check existing files and create directories automatically."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-25T20:27:06.987259Z",
     "start_time": "2019-05-25T20:27:06.289012Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import time\n",
    "import glob\n",
    "\n",
    "import pandas as pd\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from constants import (\n",
    "    DATA_DIR,\n",
    "    SEASON_DATES,\n",
    "    SECONDS_SLEEP,\n",
    "    URL_BBREF,\n",
    "    BOXSCORE_FORMAT,\n",
    "    COLUMN_HEADERS,\n",
    "    URL_ROTO_FORMAT,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-25T20:37:52.027084Z",
     "start_time": "2019-05-25T20:37:51.983762Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# Scraping Historical Game Data from Basketball-Reference.com\n",
    "def get_boxscores(season, date_list):\n",
    "    print(\"Scraping boxscores from the {} regular season\".format(season))\n",
    "\n",
    "    for date in tqdm(date_list):\n",
    "        # BeautifulSoup object for a list of boxscores on a given day\n",
    "        url_summaries = URL_BBREF + BOXSCORE_FORMAT.format(\n",
    "            month=date[4:6], day=date[6:8], year=date[0:4]\n",
    "        )\n",
    "        soup_summaries = BeautifulSoup(urlopen(url_summaries), \"lxml\")\n",
    "        games = soup_summaries.find_all(\"div\", class_=\"game_summary expanded nohover\")\n",
    "\n",
    "        for game in games:\n",
    "            summary = {}\n",
    "\n",
    "            host = game.find_all(\"table\")[1].find_all(\"a\")[1][\"href\"][7:10]\n",
    "\n",
    "            winner = game.find(\"tr\", class_=\"winner\").find_all(\"td\")\n",
    "            loser = game.find(\"tr\", class_=\"loser\").find_all(\"td\")\n",
    "\n",
    "            summary[\"winner\"] = [\n",
    "                winner[0].find(\"a\")[\"href\"][7:10],\n",
    "                int(winner[1].get_text()),\n",
    "            ]\n",
    "            summary[\"loser\"] = [\n",
    "                loser[0].find(\"a\")[\"href\"][7:10],\n",
    "                int(loser[1].get_text()),\n",
    "            ]\n",
    "\n",
    "            url_game = URL_BBREF + game.find(\"a\", string=\"Box Score\")[\"href\"]\n",
    "            soup_game = BeautifulSoup(urlopen(url_game), \"lxml\")\n",
    "\n",
    "            teams = [\"winner\", \"loser\"]\n",
    "            basic_stat_template = \"box-{team}-game-basic\"\n",
    "            advanced_stat_template = \"box-{team}-game-advanced\"\n",
    "\n",
    "            for team in teams:\n",
    "                if summary[team][0] == host:\n",
    "                    home = 1\n",
    "                else:\n",
    "                    home = 0\n",
    "\n",
    "                basic_stat = basic_stat_template.format(team=summary[team][0].upper())\n",
    "                advanced_stat = advanced_stat_template.format(\n",
    "                    team=summary[team][0].upper()\n",
    "                )\n",
    "\n",
    "                game_data = [\n",
    "                    date,\n",
    "                    summary[team][0],\n",
    "                    home,\n",
    "                    summary[\"winner\"][0],\n",
    "                    summary[\"winner\"][1],\n",
    "                    summary[\"loser\"][0],\n",
    "                    summary[\"loser\"][1],\n",
    "                ]\n",
    "\n",
    "                data_basic = (\n",
    "                    soup_game.find(\"table\", id=basic_stat)\n",
    "                    .find(\"tbody\")\n",
    "                    .find_all(\"tr\", class_=None)\n",
    "                )\n",
    "                data_advanced = (\n",
    "                    soup_game.find(\"table\", id=advanced_stat)\n",
    "                    .find(\"tbody\")\n",
    "                    .find_all(\"tr\", class_=None)\n",
    "                )\n",
    "\n",
    "                n = len(data_basic)\n",
    "\n",
    "                player_names = [data_basic[i].find(\"a\").get_text() for i in range(n)]\n",
    "\n",
    "                player_data = []\n",
    "                injury_keywords = [\"Did Not Play\", \"Not With Team\"]\n",
    "\n",
    "                for i in range(n):\n",
    "                    if data_basic[i].find(\"td\").get_text() not in injury_keywords:\n",
    "                        data = (\n",
    "                            [player_names[i]]\n",
    "                            + game_data\n",
    "                            + [td.get_text() for td in data_basic[i].find_all(\"td\")]\n",
    "                            + [\n",
    "                                td.get_text()\n",
    "                                for td in data_advanced[i].find_all(\"td\")[1:]\n",
    "                            ]\n",
    "                        )\n",
    "\n",
    "                        player_data.append(data)\n",
    "\n",
    "                df = pd.DataFrame(player_data, columns=COLUMN_HEADERS)\n",
    "                df = df.fillna(0)\n",
    "\n",
    "                df[\"MP\"] = [\n",
    "                    0.00\n",
    "                    if \":\" not in t\n",
    "                    else round(int(t.split(\":\")[0]) + int(t.split(\":\")[1]) / 60, 2)\n",
    "                    for t in df[\"MP\"]\n",
    "                ]\n",
    "                df.to_csv(\n",
    "                    os.path.join(\n",
    "                        *[\n",
    "                            DATA_DIR,\n",
    "                            \"Boxscores\",\n",
    "                            season,\n",
    "                            date + \"-\" + summary[team][0] + \".csv\",\n",
    "                        ]\n",
    "                    ),\n",
    "                    index=False,\n",
    "                )\n",
    "\n",
    "            time.sleep(SECONDS_SLEEP)\n",
    "    return None\n",
    "\n",
    "\n",
    "# Scraping DraftKings salary data from RotoGuru.com\n",
    "def get_fantasy_salary(season, date_list):\n",
    "    print(\"Scraping salary information from the {} regular season\".format(season))\n",
    "\n",
    "    for date in tqdm(date_list):\n",
    "        print(date)\n",
    "        teams, positions, players, starters, salaries = [], [], [], [], []\n",
    "\n",
    "        url_date = URL_ROTO_FORMAT.format(\n",
    "            month=date[4:6], day=date[6:8], year=date[0:4]\n",
    "        )\n",
    "        soup = BeautifulSoup(urlopen(url_date), \"lxml\")\n",
    "\n",
    "        # Check if there were any games on a given date\n",
    "        soup_table = soup.find(\"body\").find(\"table\", border=\"0\", cellspacing=\"5\")\n",
    "\n",
    "        soup_rows = soup_table.find_all(\"tr\")\n",
    "\n",
    "        for row in soup_rows:\n",
    "            if row.find(\"td\").has_attr(\"colspan\") == False:\n",
    "                if row.find(\"a\").get_text() != \"\":\n",
    "                    position = row.find_all(\"td\")[0].get_text()\n",
    "\n",
    "                    player_tmp = row.find(\"a\").get_text().split(\", \")\n",
    "                    player = player_tmp[1] + \" \" + player_tmp[0]\n",
    "\n",
    "                    starter_tmp = row.find_all(\"td\")[1].get_text()\n",
    "\n",
    "                    if \"^\" in starter_tmp:\n",
    "                        starter = 1\n",
    "                    else:\n",
    "                        starter = 0\n",
    "\n",
    "                    salary_tmp = row.find_all(\"td\")[3].get_text()\n",
    "                    salary = re.sub(\"[$,]\", \"\", salary_tmp)\n",
    "\n",
    "                    team = row.find_all(\"td\")[4].get_text()\n",
    "\n",
    "                    positions.append(position)\n",
    "                    players.append(player)\n",
    "                    starters.append(starter)\n",
    "                    salaries.append(salary)\n",
    "                    teams.append(team)\n",
    "\n",
    "        df = pd.DataFrame(\n",
    "            {\n",
    "                \"Date\": [date for i in range(len(players))],\n",
    "                \"Team\": [team.upper() for team in teams],\n",
    "                \"Starter\": starters,\n",
    "                \"Pos\": positions,\n",
    "                \"Name\": players,\n",
    "                \"Salary\": salaries,\n",
    "            }\n",
    "        )\n",
    "\n",
    "        df = df.loc[:, [\"Date\", \"Team\", \"Pos\", \"Name\", \"Starter\", \"Salary\"]]\n",
    "        df.to_csv(\n",
    "            os.path.join(DATA_DIR, \"DKSalary\", season, \"salary_\" + date + \".csv\"),\n",
    "            index=False,\n",
    "        )\n",
    "\n",
    "        time.sleep(SECONDS_SLEEP)\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-25T20:59:26.409261Z",
     "start_time": "2019-05-25T20:37:59.724959Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Comment out season dates in SEASON_DATES in constants.py to extract data for specific seasons\n",
    "for data_type in [\"Boxscores\", \"DKSalary\"]:\n",
    "    for season in SEASON_DATES.keys():\n",
    "        if not os.path.exists(os.path.join(DATA_DIR, data_type, season)):\n",
    "            # Create a new directory and scrape the entire season\n",
    "            os.mkdir(os.path.join(DATA_DIR, data_type, season))\n",
    "            start_date = SEASON_DATES[season][0]\n",
    "            end_date = SEASON_DATES[season][1]\n",
    "            date_list = [\n",
    "                d.strftime(\"%Y%m%d\") for d in pd.date_range(start_date, end_date)\n",
    "            ]\n",
    "\n",
    "            if data_type == \"Boxscores\":\n",
    "                get_boxscores(season, date_list)\n",
    "            else:\n",
    "                get_fantasy_salary(season, date_list)\n",
    "\n",
    "        elif os.path.exists(os.path.join(DATA_DIR, data_type, season)):\n",
    "            # Iterate over the existing files by name and scrape missing dates\n",
    "            start_date = SEASON_DATES[season][0]\n",
    "            end_date = SEASON_DATES[season][1]\n",
    "            # Dates to scrape box scores from\n",
    "            date_list = [\n",
    "                d.strftime(\"%Y%m%d\") for d in pd.date_range(start_date, end_date)\n",
    "            ]\n",
    "\n",
    "            if data_type == \"Boxscores\":\n",
    "                for date in date_list:\n",
    "                    # Check if csv files of the form {date}-{hometeam}.csv (i.e. 20131029-CHI.csv) exists\n",
    "                    if (\n",
    "                        len(\n",
    "                            glob.glob(\n",
    "                                os.path.join(\n",
    "                                    DATA_DIR, data_type, season, str(date) + \"*.csv\"\n",
    "                                )\n",
    "                            )\n",
    "                        )\n",
    "                        > 0\n",
    "                    ):\n",
    "                        # Set back the start day by\n",
    "                        date_list = date_list[date_list.index(date) :]\n",
    "\n",
    "                get_boxscores(season, date_list)\n",
    "\n",
    "            else:\n",
    "                for date in date_list:\n",
    "                    # Check if csv files of the form salary_{date}.csv (i.e. salary_20131029.csv) exists\n",
    "                    if os.path.exists(\n",
    "                        os.path.join(\n",
    "                            DATA_DIR, data_type, season, \"salary_{}.csv\".format(date)\n",
    "                        )\n",
    "                    ):\n",
    "                        date_list = date_list[date_list.index(date) :]\n",
    "\n",
    "                get_fantasy_salary(season, date_list)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6 (default, Oct 18 2022, 12:41:40) \n[Clang 14.0.0 (clang-1400.0.29.202)]"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "vscode": {
   "interpreter": {
    "hash": "2e9f85711d675251e4e6c92db989e987b0f16da61e903a803bff5971dd18ff93"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}