CLEANED_athletic_net_ranks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7b0abd0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Enter your athletic.net log in information ##\n",
    "\n",
    "my_email = 'chad_r_smith@charter.net'\n",
    "my_password = 'maudMenten1!'\n",
    "\n",
    "## ------------------------------------------ ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6c07bd78",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "start = time.time()\n",
    "import csv\n",
    "import requests\n",
    "import re\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import random\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f61f9019",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_dict = {145512:\"A\", 145517:\"B\", 145524:\"C\", 145534:\"D\"}\n",
    "genders_dict = {\"m\":\"Boys\", \"f\":\"Girls\"}\n",
    "classes = [145512, 145517, 145524, 145534]\n",
    "genders = [\"m\", 'f']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1df06561",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Launch a browser and website \n",
    "driver = webdriver.Chrome()\n",
    "driver.get('https://www.athletic.net/TrackAndField/rankings/list/145512/m?depth=25')\n",
    "\n",
    "# Find the log in button by its class name\n",
    "buttons = WebDriverWait(driver, 10).until(\n",
    "    EC.presence_of_all_elements_located((By.CLASS_NAME, \"a-btn.athletic-header--btn.ng-star-inserted\")))\n",
    "\n",
    "buttons[1].click()\n",
    "\n",
    "# input email and password into fields\n",
    "password_input = WebDriverWait(driver, 10).until(\n",
    "    EC.visibility_of_element_located((By.CSS_SELECTOR, 'input[placeholder=\"Enter Password\"][formcontrolname=\"password\"]')))\n",
    "email_input = WebDriverWait(driver, 10).until(\n",
    "    EC.visibility_of_element_located((By.CSS_SELECTOR, 'input[placeholder=\"Enter Email\"][formcontrolname=\"email\"]')))\n",
    "\n",
    "email_input.send_keys(my_email)\n",
    "password_input.send_keys(my_password)\n",
    "\n",
    "# click button to submit log in \n",
    "login_button = WebDriverWait(driver, 10).until(\n",
    "    EC.element_to_be_clickable((By.CSS_SELECTOR, 'button[type=\"submit\"]')))\n",
    "\n",
    "login_button.click()\n",
    "\n",
    "# pause to allow log in to compete\n",
    "time.sleep(5)\n",
    "\n",
    "# get page content\n",
    "page_source = driver.page_source\n",
    "\n",
    "# Parse the page source with BeautifulSoup\n",
    "soup = BeautifulSoup(page_source, 'html.parser')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f9ab10e9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BoysA - length is 457\n",
      "BoysB - length is 467\n",
      "BoysC - length is 467\n",
      "BoysD - length is 475\n",
      "GirlsA - length is 467\n",
      "GirlsB - length is 476\n",
      "GirlsC - length is 483\n",
      "GirlsD - length is 470\n"
     ]
    }
   ],
   "source": [
    "headers = ['Event', 'rank', 'profile', 'Athlete', 'blank_1', 'Mark', 'blank_2', 'ST', 'Age', 'Grade', 'blank_3', 'Team', 'Date', 'Meet']\n",
    "\n",
    "g_class_dict = {}\n",
    "agg_df = pd.DataFrame(columns=[\"Event\", \"rank\", \"Age\", \"Grade\", \"Athlete\", \"Mark\", \"ST\", \"Team\", \"Date\", \"Meet\"])\n",
    "\n",
    "for g in genders:\n",
    "    for class_ in classes:\n",
    "        \n",
    "        url = f'https://www.athletic.net/TrackAndField/rankings/list/{class_}/{g}?depth=25'\n",
    "        driver.get(url)\n",
    "        \n",
    "        time.sleep(4)\n",
    "\n",
    "        # get page content\n",
    "        page_source = driver.page_source\n",
    "\n",
    "        # Parse the page source with BeautifulSoup\n",
    "        soup = BeautifulSoup(page_source, 'html.parser')\n",
    "\n",
    "\n",
    "        data = [headers]\n",
    "        rows = soup.find_all('tr')\n",
    "\n",
    "        for i, row in enumerate(rows):  # Skip the first row (headers)\n",
    "            row_list = []\n",
    "\n",
    "            cells = row.find_all(['td', 'th'])  # Extract data from each cell in the row\n",
    "\n",
    "            for header, cell in zip(headers, cells):\n",
    "                cell_contents = str(cell.get_text().strip().replace('*', ''))\n",
    "                row_list.append(cell_contents)\n",
    "\n",
    "            if 'Gr' not in row_list:\n",
    "                data.append(row_list)\n",
    "\n",
    "\n",
    "        df = pd.DataFrame(data)\n",
    "\n",
    "        df.columns = df.loc[0, :]\n",
    "        df = df.drop(0).reset_index(drop=True)\n",
    "        df.drop(columns=['blank_1', 'blank_2', 'blank_3', 'profile'], inplace=True)\n",
    "        df['rank'] = df['rank'].apply(lambda x: x.replace('.', ''))\n",
    "\n",
    "        df['Event'] = df['Event'].apply(lambda x: x[:-15].rstrip())\n",
    "        df = df[[\"Event\", \"rank\", \"Age\", \"Grade\", \"Athlete\", \"Mark\", \"ST\", \"Team\", \"Date\", \"Meet\"]]\n",
    "\n",
    "        class_g_label = genders_dict[g] + str(class_dict[class_])\n",
    "        print(f\"{class_g_label} - length is {len(df)}\")\n",
    "        \n",
    "        agg_df = pd.concat([agg_df, df], ignore_index=True)\n",
    "        \n",
    "        g_class_dict[class_g_label] = df\n",
    "\n",
    "        time.sleep(2)\n",
    "\n",
    "    \n",
    "# Close the WebDriver\n",
    "driver.quit()\n",
    "\n",
    "\n",
    "import datetime as dt\n",
    "curr_date = dt.date.today().strftime('%m_%d')\n",
    "agg_df.to_csv(f'{curr_date}_Track_and_Field_leaderboard.csv', header=False, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f4abfad1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken: 96.279s\n"
     ]
    }
   ],
   "source": [
    "end = time.time()\n",
    "print(f\"Time taken: {(end-start):.03f}s\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}