CDC_Opioid_Prescription_webscrape.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Project 3\n",
    "## Scrape data from CDC Opioid Prescription website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import pandas\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[CDC Opioid Prescription Site](https://www.cdc.gov/drugoverdose/rxrate-maps/index.html). Inspect the page to identify which elements to scrape."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.cdc.gov/drugoverdose/rxrate-maps/index.html\n",
    "basestateUrl = 'https://www.cdc.gov/drugoverdose/rxrate-maps/state'\n",
    "basecountyUrl = 'https://www.cdc.gov/drugoverdose/rxrate-maps/county'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  all years is all years available on the site,  years is the range of data to pull for the website\n",
    "# allyears = [2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019]\n",
    "years = [2013, 2014, 2015, 2016, 2017, 2018, 2019]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scrape state level table info and write to a csv\n",
    "for year in years:\n",
    "    url = f'{basestateUrl}{year}.html'\n",
    "    # print(url)\n",
    "    tablelist = pd.read_html(url)\n",
    "    tabledf = tablelist[0]\n",
    "    tabledf['Year'] = year\n",
    "    # tabledf[f'{year} ODR per 100'] = tabledf['Opioid Dispensing Rate per 100']\n",
    "    # print(tabledf.head(5))\n",
    "    # print(len(tabledf))\n",
    "    tabledf.to_csv(f'Resources/DispenseRate/DRState{year}.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scrape county level table info and write to a csv - not using county level data keep just in case\n",
    "# for year in years:\n",
    "#     url = f'{basecountyUrl}{year}.html'\n",
    "#     # print(url)\n",
    "#     tablelist = pd.read_html(url)\n",
    "#     tabledf = tablelist[0]\n",
    "#     tabledf['Year'] = year\n",
    "#     # print(tabledf.head(5))\n",
    "#     # print(len(tabledf))\n",
    "#     tabledf.to_csv(f'Resources/DispenseRate/County/DRCounty{year}.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DRState2013.csv',\n",
       " 'DRState2014.csv',\n",
       " 'DRState2015.csv',\n",
       " 'DRState2016.csv',\n",
       " 'DRState2017.csv',\n",
       " 'DRState2018.csv',\n",
       " 'DRState2019.csv']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get list of csvs in State directory\n",
    "# listCSVs = os.listdir('Resources/DispenseRate/State')\n",
    "# listCSVs\n",
    "allFiles = os.listdir(\"Resources/DispenseRate\")    \n",
    "csvFiles = list(filter(lambda f: f.endswith('.csv'), allFiles))\n",
    "csvFiles\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create list of dataframes\n",
    "dfs = []\n",
    "for csvFile in csvFiles:\n",
    "    dfs.append(pd.read_csv(f'Resources/DispenseRate/{csvFile}'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     State State Abbreviation  Opioid Dispensing Rate per 100  Year\n",
      "0  Alabama                 AL                           121.0  2016\n",
      "1   Alaska                 AK                            58.9  2016\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>Abbreviation</th>\n",
       "      <th>Opioid Dispensing Rate per 100</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>59.0</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>AK</td>\n",
       "      <td>52.0</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           State Abbreviation  Opioid Dispensing Rate per 100  Year\n",
       "0  United States           US                            59.0  2017\n",
       "1         Alaska           AK                            52.0  2017"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  check some dfs in list\n",
    "print(dfs[3].head(2))\n",
    "dfs[4].head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new list of dataframes with standardized column names (changes between 2016-2017)\n",
    "dffy = []\n",
    "for dff in dfs:\n",
    "   dfff = dff.rename(columns = {'State Abbreviation': 'Abbreviation'})\n",
    "   dffy.append(dfff)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     State Abbreviation  Opioid Dispensing Rate per 100  Year\n",
      "0  Alabama           AL                           121.0  2016\n",
      "1   Alaska           AK                            58.9  2016\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>Abbreviation</th>\n",
       "      <th>Opioid Dispensing Rate per 100</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>59.0</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>AK</td>\n",
       "      <td>52.0</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           State Abbreviation  Opioid Dispensing Rate per 100  Year\n",
       "0  United States           US                            59.0  2017\n",
       "1         Alaska           AK                            52.0  2017"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  check some dfs in list\n",
    "print(dffy[3].head(2))\n",
    "dffy[4].head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check to see that info was written into dffy list\n",
    "len(dffy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dlite list will be used to create ODisp2013-2013.csv \n",
    "dlite = []\n",
    "for d in dfs:\n",
    "    year = d['Year'].min()\n",
    "    dd = d.rename(columns = {'Opioid Dispensing Rate per 100':f'{year} ODR per 100'}) \n",
    "    dds = dd.rename(columns = {'State Abbreviation': 'Abbreviation'})\n",
    "    dds2 = dds.drop(columns = ['Year'])\n",
    "    # print(dds2)\n",
    "    dlite.append(dds2)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check to see that info was written into dlite list\n",
    "len(dlite)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     State State Abbreviation  Opioid Dispensing Rate per 100  Year\n",
      "0  Alabama                 AL                           121.0  2016\n",
      "1   Alaska                 AK                            58.9  2016\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>Abbreviation</th>\n",
       "      <th>Opioid Dispensing Rate per 100</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>59.0</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>AK</td>\n",
       "      <td>52.0</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           State Abbreviation  Opioid Dispensing Rate per 100  Year\n",
       "0  United States           US                            59.0  2017\n",
       "1         Alaska           AK                            52.0  2017"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  check some dfs in list\n",
    "print(dfs[3].head(2))\n",
    "dfs[4].head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Join pairs of dfs  \n",
    "wideFramedf0 = pd.merge(dlite[0],dlite[1], on=['State','Abbreviation'], how = 'outer')\n",
    "wideFramedf1 = pd.merge(dlite[2],dlite[3], on=['State','Abbreviation'], how = 'outer')\n",
    "wideFramedf2 = pd.merge(dlite[4],dlite[5], on=['State','Abbreviation'], how = 'outer')\n",
    "# wideFramedf3 = pd.merge(dlite[6],dlite[7], on=['State','Abbreviation'], how = 'outer')\n",
    "# wideFramedf4 = pd.merge(dlite[8],dlite[9], on=['State','Abbreviation'], how = 'outer')\n",
    "# wideFramedf5 = pd.merge(dlite[10],dlite[11], on=['State','Abbreviation'], how = 'outer')\n",
    "# wideFramedf6 = pd.merge(dlite[12],dlite[13], on=['State','Abbreviation'], how = 'outer')\n",
    "# join originally written for years 2006-2019\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join quads of dfs\n",
    "wf0 = pd.merge(wideFramedf0,wideFramedf1, on=['State','Abbreviation'], how = 'outer')\n",
    "wf1 = pd.merge(wideFramedf2,dlite[6], on=['State','Abbreviation'], how = 'outer')\n",
    "# wf2 = pd.merge(wideFramedf4,wideFramedf5, on=['State','Abbreviation'], how = 'outer')\n",
    "# join originally written for years 2006-2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extra join step needed for joining years 2006-2019\n",
    "# wff0 = pd.merge(wf0,wf1, on=['State','Abbreviation'], how = 'outer')\n",
    "# # wff1 = pd.merge(wf2, wideFramedf6, on=['State','Abbreviation'], how = 'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>Abbreviation</th>\n",
       "      <th>2013 ODR per 100</th>\n",
       "      <th>2014 ODR per 100</th>\n",
       "      <th>2015 ODR per 100</th>\n",
       "      <th>2016 ODR per 100</th>\n",
       "      <th>2017 ODR per 100</th>\n",
       "      <th>2018 ODR per 100</th>\n",
       "      <th>2019 ODR per 100</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alabama</td>\n",
       "      <td>AL</td>\n",
       "      <td>142.4</td>\n",
       "      <td>135.2</td>\n",
       "      <td>125.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>108.8</td>\n",
       "      <td>97.5</td>\n",
       "      <td>85.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>AK</td>\n",
       "      <td>63.7</td>\n",
       "      <td>62.7</td>\n",
       "      <td>60.8</td>\n",
       "      <td>58.9</td>\n",
       "      <td>52.0</td>\n",
       "      <td>44.9</td>\n",
       "      <td>39.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     State Abbreviation  2013 ODR per 100  2014 ODR per 100  2015 ODR per 100  \\\n",
       "0  Alabama           AL             142.4             135.2             125.0   \n",
       "1   Alaska           AK              63.7              62.7              60.8   \n",
       "\n",
       "   2016 ODR per 100  2017 ODR per 100  2018 ODR per 100  2019 ODR per 100  \n",
       "0             121.0             108.8              97.5              85.8  \n",
       "1              58.9              52.0              44.9              39.1  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# final join of all dfs\n",
    "wideframe = pd.merge(wf0, wf1, on=['State','Abbreviation'], how = 'outer')\n",
    "print(len(wideframe)) # all 50 states, District of Columbia and overall US\n",
    "wideframe.to_csv('Resources/ODisp2013_2019.csv', index = False)\n",
    "wideframe.head(2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# update table to prep for transposing in next step\n",
    "wf = wideframe.drop(columns = ['State'])\n",
    "wff = wf.rename(columns = {'Abbreviation':'Opioid_Dispense_Rates_per_Year','2013 ODR per 100':'2013','2014 ODR per 100':'2014','2015 ODR per 100':'2015','2016 ODR per 100':'2016','2017 ODR per 100':'2017','2018 ODR per 100':'2018','2019 ODR per 100':'2019'})\n",
    "\n",
    "wfff = wff.set_index('Opioid_Dispense_Rates_per_Year')# to prepare for next step - transpose table\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Opioid_Dispense_Rates_per_Year</th>\n",
       "      <th>Year</th>\n",
       "      <th>AL</th>\n",
       "      <th>AK</th>\n",
       "      <th>AZ</th>\n",
       "      <th>AR</th>\n",
       "      <th>CA</th>\n",
       "      <th>CO</th>\n",
       "      <th>CT</th>\n",
       "      <th>DE</th>\n",
       "      <th>DC</th>\n",
       "      <th>...</th>\n",
       "      <th>TN</th>\n",
       "      <th>TX</th>\n",
       "      <th>UT</th>\n",
       "      <th>VT</th>\n",
       "      <th>VA</th>\n",
       "      <th>WA</th>\n",
       "      <th>WV</th>\n",
       "      <th>WI</th>\n",
       "      <th>WY</th>\n",
       "      <th>US</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013</td>\n",
       "      <td>142.4</td>\n",
       "      <td>63.7</td>\n",
       "      <td>80.4</td>\n",
       "      <td>120.9</td>\n",
       "      <td>54.4</td>\n",
       "      <td>71.2</td>\n",
       "      <td>67.4</td>\n",
       "      <td>92.7</td>\n",
       "      <td>41.1</td>\n",
       "      <td>...</td>\n",
       "      <td>127.1</td>\n",
       "      <td>70.0</td>\n",
       "      <td>82.1</td>\n",
       "      <td>52.2</td>\n",
       "      <td>76.6</td>\n",
       "      <td>75.2</td>\n",
       "      <td>129.0</td>\n",
       "      <td>73.8</td>\n",
       "      <td>81.5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2014</td>\n",
       "      <td>135.2</td>\n",
       "      <td>62.7</td>\n",
       "      <td>79.7</td>\n",
       "      <td>123.2</td>\n",
       "      <td>52.7</td>\n",
       "      <td>69.6</td>\n",
       "      <td>66.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>40.1</td>\n",
       "      <td>...</td>\n",
       "      <td>121.3</td>\n",
       "      <td>67.0</td>\n",
       "      <td>78.8</td>\n",
       "      <td>50.4</td>\n",
       "      <td>73.5</td>\n",
       "      <td>74.2</td>\n",
       "      <td>126.4</td>\n",
       "      <td>71.9</td>\n",
       "      <td>80.9</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Opioid_Dispense_Rates_per_Year  Year     AL    AK    AZ     AR    CA    CO  \\\n",
       "0                               2013  142.4  63.7  80.4  120.9  54.4  71.2   \n",
       "1                               2014  135.2  62.7  79.7  123.2  52.7  69.6   \n",
       "\n",
       "Opioid_Dispense_Rates_per_Year    CT    DE    DC  ...     TN    TX    UT  \\\n",
       "0                               67.4  92.7  41.1  ...  127.1  70.0  82.1   \n",
       "1                               66.0  91.0  40.1  ...  121.3  67.0  78.8   \n",
       "\n",
       "Opioid_Dispense_Rates_per_Year    VT    VA    WA     WV    WI    WY  US  \n",
       "0                               52.2  76.6  75.2  129.0  73.8  81.5 NaN  \n",
       "1                               50.4  73.5  74.2  126.4  71.9  80.9 NaN  \n",
       "\n",
       "[2 rows x 53 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transpose table into format for barchart use and save to csv\n",
    "wideframetrans = pd.DataFrame(wfff).transpose().reset_index()\n",
    "# wideframetrans\n",
    "# wft = wideframetrans.reset_index()\n",
    "wft= wideframetrans.rename(columns = {'index': 'Year'})\n",
    "wft\n",
    "wft.to_csv('Resources/ODispTransposed2013_2019.csv', index = False)\n",
    "wft.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create json file from csv - json will be used for interactive barcharts on the website\n",
    "df = pd.read_csv (r'Resources\\ODispTransposed2013_2019.csv')\n",
    "df.to_json (r'BarCharts\\ODispTransposed2013_2019.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create general use csv for potential use in other visualizations\n",
    "bigFrame = pd.concat(dffy, ignore_index = True)\n",
    "bigFrame.to_csv('Resources/ODisp2013_2019bigframe.csv', index = False)\n",
    "bigFrame.head(2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get column names\n",
    "bigFrame.columns"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7 (default, May  6 2020, 11:45:54) [MSC v.1916 64 bit (AMD64)]"
  },
  "vscode": {
   "interpreter": {
    "hash": "e09424b516901a071045f3ed6bcd6beb164e9564e937a7754100a941fb2722f2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}