political_lean_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6b0be938-0cf8-4290-9b1d-0dd0dde53bbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly as pl\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6ce0b5f2-650f-4e64-9927-373c93948f1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>state</th>\n",
       "      <th>state_po</th>\n",
       "      <th>state_fips</th>\n",
       "      <th>state_cen</th>\n",
       "      <th>state_ic</th>\n",
       "      <th>office</th>\n",
       "      <th>district</th>\n",
       "      <th>stage</th>\n",
       "      <th>runoff</th>\n",
       "      <th>special</th>\n",
       "      <th>candidate</th>\n",
       "      <th>party</th>\n",
       "      <th>writein</th>\n",
       "      <th>mode</th>\n",
       "      <th>candidatevotes</th>\n",
       "      <th>totalvotes</th>\n",
       "      <th>unofficial</th>\n",
       "      <th>version</th>\n",
       "      <th>fusion_ticket</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11083</th>\n",
       "      <td>1994</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>AL</td>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>41</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>1</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>DON WOMACK</td>\n",
       "      <td>DEMOCRAT</td>\n",
       "      <td>False</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>50227</td>\n",
       "      <td>153767</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11084</th>\n",
       "      <td>1994</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>AL</td>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>41</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>1</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>SONNY CALLAHAN</td>\n",
       "      <td>REPUBLICAN</td>\n",
       "      <td>False</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>103431</td>\n",
       "      <td>153767</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11085</th>\n",
       "      <td>1994</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>AL</td>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>41</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>1</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>WRITEIN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>109</td>\n",
       "      <td>153767</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11086</th>\n",
       "      <td>1994</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>AL</td>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>41</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>2</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>BRIAN DOWLING</td>\n",
       "      <td>DEMOCRAT</td>\n",
       "      <td>False</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>44694</td>\n",
       "      <td>169213</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11087</th>\n",
       "      <td>1994</td>\n",
       "      <td>ALABAMA</td>\n",
       "      <td>AL</td>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>41</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>2</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>TERRY EVERETT</td>\n",
       "      <td>REPUBLICAN</td>\n",
       "      <td>False</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>124465</td>\n",
       "      <td>169213</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31098</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>WY</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>0</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>LYNNETTE GREY BULL</td>\n",
       "      <td>DEMOCRAT</td>\n",
       "      <td>False</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>66576</td>\n",
       "      <td>278503</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31099</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>WY</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>0</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>OVERVOTES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>1274</td>\n",
       "      <td>278503</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31100</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>WY</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>0</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>RICHARD BRUBAKER</td>\n",
       "      <td>LIBERTARIAN</td>\n",
       "      <td>False</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>10154</td>\n",
       "      <td>278503</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31101</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>WY</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>0</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>UNDERVOTES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>6337</td>\n",
       "      <td>278503</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31102</th>\n",
       "      <td>2020</td>\n",
       "      <td>WYOMING</td>\n",
       "      <td>WY</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>US HOUSE</td>\n",
       "      <td>0</td>\n",
       "      <td>GEN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>WRITEIN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>TOTAL</td>\n",
       "      <td>525</td>\n",
       "      <td>278503</td>\n",
       "      <td>False</td>\n",
       "      <td>20220331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20007 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       year    state state_po  state_fips  state_cen  state_ic    office  \\\n",
       "11083  1994  ALABAMA       AL           1         63        41  US HOUSE   \n",
       "11084  1994  ALABAMA       AL           1         63        41  US HOUSE   \n",
       "11085  1994  ALABAMA       AL           1         63        41  US HOUSE   \n",
       "11086  1994  ALABAMA       AL           1         63        41  US HOUSE   \n",
       "11087  1994  ALABAMA       AL           1         63        41  US HOUSE   \n",
       "...     ...      ...      ...         ...        ...       ...       ...   \n",
       "31098  2020  WYOMING       WY          56         83        68  US HOUSE   \n",
       "31099  2020  WYOMING       WY          56         83        68  US HOUSE   \n",
       "31100  2020  WYOMING       WY          56         83        68  US HOUSE   \n",
       "31101  2020  WYOMING       WY          56         83        68  US HOUSE   \n",
       "31102  2020  WYOMING       WY          56         83        68  US HOUSE   \n",
       "\n",
       "       district stage runoff  special           candidate        party  \\\n",
       "11083         1   GEN  False    False          DON WOMACK     DEMOCRAT   \n",
       "11084         1   GEN  False    False      SONNY CALLAHAN   REPUBLICAN   \n",
       "11085         1   GEN  False    False             WRITEIN          NaN   \n",
       "11086         2   GEN  False    False       BRIAN DOWLING     DEMOCRAT   \n",
       "11087         2   GEN  False    False       TERRY EVERETT   REPUBLICAN   \n",
       "...         ...   ...    ...      ...                 ...          ...   \n",
       "31098         0   GEN  False    False  LYNNETTE GREY BULL     DEMOCRAT   \n",
       "31099         0   GEN  False    False           OVERVOTES          NaN   \n",
       "31100         0   GEN  False    False    RICHARD BRUBAKER  LIBERTARIAN   \n",
       "31101         0   GEN  False    False          UNDERVOTES          NaN   \n",
       "31102         0   GEN  False    False             WRITEIN          NaN   \n",
       "\n",
       "       writein   mode  candidatevotes  totalvotes  unofficial   version  \\\n",
       "11083    False  TOTAL           50227      153767       False  20220331   \n",
       "11084    False  TOTAL          103431      153767       False  20220331   \n",
       "11085     True  TOTAL             109      153767       False  20220331   \n",
       "11086    False  TOTAL           44694      169213       False  20220331   \n",
       "11087    False  TOTAL          124465      169213       False  20220331   \n",
       "...        ...    ...             ...         ...         ...       ...   \n",
       "31098    False  TOTAL           66576      278503       False  20220331   \n",
       "31099    False  TOTAL            1274      278503       False  20220331   \n",
       "31100    False  TOTAL           10154      278503       False  20220331   \n",
       "31101    False  TOTAL            6337      278503       False  20220331   \n",
       "31102     True  TOTAL             525      278503       False  20220331   \n",
       "\n",
       "       fusion_ticket  \n",
       "11083          False  \n",
       "11084          False  \n",
       "11085          False  \n",
       "11086          False  \n",
       "11087          False  \n",
       "...              ...  \n",
       "31098          False  \n",
       "31099          False  \n",
       "31100          False  \n",
       "31101          False  \n",
       "31102          False  \n",
       "\n",
       "[20007 rows x 20 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poli_df = pd.read_csv(\"1976-2020-house.csv\")\n",
    "poli_df = poli_df[poli_df['year'] >= 1993]\n",
    "poli_df = poli_df[~poli_df['state_po'].isin(['DC'])]\n",
    "poli_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2defa3fd-ff14-44e9-b9ce-d90fb5618e32",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>state_po</th>\n",
       "      <th>year</th>\n",
       "      <th>party</th>\n",
       "      <th>state_fips</th>\n",
       "      <th>state_cen</th>\n",
       "      <th>state_ic</th>\n",
       "      <th>district</th>\n",
       "      <th>special</th>\n",
       "      <th>writein</th>\n",
       "      <th>candidatevotes</th>\n",
       "      <th>totalvotes</th>\n",
       "      <th>unofficial</th>\n",
       "      <th>version</th>\n",
       "      <th>fusion_ticket</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AK</td>\n",
       "      <td>1994</td>\n",
       "      <td>DEMOCRAT</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>81</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>68172</td>\n",
       "      <td>208240</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AK</td>\n",
       "      <td>1994</td>\n",
       "      <td>GREEN</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>81</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21277</td>\n",
       "      <td>208240</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AK</td>\n",
       "      <td>1994</td>\n",
       "      <td>REPUBLICAN</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>81</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>118537</td>\n",
       "      <td>208240</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AK</td>\n",
       "      <td>1996</td>\n",
       "      <td>ALASKAN INDEPENDENCE</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>81</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5017</td>\n",
       "      <td>233700</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AK</td>\n",
       "      <td>1996</td>\n",
       "      <td>DEMOCRAT</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>81</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>85114</td>\n",
       "      <td>233700</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3017</th>\n",
       "      <td>WY</td>\n",
       "      <td>2018</td>\n",
       "      <td>REPUBLICAN</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>127963</td>\n",
       "      <td>201245</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3018</th>\n",
       "      <td>WY</td>\n",
       "      <td>2020</td>\n",
       "      <td>CONSTITUTION</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7905</td>\n",
       "      <td>278503</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3019</th>\n",
       "      <td>WY</td>\n",
       "      <td>2020</td>\n",
       "      <td>DEMOCRAT</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>66576</td>\n",
       "      <td>278503</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3020</th>\n",
       "      <td>WY</td>\n",
       "      <td>2020</td>\n",
       "      <td>LIBERTARIAN</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10154</td>\n",
       "      <td>278503</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3021</th>\n",
       "      <td>WY</td>\n",
       "      <td>2020</td>\n",
       "      <td>REPUBLICAN</td>\n",
       "      <td>56</td>\n",
       "      <td>83</td>\n",
       "      <td>68</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>185732</td>\n",
       "      <td>278503</td>\n",
       "      <td>0</td>\n",
       "      <td>20220331</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3022 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     state_po  year                 party  state_fips  state_cen  state_ic  \\\n",
       "0          AK  1994              DEMOCRAT           2         94        81   \n",
       "1          AK  1994                 GREEN           2         94        81   \n",
       "2          AK  1994            REPUBLICAN           2         94        81   \n",
       "3          AK  1996  ALASKAN INDEPENDENCE           2         94        81   \n",
       "4          AK  1996              DEMOCRAT           2         94        81   \n",
       "...       ...   ...                   ...         ...        ...       ...   \n",
       "3017       WY  2018            REPUBLICAN          56         83        68   \n",
       "3018       WY  2020          CONSTITUTION          56         83        68   \n",
       "3019       WY  2020              DEMOCRAT          56         83        68   \n",
       "3020       WY  2020           LIBERTARIAN          56         83        68   \n",
       "3021       WY  2020            REPUBLICAN          56         83        68   \n",
       "\n",
       "      district  special  writein  candidatevotes  totalvotes  unofficial  \\\n",
       "0            0        0        0           68172      208240           0   \n",
       "1            0        0        0           21277      208240           0   \n",
       "2            0        0        0          118537      208240           0   \n",
       "3            0        0        0            5017      233700           0   \n",
       "4            0        0        0           85114      233700           0   \n",
       "...        ...      ...      ...             ...         ...         ...   \n",
       "3017         0        0        0          127963      201245           0   \n",
       "3018         0        0        0            7905      278503           0   \n",
       "3019         0        0        0           66576      278503           0   \n",
       "3020         0        0        0           10154      278503           0   \n",
       "3021         0        0        0          185732      278503           0   \n",
       "\n",
       "       version  fusion_ticket  \n",
       "0     20220331              0  \n",
       "1     20220331              0  \n",
       "2     20220331              0  \n",
       "3     20220331              0  \n",
       "4     20220331              0  \n",
       "...        ...            ...  \n",
       "3017  20220331              0  \n",
       "3018  20220331              0  \n",
       "3019  20220331              0  \n",
       "3020  20220331              0  \n",
       "3021  20220331              0  \n",
       "\n",
       "[3022 rows x 14 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tot_party_votes = poli_df.groupby(['state_po','year','party']).sum().reset_index()\n",
    "tot_party_votes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9327a154-db00-4a8a-ae2e-d33ac7df8b73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>candidatevotes</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_po</th>\n",
       "      <th>year</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">AK</th>\n",
       "      <th>1994</th>\n",
       "      <td>118537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>138834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998</th>\n",
       "      <td>139676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000</th>\n",
       "      <td>190862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2002</th>\n",
       "      <td>169685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">WY</th>\n",
       "      <th>2012</th>\n",
       "      <td>166452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014</th>\n",
       "      <td>113038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016</th>\n",
       "      <td>156176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018</th>\n",
       "      <td>127963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020</th>\n",
       "      <td>185732</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>698 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               candidatevotes\n",
       "state_po year                \n",
       "AK       1994          118537\n",
       "         1996          138834\n",
       "         1998          139676\n",
       "         2000          190862\n",
       "         2002          169685\n",
       "...                       ...\n",
       "WY       2012          166452\n",
       "         2014          113038\n",
       "         2016          156176\n",
       "         2018          127963\n",
       "         2020          185732\n",
       "\n",
       "[698 rows x 1 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "votes_by_party = poli_df.groupby(['state_po','year','party'])['candidatevotes'].sum().reset_index()\n",
    "votes_by_party = votes_by_party[votes_by_party['party'] == 'REPUBLICAN']\n",
    "#total_votes = poli_df.groupby(['state_po','year','party'])['totalvotes'].first().reset_index()\n",
    "#votes_by_party['totalvotes'] = total_votes[total_votes['party'] == 'DEMOCRAT']['totalvotes']\n",
    "#votes_by_party['prop_dem'] = votes_by_party['candidatevotes']/votes_by_party['totalvotes']\n",
    "votes_by_party = votes_by_party.set_index(['state_po','year']).drop(columns=['party'])\n",
    "votes_by_party"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "09389109-5ad2-4656-a911-81c66e5dbe1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>totalvotes</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_po</th>\n",
       "      <th>year</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">AK</th>\n",
       "      <th>1994</th>\n",
       "      <td>208240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>233700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998</th>\n",
       "      <td>223300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000</th>\n",
       "      <td>274393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2002</th>\n",
       "      <td>227725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">WY</th>\n",
       "      <th>2012</th>\n",
       "      <td>250700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014</th>\n",
       "      <td>171153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016</th>\n",
       "      <td>258788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018</th>\n",
       "      <td>201245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020</th>\n",
       "      <td>278503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>700 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               totalvotes\n",
       "state_po year            \n",
       "AK       1994      208240\n",
       "         1996      233700\n",
       "         1998      223300\n",
       "         2000      274393\n",
       "         2002      227725\n",
       "...                   ...\n",
       "WY       2012      250700\n",
       "         2014      171153\n",
       "         2016      258788\n",
       "         2018      201245\n",
       "         2020      278503\n",
       "\n",
       "[700 rows x 1 columns]"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_votes = poli_df.groupby(['state_po','year','district'])['totalvotes'].first().reset_index()\n",
    "total_votes = total_votes.groupby(['state_po','year'])['totalvotes'].sum().reset_index()\n",
    "total_votes = total_votes.set_index(['state_po','year'])\n",
    "total_votes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "8753d2ad-9ded-4678-9f52-6087f13fecea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('VT', 2008)\n",
      "('VT', 2016)\n"
     ]
    }
   ],
   "source": [
    "for val in total_votes.index:\n",
    "    if val not in votes_by_party.index:\n",
    "        print(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "cb397d10-335b-4849-81d9-ae6dabdb8498",
   "metadata": {},
   "outputs": [],
   "source": [
    "poli_df.loc[(poli_df['state_po'] == 'MN') & (poli_df['year'].isin([1992])) & (poli_df['party'].isin([\"INDEPENDENT-REPUBLICAN\"])),'party'] = \"REPUBLICAN\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "d5e85286-a24a-4a8b-a03e-6b17f295b8c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['INDEPENDENT', 'LIBERTY UNION', 'ENERGY INDEPENDENCE', 'DEMOCRAT',\n",
       "       'PROGRESSIVE', nan], dtype=object)"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poli_df.loc[(poli_df['state_po'] == 'HI') & (poli_df['year'].isin([1982])),:]['party'].unique()\n",
    "poli_df.loc[(poli_df['state_po'] == 'DC') & (poli_df['year'].isin([2020])),:]['party'].unique()\n",
    "poli_df.loc[(poli_df['state_po'] == 'VT') & (poli_df['year'].isin([2008,2016])),:]['party'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "774c877d-fca4-4637-90b7-6d2cd18c6b44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>totalvotes</th>\n",
       "      <th>candidatevotes</th>\n",
       "      <th>prop_rep_votes</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_po</th>\n",
       "      <th>year</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">AK</th>\n",
       "      <th>1994</th>\n",
       "      <td>208240</td>\n",
       "      <td>118537.0</td>\n",
       "      <td>0.569233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>233700</td>\n",
       "      <td>138834.0</td>\n",
       "      <td>0.594069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998</th>\n",
       "      <td>223300</td>\n",
       "      <td>139676.0</td>\n",
       "      <td>0.625508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000</th>\n",
       "      <td>274393</td>\n",
       "      <td>190862.0</td>\n",
       "      <td>0.695579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2002</th>\n",
       "      <td>227725</td>\n",
       "      <td>169685.0</td>\n",
       "      <td>0.745131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">WY</th>\n",
       "      <th>2012</th>\n",
       "      <td>250700</td>\n",
       "      <td>166452.0</td>\n",
       "      <td>0.663949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014</th>\n",
       "      <td>171153</td>\n",
       "      <td>113038.0</td>\n",
       "      <td>0.660450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016</th>\n",
       "      <td>258788</td>\n",
       "      <td>156176.0</td>\n",
       "      <td>0.603490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018</th>\n",
       "      <td>201245</td>\n",
       "      <td>127963.0</td>\n",
       "      <td>0.635857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020</th>\n",
       "      <td>278503</td>\n",
       "      <td>185732.0</td>\n",
       "      <td>0.666894</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>700 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               totalvotes  candidatevotes  prop_rep_votes\n",
       "state_po year                                            \n",
       "AK       1994      208240        118537.0        0.569233\n",
       "         1996      233700        138834.0        0.594069\n",
       "         1998      223300        139676.0        0.625508\n",
       "         2000      274393        190862.0        0.695579\n",
       "         2002      227725        169685.0        0.745131\n",
       "...                   ...             ...             ...\n",
       "WY       2012      250700        166452.0        0.663949\n",
       "         2014      171153        113038.0        0.660450\n",
       "         2016      258788        156176.0        0.603490\n",
       "         2018      201245        127963.0        0.635857\n",
       "         2020      278503        185732.0        0.666894\n",
       "\n",
       "[700 rows x 3 columns]"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df = total_votes.merge(votes_by_party, left_index=True, right_index=True, how=\"left\")\n",
    "merged_df['candidatevotes'] = merged_df['candidatevotes'].fillna(0)\n",
    "merged_df['prop_rep_votes'] = merged_df['candidatevotes']/merged_df['totalvotes']\n",
    "merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "04da33e5-acf6-4e67-820b-290d0b78f0d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>state_po</th>\n",
       "      <th>year</th>\n",
       "      <th>totalvotes</th>\n",
       "      <th>candidatevotes</th>\n",
       "      <th>prop_rep_votes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AK</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>208240.0</td>\n",
       "      <td>118537.0</td>\n",
       "      <td>0.569233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AK</td>\n",
       "      <td>1996.0</td>\n",
       "      <td>233700.0</td>\n",
       "      <td>138834.0</td>\n",
       "      <td>0.594069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AK</td>\n",
       "      <td>1998.0</td>\n",
       "      <td>223300.0</td>\n",
       "      <td>139676.0</td>\n",
       "      <td>0.625508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AK</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>274393.0</td>\n",
       "      <td>190862.0</td>\n",
       "      <td>0.695579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AK</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>227725.0</td>\n",
       "      <td>169685.0</td>\n",
       "      <td>0.745131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1345</th>\n",
       "      <td>WY</td>\n",
       "      <td>2011.0</td>\n",
       "      <td>190822.0</td>\n",
       "      <td>131661.0</td>\n",
       "      <td>0.689968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346</th>\n",
       "      <td>WY</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>250700.0</td>\n",
       "      <td>166452.0</td>\n",
       "      <td>0.663949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1347</th>\n",
       "      <td>WY</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>171153.0</td>\n",
       "      <td>113038.0</td>\n",
       "      <td>0.660450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1348</th>\n",
       "      <td>WY</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>258788.0</td>\n",
       "      <td>156176.0</td>\n",
       "      <td>0.603490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1349</th>\n",
       "      <td>WY</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>201245.0</td>\n",
       "      <td>127963.0</td>\n",
       "      <td>0.635857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1350 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     state_po    year  totalvotes  candidatevotes  prop_rep_votes\n",
       "0          AK  1994.0    208240.0        118537.0        0.569233\n",
       "1          AK  1996.0    233700.0        138834.0        0.594069\n",
       "2          AK  1998.0    223300.0        139676.0        0.625508\n",
       "3          AK  2000.0    274393.0        190862.0        0.695579\n",
       "4          AK  2002.0    227725.0        169685.0        0.745131\n",
       "...       ...     ...         ...             ...             ...\n",
       "1345       WY  2011.0    190822.0        131661.0        0.689968\n",
       "1346       WY  2013.0    250700.0        166452.0        0.663949\n",
       "1347       WY  2015.0    171153.0        113038.0        0.660450\n",
       "1348       WY  2017.0    258788.0        156176.0        0.603490\n",
       "1349       WY  2019.0    201245.0        127963.0        0.635857\n",
       "\n",
       "[1350 rows x 5 columns]"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_index_merged_df = merged_df.reset_index()\n",
    "for state in no_index_merged_df['state_po'].unique():\n",
    "    for year in np.arange(1995,2020,2):\n",
    "#         print(state,year)\n",
    "#         print(no_index_merged_df.loc[\n",
    "#             (no_index_merged_df['state_po'] == state) & (no_index_merged_df['year'] == year-1),:].iloc[0])\n",
    "#         print(no_index_merged_df.loc[0,:])\n",
    "        try:\n",
    "            no_index_merged_df.loc[no_index_merged_df.index.max()+1,:] = no_index_merged_df.loc[\n",
    "                (no_index_merged_df['state_po'] == state) & (no_index_merged_df['year'] == year-1),:].iloc[0]\n",
    "            #print(no_index_merged_df.loc[no_index_merged_df.index.max(),:]['year'])\n",
    "            no_index_merged_df.loc[no_index_merged_df.index.max(),'year']=year\n",
    "        except:\n",
    "            #print('oops')\n",
    "            print(state,year)\n",
    "no_index_merged_df        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a2746b2-6a7f-4b94-b0ce-888fe5c5b96d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}