KNN_SVM_DETAILED_GROUPS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np                   # Numerical computation library\n",
    "import pandas as pa                  # Dataframe and Data manipulation Library\n",
    "import matplotlib.pyplot as plt      # Basic plotting functionality Library\n",
    "import seaborn as sn                 # Advanced Data visualization Library\n",
    "import numpy.random as rn\n",
    "from sklearn.linear_model import LinearRegression\n",
    "import matplotlib.colors as pltco # A package to help with coloring plots\n",
    "\n",
    "\n",
    "[x for x in dir(list) if not '__' in x]\n",
    "  \n",
    "# Curreny Exchange Rates compared to the US dollar from around the world 22 for 2008-2018\n",
    "gdp_per_industry = pa.read_csv('https://www.dropbox.com/s/emd7111t9g0rtd2/career_projections_group.csv?dl=1')\n",
    "workforce_demographic2018 = pa.read_csv('https://www.dropbox.com/s/w1zzumsadlgg9pn/Colorado_Current_Workforce_Demographic.csv?dl=1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdp_per_industry = gdp_per_industry.drop(0)\n",
    "workforce_demographic2018 = workforce_demographic2018.drop(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "workforce_demographic2018['OCC_GROUP'] = workforce_demographic2018['OCC_GROUP'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "detailed_groups_df = workforce_demographic2018.loc[workforce_demographic2018.OCC_GROUP.apply(lambda cat: 'detailed' in cat)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "detailed_groups_raw_data = detailed_groups_df.drop(['AREA','STATE','ST','OCC_GROUP','OCC_TITLE','ANNUAL','HOURLY'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OCC_CODE</th>\n",
       "      <th>TOT_EMP</th>\n",
       "      <th>EMP_PRSE</th>\n",
       "      <th>JOBS_1000</th>\n",
       "      <th>LOC_Q</th>\n",
       "      <th>H_MEAN</th>\n",
       "      <th>A_MEAN</th>\n",
       "      <th>MEAN_PRSE</th>\n",
       "      <th>H_PCT10</th>\n",
       "      <th>H_PCT25</th>\n",
       "      <th>H_MEDIAN</th>\n",
       "      <th>H_PCT75</th>\n",
       "      <th>H_PCT90</th>\n",
       "      <th>A_PCT10</th>\n",
       "      <th>A_PCT25</th>\n",
       "      <th>A_MEDIAN</th>\n",
       "      <th>A_PCT75</th>\n",
       "      <th>A_PCT90</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11-1011</td>\n",
       "      <td>570</td>\n",
       "      <td>5.3</td>\n",
       "      <td>0.216</td>\n",
       "      <td>0.16</td>\n",
       "      <td>95.25</td>\n",
       "      <td>198,130</td>\n",
       "      <td>5.2</td>\n",
       "      <td>40.92</td>\n",
       "      <td>56.83</td>\n",
       "      <td>86.63</td>\n",
       "      <td>#</td>\n",
       "      <td>#</td>\n",
       "      <td>85,110</td>\n",
       "      <td>118,210</td>\n",
       "      <td>180,190</td>\n",
       "      <td>#</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11-1021</td>\n",
       "      <td>39,200</td>\n",
       "      <td>1.9</td>\n",
       "      <td>14.958</td>\n",
       "      <td>0.95</td>\n",
       "      <td>63.42</td>\n",
       "      <td>131,920</td>\n",
       "      <td>1.1</td>\n",
       "      <td>22.27</td>\n",
       "      <td>32.52</td>\n",
       "      <td>51.51</td>\n",
       "      <td>81.71</td>\n",
       "      <td>#</td>\n",
       "      <td>46,310</td>\n",
       "      <td>67,650</td>\n",
       "      <td>107,140</td>\n",
       "      <td>169,970</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11-1031</td>\n",
       "      <td>630</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.239</td>\n",
       "      <td>0.69</td>\n",
       "      <td>*</td>\n",
       "      <td>57,570</td>\n",
       "      <td>4.5</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>*</td>\n",
       "      <td>21,230</td>\n",
       "      <td>23,970</td>\n",
       "      <td>48,260</td>\n",
       "      <td>68,270</td>\n",
       "      <td>100,430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>11-2011</td>\n",
       "      <td>390</td>\n",
       "      <td>25.9</td>\n",
       "      <td>0.147</td>\n",
       "      <td>0.84</td>\n",
       "      <td>59.12</td>\n",
       "      <td>122,970</td>\n",
       "      <td>7.1</td>\n",
       "      <td>27.10</td>\n",
       "      <td>45.76</td>\n",
       "      <td>58.58</td>\n",
       "      <td>68.07</td>\n",
       "      <td>87.05</td>\n",
       "      <td>56,380</td>\n",
       "      <td>95,180</td>\n",
       "      <td>121,850</td>\n",
       "      <td>141,580</td>\n",
       "      <td>181,070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11-2021</td>\n",
       "      <td>3,840</td>\n",
       "      <td>6.2</td>\n",
       "      <td>1.465</td>\n",
       "      <td>0.88</td>\n",
       "      <td>81.55</td>\n",
       "      <td>169,630</td>\n",
       "      <td>2.3</td>\n",
       "      <td>44.42</td>\n",
       "      <td>57.76</td>\n",
       "      <td>77.82</td>\n",
       "      <td>96.01</td>\n",
       "      <td>#</td>\n",
       "      <td>92,380</td>\n",
       "      <td>120,150</td>\n",
       "      <td>161,870</td>\n",
       "      <td>199,690</td>\n",
       "      <td>#</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  OCC_CODE TOT_EMP EMP_PRSE JOBS_1000 LOC_Q H_MEAN   A_MEAN MEAN_PRSE H_PCT10  \\\n",
       "2  11-1011     570      5.3     0.216  0.16  95.25  198,130       5.2   40.92   \n",
       "3  11-1021  39,200      1.9    14.958  0.95  63.42  131,920       1.1   22.27   \n",
       "4  11-1031     630      8.0     0.239  0.69      *   57,570       4.5       *   \n",
       "5  11-2011     390     25.9     0.147  0.84  59.12  122,970       7.1   27.10   \n",
       "6  11-2021   3,840      6.2     1.465  0.88  81.55  169,630       2.3   44.42   \n",
       "\n",
       "  H_PCT25 H_MEDIAN H_PCT75 H_PCT90 A_PCT10  A_PCT25 A_MEDIAN  A_PCT75  A_PCT90  \n",
       "2   56.83    86.63       #       #  85,110  118,210  180,190        #        #  \n",
       "3   32.52    51.51   81.71       #  46,310   67,650  107,140  169,970        #  \n",
       "4       *        *       *       *  21,230   23,970   48,260   68,270  100,430  \n",
       "5   45.76    58.58   68.07   87.05  56,380   95,180  121,850  141,580  181,070  \n",
       "6   57.76    77.82   96.01       #  92,380  120,150  161,870  199,690        #  "
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detailed_groups_raw_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "detailed_groups_raw_data = detailed_groups_raw_data.replace('-','',regex=True)\n",
    "gdp_per_industry = gdp_per_industry.replace('-','',regex=True)\n",
    "detailed_groups_raw_data = detailed_groups_raw_data.replace(',','',regex=True)\n",
    "detailed_groups_raw_data = detailed_groups_raw_data.replace('#',np.nan,regex=True)\n",
    "detailed_groups_raw_data = detailed_groups_raw_data.replace('*',np.nan)\n",
    "detailed_groups_raw_data = detailed_groups_raw_data.replace('**',np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "detailed_groups_final = detailed_groups_raw_data.apply(pa.to_numeric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OCC_CODE       int64\n",
       "TOT_EMP      float64\n",
       "EMP_PRSE     float64\n",
       "JOBS_1000    float64\n",
       "LOC_Q        float64\n",
       "H_MEAN       float64\n",
       "A_MEAN       float64\n",
       "MEAN_PRSE    float64\n",
       "H_PCT10      float64\n",
       "H_PCT25      float64\n",
       "H_MEDIAN     float64\n",
       "H_PCT75      float64\n",
       "H_PCT90      float64\n",
       "A_PCT10      float64\n",
       "A_PCT25      float64\n",
       "A_MEDIAN     float64\n",
       "A_PCT75      float64\n",
       "A_PCT90      float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detailed_groups_final.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "x = detailed_groups_final.values #returns a numpy array   .loc[:,'TOT_EMP':]\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "x_scaled = min_max_scaler.fit_transform(x)\n",
    "\n",
    "detailed_groups_normalized = pa.DataFrame(x_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "detailed_groups_occ_code = pa.DataFrame() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.006237</td>\n",
       "      <td>0.108607</td>\n",
       "      <td>0.006175</td>\n",
       "      <td>0.010355</td>\n",
       "      <td>0.664933</td>\n",
       "      <td>0.664987</td>\n",
       "      <td>0.223810</td>\n",
       "      <td>0.364629</td>\n",
       "      <td>0.644684</td>\n",
       "      <td>0.878290</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.364607</td>\n",
       "      <td>0.644752</td>\n",
       "      <td>0.878329</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.452414</td>\n",
       "      <td>0.038934</td>\n",
       "      <td>0.452402</td>\n",
       "      <td>0.127219</td>\n",
       "      <td>0.414106</td>\n",
       "      <td>0.414154</td>\n",
       "      <td>0.028571</td>\n",
       "      <td>0.143264</td>\n",
       "      <td>0.308586</td>\n",
       "      <td>0.474658</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.143183</td>\n",
       "      <td>0.308649</td>\n",
       "      <td>0.474693</td>\n",
       "      <td>0.833455</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000047</td>\n",
       "      <td>0.006930</td>\n",
       "      <td>0.163934</td>\n",
       "      <td>0.006871</td>\n",
       "      <td>0.088757</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.132482</td>\n",
       "      <td>0.190476</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.018281</td>\n",
       "      <td>0.149354</td>\n",
       "      <td>0.263547</td>\n",
       "      <td>0.413381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.002346</td>\n",
       "      <td>0.004158</td>\n",
       "      <td>0.530738</td>\n",
       "      <td>0.004086</td>\n",
       "      <td>0.110947</td>\n",
       "      <td>0.380221</td>\n",
       "      <td>0.380247</td>\n",
       "      <td>0.314286</td>\n",
       "      <td>0.200593</td>\n",
       "      <td>0.491636</td>\n",
       "      <td>0.555913</td>\n",
       "      <td>0.674359</td>\n",
       "      <td>0.855595</td>\n",
       "      <td>0.200651</td>\n",
       "      <td>0.491657</td>\n",
       "      <td>0.555973</td>\n",
       "      <td>0.674363</td>\n",
       "      <td>0.855607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.002370</td>\n",
       "      <td>0.044006</td>\n",
       "      <td>0.127049</td>\n",
       "      <td>0.043981</td>\n",
       "      <td>0.116864</td>\n",
       "      <td>0.556974</td>\n",
       "      <td>0.557016</td>\n",
       "      <td>0.085714</td>\n",
       "      <td>0.406172</td>\n",
       "      <td>0.657542</td>\n",
       "      <td>0.777037</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.406095</td>\n",
       "      <td>0.657648</td>\n",
       "      <td>0.777102</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4         5         6   \\\n",
       "0  0.000000  0.006237  0.108607  0.006175  0.010355  0.664933  0.664987   \n",
       "1  0.000023  0.452414  0.038934  0.452402  0.127219  0.414106  0.414154   \n",
       "2  0.000047  0.006930  0.163934  0.006871  0.088757       NaN  0.132482   \n",
       "3  0.002346  0.004158  0.530738  0.004086  0.110947  0.380221  0.380247   \n",
       "4  0.002370  0.044006  0.127049  0.043981  0.116864  0.556974  0.557016   \n",
       "\n",
       "         7         8         9         10        11        12        13  \\\n",
       "0  0.223810  0.364629  0.644684  0.878290       NaN       NaN  0.364607   \n",
       "1  0.028571  0.143264  0.308586  0.474658  0.833333       NaN  0.143183   \n",
       "2  0.190476       NaN       NaN       NaN       NaN       NaN  0.000057   \n",
       "3  0.314286  0.200593  0.491636  0.555913  0.674359  0.855595  0.200651   \n",
       "4  0.085714  0.406172  0.657542  0.777037  1.000000       NaN  0.406095   \n",
       "\n",
       "         14        15        16        17  \n",
       "0  0.644752  0.878329       NaN       NaN  \n",
       "1  0.308649  0.474693  0.833455       NaN  \n",
       "2  0.018281  0.149354  0.263547  0.413381  \n",
       "3  0.491657  0.555973  0.674363  0.855607  \n",
       "4  0.657648  0.777102  1.000000       NaN  "
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detailed_groups_normalized.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "detailed_groups_occ_code = detailed_groups_final.loc[:,:'OCC_CODE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "detailed_groups_normalized.iloc[:,0] = np.where(detailed_groups_normalized.iloc[:,0] >= 0, detailed_groups_occ_code.iloc[:,0], detailed_groups_normalized.iloc[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111011.0</td>\n",
       "      <td>0.006237</td>\n",
       "      <td>0.108607</td>\n",
       "      <td>0.006175</td>\n",
       "      <td>0.010355</td>\n",
       "      <td>0.664933</td>\n",
       "      <td>0.664987</td>\n",
       "      <td>0.223810</td>\n",
       "      <td>0.364629</td>\n",
       "      <td>0.644684</td>\n",
       "      <td>0.878290</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.364607</td>\n",
       "      <td>0.644752</td>\n",
       "      <td>0.878329</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>111021.0</td>\n",
       "      <td>0.452414</td>\n",
       "      <td>0.038934</td>\n",
       "      <td>0.452402</td>\n",
       "      <td>0.127219</td>\n",
       "      <td>0.414106</td>\n",
       "      <td>0.414154</td>\n",
       "      <td>0.028571</td>\n",
       "      <td>0.143264</td>\n",
       "      <td>0.308586</td>\n",
       "      <td>0.474658</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.143183</td>\n",
       "      <td>0.308649</td>\n",
       "      <td>0.474693</td>\n",
       "      <td>0.833455</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>111031.0</td>\n",
       "      <td>0.006930</td>\n",
       "      <td>0.163934</td>\n",
       "      <td>0.006871</td>\n",
       "      <td>0.088757</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.132482</td>\n",
       "      <td>0.190476</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.018281</td>\n",
       "      <td>0.149354</td>\n",
       "      <td>0.263547</td>\n",
       "      <td>0.413381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>112011.0</td>\n",
       "      <td>0.004158</td>\n",
       "      <td>0.530738</td>\n",
       "      <td>0.004086</td>\n",
       "      <td>0.110947</td>\n",
       "      <td>0.380221</td>\n",
       "      <td>0.380247</td>\n",
       "      <td>0.314286</td>\n",
       "      <td>0.200593</td>\n",
       "      <td>0.491636</td>\n",
       "      <td>0.555913</td>\n",
       "      <td>0.674359</td>\n",
       "      <td>0.855595</td>\n",
       "      <td>0.200651</td>\n",
       "      <td>0.491657</td>\n",
       "      <td>0.555973</td>\n",
       "      <td>0.674363</td>\n",
       "      <td>0.855607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>112021.0</td>\n",
       "      <td>0.044006</td>\n",
       "      <td>0.127049</td>\n",
       "      <td>0.043981</td>\n",
       "      <td>0.116864</td>\n",
       "      <td>0.556974</td>\n",
       "      <td>0.557016</td>\n",
       "      <td>0.085714</td>\n",
       "      <td>0.406172</td>\n",
       "      <td>0.657542</td>\n",
       "      <td>0.777037</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.406095</td>\n",
       "      <td>0.657648</td>\n",
       "      <td>0.777102</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4         5         6   \\\n",
       "0  111011.0  0.006237  0.108607  0.006175  0.010355  0.664933  0.664987   \n",
       "1  111021.0  0.452414  0.038934  0.452402  0.127219  0.414106  0.414154   \n",
       "2  111031.0  0.006930  0.163934  0.006871  0.088757       NaN  0.132482   \n",
       "3  112011.0  0.004158  0.530738  0.004086  0.110947  0.380221  0.380247   \n",
       "4  112021.0  0.044006  0.127049  0.043981  0.116864  0.556974  0.557016   \n",
       "\n",
       "         7         8         9         10        11        12        13  \\\n",
       "0  0.223810  0.364629  0.644684  0.878290       NaN       NaN  0.364607   \n",
       "1  0.028571  0.143264  0.308586  0.474658  0.833333       NaN  0.143183   \n",
       "2  0.190476       NaN       NaN       NaN       NaN       NaN  0.000057   \n",
       "3  0.314286  0.200593  0.491636  0.555913  0.674359  0.855595  0.200651   \n",
       "4  0.085714  0.406172  0.657542  0.777037  1.000000       NaN  0.406095   \n",
       "\n",
       "         14        15        16        17  \n",
       "0  0.644752  0.878329       NaN       NaN  \n",
       "1  0.308649  0.474693  0.833455       NaN  \n",
       "2  0.018281  0.149354  0.263547  0.413381  \n",
       "3  0.491657  0.555973  0.674363  0.855607  \n",
       "4  0.657648  0.777102  1.000000       NaN  "
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detailed_groups_normalized.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdp_per_industry['Occupation Code'] = gdp_per_industry['Occupation Code'].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdp_per_industry.drop(gdp_per_industry.columns.difference(['Occupation Code','Projection']), 1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(651, 20)"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_inner = pa.merge(left=detailed_groups_normalized, right=gdp_per_industry, left_on=0, right_on='Occupation Code')\n",
    "# In this case `species_id` is the only column name in  both dataframes, so if we skipped `left_on`\n",
    "# And `right_on` arguments we would still get the same result\n",
    "\n",
    "# What's the size of the output data?\n",
    "merged_inner.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>Occupation Code</th>\n",
       "      <th>Projection</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111011.0</td>\n",
       "      <td>0.006237</td>\n",
       "      <td>0.108607</td>\n",
       "      <td>0.006175</td>\n",
       "      <td>0.010355</td>\n",
       "      <td>0.664933</td>\n",
       "      <td>0.664987</td>\n",
       "      <td>0.223810</td>\n",
       "      <td>0.364629</td>\n",
       "      <td>0.644684</td>\n",
       "      <td>0.878290</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.364607</td>\n",
       "      <td>0.644752</td>\n",
       "      <td>0.878329</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>111011.0</td>\n",
       "      <td>1310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>111021.0</td>\n",
       "      <td>0.452414</td>\n",
       "      <td>0.038934</td>\n",
       "      <td>0.452402</td>\n",
       "      <td>0.127219</td>\n",
       "      <td>0.414106</td>\n",
       "      <td>0.414154</td>\n",
       "      <td>0.028571</td>\n",
       "      <td>0.143264</td>\n",
       "      <td>0.308586</td>\n",
       "      <td>0.474658</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.143183</td>\n",
       "      <td>0.308649</td>\n",
       "      <td>0.474693</td>\n",
       "      <td>0.833455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>111021.0</td>\n",
       "      <td>48450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>111031.0</td>\n",
       "      <td>0.006930</td>\n",
       "      <td>0.163934</td>\n",
       "      <td>0.006871</td>\n",
       "      <td>0.088757</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.132482</td>\n",
       "      <td>0.190476</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.018281</td>\n",
       "      <td>0.149354</td>\n",
       "      <td>0.263547</td>\n",
       "      <td>0.413381</td>\n",
       "      <td>111031.0</td>\n",
       "      <td>750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>112011.0</td>\n",
       "      <td>0.004158</td>\n",
       "      <td>0.530738</td>\n",
       "      <td>0.004086</td>\n",
       "      <td>0.110947</td>\n",
       "      <td>0.380221</td>\n",
       "      <td>0.380247</td>\n",
       "      <td>0.314286</td>\n",
       "      <td>0.200593</td>\n",
       "      <td>0.491636</td>\n",
       "      <td>0.555913</td>\n",
       "      <td>0.674359</td>\n",
       "      <td>0.855595</td>\n",
       "      <td>0.200651</td>\n",
       "      <td>0.491657</td>\n",
       "      <td>0.555973</td>\n",
       "      <td>0.674363</td>\n",
       "      <td>0.855607</td>\n",
       "      <td>112011.0</td>\n",
       "      <td>610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>112021.0</td>\n",
       "      <td>0.044006</td>\n",
       "      <td>0.127049</td>\n",
       "      <td>0.043981</td>\n",
       "      <td>0.116864</td>\n",
       "      <td>0.556974</td>\n",
       "      <td>0.557016</td>\n",
       "      <td>0.085714</td>\n",
       "      <td>0.406172</td>\n",
       "      <td>0.657542</td>\n",
       "      <td>0.777037</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.406095</td>\n",
       "      <td>0.657648</td>\n",
       "      <td>0.777102</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>112021.0</td>\n",
       "      <td>3610</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  111011.0  0.006237  0.108607  0.006175  0.010355  0.664933  0.664987   \n",
       "1  111021.0  0.452414  0.038934  0.452402  0.127219  0.414106  0.414154   \n",
       "2  111031.0  0.006930  0.163934  0.006871  0.088757       NaN  0.132482   \n",
       "3  112011.0  0.004158  0.530738  0.004086  0.110947  0.380221  0.380247   \n",
       "4  112021.0  0.044006  0.127049  0.043981  0.116864  0.556974  0.557016   \n",
       "\n",
       "          7         8         9        10        11        12        13  \\\n",
       "0  0.223810  0.364629  0.644684  0.878290       NaN       NaN  0.364607   \n",
       "1  0.028571  0.143264  0.308586  0.474658  0.833333       NaN  0.143183   \n",
       "2  0.190476       NaN       NaN       NaN       NaN       NaN  0.000057   \n",
       "3  0.314286  0.200593  0.491636  0.555913  0.674359  0.855595  0.200651   \n",
       "4  0.085714  0.406172  0.657542  0.777037  1.000000       NaN  0.406095   \n",
       "\n",
       "         14        15        16        17  Occupation Code  Projection  \n",
       "0  0.644752  0.878329       NaN       NaN         111011.0        1310  \n",
       "1  0.308649  0.474693  0.833455       NaN         111021.0       48450  \n",
       "2  0.018281  0.149354  0.263547  0.413381         111031.0         750  \n",
       "3  0.491657  0.555973  0.674363  0.855607         112011.0         610  \n",
       "4  0.657648  0.777102  1.000000       NaN         112021.0        3610  "
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_inner.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_inner.dropna();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(merged_inner.iloc[:,1:17])\n",
    "y = np.array(merged_inner.iloc[:,19:20])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test=train_test_split(X,y,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "knn = KNeighborsRegressor(n_neighbors=5, weights='uniform', algorithm='auto')#algorithm='brute',metric='euclidean',algorithm='brute'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn.fit(x_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = knn.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_array = np.array(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Distance between y_test and K-Nearest Neighbors predictins: [[249.85321101]]\n"
     ]
    }
   ],
   "source": [
    "total_sum = 0\n",
    "\n",
    "for i in range (len(y_test)):\n",
    "    distance = y_test_array[[i],:] - predictions[[i],:]\n",
    "    total_sum += distance\n",
    "    \n",
    "total_average = abs(total_sum/len(y_test))   ## Prints the average distance between my prediction and y_train. could easily be switched to y_test - going to try for 'detailed' group\n",
    "print(\"Average Distance between y_test and K-Nearest Neighbors predictins:\", total_average)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2837.4742160247915\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error    # i dont really know what this is but I think it is how far im off on my predictions, on average\n",
    "mse = mean_squared_error(y_test, predictions)\n",
    "rmse = mse ** (1/2)\n",
    "print(rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import scale\n",
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = svm.SVC(C=1.0, kernel='rbf', degree=3, gamma='scale', coef0=0.0, shrinking=True, probability=False, tol=0.001, cache_size=200, class_weight=None, verbose=False, max_iter=-1, decision_function_shape='ovr', random_state=None)      # remove all the stuff inside to see what can go inside this\n",
    "clf.fit(x_train, y_train.ravel());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions1 = clf.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Distance between y_test and SVM predictins: [[3480]]\n"
     ]
    }
   ],
   "source": [
    "total_sum1 = 0\n",
    "\n",
    "for i in range (len(y_test)):\n",
    "        distance = y_test_array[[i],:] - predictions1[i]\n",
    "        total_sum1 += distance\n",
    "    \n",
    "total_average1 = abs(total_sum1//len(y_test))\n",
    "print(\"Average Distance between y_test and SVM predictins:\",total_average1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}