Major_Data_tocsv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np                   # Numerical computation library\n",
    "import pandas as pa                  # Dataframe and Data manipulation Library\n",
    "import matplotlib.pyplot as plt      # Basic plotting functionality Library\n",
    "import seaborn as sn                 # Advanced Data visualization Library\n",
    "import numpy.random as rn\n",
    "from sklearn.linear_model import LinearRegression\n",
    "import matplotlib.colors as pltco # A package to help with coloring plots\n",
    "\n",
    "\n",
    "[x for x in dir(list) if not '__' in x]\n",
    "  \n",
    "# Curreny Exchange Rates compared to the US dollar from around the world 22 for 2008-2018\n",
    "gdp_per_industry = pa.read_csv('https://www.dropbox.com/s/emd7111t9g0rtd2/career_projections_group.csv?dl=1')\n",
    "workforce_demographic2018 = pa.read_csv('https://www.dropbox.com/s/w1zzumsadlgg9pn/Colorado_Current_Workforce_Demographic.csv?dl=1')\n",
    "\n",
    "gdp_per_industry = gdp_per_industry.drop(0)\n",
    "workforce_demographic2018 = workforce_demographic2018.drop(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "import numpy as np\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.linear_model import LinearRegression\n",
    "import matplotlib.cm as cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "workforce_demographic2018['OCC_GROUP'] = workforce_demographic2018['OCC_GROUP'].astype(str)\n",
    "detailed_groups_df = workforce_demographic2018.loc[workforce_demographic2018.OCC_GROUP.apply(lambda cat: 'detailed' in cat)]\n",
    "major_groups_df = workforce_demographic2018.loc[workforce_demographic2018.OCC_GROUP.apply(lambda cat: 'major' in cat)]\n",
    "#detailed_groups_raw_data = detailed_groups_df.drop(['AREA','STATE','ST','OCC_GROUP','OCC_TITLE','ANNUAL','HOURLY'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "major_groups_clean = major_groups_df.drop(['AREA','STATE','ST','OCC_GROUP','ANNUAL','HOURLY','H_PCT90','A_PCT90','OCC_CODE'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OCC_TITLE</th>\n",
       "      <th>TOT_EMP</th>\n",
       "      <th>EMP_PRSE</th>\n",
       "      <th>JOBS_1000</th>\n",
       "      <th>LOC_Q</th>\n",
       "      <th>H_MEAN</th>\n",
       "      <th>A_MEAN</th>\n",
       "      <th>MEAN_PRSE</th>\n",
       "      <th>H_PCT10</th>\n",
       "      <th>H_PCT25</th>\n",
       "      <th>H_MEDIAN</th>\n",
       "      <th>H_PCT75</th>\n",
       "      <th>A_PCT10</th>\n",
       "      <th>A_PCT25</th>\n",
       "      <th>A_MEDIAN</th>\n",
       "      <th>A_PCT75</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Management Occupations</td>\n",
       "      <td>116,770</td>\n",
       "      <td>1.1</td>\n",
       "      <td>44.556</td>\n",
       "      <td>0.85</td>\n",
       "      <td>63.34</td>\n",
       "      <td>131,750</td>\n",
       "      <td>0.8</td>\n",
       "      <td>26.78</td>\n",
       "      <td>38.28</td>\n",
       "      <td>56.15</td>\n",
       "      <td>78.38</td>\n",
       "      <td>55,700</td>\n",
       "      <td>79,630</td>\n",
       "      <td>116,800</td>\n",
       "      <td>163,030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Business and Financial Operations Occupations</td>\n",
       "      <td>190,330</td>\n",
       "      <td>1.0</td>\n",
       "      <td>72.626</td>\n",
       "      <td>1.36</td>\n",
       "      <td>37.87</td>\n",
       "      <td>78,780</td>\n",
       "      <td>0.7</td>\n",
       "      <td>19.47</td>\n",
       "      <td>25.71</td>\n",
       "      <td>34.14</td>\n",
       "      <td>46.03</td>\n",
       "      <td>40,510</td>\n",
       "      <td>53,470</td>\n",
       "      <td>71,000</td>\n",
       "      <td>95,750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>Computer and Mathematical Occupations</td>\n",
       "      <td>116,930</td>\n",
       "      <td>1.9</td>\n",
       "      <td>44.620</td>\n",
       "      <td>1.47</td>\n",
       "      <td>46.58</td>\n",
       "      <td>96,880</td>\n",
       "      <td>0.9</td>\n",
       "      <td>24.51</td>\n",
       "      <td>33.07</td>\n",
       "      <td>45.06</td>\n",
       "      <td>58.30</td>\n",
       "      <td>50,990</td>\n",
       "      <td>68,790</td>\n",
       "      <td>93,730</td>\n",
       "      <td>121,260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>Architecture and Engineering Occupations</td>\n",
       "      <td>61,970</td>\n",
       "      <td>2.2</td>\n",
       "      <td>23.647</td>\n",
       "      <td>1.34</td>\n",
       "      <td>43.80</td>\n",
       "      <td>91,090</td>\n",
       "      <td>1.1</td>\n",
       "      <td>22.35</td>\n",
       "      <td>29.72</td>\n",
       "      <td>39.62</td>\n",
       "      <td>54.67</td>\n",
       "      <td>46,480</td>\n",
       "      <td>61,820</td>\n",
       "      <td>82,400</td>\n",
       "      <td>113,710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>Life, Physical, and Social Science Occupations</td>\n",
       "      <td>30,330</td>\n",
       "      <td>2.0</td>\n",
       "      <td>11.575</td>\n",
       "      <td>1.43</td>\n",
       "      <td>36.62</td>\n",
       "      <td>76,170</td>\n",
       "      <td>2.3</td>\n",
       "      <td>16.74</td>\n",
       "      <td>22.85</td>\n",
       "      <td>32.42</td>\n",
       "      <td>45.63</td>\n",
       "      <td>34,810</td>\n",
       "      <td>47,520</td>\n",
       "      <td>67,440</td>\n",
       "      <td>94,910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          OCC_TITLE  TOT_EMP EMP_PRSE  \\\n",
       "1                            Management Occupations  116,770      1.1   \n",
       "35    Business and Financial Operations Occupations  190,330      1.0   \n",
       "64            Computer and Mathematical Occupations  116,930      1.9   \n",
       "83         Architecture and Engineering Occupations   61,970      2.2   \n",
       "116  Life, Physical, and Social Science Occupations   30,330      2.0   \n",
       "\n",
       "    JOBS_1000 LOC_Q H_MEAN   A_MEAN MEAN_PRSE H_PCT10 H_PCT25 H_MEDIAN  \\\n",
       "1      44.556  0.85  63.34  131,750       0.8   26.78   38.28    56.15   \n",
       "35     72.626  1.36  37.87   78,780       0.7   19.47   25.71    34.14   \n",
       "64     44.620  1.47  46.58   96,880       0.9   24.51   33.07    45.06   \n",
       "83     23.647  1.34  43.80   91,090       1.1   22.35   29.72    39.62   \n",
       "116    11.575  1.43  36.62   76,170       2.3   16.74   22.85    32.42   \n",
       "\n",
       "    H_PCT75 A_PCT10 A_PCT25 A_MEDIAN  A_PCT75  \n",
       "1     78.38  55,700  79,630  116,800  163,030  \n",
       "35    46.03  40,510  53,470   71,000   95,750  \n",
       "64    58.30  50,990  68,790   93,730  121,260  \n",
       "83    54.67  46,480  61,820   82,400  113,710  \n",
       "116   45.63  34,810  47,520   67,440   94,910  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "major_groups_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "major_groups_numeric = major_groups_clean.replace(',','',regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "major_groups_numeric['OCC_TITLE'] = major_groups_numeric['OCC_TITLE'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "major_groups_numeric = major_groups_numeric.iloc[:,1:].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "major_groups_numeric.astype(float);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "major_groups_numeric.to_csv('major_groups_unnormalized.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "x = major_groups_numeric.values \n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "x_scaled = min_max_scaler.fit_transform(x)\n",
    "\n",
    "major_data_normalized = pa.DataFrame(x_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "occ_titles = []\n",
    "occ_titles = major_groups_clean.iloc[:,0:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_one_list = occ_titles['OCC_TITLE'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexed_occ_title = pa.DataFrame(col_one_list) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "major_data_normalized.insert(0, \"OCC TITLE\", indexed_occ_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OCC TITLE</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Management Occupations</td>\n",
       "      <td>0.309646</td>\n",
       "      <td>0.029851</td>\n",
       "      <td>0.309635</td>\n",
       "      <td>0.270588</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Business and Financial Operations Occupations</td>\n",
       "      <td>0.517243</td>\n",
       "      <td>0.014925</td>\n",
       "      <td>0.517236</td>\n",
       "      <td>0.870588</td>\n",
       "      <td>0.492832</td>\n",
       "      <td>0.492867</td>\n",
       "      <td>0.051282</td>\n",
       "      <td>0.559107</td>\n",
       "      <td>0.552191</td>\n",
       "      <td>0.508266</td>\n",
       "      <td>0.496655</td>\n",
       "      <td>0.559455</td>\n",
       "      <td>0.552055</td>\n",
       "      <td>0.508109</td>\n",
       "      <td>0.496671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Computer and Mathematical Occupations</td>\n",
       "      <td>0.310098</td>\n",
       "      <td>0.149254</td>\n",
       "      <td>0.310108</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666268</td>\n",
       "      <td>0.666156</td>\n",
       "      <td>0.102564</td>\n",
       "      <td>0.863088</td>\n",
       "      <td>0.814393</td>\n",
       "      <td>0.752234</td>\n",
       "      <td>0.687568</td>\n",
       "      <td>0.863399</td>\n",
       "      <td>0.814384</td>\n",
       "      <td>0.752229</td>\n",
       "      <td>0.687514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Architecture and Engineering Occupations</td>\n",
       "      <td>0.154992</td>\n",
       "      <td>0.194030</td>\n",
       "      <td>0.154995</td>\n",
       "      <td>0.847059</td>\n",
       "      <td>0.610912</td>\n",
       "      <td>0.610723</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.732811</td>\n",
       "      <td>0.695048</td>\n",
       "      <td>0.630697</td>\n",
       "      <td>0.631088</td>\n",
       "      <td>0.732599</td>\n",
       "      <td>0.695034</td>\n",
       "      <td>0.630545</td>\n",
       "      <td>0.631032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Life, Physical, and Social Science Occupations</td>\n",
       "      <td>0.065700</td>\n",
       "      <td>0.164179</td>\n",
       "      <td>0.065712</td>\n",
       "      <td>0.952941</td>\n",
       "      <td>0.467941</td>\n",
       "      <td>0.467879</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.394451</td>\n",
       "      <td>0.450303</td>\n",
       "      <td>0.469839</td>\n",
       "      <td>0.490431</td>\n",
       "      <td>0.394142</td>\n",
       "      <td>0.450171</td>\n",
       "      <td>0.469874</td>\n",
       "      <td>0.490387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Community and Social Service Occupations</td>\n",
       "      <td>0.091494</td>\n",
       "      <td>0.164179</td>\n",
       "      <td>0.091509</td>\n",
       "      <td>0.447059</td>\n",
       "      <td>0.216249</td>\n",
       "      <td>0.216084</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.207479</td>\n",
       "      <td>0.249377</td>\n",
       "      <td>0.250447</td>\n",
       "      <td>0.230279</td>\n",
       "      <td>0.207367</td>\n",
       "      <td>0.249486</td>\n",
       "      <td>0.250456</td>\n",
       "      <td>0.230194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Legal Occupations</td>\n",
       "      <td>0.043715</td>\n",
       "      <td>0.298507</td>\n",
       "      <td>0.043717</td>\n",
       "      <td>0.564706</td>\n",
       "      <td>0.824771</td>\n",
       "      <td>0.824701</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.682147</td>\n",
       "      <td>0.662273</td>\n",
       "      <td>0.674710</td>\n",
       "      <td>0.878948</td>\n",
       "      <td>0.682425</td>\n",
       "      <td>0.662158</td>\n",
       "      <td>0.674686</td>\n",
       "      <td>0.878881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Education, Training, and Library Occupations</td>\n",
       "      <td>0.383925</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.383918</td>\n",
       "      <td>0.329412</td>\n",
       "      <td>0.264436</td>\n",
       "      <td>0.264337</td>\n",
       "      <td>0.820513</td>\n",
       "      <td>0.159228</td>\n",
       "      <td>0.230139</td>\n",
       "      <td>0.252458</td>\n",
       "      <td>0.260152</td>\n",
       "      <td>0.159223</td>\n",
       "      <td>0.230137</td>\n",
       "      <td>0.252497</td>\n",
       "      <td>0.260043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Arts, Design, Entertainment, Sports, and Media...</td>\n",
       "      <td>0.094796</td>\n",
       "      <td>0.194030</td>\n",
       "      <td>0.094800</td>\n",
       "      <td>0.623529</td>\n",
       "      <td>0.267423</td>\n",
       "      <td>0.267401</td>\n",
       "      <td>0.307692</td>\n",
       "      <td>0.045838</td>\n",
       "      <td>0.154970</td>\n",
       "      <td>0.242851</td>\n",
       "      <td>0.286603</td>\n",
       "      <td>0.045824</td>\n",
       "      <td>0.154966</td>\n",
       "      <td>0.242938</td>\n",
       "      <td>0.286527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Healthcare Practitioners and Technical Occupat...</td>\n",
       "      <td>0.392702</td>\n",
       "      <td>0.089552</td>\n",
       "      <td>0.392720</td>\n",
       "      <td>0.364706</td>\n",
       "      <td>0.542214</td>\n",
       "      <td>0.542173</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.458384</td>\n",
       "      <td>0.548985</td>\n",
       "      <td>0.513628</td>\n",
       "      <td>0.492765</td>\n",
       "      <td>0.458527</td>\n",
       "      <td>0.548801</td>\n",
       "      <td>0.513586</td>\n",
       "      <td>0.492631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Healthcare Support Occupations</td>\n",
       "      <td>0.165632</td>\n",
       "      <td>0.179104</td>\n",
       "      <td>0.165637</td>\n",
       "      <td>0.305882</td>\n",
       "      <td>0.078654</td>\n",
       "      <td>0.078602</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.072376</td>\n",
       "      <td>0.110082</td>\n",
       "      <td>0.105228</td>\n",
       "      <td>0.081064</td>\n",
       "      <td>0.071926</td>\n",
       "      <td>0.110103</td>\n",
       "      <td>0.105252</td>\n",
       "      <td>0.081020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Protective Service Occupations</td>\n",
       "      <td>0.146836</td>\n",
       "      <td>0.238806</td>\n",
       "      <td>0.146845</td>\n",
       "      <td>0.388235</td>\n",
       "      <td>0.217443</td>\n",
       "      <td>0.217425</td>\n",
       "      <td>0.564103</td>\n",
       "      <td>0.075995</td>\n",
       "      <td>0.135732</td>\n",
       "      <td>0.200626</td>\n",
       "      <td>0.277112</td>\n",
       "      <td>0.075696</td>\n",
       "      <td>0.135616</td>\n",
       "      <td>0.200516</td>\n",
       "      <td>0.277026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Food Preparation and Serving Related Occupations</td>\n",
       "      <td>0.700457</td>\n",
       "      <td>0.044776</td>\n",
       "      <td>0.700446</td>\n",
       "      <td>0.505882</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.051282</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Building and Grounds Cleaning and Maintenance ...</td>\n",
       "      <td>0.214681</td>\n",
       "      <td>0.134328</td>\n",
       "      <td>0.214694</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>0.037634</td>\n",
       "      <td>0.037530</td>\n",
       "      <td>0.128205</td>\n",
       "      <td>0.013269</td>\n",
       "      <td>0.045957</td>\n",
       "      <td>0.052279</td>\n",
       "      <td>0.047145</td>\n",
       "      <td>0.013051</td>\n",
       "      <td>0.045890</td>\n",
       "      <td>0.052196</td>\n",
       "      <td>0.047056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Personal Care and Service Occupations</td>\n",
       "      <td>0.243326</td>\n",
       "      <td>0.194030</td>\n",
       "      <td>0.243331</td>\n",
       "      <td>0.376471</td>\n",
       "      <td>0.037634</td>\n",
       "      <td>0.037530</td>\n",
       "      <td>0.179487</td>\n",
       "      <td>0.000603</td>\n",
       "      <td>0.022444</td>\n",
       "      <td>0.020107</td>\n",
       "      <td>0.038587</td>\n",
       "      <td>0.000290</td>\n",
       "      <td>0.022603</td>\n",
       "      <td>0.020084</td>\n",
       "      <td>0.038603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Sales and Related Occupations</td>\n",
       "      <td>0.803889</td>\n",
       "      <td>0.074627</td>\n",
       "      <td>0.803906</td>\n",
       "      <td>0.576471</td>\n",
       "      <td>0.196933</td>\n",
       "      <td>0.196745</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.014475</td>\n",
       "      <td>0.046313</td>\n",
       "      <td>0.094057</td>\n",
       "      <td>0.200716</td>\n",
       "      <td>0.014211</td>\n",
       "      <td>0.046404</td>\n",
       "      <td>0.094082</td>\n",
       "      <td>0.200643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Office and Administrative Support Occupations</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.341176</td>\n",
       "      <td>0.130625</td>\n",
       "      <td>0.130493</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.074789</td>\n",
       "      <td>0.136088</td>\n",
       "      <td>0.149017</td>\n",
       "      <td>0.146569</td>\n",
       "      <td>0.074536</td>\n",
       "      <td>0.135959</td>\n",
       "      <td>0.148964</td>\n",
       "      <td>0.146555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Farming, Fishing, and Forestry Occupations</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.223529</td>\n",
       "      <td>0.049383</td>\n",
       "      <td>0.049306</td>\n",
       "      <td>0.307692</td>\n",
       "      <td>0.000603</td>\n",
       "      <td>0.030638</td>\n",
       "      <td>0.052502</td>\n",
       "      <td>0.060370</td>\n",
       "      <td>0.000290</td>\n",
       "      <td>0.030822</td>\n",
       "      <td>0.052626</td>\n",
       "      <td>0.060223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Construction and Extraction Occupations</td>\n",
       "      <td>0.377152</td>\n",
       "      <td>0.089552</td>\n",
       "      <td>0.377151</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.217244</td>\n",
       "      <td>0.217233</td>\n",
       "      <td>0.051282</td>\n",
       "      <td>0.249698</td>\n",
       "      <td>0.261133</td>\n",
       "      <td>0.249106</td>\n",
       "      <td>0.227167</td>\n",
       "      <td>0.249710</td>\n",
       "      <td>0.261301</td>\n",
       "      <td>0.249060</td>\n",
       "      <td>0.227127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Installation, Maintenance, and Repair Occupations</td>\n",
       "      <td>0.266411</td>\n",
       "      <td>0.089552</td>\n",
       "      <td>0.266413</td>\n",
       "      <td>0.447059</td>\n",
       "      <td>0.231581</td>\n",
       "      <td>0.231498</td>\n",
       "      <td>0.025641</td>\n",
       "      <td>0.174306</td>\n",
       "      <td>0.244745</td>\n",
       "      <td>0.264298</td>\n",
       "      <td>0.257507</td>\n",
       "      <td>0.174014</td>\n",
       "      <td>0.244863</td>\n",
       "      <td>0.264204</td>\n",
       "      <td>0.257425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Production Occupations</td>\n",
       "      <td>0.269487</td>\n",
       "      <td>0.298507</td>\n",
       "      <td>0.269483</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.131223</td>\n",
       "      <td>0.131067</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.059710</td>\n",
       "      <td>0.111507</td>\n",
       "      <td>0.132484</td>\n",
       "      <td>0.144391</td>\n",
       "      <td>0.059165</td>\n",
       "      <td>0.111644</td>\n",
       "      <td>0.132424</td>\n",
       "      <td>0.144385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Transportation and Material Moving Occupations</td>\n",
       "      <td>0.405910</td>\n",
       "      <td>0.164179</td>\n",
       "      <td>0.405914</td>\n",
       "      <td>0.223529</td>\n",
       "      <td>0.143767</td>\n",
       "      <td>0.143705</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.024729</td>\n",
       "      <td>0.080513</td>\n",
       "      <td>0.121761</td>\n",
       "      <td>0.136456</td>\n",
       "      <td>0.024652</td>\n",
       "      <td>0.080479</td>\n",
       "      <td>0.121684</td>\n",
       "      <td>0.136306</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            OCC TITLE         0         1  \\\n",
       "0                              Management Occupations  0.309646  0.029851   \n",
       "1       Business and Financial Operations Occupations  0.517243  0.014925   \n",
       "2               Computer and Mathematical Occupations  0.310098  0.149254   \n",
       "3            Architecture and Engineering Occupations  0.154992  0.194030   \n",
       "4      Life, Physical, and Social Science Occupations  0.065700  0.164179   \n",
       "5            Community and Social Service Occupations  0.091494  0.164179   \n",
       "6                                   Legal Occupations  0.043715  0.298507   \n",
       "7        Education, Training, and Library Occupations  0.383925  0.000000   \n",
       "8   Arts, Design, Entertainment, Sports, and Media...  0.094796  0.194030   \n",
       "9   Healthcare Practitioners and Technical Occupat...  0.392702  0.089552   \n",
       "10                     Healthcare Support Occupations  0.165632  0.179104   \n",
       "11                     Protective Service Occupations  0.146836  0.238806   \n",
       "12   Food Preparation and Serving Related Occupations  0.700457  0.044776   \n",
       "13  Building and Grounds Cleaning and Maintenance ...  0.214681  0.134328   \n",
       "14              Personal Care and Service Occupations  0.243326  0.194030   \n",
       "15                      Sales and Related Occupations  0.803889  0.074627   \n",
       "16      Office and Administrative Support Occupations  1.000000  0.000000   \n",
       "17         Farming, Fishing, and Forestry Occupations  0.000000  1.000000   \n",
       "18            Construction and Extraction Occupations  0.377152  0.089552   \n",
       "19  Installation, Maintenance, and Repair Occupations  0.266411  0.089552   \n",
       "20                             Production Occupations  0.269487  0.298507   \n",
       "21     Transportation and Material Moving Occupations  0.405910  0.164179   \n",
       "\n",
       "           2         3         4         5         6         7         8  \\\n",
       "0   0.309635  0.270588  1.000000  1.000000  0.076923  1.000000  1.000000   \n",
       "1   0.517236  0.870588  0.492832  0.492867  0.051282  0.559107  0.552191   \n",
       "2   0.310108  1.000000  0.666268  0.666156  0.102564  0.863088  0.814393   \n",
       "3   0.154995  0.847059  0.610912  0.610723  0.153846  0.732811  0.695048   \n",
       "4   0.065712  0.952941  0.467941  0.467879  0.461538  0.394451  0.450303   \n",
       "5   0.091509  0.447059  0.216249  0.216084  0.153846  0.207479  0.249377   \n",
       "6   0.043717  0.564706  0.824771  0.824701  1.000000  0.682147  0.662273   \n",
       "7   0.383918  0.329412  0.264436  0.264337  0.820513  0.159228  0.230139   \n",
       "8   0.094800  0.623529  0.267423  0.267401  0.307692  0.045838  0.154970   \n",
       "9   0.392720  0.364706  0.542214  0.542173  0.153846  0.458384  0.548985   \n",
       "10  0.165637  0.305882  0.078654  0.078602  0.076923  0.072376  0.110082   \n",
       "11  0.146845  0.388235  0.217443  0.217425  0.564103  0.075995  0.135732   \n",
       "12  0.700446  0.505882  0.000000  0.000000  0.051282  0.000000  0.000000   \n",
       "13  0.214694  0.494118  0.037634  0.037530  0.128205  0.013269  0.045957   \n",
       "14  0.243331  0.376471  0.037634  0.037530  0.179487  0.000603  0.022444   \n",
       "15  0.803906  0.576471  0.196933  0.196745  0.153846  0.014475  0.046313   \n",
       "16  1.000000  0.341176  0.130625  0.130493  0.000000  0.074789  0.136088   \n",
       "17  0.000000  0.223529  0.049383  0.049306  0.307692  0.000603  0.030638   \n",
       "18  0.377151  0.800000  0.217244  0.217233  0.051282  0.249698  0.261133   \n",
       "19  0.266413  0.447059  0.231581  0.231498  0.025641  0.174306  0.244745   \n",
       "20  0.269483  0.000000  0.131223  0.131067  0.076923  0.059710  0.111507   \n",
       "21  0.405914  0.223529  0.143767  0.143705  0.948718  0.024729  0.080513   \n",
       "\n",
       "           9        10        11        12        13        14  \n",
       "0   1.000000  1.000000  1.000000  1.000000  1.000000  1.000000  \n",
       "1   0.508266  0.496655  0.559455  0.552055  0.508109  0.496671  \n",
       "2   0.752234  0.687568  0.863399  0.814384  0.752229  0.687514  \n",
       "3   0.630697  0.631088  0.732599  0.695034  0.630545  0.631032  \n",
       "4   0.469839  0.490431  0.394142  0.450171  0.469874  0.490387  \n",
       "5   0.250447  0.230279  0.207367  0.249486  0.250456  0.230194  \n",
       "6   0.674710  0.878948  0.682425  0.662158  0.674686  0.878881  \n",
       "7   0.252458  0.260152  0.159223  0.230137  0.252497  0.260043  \n",
       "8   0.242851  0.286603  0.045824  0.154966  0.242938  0.286527  \n",
       "9   0.513628  0.492765  0.458527  0.548801  0.513586  0.492631  \n",
       "10  0.105228  0.081064  0.071926  0.110103  0.105252  0.081020  \n",
       "11  0.200626  0.277112  0.075696  0.135616  0.200516  0.277026  \n",
       "12  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "13  0.052279  0.047145  0.013051  0.045890  0.052196  0.047056  \n",
       "14  0.020107  0.038587  0.000290  0.022603  0.020084  0.038603  \n",
       "15  0.094057  0.200716  0.014211  0.046404  0.094082  0.200643  \n",
       "16  0.149017  0.146569  0.074536  0.135959  0.148964  0.146555  \n",
       "17  0.052502  0.060370  0.000290  0.030822  0.052626  0.060223  \n",
       "18  0.249106  0.227167  0.249710  0.261301  0.249060  0.227127  \n",
       "19  0.264298  0.257507  0.174014  0.244863  0.264204  0.257425  \n",
       "20  0.132484  0.144391  0.059165  0.111644  0.132424  0.144385  \n",
       "21  0.121761  0.136456  0.024652  0.080479  0.121684  0.136306  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "major_data_normalized.head(22)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "major_data_normalized.to_csv('major_data_normalized.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(major_data_normalized.iloc[:,1:15])\n",
    "y = np.array(major_data_normalized.iloc[:,0:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test=train_test_split(X,y,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_x_train = pa.DataFrame(x_train)\n",
    "df_y_train = pa.DataFrame(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_x_test = pa.DataFrame(x_test)\n",
    "df_y_test = pa.DataFrame(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_x_train.insert(0, \"OCC TITLE\", df_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_x_test.insert(0, \"OCC TITLE\", df_y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_x_train.drop('OCC TITLE',axis=1);   #drops the title column in both testing and training so that my neural network can just go straight to working with float and int dtypes\n",
    "df_x_test.drop('OCC TITLE',axis=1);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_x_train.to_csv('training_data_TF1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_x_test.to_csv('testing_data_TF1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}