create_political_voice_xgboost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datojen haku ja esikäsittely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import shap\n",
    "\n",
    "from create_shap_values_via_xgboost import create_shap_values_via_xgboost\n",
    "from create_target_columns import create_target_columns\n",
    "from print_examples import show_all_results, create_party_results, show_one_results\n",
    "from show_election_result import show_election_result\n",
    "from set_path import set_path\n",
    "from read_and_prepare_data import read_and_prepare_data\n",
    "from selected_cols import selected_cols\n",
    "from create_prediction import optimize_one_par\n",
    "\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "pd.options.display.max_colwidth = 100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read_and_prepare_data  read from file\n"
     ]
    }
   ],
   "source": [
    "mainpath, path = set_path('areadata')\n",
    "stat, post, kunta_stat, vaalidata = read_and_prepare_data(path)\n",
    "numeric_features, categorical_features = selected_cols(largeset=False, parties=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set values to feature engineering attributes\n",
    "\n",
    "test_size = 0.3\n",
    "metric = 'rmse'\n",
    "Skfold=False\n",
    "Verbose = False\n",
    "testing=None\n",
    "scaled = False\n",
    "\n",
    "initial_params = {\n",
    "        #Initial xgboost parameters to be automatically tuned\n",
    "        'objective':'reg:squarederror',\n",
    "        'booster' : 'gbtree',\n",
    "        'eval_metric' : metric,\n",
    "        'seed' : 42,\n",
    "        }\n",
    "\n",
    "filename = 'party_xgboost_vote_lkm_'\n",
    "\n",
    "list_of_parties = ['VIHR', 'KOK', 'SDP', 'KD', 'KESK', 'RKP', 'PS', 'VAS']\n",
    "selected_parties = list_of_parties\n",
    "included_col_start = 'Ääniosuus '\n",
    "all_included_columns = create_target_columns(list_of_parties, included_col_start)\n",
    "\n",
    "target_col_start = 'Ääniosuus '\n",
    "target = create_target_columns(selected_parties, target_col_start)\n",
    "\n",
    "\n",
    "list_of_parties = ['VIHR', 'KOK', 'SDP', 'KD', 'KESK', 'RKP', 'PS', 'VAS']\n",
    "\n",
    "target_col_start = 'Äänet yhteensä lkm '\n",
    "target = create_target_columns(list_of_parties, target_col_start)\n",
    "\n",
    "included_col_start = 'Ääniosuus '\n",
    "all_included_columns = create_target_columns(list_of_parties, included_col_start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Äänet yhteensä lkm VIHR\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm VIHR_model.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [01:32,  1.56s/it]                                                        \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm VIHR_shap.pkl\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm VIHR_features.pkl\n",
      "--- 687.3985805511475 seconds ---\n",
      "\n",
      "Äänet yhteensä lkm KOK\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm KOK_model.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [01:29,  1.52s/it]                                                        \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm KOK_shap.pkl\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm KOK_features.pkl\n",
      "--- 643.4323878288269 seconds ---\n",
      "\n",
      "Äänet yhteensä lkm SDP\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm SDP_model.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [01:29,  1.52s/it]                                                        \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm SDP_shap.pkl\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm SDP_features.pkl\n",
      "--- 757.5791866779327 seconds ---\n",
      "\n",
      "Äänet yhteensä lkm KD\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm KD_model.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [01:28,  1.50s/it]                                                        \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm KD_shap.pkl\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm KD_features.pkl\n",
      "--- 638.4097435474396 seconds ---\n",
      "\n",
      "Äänet yhteensä lkm KESK\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm KESK_model.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [01:25,  1.45s/it]                                                        \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm KESK_shap.pkl\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm KESK_features.pkl\n",
      "--- 3094.0316088199615 seconds ---\n",
      "\n",
      "Äänet yhteensä lkm RKP\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm RKP_model.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [01:26,  1.46s/it]                                                        \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm RKP_shap.pkl\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm RKP_features.pkl\n",
      "--- 1068.0015141963959 seconds ---\n",
      "\n",
      "Äänet yhteensä lkm PS\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm PS_model.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [01:28,  1.50s/it]                                                        \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm PS_shap.pkl\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm PS_features.pkl\n",
      "--- 824.738751411438 seconds ---\n",
      "\n",
      "Äänet yhteensä lkm VAS\n",
      "Load model filename  C:\\Users\\risto\\Documents\\GitHub\\areadata\\data\\party_xgboost_vote_lkm_Äänet yhteensä lkm VAS_model.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [01:31,  1.55s/it]                                                        \n"
     ]
    }
   ],
   "source": [
    "data, X, y, test, X_test, y_test_pred, model_list, model_params, shap_data, features_dict = create_shap_values_via_xgboost(path,\n",
    "                               filename,\n",
    "                               stat,\n",
    "                               stat,\n",
    "                               target,\n",
    "                               initial_params,\n",
    "                               numeric_features=numeric_features,\n",
    "                               categorical_features=categorical_features,\n",
    "                               scaled=scaled,\n",
    "                               test_size=test_size,\n",
    "                               Skfold=Skfold,\n",
    "                               Verbose=False,\n",
    "                               testing=testing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = show_election_result(data, y_test_pred, target_col_start, vaalidata, selected_parties)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_value = 1.5\n",
    "columns = 12\n",
    "scaled = True\n",
    "included_columns = ['Miehet, 2018 (HE) osuudesta asukkaat']\n",
    "samples = 5\n",
    "show_cols = ['Postinumero','nimi', 'muncipality_name','Asukkaat yhteensä, 2018 (HE)', 'Suurin_puolue', 'Ennustettu Suurin_puolue']\n",
    "comp_col = 'Miehet, 2018 (HE) osuudesta asukkaat'\n",
    "compare = {}\n",
    "stats_data = {}\n",
    "k = {}\n",
    "k_compare = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "party = 'VIHR'\n",
    "compare_to, t = create_party_results(target_col_start, party, data, vaalidata, compare_value)\n",
    "compare[t] = compare_to\n",
    "stats_data[t], k[t], col_list = show_all_results(compare_to, data, X, y[t], shap_data[t], t, columns, all_included_columns, show_cols, comp_col, scaled=scaled, included_columns=included_columns, samples=samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "party = 'KOK'\n",
    "compare_to, t = create_party_results(target_col_start, party, data, vaalidata, compare_value)\n",
    "compare[t] = compare_to\n",
    "stats_data[t], k[t], col_list = show_all_results(compare_to, data, X, y[t], shap_data[t], t, columns, all_included_columns, show_cols, comp_col, scaled=scaled, included_columns=included_columns, samples=samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "party = 'SDP'\n",
    "compare_to, t = create_party_results(target_col_start, party, data, vaalidata, compare_value)\n",
    "compare[t] = compare_to\n",
    "stats_data[t], k[t], col_list = show_all_results(compare_to, data, X, y[t], shap_data[t], t, columns, all_included_columns, show_cols, comp_col, scaled=scaled, included_columns=included_columns, samples=samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "party ='KD'\n",
    "compare_to, t = create_party_results(target_col_start, party, data, vaalidata, compare_value)\n",
    "compare[t] = compare_to\n",
    "stats_data[t], k[t], col_list = show_all_results(compare_to, data, X, y[t], shap_data[t], t, columns, all_included_columns, show_cols, comp_col, scaled=scaled, included_columns=included_columns, samples=samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "party ='RKP'\n",
    "compare_to, t = create_party_results(target_col_start, party, data, vaalidata, compare_value)\n",
    "compare[t] = compare_to\n",
    "stats_data[t], k[t], col_list = show_all_results(compare_to, data, X, y[t], shap_data[t], t, columns, all_included_columns, show_cols, scaled=scaled, included_columns=included_columns, samples=samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "party ='KESK'\n",
    "compare_to, t = create_party_results(target_col_start, party, data, vaalidata, compare_value)\n",
    "compare[t] = compare_to\n",
    "stats_data[t], k[t], col_list = show_all_results(compare_to, data, X, y[t], shap_data[t], t, columns, all_included_columns, show_cols, comp_col, scaled=scaled, included_columns=included_columns, samples=samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "party ='PS'\n",
    "compare_to, t = create_party_results(target_col_start, party, data, vaalidata, compare_value)\n",
    "compare[t] = compare_to\n",
    "stats_data[t], k[t], col_list = show_all_results(compare_to, data, X, y[t], shap_data[t], t, columns, all_included_columns, show_cols, comp_col, scaled=scaled, included_columns=included_columns, samples=samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "party ='VAS'\n",
    "compare_to, t = create_party_results(target_col_start, party, data, vaalidata, compare_value)\n",
    "compare[t] = compare_to\n",
    "stats_data[t], k[t], col_list = show_all_results(compare_to, data, X, y[t], shap_data[t], t, columns, all_included_columns, show_cols, comp_col, scaled=scaled, included_columns=included_columns, samples=samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pnro ='01300'\n",
    "show_one_results(data, X, y, model_list, shap_data, target, pnro, columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pnro ='00260'\n",
    "show_one_results(data, X, y, model_list, shap_data, target, pnro, columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pnro ='00320'\n",
    "show_one_results(data, X, y, model_list, shap_data, target, pnro, columns)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gpa",
   "language": "python",
   "name": "gpa"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}