model-scraped.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3bd7b73c",
   "metadata": {},
   "source": [
    "# Stress Detection Models:\n",
    "#### RoBERTa Classifier Adapted on Scraped Data with LIWC/Empath "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dd5bead9",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_scr = r'domain-adaptation\\predictions\\train-roberta-scraped.csv'\n",
    "test_scr = r'domain-adaptation\\predictions\\test-roberta-scraped.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a86d55b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import phik\n",
    "import warnings\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import *\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import recall_score\n",
    "from empath import Empath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "df186a56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for reproducibility:\n",
    "seed = 1\n",
    "def set_seed(seed: int):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "set_seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9851f09",
   "metadata": {},
   "source": [
    "## Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "98c9518d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepares training and testing data for classification from given paths\n",
    "def prepare_data(dataset_path_train = None, dataset_path_test = None, features=None):\n",
    "    if (dataset_path_train is None or dataset_path_test is None):\n",
    "        raise RuntimeException(\"Error! Dataset must be provided\")\n",
    "    X_train = pd.read_csv(dataset_path_train)\n",
    "    X_test = pd.read_csv(dataset_path_test)\n",
    "    \n",
    "    y_train = X_train['label']\n",
    "    y_test = X_test['label']\n",
    "        \n",
    "    X_train = X_train.drop(columns=['label'])\n",
    "    X_test = X_test.drop(columns=['label'])\n",
    "    \n",
    "    \n",
    "    if features is not None:\n",
    "        X_train = X_train[features]\n",
    "        X_test = X_test[features]\n",
    "    \n",
    "    return X_train, y_train, X_test, y_test\n",
    "\n",
    "# Calculates Empath features and adds them to the given dataframe\n",
    "def empath(X):\n",
    "    rows = []\n",
    "    for t in X['text']:\n",
    "        empath = Empath().analyze(t, normalize=True)\n",
    "        rows.append(pd.DataFrame([empath]))\n",
    "    df = pd.concat(rows, ignore_index=True)\n",
    "    X = pd.concat([X,df], axis=1)\n",
    "    return X.drop(columns=['text'])\n",
    "\n",
    "# Returns only lexical features from a given dataframe\n",
    "def lex(X):\n",
    "    features = []\n",
    "    for feat in X.columns.values:\n",
    "        if 'lex' in feat:\n",
    "            features.append(feat)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "626aceb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "basic = ['roberta_prediction', 'text'] # for creating Empath features\n",
    "lexic = lex(pd.read_csv(train_scr))\n",
    "lexic.extend(['roberta_prediction']) # for getting LIWC features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4fafbb14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluates performance of a model given its predictions on the test set\n",
    "def evaluate(y_test, y_pred):\n",
    "    print(\"Accuracy: \", accuracy_score(y_test, y_pred))\n",
    "    print(\"Precision: \", precision_score(y_test, y_pred, zero_division=1))\n",
    "    print(\"Recall: \", recall_score(y_test, y_pred, zero_division=1))\n",
    "    print(\"F1: \", f1_score(y_test, y_pred, zero_division=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c56f7ccd",
   "metadata": {},
   "source": [
    "# LIWC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "612e422b",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train, X_test, y_test = prepare_data(dataset_path_train=train_scr, \\\n",
    "                                                dataset_path_test=test_scr, \\\n",
    "                                                features=lexic)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dd9e982",
   "metadata": {},
   "source": [
    "## Model training and testing\n",
    "### Logistic Regression\n",
    "#### Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "65f9053d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clf__C': 10, 'clf__penalty': 'l2', 'clf__solver': 'liblinear'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "warnings.filterwarnings('ignore')\n",
    "hp = {\n",
    "        'clf__solver': ['liblinear', 'newton-cg', 'lbfgs'],\n",
    "        'clf__penalty': ['l2', 'elasticnet'],\n",
    "        'clf__C': [0.1, 1, 10],\n",
    "}\n",
    "\n",
    "pipe = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', LogisticRegression(max_iter=1000000, random_state=seed))])\n",
    "\n",
    "gs_model = GridSearchCV(pipe, hp, n_jobs=-1, scoring='f1')\n",
    "gs_model.fit(X_train, y_train)\n",
    "gs_model.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bff6fb4",
   "metadata": {},
   "source": [
    "#### Testing model with best hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "749f4928",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8223776223776224\n",
      "Precision:  0.7880952380952381\n",
      "Recall:  0.8970189701897019\n",
      "F1:  0.8390367553865653\n"
     ]
    }
   ],
   "source": [
    "model = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', LogisticRegression(max_iter=1000000, C=10, penalty='l2', solver='liblinear', random_state=seed))])\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "evaluate(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "16cb9678",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          coefficient\n",
      "roberta_prediction           1.941677\n",
      "lex_liwc_function            0.364690\n",
      "lex_liwc_feel                0.340625\n",
      "lex_liwc_social             -0.294864\n",
      "lex_liwc_OtherP             -0.267551\n",
      "lex_liwc_swear               0.241359\n",
      "lex_liwc_auxverb            -0.233335\n",
      "lex_liwc_tentat              0.231733\n",
      "lex_liwc_Dic                 0.226625\n",
      "lex_liwc_Apostro             0.208648\n",
      "lex_liwc_ingest             -0.205350\n",
      "lex_liwc_Tone               -0.204162\n",
      "lex_liwc_percept            -0.199847\n",
      "lex_liwc_death               0.180748\n",
      "lex_liwc_Clout              -0.172249\n",
      "lex_liwc_AllPunc            -0.164019\n",
      "lex_liwc_informal           -0.159971\n",
      "lex_liwc_prep               -0.159533\n",
      "lex_liwc_certain             0.156124\n",
      "lex_liwc_ipron              -0.149457\n",
      "lex_liwc_anx                 0.144994\n",
      "lex_dal_avg_pleasantness    -0.139682\n",
      "lex_dal_min_pleasantness    -0.138055\n",
      "lex_liwc_focuspresent        0.137293\n",
      "lex_liwc_friend              0.137167\n",
      "lex_liwc_compare            -0.135415\n",
      "lex_dal_avg_activation       0.128303\n",
      "lex_liwc_body               -0.126934\n",
      "lex_liwc_article            -0.122307\n",
      "lex_liwc_differ             -0.122129\n",
      "lex_dal_min_activation      -0.121253\n",
      "lex_liwc_sexual              0.120385\n",
      "lex_liwc_cogproc            -0.120263\n",
      "lex_liwc_WPS                 0.117975\n",
      "lex_liwc_motion             -0.111271\n",
      "lex_liwc_we                  0.104971\n",
      "lex_liwc_Authentic          -0.103810\n",
      "lex_liwc_conj               -0.103694\n",
      "lex_dal_avg_imagery          0.103466\n",
      "lex_liwc_Colon              -0.100643\n",
      "lex_liwc_Parenth            -0.093079\n",
      "lex_liwc_focusfuture        -0.091207\n",
      "lex_liwc_Dash                0.089078\n",
      "lex_liwc_hear                0.088897\n",
      "lex_liwc_pronoun            -0.087698\n",
      "lex_liwc_you                 0.087499\n",
      "lex_liwc_family              0.086937\n",
      "lex_dal_max_imagery          0.083986\n",
      "lex_liwc_netspeak            0.080944\n",
      "lex_liwc_negemo             -0.078656\n",
      "lex_liwc_WC                  0.076394\n",
      "lex_liwc_bio                 0.074330\n",
      "lex_liwc_negate             -0.072317\n",
      "lex_liwc_assent              0.071857\n",
      "lex_liwc_QMark               0.070585\n",
      "lex_liwc_work                0.069130\n",
      "lex_liwc_nonflu              0.068068\n",
      "lex_liwc_they               -0.057349\n",
      "lex_liwc_sad                 0.056190\n",
      "lex_liwc_affect              0.055449\n",
      "lex_liwc_verb                0.054795\n",
      "lex_liwc_Period              0.053707\n",
      "lex_dal_max_pleasantness    -0.052816\n",
      "lex_liwc_Comma              -0.052203\n",
      "lex_liwc_anger               0.050907\n",
      "lex_liwc_quant              -0.049339\n",
      "lex_liwc_leisure            -0.047197\n",
      "lex_liwc_posemo              0.047102\n",
      "lex_dal_max_activation      -0.046948\n",
      "lex_liwc_focuspast          -0.046545\n",
      "lex_liwc_risk                0.045074\n",
      "lex_liwc_number              0.043994\n",
      "lex_liwc_home                0.041113\n",
      "lex_liwc_Quote              -0.040635\n",
      "lex_liwc_health              0.039492\n",
      "lex_liwc_power              -0.037993\n",
      "lex_liwc_reward             -0.037522\n",
      "lex_liwc_Exclam              0.035718\n",
      "lex_liwc_i                  -0.029505\n",
      "lex_liwc_SemiC               0.029209\n",
      "lex_liwc_money              -0.028723\n",
      "lex_liwc_female             -0.028202\n",
      "lex_liwc_Sixltr              0.027975\n",
      "lex_liwc_male               -0.027853\n",
      "lex_liwc_cause               0.026234\n",
      "lex_liwc_affiliation         0.026122\n",
      "lex_liwc_insight             0.026000\n",
      "lex_liwc_relig              -0.024612\n",
      "lex_liwc_ppron               0.024315\n",
      "lex_liwc_see                 0.018771\n",
      "lex_liwc_filler              0.018123\n",
      "lex_liwc_Analytic           -0.015684\n",
      "lex_liwc_adj                 0.013713\n",
      "lex_liwc_space              -0.013124\n",
      "lex_liwc_discrep            -0.012110\n",
      "lex_liwc_adverb             -0.011136\n",
      "lex_liwc_relativ            -0.010961\n",
      "lex_liwc_achieve             0.010131\n",
      "lex_liwc_shehe              -0.006966\n",
      "lex_liwc_interrog           -0.006667\n",
      "lex_liwc_drives             -0.005607\n",
      "lex_dal_min_imagery          0.005500\n",
      "lex_liwc_time                0.004088\n"
     ]
    }
   ],
   "source": [
    "cdf = pd.DataFrame(model.get_params()['clf'].coef_.transpose(), X_train.columns, columns=['coefficient'])\n",
    "print(cdf.sort_values(by='coefficient',  key=abs, ascending=False).to_string())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a18f3bf",
   "metadata": {},
   "source": [
    "### Support Vector Machine\n",
    "#### Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "65964267",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clf__C': 0.01, 'clf__gamma': 1000, 'clf__kernel': 'linear'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hp = {\n",
    "        'clf__kernel': ['linear'],\n",
    "        'clf__gamma': [1000, 100, 10, 1, 0.1, 0.01],\n",
    "        'clf__C': [0.01, 0.1, 1, 10],\n",
    "}\n",
    "\n",
    "pipe = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', SVC(random_state=seed))])\n",
    "\n",
    "gs_model = GridSearchCV(pipe, hp, n_jobs=-1, scoring='f1')\n",
    "gs_model.fit(X_train, y_train)\n",
    "gs_model.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "edd1dbb3",
   "metadata": {},
   "source": [
    "#### Testing model with best hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d63ffec1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8167832167832167\n",
      "Precision:  0.7741935483870968\n",
      "Recall:  0.9105691056910569\n",
      "F1:  0.8368617683686178\n"
     ]
    }
   ],
   "source": [
    "model = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', SVC(C=0.01, gamma=1000, kernel='linear', random_state=seed))])\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "evaluate(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d63c561",
   "metadata": {},
   "source": [
    "### Random Forest Classifier\n",
    "#### Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "10becab6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 24 candidates, totalling 120 fits\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'clf__criterion': 'gini',\n",
       " 'clf__max_features': 'auto',\n",
       " 'clf__min_samples_split': 4,\n",
       " 'clf__n_estimators': 200}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hp = {\n",
    "        'clf__n_estimators': [50, 200],\n",
    "        'clf__max_features': ['auto', 'sqrt', 'log2'],\n",
    "        'clf__criterion': ['gini', 'entropy'],\n",
    "        'clf__min_samples_split': [2, 4]\n",
    "}\n",
    "\n",
    "pipe = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf',  RandomForestClassifier(random_state=seed))])\n",
    "\n",
    "gs_model = GridSearchCV(pipe, hp, n_jobs=-1, scoring='f1', verbose=4)\n",
    "gs_model.fit(X_train, y_train)\n",
    "gs_model.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e557aed",
   "metadata": {},
   "source": [
    "#### Testing model with best hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "833a559b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8083916083916084\n",
      "Precision:  0.7697674418604651\n",
      "Recall:  0.8970189701897019\n",
      "F1:  0.8285356695869837\n"
     ]
    }
   ],
   "source": [
    "model = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', RandomForestClassifier(criterion='gini', max_features='auto', min_samples_split=4, n_estimators=200, random_state=seed))])\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "evaluate(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c194a181",
   "metadata": {},
   "source": [
    "# Empath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "027feae0",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train, X_test, y_test = prepare_data(dataset_path_train=train_scr, \\\n",
    "                                                dataset_path_test=test_scr, \\\n",
    "                                                features=basic)\n",
    "X_train = empath(X_train)\n",
    "X_test = empath(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a459898c",
   "metadata": {},
   "source": [
    "## Features Correlation Analysis and Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c4fe7514",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determines which features to drop based on the given threshold from phik correlation matrix\n",
    "def cutoff_features(corr_matrix, threshold):\n",
    "    upper_tri = corr_matrix.abs().where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool_))\n",
    "    to_drop = [column for column in upper_tri.columns if any(upper_tri[column] > float(threshold))]\n",
    "    return to_drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e63d88c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "corrs = X_train.phik_matrix(interval_cols=X_train.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0e60cc48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['health',\n",
       " 'banking',\n",
       " 'night',\n",
       " 'college',\n",
       " 'exasperation',\n",
       " 'reading',\n",
       " 'worship',\n",
       " 'eating',\n",
       " 'water',\n",
       " 'legend',\n",
       " 'neglect',\n",
       " 'swimming',\n",
       " 'love',\n",
       " 'sympathy',\n",
       " 'vehicle',\n",
       " 'disgust',\n",
       " 'sound',\n",
       " 'sailing',\n",
       " 'warmth',\n",
       " 'fun',\n",
       " 'joy',\n",
       " 'affection',\n",
       " 'lust',\n",
       " 'shame',\n",
       " 'anger',\n",
       " 'car',\n",
       " 'technology',\n",
       " 'power',\n",
       " 'white_collar_job',\n",
       " 'party',\n",
       " 'cleaning',\n",
       " 'competing',\n",
       " 'ocean',\n",
       " 'contentment',\n",
       " 'musical']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_drop = cutoff_features(corrs, 0.85)\n",
    "X_train = X_train.drop(columns=to_drop)\n",
    "X_test = X_test.drop(columns=to_drop)\n",
    "to_drop"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b0d0d18",
   "metadata": {},
   "source": [
    "## Model training and testing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f12118a",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac9ab26d",
   "metadata": {},
   "source": [
    "#### Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d26e5fce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clf__C': 0.1, 'clf__penalty': 'l2', 'clf__solver': 'liblinear'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "warnings.filterwarnings('ignore')\n",
    "hp = {\n",
    "        'clf__solver': ['liblinear', 'newton-cg', 'lbfgs'],\n",
    "        'clf__penalty': ['l2', 'elasticnet'],\n",
    "        'clf__C': [0.1, 1, 10],\n",
    "}\n",
    "\n",
    "pipe = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', LogisticRegression(max_iter=1000000, random_state=seed))])\n",
    "\n",
    "gs_model = GridSearchCV(pipe, hp, n_jobs=-1, scoring='f1')\n",
    "gs_model.fit(X_train, y_train)\n",
    "gs_model.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e374a4a",
   "metadata": {},
   "source": [
    "#### Testing model with best hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7e9be2ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8223776223776224\n",
      "Precision:  0.7880952380952381\n",
      "Recall:  0.8970189701897019\n",
      "F1:  0.8390367553865653\n"
     ]
    }
   ],
   "source": [
    "model = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', LogisticRegression(max_iter=1000000, C=0.1, penalty='l2', solver='liblinear', random_state=seed))])\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "evaluate(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0c659713",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                       coefficient\n",
      "roberta_prediction        2.221256\n",
      "beauty                   -0.231650\n",
      "domestic_work             0.229742\n",
      "childish                 -0.219444\n",
      "tourism                  -0.209199\n",
      "order                     0.199517\n",
      "science                   0.186974\n",
      "negative_emotion          0.183572\n",
      "rage                     -0.180759\n",
      "nervousness               0.159488\n",
      "heroic                   -0.156076\n",
      "toy                       0.151255\n",
      "tool                     -0.147449\n",
      "prison                   -0.143879\n",
      "cheerfulness              0.142850\n",
      "smell                    -0.141665\n",
      "irritability              0.141119\n",
      "wealthy                   0.130571\n",
      "money                    -0.130535\n",
      "torment                   0.130257\n",
      "dance                     0.130222\n",
      "listen                   -0.125616\n",
      "economics                 0.120398\n",
      "family                   -0.119987\n",
      "music                    -0.115232\n",
      "politeness               -0.108916\n",
      "restaurant               -0.102343\n",
      "violence                  0.101150\n",
      "vacation                  0.099485\n",
      "feminine                  0.099337\n",
      "liquid                    0.099275\n",
      "confusion                 0.098980\n",
      "kill                      0.097040\n",
      "computer                 -0.096910\n",
      "fabric                    0.096743\n",
      "divine                    0.092891\n",
      "journalism               -0.092824\n",
      "body                      0.092401\n",
      "healing                  -0.092218\n",
      "sadness                   0.091139\n",
      "writing                   0.090983\n",
      "medical_emergency         0.089889\n",
      "hiking                    0.089426\n",
      "envy                      0.089219\n",
      "law                       0.088532\n",
      "meeting                  -0.087537\n",
      "hearing                   0.085360\n",
      "clothing                 -0.084255\n",
      "terrorism                 0.082998\n",
      "breaking                 -0.079509\n",
      "play                     -0.079088\n",
      "stealing                 -0.077615\n",
      "internet                 -0.077561\n",
      "art                      -0.076269\n",
      "payment                  -0.075303\n",
      "rural                     0.074942\n",
      "weapon                    0.074795\n",
      "leisure                   0.073609\n",
      "achievement               0.073261\n",
      "speaking                 -0.073013\n",
      "optimism                 -0.071561\n",
      "strength                 -0.068776\n",
      "surprise                  0.068513\n",
      "home                     -0.068502\n",
      "farming                  -0.067725\n",
      "military                 -0.067144\n",
      "fire                     -0.067104\n",
      "real_estate              -0.066161\n",
      "suffering                 0.065159\n",
      "sleep                    -0.062777\n",
      "crime                     0.062218\n",
      "plant                     0.062137\n",
      "magic                    -0.059178\n",
      "furniture                 0.058814\n",
      "pain                     -0.057691\n",
      "anticipation              0.057262\n",
      "ugliness                  0.057131\n",
      "alcohol                  -0.056291\n",
      "communication            -0.055082\n",
      "air_travel                0.054189\n",
      "religion                 -0.053498\n",
      "swearing_terms            0.053440\n",
      "occupation               -0.053295\n",
      "positive_emotion         -0.051691\n",
      "negotiate                 0.049585\n",
      "blue_collar_job           0.048546\n",
      "hygiene                   0.048015\n",
      "dispute                   0.047269\n",
      "traveling                -0.047084\n",
      "disappointment            0.046775\n",
      "sexual                    0.044551\n",
      "business                  0.044481\n",
      "monster                  -0.044455\n",
      "messaging                -0.043872\n",
      "morning                  -0.043812\n",
      "urban                    -0.043581\n",
      "fight                    -0.042726\n",
      "deception                -0.041977\n",
      "fashion                  -0.041679\n",
      "death                     0.040081\n",
      "animal                    0.038311\n",
      "medieval                  0.037928\n",
      "masculine                -0.036627\n",
      "zest                      0.034992\n",
      "cold                     -0.034369\n",
      "appearance               -0.032894\n",
      "dominant_personality     -0.032845\n",
      "hate                      0.032756\n",
      "phone                     0.031381\n",
      "ridicule                 -0.030799\n",
      "horror                    0.030629\n",
      "philosophy               -0.030614\n",
      "pet                       0.030424\n",
      "weakness                  0.029764\n",
      "government               -0.029400\n",
      "friends                   0.028976\n",
      "emotional                -0.028917\n",
      "school                   -0.028695\n",
      "beach                    -0.027682\n",
      "sports                   -0.025467\n",
      "ancient                  -0.025453\n",
      "poor                     -0.025164\n",
      "weather                  -0.025050\n",
      "politics                 -0.024288\n",
      "hipster                  -0.024138\n",
      "fear                      0.024047\n",
      "wedding                   0.023648\n",
      "movement                  0.022693\n",
      "work                      0.019244\n",
      "driving                   0.018662\n",
      "superhero                 0.018229\n",
      "attractive               -0.017103\n",
      "shape_and_size           -0.015516\n",
      "timidity                  0.014886\n",
      "valuable                  0.014123\n",
      "office                   -0.013981\n",
      "shopping                  0.013834\n",
      "aggression               -0.012432\n",
      "injury                   -0.012204\n",
      "trust                     0.010726\n",
      "help                     -0.010430\n",
      "noise                     0.009658\n",
      "giving                   -0.009371\n",
      "exotic                    0.009237\n",
      "programming               0.008817\n",
      "royalty                  -0.008772\n",
      "leader                   -0.008160\n",
      "anonymity                -0.007500\n",
      "ship                     -0.006669\n",
      "exercise                 -0.004554\n",
      "independence              0.004458\n",
      "celebration               0.004401\n",
      "gain                     -0.003545\n",
      "dominant_heirarchical    -0.002226\n",
      "children                  0.001800\n",
      "war                      -0.001624\n",
      "pride                    -0.001413\n",
      "youth                    -0.001390\n",
      "cooking                  -0.000836\n",
      "social_media             -0.000258\n"
     ]
    }
   ],
   "source": [
    "cdf = pd.DataFrame(model.get_params()['clf'].coef_.transpose(), X_train.columns, columns=['coefficient'])\n",
    "print(cdf.sort_values(by='coefficient',  key=abs, ascending=False).to_string())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41a5e413",
   "metadata": {},
   "source": [
    "### Support Vector Machine"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd51b65c",
   "metadata": {},
   "source": [
    "#### Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "61ff13ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clf__C': 0.01, 'clf__gamma': 1000, 'clf__kernel': 'linear'}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hp = {\n",
    "        'clf__kernel': ['linear'],\n",
    "        'clf__gamma': [1000, 100, 10, 1, 0.1, 0.01],\n",
    "        'clf__C': [0.01, 0.1, 1, 10],\n",
    "}\n",
    "\n",
    "pipe = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', SVC(random_state=seed))])\n",
    "\n",
    "gs_model = GridSearchCV(pipe, hp, n_jobs=-1, scoring='f1')\n",
    "gs_model.fit(X_train, y_train)\n",
    "gs_model.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "817c9e78",
   "metadata": {},
   "source": [
    "#### Testing model with best hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "43343f86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8167832167832167\n",
      "Precision:  0.7741935483870968\n",
      "Recall:  0.9105691056910569\n",
      "F1:  0.8368617683686178\n"
     ]
    }
   ],
   "source": [
    "model = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', SVC(C=0.01, gamma=1000, kernel='linear', random_state=seed))])\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "evaluate(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c8ade53",
   "metadata": {},
   "source": [
    "### Random Forest Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c67aa6f",
   "metadata": {},
   "source": [
    "#### Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6a115b1c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 24 candidates, totalling 120 fits\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'clf__criterion': 'entropy',\n",
       " 'clf__max_features': 'auto',\n",
       " 'clf__min_samples_split': 4,\n",
       " 'clf__n_estimators': 200}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hp = {\n",
    "        'clf__n_estimators': [50, 200],\n",
    "        'clf__max_features': ['auto', 'sqrt', 'log2'],\n",
    "        'clf__criterion': ['gini', 'entropy'],\n",
    "        'clf__min_samples_split': [2, 4]\n",
    "}\n",
    "\n",
    "pipe = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf',  RandomForestClassifier(random_state=seed))])\n",
    "\n",
    "gs_model = GridSearchCV(pipe, hp, n_jobs=-1, scoring='f1', verbose=4)\n",
    "gs_model.fit(X_train, y_train)\n",
    "gs_model.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a373f0c",
   "metadata": {},
   "source": [
    "#### Testing model with best hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "fb791c2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.813986013986014\n",
      "Precision:  0.7757009345794392\n",
      "Recall:  0.8997289972899729\n",
      "F1:  0.8331242158092848\n"
     ]
    }
   ],
   "source": [
    "model = Pipeline([\n",
    "        ('scale', StandardScaler()),\n",
    "        ('clf', RandomForestClassifier(criterion='entropy', max_features='auto', min_samples_split=4, n_estimators=200, random_state=seed))])\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "evaluate(y_test, y_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}