code/4_train_classifiers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train classification models for misinformation detection\n",
    "\n",
    "Trains k-Nearest Neighbors, Random Forest, Decision Tree, Multinomial Naive Bayes, Logistic Regression, Support Vector Machine, and Multi-Layer Perceptron. Uses 10-fold cross-validation to select the best model.\n",
    "\n",
    "## Inputs\n",
    "list of validated labeled tweets with filename format 'myth_{myth_name}_sample_{sample_size}_{date}-labeled.csv'. If negative cases are needed for a focal myth, may also borrow from labeled positives for other myths (very unlikely a tweet contains both myths)\n",
    "\n",
    "## Outputs\n",
    "ML models and vectorizers (with TF-IDF weighting)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "######################################################\n",
    "# Import libraries\n",
    "######################################################\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import regex as re\n",
    "from collections import Counter\n",
    "from datetime import date\n",
    "from tqdm import tqdm\n",
    "import emoji\n",
    "from os.path import join\n",
    "\n",
    "import nltk\n",
    "from nltk import word_tokenize, sent_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "\n",
    "stemmer = WordNetLemmatizer()\n",
    "\n",
    "from pandas.plotting import scatter_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import joblib\n",
    "import csv\n",
    "\n",
    "from sklearn.datasets import load_files\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import cross_val_score, cross_val_predict, train_test_split, KFold\n",
    "\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load & inspect data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define filepaths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "thisday = date.today().strftime(\"%m%d%y\")\n",
    "DATA_DIR = str() # set path to check for data\n",
    "\n",
    "# May be multiple labeled data files for focal myth(s)\n",
    "myth_fps = [fp for fp in glob.glob(DATA_DIR) if\n",
    "           (fp.endswith('labeled.csv'))\n",
    "          ]\n",
    "# negative cases\n",
    "neg_myth_fps = [fp for fp in glob.glob(join(DATA_DIR, 'negative_cases/*')) if\n",
    "               (fp.endswith('labeled.csv'))\n",
    "               ]\n",
    "\n",
    "# specify shorthand for focal myth\n",
    "# use different, specific naming for multiple myths (e.g., myth1, myth2)\n",
    "MYTH = str()\n",
    "myth_vec_fp = f'{MYTH}_VEC_{str(thisday)}.joblib'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load & rename data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For multiple files holding labeled data, load each one and add to empty DF\n",
    "\n",
    "# focal myth name defined above\n",
    "myth_df = pd.DataFrame() # initialize\n",
    "for fp in myth_fps:\n",
    "    temp = pd.read_csv(fp, low_memory=False) # read file\n",
    "    myth_df = pd.concat([myth_df, temp]) # add positive tweets to main DF\n",
    "#add negative tweets\n",
    "for fp in neg_myth_fps: \n",
    "    temp = pd.read_csv(fp, low_memory=False) # read file\n",
    "    temp = temp[temp.myth_score == 1.0] # keep only negative tweets with score of 1.0 for other myth\n",
    "    temp = temp.assign(myth_score = 0, myth_supports_score = 0) # change myth scores to zero\n",
    "    temp = temp.assign(is_myth = \"no\", is_myth_supports = \"no\") # change myth labels to \"no\"\n",
    "    myth_df = pd.concat([myth_df, temp]) # add negative tweets to main df\n",
    "    \n",
    "# Rename each DF's myth columns for clarity\n",
    "myth_df = myth_df.reset_index(drop=True).rename(\n",
    "    copy = False, columns = \n",
    "    {'is_myth': f'{MYTH}_is_myth', f'{MYTH}_score': f'{MYTH}_myth_score', \n",
    "     'is_myth_supports': f'{MYTH}_is_myth_supports', 'myth_supports_score': f'{MYTH}_myth_supports_score'})\n",
    "\n",
    "myth_df.tail(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summarize the numerical data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Look at the number of instances of each is_myth class distribution\n",
    "print(myth_df.groupby(f'{MYTH}_is_myth').size())\n",
    "print()\n",
    "\n",
    "# Look at the number of instances of each is_myth_supports class distribution\n",
    "print(myth_df.groupby(f'{MYTH}_is_myth_supports').size())\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess tweet text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tweet Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_tweets(tweet):\n",
    "    '''\n",
    "    Preprocesses raw text of a tweet, skipping any retweets. \n",
    "    Steps: lower-casing; removing punctuation, newlines, URLs, usernames, and emojis;\n",
    "    stripping whitespace, replacing hashtags, and finally, lemmatization.\n",
    "    \n",
    "    args:\n",
    "        tweet: raw text of a tweet\n",
    "    \n",
    "    returns:\n",
    "        string: cleaned tweet text\n",
    "    '''\n",
    "    \n",
    "    # Skip retweets and non-strings\n",
    "    retweet_pattern = r'^RT\\s+' # recognize retweets by starting with 'RT'\n",
    "    if not isinstance(tweet, str) or re.search(retweet_pattern, tweet):\n",
    "        return ''\n",
    "    \n",
    "    # Convert to lowercase\n",
    "    tweet = tweet.lower()\n",
    "    \n",
    "    # Remove punctuation with regex: match all punctuation (\\p{P}) and symbols (\\p{S}), \n",
    "    # then check that it is not a wasn't a hashtag or @-symbol using a negative look-behind.\n",
    "    punc_pattern = r\"[\\p{P}\\p{S}](?<![@#\\'\\/:])\"\n",
    "    tweet = re.sub(punc_pattern, \"\", tweet)   \n",
    "    \n",
    "    # Repair hashtag and remove newline character\n",
    "    # from text_helpers.tweet_text_cleanup\n",
    "    tweet = tweet.replace(\"# \", \"#\")\n",
    "    tweet = tweet.replace(\"\\n\", \" \")\n",
    "    \n",
    "    # remove URLs and @mentions\n",
    "    # Simple regular expression to match URLs starting with `https` or `http`\n",
    "    # More complex regex an be found here: https://mathiasbynens.be/demo/url-regex\n",
    "    url_regex = r\"https?://\\S*\"\n",
    "    # Regex to match mentions\n",
    "    mention_regex = r\"@\\S*\"\n",
    "    tweet = re.sub(url_regex, \"\", tweet)\n",
    "    tweet = re.sub(mention_regex, \"\", tweet)\n",
    "        \n",
    "    # Remove additional white spaces\n",
    "    whitespace_pattern = r'\\s+'\n",
    "    tweet = re.sub(whitespace_pattern, ' ', tweet) # strip whitespaces in between words\n",
    "    tweet = tweet.strip() # strip whitespaces at start & end\n",
    "    \n",
    "    # Replace #word with word\n",
    "    tweet = re.sub(r'#([^\\s]+)', r'\\1', tweet)\n",
    "    \n",
    "    # Remove emojis\n",
    "    tweet = emoji.get_emoji_regexp().sub(u'', tweet)\n",
    "    \n",
    "    # Lemmatization\n",
    "    tweet = tweet.split()\n",
    "    tweet = ' '.join([stemmer.lemmatize(word) for word in tweet])\n",
    "    \n",
    "    return tweet\n",
    "\n",
    "\n",
    "myth_df['text_cleaned'] = myth_df['text'].apply(lambda x: process_tweets(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorize texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use TFIDF weighted DTM because does better overall than unweighted\n",
    "myth_vectorizer = TfidfVectorizer(max_features=10000, min_df=1, max_df=0.8, stop_words=stopwords.words('english')) # TFIDF\n",
    "\n",
    "# creates sparse DTM X\n",
    "# use X.toarray() to get with zero representation\n",
    "\n",
    "myth_tweets = [] # make empty list to add tweets to\n",
    "myth_df['text_cleaned'].apply(lambda x: myth_tweets.append(x)) # add tweet from each row of DF\n",
    "\n",
    "X_myth = myth_vectorizer.fit_transform(myth_tweets)\n",
    "\n",
    "# save vectorizers\n",
    "joblib.dump(myth_vectorizer, open(myth_vec_fp, \"wb\"))\n",
    "\n",
    "\n",
    "print('Number of features in vectorizer (total vocabulary):', len(myth_vectorizer.get_feature_names()))\n",
    "print()\n",
    "\n",
    "print(myth_vectorizer.get_feature_names()[::100]) # get every 100th word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Specify the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_is_myth = myth_df[['text_cleaned',f'{MYTH}_is_myth']].copy()\n",
    "\n",
    "df_is_myth_supports = myth_df[['text_cleaned',f'{MYTH}_is_myth_supports']].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert No/Yes to [0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def no_yes_convert(convert_df, column_name, has_unsure = False):\n",
    "    '''\n",
    "    args\n",
    "        convert_df: df containing column to convert\n",
    "        column_name: column to convert from 'yes','no','unsure' to float. Scoring scheme:\n",
    "            no: 0\n",
    "            unsure: 0.5\n",
    "            yes: 1\n",
    "        has_unsure: boolean, indicates whether convert_df has 'unsure' in column_name\n",
    "    '''\n",
    "    \n",
    "    # Already converted to float\n",
    "    if convert_df[column_name].dtype == 'float64':\n",
    "        return convert_df\n",
    "    \n",
    "    new_df = convert_df.loc[:, convert_df.columns != column_name]\n",
    "    \n",
    "    for num in range(0,len(new_df)):\n",
    "        row_index = new_df.index[num]\n",
    "        \n",
    "        if convert_df.loc[num,column_name] == 'no' or convert_df.loc[num,column_name] == 'unsure':\n",
    "            new_df.loc[row_index,column_name] = 0.0\n",
    "            \n",
    "        elif convert_df.loc[num,column_name] == 'yes':\n",
    "            new_df.loc[row_index,column_name] = 1.0\n",
    "            \n",
    "#         elif has_unsure == True and convert_df.loc[num,column_name] == 'unsure':\n",
    "#             new_df.loc[row_index,column_name] = 0.5\n",
    "            \n",
    "    return new_df\n",
    "\n",
    "df_is_myth = no_yes_convert(df_is_myth,f'{MYTH}_is_myth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup for modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Balance x_train, y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resample_data(X_train, Y_train, undersample = False, sampling_ratio = 0.5):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        X_train: X training data\n",
    "        Y_train: Y training data\n",
    "        undersample: boolean for over or undersampling\n",
    "        sampling_ratio: ratio of minority to total\n",
    "        \n",
    "        archived/not used:\n",
    "        sampling_strategy: strategy for resampled distribution\n",
    "            if oversample: 'majority' makes minority = to majority\n",
    "            if undersample: 'minority' makes majority = to minority\n",
    "            \n",
    "    Returns:\n",
    "        X_balanced: predictors at balanced ratio\n",
    "        Y_balanced: outcomes at balanced ratio\n",
    "    \"\"\"\n",
    "    \n",
    "    if undersample == True:\n",
    "        #TODO: Implement real_sampling_ratio for undersample\n",
    "        undersample = RandomUnderSampler(sampling_strategy=sampling_ratio)\n",
    "        X_balanced, Y_balanced = undersample.fit_resample(X_train, Y_train)\n",
    "    else:\n",
    "        # real_sampling_ratio is the ratio of the minority to majority, keeping majority constant\n",
    "        real_sampling_ratio = ((sampling_ratio * len(Counter(Y_train)))/(1-sampling_ratio))/len(Counter(Y_train))\n",
    "        oversample = RandomOverSampler(sampling_strategy=real_sampling_ratio)\n",
    "        X_balanced, Y_balanced = oversample.fit_resample(X_train, Y_train)\n",
    "    \n",
    "    print(f'Y_train: {Counter(Y_train)}\\nY_resample: {Counter(Y_balanced)}')\n",
    "    \n",
    "    return X_balanced, Y_balanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_predictions(text, vectorizer_model, class_model):\n",
    "    '''\n",
    "    Predicts the label for an input text using a given model trained to classify the texts. \n",
    "    Uses vectorizer_model to restrict the vocab of the input text so it's consistent with vocab in class_model (avoids errors).\n",
    "    \n",
    "    Args:\n",
    "        text: preprocessed text in format of list of sentences, each a str or list of tokens\n",
    "        vectorizer_model: fitted text vectorizer\n",
    "        class_model: trained classification model\n",
    "    Returns:\n",
    "        label: label for text predicted by model, false for tie\n",
    "        prob: probability for label\n",
    "    '''\n",
    "    \n",
    "    X = vectorizer_model.transform(text) # create TF-IDF-weighted DTM from text\n",
    "    try:\n",
    "        probabilities = class_model.predict_proba(X)\n",
    "    except: \n",
    "        return\n",
    "    \n",
    "    label = 'no'\n",
    "    prob_no = probabilities[0][0]\n",
    "    prob_yes = probabilities[0][1]\n",
    "    \n",
    "    # predicted label is one with greater probability\n",
    "    if probabilities[0][0] < probabilities[0][1]:\n",
    "        label = 'yes'\n",
    "        \n",
    "    return label, prob_yes, prob_no"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup 10-fold cross validation for model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define test options for k-fold CV\n",
    "num_folds = 10 \n",
    "seed = 3\n",
    "scoring='f1_weighted' # set scoring metric (not used here)\n",
    "\n",
    "def show_kfold_output(models,  \n",
    "                      X, \n",
    "                      Y, \n",
    "                      df, \n",
    "                      text_col, \n",
    "                      vectorizer, \n",
    "                      num_folds = num_folds, \n",
    "                      random_state = seed, \n",
    "                      shuffle = True):\n",
    "    '''\n",
    "    Estimates the accuracy of different model algorithms, adds results to a results array and returns.\n",
    "    Prints the accuracy results: averages and std.\n",
    "    Uses cross_val_predict, which unlike cross_val_score cannot define scoring option/evaluation metric.\n",
    "    \n",
    "    Args:\n",
    "        models: list of (name, model) tuples\n",
    "        X: predictors\n",
    "        Y: outcomes\n",
    "        num_folds: Split data randomly into num_folds parts: (num_folds-1) for training, 1 for scoring\n",
    "        random_state: seed\n",
    "        shuffle: \n",
    "    \n",
    "    Returns:\n",
    "        results: list of model results\n",
    "        names: list of model names (matches results)\n",
    "        \n",
    "    Source: \n",
    "        https://stackoverflow.com/questions/40057049/using-confusion-matrix-as-scoring-metric-in-cross-validation-in-scikit-learn\n",
    "    '''\n",
    "    \n",
    "    results = []\n",
    "    names = []\n",
    "    \n",
    "    for name, model in models:\n",
    "        # Print name of model\n",
    "        print(f'{name}:')\n",
    "        print()\n",
    "        \n",
    "        # Setup model options\n",
    "        kfold = KFold(\n",
    "            n_splits=num_folds, \n",
    "            random_state=seed, \n",
    "            shuffle=True)\n",
    "        \n",
    "        # Get kfold results\n",
    "        cv_results = cross_val_predict(\n",
    "            model.fit(X, Y), \n",
    "            X, \n",
    "            Y, \n",
    "            cv=kfold, \n",
    "            #scoring=scoring, \n",
    "            n_jobs=-1) # use all cores = faster\n",
    "        \n",
    "        # Add results and name of each algorithm to the model array\n",
    "        results.append(cv_results)\n",
    "        names.append(name)\n",
    "        \n",
    "        # Validation step: Predict class probabilities in labeled data, compare to actual labels\n",
    "        try:\n",
    "            tqdm.pandas(desc=\"Computing predictions\")\n",
    "            df[['prediction', 'prediction_prob_yes', 'prediction_prob_no']] = df[text_col].progress_apply(\n",
    "            lambda text: pd.Series(compute_predictions([sent for sent in sent_tokenize(text)], vectorizer, model)))\n",
    "            print(\"Distribution of predicted labels:\\n\", df['prediction'].value_counts()) # show predicted distribution, compare to labeled distribution\n",
    "            print()\n",
    "        except: \n",
    "            pass\n",
    "        \n",
    "        # Print results\n",
    "        print(f'Mean (std):\\t {round(cv_results.mean(),4)} ({round(cv_results.std(),4)})')\n",
    "        print(f'Accuracy:\\t', {round(accuracy_score(Y, cv_results)), 4})\n",
    "        print()\n",
    "        print('Confusion matrix:\\n', confusion_matrix(Y, cv_results))\n",
    "        print()\n",
    "        print('Report:\\n', classification_report(Y, cv_results))\n",
    "        print()\n",
    "        \n",
    "    # Return arrays\n",
    "    return results, names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare training and validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate training and final validation data set. First remove class\n",
    "# label from data (X). Setup target class (Y)\n",
    "# Then make the validation set 10% of the entire\n",
    "# set of labeled data (X_validate, Y_validate)\n",
    "\n",
    "valueArray = df_is_myth.values\n",
    "Y = valueArray[:,1]\n",
    "Y = Y.astype('float')\n",
    "test_size = 0.2\n",
    "seed = 3\n",
    "X_train, X_validate, Y_train, Y_validate = train_test_split(\n",
    "    X_myth, \n",
    "    Y, \n",
    "    test_size=test_size, \n",
    "    random_state=seed)\n",
    "\n",
    "print(f'Y_train Distribution: {Counter(Y_train).most_common()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "######################################################\n",
    "# Oversample to desirable ratio\n",
    "######################################################\n",
    "\n",
    "sampling_ratio = 0.5 # ratio of minority to total cases\n",
    "undersample = False # whether to undersample or oversample\n",
    "\n",
    "X_balanced, Y_balanced = resample_data(\n",
    "    X_myth, #X_train, \n",
    "    Y, #Y_train, \n",
    "    undersample=undersample, \n",
    "    sampling_ratio=sampling_ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10-Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Use different algorithms to build models\n",
    "models = []\n",
    "models.append(('K-Nearest Neighbors (KNN)', KNeighborsClassifier()))\n",
    "models.append(('Random Forest (RF)', RandomForestClassifier(n_estimators=40, random_state=seed)))\n",
    "models.append(('Decision Tree (DT)', DecisionTreeClassifier(random_state=seed)))\n",
    "models.append(('Multinomial Naive Bayes (MNB)', MultinomialNB()))\n",
    "models.append(('Logistic Regression (LR)', LogisticRegression(random_state=seed)))\n",
    "models.append(('Support Vector Machine (SVM)', SVC(gamma='auto')))\n",
    "models.append(('Multi-Layer Perceptron (MLP)', MLPClassifier(max_iter=100, activation='relu')))\n",
    "\n",
    "# Baseline: distribution of labeled data\n",
    "print(f'Distribution of labeled disinfectants tweets: {Counter(Y_balanced).most_common()}')\n",
    "print()\n",
    "\n",
    "# Evaluate algorithms using 10-fold cross validation\n",
    "results, names = show_kfold_output(models=models, \n",
    "                                   X=X_balanced,\n",
    "                                   Y=Y_balanced, \n",
    "                                   df=myth_df, \n",
    "                                   text_col='text_cleaned', \n",
    "                                   vectorizer=myth_vectorizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save best model\n",
    "\n",
    "# Model file output format\n",
    "best_model_suffix = \"LR\" # this model has best accuracy\n",
    "thisday = date.today().strftime(\"%m%d%y\")\n",
    "\n",
    "MYTH_NAME_mod_fp = f'classifier_{MYTH}_{str(best_model_suffix)}_{str(thisday)}.joblib'\n",
    "\n",
    "best_model = LogisticRegression(random_state=seed).fit(X_train, Y_train)\n",
    "\n",
    "joblib.dump(best_model, MYTH_NAME_mod_fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspect featured keywords (Random Forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get feature names\n",
    "# feature_names = [f\"feature {i}\" for i in range(X_balanced.shape[1])]\n",
    "feature_names = myth_vectorizer.get_feature_names_out()\n",
    "rf_model = RandomForestClassifier(n_estimators=40, random_state=seed).fit(X_balanced, Y_balanced)\n",
    "\n",
    "# Get feature importances from the randome forest model\n",
    "importances = rf_model.feature_importances_\n",
    "std = np.std(\n",
    "    [tree.feature_importances_ for tree in rf_model.estimators_], axis=0)\n",
    "\n",
    "print(type(feature_names))\n",
    "print(type(X_balanced))\n",
    "\n",
    "print(X_balanced.shape)\n",
    "print(feature_names.shape)\n",
    "\n",
    "# Build dataframe\n",
    "forest_importances = pd.DataFrame()\n",
    "forest_importances[\"importances\"] = importances\n",
    "forest_importances[\"std\"] = std\n",
    "forest_importances.index = feature_names\n",
    "forest_importances = forest_importances.sort_values(\n",
    "    by=[\"importances\", \"std\"], ascending=False)\n",
    "\n",
    "# See top k important features\n",
    "top_k = 10\n",
    "top_k_forest_importances = forest_importances.head(top_k)\n",
    "print(list(top_k_forest_importances.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "top_k_forest_importances[\"importances\"].plot.bar(\n",
    "    yerr=top_k_forest_importances[\"std\"], ax=ax)\n",
    "ax.set_title(\"Feature importances\")\n",
    "ax.set_ylabel(\"Mean decrease in impurity\")\n",
    "fig.tight_layout()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}