Part3_BoW_Model_Search.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e0b065b8-4e3b-40b6-ad48-a5caf61702ea",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Imports/setting up directories\n",
    "Note: I'm using the dark grid style for seaborn (my prefered plotting library)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2e95b35c-1945-424c-9b28-dab9401ab574",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "                <script type=\"application/javascript\" id=\"jupyter_black\">\n",
       "                (function() {\n",
       "                    if (window.IPython === undefined) {\n",
       "                        return\n",
       "                    }\n",
       "                    var msg = \"WARNING: it looks like you might have loaded \" +\n",
       "                        \"jupyter_black in a non-lab notebook with \" +\n",
       "                        \"`is_lab=True`. Please double check, and if \" +\n",
       "                        \"loading with `%load_ext` please review the README!\"\n",
       "                    console.log(msg)\n",
       "                    alert(msg)\n",
       "                })()\n",
       "                </script>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext jupyter_black\n",
    "# Utilities/Misc\n",
    "import os\n",
    "from pathlib import Path\n",
    "import pickle as pkl\n",
    "import warnings\n",
    "\n",
    "# Data handling\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Dimension reduction\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "# ML Modeling/Optimization\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from skopt import BayesSearchCV\n",
    "\n",
    "# Scoring\n",
    "from sklearn.metrics import (\n",
    "    f1_score,\n",
    "    accuracy_score,\n",
    "    balanced_accuracy_score,\n",
    "    roc_auc_score,\n",
    ")\n",
    "\n",
    "# NLP-specific tools\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "\n",
    "# Custom modules\n",
    "from src.preprocessing import EmbeddingAwareTokenizer, do_nothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3d44aea4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Paths and directories\n",
    "CWD = Path(os.getcwd())\n",
    "DATA_DIR = CWD / \"data\"\n",
    "CORPUS_DIR = DATA_DIR / \"corpus_files\"\n",
    "OBJ_DIR = DATA_DIR / \"objects\"\n",
    "MODEL_SEARCH = DATA_DIR / \"model_search_results\"\n",
    "MODEL_SEARCH.mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4b301aee-fd1c-42f6-8e56-36843367d43b",
   "metadata": {},
   "source": [
    "# Load/Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "316812fa-d199-4b9a-aa6b-7adc9dec7144",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3201 entries, 0 to 3200\n",
      "Data columns (total 56 columns):\n",
      " #   Column                     Non-Null Count  Dtype         \n",
      "---  ------                     --------------  -----         \n",
      " 0   UID                        3201 non-null   int16         \n",
      " 1   clean_text                 3201 non-null   object        \n",
      " 2   num_sections               3201 non-null   int64         \n",
      " 3   credit_text                3201 non-null   object        \n",
      " 4   num_credit_sections        3201 non-null   int64         \n",
      " 5   clean_word_tokens          3201 non-null   object        \n",
      " 6   credit_word_tokens         3201 non-null   object        \n",
      " 7   link                       3201 non-null   object        \n",
      " 8   submission_author          3189 non-null   object        \n",
      " 9   submission_id              3201 non-null   object        \n",
      " 10  submission_title           3201 non-null   object        \n",
      " 11  subreddit                  3201 non-null   category      \n",
      " 12  submission_flair           3201 non-null   category      \n",
      " 13  submission_score           3201 non-null   Int32         \n",
      " 14  submission_upvote_ratio    3198 non-null   float32       \n",
      " 15  submission_date            1983 non-null   datetime64[ns]\n",
      " 16  comment_author             2597 non-null   object        \n",
      " 17  comment_id                 2597 non-null   object        \n",
      " 18  comment_score              2597 non-null   Int32         \n",
      " 19  comment_body               2597 non-null   object        \n",
      " 20  comment_date               1849 non-null   datetime64[ns]\n",
      " 21  src_url                    3201 non-null   object        \n",
      " 22  manually_reviewed          3201 non-null   boolean       \n",
      " 23  related_link               1445 non-null   boolean       \n",
      " 24  raw_markdown               3201 non-null   object        \n",
      " 25  clean_markdown             3201 non-null   object        \n",
      " 26  lang                       3201 non-null   object        \n",
      " 27  raw_md_char_count          3201 non-null   Int32         \n",
      " 28  raw_md_hastag_runs         3201 non-null   Int32         \n",
      " 29  raw_md_asterisk_runs       3201 non-null   Int32         \n",
      " 30  clean_md_char_count        3201 non-null   Int32         \n",
      " 31  clean_md_hastag_runs       3201 non-null   Int32         \n",
      " 32  clean_md_asterisk_runs     3201 non-null   Int32         \n",
      " 33  clean_md_stopword_count    3201 non-null   Int32         \n",
      " 34  clean_md_word_count        3201 non-null   Int32         \n",
      " 35  clean_md_avg_word_len      3201 non-null   float64       \n",
      " 36  clean_md_sent_count        3201 non-null   Int32         \n",
      " 37  clean_md_avg_sent_len      3201 non-null   float64       \n",
      " 38  doc_main_char_count        3201 non-null   Int32         \n",
      " 39  doc_main_hastag_runs       3201 non-null   Int32         \n",
      " 40  doc_main_asterisk_runs     3201 non-null   Int32         \n",
      " 41  doc_credit_char_count      3201 non-null   Int32         \n",
      " 42  doc_credit_hastag_runs     3201 non-null   Int32         \n",
      " 43  doc_credit_asterisk_runs   3201 non-null   Int32         \n",
      " 44  doc_main_stopword_count    3201 non-null   Int32         \n",
      " 45  doc_credit_stopword_count  3201 non-null   Int32         \n",
      " 46  doc_main_word_count        3201 non-null   Int32         \n",
      " 47  doc_main_avg_word_len      3201 non-null   float64       \n",
      " 48  doc_main_sent_count        3201 non-null   Int32         \n",
      " 49  doc_main_avg_sent_len      3201 non-null   float64       \n",
      " 50  doc_credit_word_count      3201 non-null   Int32         \n",
      " 51  doc_credit_avg_word_len    3201 non-null   float64       \n",
      " 52  doc_credit_sent_count      3201 non-null   Int32         \n",
      " 53  doc_credit_avg_sent_len    3201 non-null   float64       \n",
      " 54  full_text                  3201 non-null   object        \n",
      " 55  split                      3042 non-null   object        \n",
      "dtypes: Int32(23), boolean(2), category(2), datetime64[ns](2), float32(1), float64(6), int16(1), int64(2), object(17)\n",
      "memory usage: 1.0+ MB\n"
     ]
    }
   ],
   "source": [
    "doc_corpus_path = CORPUS_DIR / \"document_corpus.pkl\"\n",
    "doc_df: pd.DataFrame = pd.read_pickle(doc_corpus_path)\n",
    "doc_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c0407351",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Subclass      1187\n",
       "Monster        562\n",
       "Class          478\n",
       "Race           229\n",
       "Spell          194\n",
       "Item           166\n",
       "Compendium     152\n",
       "Feat           122\n",
       "Mechanic       104\n",
       "Background       7\n",
       "Name: submission_flair, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_df[\"submission_flair\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "31edfe7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"split\" not in doc_df.columns:\n",
    "    model_data = doc_df[~doc_df[\"submission_flair\"].isin([\"Background\", \"Compendium\"])]\n",
    "    train_df, test_df = train_test_split(\n",
    "        model_data, train_size=0.75, random_state=29359\n",
    "    )\n",
    "    doc_df[\"split\"] = None\n",
    "    doc_df.loc[doc_df[\"UID\"].isin(train_df[\"UID\"]), \"split\"] = \"train\"\n",
    "    doc_df.loc[doc_df[\"UID\"].isin(test_df[\"UID\"]), \"split\"] = \"test\"\n",
    "    doc_df.to_pickle(doc_corpus_path)\n",
    "else:\n",
    "    train_df = doc_df[doc_df[\"split\"].isin([\"train\", \"validation\"])].copy()\n",
    "    test_df = doc_df[doc_df[\"split\"] == \"test\"].copy()\n",
    "\n",
    "train_y = train_df[\"submission_flair\"].values\n",
    "test_y = test_df[\"submission_flair\"].values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3d251e0",
   "metadata": {},
   "source": [
    "# Full Text\n",
    "\n",
    "I'd like to compare my credit/introduction-cleaned model to the raw text to see how much it helped (or hindered) classification overall.\n",
    "\n",
    "I already know *a priori* that it has many features unique to the GMBinder/Homebrewery format that are not found in \"real\" texts (i.e. professionally published texts, where credits and indexes only appear at the start of an overall book, not next to specific elements).\n",
    "\n",
    "The cleaning I performed is a good thing for that reason alone, but I'm curious if it will help generalizability even within my scraped dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10621a03",
   "metadata": {},
   "source": [
    "### Vectorize Raw Texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ab466c32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculated 1781 components\n"
     ]
    }
   ],
   "source": [
    "# Get stop words\n",
    "stop_words = stopwords.words(\"english\")\n",
    "stop_words = [word_tokenize(sw) for sw in stop_words]\n",
    "stop_words = [token for sw_tokens in stop_words for token in sw_tokens]\n",
    "\n",
    "# Get list of rare words\n",
    "full_count_vectorizer = CountVectorizer(\n",
    "    tokenizer=word_tokenize,\n",
    "    lowercase=True,\n",
    "    token_pattern=None,\n",
    "    stop_words=stop_words,\n",
    ")\n",
    "full_counts = full_count_vectorizer.fit_transform(train_df[\"full_text\"]).toarray()\n",
    "full_vocabulary = full_count_vectorizer.vocabulary_\n",
    "full_infrequent_words = [\n",
    "    word for word, index in full_vocabulary.items() if full_counts[:, index].sum() <= 5\n",
    "]\n",
    "\n",
    "# Expand stop words to include rare terms\n",
    "stop_words.extend(full_infrequent_words)\n",
    "stop_words = [word_tokenize(sw) for sw in stop_words]\n",
    "stop_words = [token for sw_tokens in stop_words for token in sw_tokens]\n",
    "\n",
    "# Get vectors\n",
    "full_tfidf_vectorizer = TfidfVectorizer(\n",
    "    tokenizer=word_tokenize,\n",
    "    lowercase=True,\n",
    "    token_pattern=None,\n",
    "    stop_words=stop_words,\n",
    ")\n",
    "full_train_X = full_tfidf_vectorizer.fit_transform(train_df[\"full_text\"])\n",
    "full_test_X = full_tfidf_vectorizer.transform(test_df[\"full_text\"])\n",
    "\n",
    "# Get dense array\n",
    "full_test_X = full_test_X.toarray()\n",
    "full_train_X = full_train_X.toarray()\n",
    "\n",
    "# Fit/get PCs\n",
    "full_pca = PCA(n_components=1781)\n",
    "full_train_X = full_pca.fit_transform(full_train_X)\n",
    "full_test_X = full_pca.transform(full_test_X)\n",
    "print(f\"Calculated {full_pca.n_components_} components\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc2a8f75",
   "metadata": {},
   "source": [
    "### Perform Bayes Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "305d7dd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "382.62 total minutes to fit all 320 models\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>BayesSearchCV(cv=5,\n",
       "              estimator=OneVsRestClassifier(estimator=SVC(class_weight=&#x27;balanced&#x27;,\n",
       "                                                          probability=True)),\n",
       "              n_iter=64, n_jobs=-1, scoring=&#x27;f1_macro&#x27;,\n",
       "              search_spaces={&#x27;estimator__C&#x27;: (1e-06, 1000000.0, &#x27;log-uniform&#x27;),\n",
       "                             &#x27;estimator__degree&#x27;: (1, 8),\n",
       "                             &#x27;estimator__gamma&#x27;: (1e-06, 10.0, &#x27;log-uniform&#x27;),\n",
       "                             &#x27;estimator__kernel&#x27;: [&#x27;linear&#x27;, &#x27;poly&#x27;, &#x27;rbf&#x27;]})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">BayesSearchCV</label><div class=\"sk-toggleable__content\"><pre>BayesSearchCV(cv=5,\n",
       "              estimator=OneVsRestClassifier(estimator=SVC(class_weight=&#x27;balanced&#x27;,\n",
       "                                                          probability=True)),\n",
       "              n_iter=64, n_jobs=-1, scoring=&#x27;f1_macro&#x27;,\n",
       "              search_spaces={&#x27;estimator__C&#x27;: (1e-06, 1000000.0, &#x27;log-uniform&#x27;),\n",
       "                             &#x27;estimator__degree&#x27;: (1, 8),\n",
       "                             &#x27;estimator__gamma&#x27;: (1e-06, 10.0, &#x27;log-uniform&#x27;),\n",
       "                             &#x27;estimator__kernel&#x27;: [&#x27;linear&#x27;, &#x27;poly&#x27;, &#x27;rbf&#x27;]})</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: OneVsRestClassifier</label><div class=\"sk-toggleable__content\"><pre>OneVsRestClassifier(estimator=SVC(class_weight=&#x27;balanced&#x27;, probability=True))</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(class_weight=&#x27;balanced&#x27;, probability=True)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(class_weight=&#x27;balanced&#x27;, probability=True)</pre></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "BayesSearchCV(cv=5,\n",
       "              estimator=OneVsRestClassifier(estimator=SVC(class_weight='balanced',\n",
       "                                                          probability=True)),\n",
       "              n_iter=64, n_jobs=-1, scoring='f1_macro',\n",
       "              search_spaces={'estimator__C': (1e-06, 1000000.0, 'log-uniform'),\n",
       "                             'estimator__degree': (1, 8),\n",
       "                             'estimator__gamma': (1e-06, 10.0, 'log-uniform'),\n",
       "                             'estimator__kernel': ['linear', 'poly', 'rbf']})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_search_path = MODEL_SEARCH / \"full_bayes.pkl\"\n",
    "full_model_path = OBJ_DIR / \"full_ovr_svr_model.pkl\"\n",
    "full_search_results_path = MODEL_SEARCH / \"full_bayes_search_ovr_svc.csv\"\n",
    "\n",
    "if full_search_path.is_file():\n",
    "    with open(full_search_path, \"rb\") as bs:\n",
    "        full_search: BayesSearchCV = pkl.load(bs)\n",
    "        full_results = pd.read_csv(full_search_results_path)\n",
    "\n",
    "    if full_model_path.is_file():\n",
    "        with open(full_model_path, \"rb\") as bm:\n",
    "            full_model: OneVsRestClassifier = pkl.load(bm)\n",
    "    elif isinstance(full_search.best_estimator_, OneVsRestClassifier):\n",
    "        main_text_model = full_search.best_estimator_\n",
    "    else:\n",
    "        warnings.warn(\n",
    "            \"Unable to retrieve best estimator, check top params.\", UserWarning\n",
    "        )\n",
    "else:\n",
    "    full_svc = SVC(class_weight=\"balanced\", probability=True)\n",
    "    full_ovr = OneVsRestClassifier(full_svc)\n",
    "    full_search = BayesSearchCV(\n",
    "        full_ovr,\n",
    "        {\n",
    "            \"estimator__C\": (1e-6, 1e6, \"log-uniform\"),\n",
    "            \"estimator__gamma\": (1e-6, 1e1, \"log-uniform\"),\n",
    "            \"estimator__degree\": (1, 8),  # integer valued parameter\n",
    "            \"estimator__kernel\": [\"linear\", \"poly\", \"rbf\"],  # categorical parameter\n",
    "        },\n",
    "        n_iter=64,\n",
    "        cv=5,\n",
    "        n_jobs=-1,\n",
    "        scoring=\"f1_macro\",\n",
    "    )\n",
    "\n",
    "    full_search.fit(full_train_X, train_y)\n",
    "    full_model = full_search.best_estimator_\n",
    "    with open(full_search_path, \"wb\") as bs:\n",
    "        pkl.dump(full_search, bs)\n",
    "    with open(full_model_path, \"wb\") as bm:\n",
    "        pkl.dump(full_model, bm)\n",
    "    full_results = pd.DataFrame(full_search.cv_results_)\n",
    "    full_results.to_csv(full_search_results_path)\n",
    "\n",
    "fit_time = full_results[\"mean_fit_time\"].sum() * 5\n",
    "n_models = len(full_results) * 5\n",
    "print(f\"{fit_time/60:.2f} total minutes to fit all {n_models} models\")\n",
    "\n",
    "full_search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e40a7b9",
   "metadata": {},
   "source": [
    "## Score Full Text Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3c96f8c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W/ Credit: Accuracy = 96.76%; class-balanced accuracy = 96.79%\n",
      "W/ Credit: Micro-averaged F1 = 0.97; macro-averaged F1 = 0.96\n"
     ]
    }
   ],
   "source": [
    "full_train_preds = full_model.predict(full_train_X)\n",
    "\n",
    "full_train_acc = accuracy_score(train_y, full_train_preds)\n",
    "full_train_balanced_acc = balanced_accuracy_score(train_y, full_train_preds)\n",
    "print(\n",
    "    f\"W/ Credit: Accuracy = {full_train_acc*100:.2f}%; class-balanced accuracy = {full_train_balanced_acc*100:.2f}%\"\n",
    ")\n",
    "full_train_f1_micro = f1_score(train_y, full_train_preds, average=\"micro\")\n",
    "full_train_f1_macro = f1_score(train_y, full_train_preds, average=\"macro\")\n",
    "print(\n",
    "    f\"W/ Credit: Micro-averaged F1 = {full_train_f1_micro:.2f}; macro-averaged F1 = {full_train_f1_macro:.2f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cd050f92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W/O Credit: Micro-averaged OVR AUC-ROC = 1.00; macro-averaged OVR AUC-ROC = 1.00\n"
     ]
    }
   ],
   "source": [
    "train_Y = full_model.label_binarizer_.transform(train_y)\n",
    "full_train_probs = full_model.predict_proba(full_train_X)\n",
    "\n",
    "train_micro_roc_auc_ovr = roc_auc_score(\n",
    "    train_Y.toarray(),\n",
    "    full_train_probs,\n",
    "    multi_class=\"ovr\",\n",
    "    average=\"micro\",\n",
    ")\n",
    "train_macro_roc_auc_ovr = roc_auc_score(\n",
    "    train_Y.toarray(),\n",
    "    full_train_probs,\n",
    "    multi_class=\"ovr\",\n",
    "    average=\"macro\",\n",
    ")\n",
    "print(\n",
    "    f\"W/O Credit: Micro-averaged OVR AUC-ROC = {train_micro_roc_auc_ovr:.2f}; macro-averaged OVR AUC-ROC = {train_macro_roc_auc_ovr:.2f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2348a350",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W/ Credit: Accuracy = 92.12%; class-balanced accuracy = 87.86%\n",
      "W/ Credit: Micro-averaged F1 = 0.92; macro-averaged F1 = 0.88\n"
     ]
    }
   ],
   "source": [
    "full_test_preds = full_model.predict(full_test_X)\n",
    "\n",
    "full_test_acc = accuracy_score(test_y, full_test_preds)\n",
    "full_test_balanced_acc = balanced_accuracy_score(test_y, full_test_preds)\n",
    "print(\n",
    "    f\"W/ Credit: Accuracy = {full_test_acc*100:.2f}%; class-balanced accuracy = {full_test_balanced_acc*100:.2f}%\"\n",
    ")\n",
    "full_test_f1_micro = f1_score(test_y, full_test_preds, average=\"micro\")\n",
    "full_test_f1_macro = f1_score(test_y, full_test_preds, average=\"macro\")\n",
    "print(\n",
    "    f\"W/ Credit: Micro-averaged F1 = {full_test_f1_micro:.2f}; macro-averaged F1 = {full_test_f1_macro:.2f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4ca45b5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W/O Credit: Micro-averaged OVR AUC-ROC = 0.99; macro-averaged OVR AUC-ROC = 0.99\n"
     ]
    }
   ],
   "source": [
    "test_Y = full_model.label_binarizer_.transform(test_y)\n",
    "full_test_probs = full_model.predict_proba(full_test_X)\n",
    "\n",
    "test_micro_roc_auc_ovr = roc_auc_score(\n",
    "    test_Y.toarray(),\n",
    "    full_test_probs,\n",
    "    multi_class=\"ovr\",\n",
    "    average=\"micro\",\n",
    ")\n",
    "test_macro_roc_auc_ovr = roc_auc_score(\n",
    "    test_Y.toarray(),\n",
    "    full_test_probs,\n",
    "    multi_class=\"ovr\",\n",
    "    average=\"macro\",\n",
    ")\n",
    "print(\n",
    "    f\"W/O Credit: Micro-averaged OVR AUC-ROC = {test_micro_roc_auc_ovr:.2f}; macro-averaged OVR AUC-ROC = {test_macro_roc_auc_ovr:.2f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb73c825",
   "metadata": {},
   "source": [
    "## Credit-Cleaned Text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14d34acf",
   "metadata": {},
   "source": [
    "### Pre-Processing of Credit-Cleaned Text\n",
    "* Vectorize Pre-Tokenized Inputs\n",
    "* Reduce Dimensions for Efficiency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "07e3a15e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculated 1791 components\n"
     ]
    }
   ],
   "source": [
    "# Define stop words\n",
    "with open(OBJ_DIR / \"tokenizer.pkl\", \"rb\") as p:\n",
    "    ea_tokenizer: EmbeddingAwareTokenizer = pkl.load(p)\n",
    "\n",
    "stop_words = stopwords.words(\"english\")\n",
    "stop_words = [ea_tokenizer.tokenize(sw) for sw in stop_words]\n",
    "stop_words = [token for sw_tokens in stop_words for token in sw_tokens]\n",
    "stop_words = [token for token in stop_words if \"<\" not in token]\n",
    "\n",
    "# Get vectors\n",
    "main_text_tfidf_vectorizer = TfidfVectorizer(\n",
    "    tokenizer=do_nothing,\n",
    "    lowercase=False,\n",
    "    token_pattern=None,\n",
    "    stop_words=stop_words,\n",
    ")\n",
    "\n",
    "main_text_train_X = main_text_tfidf_vectorizer.fit_transform(\n",
    "    train_df[\"clean_word_tokens\"]\n",
    ")\n",
    "main_text_test_X = main_text_tfidf_vectorizer.transform(test_df[\"clean_word_tokens\"])\n",
    "\n",
    "# Convert to dense\n",
    "main_text_test_X = main_text_test_X.toarray()\n",
    "main_text_train_X = main_text_train_X.toarray()\n",
    "\n",
    "# Get PCs\n",
    "main_text_pca = PCA(n_components=1791)\n",
    "main_text_train_X = main_text_pca.fit_transform(main_text_train_X)\n",
    "main_text_test_X = main_text_pca.transform(main_text_test_X)\n",
    "print(f\"Calculated {main_text_pca.n_components_} components\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3da03f52",
   "metadata": {},
   "source": [
    "### Perform Bayes Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2259becd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "431.44 total minutes to fit all 320 models\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>BayesSearchCV(cv=5,\n",
       "              estimator=OneVsRestClassifier(estimator=SVC(class_weight=&#x27;balanced&#x27;,\n",
       "                                                          probability=True)),\n",
       "              n_iter=64, n_jobs=-1, scoring=&#x27;f1_macro&#x27;,\n",
       "              search_spaces={&#x27;estimator__C&#x27;: (1e-06, 1000000.0, &#x27;log-uniform&#x27;),\n",
       "                             &#x27;estimator__degree&#x27;: (1, 8),\n",
       "                             &#x27;estimator__gamma&#x27;: (1e-06, 10.0, &#x27;log-uniform&#x27;),\n",
       "                             &#x27;estimator__kernel&#x27;: [&#x27;linear&#x27;, &#x27;poly&#x27;, &#x27;rbf&#x27;]})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">BayesSearchCV</label><div class=\"sk-toggleable__content\"><pre>BayesSearchCV(cv=5,\n",
       "              estimator=OneVsRestClassifier(estimator=SVC(class_weight=&#x27;balanced&#x27;,\n",
       "                                                          probability=True)),\n",
       "              n_iter=64, n_jobs=-1, scoring=&#x27;f1_macro&#x27;,\n",
       "              search_spaces={&#x27;estimator__C&#x27;: (1e-06, 1000000.0, &#x27;log-uniform&#x27;),\n",
       "                             &#x27;estimator__degree&#x27;: (1, 8),\n",
       "                             &#x27;estimator__gamma&#x27;: (1e-06, 10.0, &#x27;log-uniform&#x27;),\n",
       "                             &#x27;estimator__kernel&#x27;: [&#x27;linear&#x27;, &#x27;poly&#x27;, &#x27;rbf&#x27;]})</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: OneVsRestClassifier</label><div class=\"sk-toggleable__content\"><pre>OneVsRestClassifier(estimator=SVC(class_weight=&#x27;balanced&#x27;, probability=True))</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(class_weight=&#x27;balanced&#x27;, probability=True)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(class_weight=&#x27;balanced&#x27;, probability=True)</pre></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "BayesSearchCV(cv=5,\n",
       "              estimator=OneVsRestClassifier(estimator=SVC(class_weight='balanced',\n",
       "                                                          probability=True)),\n",
       "              n_iter=64, n_jobs=-1, scoring='f1_macro',\n",
       "              search_spaces={'estimator__C': (1e-06, 1000000.0, 'log-uniform'),\n",
       "                             'estimator__degree': (1, 8),\n",
       "                             'estimator__gamma': (1e-06, 10.0, 'log-uniform'),\n",
       "                             'estimator__kernel': ['linear', 'poly', 'rbf']})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_text_search_path = MODEL_SEARCH / \"main_text_bayes.pkl\"\n",
    "main_text_model_path = OBJ_DIR / \"main_text_ovr_svr_model.pkl\"\n",
    "main_text_search_results_path = MODEL_SEARCH / \"main_text_bayes_search_ovr_svc.csv\"\n",
    "\n",
    "if main_text_search_path.is_file():\n",
    "    with open(main_text_search_path, \"rb\") as bs:\n",
    "        main_text_search: BayesSearchCV = pkl.load(bs)\n",
    "        main_text_results = pd.read_csv(main_text_search_results_path)\n",
    "\n",
    "    if main_text_model_path.is_file():\n",
    "        with open(main_text_model_path, \"rb\") as bm:\n",
    "            main_text_model: OneVsRestClassifier = pkl.load(bm)\n",
    "    elif isinstance(main_text_search.best_estimator_, OneVsRestClassifier):\n",
    "        main_text_model = main_text_search.best_estimator_\n",
    "    else:\n",
    "        warnings.warn(\n",
    "            \"Unable to retrieve best estimator, check top params.\", UserWarning\n",
    "        )\n",
    "\n",
    "else:\n",
    "    main_text_svc = SVC(class_weight=\"balanced\", probability=True)\n",
    "    main_text_ovr = OneVsRestClassifier(main_text_svc)\n",
    "    main_text_search = BayesSearchCV(\n",
    "        main_text_ovr,\n",
    "        {\n",
    "            \"estimator__C\": (1e-6, 1e6, \"log-uniform\"),\n",
    "            \"estimator__gamma\": (1e-6, 1e1, \"log-uniform\"),\n",
    "            \"estimator__degree\": (1, 8),  # integer valued parameter\n",
    "            \"estimator__kernel\": [\"linear\", \"poly\", \"rbf\"],  # categorical parameter\n",
    "        },\n",
    "        n_iter=64,\n",
    "        cv=5,\n",
    "        n_jobs=-1,\n",
    "        scoring=\"f1_macro\",\n",
    "    )\n",
    "\n",
    "    main_text_search.fit(main_text_train_X, train_y)\n",
    "    main_text_model = main_text_search.best_estimator_\n",
    "    with open(main_text_search_path, \"wb\") as bs:\n",
    "        pkl.dump(main_text_search, bs)\n",
    "    with open(main_text_model_path, \"wb\") as bm:\n",
    "        pkl.dump(main_text_model, bm)\n",
    "    main_text_results = pd.DataFrame(main_text_search.cv_results_)\n",
    "    main_text_results.to_csv(main_text_search_results_path)\n",
    "\n",
    "\n",
    "fit_time = main_text_results[\"mean_fit_time\"].sum() * 5\n",
    "n_models = len(main_text_results) * 5\n",
    "print(f\"{fit_time/60:.2f} total minutes to fit all {n_models} models\")\n",
    "\n",
    "main_text_search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddee91e7",
   "metadata": {},
   "source": [
    "## Score Cleaned Text Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2164eb09",
   "metadata": {},
   "source": [
    "### Training Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c9343144",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W/O Credit: Accuracy = 98.11%; class-balanced accuracy = 98.25%\n",
      "W/O Credit: Micro-averaged F1 = 0.98; macro-averaged F1 = 0.98\n"
     ]
    }
   ],
   "source": [
    "main_text_train_preds = main_text_model.predict(main_text_train_X)\n",
    "\n",
    "main_text_train_acc = accuracy_score(train_y, main_text_train_preds)\n",
    "main_text_train_balanced_acc = balanced_accuracy_score(train_y, main_text_train_preds)\n",
    "print(\n",
    "    f\"W/O Credit: Accuracy = {main_text_train_acc*100:.2f}%; class-balanced accuracy = {main_text_train_balanced_acc*100:.2f}%\"\n",
    ")\n",
    "main_text_train_f1_micro = f1_score(train_y, main_text_train_preds, average=\"micro\")\n",
    "main_text_train_f1_macro = f1_score(train_y, main_text_train_preds, average=\"macro\")\n",
    "print(\n",
    "    f\"W/O Credit: Micro-averaged F1 = {main_text_train_f1_micro:.2f}; macro-averaged F1 = {main_text_train_f1_macro:.2f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "00c0be9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W/O Credit: Micro-averaged OVR AUC-ROC = 1.00; macro-averaged OVR AUC-ROC = 1.00\n"
     ]
    }
   ],
   "source": [
    "train_Y = main_text_model.label_binarizer_.transform(train_y)\n",
    "main_text_train_probs = main_text_model.predict_proba(main_text_train_X)\n",
    "\n",
    "train_micro_roc_auc_ovr = roc_auc_score(\n",
    "    train_Y.toarray(),\n",
    "    main_text_train_probs,\n",
    "    multi_class=\"ovr\",\n",
    "    average=\"micro\",\n",
    ")\n",
    "train_macro_roc_auc_ovr = roc_auc_score(\n",
    "    train_Y.toarray(),\n",
    "    main_text_train_probs,\n",
    "    multi_class=\"ovr\",\n",
    "    average=\"macro\",\n",
    ")\n",
    "print(\n",
    "    f\"W/O Credit: Micro-averaged OVR AUC-ROC = {train_micro_roc_auc_ovr:.2f}; macro-averaged OVR AUC-ROC = {train_macro_roc_auc_ovr:.2f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11534d9d",
   "metadata": {},
   "source": [
    "### Testing Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bdca95e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W/O Credit: Accuracy = 91.85%; class-balanced accuracy = 87.72%\n",
      "W/O Credit: Micro-averaged F1 = 0.92; macro-averaged F1 = 0.88\n"
     ]
    }
   ],
   "source": [
    "main_text_test_preds = main_text_model.predict(main_text_test_X)\n",
    "\n",
    "# Accuracy\n",
    "main_text_test_acc = accuracy_score(test_y, main_text_test_preds)\n",
    "main_text_test_balanced_acc = balanced_accuracy_score(test_y, main_text_test_preds)\n",
    "print(\n",
    "    f\"W/O Credit: Accuracy = {main_text_test_acc*100:.2f}%; class-balanced accuracy = {main_text_test_balanced_acc*100:.2f}%\"\n",
    ")\n",
    "\n",
    "main_text_test_f1_micro = f1_score(test_y, main_text_test_preds, average=\"micro\")\n",
    "main_text_test_f1_macro = f1_score(test_y, main_text_test_preds, average=\"macro\")\n",
    "print(\n",
    "    f\"W/O Credit: Micro-averaged F1 = {main_text_test_f1_micro:.2f}; macro-averaged F1 = {main_text_test_f1_macro:.2f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "04457fa2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W/O Credit: Micro-averaged OVR AUC-ROC = 0.99; macro-averaged OVR AUC-ROC = 0.99\n"
     ]
    }
   ],
   "source": [
    "test_Y = main_text_model.label_binarizer_.transform(test_y)\n",
    "main_text_test_probs = main_text_model.predict_proba(main_text_test_X)\n",
    "\n",
    "test_micro_roc_auc_ovr = roc_auc_score(\n",
    "    test_Y.toarray(),\n",
    "    main_text_test_probs,\n",
    "    multi_class=\"ovr\",\n",
    "    average=\"micro\",\n",
    ")\n",
    "test_macro_roc_auc_ovr = roc_auc_score(\n",
    "    test_Y.toarray(),\n",
    "    main_text_test_probs,\n",
    "    multi_class=\"ovr\",\n",
    "    average=\"macro\",\n",
    ")\n",
    "print(\n",
    "    f\"W/O Credit: Micro-averaged OVR AUC-ROC = {test_micro_roc_auc_ovr:.2f}; macro-averaged OVR AUC-ROC = {test_macro_roc_auc_ovr:.2f}\"\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}