dummies-stratified-scaled-kfold-logreg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from base.ipynb\n"
     ]
    }
   ],
   "source": [
    "import import_ipynb\n",
    "import base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run base.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import original Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = get_original_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Category</th>\n",
       "      <th>Subcategory</th>\n",
       "      <th>Country</th>\n",
       "      <th>Launched</th>\n",
       "      <th>Deadline</th>\n",
       "      <th>Goal</th>\n",
       "      <th>Pledged</th>\n",
       "      <th>Backers</th>\n",
       "      <th>State</th>\n",
       "      <th>Duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Fashion</td>\n",
       "      <td>Fashion</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-21 21:02:48</td>\n",
       "      <td>2009-05-31</td>\n",
       "      <td>1000</td>\n",
       "      <td>625</td>\n",
       "      <td>30</td>\n",
       "      <td>Failed</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Film__Video</td>\n",
       "      <td>Shorts</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-23 00:07:53</td>\n",
       "      <td>2009-07-20</td>\n",
       "      <td>80000</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>Failed</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Art</td>\n",
       "      <td>Illustration</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-24 21:52:03</td>\n",
       "      <td>2009-05-03</td>\n",
       "      <td>20</td>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>Successful</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Technology</td>\n",
       "      <td>Software</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-25 17:36:21</td>\n",
       "      <td>2009-07-14</td>\n",
       "      <td>99</td>\n",
       "      <td>145</td>\n",
       "      <td>25</td>\n",
       "      <td>Successful</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fashion</td>\n",
       "      <td>Fashion</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-27 14:10:39</td>\n",
       "      <td>2009-05-26</td>\n",
       "      <td>1900</td>\n",
       "      <td>387</td>\n",
       "      <td>10</td>\n",
       "      <td>Failed</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Category   Subcategory        Country            Launched   Deadline   \n",
       "0      Fashion       Fashion  United_States 2009-04-21 21:02:48 2009-05-31  \\\n",
       "1  Film__Video        Shorts  United_States 2009-04-23 00:07:53 2009-07-20   \n",
       "2          Art  Illustration  United_States 2009-04-24 21:52:03 2009-05-03   \n",
       "3   Technology      Software  United_States 2009-04-25 17:36:21 2009-07-14   \n",
       "4      Fashion       Fashion  United_States 2009-04-27 14:10:39 2009-05-26   \n",
       "\n",
       "    Goal  Pledged  Backers       State  Duration  \n",
       "0   1000      625       30      Failed        39  \n",
       "1  80000       22        3      Failed        87  \n",
       "2     20       35        3  Successful         8  \n",
       "3     99      145       25  Successful        79  \n",
       "4   1900      387       10      Failed        28  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean categorical strings from spaces and troublesome special characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# Define a function to clean up string values\n",
    "def clean_string(s):\n",
    "    # Remove leading/trailing spaces\n",
    "    s = s.strip()\n",
    "    # Replace all spaces with underscores (or remove them if desired)\n",
    "    s = re.sub(r'\\s+', '_', s)  # Replace spaces with underscores\n",
    "    # Remove problematic special characters\n",
    "    s = re.sub(r'[^\\w\\s]', '', s)\n",
    "    return s\n",
    "\n",
    "# Apply the function to all columns in the DataFrame\n",
    "df = df.applymap(lambda x: clean_string(x) if isinstance(x, str) else x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Category</th>\n",
       "      <th>Subcategory</th>\n",
       "      <th>Country</th>\n",
       "      <th>Launched</th>\n",
       "      <th>Deadline</th>\n",
       "      <th>Goal</th>\n",
       "      <th>Pledged</th>\n",
       "      <th>Backers</th>\n",
       "      <th>State</th>\n",
       "      <th>Duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Fashion</td>\n",
       "      <td>Fashion</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-21 21:02:48</td>\n",
       "      <td>2009-05-31</td>\n",
       "      <td>1000</td>\n",
       "      <td>625</td>\n",
       "      <td>30</td>\n",
       "      <td>Failed</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Film__Video</td>\n",
       "      <td>Shorts</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-23 00:07:53</td>\n",
       "      <td>2009-07-20</td>\n",
       "      <td>80000</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>Failed</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Art</td>\n",
       "      <td>Illustration</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-24 21:52:03</td>\n",
       "      <td>2009-05-03</td>\n",
       "      <td>20</td>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>Successful</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Technology</td>\n",
       "      <td>Software</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-25 17:36:21</td>\n",
       "      <td>2009-07-14</td>\n",
       "      <td>99</td>\n",
       "      <td>145</td>\n",
       "      <td>25</td>\n",
       "      <td>Successful</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fashion</td>\n",
       "      <td>Fashion</td>\n",
       "      <td>United_States</td>\n",
       "      <td>2009-04-27 14:10:39</td>\n",
       "      <td>2009-05-26</td>\n",
       "      <td>1900</td>\n",
       "      <td>387</td>\n",
       "      <td>10</td>\n",
       "      <td>Failed</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Category   Subcategory        Country            Launched   Deadline   \n",
       "0      Fashion       Fashion  United_States 2009-04-21 21:02:48 2009-05-31  \\\n",
       "1  Film__Video        Shorts  United_States 2009-04-23 00:07:53 2009-07-20   \n",
       "2          Art  Illustration  United_States 2009-04-24 21:52:03 2009-05-03   \n",
       "3   Technology      Software  United_States 2009-04-25 17:36:21 2009-07-14   \n",
       "4      Fashion       Fashion  United_States 2009-04-27 14:10:39 2009-05-26   \n",
       "\n",
       "    Goal  Pledged  Backers       State  Duration  \n",
       "0   1000      625       30      Failed        39  \n",
       "1  80000       22        3      Failed        87  \n",
       "2     20       35        3  Successful         8  \n",
       "3     99      145       25  Successful        79  \n",
       "4   1900      387       10      Failed        28  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 331462 entries, 0 to 374605\n",
      "Data columns (total 10 columns):\n",
      " #   Column       Non-Null Count   Dtype         \n",
      "---  ------       --------------   -----         \n",
      " 0   Category     331462 non-null  object        \n",
      " 1   Subcategory  331462 non-null  object        \n",
      " 2   Country      331462 non-null  object        \n",
      " 3   Launched     331462 non-null  datetime64[ns]\n",
      " 4   Deadline     331462 non-null  datetime64[ns]\n",
      " 5   Goal         331462 non-null  int64         \n",
      " 6   Pledged      331462 non-null  int64         \n",
      " 7   Backers      331462 non-null  int64         \n",
      " 8   State        331462 non-null  object        \n",
      " 9   Duration     331462 non-null  int64         \n",
      "dtypes: datetime64[ns](2), int64(4), object(4)\n",
      "memory usage: 27.8+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode categorical features as dummy (binary) variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.get_dummies(df, drop_first=True, columns=df.select_dtypes(include=['object']).columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Launched</th>\n",
       "      <th>Deadline</th>\n",
       "      <th>Goal</th>\n",
       "      <th>Pledged</th>\n",
       "      <th>Backers</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Category_Comics</th>\n",
       "      <th>Category_Crafts</th>\n",
       "      <th>Category_Dance</th>\n",
       "      <th>Category_Design</th>\n",
       "      <th>...</th>\n",
       "      <th>Country_Netherlands</th>\n",
       "      <th>Country_New_Zealand</th>\n",
       "      <th>Country_Norway</th>\n",
       "      <th>Country_Singapore</th>\n",
       "      <th>Country_Spain</th>\n",
       "      <th>Country_Sweden</th>\n",
       "      <th>Country_Switzerland</th>\n",
       "      <th>Country_United_Kingdom</th>\n",
       "      <th>Country_United_States</th>\n",
       "      <th>State_Successful</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2009-04-21 21:02:48</td>\n",
       "      <td>2009-05-31</td>\n",
       "      <td>1000</td>\n",
       "      <td>625</td>\n",
       "      <td>30</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2009-04-23 00:07:53</td>\n",
       "      <td>2009-07-20</td>\n",
       "      <td>80000</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>87</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2009-04-24 21:52:03</td>\n",
       "      <td>2009-05-03</td>\n",
       "      <td>20</td>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2009-04-25 17:36:21</td>\n",
       "      <td>2009-07-14</td>\n",
       "      <td>99</td>\n",
       "      <td>145</td>\n",
       "      <td>25</td>\n",
       "      <td>79</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2009-04-27 14:10:39</td>\n",
       "      <td>2009-05-26</td>\n",
       "      <td>1900</td>\n",
       "      <td>387</td>\n",
       "      <td>10</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 200 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Launched   Deadline   Goal  Pledged  Backers  Duration   \n",
       "0 2009-04-21 21:02:48 2009-05-31   1000      625       30        39  \\\n",
       "1 2009-04-23 00:07:53 2009-07-20  80000       22        3        87   \n",
       "2 2009-04-24 21:52:03 2009-05-03     20       35        3         8   \n",
       "3 2009-04-25 17:36:21 2009-07-14     99      145       25        79   \n",
       "4 2009-04-27 14:10:39 2009-05-26   1900      387       10        28   \n",
       "\n",
       "   Category_Comics  Category_Crafts  Category_Dance  Category_Design  ...   \n",
       "0            False            False           False            False  ...  \\\n",
       "1            False            False           False            False  ...   \n",
       "2            False            False           False            False  ...   \n",
       "3            False            False           False            False  ...   \n",
       "4            False            False           False            False  ...   \n",
       "\n",
       "   Country_Netherlands  Country_New_Zealand  Country_Norway   \n",
       "0                False                False           False  \\\n",
       "1                False                False           False   \n",
       "2                False                False           False   \n",
       "3                False                False           False   \n",
       "4                False                False           False   \n",
       "\n",
       "   Country_Singapore  Country_Spain  Country_Sweden  Country_Switzerland   \n",
       "0              False          False           False                False  \\\n",
       "1              False          False           False                False   \n",
       "2              False          False           False                False   \n",
       "3              False          False           False                False   \n",
       "4              False          False           False                False   \n",
       "\n",
       "   Country_United_Kingdom  Country_United_States  State_Successful  \n",
       "0                   False                   True             False  \n",
       "1                   False                   True             False  \n",
       "2                   False                   True              True  \n",
       "3                   False                   True              True  \n",
       "4                   False                   True             False  \n",
       "\n",
       "[5 rows x 200 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(331462, 200)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define target & features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import StratifiedKFold, GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import f1_score, make_scorer\n",
    "\n",
    "# Drop or convert datetime columns\n",
    "X = df.drop(columns=['State_Successful', 'Launched', 'Deadline', 'Pledged']) # Drop datetime columns + Pledged (data leakage)\n",
    "y = df['State_Successful']  # Target column\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract features from datetime columns (here only the year to avoid too many features)\n",
    "X['year_launched'] = df['Launched'].dt.year\n",
    "# X['month_launched'] = df['Launched'].dt.month\n",
    "# X['day_launched'] = df['Launched'].dt.day\n",
    "\n",
    "X['year_deadline'] = df['Deadline'].dt.year\n",
    "# X['year_deadline'] = df['Deadline'].dt.month\n",
    "# X['year_deadline'] = df['Deadline'].dt.day"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split the data into training and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scale the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale numerical data using StandarScaler:\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# Identify numerical columns (excluding date-time and timedelta types)\n",
    "numerical_cols = X.select_dtypes(include=['number']).columns.tolist()\n",
    "\n",
    "# Initialize StandardScaler\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Fit and transform only the numerical columns to the training data\n",
    "X_train[numerical_cols] = scaler.fit_transform(X_train[numerical_cols])\n",
    "\n",
    "# Use the same scaler to transform the test data\n",
    "X_test[numerical_cols] = scaler.transform(X_test[numerical_cols])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Resample the data for imbalanced target classes\n",
    "Since the resampling approach using SMOTE from the imbalanced-learn library completely crashed the local machine (MacBook Air M2 2022, 8GB, Sonoma 14.5) and the approach of randomly under-sampling the majority target class, while being computationally cheaper, will risk us losing meaningful information, there doesn't seem to be a clear optimal solution yet. By using the simplest and computationally cheapest form of resampling imbalanced target classes, namely by randomly under-sampling the majority class of the target variable, we apply the most (and only) feasible resampling method to handle imbalanced target classes within our limited computational means:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply random undersampling to the training set\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "undersampler = RandomUnderSampler(random_state=42)\n",
    "X_train_resampled, y_train_resampled = undersampler.fit_resample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare the cross-validation strategy\n",
    "- Stratified K-fold for better representation of classes\n",
    "- Saga solver for faster convergence times\n",
    "- Small number of max. iteration for computational limitations\n",
    "- logistic regression with weights for additional class balance\n",
    "- Ridge and Lasso regularization for feature selection insights\n",
    "- Smaller regularization options for more aggressive feature selection due to large dimensionality of dummy encoded data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import StratifiedKFold, GridSearchCV\n",
    "from sklearn.metrics import make_scorer, f1_score\n",
    "\n",
    "# Define the StratifiedKFold cross-validator\n",
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "# Define F1 scorer\n",
    "f1_scorer = make_scorer(f1_score, average='weighted')\n",
    "\n",
    "# Define the logistic regression model with class_weight='balanced'\n",
    "logistic_regression = LogisticRegression(solver='saga', max_iter=100, class_weight='balanced')\n",
    "\n",
    "# Define the parameter grid for Logistic Regression\n",
    "param_grid_lr = {\n",
    "    'penalty': ['l1', 'l2'],\n",
    "    'C': [0.01, 0.1, 1, 10]\n",
    "    }\n",
    "\n",
    "# GridSearchCV for Logistic Regression\n",
    "grid_search_lr = GridSearchCV(logistic_regression, param_grid_lr, cv=skf, scoring=f1_scorer, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply stratified k-fold cross validation gridsearch\n",
    "Fitted to the resampled training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=LogisticRegression(class_weight=&#x27;balanced&#x27;,\n",
       "                                          solver=&#x27;saga&#x27;),\n",
       "             n_jobs=-1,\n",
       "             param_grid={&#x27;C&#x27;: [0.01, 0.1, 1, 10], &#x27;penalty&#x27;: [&#x27;l1&#x27;, &#x27;l2&#x27;]},\n",
       "             scoring=make_scorer(f1_score, average=weighted))</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=LogisticRegression(class_weight=&#x27;balanced&#x27;,\n",
       "                                          solver=&#x27;saga&#x27;),\n",
       "             n_jobs=-1,\n",
       "             param_grid={&#x27;C&#x27;: [0.01, 0.1, 1, 10], &#x27;penalty&#x27;: [&#x27;l1&#x27;, &#x27;l2&#x27;]},\n",
       "             scoring=make_scorer(f1_score, average=weighted))</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(class_weight=&#x27;balanced&#x27;, solver=&#x27;saga&#x27;)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(class_weight=&#x27;balanced&#x27;, solver=&#x27;saga&#x27;)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=LogisticRegression(class_weight='balanced',\n",
       "                                          solver='saga'),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'C': [0.01, 0.1, 1, 10], 'penalty': ['l1', 'l2']},\n",
       "             scoring=make_scorer(f1_score, average=weighted))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_lr.fit(X_train_resampled, y_train_resampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Logistic Regression Model: LogisticRegression(C=0.01, class_weight='balanced', penalty='l1', solver='saga')\n",
      "Logistic Regression F1 Score: 0.7975915850439066\n",
      "                      Feature   Coefficient\n",
      "1                     Backers  1.015463e+01\n",
      "0                        Goal -2.289651e+00\n",
      "84         Subcategory_HipHop -9.202204e-01\n",
      "16           Category_Theater  8.521464e-01\n",
      "164   Subcategory_Video_Games -7.783138e-01\n",
      "..                        ...           ...\n",
      "56          Subcategory_Drama  1.381033e-02\n",
      "55    Subcategory_Documentary -1.449379e-03\n",
      "115  Subcategory_Performances  9.369830e-04\n",
      "74           Subcategory_Food  3.724464e-08\n",
      "106         Subcategory_Music -3.476722e-09\n",
      "\n",
      "[61 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "# Best Logistic Regression Model\n",
    "best_lr = grid_search_lr.best_estimator_\n",
    "\n",
    "# Output the best model and its corresponding hyperparameters\n",
    "print(\"Best Logistic Regression Model:\", best_lr)\n",
    "print(\"Logistic Regression F1 Score:\", f1_score(y_test, best_lr.predict(X_test), average='weighted'))\n",
    "\n",
    "# Output the best hyperparameters\n",
    "\n",
    "coefficients = best_lr.coef_.flatten()\n",
    "feature_names = X_test.columns\n",
    "coeff_df = pd.DataFrame({\n",
    "    'Feature': feature_names,\n",
    "    'Coefficient': coefficients\n",
    "})\n",
    "selected_features = coeff_df[coeff_df['Coefficient'] != 0]\n",
    "selected_features = selected_features.reindex(selected_features['Coefficient'].abs().sort_values(ascending=False).index)\n",
    "print(selected_features)\n",
    "\n",
    "# Best Logistic Regression Model: LogisticRegression(C=0.01, class_weight='balanced', penalty='l1', solver='saga')\n",
    "# Logistic Regression F1 Score: 0.7975915850439066\n",
    "# 61 features selected"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Pre-Selection: \n",
    "Since the categorical features include a large number of unique values and the data set is of a 6-figure order, we might need to perform a more selective lasso regression prior to the modelling for this to be computationally feasible. This also means decreasing the regularization constant (C):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of selected features: 36\n",
      "Selected feature indices: [  0   1   2   3   4   5   6   7   9  10  11  12  15  16  23  24  26  39\n",
      "  41  46  52  70  84  86  88 104 111 129 130 141 145 152 157 164 167 194]\n",
      "Selected feature names: ['Goal' 'Backers' 'Duration' 'Category_Comics' 'Category_Crafts'\n",
      " 'Category_Dance' 'Category_Design' 'Category_Fashion' 'Category_Food'\n",
      " 'Category_Games' 'Category_Journalism' 'Category_Music'\n",
      " 'Category_Technology' 'Category_Theater' 'Subcategory_Apparel'\n",
      " 'Subcategory_Apps' 'Subcategory_Art' 'Subcategory_Classical_Music'\n",
      " 'Subcategory_Comic_Books' 'Subcategory_Country__Folk' 'Subcategory_Dance'\n",
      " 'Subcategory_Fiction' 'Subcategory_HipHop' 'Subcategory_Illustration'\n",
      " 'Subcategory_Indie_Rock' 'Subcategory_Mobile_Games'\n",
      " 'Subcategory_Nonfiction' 'Subcategory_Product_Design'\n",
      " 'Subcategory_Public_Art' 'Subcategory_Rock' 'Subcategory_Shorts'\n",
      " 'Subcategory_Tabletop_Games' 'Subcategory_Theater'\n",
      " 'Subcategory_Video_Games' 'Subcategory_Web' 'Country_United_Kingdom']\n",
      "Shape of the reduced feature set: (214162, 36)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "# Step 1: Define a more selective Logistic Regression model by decreasing C\n",
    "# Smaller C values increase regularization strength\n",
    "logreg = LogisticRegression(penalty=\"l1\", solver=\"saga\", max_iter=100, C=0.01)\n",
    "\n",
    "# Step 2: Fit the feature selector model with the more selective logistic regression\n",
    "selector = SelectFromModel(logreg, threshold=\"mean\")\n",
    "selector.fit(X_train_resampled, y_train_resampled)\n",
    "\n",
    "# Step 3: Apply the selector to reduce dimensionality\n",
    "X_train_reduced = selector.transform(X_train_resampled)\n",
    "X_test_reduced = selector.transform(X_test)\n",
    "\n",
    "# Step 4: Retrieve the indices and names of selected features\n",
    "selected_feature_indices = selector.get_support(indices=True)\n",
    "selected_feature_names = np.array(feature_names)[selected_feature_indices]\n",
    "\n",
    "print(\"Number of selected features:\", len(selected_feature_indices))\n",
    "print(\"Selected feature indices:\", selected_feature_indices)\n",
    "print(\"Selected feature names:\", selected_feature_names)\n",
    "print(\"Shape of the reduced feature set:\", X_train_reduced.shape)\n",
    "\n",
    "# C=0.0001, 0.01\n",
    "# Number of selected features: 36\n",
    "    \"\"\"_summary_\n",
    "    Selected feature indices: [  0   1   2   3   4   5   6   7   9  10  11  12  15  16  23  24  26  39\n",
    "  41  46  52  70  84  86  88 104 111 129 130 141 145 152 157 164 167 194]\n",
    "Selected feature names: ['Goal' 'Backers' 'Duration' 'Category_Comics' 'Category_Crafts'\n",
    " 'Category_Dance' 'Category_Design' 'Category_Fashion' 'Category_Food'\n",
    " 'Category_Games' 'Category_Journalism' 'Category_Music'\n",
    " 'Category_Technology' 'Category_Theater' 'Subcategory_Apparel'\n",
    " 'Subcategory_Apps' 'Subcategory_Art' 'Subcategory_Classical_Music'\n",
    " 'Subcategory_Comic_Books' 'Subcategory_Country__Folk' 'Subcategory_Dance'\n",
    " 'Subcategory_Fiction' 'Subcategory_HipHop' 'Subcategory_Illustration'\n",
    " 'Subcategory_Indie_Rock' 'Subcategory_Mobile_Games'\n",
    " 'Subcategory_Nonfiction' 'Subcategory_Product_Design'\n",
    " 'Subcategory_Public_Art' 'Subcategory_Rock' 'Subcategory_Shorts'\n",
    " 'Subcategory_Tabletop_Games' 'Subcategory_Theater'\n",
    " 'Subcategory_Video_Games' 'Subcategory_Web' 'Country_United_Kingdom']\n",
    "Shape of the reduced feature set: (214162, 36)\n",
    "\n",
    "    \"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/model_selection/_validation.py:378: FitFailedWarning: \n",
      "50 fits failed out of a total of 75.\n",
      "The score on these train-test partitions for these parameters will be set to nan.\n",
      "If these failures are not expected, you can try to debug them by setting error_score='raise'.\n",
      "\n",
      "Below are more details about the failures:\n",
      "--------------------------------------------------------------------------------\n",
      "9 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 1160, in fit\n",
      "    self._validate_params()\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/base.py\", line 600, in _validate_params\n",
      "    validate_parameter_constraints(\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/utils/_param_validation.py\", line 97, in validate_parameter_constraints\n",
      "    raise InvalidParameterError(\n",
      "sklearn.utils._param_validation.InvalidParameterError: The 'penalty' parameter of LogisticRegression must be a str among {'none' (deprecated), 'elasticnet', 'l2', 'l1'} or None. Got 'None' instead.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "1 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 1160, in fit\n",
      "    self._validate_params()\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/base.py\", line 600, in _validate_params\n",
      "    validate_parameter_constraints(\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/utils/_param_validation.py\", line 97, in validate_parameter_constraints\n",
      "    raise InvalidParameterError(\n",
      "sklearn.utils._param_validation.InvalidParameterError: The 'penalty' parameter of LogisticRegression must be a str among {'none' (deprecated), 'l2', 'l1', 'elasticnet'} or None. Got 'None' instead.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "25 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 1291, in fit\n",
      "    fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, prefer=prefer)(\n",
      "                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/utils/parallel.py\", line 63, in __call__\n",
      "    return super().__call__(iterable_with_config)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/joblib/parallel.py\", line 1918, in __call__\n",
      "    return output if self.return_generator else list(output)\n",
      "                                                ^^^^^^^^^^^^\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/joblib/parallel.py\", line 1847, in _get_sequential_output\n",
      "    res = func(*args, **kwargs)\n",
      "          ^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/utils/parallel.py\", line 123, in __call__\n",
      "    return self.function(*args, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 521, in _logistic_regression_path\n",
      "    alpha = (1.0 / C) * (1 - l1_ratio)\n",
      "                         ~~^~~~~~~~~~\n",
      "TypeError: unsupported operand type(s) for -: 'int' and 'NoneType'\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "3 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 1160, in fit\n",
      "    self._validate_params()\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/base.py\", line 600, in _validate_params\n",
      "    validate_parameter_constraints(\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/utils/_param_validation.py\", line 97, in validate_parameter_constraints\n",
      "    raise InvalidParameterError(\n",
      "sklearn.utils._param_validation.InvalidParameterError: The 'penalty' parameter of LogisticRegression must be a str among {'l1', 'l2', 'elasticnet', 'none' (deprecated)} or None. Got 'None' instead.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "2 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 1160, in fit\n",
      "    self._validate_params()\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/base.py\", line 600, in _validate_params\n",
      "    validate_parameter_constraints(\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/utils/_param_validation.py\", line 97, in validate_parameter_constraints\n",
      "    raise InvalidParameterError(\n",
      "sklearn.utils._param_validation.InvalidParameterError: The 'penalty' parameter of LogisticRegression must be a str among {'l1', 'elasticnet', 'l2', 'none' (deprecated)} or None. Got 'None' instead.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "5 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 1160, in fit\n",
      "    self._validate_params()\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/base.py\", line 600, in _validate_params\n",
      "    validate_parameter_constraints(\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/utils/_param_validation.py\", line 97, in validate_parameter_constraints\n",
      "    raise InvalidParameterError(\n",
      "sklearn.utils._param_validation.InvalidParameterError: The 'penalty' parameter of LogisticRegression must be a str among {'l1', 'none' (deprecated), 'l2', 'elasticnet'} or None. Got 'None' instead.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "5 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 1160, in fit\n",
      "    self._validate_params()\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/base.py\", line 600, in _validate_params\n",
      "    validate_parameter_constraints(\n",
      "  File \"/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/utils/_param_validation.py\", line 97, in validate_parameter_constraints\n",
      "    raise InvalidParameterError(\n",
      "sklearn.utils._param_validation.InvalidParameterError: The 'penalty' parameter of LogisticRegression must be a str among {'l2', 'none' (deprecated), 'elasticnet', 'l1'} or None. Got 'None' instead.\n",
      "\n",
      "  warnings.warn(some_fits_failed_message, FitFailedWarning)\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/model_selection/_search.py:952: UserWarning: One or more of the test scores are non-finite: [0.77022697        nan        nan 0.82605299        nan        nan\n",
      " 0.84994351        nan        nan 0.85304871        nan        nan\n",
      " 0.85323814        nan        nan]\n",
      "  warnings.warn(\n",
      "/Users/laylanyrabia/neuefische/kickstarter/project_kickstarter/.venv/lib/python3.11/site-packages/sklearn/linear_model/_sag.py:350: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=LogisticRegression(class_weight=&#x27;balanced&#x27;,\n",
       "                                          max_iter=1000, solver=&#x27;saga&#x27;),\n",
       "             n_jobs=-1,\n",
       "             param_grid={&#x27;C&#x27;: [0.01, 0.1, 1, 10, 100],\n",
       "                         &#x27;penalty&#x27;: [&#x27;l2&#x27;, &#x27;None&#x27;, &#x27;elasticnet&#x27;]},\n",
       "             scoring=make_scorer(f1_score, average=weighted))</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=LogisticRegression(class_weight=&#x27;balanced&#x27;,\n",
       "                                          max_iter=1000, solver=&#x27;saga&#x27;),\n",
       "             n_jobs=-1,\n",
       "             param_grid={&#x27;C&#x27;: [0.01, 0.1, 1, 10, 100],\n",
       "                         &#x27;penalty&#x27;: [&#x27;l2&#x27;, &#x27;None&#x27;, &#x27;elasticnet&#x27;]},\n",
       "             scoring=make_scorer(f1_score, average=weighted))</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(class_weight=&#x27;balanced&#x27;, max_iter=1000, solver=&#x27;saga&#x27;)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(class_weight=&#x27;balanced&#x27;, max_iter=1000, solver=&#x27;saga&#x27;)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=LogisticRegression(class_weight='balanced',\n",
       "                                          max_iter=1000, solver='saga'),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'C': [0.01, 0.1, 1, 10, 100],\n",
       "                         'penalty': ['l2', 'None', 'elasticnet']},\n",
       "             scoring=make_scorer(f1_score, average=weighted))"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the logistic regression model with class_weight='balanced'\n",
    "logistic_regression = LogisticRegression(solver='saga', max_iter=1000, class_weight='balanced')\n",
    "\n",
    "# Define the parameter grid for Logistic Regression\n",
    "param_grid_lr = {\n",
    "    'penalty': ['l2', 'None', 'elasticnet'],\n",
    "    'C': [0.01, 0.1, 1, 10, 100]\n",
    "    }\n",
    "grid_search_lr = GridSearchCV(logistic_regression, param_grid_lr, cv=skf, scoring=f1_scorer, n_jobs=-1)\n",
    "grid_search_lr.fit(X_test_reduced, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Logistic Regression Model: LogisticRegression(C=100, class_weight='balanced', max_iter=1000, solver='saga')\n",
      "Logistic Regression F1 Score: 0.8561574779080967\n"
     ]
    }
   ],
   "source": [
    "# Best Logistic Regression Model\n",
    "best_lr = grid_search_lr.best_estimator_\n",
    "\n",
    "# Output the best model and its corresponding hyperparameters\n",
    "print(\"Best Logistic Regression Model:\", best_lr)\n",
    "print(\"Logistic Regression F1 Score:\", f1_score(y_test, best_lr.predict(X_test_reduced), average='weighted'))\n",
    "\n",
    "# BBest Logistic Regression Model: LogisticRegression(C=100, class_weight='balanced', max_iter=1000, solver='saga')\n",
    "# Logistic Regression F1 Score: 0.856157477908096"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Coefficients: [[-9.59989784e+01  3.77829630e+01 -2.23376787e-01 -3.75060647e-01\n",
      "  -3.55940652e-01  3.53729502e-01 -2.63556514e-01 -4.24585298e-01\n",
      "   8.82002147e-02 -3.72916624e-01 -8.07207222e-01  3.67990206e-01\n",
      "   1.75149404e-01 -3.20280063e-01 -2.49874419e-01 -2.13834026e-02\n",
      "   8.09541193e-01 -7.79343346e-02  1.72412265e-01 -5.60234106e-01\n",
      "  -2.93860926e-01 -5.43553811e-01  1.22064047e-01 -4.01422802e-01\n",
      "  -1.46426455e+00  2.25247563e-01  1.75634473e-01  1.45181035e-01\n",
      "  -7.34003594e-01 -1.41778741e-01  2.16223535e-02 -2.70711563e-01\n",
      "   1.54964689e-01 -6.29641873e-01  1.05919643e-01 -1.50593383e-01\n",
      "  -4.81926015e-01  5.31861448e-01  1.10601732e-01  7.87173606e-01\n",
      "  -2.02879634e-02  5.88207210e-01  1.95043935e-01  9.94170080e-02\n",
      "   3.50387848e-03 -6.97309681e-01  3.55859727e-01  1.55645319e-01\n",
      "  -1.28184771e-01 -7.57334468e-02 -3.97371150e-01 -2.56802447e-01\n",
      "   5.86470519e-01 -1.73344886e-01 -3.03163789e-01 -1.94866877e-01\n",
      "   3.78718637e-01 -2.11292848e-01 -3.71385820e-01 -6.41290624e-01\n",
      "  -9.38123417e-02 -9.49604558e-02  2.91020857e-02  1.25690896e-02\n",
      "  -5.83478662e-01  9.77660326e-03 -1.76240108e-01  7.91707054e-02\n",
      "  -1.76348163e-01  2.21840551e-01 -2.63576295e-01  4.45341635e-02\n",
      "   6.06271094e-01 -3.28575672e-01 -1.82397775e-01 -3.85436319e-01\n",
      "   7.04433075e-02 -9.88161777e-01  9.90518691e-02  3.52883027e-01\n",
      "   6.26451146e-02 -1.66995972e-01  6.54355230e-02 -7.22153357e-01\n",
      "  -9.53422536e-01 -9.67887129e-02  3.25381291e-01 -1.52694438e-01\n",
      "   5.17203437e-01  4.43115942e-01 -4.09855897e-01  3.83430271e-01\n",
      "   8.87704779e-02 -7.09858910e-01 -7.34477560e-02  3.02928265e-01\n",
      "  -3.59707301e-01  1.65757590e-01  5.32294824e-01  8.30525275e-01\n",
      "   4.64907977e-01  1.30980310e-01 -5.89610894e-03  4.25378803e-02\n",
      "  -3.79207872e-01 -2.71062244e-01  4.17116538e-02  2.02763450e-02\n",
      "  -4.78433010e-02  9.33091457e-02 -2.72092671e-01 -2.70237048e-01\n",
      "   1.35682772e-01 -1.88486895e-01  2.82293309e-01  5.57699295e-01\n",
      "  -9.68554485e-02  3.16165567e-01 -1.04785758e+00  1.76464853e-01\n",
      "   6.24008303e-02 -4.10976350e-01 -5.75643903e-01  5.08665965e-02\n",
      "   3.15352151e-01 -1.67748291e-02  3.53367422e-01 -8.14089528e-01\n",
      "  -5.87672383e-02 -5.03294656e-01  3.63761044e-01 -1.86705836e-01\n",
      "   2.01827793e-01  2.99266612e-01 -4.94826321e-02 -5.89474289e-01\n",
      "   1.55891773e-01 -1.68325255e-01  1.03646018e+00 -1.24226947e-01\n",
      "   4.03678112e-01  2.80019296e-01  1.57085516e-01  1.60467087e-01\n",
      "   1.66411874e-01  5.46092634e-01 -2.96221845e-02 -1.07858627e+00\n",
      "   4.05319996e-01  3.96020711e-01 -3.02523353e-01  2.26410334e-01\n",
      "   1.04549607e-01  2.46635846e-01 -8.69370400e-01 -4.13538836e-01\n",
      "  -2.88230393e-01  9.71441197e-02  1.14510704e-01 -1.26742709e-01\n",
      "   6.54085603e-01 -1.22655846e-01 -1.34983273e+00 -3.22257520e-01\n",
      "  -1.17301454e+00 -5.41848177e-01  2.09098622e-01 -1.32749925e+00\n",
      "   1.81226733e-01 -4.50776691e-01  8.36192005e-02 -2.61038194e-01\n",
      "  -1.20808918e-01 -3.10515876e-01  2.87706807e-01 -2.70709181e-01\n",
      "   1.21614878e-01  1.10640677e-01  4.58967026e-01  3.11757567e-01\n",
      "  -5.16772104e-02  7.49833556e-01 -2.30060245e-01 -2.31736697e-01\n",
      "   1.03923152e-01  9.11666043e-02 -1.31263319e-01  3.29244170e-03\n",
      "   1.02788569e-01  9.51252148e-02  9.50279745e-02 -9.52200460e-02\n",
      "  -1.00230883e-02  6.19956311e-01  2.36268153e-01  1.56799670e-01\n",
      "  -6.10464292e-02  2.31864950e-03]]\n",
      "Model Intercept: [-0.04422358]\n",
      "Model Score: [False False False ... False False  True]\n",
      "Confusion Matrix:\n",
      " [[36041  3482]\n",
      " [ 3853 22917]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8620436720645489"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.logistic_regression(X_train_resampled, X_test, y_train_resampled, y_test, metric=\"f1\",verbose=True)\n",
    "# f1 0.8620436720645489"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Coefficients: [[-9.59989784e+01  3.77829630e+01 -2.23376787e-01 -3.75060647e-01\n",
      "  -3.55940652e-01  3.53729502e-01 -2.63556514e-01 -4.24585298e-01\n",
      "   8.82002147e-02 -3.72916624e-01 -8.07207222e-01  3.67990206e-01\n",
      "   1.75149404e-01 -3.20280063e-01 -2.49874419e-01 -2.13834026e-02\n",
      "   8.09541193e-01 -7.79343346e-02  1.72412265e-01 -5.60234106e-01\n",
      "  -2.93860926e-01 -5.43553811e-01  1.22064047e-01 -4.01422802e-01\n",
      "  -1.46426455e+00  2.25247563e-01  1.75634473e-01  1.45181035e-01\n",
      "  -7.34003594e-01 -1.41778741e-01  2.16223535e-02 -2.70711563e-01\n",
      "   1.54964689e-01 -6.29641873e-01  1.05919643e-01 -1.50593383e-01\n",
      "  -4.81926015e-01  5.31861448e-01  1.10601732e-01  7.87173606e-01\n",
      "  -2.02879634e-02  5.88207210e-01  1.95043935e-01  9.94170080e-02\n",
      "   3.50387848e-03 -6.97309681e-01  3.55859727e-01  1.55645319e-01\n",
      "  -1.28184771e-01 -7.57334468e-02 -3.97371150e-01 -2.56802447e-01\n",
      "   5.86470519e-01 -1.73344886e-01 -3.03163789e-01 -1.94866877e-01\n",
      "   3.78718637e-01 -2.11292848e-01 -3.71385820e-01 -6.41290624e-01\n",
      "  -9.38123417e-02 -9.49604558e-02  2.91020857e-02  1.25690896e-02\n",
      "  -5.83478662e-01  9.77660326e-03 -1.76240108e-01  7.91707054e-02\n",
      "  -1.76348163e-01  2.21840551e-01 -2.63576295e-01  4.45341635e-02\n",
      "   6.06271094e-01 -3.28575672e-01 -1.82397775e-01 -3.85436319e-01\n",
      "   7.04433075e-02 -9.88161777e-01  9.90518691e-02  3.52883027e-01\n",
      "   6.26451146e-02 -1.66995972e-01  6.54355230e-02 -7.22153357e-01\n",
      "  -9.53422536e-01 -9.67887129e-02  3.25381291e-01 -1.52694438e-01\n",
      "   5.17203437e-01  4.43115942e-01 -4.09855897e-01  3.83430271e-01\n",
      "   8.87704779e-02 -7.09858910e-01 -7.34477560e-02  3.02928265e-01\n",
      "  -3.59707301e-01  1.65757590e-01  5.32294824e-01  8.30525275e-01\n",
      "   4.64907977e-01  1.30980310e-01 -5.89610894e-03  4.25378803e-02\n",
      "  -3.79207872e-01 -2.71062244e-01  4.17116538e-02  2.02763450e-02\n",
      "  -4.78433010e-02  9.33091457e-02 -2.72092671e-01 -2.70237048e-01\n",
      "   1.35682772e-01 -1.88486895e-01  2.82293309e-01  5.57699295e-01\n",
      "  -9.68554485e-02  3.16165567e-01 -1.04785758e+00  1.76464853e-01\n",
      "   6.24008303e-02 -4.10976350e-01 -5.75643903e-01  5.08665965e-02\n",
      "   3.15352151e-01 -1.67748291e-02  3.53367422e-01 -8.14089528e-01\n",
      "  -5.87672383e-02 -5.03294656e-01  3.63761044e-01 -1.86705836e-01\n",
      "   2.01827793e-01  2.99266612e-01 -4.94826321e-02 -5.89474289e-01\n",
      "   1.55891773e-01 -1.68325255e-01  1.03646018e+00 -1.24226947e-01\n",
      "   4.03678112e-01  2.80019296e-01  1.57085516e-01  1.60467087e-01\n",
      "   1.66411874e-01  5.46092634e-01 -2.96221845e-02 -1.07858627e+00\n",
      "   4.05319996e-01  3.96020711e-01 -3.02523353e-01  2.26410334e-01\n",
      "   1.04549607e-01  2.46635846e-01 -8.69370400e-01 -4.13538836e-01\n",
      "  -2.88230393e-01  9.71441197e-02  1.14510704e-01 -1.26742709e-01\n",
      "   6.54085603e-01 -1.22655846e-01 -1.34983273e+00 -3.22257520e-01\n",
      "  -1.17301454e+00 -5.41848177e-01  2.09098622e-01 -1.32749925e+00\n",
      "   1.81226733e-01 -4.50776691e-01  8.36192005e-02 -2.61038194e-01\n",
      "  -1.20808918e-01 -3.10515876e-01  2.87706807e-01 -2.70709181e-01\n",
      "   1.21614878e-01  1.10640677e-01  4.58967026e-01  3.11757567e-01\n",
      "  -5.16772104e-02  7.49833556e-01 -2.30060245e-01 -2.31736697e-01\n",
      "   1.03923152e-01  9.11666043e-02 -1.31263319e-01  3.29244170e-03\n",
      "   1.02788569e-01  9.51252148e-02  9.50279745e-02 -9.52200460e-02\n",
      "  -1.00230883e-02  6.19956311e-01  2.36268153e-01  1.56799670e-01\n",
      "  -6.10464292e-02  2.31864950e-03]]\n",
      "Model Intercept: [-0.04422358]\n",
      "Model Score: [False False False ... False False  True]\n",
      "Confusion Matrix:\n",
      " [[36041  3482]\n",
      " [ 3853 22917]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8681010644342588"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.logistic_regression(X_train_resampled, X_test, y_train_resampled, y_test, metric=\"precision\")\n",
    "# precision 0.8681010644342588"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Applying preselection with the preset models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8604510596501831"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.logistic_regression(X_train_reduced, X_test_reduced, y_train_resampled, y_test, metric=\"f1\")\n",
    "# 0.8604510596501831 f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8702812165673238"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.logistic_regression(X_train_reduced, X_test_reduced, y_train_resampled, y_test, metric=\"precision\")\n",
    "# 0.8702812165673238 precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.879720536863394"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base.decision_tree(X_train_reduced,X_test_reduced,y_train_resampled,y_test,metric=\"f1\")\n",
    "# 0.879720536863394"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting xgboost\n",
      "  Downloading xgboost-2.1.1-py3-none-macosx_12_0_arm64.whl.metadata (2.1 kB)\n",
      "Requirement already satisfied: numpy in ./.venv/lib/python3.11/site-packages (from xgboost) (1.24.3)\n",
      "Requirement already satisfied: scipy in ./.venv/lib/python3.11/site-packages (from xgboost) (1.14.0)\n",
      "Downloading xgboost-2.1.1-py3-none-macosx_12_0_arm64.whl (1.9 MB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: xgboost\n",
      "Successfully installed xgboost-2.1.1\n"
     ]
    }
   ],
   "source": [
    "!pip install xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1\n",
      "logistic regression 0.8620436720645489\n",
      "decision tree 0.8838409723239127\n",
      "accuracy\n",
      "logistic regression 0.889354833843694\n",
      "decision tree 0.9037454934910172\n",
      "precision\n",
      "logistic regression 0.8681010644342588\n",
      "decision tree 0.8671479366447581\n",
      "recall\n",
      "logistic regression 0.8560702278670154\n",
      "decision tree 0.9015315651849085\n"
     ]
    }
   ],
   "source": [
    "for metric in [\"f1\",\"accuracy\", \"precision\", \"recall\"]:\n",
    "    print(metric)\n",
    "    print(f\"logistic regression {base.logistic_regression(X_train_resampled, X_test, y_train_resampled, y_test, metric=metric)}\")\n",
    "    print(f\"decision tree {base.decision_tree(X_train_resampled, X_test, y_train_resampled, y_test, metric=metric)}\")\n",
    "    # print(f\"XGBoost {base.xgb(X_train_resampled, X_test, y_train_resampled, y_test, metric=metric)}\")\n",
    "\n",
    "    \"\"\"\n",
    "f1\n",
    "logistic regression 0.8620436720645489\n",
    "decision tree 0.8838409723239127\n",
    "\n",
    "accuracy\n",
    "logistic regression 0.889354833843694\n",
    "decision tree 0.9037454934910172\n",
    "\n",
    "precision\n",
    "logistic regression 0.8681010644342588\n",
    "decision tree 0.8671479366447581\n",
    "\n",
    "recall\n",
    "logistic regression 0.8560702278670154\n",
    "decision tree 0.9015315651849085\n",
    "    \"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}