for_tabular_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from lightgbm import LGBMClassifier\n",
    "# Decicion Tree Classifier Model\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "# Random Forest Model \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "# Extreme gradient boosting model\n",
    "from xgboost import XGBClassifier\n",
    "# Categorical Boosting Model \n",
    "from catboost import CatBoostClassifier, Pool\n",
    "# KFold (Multiple model runs)\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train=pd.read_csv('data/train.csv')\n",
    "label_encoder = LabelEncoder()\n",
    "df_train['Target'] = label_encoder.fit_transform(df_train['Target'])\n",
    "X=df_train.drop('Target',axis=1)\n",
    "y=df_train['Target']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_lgbm_model(X, y, lgbm_params=None,cv=5):\n",
    "    '''\n",
    "    This function tests the LGBM model and prints scores.\n",
    "\n",
    "    Args:\n",
    "        X: Dataframe with predictor features.\n",
    "        y: Series with the target feature.\n",
    "        lgbm_params: A dict of the LGBM tuned parameters (default=None).\n",
    "        cv: cross validation\n",
    "\n",
    "    Returns:\n",
    "        average_score: float representing the average score calculated after 5 model folds.\n",
    "        label_scores: list containing scores for each target feature (if by_target is True in test_params)\n",
    "    '''\n",
    "    folds = StratifiedKFold(n_splits=cv, random_state=123, shuffle=True)\n",
    "    fin_scores = []\n",
    "    label_scores = []\n",
    "    unique_targets = y.unique()\n",
    "    \n",
    "    for i, (train_index, test_index) in enumerate(folds.split(X, y)):\n",
    "        Xf_train = X.iloc[train_index]\n",
    "        yf_train = y.iloc[train_index]\n",
    "\n",
    "        X_test = X.iloc[test_index]\n",
    "        y_test = y.iloc[test_index]\n",
    "\n",
    "        if lgbm_params is not None:\n",
    "            lgbmc = LGBMClassifier(**lgbm_params)\n",
    "        else:\n",
    "            lgbmc = LGBMClassifier(verbose=-1)\n",
    "\n",
    "        lgbmc.fit(X=Xf_train, y=yf_train)\n",
    "\n",
    "        y_pred = lgbmc.predict(X=X_test)\n",
    "\n",
    "        target_scores=[]\n",
    "        for target in unique_targets:\n",
    "            target_mask = y_test == target\n",
    "            target_pred = y_pred[target_mask]\n",
    "            target_true = y_test[target_mask]\n",
    "            score = accuracy_score(target_true, target_pred)\n",
    "            target_scores.append(score)\n",
    "\n",
    "        fin_scores.append(np.mean(target_scores))\n",
    "        label_scores.extend(target_scores) \n",
    "\n",
    "    average_score = np.mean(fin_scores)\n",
    "\n",
    "    return average_score, label_scores\n",
    "# test_lgbm_model(X,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_random_forest(X, y, by_target=False, rf_params=None,cv=5):\n",
    "    '''\n",
    "    This function tests the Random Forest model and prints scores.\n",
    "\n",
    "    Args:\n",
    "        X: Dataframe with predictor features.\n",
    "        y: Series with the target feature.\n",
    "        by_target: Print scores for each target (default=False).\n",
    "        cv: cross validation\n",
    "\n",
    "    Returns:\n",
    "        average_score: float representing the average score calculated after 5 model folds.\n",
    "        label_scores: list containing scores for each target feature (if by_target is True)\n",
    "    '''\n",
    "    folds = StratifiedKFold(n_splits=cv, random_state=123, shuffle=True)\n",
    "    fin_scores = []\n",
    "    label_scores = []\n",
    "    unique_targets = y.unique()\n",
    "    \n",
    "    for i, (train_index, test_index) in enumerate(folds.split(X, y)):\n",
    "        Xf_train = X.iloc[train_index]\n",
    "        yf_train = y.iloc[train_index]\n",
    "\n",
    "        X_test = X.iloc[test_index]\n",
    "        y_test = y.iloc[test_index]\n",
    "        \n",
    "        if rf_params is not None:\n",
    "            rfc = RandomForestClassifier(**rf_params)\n",
    "        else:\n",
    "            rfc = RandomForestClassifier()\n",
    "            \n",
    "        rfc.fit(Xf_train, yf_train)\n",
    "\n",
    "        y_pred = rfc.predict(X_test)\n",
    "        \n",
    "        target_scores=[]\n",
    "        for target in unique_targets:\n",
    "            target_mask = y_test == target\n",
    "            target_pred = y_pred[target_mask]\n",
    "            target_true = y_test[target_mask]\n",
    "            score = accuracy_score(target_true, target_pred)\n",
    "            target_scores.append(score)\n",
    "\n",
    "        fin_scores.append(np.mean(target_scores))  \n",
    "        label_scores.extend(target_scores) \n",
    "        \n",
    "    average_score = np.mean(fin_scores)\n",
    "    \n",
    "    return average_score, label_scores\n",
    "# test_random_forest(X,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_xgboost(X, y, by_target=False, xgb_params=None,cv=5):\n",
    "    '''\n",
    "    This function tests the XGBoost model and prints scores.\n",
    "\n",
    "    Args:\n",
    "        X: Dataframe with predictor features.\n",
    "        y: Series with the target feature.\n",
    "        by_target: Print scores for each target (default=False).\n",
    "        cv: cross validation\n",
    "\n",
    "    Returns:\n",
    "        average_score: float representing the average score calculated after 5 model folds.\n",
    "        label_scores: list containing scores for each target feature (if by_target is True)\n",
    "    '''\n",
    "    folds = StratifiedKFold(n_splits=cv, random_state=123, shuffle=True)\n",
    "    fin_scores = []\n",
    "    label_scores = []\n",
    "    unique_targets = y.unique()\n",
    "    \n",
    "    for i, (train_index, test_index) in enumerate(folds.split(X, y)):\n",
    "        Xf_train = X.iloc[train_index]\n",
    "        yf_train = y.iloc[train_index]\n",
    "\n",
    "        X_test = X.iloc[test_index]\n",
    "        y_test = y.iloc[test_index]\n",
    "\n",
    "        if xgb_params is not None:\n",
    "            xgb = XGBClassifier(**xgb_params)\n",
    "        else:\n",
    "            xgb = XGBClassifier(verbose=-1)\n",
    "\n",
    "        xgb.fit(X=Xf_train, y=yf_train)\n",
    "\n",
    "        y_pred = xgb.predict(X=X_test)\n",
    "\n",
    "        target_scores=[]\n",
    "        for target in unique_targets:\n",
    "            target_mask = y_test == target\n",
    "            target_pred = y_pred[target_mask]\n",
    "            target_true = y_test[target_mask]\n",
    "            score = accuracy_score(target_true, target_pred)\n",
    "            target_scores.append(score)\n",
    "\n",
    "        fin_scores.append(np.mean(target_scores)) \n",
    "        label_scores.extend(target_scores)  \n",
    "\n",
    "    average_score = np.mean(fin_scores)\n",
    "\n",
    "    return average_score, label_scores\n",
    "# test_xgboost(X,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_catboost(X, y, by_target=False, cat_params=None,cv=5):\n",
    "    '''\n",
    "    This function tests the CatBoost model and prints scores.\n",
    "\n",
    "    Args:\n",
    "        X: Dataframe with predictor features.\n",
    "        y: Series with the target feature.\n",
    "        by_target: Print scores for each target (default=False).\n",
    "        cv: cross valiadtion\n",
    "\n",
    "    Returns:\n",
    "        average_score: float representing the average score calculated after 5 model folds.\n",
    "        label_scores: list containing scores for each target feature (if by_target is True)\n",
    "    '''\n",
    "    folds = StratifiedKFold(n_splits=cv, random_state=123, shuffle=True)\n",
    "    fin_scores = []\n",
    "    label_scores = []\n",
    "    unique_targets = y.unique()\n",
    "    \n",
    "    for i, (train_index, test_index) in enumerate(folds.split(X, y)):\n",
    "        Xf_train = X.iloc[train_index]\n",
    "        yf_train = y.iloc[train_index]\n",
    "\n",
    "        X_test = X.iloc[test_index]\n",
    "        y_test = y.iloc[test_index]\n",
    "        \n",
    "        if cat_params is not None:\n",
    "            cat = CatBoostClassifier(**cat_params)\n",
    "        else:\n",
    "            cat = CatBoostClassifier(verbose=0)\n",
    "            \n",
    "        train_pool = Pool(\n",
    "                data=Xf_train,\n",
    "                label=yf_train\n",
    "                )\n",
    "        \n",
    "        cat.fit(train_pool)\n",
    "\n",
    "        y_pred = cat.predict(X_test)\n",
    "\n",
    "        target_scores=[]\n",
    "        for target in unique_targets:\n",
    "            target_mask = y_test == target\n",
    "            target_pred = y_pred[target_mask]\n",
    "            target_true = y_test[target_mask]\n",
    "            score = accuracy_score(target_true, target_pred)\n",
    "            target_scores.append(score)\n",
    "\n",
    "        fin_scores.append(np.mean(target_scores)) \n",
    "        label_scores.extend(target_scores) \n",
    "\n",
    "    average_score = np.mean(fin_scores)\n",
    "\n",
    "    return average_score, label_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbm_params = {\n",
    "                 'objective': 'multiclass', # multiclass target: 'Graduated', 'Dropout', or 'Enrolled'\n",
    "                 'data_sample_strategy': 'goss', # Gradient-based One-Sided Sampling\n",
    "                 'tree_learner': 'feature', # split nodes based on the best feature\n",
    "                 'n_estimators': 743, # number of boosting iterations\n",
    "                 'learning_rate': 0.02636616162598401, # step size for updatig model weights\n",
    "                 'feature_fraction': 0.298183729482288, # about 30% of features considered at each split\n",
    "                 'lambda_l1': 8.242410039948067e-07, # L1 regulation penalization - adding magnitude of weights to the loss\n",
    "                 'lambda_l2': 0.4063299210212167, # L2 regulation penalization = adding the square of weights to the loss\n",
    "                 'num_leaves': 699, # Maximum number of leaves (terminal nodes) to use\n",
    "                 'max_depth': 8, # Maximum tree depth (levels) allowed\n",
    "                 'colsample_bytree': 0.7975468653525116, # proportion of samples to randomly choose at each iteration\n",
    "                 'min_child_samples': 102, # Minimum number of samples needed per leaf\n",
    "                 'min_sum_hessian_in_leaf': 5.440582524630883, # Minimum sum of squared gradients allowed in a leaf node\n",
    "                 'min_gain_to_split': 0.7247318987185962, # Minumum gain (model score improvement) to make further leaf partitions\n",
    "                 'max_bin': 156, # Maximum numer of bins used for discretitizing features before tree splits\n",
    "                 'top_rate': 0.6132659772851583, # Top proportion of features to choose (~61%)\n",
    "                 'verbose': -1, # Turn off warnings and model logs for a cleaner look\n",
    "                 'random_state': 123 # Random state value for repeatablity\n",
    "}\n",
    "rf_params = {\n",
    "    'n_estimators': 100, # number of trees in the forest\n",
    "    'max_depth': 10, # maximum depth of individual trees\n",
    "    'min_samples_split': 2, # minimum samples required to split a node\n",
    "    'min_samples_leaf': 1, # minimum samples required at each leaf node\n",
    "    'max_features': 'auto', # number of features considered at each split (default: 'auto')\n",
    "    'bootstrap': True, # whether to use bootstrap aggregating (bagging)\n",
    "    'random_state': 123 # for model repeatability\n",
    "}\n",
    "xgb_params = {\n",
    "    'objective': 'multiclass', # Adjust for your problem type (classification or regression)\n",
    "    'n_estimators': 100, # Number of boosting rounds\n",
    "    'learning_rate': 0.1, # Learning rate (controls step size)\n",
    "    'max_depth': 6, # Maximum depth of individual trees\n",
    "    'min_child_weight': 1, # Minimum sum of hessian for child nodes\n",
    "    'gamma': 0, # Minimum loss reduction required for a split\n",
    "    'subsample': 0.8, # Fraction of samples for each boosting round\n",
    "    'colsample_bytree': 0.8, # Fraction of features considered per tree\n",
    "    'colsample_bylevel': 1, # Fraction of features considered per level\n",
    "    'reg_alpha': 1, # L1 regularization term\n",
    "    'reg_lambda': 0, # L2 regularization term\n",
    "    'random_state': 123 # for model repeatability\n",
    "}\n",
    "cat_params = {\n",
    "    #'loss_function': 'MultiClass', # adjust for your problem type\n",
    "    #'eval_metric': 'MultiClass', # adjust for your problem (classification metric)\n",
    "    'iterations': 100, # number of boosting rounds\n",
    "    #'learning_rate': 0.1, # learning rate (controls step size)\n",
    "    'depth': 4, # maximum depth of individual trees\n",
    "    #'l2_leaf_reg': 3, # L2 regularization term on leaf values\n",
    "    #'min_data_in_leaf': 10, # minimum samples required at each leaf node\n",
    "    #'random_strength': 1, # amount of randomness in feature selection\n",
    "    #'od_wait': 20, # patience for early stopping based on validation metric\n",
    "    'verbose': 0, # turn off log output for readability\n",
    "    'random_state': 123 # for model repeatability\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_params(X, y, by_target=False, model_type=\"lgbm\", lgbm_params=None, rf_params=None, xgb_params=None, cat_params=None):\n",
    "    '''\n",
    "    This function checks the model type and calls the appropriate model testing function.\n",
    "\n",
    "    Args:\n",
    "        X: Dataframe with predictor features.\n",
    "        y: Series with the target feature.\n",
    "        by_target: Print scores for each target (default=False).\n",
    "        model_type: Type of model to test (default=\"lgbm\").\n",
    "        lgbm_params: A dict of the LGBM tuned parameters (default=None).\n",
    "        rf_params: A dict of the Random Forest tuned parameters (default=None).\n",
    "\n",
    "    Returns:\n",
    "        None - if only one score return None.\n",
    "        max(scores) - if multiple scores, return the highest one.\n",
    "  '''\n",
    "    scores = []\n",
    "    target_names = ['Graduate', 'Dropout', 'Enrolled']\n",
    "    \n",
    "    if model_type == \"lgbm\":\n",
    "        average_score, label_scores = test_lgbm_model(X, y, lgbm_params)\n",
    "    elif model_type == \"random_forest\":\n",
    "        average_score, label_scores = test_random_forest(X, y, rf_params=rf_params)\n",
    "    elif model_type == 'xgboost':\n",
    "        average_score, label_scores = test_xgboost(X, y, xgb_params=xgb_params)\n",
    "    elif model_type == 'catboost':\n",
    "        average_score, label_scores = test_catboost(X, y, cat_params=cat_params)\n",
    "    else:\n",
    "        print(f\"Model type '{model_type}' not supported.\")\n",
    "        return None\n",
    "\n",
    "    if by_target:\n",
    "        unique_targets = y.unique()\n",
    "        label_score_names = unique_targets.tolist()\n",
    "        \n",
    "        for i, score in enumerate(zip(label_score_names, label_scores)):\n",
    "            target_name, target_score = score\n",
    "            print(f'Score for target {target_names[target_name]}: {target_score}')\n",
    "\n",
    "    if len(scores) > 1:\n",
    "        return max(scores)\n",
    "    else:\n",
    "        return average_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_params(X,y,by_target=True,model_type='lgbm',lgbm_params=lgbm_params)\n",
    "test_params(X,y,by_target=True,model_type='random_forest',rf_params=rf_params)\n",
    "test_params(X,y,by_target=True,model_type='xgboost',xgb_params=xgb_params)\n",
    "test_params(X,y,by_target=True,model_type='catboost',cat_params=cat_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train and voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train LightGBM Model\n",
    "LGBM_model = LGBMClassifier(**lgbm_params)  \n",
    "LGBM_model.fit(X, y)\n",
    "\n",
    "# Train XGBoost Model\n",
    "XGBoost_model = XGBClassifier(**xgb_params) \n",
    "XGBoost_model.fit(X, y)\n",
    "\n",
    "# Train Random Forest Model\n",
    "RandomForest_model = RandomForestClassifier()  \n",
    "RandomForest_model.fit(X, y)\n",
    "\n",
    "# Train CatBoost Model\n",
    "Categorical_Model = CatBoostClassifier(verbose=0) \n",
    "Categorical_Model.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test=pd.read_csv('data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions from each model\n",
    "predictions_by_model = {}\n",
    "for model_name, model in [('lgbm', LGBM_model),\n",
    "                          ('xgboost', XGBoost_model), \n",
    "                          ('random_forest', RandomForest_model),\n",
    "                          ('catboost', Categorical_Model)]:\n",
    "    predictions_by_model[model_name] = model.predict(df_test)\n",
    "\n",
    "# Final prediction selection based on class-wise strengths\n",
    "final_predictions = []\n",
    "for i in range(len(df_test)):\n",
    "    lgbm_pred = predictions_by_model['lgbm'][i]\n",
    "    xgboost_pred = predictions_by_model['xgboost'][i]\n",
    "    rf_pred = predictions_by_model['random_forest'][i]\n",
    "    cat_pred = predictions_by_model['catboost'][i]\n",
    "\n",
    "    # Choose prediction based on class and model performance\n",
    "    #if rf_pred in [0, 2]:\n",
    "        #final_predictions.append(rf_pred)  # Use Random Forest if it predicts None\n",
    "    if xgboost_pred == 2:\n",
    "        final_predictions.append(xgboost_pred)  # Use XGBoost if it predicts 0\n",
    "    elif lgbm_pred in [0, 1]:\n",
    "        final_predictions.append(lgbm_pred)  # Use LGBM if it predicts 1 or 2\n",
    "    else:\n",
    "        # Default prediction of most frequent class \n",
    "        final_predictions.append(lgbm_pred)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# output predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the id series, and the final output dataframe\n",
    "id_val = df_test['id']\n",
    "output = pd.DataFrame({\n",
    "    'id': id_val,\n",
    "    'Target': final_predictions\n",
    "})\n",
    "output['Target']=label_encoder.inverse_transform(output['Target'])\n",
    "submission = output.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py311",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}