spambase_dataset_loader_nb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ba98f19d-1bcc-46c0-87b2-12e2ca050459",
   "metadata": {},
   "source": [
    "#  Spambase Dataset Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3ef6d9ac-f713-4aa9-a45e-9bc21c471ad7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: ipynb in c:\\users\\acristobal\\appdata\\roaming\\python\\python311\\site-packages (0.5.1)\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: nbimporter in c:\\users\\acristobal\\appdata\\roaming\\python\\python311\\site-packages (0.3.4)\n"
     ]
    }
   ],
   "source": [
    "# Importing an ipynb file from another ipynb file\n",
    "!pip install ipynb\n",
    "\n",
    "# Importing functions from another jupyter notebook\n",
    "!pip install nbimporter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "63267e30-dc85-47a3-b6a3-ca2036708a85",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "import statsmodels.api as sm\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "\n",
    "class SpambaseDatasetLoader():\n",
    "    \n",
    "    def __init__(self):\n",
    "        print('init Loader notebook')\n",
    "    \n",
    "    def load_dataset(self, url='https://archive.ics.uci.edu/ml/machine-learning-databases/spambase/spambase.data'):\n",
    "        '''\n",
    "        \n",
    "        (string) --> None\n",
    "        \n",
    "        This function retrieves the spambase.data from University of California Irvine (UCI) - Dataset Repository\n",
    "        and save into two files:\n",
    "        \n",
    "        1. spambase_all.csv - The CSV file that contains all the rows from spambase.data.\n",
    "        2. spambase_balance.csv - The CSV file that rows balance between spam and not spam records.\n",
    "        \n",
    "        '''\n",
    "        print('Loading dataset.')\n",
    "        \n",
    "        columns = [\n",
    "            'word_freq_make',\n",
    "            'word_freq_address',\n",
    "            'word_freq_all',\n",
    "            'word_freq_3d',\n",
    "            'word_freq_our',\n",
    "            'word_freq_over',\n",
    "            'word_freq_remove',\n",
    "            'word_freq_internet',\n",
    "            'word_freq_order',\n",
    "            'word_freq_mail',\n",
    "            'word_freq_receive',\n",
    "            'word_freq_will',\n",
    "            'word_freq_people',\n",
    "            'word_freq_report',\n",
    "            'word_freq_addresses',\n",
    "            'word_freq_free',\n",
    "            'word_freq_business',\n",
    "            'word_freq_email',\n",
    "            'word_freq_you',\n",
    "            'word_freq_credit',\n",
    "            'word_freq_your',\n",
    "            'word_freq_font',\n",
    "            'word_freq_000',\n",
    "            'word_freq_money',\n",
    "            'word_freq_hp',\n",
    "            'word_freq_hpl',\n",
    "            'word_freq_george',\n",
    "            'word_freq_650',\n",
    "            'word_freq_lab',\n",
    "            'word_freq_labs',\n",
    "            'word_freq_telnet',\n",
    "            'word_freq_857',\n",
    "            'word_freq_data',\n",
    "            'word_freq_415',\n",
    "            'word_freq_85',\n",
    "            'word_freq_technology',\n",
    "            'word_freq_1999',\n",
    "            'word_freq_parts',\n",
    "            'word_freq_pm',\n",
    "            'word_freq_direct',\n",
    "            'word_freq_cs',\n",
    "            'word_freq_meeting',\n",
    "            'word_freq_original',\n",
    "            'word_freq_project',\n",
    "            'word_freq_re',\n",
    "            'word_freq_edu',\n",
    "            'word_freq_table',\n",
    "            'word_freq_conference',\n",
    "            'char_freq_;',\n",
    "            'char_freq_(',\n",
    "            'char_freq_[',\n",
    "            'char_freq_!',\n",
    "            'char_freq_$',\n",
    "            'char_freq_#',\n",
    "            'capital_run_length_average',\n",
    "            'capital_run_length_longest',\n",
    "            'capital_run_length_total',\n",
    "            'spam_nonspam']\n",
    "        \n",
    "        # retrieve the spam data from icu.\n",
    "        spam_data = pd.read_csv(url, header=None, names=columns, index_col=False)\n",
    "\n",
    "        # make the label the first feature.\n",
    "        spam_data.insert(0, 'target_spam_nonspam', spam_data['spam_nonspam'])\n",
    "        spam_data = spam_data.drop('spam_nonspam', axis = 1)\n",
    "\n",
    "        # save the data to the new csv.\n",
    "        spam_data.to_csv('./datasets/spambase_all.csv', index = False)\n",
    "        print('Loading spambase_all.csv completed.')\n",
    "            \n",
    "        spam = spam_data[spam_data['target_spam_nonspam']==1]\n",
    "        non_spam = spam_data[spam_data['target_spam_nonspam']==0]\n",
    "        non_spam = non_spam.sample(n=len(spam), random_state=101)\n",
    "        spam_data_balance = pd.concat([spam,non_spam],axis=0)            \n",
    "        print('Loading spambase_balance.csv completed.')\n",
    "    \n",
    "    def get_full_dataset(self):\n",
    "        '''\n",
    "        \n",
    "        (None) --> Dataframe\n",
    "        \n",
    "        This function returns the spambase imbalance dataset.\n",
    "        \n",
    "        '''\n",
    "        print('get_full_dataset')\n",
    "        return pd.read_csv('./datasets/spambase_all.csv')        \n",
    "    \n",
    "    def get_balance_dataset(self): \n",
    "        '''\n",
    "        \n",
    "        (None) --> Dataframe\n",
    "        \n",
    "        This function returns the spambase dataset based on the balance dataset.\n",
    "                \n",
    "        '''\n",
    "        print('get_balance_data')\n",
    "        return pd.read_csv('./datasets/spambase_balance.csv')\n",
    "\n",
    "    def backward_elimitation(self, data):\n",
    "        '''\n",
    "\n",
    "        (DataFrame) --> Array\n",
    "        \n",
    "        This backward elimination technique used Logistic regression-based model which \n",
    "        selects the features based on the p-value score of the feature.\n",
    "        The features with p-value less than 0.05 are considered to be the more relevant feature\n",
    "        Source: https://www.analyticsvidhya.com/blog/2021/04/discovering-the-shades-of-feature-selection-methods/\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        data: Dataframe that will be used in feature selection.\n",
    "        \n",
    "        Returns\n",
    "        ----------\n",
    "        Array of features\n",
    "        \n",
    "        '''\n",
    "        print('Feature Selection using Backward Elimination')\n",
    "        threshold = 0.05\n",
    "        selected_columns = data.columns\n",
    "        selected_columns = selected_columns[1:].values # remove the label\n",
    "        \n",
    "        X = data.iloc[:,1:]\n",
    "        y = data.iloc[:,0]\n",
    "        \n",
    "        # Add constant to predictors for statsmodels\n",
    "        X = sm.add_constant(X)\n",
    "\n",
    "        logit_model = sm.Logit(y, X).fit()\n",
    "\n",
    "        # Perform backward elimination\n",
    "        while len(X.columns) > 1:\n",
    "            # Fit Logit model with all predictors except one\n",
    "            logit_model = sm.Logit(y, X.iloc[:, :-1]).fit()\n",
    "\n",
    "            # Get p-values for each predictor\n",
    "            p_values = logit_model.pvalues\n",
    "\n",
    "            # Remove predictor with highest p-value\n",
    "            max_p_value = p_values.idxmax()\n",
    "            if p_values[max_p_value] > threshold:\n",
    "                X = X.drop(max_p_value, axis=1)\n",
    "            else:\n",
    "                break\n",
    "                \n",
    "        X = X.drop('const', axis=1) # cleanup remove the const column before returning\n",
    "        return X.columns\n",
    "    \n",
    "    def pearson_correlation(self, data):\n",
    "        '''\n",
    "        \n",
    "        (DataFrame) --> Array \n",
    "        \n",
    "        Pearson Correlation is used to construct a correlation matrix that measures the linear association \n",
    "        between two features and gives a value between -1 and 1 indicating how related the two features are to one another.\n",
    "        This measures the degree to which two features are interdependent by computing the association \n",
    "        between each feature and the target variable, the one exerting high impact on the target can be picked out\n",
    "        \n",
    "        A value of 1 indicates a positive correlation, -1 indicates a negative correlation and 0 indicates no correlation between the features.\n",
    "        Source: https://www.analyticsvidhya.com/blog/2021/04/discovering-the-shades-of-feature-selection-methods/\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        data: Dataframe that will be used in feature selection.\n",
    "        \n",
    "        Returns\n",
    "        ----------\n",
    "        Array of features       \n",
    "        \n",
    "        '''\n",
    "        print('Feature Selection using Pearson Correlation')\n",
    "        corr = data.corr()\n",
    "        cor_target = abs(corr['target_spam_nonspam'])\n",
    "        \n",
    "        #Selecting highly correlated features\n",
    "        relevant_features = cor_target[cor_target>0.2]\n",
    "        selected_columns = relevant_features.keys().to_list()\n",
    "       \n",
    "        selected_columns.remove('target_spam_nonspam')\n",
    "        return selected_columns\n",
    "    \n",
    "    def chi2(self, data):\n",
    "        '''\n",
    "        \n",
    "        (DataFrame) --> Array \n",
    "        \n",
    "        A chi-square test is used in statistical models to check the independence of attributes.\n",
    "        The model measures the degree of deviation between the expected and actual response.\n",
    "        The lower the value of Chi-square, the less dependent the variables are to one another, \n",
    "        and the higher the value the more is their correlation. \n",
    "        Source: https://www.analyticsvidhya.com/blog/2021/04/discovering-the-shades-of-feature-selection-methods/        \n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        data: Dataframe that will be used in feature selection.\n",
    "        \n",
    "        Returns\n",
    "        ----------\n",
    "        Array of features\n",
    "        \n",
    "        '''\n",
    "        print('Feature Selection using Chi-squared')\n",
    "        X = data.iloc[:,1:]\n",
    "        y = data.iloc[:,0]\n",
    "        \n",
    "        chi2_features = SelectKBest(chi2, k=19)\n",
    "        X_kbest_features = chi2_features.fit_transform(X, y)\n",
    "        mask=chi2_features.get_support()\n",
    "        selected_columns =[]\n",
    "        for bool, feature in zip(mask, X.columns):\n",
    "            if (bool):\n",
    "                selected_columns.append(feature)\n",
    "\n",
    "        return selected_columns\n",
    "        \n",
    "    \n",
    "    def perform_feature_selection(self, data, feature_selection_type):\n",
    "        '''\n",
    "                        \n",
    "        (DataFrame, float) --> Dataframe\n",
    "        \n",
    "        \n",
    "        This function performs feature selection based on the provided feature selection type.  \n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        data: Dataframe that will be used in feature selection.\n",
    "        \n",
    "        feature_selection_type: The type of feature selection to be performed.\n",
    "        1. stepwise_back - use stepwise backward elimination technique in feature selection..\n",
    "        2. pearson - perform the feature selection using pearson correlation.\n",
    "        3. chi2 - perform the feature selection using chi-squared.\n",
    "        \n",
    "        Returns\n",
    "        ----------        \n",
    "        DataFrame : The DataFrame after performing feature selection.\n",
    "        \n",
    "        '''\n",
    "        print('The shape before feature selection: {}'.format(data.shape))\n",
    "        \n",
    "        if feature_selection_type == 'stepwise_back':        \n",
    "            selected_columns = self.backward_elimitation(data)\n",
    "        elif feature_selection_type == 'pearson':\n",
    "            selected_columns = self.pearson_correlation(data)\n",
    "        elif feature_selection_type == 'chi2':\n",
    "            selected_columns = self.chi2(data)\n",
    "        else:\n",
    "            raise ValueError('Unknown type of feature selection.')\n",
    "        \n",
    "        y = pd.DataFrame()\n",
    "        y['target_spam_nonspam'] = data.iloc[:,0]\n",
    "        \n",
    "        X = pd.DataFrame(data = data.iloc[:,1:], columns = selected_columns)\n",
    "        print('The shape after feature selection: {}'.format(X.shape))\n",
    "               \n",
    "        return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "35f97743-f46a-4945-bdcb-c58f8cb30821",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "init Loader notebook\n"
     ]
    }
   ],
   "source": [
    "import nbimporter\n",
    "import spambase_dataset_loader_nb\n",
    "\n",
    "loader = spambase_dataset_loader_nb.SpambaseDatasetLoader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ebb1f362-7771-470e-ab33-fd0ff5aae686",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method load_dataset in module spambase_dataset_loader_nb:\n",
      "\n",
      "load_dataset(url='https://archive.ics.uci.edu/ml/machine-learning-databases/spambase/spambase.data') method of spambase_dataset_loader_nb.SpambaseDatasetLoader instance\n",
      "    (string) --> None\n",
      "    \n",
      "    This function retrieves the spambase.data from University of California Irvine (UCI) - Dataset Repository\n",
      "    and save into two files:\n",
      "    \n",
      "    1. spambase_all.csv - The CSV file that contains all the rows from spambase.data.\n",
      "    2. spambase_balance.csv - The CSV file that rows balance between spam and not spam records.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(loader.load_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3184c42c-cd7b-471e-bdac-cb93410c16c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method perform_feature_selection in module spambase_dataset_loader_nb:\n",
      "\n",
      "perform_feature_selection(data, feature_selection_type) method of spambase_dataset_loader_nb.SpambaseDatasetLoader instance\n",
      "    (DataFrame, float) --> Dataframe\n",
      "    \n",
      "    \n",
      "    This function performs feature selection based on the provided feature selection type.  \n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    data: Dataframe that will be used in feature selection.\n",
      "    \n",
      "    feature_selection_type: The type of feature selection to be performed.\n",
      "    1. stepwise_back - use stepwise backward elimination technique in feature selection..\n",
      "    2. pearson - perform the feature selection using pearson correlation.\n",
      "    3. chi2 - perform the feature selection using chi-squared.\n",
      "    \n",
      "    Returns\n",
      "    ----------        \n",
      "    DataFrame : The DataFrame after performing feature selection.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(loader.perform_feature_selection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "23564199-2f8d-4be7-848a-c637def67186",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method backward_elimitation in module spambase_dataset_loader_nb:\n",
      "\n",
      "backward_elimitation(data) method of spambase_dataset_loader_nb.SpambaseDatasetLoader instance\n",
      "    (DataFrame) --> Array\n",
      "    \n",
      "    This backward elimination technique used Logistic regression-based model which \n",
      "    selects the features based on the p-value score of the feature.\n",
      "    The features with p-value less than 0.05 are considered to be the more relevant feature\n",
      "    Source: https://www.analyticsvidhya.com/blog/2021/04/discovering-the-shades-of-feature-selection-methods/\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    data: Dataframe that will be used in feature selection.\n",
      "    \n",
      "    Returns\n",
      "    ----------\n",
      "    Array of features\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(loader.backward_elimitation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "46119bed-7e8b-40fe-89ed-23e9d33f500a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method pearson_correlation in module spambase_dataset_loader_nb:\n",
      "\n",
      "pearson_correlation(data) method of spambase_dataset_loader_nb.SpambaseDatasetLoader instance\n",
      "    (DataFrame) --> Array \n",
      "    \n",
      "    Pearson Correlation is used to construct a correlation matrix that measures the linear association \n",
      "    between two features and gives a value between -1 and 1 indicating how related the two features are to one another.\n",
      "    This measures the degree to which two features are interdependent by computing the association \n",
      "    between each feature and the target variable, the one exerting high impact on the target can be picked out\n",
      "    \n",
      "    A value of 1 indicates a positive correlation, -1 indicates a negative correlation and 0 indicates no correlation between the features.\n",
      "    Source: https://www.analyticsvidhya.com/blog/2021/04/discovering-the-shades-of-feature-selection-methods/\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    data: Dataframe that will be used in feature selection.\n",
      "    \n",
      "    Returns\n",
      "    ----------\n",
      "    Array of features\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(loader.pearson_correlation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5be14f0b-c5e5-4e34-8d82-cbfa3db4c989",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method chi2 in module spambase_dataset_loader_nb:\n",
      "\n",
      "chi2(data) method of spambase_dataset_loader_nb.SpambaseDatasetLoader instance\n",
      "    (DataFrame) --> Array \n",
      "    \n",
      "    A chi-square test is used in statistical models to check the independence of attributes.\n",
      "    The model measures the degree of deviation between the expected and actual response.\n",
      "    The lower the value of Chi-square, the less dependent the variables are to one another, \n",
      "    and the higher the value the more is their correlation. \n",
      "    Source: https://www.analyticsvidhya.com/blog/2021/04/discovering-the-shades-of-feature-selection-methods/        \n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    data: Dataframe that will be used in feature selection.\n",
      "    \n",
      "    Returns\n",
      "    ----------\n",
      "    Array of features\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(loader.chi2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "492a8c51-5fad-4e81-b03a-8af74b933d2c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}