Advanced House Prediction - Basic FE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading neccesary packages\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pdpipe as pdp\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "#\n",
    "\n",
    "from scipy import stats\n",
    "from scipy.stats import skew, boxcox_normmax, norm\n",
    "from scipy.special import boxcox1p\n",
    "\n",
    "#\n",
    "\n",
    "from typing import Dict, List, Tuple, Sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Initial Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_file_path(in_kaggle: bool) -> Tuple[str, str]:\n",
    "    train_set_path = ''\n",
    "    test_set_path = ''\n",
    "    \n",
    "    if in_kaggle:\n",
    "        # running in Kaggle, inside \n",
    "        # 'House Prices: Advanced Regression Techniques' competition kernel container\n",
    "        train_set_path = '../input/train.csv'\n",
    "        test_set_path = '../input/test.csv'\n",
    "    else:\n",
    "        # running locally\n",
    "        train_set_path = 'data/train.csv'\n",
    "        test_set_path = 'data/test.csv'\n",
    "    \n",
    "    return train_set_path,test_set_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Loading datasets\n",
    "in_kaggle = False\n",
    "train_set_path, test_set_path = get_data_file_path(in_kaggle)\n",
    "\n",
    "train = pd.read_csv(train_set_path)\n",
    "test = pd.read_csv(test_set_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1460, 81)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check train dimension\n",
    "display(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1459, 80)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check test dimension\n",
    "display(test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initial Data Transformation: Dropping Id col\n",
    "\n",
    "Starting this section and down the activities in **'Setting Model Data and Log Transforming the Target'** section below, we are going to reuse the feature enginering steps defined, justified and explained in https://www.kaggle.com/datafan07/beginner-eda-with-feature-eng-and-blending-models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dropping unneccessary columns, merging training and test sets\n",
    "\n",
    "train.drop('Id', axis=1, inplace=True)\n",
    "test.drop('Id', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dropping Outliers From Training Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping outliers after detecting them by eye\n",
    "\n",
    "train = train.drop(train[(train['OverallQual'] < 5)\n",
    "                                  & (train['SalePrice'] > 200000)].index)\n",
    "train = train.drop(train[(train['GrLivArea'] > 4000)\n",
    "                                  & (train['SalePrice'] < 200000)].index)\n",
    "train = train.drop(train[(train['GarageArea'] > 1200)\n",
    "                                  & (train['SalePrice'] < 200000)].index)\n",
    "train = train.drop(train[(train['TotalBsmtSF'] > 3000)\n",
    "                                  & (train['SalePrice'] > 320000)].index)\n",
    "train = train.drop(train[(train['1stFlrSF'] < 3000)\n",
    "                                  & (train['SalePrice'] > 600000)].index)\n",
    "train = train.drop(train[(train['1stFlrSF'] > 3000)\n",
    "                                  & (train['SalePrice'] < 200000)].index)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merging Train and Test Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2908, 79)\n"
     ]
    }
   ],
   "source": [
    "# Backing up target variables and dropping them from train data\n",
    "\n",
    "y = train['SalePrice'].reset_index(drop=True)\n",
    "train_features = train.drop(['SalePrice'], axis=1)\n",
    "test_features = test\n",
    "\n",
    "# Merging features\n",
    "\n",
    "features = pd.concat([train_features, test_features]).reset_index(drop=True)\n",
    "print(features.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imputing Missing Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of NaN including columns where NaN's mean none.\n",
    "none_cols = [\n",
    "    'Alley', 'PoolQC', 'MiscFeature', 'Fence', 'FireplaceQu', 'GarageType',\n",
    "    'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond',\n",
    "    'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType'\n",
    "]\n",
    "\n",
    "# List of NaN including columns where NaN's mean 0.\n",
    "\n",
    "zero_cols = [\n",
    "    'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath',\n",
    "    'BsmtHalfBath', 'GarageYrBlt', 'GarageArea', 'GarageCars', 'MasVnrArea'\n",
    "]\n",
    "\n",
    "# List of NaN including columns where NaN's actually missing gonna replaced with mode.\n",
    "\n",
    "freq_cols = [\n",
    "    'Electrical', 'Exterior1st', 'Exterior2nd', 'Functional', 'KitchenQual',\n",
    "    'SaleType', 'Utilities'\n",
    "]\n",
    "\n",
    "# Filling the list of columns above:\n",
    "\n",
    "for col in zero_cols:\n",
    "    features[col].replace(np.nan, 0, inplace=True)\n",
    "\n",
    "for col in none_cols:\n",
    "    features[col].replace(np.nan, 'None', inplace=True)\n",
    "\n",
    "for col in freq_cols:\n",
    "    features[col].replace(np.nan, features[col].mode()[0], inplace=True)\n",
    "    \n",
    "# Filling MSZoning according to MSSubClass\n",
    "features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].apply(\n",
    "    lambda x: x.fillna(x.mode()[0]))\n",
    "\n",
    "# Filling LotFrontage according to Neighborhood\n",
    "features['LotFrontage'] = features.groupby(\n",
    "    ['Neighborhood'])['LotFrontage'].apply(lambda x: x.fillna(x.median()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Numeric Features to be Treated as Categories\n",
    "\n",
    "Below we cast some numeric features to categories based on the logical assessment of the feature essence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features which numerical on data but should be treated as category.\n",
    "features['MSSubClass'] = features['MSSubClass'].astype(str)\n",
    "features['YrSold'] = features['YrSold'].astype(str)\n",
    "features['MoSold'] = features['MoSold'].astype(str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# New Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Binning the Rare Category Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    }
   ],
   "source": [
    "# Transforming rare values(less than 10) into one group - dimensionality reduction\n",
    "\n",
    "others = [\n",
    "    'Condition1', 'Condition2', 'RoofMatl', 'Exterior1st', 'Exterior2nd',\n",
    "    'Heating', 'Electrical', 'Functional', 'SaleType'\n",
    "]\n",
    "\n",
    "for col in others:\n",
    "    mask = features[col].isin(\n",
    "        features[col].value_counts()[features[col].value_counts() < 10].index)\n",
    "    features[col][mask] = 'Other'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Converting Some Categorical Variables to Numeric Ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converting some of the categorical values to numeric ones.\n",
    "\n",
    "neigh_map = {\n",
    "    'MeadowV': 1,\n",
    "    'IDOTRR': 1,\n",
    "    'BrDale': 1,\n",
    "    'BrkSide': 2,\n",
    "    'OldTown': 2,\n",
    "    'Edwards': 2,\n",
    "    'Sawyer': 3,\n",
    "    'Blueste': 3,\n",
    "    'SWISU': 3,\n",
    "    'NPkVill': 3,\n",
    "    'NAmes': 3,\n",
    "    'Mitchel': 4,\n",
    "    'SawyerW': 5,\n",
    "    'NWAmes': 5,\n",
    "    'Gilbert': 5,\n",
    "    'Blmngtn': 5,\n",
    "    'CollgCr': 5,\n",
    "    'ClearCr': 6,\n",
    "    'Crawfor': 6,\n",
    "    'Veenker': 7,\n",
    "    'Somerst': 7,\n",
    "    'Timber': 8,\n",
    "    'StoneBr': 9,\n",
    "    'NridgHt': 10,\n",
    "    'NoRidge': 10\n",
    "}\n",
    "\n",
    "features['Neighborhood'] = features['Neighborhood'].map(neigh_map).astype('int')\n",
    "ext_map = {'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}\n",
    "features['ExterQual'] = features['ExterQual'].map(ext_map).astype('int')\n",
    "features['ExterCond'] = features['ExterCond'].map(ext_map).astype('int')\n",
    "bsm_map = {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}\n",
    "features['BsmtQual'] = features['BsmtQual'].map(bsm_map).astype('int')\n",
    "features['BsmtCond'] = features['BsmtCond'].map(bsm_map).astype('int')\n",
    "bsmf_map = {\n",
    "    'None': 0,\n",
    "    'Unf': 1,\n",
    "    'LwQ': 2,\n",
    "    'Rec': 3,\n",
    "    'BLQ': 4,\n",
    "    'ALQ': 5,\n",
    "    'GLQ': 6\n",
    "}\n",
    "\n",
    "features['BsmtFinType1'] = features['BsmtFinType1'].map(bsmf_map).astype('int')\n",
    "features['BsmtFinType2'] = features['BsmtFinType2'].map(bsmf_map).astype('int')\n",
    "heat_map = {'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}\n",
    "features['HeatingQC'] = features['HeatingQC'].map(heat_map).astype('int')\n",
    "features['KitchenQual'] = features['KitchenQual'].map(heat_map).astype('int')\n",
    "features['FireplaceQu'] = features['FireplaceQu'].map(bsm_map).astype('int')\n",
    "features['GarageCond'] = features['GarageCond'].map(bsm_map).astype('int')\n",
    "features['GarageQual'] = features['GarageQual'].map(bsm_map).astype('int')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating New Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating new features  based on previous observations\n",
    "\n",
    "features['TotalSF'] = (features['BsmtFinSF1'] + features['BsmtFinSF2'] +\n",
    "                       features['1stFlrSF'] + features['2ndFlrSF'])\n",
    "features['TotalBathrooms'] = (features['FullBath'] +\n",
    "                              (0.5 * features['HalfBath']) +\n",
    "                              features['BsmtFullBath'] +\n",
    "                              (0.5 * features['BsmtHalfBath']))\n",
    "\n",
    "features['TotalPorchSF'] = (features['OpenPorchSF'] + features['3SsnPorch'] +\n",
    "                            features['EnclosedPorch'] +\n",
    "                            features['ScreenPorch'] + features['WoodDeckSF'])\n",
    "\n",
    "features['YearBlRm'] = (features['YearBuilt'] + features['YearRemodAdd'])\n",
    "\n",
    "# Merging quality and conditions\n",
    "\n",
    "features['TotalExtQual'] = (features['ExterQual'] + features['ExterCond'])\n",
    "features['TotalBsmQual'] = (features['BsmtQual'] + features['BsmtCond'] +\n",
    "                            features['BsmtFinType1'] +\n",
    "                            features['BsmtFinType2'])\n",
    "features['TotalGrgQual'] = (features['GarageQual'] + features['GarageCond'])\n",
    "features['TotalQual'] = features['OverallQual'] + features[\n",
    "    'TotalExtQual'] + features['TotalBsmQual'] + features[\n",
    "        'TotalGrgQual'] + features['KitchenQual'] + features['HeatingQC']\n",
    "\n",
    "# Creating new features by using new quality indicators\n",
    "\n",
    "features['QualGr'] = features['TotalQual'] * features['GrLivArea']\n",
    "features['QualBsm'] = features['TotalBsmQual'] * (features['BsmtFinSF1'] +\n",
    "                                                  features['BsmtFinSF2'])\n",
    "features['QualPorch'] = features['TotalExtQual'] * features['TotalPorchSF']\n",
    "features['QualExt'] = features['TotalExtQual'] * features['MasVnrArea']\n",
    "features['QualGrg'] = features['TotalGrgQual'] * features['GarageArea']\n",
    "features['QlLivArea'] = (features['GrLivArea'] -\n",
    "                         features['LowQualFinSF']) * (features['TotalQual'])\n",
    "features['QualSFNg'] = features['QualGr'] * features['Neighborhood']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating some simple features\n",
    "\n",
    "features['HasPool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)\n",
    "\n",
    "features['Has2ndFloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)\n",
    "\n",
    "features['HasGarage'] = features['QualGrg'].apply(lambda x: 1 if x > 0 else 0)\n",
    "\n",
    "features['HasBsmt'] = features['QualBsm'].apply(lambda x: 1 if x > 0 else 0)\n",
    "\n",
    "features['HasFireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)\n",
    "\n",
    "features['HasPorch'] = features['QualPorch'].apply(lambda x: 1 if x > 0 else 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transforming The Skewed Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\scipy\\stats\\stats.py:3508: PearsonRConstantInputWarning: An input array is constant; the correlation coefficent is not defined.\n",
      "  warnings.warn(PearsonRConstantInputWarning())\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\scipy\\stats\\stats.py:3538: PearsonRNearConstantInputWarning: An input array is nearly constant; the computed correlation coefficent may be inaccurate.\n",
      "  warnings.warn(PearsonRNearConstantInputWarning())\n"
     ]
    }
   ],
   "source": [
    "possible_skewed = [\n",
    "    'LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2',\n",
    "    'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'GrLivArea',\n",
    "    'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch',\n",
    "    'ScreenPorch', 'PoolArea', 'LowQualFinSF', 'MiscVal'\n",
    "]\n",
    "\n",
    "# Finding skewness of the numerical features\n",
    "\n",
    "skew_features = np.abs(features[possible_skewed].apply(lambda x: skew(x)).sort_values(\n",
    "    ascending=False))\n",
    "\n",
    "# Filtering skewed features\n",
    "\n",
    "high_skew = skew_features[skew_features > 0.3]\n",
    "\n",
    "# Taking indexes of high skew\n",
    "\n",
    "skew_index = high_skew.index\n",
    "\n",
    "# Applying boxcox transformation to fix skewness\n",
    "\n",
    "for i in skew_index:\n",
    "    features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dropping Unnecessary Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features to drop\n",
    "\n",
    "to_drop = [\n",
    "    'Utilities',\n",
    "    'PoolQC',\n",
    "    'YrSold',\n",
    "    'MoSold',\n",
    "    'ExterQual',\n",
    "    'BsmtQual',\n",
    "    'GarageQual',\n",
    "    'KitchenQual',\n",
    "    'HeatingQC',\n",
    "]\n",
    "\n",
    "# Dropping ML-irrelevant features\n",
    "\n",
    "features.drop(columns=to_drop, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Label Encoding The Categorical Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting dummy variables for ategorical data\n",
    "features = pd.get_dummies(data=features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Check of The Data Before Feature Selection and ML Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of missing values: 0\n"
     ]
    }
   ],
   "source": [
    "print(f'Number of missing values: {features.isna().sum().sum()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2908, 226)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Separating Train and Test Sets Again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separating train and test set\n",
    "\n",
    "train = features.iloc[:len(y), :]\n",
    "test = features.iloc[len(train):, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting Model Data and Log Transforming the Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting model data\n",
    "\n",
    "X = train\n",
    "X_test = test\n",
    "y = np.log1p(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# References\n",
    "\n",
    "Below are the good sources for feature engineering and data pre-processing insights for Advanced House Pricing Prediction Dataset:\n",
    "\n",
    "- https://www.kaggle.com/datafan07/beginner-eda-with-feature-eng-and-blending-models\n",
    "- https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python\n",
    "- https://www.kaggle.com/pavansanagapati/a-simple-tutorial-on-exploratory-data-analysis\n",
    "- https://www.kaggle.com/masumrumi/a-detailed-regression-guide-with-house-pricing\n",
    "- https://www.kaggle.com/tannercarbonati/detailed-data-analysis-ensemble-modeling\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}