featureEngineering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## EDA & Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyarrow.parquet as pq\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sbn\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pq.read_table(\"/Users/rahul/PycharmProjects/nycTaxi/yellow_tripdata_2024-05.parquet\")\n",
    "df = data.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>PULocationID</th>\n",
       "      <th>DOLocationID</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>fare_amount</th>\n",
       "      <th>extra</th>\n",
       "      <th>mta_tax</th>\n",
       "      <th>tip_amount</th>\n",
       "      <th>tolls_amount</th>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <th>total_amount</th>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <th>Airport_fee</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2024-05-01 00:59:15</td>\n",
       "      <td>2024-05-01 01:23:50</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>145</td>\n",
       "      <td>1</td>\n",
       "      <td>28.2</td>\n",
       "      <td>7.75</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>42.45</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-04-30 23:58:26</td>\n",
       "      <td>2024-05-01 00:29:42</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.23</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>249</td>\n",
       "      <td>1</td>\n",
       "      <td>46.4</td>\n",
       "      <td>6.00</td>\n",
       "      <td>0.5</td>\n",
       "      <td>8.72</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>66.87</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:57:17</td>\n",
       "      <td>2024-05-01 01:14:15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.02</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>170</td>\n",
       "      <td>1</td>\n",
       "      <td>35.9</td>\n",
       "      <td>6.00</td>\n",
       "      <td>0.5</td>\n",
       "      <td>10.57</td>\n",
       "      <td>6.94</td>\n",
       "      <td>1.0</td>\n",
       "      <td>65.16</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:24:47</td>\n",
       "      <td>2024-05-01 00:48:51</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.53</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>87</td>\n",
       "      <td>133</td>\n",
       "      <td>1</td>\n",
       "      <td>30.3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.5</td>\n",
       "      <td>7.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>42.36</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:11:20</td>\n",
       "      <td>2024-05-01 00:52:10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.38</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>161</td>\n",
       "      <td>165</td>\n",
       "      <td>1</td>\n",
       "      <td>61.8</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>66.80</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n",
       "0         1  2024-05-01 00:59:15   2024-05-01 01:23:50              1.0   \n",
       "1         2  2024-04-30 23:58:26   2024-05-01 00:29:42              1.0   \n",
       "2         2  2024-05-01 00:57:17   2024-05-01 01:14:15              1.0   \n",
       "3         2  2024-05-01 00:24:47   2024-05-01 00:48:51              1.0   \n",
       "4         2  2024-05-01 00:11:20   2024-05-01 00:52:10              1.0   \n",
       "\n",
       "   trip_distance  RatecodeID store_and_fwd_flag  PULocationID  DOLocationID  \\\n",
       "0           6.10         1.0                  N           138           145   \n",
       "1          11.23         1.0                  N           138           249   \n",
       "2           9.02         1.0                  N           138           170   \n",
       "3           6.53         1.0                  N            87           133   \n",
       "4          14.38         1.0                  N           161           165   \n",
       "\n",
       "   payment_type  fare_amount  extra  mta_tax  tip_amount  tolls_amount  \\\n",
       "0             1         28.2   7.75      0.5        5.00          0.00   \n",
       "1             1         46.4   6.00      0.5        8.72          0.00   \n",
       "2             1         35.9   6.00      0.5       10.57          6.94   \n",
       "3             1         30.3   1.00      0.5        7.06          0.00   \n",
       "4             1         61.8   1.00      0.5        0.00          0.00   \n",
       "\n",
       "   improvement_surcharge  total_amount  congestion_surcharge  Airport_fee  \n",
       "0                    1.0         42.45                   0.0         1.75  \n",
       "1                    1.0         66.87                   2.5         1.75  \n",
       "2                    1.0         65.16                   2.5         1.75  \n",
       "3                    1.0         42.36                   2.5         0.00  \n",
       "4                    1.0         66.80                   2.5         0.00  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>PULocationID</th>\n",
       "      <th>DOLocationID</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>fare_amount</th>\n",
       "      <th>extra</th>\n",
       "      <th>mta_tax</th>\n",
       "      <th>tip_amount</th>\n",
       "      <th>tolls_amount</th>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <th>total_amount</th>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <th>Airport_fee</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3723828</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:45:00</td>\n",
       "      <td>2024-06-01 00:09:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>88</td>\n",
       "      <td>186</td>\n",
       "      <td>0</td>\n",
       "      <td>29.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.93</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723829</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:10:52</td>\n",
       "      <td>2024-05-31 23:38:05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.81</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>231</td>\n",
       "      <td>143</td>\n",
       "      <td>0</td>\n",
       "      <td>30.71</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>34.71</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723830</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:23:57</td>\n",
       "      <td>2024-05-31 23:38:35</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>142</td>\n",
       "      <td>140</td>\n",
       "      <td>0</td>\n",
       "      <td>17.15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>4.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>25.38</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723831</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:44:12</td>\n",
       "      <td>2024-05-31 23:47:44</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.94</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>141</td>\n",
       "      <td>263</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723832</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:11:10</td>\n",
       "      <td>2024-05-31 23:38:27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>232</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>33.65</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37.65</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n",
       "3723828         2  2024-05-31 23:45:00   2024-06-01 00:09:00              NaN   \n",
       "3723829         2  2024-05-31 23:10:52   2024-05-31 23:38:05              NaN   \n",
       "3723830         2  2024-05-31 23:23:57   2024-05-31 23:38:35              NaN   \n",
       "3723831         2  2024-05-31 23:44:12   2024-05-31 23:47:44              NaN   \n",
       "3723832         2  2024-05-31 23:11:10   2024-05-31 23:38:27              NaN   \n",
       "\n",
       "         trip_distance  RatecodeID store_and_fwd_flag  PULocationID  \\\n",
       "3723828           4.07         NaN               None            88   \n",
       "3723829           4.81         NaN               None           231   \n",
       "3723830           2.03         NaN               None           142   \n",
       "3723831           0.94         NaN               None           141   \n",
       "3723832           9.25         NaN               None           232   \n",
       "\n",
       "         DOLocationID  payment_type  fare_amount  extra  mta_tax  tip_amount  \\\n",
       "3723828           186             0        29.93    0.0      0.5        0.00   \n",
       "3723829           143             0        30.71    0.0      0.5        0.00   \n",
       "3723830           140             0        17.15    0.0      0.5        4.23   \n",
       "3723831           263             0        -1.00    0.0      0.5        0.00   \n",
       "3723832            42             0        33.65    0.0      0.5        0.00   \n",
       "\n",
       "         tolls_amount  improvement_surcharge  total_amount  \\\n",
       "3723828           0.0                    1.0         33.93   \n",
       "3723829           0.0                    1.0         34.71   \n",
       "3723830           0.0                    1.0         25.38   \n",
       "3723831           0.0                    1.0          3.00   \n",
       "3723832           0.0                    1.0         37.65   \n",
       "\n",
       "         congestion_surcharge  Airport_fee  \n",
       "3723828                   NaN          NaN  \n",
       "3723829                   NaN          NaN  \n",
       "3723830                   NaN          NaN  \n",
       "3723831                   NaN          NaN  \n",
       "3723832                   NaN          NaN  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = df.iloc[300000:].dropna()\n",
    "\n",
    "random_sample = filtered_df.sample(n=5)\n",
    "    \n",
    "output_file = 'cleaned_tripdata_test.csv'\n",
    "random_sample.to_csv(output_file, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['VendorID', 'tpep_pickup_datetime', 'tpep_dropoff_datetime',\n",
       "       'passenger_count', 'trip_distance', 'RatecodeID', 'store_and_fwd_flag',\n",
       "       'PULocationID', 'DOLocationID', 'payment_type', 'fare_amount', 'extra',\n",
       "       'mta_tax', 'tip_amount', 'tolls_amount', 'improvement_surcharge',\n",
       "       'total_amount', 'congestion_surcharge', 'Airport_fee'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VendorID                      0\n",
       "tpep_pickup_datetime          0\n",
       "tpep_dropoff_datetime         0\n",
       "passenger_count          404666\n",
       "trip_distance                 0\n",
       "RatecodeID               404666\n",
       "store_and_fwd_flag       404666\n",
       "PULocationID                  0\n",
       "DOLocationID                  0\n",
       "payment_type                  0\n",
       "fare_amount                   0\n",
       "extra                         0\n",
       "mta_tax                       0\n",
       "tip_amount                    0\n",
       "tolls_amount                  0\n",
       "improvement_surcharge         0\n",
       "total_amount                  0\n",
       "congestion_surcharge     404666\n",
       "Airport_fee              404666\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3723833, 19)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>VendorID</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>1.759386</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.428868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <td>3723833</td>\n",
       "      <td>2024-05-16 02:06:29.391870</td>\n",
       "      <td>2002-12-31 16:46:07</td>\n",
       "      <td>2024-05-08 16:56:11</td>\n",
       "      <td>2024-05-15 20:48:05</td>\n",
       "      <td>2024-05-23 08:34:02</td>\n",
       "      <td>2024-06-01 23:54:14</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <td>3723833</td>\n",
       "      <td>2024-05-16 02:24:30.810017</td>\n",
       "      <td>2002-12-31 17:24:07</td>\n",
       "      <td>2024-05-08 17:19:02</td>\n",
       "      <td>2024-05-15 21:05:23</td>\n",
       "      <td>2024-05-23 08:51:22</td>\n",
       "      <td>2024-06-02 00:18:18</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>passenger_count</th>\n",
       "      <td>3319167.0</td>\n",
       "      <td>1.323312</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.806473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trip_distance</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>5.36705</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.05</td>\n",
       "      <td>1.8</td>\n",
       "      <td>3.4</td>\n",
       "      <td>249151.44</td>\n",
       "      <td>451.454445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RatecodeID</th>\n",
       "      <td>3319167.0</td>\n",
       "      <td>2.292974</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>99.0</td>\n",
       "      <td>10.815321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PULocationID</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>164.906041</td>\n",
       "      <td>1.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>234.0</td>\n",
       "      <td>265.0</td>\n",
       "      <td>64.477435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DOLocationID</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>164.036499</td>\n",
       "      <td>1.0</td>\n",
       "      <td>114.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>234.0</td>\n",
       "      <td>265.0</td>\n",
       "      <td>69.539508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>payment_type</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>1.091228</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.645757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fare_amount</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>19.740521</td>\n",
       "      <td>-866.69</td>\n",
       "      <td>9.3</td>\n",
       "      <td>14.2</td>\n",
       "      <td>23.3</td>\n",
       "      <td>334076.32</td>\n",
       "      <td>176.108167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>extra</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>1.423544</td>\n",
       "      <td>-7.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>65.99</td>\n",
       "      <td>1.848095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mta_tax</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>0.481688</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.123349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tip_amount</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>3.322133</td>\n",
       "      <td>-92.35</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>4.28</td>\n",
       "      <td>437.84</td>\n",
       "      <td>4.109948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tolls_amount</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>0.576083</td>\n",
       "      <td>-46.14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>528.56</td>\n",
       "      <td>2.267847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>0.963602</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.247665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_amount</th>\n",
       "      <td>3723833.0</td>\n",
       "      <td>28.374743</td>\n",
       "      <td>-867.69</td>\n",
       "      <td>15.96</td>\n",
       "      <td>21.36</td>\n",
       "      <td>31.08</td>\n",
       "      <td>334145.3</td>\n",
       "      <td>176.720899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <td>3319167.0</td>\n",
       "      <td>2.242073</td>\n",
       "      <td>-2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.853731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Airport_fee</th>\n",
       "      <td>3319167.0</td>\n",
       "      <td>0.149982</td>\n",
       "      <td>-1.75</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.75</td>\n",
       "      <td>0.504123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           count                        mean  \\\n",
       "VendorID               3723833.0                    1.759386   \n",
       "tpep_pickup_datetime     3723833  2024-05-16 02:06:29.391870   \n",
       "tpep_dropoff_datetime    3723833  2024-05-16 02:24:30.810017   \n",
       "passenger_count        3319167.0                    1.323312   \n",
       "trip_distance          3723833.0                     5.36705   \n",
       "RatecodeID             3319167.0                    2.292974   \n",
       "PULocationID           3723833.0                  164.906041   \n",
       "DOLocationID           3723833.0                  164.036499   \n",
       "payment_type           3723833.0                    1.091228   \n",
       "fare_amount            3723833.0                   19.740521   \n",
       "extra                  3723833.0                    1.423544   \n",
       "mta_tax                3723833.0                    0.481688   \n",
       "tip_amount             3723833.0                    3.322133   \n",
       "tolls_amount           3723833.0                    0.576083   \n",
       "improvement_surcharge  3723833.0                    0.963602   \n",
       "total_amount           3723833.0                   28.374743   \n",
       "congestion_surcharge   3319167.0                    2.242073   \n",
       "Airport_fee            3319167.0                    0.149982   \n",
       "\n",
       "                                       min                  25%  \\\n",
       "VendorID                               1.0                  2.0   \n",
       "tpep_pickup_datetime   2002-12-31 16:46:07  2024-05-08 16:56:11   \n",
       "tpep_dropoff_datetime  2002-12-31 17:24:07  2024-05-08 17:19:02   \n",
       "passenger_count                        0.0                  1.0   \n",
       "trip_distance                          0.0                 1.05   \n",
       "RatecodeID                             1.0                  1.0   \n",
       "PULocationID                           1.0                132.0   \n",
       "DOLocationID                           1.0                114.0   \n",
       "payment_type                           0.0                  1.0   \n",
       "fare_amount                        -866.69                  9.3   \n",
       "extra                                 -7.5                  0.0   \n",
       "mta_tax                               -0.5                  0.5   \n",
       "tip_amount                          -92.35                  0.0   \n",
       "tolls_amount                        -46.14                  0.0   \n",
       "improvement_surcharge                 -1.0                  1.0   \n",
       "total_amount                       -867.69                15.96   \n",
       "congestion_surcharge                  -2.5                  2.5   \n",
       "Airport_fee                          -1.75                  0.0   \n",
       "\n",
       "                                       50%                  75%  \\\n",
       "VendorID                               2.0                  2.0   \n",
       "tpep_pickup_datetime   2024-05-15 20:48:05  2024-05-23 08:34:02   \n",
       "tpep_dropoff_datetime  2024-05-15 21:05:23  2024-05-23 08:51:22   \n",
       "passenger_count                        1.0                  1.0   \n",
       "trip_distance                          1.8                  3.4   \n",
       "RatecodeID                             1.0                  1.0   \n",
       "PULocationID                         162.0                234.0   \n",
       "DOLocationID                         162.0                234.0   \n",
       "payment_type                           1.0                  1.0   \n",
       "fare_amount                           14.2                 23.3   \n",
       "extra                                  1.0                  2.5   \n",
       "mta_tax                                0.5                  0.5   \n",
       "tip_amount                             2.6                 4.28   \n",
       "tolls_amount                           0.0                  0.0   \n",
       "improvement_surcharge                  1.0                  1.0   \n",
       "total_amount                         21.36                31.08   \n",
       "congestion_surcharge                   2.5                  2.5   \n",
       "Airport_fee                            0.0                  0.0   \n",
       "\n",
       "                                       max         std  \n",
       "VendorID                               6.0    0.428868  \n",
       "tpep_pickup_datetime   2024-06-01 23:54:14         NaN  \n",
       "tpep_dropoff_datetime  2024-06-02 00:18:18         NaN  \n",
       "passenger_count                        9.0    0.806473  \n",
       "trip_distance                    249151.44  451.454445  \n",
       "RatecodeID                            99.0   10.815321  \n",
       "PULocationID                         265.0   64.477435  \n",
       "DOLocationID                         265.0   69.539508  \n",
       "payment_type                           4.0    0.645757  \n",
       "fare_amount                      334076.32  176.108167  \n",
       "extra                                65.99    1.848095  \n",
       "mta_tax                                4.0    0.123349  \n",
       "tip_amount                          437.84    4.109948  \n",
       "tolls_amount                        528.56    2.267847  \n",
       "improvement_surcharge                  1.0    0.247665  \n",
       "total_amount                      334145.3  176.720899  \n",
       "congestion_surcharge                   2.5    0.853731  \n",
       "Airport_fee                           1.75    0.504123  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe().transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_of_day(x):\n",
    "    if x in range(6,12):\n",
    "        return 'morning'\n",
    "    elif x in range(12,16):\n",
    "        return 'afternoon'\n",
    "    elif x in range(16,22):\n",
    "        return 'evening'\n",
    "    else:\n",
    "        return 'late night'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>PULocationID</th>\n",
       "      <th>DOLocationID</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>...</th>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <th>Airport_fee</th>\n",
       "      <th>trip_duration</th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>dropoff_hour</th>\n",
       "      <th>pickup_day</th>\n",
       "      <th>dropoff_day</th>\n",
       "      <th>pickup_timeofday</th>\n",
       "      <th>dropoff_timeofday</th>\n",
       "      <th>pickup_is_weekend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2024-05-01 00:59:15</td>\n",
       "      <td>2024-05-01 01:23:50</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>145</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.75</td>\n",
       "      <td>24.58</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-04-30 23:58:26</td>\n",
       "      <td>2024-05-01 00:29:42</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.23</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>249</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1.75</td>\n",
       "      <td>31.27</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:57:17</td>\n",
       "      <td>2024-05-01 01:14:15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.02</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>170</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1.75</td>\n",
       "      <td>16.97</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:24:47</td>\n",
       "      <td>2024-05-01 00:48:51</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.53</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>87</td>\n",
       "      <td>133</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>24.07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:11:20</td>\n",
       "      <td>2024-05-01 00:52:10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.38</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>161</td>\n",
       "      <td>165</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>40.83</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n",
       "0         1  2024-05-01 00:59:15   2024-05-01 01:23:50              1.0   \n",
       "1         2  2024-04-30 23:58:26   2024-05-01 00:29:42              1.0   \n",
       "2         2  2024-05-01 00:57:17   2024-05-01 01:14:15              1.0   \n",
       "3         2  2024-05-01 00:24:47   2024-05-01 00:48:51              1.0   \n",
       "4         2  2024-05-01 00:11:20   2024-05-01 00:52:10              1.0   \n",
       "\n",
       "   trip_distance  RatecodeID store_and_fwd_flag  PULocationID  DOLocationID  \\\n",
       "0           6.10         1.0                  N           138           145   \n",
       "1          11.23         1.0                  N           138           249   \n",
       "2           9.02         1.0                  N           138           170   \n",
       "3           6.53         1.0                  N            87           133   \n",
       "4          14.38         1.0                  N           161           165   \n",
       "\n",
       "   payment_type  ...  congestion_surcharge  Airport_fee  trip_duration  \\\n",
       "0             1  ...                   0.0         1.75          24.58   \n",
       "1             1  ...                   2.5         1.75          31.27   \n",
       "2             1  ...                   2.5         1.75          16.97   \n",
       "3             1  ...                   2.5         0.00          24.07   \n",
       "4             1  ...                   2.5         0.00          40.83   \n",
       "\n",
       "   pickup_hour  dropoff_hour  pickup_day  dropoff_day  pickup_timeofday  \\\n",
       "0            0             1           2            2        late night   \n",
       "1           23             0           1            2        late night   \n",
       "2            0             1           2            2        late night   \n",
       "3            0             0           2            2        late night   \n",
       "4            0             0           2            2        late night   \n",
       "\n",
       "   dropoff_timeofday  pickup_is_weekend  \n",
       "0         late night              False  \n",
       "1         late night              False  \n",
       "2         late night              False  \n",
       "3         late night              False  \n",
       "4         late night              False  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#generating the trip duration column\n",
    "df['tpep_pickup_datetime'] = pd.to_datetime(df['tpep_pickup_datetime'])\n",
    "df['tpep_dropoff_datetime'] = pd.to_datetime(df['tpep_dropoff_datetime'])\n",
    "\n",
    "#generating trip duration\n",
    "df['trip_duration'] = ((df['tpep_dropoff_datetime'] - df['tpep_pickup_datetime']).dt.total_seconds() / 60).round(2)\n",
    "\n",
    "#generating pickup hour\n",
    "df['pickup_hour'] = df['tpep_pickup_datetime'].dt.hour\n",
    "#generating dropoff hour\n",
    "df['dropoff_hour'] = df['tpep_dropoff_datetime'].dt.hour\n",
    "\n",
    "\n",
    "#generating the pickup day \n",
    "df['pickup_day'] = df['tpep_pickup_datetime'].dt.dayofweek\n",
    "#generating the dropoff day \n",
    "df['dropoff_day'] = df['tpep_dropoff_datetime'].dt.dayofweek\n",
    "\n",
    "#generating pickup time of day\n",
    "df['pickup_timeofday'] = df['pickup_hour'].apply(time_of_day)\n",
    "#generating dropoff time of day\n",
    "df['dropoff_timeofday'] = df['dropoff_hour'].apply(time_of_day)\n",
    "\n",
    "#generating is weekend \n",
    "df['pickup_is_weekend'] = df['tpep_pickup_datetime'].dt.dayofweek >= 5\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['VendorID', 'tpep_pickup_datetime', 'tpep_dropoff_datetime',\n",
       "       'passenger_count', 'trip_distance', 'RatecodeID', 'store_and_fwd_flag',\n",
       "       'PULocationID', 'DOLocationID', 'payment_type', 'fare_amount', 'extra',\n",
       "       'mta_tax', 'tip_amount', 'tolls_amount', 'improvement_surcharge',\n",
       "       'total_amount', 'congestion_surcharge', 'Airport_fee', 'trip_duration',\n",
       "       'pickup_hour', 'dropoff_hour', 'pickup_day', 'dropoff_day',\n",
       "       'pickup_timeofday', 'dropoff_timeofday', 'pickup_is_weekend'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3723833, 27)\n",
      "(3684085, 27)\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)\n",
    "#removing all rows with passenger count equals to 0\n",
    "df = df[df['passenger_count'] != 0]\n",
    "print(df.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VendorID                      0\n",
       "tpep_pickup_datetime          0\n",
       "tpep_dropoff_datetime         0\n",
       "passenger_count          404666\n",
       "trip_distance                 0\n",
       "RatecodeID               404666\n",
       "store_and_fwd_flag       404666\n",
       "PULocationID                  0\n",
       "DOLocationID                  0\n",
       "payment_type                  0\n",
       "fare_amount                   0\n",
       "extra                         0\n",
       "mta_tax                       0\n",
       "tip_amount                    0\n",
       "tolls_amount                  0\n",
       "improvement_surcharge         0\n",
       "total_amount                  0\n",
       "congestion_surcharge     404666\n",
       "Airport_fee              404666\n",
       "trip_duration                 0\n",
       "pickup_hour                   0\n",
       "dropoff_hour                  0\n",
       "pickup_day                    0\n",
       "dropoff_day                   0\n",
       "pickup_timeofday              0\n",
       "dropoff_timeofday             0\n",
       "pickup_is_weekend             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>PULocationID</th>\n",
       "      <th>DOLocationID</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>...</th>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <th>Airport_fee</th>\n",
       "      <th>trip_duration</th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>dropoff_hour</th>\n",
       "      <th>pickup_day</th>\n",
       "      <th>dropoff_day</th>\n",
       "      <th>pickup_timeofday</th>\n",
       "      <th>dropoff_timeofday</th>\n",
       "      <th>pickup_is_weekend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3319167</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:46:00</td>\n",
       "      <td>2024-05-01 00:54:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.65</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>186</td>\n",
       "      <td>79</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3319168</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:23:22</td>\n",
       "      <td>2024-05-01 00:35:48</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.42</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>170</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.43</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3319169</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:33:56</td>\n",
       "      <td>2024-05-01 00:35:50</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>238</td>\n",
       "      <td>238</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.90</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3319170</th>\n",
       "      <td>1</td>\n",
       "      <td>2024-05-01 00:01:47</td>\n",
       "      <td>2024-05-01 00:10:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>113</td>\n",
       "      <td>170</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3319171</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-01 00:45:56</td>\n",
       "      <td>2024-05-01 00:52:02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>112</td>\n",
       "      <td>157</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723828</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:45:00</td>\n",
       "      <td>2024-06-01 00:09:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>88</td>\n",
       "      <td>186</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24.00</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723829</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:10:52</td>\n",
       "      <td>2024-05-31 23:38:05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.81</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>231</td>\n",
       "      <td>143</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.22</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723830</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:23:57</td>\n",
       "      <td>2024-05-31 23:38:35</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>142</td>\n",
       "      <td>140</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.63</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723831</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:44:12</td>\n",
       "      <td>2024-05-31 23:47:44</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.94</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>141</td>\n",
       "      <td>263</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.53</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723832</th>\n",
       "      <td>2</td>\n",
       "      <td>2024-05-31 23:11:10</td>\n",
       "      <td>2024-05-31 23:38:27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>232</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.28</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>404666 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n",
       "3319167         2  2024-05-01 00:46:00   2024-05-01 00:54:00              NaN   \n",
       "3319168         2  2024-05-01 00:23:22   2024-05-01 00:35:48              NaN   \n",
       "3319169         2  2024-05-01 00:33:56   2024-05-01 00:35:50              NaN   \n",
       "3319170         1  2024-05-01 00:01:47   2024-05-01 00:10:00              NaN   \n",
       "3319171         2  2024-05-01 00:45:56   2024-05-01 00:52:02              NaN   \n",
       "...           ...                  ...                   ...              ...   \n",
       "3723828         2  2024-05-31 23:45:00   2024-06-01 00:09:00              NaN   \n",
       "3723829         2  2024-05-31 23:10:52   2024-05-31 23:38:05              NaN   \n",
       "3723830         2  2024-05-31 23:23:57   2024-05-31 23:38:35              NaN   \n",
       "3723831         2  2024-05-31 23:44:12   2024-05-31 23:47:44              NaN   \n",
       "3723832         2  2024-05-31 23:11:10   2024-05-31 23:38:27              NaN   \n",
       "\n",
       "         trip_distance  RatecodeID store_and_fwd_flag  PULocationID  \\\n",
       "3319167           1.65         NaN               None           186   \n",
       "3319168           2.42         NaN               None           170   \n",
       "3319169           0.40         NaN               None           238   \n",
       "3319170           1.10         NaN               None           113   \n",
       "3319171           2.59         NaN               None           112   \n",
       "...                ...         ...                ...           ...   \n",
       "3723828           4.07         NaN               None            88   \n",
       "3723829           4.81         NaN               None           231   \n",
       "3723830           2.03         NaN               None           142   \n",
       "3723831           0.94         NaN               None           141   \n",
       "3723832           9.25         NaN               None           232   \n",
       "\n",
       "         DOLocationID  payment_type  ...  congestion_surcharge  Airport_fee  \\\n",
       "3319167            79             0  ...                   NaN          NaN   \n",
       "3319168            50             0  ...                   NaN          NaN   \n",
       "3319169           238             0  ...                   NaN          NaN   \n",
       "3319170           170             0  ...                   NaN          NaN   \n",
       "3319171           157             0  ...                   NaN          NaN   \n",
       "...               ...           ...  ...                   ...          ...   \n",
       "3723828           186             0  ...                   NaN          NaN   \n",
       "3723829           143             0  ...                   NaN          NaN   \n",
       "3723830           140             0  ...                   NaN          NaN   \n",
       "3723831           263             0  ...                   NaN          NaN   \n",
       "3723832            42             0  ...                   NaN          NaN   \n",
       "\n",
       "         trip_duration  pickup_hour  dropoff_hour  pickup_day  dropoff_day  \\\n",
       "3319167           8.00            0             0           2            2   \n",
       "3319168          12.43            0             0           2            2   \n",
       "3319169           1.90            0             0           2            2   \n",
       "3319170           8.22            0             0           2            2   \n",
       "3319171           6.10            0             0           2            2   \n",
       "...                ...          ...           ...         ...          ...   \n",
       "3723828          24.00           23             0           4            5   \n",
       "3723829          27.22           23            23           4            4   \n",
       "3723830          14.63           23            23           4            4   \n",
       "3723831           3.53           23            23           4            4   \n",
       "3723832          27.28           23            23           4            4   \n",
       "\n",
       "         pickup_timeofday  dropoff_timeofday  pickup_is_weekend  \n",
       "3319167        late night         late night              False  \n",
       "3319168        late night         late night              False  \n",
       "3319169        late night         late night              False  \n",
       "3319170        late night         late night              False  \n",
       "3319171        late night         late night              False  \n",
       "...                   ...                ...                ...  \n",
       "3723828        late night         late night              False  \n",
       "3723829        late night         late night              False  \n",
       "3723830        late night         late night              False  \n",
       "3723831        late night         late night              False  \n",
       "3723832        late night         late night              False  \n",
       "\n",
       "[404666 rows x 27 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.isnull().any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3684085, 27)\n",
      "(3279419, 27)\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)\n",
    "df = df.dropna()\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>PULocationID</th>\n",
       "      <th>DOLocationID</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>fare_amount</th>\n",
       "      <th>extra</th>\n",
       "      <th>mta_tax</th>\n",
       "      <th>...</th>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <th>Airport_fee</th>\n",
       "      <th>trip_duration</th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>dropoff_hour</th>\n",
       "      <th>pickup_day</th>\n",
       "      <th>dropoff_day</th>\n",
       "      <th>pickup_timeofday</th>\n",
       "      <th>dropoff_timeofday</th>\n",
       "      <th>pickup_is_weekend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>138</td>\n",
       "      <td>145</td>\n",
       "      <td>1</td>\n",
       "      <td>28.2</td>\n",
       "      <td>7.75</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.75</td>\n",
       "      <td>24.58</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.23</td>\n",
       "      <td>1.0</td>\n",
       "      <td>138</td>\n",
       "      <td>249</td>\n",
       "      <td>1</td>\n",
       "      <td>46.4</td>\n",
       "      <td>6.00</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1.75</td>\n",
       "      <td>31.27</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.02</td>\n",
       "      <td>1.0</td>\n",
       "      <td>138</td>\n",
       "      <td>170</td>\n",
       "      <td>1</td>\n",
       "      <td>35.9</td>\n",
       "      <td>6.00</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1.75</td>\n",
       "      <td>16.97</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.53</td>\n",
       "      <td>1.0</td>\n",
       "      <td>87</td>\n",
       "      <td>133</td>\n",
       "      <td>1</td>\n",
       "      <td>30.3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>24.07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.38</td>\n",
       "      <td>1.0</td>\n",
       "      <td>161</td>\n",
       "      <td>165</td>\n",
       "      <td>1</td>\n",
       "      <td>61.8</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>40.83</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>late night</td>\n",
       "      <td>late night</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   VendorID  passenger_count  trip_distance  RatecodeID  PULocationID  \\\n",
       "0         1              1.0           6.10         1.0           138   \n",
       "1         2              1.0          11.23         1.0           138   \n",
       "2         2              1.0           9.02         1.0           138   \n",
       "3         2              1.0           6.53         1.0            87   \n",
       "4         2              1.0          14.38         1.0           161   \n",
       "\n",
       "   DOLocationID  payment_type  fare_amount  extra  mta_tax  ...  \\\n",
       "0           145             1         28.2   7.75      0.5  ...   \n",
       "1           249             1         46.4   6.00      0.5  ...   \n",
       "2           170             1         35.9   6.00      0.5  ...   \n",
       "3           133             1         30.3   1.00      0.5  ...   \n",
       "4           165             1         61.8   1.00      0.5  ...   \n",
       "\n",
       "   congestion_surcharge  Airport_fee  trip_duration  pickup_hour  \\\n",
       "0                   0.0         1.75          24.58            0   \n",
       "1                   2.5         1.75          31.27           23   \n",
       "2                   2.5         1.75          16.97            0   \n",
       "3                   2.5         0.00          24.07            0   \n",
       "4                   2.5         0.00          40.83            0   \n",
       "\n",
       "   dropoff_hour  pickup_day  dropoff_day  pickup_timeofday  dropoff_timeofday  \\\n",
       "0             1           2            2        late night         late night   \n",
       "1             0           1            2        late night         late night   \n",
       "2             1           2            2        late night         late night   \n",
       "3             0           2            2        late night         late night   \n",
       "4             0           2            2        late night         late night   \n",
       "\n",
       "   pickup_is_weekend  \n",
       "0              False  \n",
       "1              False  \n",
       "2              False  \n",
       "3              False  \n",
       "4              False  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop(['tpep_pickup_datetime','tpep_dropoff_datetime','store_and_fwd_flag'], axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "trip_duration\n",
       "0.10       3124\n",
       "7.17       3035\n",
       "7.00       3030\n",
       "7.73       3021\n",
       "8.13       3015\n",
       "           ... \n",
       "1435.05       1\n",
       "118.63        1\n",
       "1419.58       1\n",
       "1427.37       1\n",
       "1411.12       1\n",
       "Name: count, Length: 10242, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['trip_duration'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original shape: (3279419, 24)\n",
      "Cleaned shape: (2570334, 24)\n"
     ]
    }
   ],
   "source": [
    "def remove_outliers(df, columns):\n",
    "    Q1 = df[columns].quantile(0.25)\n",
    "    Q3 = df[columns].quantile(0.75)\n",
    "\n",
    "    IQR = Q3 - Q1\n",
    "    lower_bound = Q1 - 1.5 * IQR\n",
    "    upper_bound = Q3 + 1.5 * IQR\n",
    "\n",
    "    mask = ((df[columns] < lower_bound) | (df[columns] > upper_bound)).any(axis=1)\n",
    "    return df[~mask]\n",
    "\n",
    "\n",
    "numerical_columns = ['passenger_count']\n",
    "\n",
    "print(f\"Original shape: {df.shape}\")\n",
    "\n",
    "# Apply the function to remove outliers\n",
    "df = remove_outliers(df, numerical_columns)\n",
    "df  = df.reset_index(drop=True)\n",
    "\n",
    "print(f\"Cleaned shape: {df.shape}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "trip_duration\n",
       "7.17       2433\n",
       "8.50       2419\n",
       "6.50       2412\n",
       "6.67       2406\n",
       "7.27       2406\n",
       "           ... \n",
       "1425.88       1\n",
       "1432.82       1\n",
       "1436.95       1\n",
       "1414.77       1\n",
       "1411.12       1\n",
       "Name: count, Length: 9652, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['trip_duration'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "payment_type\n",
       "1    2133693\n",
       "2     368356\n",
       "4      49236\n",
       "3      19049\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['payment_type'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['VendorID', 'passenger_count', 'trip_distance', 'RatecodeID',\n",
       "       'PULocationID', 'DOLocationID', 'payment_type', 'fare_amount', 'extra',\n",
       "       'mta_tax', 'tip_amount', 'tolls_amount', 'improvement_surcharge',\n",
       "       'total_amount', 'congestion_surcharge', 'Airport_fee', 'trip_duration',\n",
       "       'pickup_hour', 'dropoff_hour', 'pickup_day', 'dropoff_day',\n",
       "       'pickup_timeofday', 'dropoff_timeofday', 'pickup_is_weekend'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}