flight-delay-prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'scipy'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-d64ca633c9f6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwarnings\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscipy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0msns\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mmpl\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'scipy'"
     ]
    }
   ],
   "source": [
    "import datetime, warnings, scipy \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "from matplotlib.patches import ConnectionPatch\n",
    "from collections import OrderedDict\n",
    "from matplotlib.gridspec import GridSpec\n",
    "from mpl_toolkits.basemap import Basemap\n",
    "from sklearn import metrics, linear_model\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.preprocessing import PolynomialFeatures, StandardScaler\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, cross_val_predict\n",
    "from scipy.optimize import curve_fit\n",
    "from scipy import stats\n",
    "plt.rcParams[\"patch.force_edgecolor\"] = True\n",
    "plt.style.use('fivethirtyeight')\n",
    "mpl.rc('patch', edgecolor = 'dimgray', linewidth=1)\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"last_expr\"\n",
    "pd.options.display.max_columns = 50\n",
    "%matplotlib inline\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conda install -c conda-forge basemap-data-hires"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read the flights and airlines dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flights = pd.read_csv('../input/flight-delays/flights.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "airports = pd.read_csv(\"../input/flight-delays/airports.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Overview of all Flights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_flights = flights['ORIGIN_AIRPORT'].value_counts()\n",
    "#___________________________\n",
    "plt.figure(figsize=(9,9))\n",
    "#________________________________________\n",
    "# Define the variables with their values\n",
    "colors = ['gray', 'red', 'darkblue', 'violet', 'green', 'orange']\n",
    "size_limits = [1, 100, 1000, 10000, 100000, 1000000]\n",
    "labels = []\n",
    "for i in range(len(size_limits)-1):\n",
    "    labels.append(\"{} <.< {}\".format(size_limits[i], size_limits[i+1]))\n",
    "    \n",
    "# To create the base outline of the map visualization\n",
    "map = Basemap(resolution='i',llcrnrlon=-180, urcrnrlon=-50,\n",
    "              llcrnrlat=10, urcrnrlat=75, lat_0=0, lon_0=0,)\n",
    "map.drawcoastlines()\n",
    "map.shadedrelief()\n",
    "map.drawcountries(linewidth = 1)\n",
    "map.drawstates(color='0.1')\n",
    "\n",
    "# Put airports on map\n",
    "for index, (code, y,x) in airports[['IATA_CODE', 'LATITUDE', 'LONGITUDE']].iterrows():\n",
    "    x, y = map(x, y)\n",
    "    isize = [i for i, val in enumerate(size_limits) if val < count_flights[code]]\n",
    "    ind = isize[-1]\n",
    "    map.plot(x, y, marker='o', markersize = ind+5, markeredgewidth = 1, color = colors[ind],\n",
    "             markeredgecolor='k', label = labels[ind])\n",
    "\n",
    "# Remove duplicate labels and set their order\n",
    "handles, labels = plt.gca().get_legend_handles_labels()\n",
    "by_label = OrderedDict(zip(labels, handles))\n",
    "key_order = ('1 <.< 100', '100 <.< 1000', '1000 <.< 10000',\n",
    "             '10000 <.< 100000', '100000 <.< 1000000')\n",
    "new_label = OrderedDict()\n",
    "for key in key_order:\n",
    "    new_label[key] = by_label[key]\n",
    "plt.legend(new_label.values(), new_label.keys(), loc = 1, prop= {'size':9},\n",
    "           title='Number of flights per year', frameon = True, framealpha = 1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bring records of only January due to the large size of the dataset. This is done by calling the '1' value in the dataset which depicts the month January."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flights = flights[flights['MONTH'] == 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Transformation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Converting the existing dates into YYYY-MM-DD format for easier reading."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flights['DATE'] = pd.to_datetime(flights[['YEAR','MONTH', 'DAY']])\n",
    "print(flights['DATE'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following functions help convert the confusing time format into an understanding format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert the HHMM format of scheduled departure into a datetime format\n",
    "def format_hours(time):\n",
    "    if pd.isnull(time):\n",
    "        return np.nan\n",
    "    else:\n",
    "        if time == 2400: time = 0\n",
    "        time = \"{0:04d}\".format(int(time))\n",
    "        hour = datetime.time(int(time[0:2]), int(time[2:4]))\n",
    "        return hour\n",
    "#_____________________________________________________________________\n",
    "# Function that combines a date and time to produce a datetime.datetime\n",
    "def combine_date_hour(x):\n",
    "    if pd.isnull(x[0]) or pd.isnull(x[1]):\n",
    "        return np.nan\n",
    "    else:\n",
    "        return datetime.datetime.combine(x[0],x[1])\n",
    "#_______________________________________________________________________________\n",
    "# Function that combine two columns of the dataframe to create a datetime format in one column\n",
    "def create_flight_time(flights, col):    \n",
    "    liste = []\n",
    "    for index, cols in flights[['DATE', col]].iterrows():    \n",
    "        if pd.isnull(cols[1]):\n",
    "            liste.append(np.nan)\n",
    "        elif float(cols[1]) == 2400:\n",
    "            cols[0] += datetime.timedelta(days=1)\n",
    "            cols[1] = datetime.time(0,0)\n",
    "            liste.append(combine_date_hour(cols))\n",
    "        else:\n",
    "            cols[1] = format_hours(cols[1])\n",
    "            liste.append(combine_date_hour(cols))\n",
    "    return pd.Series(liste)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calling the premade functions to modify the existing data\n",
    "flights['SCHEDULED_DEPARTURE'] = create_flight_time(flights, 'SCHEDULED_DEPARTURE')\n",
    "flights['DEPARTURE_TIME'] = flights['DEPARTURE_TIME'].apply(format_hours)\n",
    "flights['SCHEDULED_ARRIVAL'] = flights['SCHEDULED_ARRIVAL'].apply(format_hours)\n",
    "flights['ARRIVAL_TIME'] = flights['ARRIVAL_TIME'].apply(format_hours)\n",
    "# Retrieve the modified columns with the changed data\n",
    "flights.loc[:10, ['SCHEDULED_DEPARTURE', 'SCHEDULED_ARRIVAL', 'DEPARTURE_TIME','ARRIVAL_TIME', 'DEPARTURE_DELAY', 'ARRIVAL_DELAY']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Through analyzing the result, it can be shown that the variables, **DEPARTURE_TIME** and **ARRIVAL_TIME** do not show any clear understanding of whether the flight was delayed or not. In the dataset, it is not depicted in a understandable manner. It does not indicate if there is a large delay or if the flight left before time. Therefore, the variables will not be used. Another set of variables will be used to show if the flight was delayed or not."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Filling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "variables_to_remove = ['TAXI_OUT', 'TAXI_IN', 'WHEELS_ON', 'WHEELS_OFF', 'YEAR', \n",
    "                       'MONTH','DAY','DAY_OF_WEEK','DATE', 'AIR_SYSTEM_DELAY',\n",
    "                       'SECURITY_DELAY', 'AIRLINE_DELAY', 'LATE_AIRCRAFT_DELAY',\n",
    "                       'WEATHER_DELAY', 'DIVERTED', 'CANCELLED', 'CANCELLATION_REASON',\n",
    "                       'FLIGHT_NUMBER', 'TAIL_NUMBER', 'AIR_TIME']\n",
    "# Drop the columns completely and modify the data in place\n",
    "flights.drop(variables_to_remove, axis = 1)\n",
    "flights = flights[['AIRLINE', 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT',\n",
    "        'SCHEDULED_DEPARTURE', 'DEPARTURE_TIME', 'DEPARTURE_DELAY',\n",
    "        'SCHEDULED_ARRIVAL', 'ARRIVAL_TIME', 'ARRIVAL_DELAY',\n",
    "        'SCHEDULED_TIME', 'ELAPSED_TIME']]\n",
    "flights[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding the missing values and removing the entries completely. There is no viable option to assume the data for the blanks in the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the missing values of each row in the dataset\n",
    "missing_flights = flights.isnull().sum(axis=0).reset_index()\n",
    "# Create the columns to show the variable and the number of missing values\n",
    "missing_flights.columns = ['variable', 'missing values']\n",
    "# Calculate the % of whether to fill those fields or remove\n",
    "missing_flights['filling factor (%)']=(flights.shape[0]-missing_flights['missing values'])/flights.shape[0]*100\n",
    "# Drops the old index range and makes it in increasing order\n",
    "missing_flights.sort_values('filling factor (%)').reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop the missing values directly\n",
    "flights.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_stats(group):\n",
    "    return {'min': group.min(), 'max': group.max(),\n",
    "            'count': group.count(), 'mean': group.mean()}\n",
    "#_______________________________________________________________\n",
    "# Creation of a dataframe with statitical infos on each airline:\n",
    "global_stats = flights['DEPARTURE_DELAY'].groupby(flights['AIRLINE']).apply(get_stats).unstack()\n",
    "global_stats = global_stats.sort_values('count')\n",
    "global_stats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparing Airlines - Which has the most delay?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading the airlines.csv file for accessing the airline names and respective IATA codes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "airlines_names = pd.read_csv('../input/flight-delays/airlines.csv')\n",
    "airlines_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iata_airlines = airlines_names.set_index('IATA_CODE')['AIRLINE'].to_dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To check which airline has the most or least delay, we would have to create a graphic visualization to compare each airline. This visualization compares on time, short and long delays for each US airline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#_____________________________________________\n",
    "# Function that define how delays are grouped\n",
    "delay_type = lambda x:((0,1)[x > 5],2)[x > 45]\n",
    "flights['DELAY_LEVEL'] = flights['DEPARTURE_DELAY'].apply(delay_type)\n",
    "#____________________________________________________\n",
    "fig = plt.figure(1, figsize=(10,7))\n",
    "ax = sns.countplot(y=\"AIRLINE\", hue='DELAY_LEVEL', data=flights)\n",
    "#____________________________________________________________________________________\n",
    "# We replace the abbreviations by the full names of the companies and set the labels\n",
    "labels = [iata_airlines[item.get_text()] for item in ax.get_yticklabels()]\n",
    "ax.set_yticklabels(labels)\n",
    "plt.setp(ax.get_xticklabels(), fontsize=10, weight = 'normal', rotation = 0);\n",
    "plt.setp(ax.get_yticklabels(), fontsize=10, weight = 'bold', rotation = 0);\n",
    "ax.yaxis.label.set_visible(False)\n",
    "plt.xlabel('Flight Count', fontsize=16, labelpad=10)\n",
    "#________________\n",
    "# Set the legend\n",
    "L = plt.legend()\n",
    "L.get_texts()[0].set_text('on time (t < 5 min)')\n",
    "L.get_texts()[1].set_text('small delay (5 < t < 45 min)')\n",
    "L.get_texts()[2].set_text('large delay (t > 45 min)')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare Delay in Take off or Arrival"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The visualization shows that the departures are higher than the arrivals. It can be safe to assume that the average delays between the airlines is much higher during take off rather than arrival at the destination. Another assumption for this could be flights adjusting their flight speed during the landing phase which in turn can play a big role in the arrival delay."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mpl.rcParams.update(mpl.rcParamsDefault)\n",
    "mpl.rcParams['hatch.linewidth'] = 2.0  \n",
    "\n",
    "fig = plt.figure(1, figsize=(5,5))\n",
    "ax = sns.barplot(x=\"DEPARTURE_DELAY\", y=\"AIRLINE\", data=flights, color=\"grey\", ci=None)\n",
    "ax = sns.barplot(x=\"ARRIVAL_DELAY\", y=\"AIRLINE\", data=flights, hatch = '///',\n",
    "                 alpha = 0.0, ci=None)\n",
    "labels = [iata_airlines[item.get_text()] for item in ax.get_yticklabels()]\n",
    "ax.set_yticklabels(labels)\n",
    "ax.yaxis.label.set_visible(False)\n",
    "plt.xlabel('Average delay between the departures, @grey and arrivals, hatch lines',\n",
    "           fontsize=10, labelpad=10);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction of Flight Delays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The goal of this model is to predict the flight delay time for the month January 2021(the last week) during the COVID period. We would categorise this delay along with small or large delays."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = flights[flights['SCHEDULED_DEPARTURE'].apply(lambda x:x.date()) < datetime.date(2015, 1, 23)]\n",
    "df_test  = flights[flights['SCHEDULED_DEPARTURE'].apply(lambda x:x.date()) > datetime.date(2015, 1, 23)]\n",
    "df = df_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**LINEAR REGRESSION INTRODUCTION FOR ALGORITHM.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model will take into account all the airports in comparison with a single airline. All of the flights will be from the month of January. The specific airline which will be considered is American Airlines."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_flight_delays(flights, carrier, id_airport, extreme_values = False):\n",
    "    df2 = flights[(flights['AIRLINE'] == carrier) & (flights['ORIGIN_AIRPORT'] == id_airport)]\n",
    "    #_______________________________________\n",
    "    # remove extreme values before fitting\n",
    "    if extreme_values:\n",
    "        df2['DEPARTURE_DELAY'] = df2['DEPARTURE_DELAY'].apply(lambda x:x if x < 60 else np.nan)\n",
    "        df2.dropna(how = 'any')\n",
    "    #__________________________________\n",
    "    # Converting date and hour to solely hour\n",
    "    df2.sort_values('SCHEDULED_DEPARTURE', inplace = True)\n",
    "    df2['hour_depart'] =  df2['SCHEDULED_DEPARTURE'].apply(lambda x:x.time())\n",
    "    #___________________________________________________________________\n",
    "    test2 = df2['DEPARTURE_DELAY'].groupby(df2['hour_depart']).apply(get_stats).unstack()\n",
    "    test2.reset_index(inplace=True)\n",
    "    #___________________________________\n",
    "    fct = lambda x:x.hour*3600+x.minute*60+x.second\n",
    "    test2.reset_index(inplace=True)\n",
    "    test2['hour_depart_min'] = test2['hour_depart'].apply(fct)\n",
    "    return test2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merged_delays(flights, carrier):\n",
    "    listed_airports = flights[flights['AIRLINE'] == carrier]['ORIGIN_AIRPORT'].unique()\n",
    "    i = 0\n",
    "    listed_columns = ['AIRPORT_ID', 'hour_depart_min', 'mean']\n",
    "    for id_airport in listed_airports:\n",
    "        test2 = get_flight_delays(flights, carrier, id_airport, True)\n",
    "        test2.loc[:, 'AIRPORT_ID'] = id_airport\n",
    "        test2 = test2[listed_columns]\n",
    "        test2.dropna(how = 'any', inplace = True)\n",
    "        if i == 0:\n",
    "            merged_df = test2.copy()\n",
    "        else:\n",
    "            merged_df = pd.concat([merged_df, test2], ignore_index = True)\n",
    "        i += 1    \n",
    "    return merged_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filter out only AA flights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "carrier = 'AA'\n",
    "merged_df = merged_delays(flights, carrier)\n",
    "merged_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assigning different labels for each airport in the airports.csv file\n",
    "label_encoder = LabelEncoder()\n",
    "integer_encoded = label_encoder.fit_transform(merged_df['AIRPORT_ID'])\n",
    "#__________________________________________________________\n",
    "# Similarity between the codes and tags of the airports\n",
    "zipped = zip(integer_encoded, merged_df['AIRPORT_ID'])\n",
    "label_airports = list(set(list(zipped)))\n",
    "label_airports.sort(key = lambda x:x[0])\n",
    "label_airports[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**ONE HOT ENCODING EXPLANATION HERE.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Throughout the use of the one hot encoding method, we create an array which replaces the current airport labels into the previous labels which were set for the airports which is in numerical format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "onehot_encoder = OneHotEncoder(sparse=False)\n",
    "integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n",
    "onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n",
    "b = np.array(merged_df['hour_depart_min'])\n",
    "b = b.reshape(len(b),1)\n",
    "X = np.hstack((onehot_encoded, b))\n",
    "Y = np.array(merged_df['mean'])\n",
    "Y = Y.reshape(len(Y), 1)\n",
    "print(X.shape, Y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**MSE SCORE INTRODUCTION**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm = linear_model.LinearRegression()\n",
    "model = lm.fit(X,Y)\n",
    "predictions = lm.predict(X)\n",
    "print(\"MSE =\", round(metrics.mean_squared_error(predictions, Y)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tips = pd.DataFrame()\n",
    "tips[\"Airports\"] = pd.Series([float(s) for s in predictions]) \n",
    "tips[\"Mean_delays\"] = pd.Series([float(s) for s in Y]) \n",
    "sns.jointplot(x=\"Mean_delays\", y=\"Airports\", data=tips, size = 4, ratio = 5,\n",
    "              joint_kws={'line_kws':{'color':'red'}}, kind='reg')\n",
    "plt.show()\n",
    "print(\"Pearsonr:\", stats.pearsonr(tips[\"Airports\"], tips[\"Mean_delays\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Pearsonr score introduction**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model shows a positive correlation between the airport predictions along with the delays. The current MSE score is 54 which is not that great of a score as it is not well accurate with the real data. A solution would be to try a polynomial regression fit."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Polynomial Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**INTRODUCTION HERE.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "poly = PolynomialFeatures(degree = 2)\n",
    "regr = linear_model.LinearRegression()\n",
    "X_ = poly.fit_transform(X)\n",
    "regr.fit(X_, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = regr.predict(X_)\n",
    "print(\"MSE =\", round(metrics.mean_squared_error(result, Y)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tips = pd.DataFrame()\n",
    "tips[\"Airports\"] = pd.Series([float(s) for s in result]) \n",
    "tips[\"Mean_delays\"] = pd.Series([float(s) for s in Y]) \n",
    "sns.jointplot(x=\"Mean_delays\", y=\"Airports\", data=tips, size = 4, ratio = 7,\n",
    "              joint_kws={'line_kws':{'color':'red'}}, kind='reg')\n",
    "plt.show()\n",
    "print(\"Pearsonr:\", stats.pearsonr(tips[\"Airports\"], tips[\"Mean_delays\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is a definite difference between the two models in terms of MSE and Pearson r score. However, there is still a high positive correlation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Splitting The Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This process is splitting the dataset into training and test data which can help reduce overfitting and biased results. Hence, this will provide a better view of the predictions. Regularization technique will be performed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "poly = PolynomialFeatures(degree = 2)\n",
    "regr = linear_model.LinearRegression()\n",
    "X_ = poly.fit_transform(X_train)\n",
    "regr.fit(X_, Y_train)\n",
    "result = regr.predict(X_)\n",
    "score = metrics.mean_squared_error(result, Y_train)\n",
    "print(\"Mean squared error = \", score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_ = poly.fit_transform(X_test)\n",
    "result = regr.predict(X_)\n",
    "score = metrics.mean_squared_error(result, Y_test)\n",
    "print(\"Mean squared error = \", score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Ridge Regularization explanation here**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ridgereg = Ridge(alpha=0.3,normalize=True)\n",
    "poly = PolynomialFeatures(degree = 2)\n",
    "X_ = poly.fit_transform(X_train)\n",
    "ridgereg.fit(X_, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_ = poly.fit_transform(X_test)\n",
    "result = ridgereg.predict(X_)\n",
    "score = metrics.mean_squared_error(result, Y_test)\n",
    "print(\"Mean squared error = \", score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing the predictions for the month of January end."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "carrier = 'AA'\n",
    "merged_df_test = merged_delays(df_test, carrier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_ = poly.fit_transform(X_test)\n",
    "result = ridgereg.predict(X_)\n",
    "score = metrics.mean_squared_error(result, Y_test)\n",
    "'MSE = {:.2f}'.format(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'Average Delay = {:.2f} min'.format(np.sqrt(score))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}