engine.py

# -*- coding: utf-8 -*-
"""power-usage-prediction-1.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1qhEJsHqDklhkuAvxMDyBk5J_ExWPe4SU

# Downloading and Merging Data
"""

!pip install opendatasets --upgrade --quiet

import opendatasets as od

power_usage_url = "https://www.kaggle.com/srinuti/residential-power-usage-3years-data-timeseries?select=power_usage_2016_to_2020.csv"
weather_url = "https://www.kaggle.com/srinuti/residential-power-usage-3years-data-timeseries?select=weather_2016_2020_daily.csv"

od.download(power_usage_url)
od.download(weather_url)

import os
data_dir = "./residential-power-usage-3years-data-timeseries"
os.listdir(data_dir)

power_usage_csv = data_dir + "/power_usage_2016_to_2020.csv"
weather_csv = data_dir + "/weather_2016_2020_daily.csv"

!pip install pandas --quiet

import pandas as pd
power_usage_df = pd.read_csv(power_usage_csv)
weather_df = pd.read_csv(weather_csv)

power_usage_df

weather_df

n = power_usage_df.shape[0]
p1 = pd.Series(range(n), pd.period_range('2016-06-01 00:00:00', freq = '1H', periods = n))
power_usage_df['StartDate'] = p1.to_frame().index

power_usage_df['StartDate'] = power_usage_df['StartDate'].apply (lambda x: x.to_timestamp())
power_usage_df['Date'] = pd.DatetimeIndex(power_usage_df['StartDate']).date

power_usage_df

power_usage_df1 = power_usage_df.groupby('Date').sum()

power_usage_df1['day_of_week'] = power_usage_df1['day_of_week'].apply(lambda x: x/24)

notes_col = power_usage_df.groupby('Date').first()['notes'].values
power_usage_df1['notes'] = notes_col

power_usage_df1

m = weather_df.shape[0]
p2 = pd.Series(range(m), pd.period_range('2016-06-01', freq = '1D', periods = m))
weather_df['Date'] = p2.to_frame().index

weather_df['Date'] = weather_df['Date'].apply (lambda x: x.to_timestamp())

weather_df

df_temp = power_usage_df1['Value (kWh)'].values
k = power_usage_df1.shape[0]
df = weather_df[0:k]
df['kWh_usage'] = pd.Series(df_temp).to_frame()

df_temp = power_usage_df1['notes'].values
df['tags'] = pd.Series(df_temp).to_frame()

df

"""# Exploratory Data Analysis"""

df.head()

df.info()

df.isna().sum()

df.corr()

!pip install plotly matplotlib seaborn --quiet

# Commented out IPython magic to ensure Python compatibility.
import plotly.express as px
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
# %matplotlib inline

sns.set_style('darkgrid')
matplotlib.rcParams['font.size'] = 14
matplotlib.rcParams['figure.figsize'] = (10, 6)
matplotlib.rcParams['figure.facecolor'] = '#00000000'

fig = px.histogram(df, x='Temp_avg', 
                   marginal='box', 
                   title='Distribution of Avg Temp')
fig.update_layout(bargap=0.1)
fig.show()

"""Temperature distribution is not uniform. We have more warmer days than colder days. We also have very few data on days with temperature below 35. Median of Average-temperature lies at 73.8"""

fig = px.histogram(df, x='Dew_avg', 
                   marginal='box', 
                   title='Distribution of Avg Dew')
fig.update_layout(bargap=0.1)
fig.show()

fig = px.histogram(df, x='Hum_avg', 
                   marginal='box', 
                   title='Distribution of Avg Humidity')
fig.update_layout(bargap=0.1)
fig.show()

fig = px.histogram(df, x='Wind_avg', 
                   marginal='box', 
                   title='Distribution of Avg Wind')
fig.update_layout(bargap=0.1)
fig.show()

fig = px.histogram(df, x='Press_avg', 
                   marginal='box', 
                   title='Distribution of Avg Pressure')
fig.update_layout(bargap=0.1)
fig.show()

fig = px.histogram(df, x='Precipit', 
                   marginal='box', 
                   title='Distribution of Precipitation')
fig.update_layout(bargap=0.1)
fig.show()

"""# Imputing Missing Values"""

!pip install numpy --quiet

import numpy as np

numeric_cols = df.select_dtypes(include=np.number).columns.tolist()[1:-2]
categorical_cols = df.select_dtypes('object').columns.tolist()

df[numeric_cols].describe()

from sklearn.impute import SimpleImputer

imputer = SimpleImputer(strategy = 'mean')

imputer.fit(df[numeric_cols])

list(imputer.statistics_)

"""# Scaling Numeric Columns"""

df[numeric_cols].describe()

from sklearn.preprocessing import MinMaxScaler

scaler = MinMaxScaler()

scaler.fit(df[numeric_cols])

df[numeric_cols] = scaler.transform(df[numeric_cols])

df[numeric_cols].describe()

"""# Encoding Categorical Data"""

df[categorical_cols].nunique()

from sklearn.preprocessing import OneHotEncoder

encoder = OneHotEncoder(sparse=False, handle_unknown='ignore')

encoder.fit(df[categorical_cols])

encoder.categories_

encoded_cols = list(encoder.get_feature_names(categorical_cols))

df[encoded_cols] = encoder.transform(df[categorical_cols])

df.head()

"""# Train Test Split"""

plt.title('No. of Rows per Year')
sns.countplot(x=pd.to_datetime(df.Date).dt.year);

'''
year = pd.to_datetime(df.Date).dt.year
month = pd.to_datetime(df.Date).dt.month

train_df = df[year < 2020]
test_df = df[year >= 2020]
'''

train_df = df[:1120]
test_df = df[1120:]

print('train_df.shape :', train_df.shape)
print('test_df.shape :', test_df.shape)

train_df

test_df

input_cols = numeric_cols + encoded_cols
target_col = 'kWh_usage'

print(input_cols)

train_inputs = train_df[input_cols].copy()
train_targets = train_df[target_col].copy()

test_inputs = test_df[input_cols].copy()
test_targets = test_df[target_col].copy()

train_inputs

train_targets

"""# File Backup"""

print('train_inputs:', train_inputs.shape)
print('train_targets:', train_targets.shape)
print('test_inputs:', test_inputs.shape)
print('test_targets:', test_targets.shape)

!pip install pyarrow --quiet

# Commented out IPython magic to ensure Python compatibility.
# %%time
# train_inputs.to_parquet('train_inputs.parquet')
# test_inputs.to_parquet('test_inputs.parquet')
# 
# pd.DataFrame(train_targets).to_parquet('train_targets.parquet')
# pd.DataFrame(test_targets).to_parquet('test_targets.parquet')

#to verify if parquet files are generated correctly
train_inputs = pd.read_parquet('train_inputs.parquet')
test_inputs = pd.read_parquet('test_inputs.parquet')

train_targets = pd.read_parquet('train_targets.parquet')[target_col]
test_targets = pd.read_parquet('test_targets.parquet')[target_col]

print('train_inputs:', train_inputs.shape)
print('train_targets:', train_targets.shape)
print('test_inputs:', test_inputs.shape)
print('test_targets:', test_targets.shape)

"""# Model 1 - LinearRegression()"""

def rmse(targets, predictions):
    return np.sqrt(np.mean(np.square(targets - predictions)))

from sklearn.linear_model import LinearRegression
model1 = LinearRegression()

print('train_inputs:', train_inputs.shape)
print('train_targets:', train_targets.shape)
print('test_inputs:', test_inputs.shape)
print('test_targets:', test_targets.shape)

model1.fit(train_inputs, train_targets)

train_preds = model1.predict(train_inputs)

rmse(train_targets, train_preds)

model1.coef_

model1.intercept_

from sklearn.metrics import r2_score

r2_score(train_targets, train_preds)

test_preds = model1.predict(test_inputs)

rmse(test_targets, test_preds)

r2_score(test_targets, test_preds)

"""# Model 2 - SGDRegressor()"""

from sklearn.linear_model import SGDRegressor
model2 = SGDRegressor()

model2.fit(train_inputs, train_targets)

train_preds = model2.predict(train_inputs)

rmse(train_targets, train_preds)

model2.coef_

model2.intercept_

test_preds = model2.predict(test_inputs)
rmse(test_targets, test_preds)

r2_score(test_targets, test_preds)

"""# Model Backup"""

import joblib

Model1 = {
    'model': model1,
    'imputer': imputer,
    'scaler': scaler,
    'encoder': encoder,
    'input_cols': input_cols,
    'target_col': target_col,
    'numeric_cols': numeric_cols,
    'categorical_cols': categorical_cols,
    'encoded_cols': encoded_cols
}

joblib.dump(Model1, "Model1.joblib")

Model2 = {
    'model': model2,
    'imputer': imputer,
    'scaler': scaler,
    'encoder': encoder,
    'input_cols': input_cols,
    'target_col': target_col,
    'numeric_cols': numeric_cols,
    'categorical_cols': categorical_cols,
    'encoded_cols': encoded_cols
}

joblib.dump(Model2, "Model2.joblib")

"""# Helper Functions

When the input is in form of a Pandas Series:
"""

def list_predict(inputs, targets):
    preds = model1.predict(inputs) #use model1 or model2 here
    accuracy = r2_score(targets, preds)
    print("Accuracy: {:.2f}%".format(accuracy * 100))
    return preds

test_preds = list_predict(test_inputs, test_targets)

"""For a single input:"""

def single_predict(single_input):
    input_df = pd.DataFrame([single_input])
    input_df[numeric_cols] = imputer.transform(input_df[numeric_cols])
    input_df[numeric_cols] = scaler.transform(input_df[numeric_cols])
    input_df[encoded_cols] = encoder.transform(input_df[categorical_cols])
    X_input = input_df[numeric_cols + encoded_cols]
    pred = model1.predict(X_input)[0]
    print(pred)
    return pred

new_input = {
    'Date': '2021-08-18',
    'Day': 3,
    'Temp_max': 32,
    'Temp_avg': 30,
    'Temp_min': 26,
    'Dew_max': 27,
    'Dew_avg': 26,
    'Dew_min': 24,
    'Hum_max': 93,
    'Hum_avg': 92,
    'Hum_min': 90,
    'Wind_max': 7.3,
    'Wind_avg': 6,
    'Wind_min': 4.2,
    'Press_max': 1003.2,
    'Press_avg': 1003,
    'Press_min': 1003,
    'Precipit': 0,
    'day_of_week': 3,
    'tags': 'Weekday'
}

test_preds = single_predict(new_input)