model evaluation

Author: RauhanAhmed

src/components/data_transformation.py

@@ -21,7 +21,8 @@ class DataTransformationConfig:
     holidays:str = os.path.join("artifacts", "holidays.csv")
     processed_data:str = os.path.join("artifacts", "processed_data.csv")
     train_data:str = os.path.join("artifacts", "train_data.csv")
-    test_data:str = os.path.join("artifacts", "test_data.csv")
+    test_data:str = os.path.join("artifacts", "test_data.joblib")
+    test_data_covariates:str = os.path.join("artifacts", "test_data_covariates.joblib")
     timeseries_data:str = os.path.join("artifacts", "timeseries_data.joblib")
     covariates:str = os.path.join("artifacts", "covariates.joblib")
 
@@ -85,7 +86,7 @@ def integrate_data(self):
                 indices = processed_data[((processed_data["date"] == date) & (processed_data["city"] == city))].index
                 processed_data["is_holiday"][indices] = 1
 
-            processed_data.to_csv(self.datatransformationconfig.processed_data)
+            processed_data.to_csv(self.datatransformationconfig.processed_data, index = False)
 
             logging.info("data integration complete")
 
@@ -112,8 +113,8 @@ def split_data(self, number_of_test_days = 15):
             train_data = processed_data.iloc[:split_index + 1, :]
             test_data = processed_data.iloc[split_index + 1:, :]
 
-            train_data.to_csv(self.datatransformationconfig.train_data)
-            test_data.to_csv(self.datatransformationconfig.test_data)
+            train_data.to_csv(self.datatransformationconfig.train_data, index = False)
+            joblib.dump(test_data, self.datatransformationconfig.test_data)
 
             logging.info("data split complete")
 
@@ -131,7 +132,7 @@ def transform_data(self):
         logging.info("executing transform_data function")
         try:
             train_data = pd.read_csv(self.datatransformationconfig.train_data)
-            test_data = pd.read_csv(self.datatransformationconfig.test_data)
+            test_data = joblib.load(self.datatransformationconfig.test_data)
 
             train_data.drop(["id", "city", "store_type", "state", "cluster"], axis = 1, inplace = True)
             test_data.drop(["id", "city", "store_type", "state", "cluster"], axis = 1, inplace = True)
@@ -160,6 +161,19 @@ def transform_data(self):
                     covariates[cov].loc[date, :] = [np.NaN] * covariates[cov].shape[1]
                 covariates[cov] = covariates[cov].ffill()  
 
+            logging.info("reformatting test_data")
+
+            test_sales = {}
+            test_covariates = {}
+            for group, data_slice in test_data.groupby(by = ["store_nbr", "family"]):
+                data_slice.set_index("date", drop = True, inplace = True)
+                test_covariate = data_slice[["onpromotion", "dcoilwtico", "is_holiday"]]
+                test_sales_series = data_slice["sales"]
+                test_sales[group] = test_sales_series
+                test_covariates[str(group)] = test_covariate          
+
+            test_data = pd.DataFrame(data = test_sales)
+
             logging.info("detecting and removing outliers from different series")    
 
             temp = series_dataset.apply(lambda x : hampel(x, window_size = 7, n_sigma = 3.0).filtered_data)
@@ -178,24 +192,34 @@ def transform_data(self):
                     constant_features.append(feature)
             features_to_keep = set(series_dataset.columns).difference(set(constant_features))
             series_dataset = series_dataset[features_to_keep]
-            for constant_feature in constant_features:
-                test_data[~((test_data["store_nbr"] == constant_feature[0]) & (test_data["family"] == constant_feature[1]))]
+
             series_dataset = series_dataset[sorted(series_dataset.columns)]
+            test_data = test_data[series_dataset.columns]
 
-            logging.info("converting sales series and covariates into Darta TimeSeries")
+            logging.info("converting sales series and covariates into Darts TimeSeries")
 
             series_dataset.set_index(pd.to_datetime(series_dataset.index), inplace = True)
+            test_data.set_index(pd.to_datetime(test_data.index), inplace = True)
+
             timeseries_data = TimeSeries.from_dataframe(series_dataset)
+            test_data = TimeSeries.from_dataframe(test_data)
 
             for cov_key in covariates:
                 temp_cov = covariates[cov_key]
                 temp_cov.set_index(pd.to_datetime(temp_cov.index), inplace = True)
                 covariates[cov_key] = TimeSeries.from_dataframe(temp_cov)
 
+            for cov_key in test_covariates:
+                temp_cov = test_covariates[cov_key]
+                temp_cov.set_index(pd.to_datetime(temp_cov.index), inplace = True)
+                test_covariates[cov_key] = TimeSeries.from_dataframe(temp_cov)      
+
             joblib.dump(timeseries_data, self.datatransformationconfig.timeseries_data)
             joblib.dump(covariates, self.datatransformationconfig.covariates)
+            joblib.dump(test_data, self.datatransformationconfig.test_data)
+            joblib.dump(test_covariates, self.datatransformationconfig.test_data_covariates)
 
-            logging.info("saved timeseries_data and covariates to artifacts")
+            logging.info("saved timeseries_data, test_data and covariates to artifacts")
             logging.info(">>> DATA TRANSFORMATION COMPLETE <<<")
 
         except Exception as e:

src/components/model_evaluation.py

@@ -0,0 +1,74 @@
+from src.utils.exception import CustomException
+from src.utils.logger import logging
+from src.utils import generate_covariates
+from dataclasses import dataclass
+from sklearn.metrics import mean_squared_error
+from sklearn.preprocessing import MinMaxScaler
+import os
+import numpy as np
+import pandas as pd
+import joblib
+
+@dataclass
+class ModelEvaluationConfig:
+    trained_model_path:str = os.path.join("artifacts", "trained_model.joblib")
+    oil_model_path:str = os.path.join("artifacts", "oil_model.joblib")
+    covariates:str = os.path.join("artifacts", "covariates.joblib")
+    test_data_path:str = os.path.join("artifacts", "test_data.joblib")
+    timeseries_data_path:str = os.path.join("artifacts", "timeseries_data.joblib")
+    test_covariates_path:str = os.path.join("artifacts", "test_data_covariates.joblib")
+
+class ModelEvaluation:
+    def __init__(self):
+        self.modelevaluationconfig = ModelEvaluationConfig()
+
+    def generate_predictions(self):
+        try:
+            trained_model = joblib.load(self.modelevaluationconfig.trained_model_path)
+            oil_model = joblib.load(self.modelevaluationconfig.oil_model_path)
+            covariates = joblib.load(self.modelevaluationconfig.covariates)
+            test_data = joblib.load(self.modelevaluationconfig.test_data_path)
+            test_data_covariates = joblib.load(self.modelevaluationconfig.test_covariates_path)
+            timeseries_data = joblib.load(self.modelevaluationconfig.timeseries_data_path)
+
+            oil_forecasts = oil_model.predict(n = len(test_data)).pd_series().to_list()
+            new_covariates = [
+                covariates[cov].append(generate_covariates(
+                    horizon = len(test_data),
+                    onpromotion = test_data_covariates[cov].pd_dataframe()["onpromotion"],
+                    oil_forecasts = oil_forecasts,
+                    is_holiday = test_data_covariates[cov].pd_dataframe()["is_holiday"],
+                    trained_last_date = oil_model.training_series.end_time()
+                )) for cov in test_data.components
+            ]
+
+            predictions = trained_model.predict(
+                n = len(test_data),
+                series = [timeseries_data[series] for series in timeseries_data.components],
+                past_covariates = new_covariates
+            )
+
+            predictions_df = pd.DataFrame()
+            for prediction in predictions:
+                predictions_df[prediction.components[0]] = list(prediction.pd_series())
+
+            return timeseries_data, test_data, predictions_df
+        except Exception as e:
+            print(CustomException(e))
+
+    def evaluate_predictions(self, train_data, targets, predictions):
+        try:
+            scaler = MinMaxScaler()
+            scaler.fit(np.array(train_data))
+            real_values = scaler.transform(np.array(targets))
+            predicted_values = scaler.transform(np.array(predictions))
+
+            real = []
+            pred = []
+            for col in range(real_values.shape[1]):
+                real += real_values[:, col]
+                pred += predicted_values[:, col]
+
+            logging.info(mean_squared_error(real, pred))
+        except Exception as e:
+            print(CustomException(e))