Replace current error handling with "empty record" error handling (#361)

* Replace current error handling with "empty record" error handling

* Fix stupid web editor merge

* Improve warning messages

Author: nnansters

nannyml/drift/univariate/calculator.py

@@ -30,8 +30,10 @@
 from __future__ import annotations
 
 import warnings
+from logging import Logger
 from typing import Any, Dict, List, Optional, Union
 
+import numpy as np
 import pandas as pd
 from pandas import MultiIndex
 
@@ -344,7 +346,7 @@ def _calculate(self, data: pd.DataFrame, *args, **kwargs) -> Result:
             for column_name in self.continuous_column_names:
                 for method in self._column_to_models_mapping[column_name]:
                     try:
-                        for k, v in _calculate_for_column(chunk.data, column_name, method).items():
+                        for k, v in _calculate_for_column(chunk.data, column_name, method, self._logger).items():
                             row[f'{column_name}_{method.column_name}_{k}'] = v
                     except Exception as exc:
                         self._logger.error(
@@ -356,7 +358,7 @@ def _calculate(self, data: pd.DataFrame, *args, **kwargs) -> Result:
             for column_name in self.categorical_column_names:
                 for method in self._column_to_models_mapping[column_name]:
                     try:
-                        for k, v in _calculate_for_column(chunk.data, column_name, method).items():
+                        for k, v in _calculate_for_column(chunk.data, column_name, method, self._logger).items():
                             row[f'{column_name}_{method.column_name}_{k}'] = v
                     except Exception as exc:
                         self._logger.error(
@@ -400,14 +402,27 @@ def _calculate(self, data: pd.DataFrame, *args, **kwargs) -> Result:
         return self.result
 
 
-def _calculate_for_column(data: pd.DataFrame, column_name: str, method: Method) -> Dict[str, Any]:
+def _calculate_for_column(
+    data: pd.DataFrame, column_name: str, method: Method, logger: Optional[Logger] = None
+) -> Dict[str, Any]:
     result = {}
-    value = method.calculate(data[column_name])
-    result['value'] = value
-    result['upper_threshold'] = method.upper_threshold_value
-    result['lower_threshold'] = method.lower_threshold_value
-    result['alert'] = method.alert(value)
-    return result
+    try:
+        value = method.calculate(data[column_name])
+        result['value'] = value
+        result['upper_threshold'] = method.upper_threshold_value
+        result['lower_threshold'] = method.lower_threshold_value
+        result['alert'] = method.alert(value)
+    except Exception as exc:
+        if logger:
+            logger.error(
+                f"an unexpected exception occurred during calculation of method '{method.display_name}': " f"{exc}"
+            )
+        result['value'] = np.NaN
+        result['upper_threshold'] = method.upper_threshold_value
+        result['lower_threshold'] = method.lower_threshold_value
+        result['alert'] = np.NaN
+    finally:
+        return result
 
 
 def _create_multilevel_index(

nannyml/performance_calculation/calculator.py

@@ -354,14 +354,8 @@ def _calculate(self, data: pd.DataFrame, *args, **kwargs) -> Result:
     def _calculate_metrics_for_chunk(self, chunk: Chunk) -> Dict:
         chunk_records: Dict[str, Any] = {}
         for metric in self.metrics:
-            try:
-                chunk_record = metric.get_chunk_record(chunk.data)
-                chunk_records.update(chunk_record)
-            except Exception as exc:
-                self._logger.error(
-                    f"an unexpected error occurred while calculating metric {metric.display_name}: {exc}"
-                )
-                continue
+            chunk_record = metric.get_chunk_record(chunk.data)
+            chunk_records.update(chunk_record)
         return chunk_records
 
 

nannyml/performance_calculation/metrics/base.py

@@ -182,16 +182,27 @@ def get_chunk_record(self, chunk_data: pd.DataFrame) -> Dict:
 
         chunk_record = {}
 
-        realized_value = self.calculate(chunk_data)
-        sampling_error = self.sampling_error(chunk_data)
-
-        chunk_record[f'{column_name}_sampling_error'] = sampling_error
-        chunk_record[f'{column_name}'] = realized_value
-        chunk_record[f'{column_name}_upper_threshold'] = self.upper_threshold_value
-        chunk_record[f'{column_name}_lower_threshold'] = self.lower_threshold_value
-        chunk_record[f'{column_name}_alert'] = self.alert(realized_value)
-
-        return chunk_record
+        try:
+            realized_value = self.calculate(chunk_data)
+            sampling_error = self.sampling_error(chunk_data)
+
+            chunk_record[f'{column_name}_sampling_error'] = sampling_error
+            chunk_record[f'{column_name}'] = realized_value
+            chunk_record[f'{column_name}_upper_threshold'] = self.upper_threshold_value
+            chunk_record[f'{column_name}_lower_threshold'] = self.lower_threshold_value
+            chunk_record[f'{column_name}_alert'] = self.alert(realized_value)
+        except Exception as exc:
+            if self._logger:
+                self._logger.error(
+                    f"an unexpected exception occurred during calculation of method '{self.display_name}': " f"{exc}"
+                )
+            chunk_record[f'{column_name}_sampling_error'] = np.NaN
+            chunk_record[f'{column_name}'] = np.NaN
+            chunk_record[f'{column_name}_upper_threshold'] = self.upper_threshold_value
+            chunk_record[f'{column_name}_lower_threshold'] = self.lower_threshold_value
+            chunk_record[f'{column_name}_alert'] = np.NaN
+        finally:
+            return chunk_record
 
     @property
     def display_name(self) -> str:

nannyml/performance_calculation/metrics/binary_classification.py

@@ -99,8 +99,11 @@ def _calculate(self, data: pd.DataFrame):
         y_pred = data[self.y_pred_proba]
 
         if y_true.nunique() <= 1:
-            warnings.warn("Calculated ROC-AUC score contains NaN values.")
-            return np.nan
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             return roc_auc_score(y_true, y_pred)
 
@@ -166,9 +169,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated F1-score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             return f1_score(y_true, y_pred)
 
@@ -233,9 +245,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Precision score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             return precision_score(y_true, y_pred)
 
@@ -300,9 +321,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Recall score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             return recall_score(y_true, y_pred)
 
@@ -367,9 +397,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Specificity score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
             return tn / (tn + fp)
@@ -435,9 +474,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Accuracy score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
             return (tp + tn) / (tp + tn + fp + fn)
@@ -537,7 +585,7 @@ def _calculate(self, data: pd.DataFrame):
         y_pred = data[self.y_pred]
 
         if y_true.shape[0] == 0:
-            warnings.warn("Calculated Business Value contains NaN values.")
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate business value. Returning NaN.")
             return np.NaN
 
         tp_value = self.business_value_matrix[1, 1]
@@ -600,7 +648,7 @@ def __init__(
                 ('False Positive', 'false_positive'),
                 ('False Negative', 'false_negative'),
             ],
-            lower_threshold_limit=0
+            lower_threshold_limit=0,
         )
 
         self.upper_threshold_value_limit: Optional[float] = 1.0 if normalize_confusion_matrix else None
@@ -793,8 +841,8 @@ def _calculate_false_negatives(self, data: pd.DataFrame) -> float:
         y_pred = data[self.y_pred]
 
         if y_true.empty or y_pred.empty:
-            warnings.warn("Calculated false_negatives contain NaN values.")
-            return np.nan
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
 
         num_fn = np.sum(np.logical_and(np.logical_not(y_pred), y_true))
         num_tn = np.sum(np.logical_and(np.logical_not(y_pred), np.logical_not(y_true)))

nannyml/performance_calculation/metrics/multiclass_classification.py

@@ -132,8 +132,11 @@ def _calculate(self, data: pd.DataFrame):
             )
 
         if y_true.nunique() <= 1:
-            warnings.warn("Calculated ROC-AUC score contains NaN values.")
-            return np.nan
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                "Returning NaN."
+            )
+            return np.NaN
         else:
             return roc_auc_score(y_true, y_pred_proba, multi_class='ovr', average='macro', labels=labels)
 
@@ -219,9 +222,16 @@ def _calculate(self, data: pd.DataFrame):
                 f"could not calculate metric {self.display_name}: " "prediction column contains no data"
             )
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated F1-score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
         else:
             return f1_score(y_true, y_pred, average='macro', labels=labels)
 
@@ -307,9 +317,16 @@ def _calculate(self, data: pd.DataFrame):
                 f"could not calculate metric {self.display_name}: " "prediction column contains no data"
             )
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Precision score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
         else:
             return precision_score(y_true, y_pred, average='macro', labels=labels)
 
@@ -395,9 +412,16 @@ def _calculate(self, data: pd.DataFrame):
                 f"could not calculate metric {self.display_name}: " "prediction column contains no data"
             )
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Recall score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
         else:
             return recall_score(y_true, y_pred, average='macro', labels=labels)
 
@@ -483,9 +507,16 @@ def _calculate(self, data: pd.DataFrame):
                 f"could not calculate metric {self.display_name}: prediction column contains no data"
             )
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Specificity score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
         else:
             MCM = multilabel_confusion_matrix(y_true, y_pred, labels=labels)
             tn_sum = MCM[:, 0, 0]
@@ -596,7 +627,7 @@ def __init__(
             threshold=threshold,
             y_pred_proba=y_pred_proba,
             components=[("None", "none")],
-            lower_threshold_limit=0
+            lower_threshold_limit=0,
         )
 
         self.normalize_confusion_matrix: Optional[str] = normalize_confusion_matrix