diff --git a/nannyml/performance_calculation/metrics/binary_classification.py b/nannyml/performance_calculation/metrics/binary_classification.py
index a3e6d1eb..76fd80ac 100644
--- a/nannyml/performance_calculation/metrics/binary_classification.py
+++ b/nannyml/performance_calculation/metrics/binary_classification.py
@@ -99,8 +99,11 @@ def _calculate(self, data: pd.DataFrame):
         y_pred = data[self.y_pred_proba]
 
         if y_true.nunique() <= 1:
-            warnings.warn("Calculated ROC-AUC score contains NaN values.")
-            return np.nan
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             return roc_auc_score(y_true, y_pred)
 
@@ -166,9 +169,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated F1-score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             return f1_score(y_true, y_pred)
 
@@ -233,9 +245,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Precision score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             return precision_score(y_true, y_pred)
 
@@ -300,9 +321,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Recall score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             return recall_score(y_true, y_pred)
 
@@ -367,9 +397,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Specificity score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
             return tn / (tn + fp)
@@ -435,9 +474,18 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Accuracy score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                f"Returning NaN."
+            )
+            return np.NaN
         else:
             tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
             return (tp + tn) / (tp + tn + fp + fn)
@@ -537,7 +585,7 @@ def _calculate(self, data: pd.DataFrame):
         y_pred = data[self.y_pred]
 
         if y_true.shape[0] == 0:
-            warnings.warn("Calculated Business Value contains NaN values.")
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate business value. Returning NaN.")
             return np.NaN
 
         tp_value = self.business_value_matrix[1, 1]
@@ -600,7 +648,7 @@ def __init__(
                 ('False Positive', 'false_positive'),
                 ('False Negative', 'false_negative'),
             ],
-            lower_threshold_limit=0
+            lower_threshold_limit=0,
         )
 
         self.upper_threshold_value_limit: Optional[float] = 1.0 if normalize_confusion_matrix else None
@@ -793,8 +841,8 @@ def _calculate_false_negatives(self, data: pd.DataFrame) -> float:
         y_pred = data[self.y_pred]
 
         if y_true.empty or y_pred.empty:
-            warnings.warn("Calculated false_negatives contain NaN values.")
-            return np.nan
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
 
         num_fn = np.sum(np.logical_and(np.logical_not(y_pred), y_true))
         num_tn = np.sum(np.logical_and(np.logical_not(y_pred), np.logical_not(y_true)))
diff --git a/nannyml/performance_calculation/metrics/multiclass_classification.py b/nannyml/performance_calculation/metrics/multiclass_classification.py
index 6179477c..e425f91a 100644
--- a/nannyml/performance_calculation/metrics/multiclass_classification.py
+++ b/nannyml/performance_calculation/metrics/multiclass_classification.py
@@ -132,8 +132,11 @@ def _calculate(self, data: pd.DataFrame):
             )
 
         if y_true.nunique() <= 1:
-            warnings.warn("Calculated ROC-AUC score contains NaN values.")
-            return np.nan
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class for chunk, cannot calculate {self.display_name}. "
+                "Returning NaN."
+            )
+            return np.NaN
         else:
             return roc_auc_score(y_true, y_pred_proba, multi_class='ovr', average='macro', labels=labels)
 
@@ -219,9 +222,16 @@ def _calculate(self, data: pd.DataFrame):
                 f"could not calculate metric {self.display_name}: " "prediction column contains no data"
             )
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated F1-score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
         else:
             return f1_score(y_true, y_pred, average='macro', labels=labels)
 
@@ -307,9 +317,16 @@ def _calculate(self, data: pd.DataFrame):
                 f"could not calculate metric {self.display_name}: " "prediction column contains no data"
             )
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Precision score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
         else:
             return precision_score(y_true, y_pred, average='macro', labels=labels)
 
@@ -395,9 +412,16 @@ def _calculate(self, data: pd.DataFrame):
                 f"could not calculate metric {self.display_name}: " "prediction column contains no data"
             )
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Recall score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
         else:
             return recall_score(y_true, y_pred, average='macro', labels=labels)
 
@@ -483,9 +507,16 @@ def _calculate(self, data: pd.DataFrame):
                 f"could not calculate metric {self.display_name}: prediction column contains no data"
             )
 
-        if (y_true.nunique() <= 1) or (y_pred.nunique() <= 1):
-            warnings.warn("Calculated Specificity score contains NaN values.")
-            return np.nan
+        if y_true.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_true}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
+        elif y_pred.nunique() <= 1:
+            warnings.warn(
+                f"'{self.y_pred}' only contains a single class, cannot calculate {self.display_name}. Returning NaN."
+            )
+            return np.NaN
         else:
             MCM = multilabel_confusion_matrix(y_true, y_pred, labels=labels)
             tn_sum = MCM[:, 0, 0]
@@ -596,7 +627,7 @@ def __init__(
             threshold=threshold,
             y_pred_proba=y_pred_proba,
             components=[("None", "none")],
-            lower_threshold_limit=0
+            lower_threshold_limit=0,
         )
 
         self.normalize_confusion_matrix: Optional[str] = normalize_confusion_matrix
diff --git a/nannyml/performance_calculation/metrics/regression.py b/nannyml/performance_calculation/metrics/regression.py
index c2cc00c1..cf936d2b 100644
--- a/nannyml/performance_calculation/metrics/regression.py
+++ b/nannyml/performance_calculation/metrics/regression.py
@@ -1,6 +1,7 @@
 #  Author:   Niels Nuyttens  <niels@nannyml.com>
 #
 #  License: Apache Software License 2.0
+import warnings
 from typing import Optional, Tuple
 
 import numpy as np
@@ -81,8 +82,12 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if y_true.empty or y_pred.empty:
-            return np.nan
+        if y_true.empty:
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
+        elif y_pred.empty:
+            warnings.warn(f"'{self.y_pred}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
 
         return mean_absolute_error(y_true, y_pred)
 
@@ -139,8 +144,12 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if y_true.empty or y_pred.empty:
-            return np.nan
+        if y_true.empty:
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
+        elif y_pred.empty:
+            warnings.warn(f"'{self.y_pred}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
 
         return mean_absolute_percentage_error(y_true, y_pred)
 
@@ -197,8 +206,12 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if y_true.empty or y_pred.empty:
-            return np.nan
+        if y_true.empty:
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
+        elif y_pred.empty:
+            warnings.warn(f"'{self.y_pred}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
 
         return mean_squared_error(y_true, y_pred)
 
@@ -255,8 +268,12 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if y_true.empty or y_pred.empty:
-            return np.nan
+        if y_true.empty:
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
+        elif y_pred.empty:
+            warnings.warn(f"'{self.y_pred}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
 
         # TODO: include option to drop negative values as well?
 
@@ -318,8 +335,12 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if y_true.empty or y_pred.empty:
-            return np.nan
+        if y_true.empty:
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
+        elif y_pred.empty:
+            warnings.warn(f"'{self.y_pred}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
 
         return mean_squared_error(y_true, y_pred, squared=False)
 
@@ -376,8 +397,12 @@ def _calculate(self, data: pd.DataFrame):
         y_true = data[self.y_true]
         y_pred = data[self.y_pred]
 
-        if y_true.empty or y_pred.empty:
-            return np.nan
+        if y_true.empty:
+            warnings.warn(f"'{self.y_true}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
+        elif y_pred.empty:
+            warnings.warn(f"'{self.y_pred}' contains no data, cannot calculate {self.display_name}. Returning NaN.")
+            return np.NaN
 
         # TODO: include option to drop negative values as well?
 
diff --git a/nannyml/performance_estimation/confidence_based/metrics.py b/nannyml/performance_estimation/confidence_based/metrics.py
index 5dcd94dd..98246961 100644
--- a/nannyml/performance_estimation/confidence_based/metrics.py
+++ b/nannyml/performance_estimation/confidence_based/metrics.py
@@ -498,15 +498,23 @@ def _realized_performance(self, data: pd.DataFrame) -> float:
         _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba)
 
         if y_true is None:
-            warnings.warn("No 'y_true' values given for chunk, returning NaN as realized F1 score.")
+            warnings.warn(
+                f"No '{self.y_true}' values given for chunk, returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_true.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_true', returning NaN as realized F1 score.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_true}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_pred.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized F1 score.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_pred}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         return f1_score(y_true=y_true, y_pred=y_pred)
@@ -586,15 +594,23 @@ def _realized_performance(self, data: pd.DataFrame) -> float:
         _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba)
 
         if y_true is None:
-            warnings.warn("No 'y_true' values given for chunk, returning NaN as realized precision.")
+            warnings.warn(
+                f"No '{self.y_true}' values given for chunk, returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_true.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_true', returning NaN as realized precision.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_true}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_pred.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized precision.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_pred}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         return precision_score(y_true=y_true, y_pred=y_pred)
@@ -672,15 +688,23 @@ def _realized_performance(self, data: pd.DataFrame) -> float:
         _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba)
 
         if y_true is None:
-            warnings.warn("No 'y_true' values given for chunk, returning NaN as realized recall.")
+            warnings.warn(
+                f"No '{self.y_true}' values given for chunk, returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_true.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_true', returning NaN as recall precision.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_true}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_pred.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_pred', returning NaN as recall precision.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_pred}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         return recall_score(y_true=y_true, y_pred=y_pred)
@@ -758,15 +782,23 @@ def _realized_performance(self, data: pd.DataFrame) -> float:
         _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba)
 
         if y_true is None:
-            warnings.warn("No 'y_true' values given for chunk, returning NaN as realized specificity.")
+            warnings.warn(
+                f"No '{self.y_true}' values given for chunk, returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_true.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_true', returning NaN as realized specificity.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_true}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_pred.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized specificity.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_pred}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
@@ -849,15 +881,23 @@ def _realized_performance(self, data: pd.DataFrame) -> float:
         _, y_pred, y_true = self._common_cleaning(data, y_pred_proba_column_name=self.uncalibrated_y_pred_proba)
 
         if y_true is None:
-            warnings.warn("No 'y_true' values given for chunk, returning NaN as realized accuracy.")
+            warnings.warn(
+                f"No '{self.y_true}' values given for chunk, returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_true.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_true', returning NaN as realized accuracy.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_true}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         if y_pred.nunique() <= 1:
-            warnings.warn("Too few unique values present in 'y_pred', returning NaN as realized accuracy.")
+            warnings.warn(
+                f"Too few unique values present in '{self.y_pred}', "
+                f"returning NaN as realized {self.display_name} score."
+            )
             return np.NaN
 
         return accuracy_score(y_true=y_true, y_pred=y_pred)