Code fixes (#359)

* glossary cov shift update

* remove duplicate confidence upper bound functionality

* mypy and flake8 cbpe fixes

* dont allow confidence range to go above/below threshold limits

* fix check error on 0

* fix confidence range tests for BC CBPE when limit is 0

* fix confidence range tests for MC CBPE when limit is 0

* fix lower limit 0 for stats std values thresholds and confidence

* fix binary classification CM elements threshold logic

* fix realized perf MC CM threshold logic

docs/glossary.rst

@@ -52,7 +52,7 @@ Glossary
         model called :term:`nanny model`.
 
     Concept Drift
-        A change in the underlying pattern (or mapping) between the :term:`Model Inputs` and the :term:`Target` (P(y|X)).
+        A change in the underlying pattern (or mapping) between the :term:`Model Inputs` and the :term:`Target` (P(Y|X)).
 
     Confidence Band
         When we estimate a statistic from a sample, our estimation has to take into account the variance of that statistic
@@ -71,7 +71,8 @@ Glossary
         the actual *probability*. Regardless of the algorithm type, all classification models calculate some form of
         confidence scores. These scores are then thresholded to return the predicted class. Confidence scores can be
         turned into calibrated probabilities and used to estimate the performance of classification models in the absence
-        of ground truth, to learn more about this check out our :ref:`Confidence-based Performance Estimation Deep Dive<how-it-works-cbpe>`).
+        of ground truth, to learn more about this check out our
+        :ref:`Confidence-based Performance Estimation Deep Dive<how-it-works-cbpe>`.
 
     Confusion Matrix
         A confusion matrix is a table that is often used to describe the performance of a classification model (or
@@ -81,10 +82,13 @@ Glossary
         For more information on the confusion matrix, see the `Wikipedia Confusion Matrix page`_.
 
     Covariate Shift
-        A change in joint distribution of :term:`Model Inputs`, :math:`P(\mathbf{X})`.
+        A change in the distribution of :term:`Model Inputs`, :math:`P(\mathbf{X})`. Note that under covariate shift
+        while the distribution of model inputs changes the conditional probability :math:`P(Y|\mathbf{X})` does not change.
+        The latter is called :term:`Concept Drift`.
 
     Data Drift
-        A synonym for :term:`Covariate Shift`.
+        A change in joint distribution of :term:`Model Inputs` and model :term:`targets<Target>`, denoted as
+        :math:`P(\mathbf{X}, Y)`.
 
     Data Chunk
         A data chunk is simply a sample of data. All the results generated by NannyML are calculated and presented on the

nannyml/data_quality/missing/calculator.py

@@ -62,6 +62,9 @@ def __init__(
             Only one of `chunk_size`, `chunk_number` or `chunk_period` should be given.
         chunker : Chunker
             The `Chunker` used to split the data sets into a lists of chunks.
+        threshold: Threshold, default=StandardDeviationThreshold
+            The threshold you wish to evaluate values on. Defaults to a StandardDeviationThreshold with default
+            options. The other available value is ConstantThreshold.
 
 
         Examples
@@ -102,14 +105,13 @@ def __init__(
         self._lower_alert_thresholds: Dict[str, Optional[float]] = {column_name: 0 for column_name in self.column_names}
 
         self.lower_threshold_value_limit: float = 0
-        self.upper_threshold_value_limit: float
+        self.upper_threshold_value_limit: Optional[float] = None
         self.normalize = normalize
         if self.normalize:
             self.data_quality_metric = 'missing_values_rate'
             self.upper_threshold_value_limit = 1
         else:
             self.data_quality_metric = 'missing_values_count'
-            self.upper_threshold_value_limit = np.nan
 
     def _calculate_missing_value_stats(self, data: pd.Series):
         count_tot = data.shape[0]
@@ -217,8 +219,14 @@ def _calculate_for_column(self, data: pd.DataFrame, column_name: str) -> Dict[st
         else:
             result['sampling_error'] = serr * np.sqrt(tot)
 
-        result['upper_confidence_boundary'] = result['value'] + SAMPLING_ERROR_RANGE * result['sampling_error']
-        result['lower_confidence_boundary'] = result['value'] - SAMPLING_ERROR_RANGE * result['sampling_error']
+        result['upper_confidence_boundary'] = np.minimum(
+            result['value'] + SAMPLING_ERROR_RANGE * result['sampling_error'],
+            np.inf if self.upper_threshold_value_limit is None else self.upper_threshold_value_limit
+        )
+        result['lower_confidence_boundary'] = np.maximum(
+            result['value'] - SAMPLING_ERROR_RANGE * result['sampling_error'],
+            -np.inf if self.lower_threshold_value_limit is None else self.lower_threshold_value_limit
+        )
 
         result['upper_threshold'] = self._upper_alert_thresholds[column_name]
         result['lower_threshold'] = self._lower_alert_thresholds[column_name]

nannyml/data_quality/missing/result.py

@@ -24,7 +24,10 @@
 
 
 class Result(PerColumnResult, ResultCompareMixin):
-    """Contains the results of the univariate statistical drift calculation and provides plotting functionality."""
+    """Missing Values Result Class.
+
+    Contains calculation results and provides plotting functionality.
+    """
 
     def __init__(
         self,
@@ -34,13 +37,14 @@ def __init__(
         timestamp_column_name: Optional[str],
         chunker: Chunker,
     ):
+        """Initialize Missing Values Result Class."""
         super().__init__(results_data, column_names)
 
         self.timestamp_column_name = timestamp_column_name
         self.data_quality_metric = data_quality_metric
         self.chunker = chunker
 
-    def keys(self) -> List[Key]:
+    def keys(self) -> List[Key]:  # noqa: D102
         return [
             Key(
                 properties=(column_name,),
@@ -55,10 +59,7 @@ def plot(
         *args,
         **kwargs,
     ) -> go.Figure:
-        """
-
-        Parameters
-        ----------
+        """Plot Missing Values results.
 
         Returns
         -------
@@ -82,7 +83,6 @@ def plot(
         ...     res = res.filter(period='analysis', column_name=column_name).plot().show()
 
         """
-
         return plot_metrics(
             self,
             title='Data Quality ',

nannyml/performance_calculation/calculator.py

@@ -17,7 +17,6 @@
 
 Examples
 --------
-
 >>> import nannyml as nml
 >>> from IPython.display import display
 >>> reference_df, analysis_df, analysis_targets_df = nml.load_synthetic_car_loan_dataset()
@@ -113,6 +112,7 @@ def __init__(
                 - 'regression'
                 - 'classification_binary'
                 - 'classification_multiclass'
+
         y_pred_proba: ModelOutputsType, default=None
             Name(s) of the column(s) containing your model output.
             Pass a single string when there is only a single model output column, e.g. in binary classification cases.
@@ -254,7 +254,7 @@ def __init__(
 
         self.result: Optional[Result] = None
 
-    def __str__(self):
+    def __str__(self):  # noqa: D105
         return f"PerformanceCalculator[metrics={str(self.metrics)}]"
 
     @log_usage(UsageEvent.PERFORMANCE_CALC_FIT, metadata_from_self=['metrics', 'problem_type'])

nannyml/performance_calculation/metrics/binary_classification.py

@@ -600,13 +600,11 @@ def __init__(
                 ('False Positive', 'false_positive'),
                 ('False Negative', 'false_negative'),
             ],
+            lower_threshold_limit=0
         )
 
-        self.lower_threshold_limit: Optional[float] = 0.0 if normalize_confusion_matrix else None
-        self.upper_threshold_limit: Optional[float] = 1.0 if normalize_confusion_matrix else None
-
+        self.upper_threshold_value_limit: Optional[float] = 1.0 if normalize_confusion_matrix else None
         self.normalize_confusion_matrix: Optional[str] = normalize_confusion_matrix
-
         # sampling error
         self._sampling_error_components: Tuple = ()
 
@@ -683,8 +681,8 @@ def _calculate_confusion_matrix_alert_thresholds(
         lower_threshold_value, upper_threshold_value = calculate_threshold_values(
             threshold=self.threshold,
             data=np.asarray(chunked_reference_metric),
-            lower_threshold_value_limit=self.lower_threshold_limit,
-            upper_threshold_value_limit=self.upper_threshold_limit,
+            lower_threshold_value_limit=self.lower_threshold_value_limit,
+            upper_threshold_value_limit=self.upper_threshold_value_limit,
             logger=self._logger,
             metric_name=self.display_name,
         )

nannyml/performance_calculation/metrics/multiclass_classification.py

@@ -596,9 +596,11 @@ def __init__(
             threshold=threshold,
             y_pred_proba=y_pred_proba,
             components=[("None", "none")],
+            lower_threshold_limit=0
         )
 
         self.normalize_confusion_matrix: Optional[str] = normalize_confusion_matrix
+        self.upper_threshold_value_limit: Optional[float] = 1.0 if normalize_confusion_matrix else None
 
         self.classes: Optional[List[str]] = None
 

nannyml/performance_calculation/result.py

@@ -84,9 +84,7 @@ def __init__(
         self.analysis_data = analysis_data
 
     def keys(self) -> List[Key]:
-        """
-        Creates a list of keys where each Key is a `namedtuple('Key', 'properties display_names')`
-        """
+        """Creates a list of keys where each Key is a `namedtuple('Key', 'properties display_names')`."""
         return [
             Key(
                 properties=(component[1],),
@@ -108,6 +106,7 @@ def plot(
         **kwargs,
     ) -> go.Figure:
         """Render realized performance metrics.
+
         This function will return a :class:`plotly.graph_objects.Figure` object.
 
         Parameters

nannyml/performance_estimation/confidence_based/cbpe.py

@@ -298,14 +298,6 @@ def __init__(
 
         self.result: Optional[Result] = None
 
-    def __deepcopy__(self, memodict={}):
-        cls = self.__class__
-        result = cls.__new__(cls, y_pred_proba=self.y_pred_proba, problem_type=self.problem_type)
-        memodict[id(self)] = result
-        for k, v in self.__dict__.items():
-            setattr(result, k, copy.deepcopy(v, memodict))
-        return result
-
     @log_usage(UsageEvent.CBPE_ESTIMATOR_FIT, metadata_from_self=['metrics', 'problem_type'])
     def _fit(self, reference_data: pd.DataFrame, *args, **kwargs) -> CBPE:
         """Fits the drift calculator using a set of reference data.