Feature: events strategy evaluation

ats/evaluators.py

@@ -114,9 +114,6 @@ def _copy_dataset(self,dataset,models):
         return dataset_copies
 
     def evaluate(self,models={},granularity='point',strategy='flags',breakdown=False):
-        if strategy != 'flags':
-            raise NotImplementedError(f'Evaluation strategy {strategy} is not implemented')
-
         if not models:
             raise ValueError('There are no models to evaluate')
         if not self.test_data:
@@ -138,11 +135,23 @@ def evaluate(self,models={},granularity='point',strategy='flags',breakdown=False
             flagged_dataset = _get_model_output(dataset_copies[j],model)
             for i,sample_df in enumerate(flagged_dataset):
                 if granularity == 'point':
-                    single_model_evaluation[f'sample_{i+1}'] = _point_granularity_evaluation(sample_df,anomaly_labels_list[i],breakdown=breakdown)
+                    if strategy == 'flags':
+                        single_model_evaluation[f'sample_{i+1}'] = _point_granularity_evaluation(sample_df,anomaly_labels_list[i],breakdown=breakdown)
+                    elif strategy == 'events':
+                        single_model_evaluation[f'sample_{i+1}'] = _point_eval_with_events_strategy(sample_df,anomaly_labels_list[i],breakdown=breakdown)
+
                 elif granularity == 'variable':
-                    single_model_evaluation[f'sample_{i+1}'] = _variable_granularity_evaluation(sample_df,anomaly_labels_list[i], breakdown = breakdown)
+                    if strategy == 'flags':
+                        single_model_evaluation[f'sample_{i+1}'] = _variable_granularity_evaluation(sample_df,anomaly_labels_list[i], breakdown = breakdown)
+                    elif strategy == 'events':
+                        single_model_evaluation[f'sample_{i+1}'] = _variable_eval_with_events_strategy(sample_df,anomaly_labels_list[i],breakdown=breakdown)
+
                 elif granularity == 'series':
-                    single_model_evaluation[f'sample_{i+1}'] = _series_granularity_evaluation(sample_df,anomaly_labels_list[i], breakdown = breakdown)
+                    if strategy == 'flags':
+                        single_model_evaluation[f'sample_{i+1}'] = _series_granularity_evaluation(sample_df,anomaly_labels_list[i], breakdown = breakdown)
+                    elif strategy == 'events':
+                        single_model_evaluation[f'sample_{i+1}'] = _series_eval_with_events_strategy(sample_df,anomaly_labels_list[i],breakdown=breakdown)
+
                 else:
                     raise ValueError(f'Unknown granularity {granularity}')
 
@@ -360,3 +369,88 @@ def _series_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,break
         return one_series_evaluation_result | breakdown_info
     else:
         return one_series_evaluation_result
+
+def _variable_eval_with_events_strategy(sample_df,anomaly_labels_df,breakdown=False):
+    raise NotImplementedError('Evaluation with events strategy and variable granularity not implemented')
+
+def _point_eval_with_events_strategy(flagged_timeseries_df,anomaly_labels_df,breakdown=False):
+    detected_events_n = 0
+    false_positives_n = 0
+    total_events_n = 0
+    events_n, event_type_counts, event_time_slots = _count_anomalous_events(anomaly_labels_df)
+    evaluation_result = {}
+    breakdown_info = {}
+
+    flags_df = flagged_timeseries_df.filter(like='anomaly')
+    previous_point_info = {'label': 0}
+    for timestamp in flagged_timeseries_df.index:
+        anomaly_label = anomaly_labels_df.loc[timestamp]
+        # True if there is at least 1 variable detected as anomalous
+        is_anomalous = flags_df.loc[timestamp].any()
+        point_info = {anomaly_label: is_anomalous}
+        if point_info != previous_point_info and is_anomalous:
+            if anomaly_label is None:
+                false_positives_n += 1
+        previous_point_info = point_info
+
+    evaluation_result['false_positives_count'] = false_positives_n
+    evaluation_result['false_positives_ratio'] = false_positives_n/len(anomaly_labels_df)
+
+    for anomaly, anomaly_time_slots in event_time_slots.items():
+        anomaly_n = int(len(anomaly_time_slots))/2
+        total_events_n += anomaly_n
+        detected_anomaly_n = 0
+        for i in range(0,len(anomaly_time_slots),2):
+            start = anomaly_time_slots[i]
+            stop = anomaly_time_slots[i+1]
+            is_detected = flags_df.loc[start:stop].any().any()
+            if is_detected:
+                detected_anomaly_n +=1
+        breakdown_info[anomaly + '_true_positives_count'] = detected_anomaly_n
+        breakdown_info[anomaly + '_true_positives_rate'] = detected_anomaly_n/anomaly_n
+        detected_events_n += detected_anomaly_n
+
+    evaluation_result['true_positives_count'] = detected_events_n
+    if events_n:
+        evaluation_result['true_positives_rate'] = detected_events_n/total_events_n
+    else:
+        evaluation_result['true_positives_rate'] = None
+
+    if breakdown:
+        return evaluation_result | breakdown_info
+    else:
+        return evaluation_result
+
+def _series_eval_with_events_strategy(sample_df,anomaly_labels_df,breakdown=False):
+    raise NotImplementedError('Evaluation with events strategy and series granularity not implemented')
+
+def _count_anomalous_events(anomaly_labels_df):
+    events_n = 0
+    event_type_counts = {}
+    event_time_slots = {}
+    previous_anomaly_label = None
+    for timestamp in anomaly_labels_df.index:
+        anomaly_label = anomaly_labels_df.loc[timestamp]
+        if anomaly_label != previous_anomaly_label:
+            if anomaly_label is not None:
+                events_n += 1
+                # To manage series with adjoining anomalies
+                if previous_anomaly_label is not None:
+                    event_time_slots[key].append(previous_timestamp)
+                key = anomaly_label
+                if anomaly_label in event_type_counts.keys():
+                    event_type_counts[key] +=1
+                    event_time_slots[key].append(timestamp)
+                else:
+                    event_type_counts[key] =1
+                    event_time_slots[key] = [timestamp]
+
+            else:
+                event_time_slots[key].append(previous_timestamp)
+
+        previous_timestamp = timestamp
+        previous_anomaly_label = anomaly_label
+    # To manage series ending with an anomaly
+    if previous_anomaly_label is not None:
+        event_time_slots[key].append(previous_timestamp)
+    return events_n , event_type_counts, event_time_slots