From 1b730c12da8035c456c3eb74777347d80a793415 Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Mon, 24 Nov 2025 16:04:31 +0100 Subject: [PATCH 01/10] Add "granularity" argument in "evaluate()" and "_calculate_model_scores()" This argument has 'data_point' as default --- ats/evaluators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ats/evaluators.py b/ats/evaluators.py index 4bbce18..ae30518 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -46,7 +46,7 @@ def evaluate_anomaly_detector(evaluated_timeseries_df, anomaly_labels, details=F return evaluation_results -def _calculate_model_scores(single_model_evaluation={}): +def _calculate_model_scores(single_model_evaluation={},granularity='data_point'): anomalies = list(single_model_evaluation['sample_1'].keys()) samples_n = len(single_model_evaluation) detections_per_anomaly = {} @@ -80,7 +80,7 @@ def _copy_dataset(self,dataset,models): dataset_copies.append(dataset_copy) return dataset_copies - def evaluate(self,models={}): + def evaluate(self,models={},granularity='data_point'): if not models: raise ValueError('There are no models to evaluate') if not self.test_data: @@ -102,7 +102,7 @@ def evaluate(self,models={}): flagged_dataset = _get_model_output(dataset_copies[j],model) for i,sample_df in enumerate(flagged_dataset): single_model_evaluation[f'sample_{i+1}'] = evaluate_anomaly_detector(sample_df,anomaly_labels_list[i]) - models_scores[model_name] = _calculate_model_scores(single_model_evaluation) + models_scores[model_name] = _calculate_model_scores(single_model_evaluation,granularity=granularity) j+=1 return models_scores From 8407da70bf34100a5e858853009eaaed1a938892 Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Mon, 24 Nov 2025 19:13:27 +0100 Subject: [PATCH 02/10] Add "_variable_granularity_evaluation()" This function evaluates the model with "variable" granularity on a single timeseries --- ats/evaluators.py | 19 +++++++++++++++++++ ats/tests/test_evaluators.py | 16 ++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/ats/evaluators.py b/ats/evaluators.py index ae30518..4973d11 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -123,3 +123,22 @@ def _get_model_output(dataset,model): flagged_dataset.append(flagged_series) return flagged_dataset + +def _variable_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df): + one_series_evaluation_result = {} + flag_columns_n = len(flagged_timeseries_df.filter(like='anomaly').columns) + variables_n = len(flagged_timeseries_df.columns) - flag_columns_n + if variables_n != 1 and variables_n != flag_columns_n: + raise ValueError('Variable granularity is not for this model') + normalization_factor = variables_n * len(flagged_timeseries_df) + + for anomaly,frequency in anomaly_labels_df.value_counts(dropna=False).items(): + anomaly_count = 0 + for timestamp in flagged_timeseries_df.index: + if anomaly_labels_df[timestamp] == anomaly: + for column in flagged_timeseries_df.filter(like='anomaly').columns: + anomaly_count += flagged_timeseries_df.loc[timestamp,column] + one_series_evaluation_result[anomaly] = anomaly_count / normalization_factor + + one_series_evaluation_result['false_positives'] = one_series_evaluation_result.pop(None) + return one_series_evaluation_result \ No newline at end of file diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index c0cc581..e3ee33b 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -6,6 +6,7 @@ from ..evaluators import _format_for_anomaly_detector from ..evaluators import _calculate_model_scores from ..evaluators import Evaluator +from ..evaluators import _variable_granularity_evaluation import unittest import pandas as pd import random as rnd @@ -300,3 +301,18 @@ def test_copy_dataset(self): self.assertEqual(len(dataset_copies[0]),2) self.assertIsInstance(dataset_copies[1],list) self.assertEqual(len(dataset_copies[1]),2) + + def test_variable_granularity_evaluation(self): + series_generator = HumiTempTimeseriesGenerator() + series = series_generator.generate(anomalies=['step_uv']) + minmax = MinMaxAnomalyDetector() + formatted_series,anomaly_labels = _format_for_anomaly_detector(series,synthetic=True) + flagged_series = minmax.apply(formatted_series) + evaluation_result = _variable_granularity_evaluation(flagged_series,anomaly_labels) + # evaluation_result: + # { 'step_uv': 0.00034722222222222224 + # 'false_positives': 0.00034722222222222224 + # } + self.assertEqual(len(evaluation_result),2) + self.assertAlmostEqual(evaluation_result['step_uv'],1/len(series)) + self.assertAlmostEqual(evaluation_result['false_positives'],1/len(series)) From d5e1eecee088bd98d6911b7d0fe9fbc796485997 Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Mon, 24 Nov 2025 19:31:36 +0100 Subject: [PATCH 03/10] Add "_point_granularity_evaluation()" function This function evaluates the model with point granularity on a single series --- ats/evaluators.py | 19 ++++++++++++++++++- ats/tests/test_evaluators.py | 16 ++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/ats/evaluators.py b/ats/evaluators.py index 4973d11..2656221 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -141,4 +141,21 @@ def _variable_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df): one_series_evaluation_result[anomaly] = anomaly_count / normalization_factor one_series_evaluation_result['false_positives'] = one_series_evaluation_result.pop(None) - return one_series_evaluation_result \ No newline at end of file + return one_series_evaluation_result + +def _point_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df): + one_series_evaluation_result = {} + normalization_factor = len(flagged_timeseries_df) + + for anomaly,frequency in anomaly_labels_df.value_counts(dropna=False).items(): + anomaly_count = 0 + for timestamp in flagged_timeseries_df.index: + if anomaly_labels_df[timestamp] == anomaly: + for column in flagged_timeseries_df.filter(like='anomaly').columns: + if flagged_timeseries_df.loc[timestamp,column]: + anomaly_count += 1 + break + one_series_evaluation_result[anomaly] = anomaly_count / normalization_factor + + one_series_evaluation_result['false_positives'] = one_series_evaluation_result.pop(None) + return one_series_evaluation_result diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index e3ee33b..13594de 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -7,6 +7,7 @@ from ..evaluators import _calculate_model_scores from ..evaluators import Evaluator from ..evaluators import _variable_granularity_evaluation +from ..evaluators import _point_granularity_evaluation import unittest import pandas as pd import random as rnd @@ -316,3 +317,18 @@ def test_variable_granularity_evaluation(self): self.assertEqual(len(evaluation_result),2) self.assertAlmostEqual(evaluation_result['step_uv'],1/len(series)) self.assertAlmostEqual(evaluation_result['false_positives'],1/len(series)) + + def test_point_granularity_evaluation(self): + series_generator = HumiTempTimeseriesGenerator() + series = series_generator.generate(anomalies=['step_uv']) + minmax = MinMaxAnomalyDetector() + formatted_series,anomaly_labels = _format_for_anomaly_detector(series,synthetic=True) + flagged_series = minmax.apply(formatted_series) + evaluation_result = _point_granularity_evaluation(flagged_series,anomaly_labels) + # evaluation_result: + # { 'step_uv': 0.0006944444444444445 + # 'false_positives': 0.0006944444444444445 + # } + self.assertEqual(len(evaluation_result),2) + self.assertAlmostEqual(evaluation_result['step_uv'],2/len(series)) + self.assertAlmostEqual(evaluation_result['false_positives'],2/len(series)) From 81e38eede4e1b8b43ce58e0750736c3b04af623d Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Mon, 24 Nov 2025 20:21:42 +0100 Subject: [PATCH 04/10] Add function "_series_granularity_evaluation()" This function evaluates the model with series granularity on a single timeseries --- ats/evaluators.py | 23 +++++++++++++++++++++++ ats/tests/test_evaluators.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/ats/evaluators.py b/ats/evaluators.py index 2656221..228595b 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -159,3 +159,26 @@ def _point_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df): one_series_evaluation_result['false_positives'] = one_series_evaluation_result.pop(None) return one_series_evaluation_result + +def _series_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df): + anomalies = [] + for anomaly,frequency in anomaly_labels_df.value_counts(dropna=False).items(): + if anomaly is not None: + anomalies.append(anomaly) + anomalies_n = len(anomalies) + if anomalies_n > 1: + raise ValueError('Evaluation with series granularity supports series with only one anomaly') + + one_series_evaluation_result = {} + is_series_anomalous = 0 + for timestamp in flagged_timeseries_df.index: + for column in flagged_timeseries_df.filter(like='anomaly').columns: + if flagged_timeseries_df.loc[timestamp,column]: + is_series_anomalous = 1 + break + if is_series_anomalous and not anomalies: + one_series_evaluation_result['false_positives'] = 1 + elif is_series_anomalous and anomalies: + one_series_evaluation_result[anomalies[0]] = 1 + + return one_series_evaluation_result \ No newline at end of file diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index 13594de..e9a4cc0 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -8,6 +8,7 @@ from ..evaluators import Evaluator from ..evaluators import _variable_granularity_evaluation from ..evaluators import _point_granularity_evaluation +from ..evaluators import _series_granularity_evaluation import unittest import pandas as pd import random as rnd @@ -332,3 +333,35 @@ def test_point_granularity_evaluation(self): self.assertEqual(len(evaluation_result),2) self.assertAlmostEqual(evaluation_result['step_uv'],2/len(series)) self.assertAlmostEqual(evaluation_result['false_positives'],2/len(series)) + + def test_series_granularity_evaluation(self): + series_generator = HumiTempTimeseriesGenerator() + series = series_generator.generate(anomalies=['step_uv']) + minmax = MinMaxAnomalyDetector() + formatted_series,anomaly_labels = _format_for_anomaly_detector(series,synthetic=True) + flagged_series = minmax.apply(formatted_series) + evaluation_result = _series_granularity_evaluation(flagged_series,anomaly_labels) + # evaluation_result: + # { 'step_uv': 1 + # } + self.assertEqual(len(evaluation_result),1) + self.assertAlmostEqual(evaluation_result['step_uv'],1) + + series1 = series_generator.generate(anomalies=[]) + minmax1 = MinMaxAnomalyDetector() + formatted_series1,anomaly_labels1 = _format_for_anomaly_detector(series1,synthetic=True) + flagged_series1 = minmax.apply(formatted_series1) + evaluation_result1 = _series_granularity_evaluation(flagged_series1,anomaly_labels1) + self.assertEqual(len(evaluation_result1),1) + self.assertAlmostEqual(evaluation_result1['false_positives'],1) + # evaluation_result1: + # { 'false_positives': 1 + # } + + try: + series2 = series_generator.generate(anomalies=['spike_uv','step_uv']) + formatted_series2,anomaly_labels2 = _format_for_anomaly_detector(series2,synthetic=True) + flagged_series2 = minmax.apply(formatted_series2) + evaluation_result2 = _series_granularity_evaluation(flagged_series2,anomaly_labels2) + except Exception as e: + self.assertIsInstance(e,ValueError) From fdfc76b23e1bcd96ed250a605b28651f5373bff1 Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Thu, 27 Nov 2025 13:07:07 +0100 Subject: [PATCH 05/10] Change the structure of "evaluate()" function This changes have been made in order to include the parameter granularity in the eveluation --- ats/evaluators.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ats/evaluators.py b/ats/evaluators.py index 228595b..f6e4e4d 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -101,7 +101,13 @@ def evaluate(self,models={},granularity='data_point'): single_model_evaluation = {} flagged_dataset = _get_model_output(dataset_copies[j],model) for i,sample_df in enumerate(flagged_dataset): - single_model_evaluation[f'sample_{i+1}'] = evaluate_anomaly_detector(sample_df,anomaly_labels_list[i]) + if granularity == 'data_point': + single_model_evaluation[f'sample_{i+1}'] = _point_granularity_evaluation(sample_df,anomaly_labels) + if granularity == 'variable': + single_model_evaluation[f'sample_{i+1}'] = _variable_granularity_evaluation(sample_df,anomaly_labels) + if granularity == 'series': + single_model_evaluation[f'sample_{i+1}'] = _series_granularity_evaluation(sample_df,anomaly_labels) + models_scores[model_name] = _calculate_model_scores(single_model_evaluation,granularity=granularity) j+=1 From 84dd3e864ba4e185b220f154e742badfd8509500 Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Thu, 27 Nov 2025 21:49:43 +0100 Subject: [PATCH 06/10] Modify "_calculate_model_scores()" to include granularity --- ats/evaluators.py | 32 ++++++++++----------- ats/tests/test_evaluators.py | 54 ++++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 25 deletions(-) diff --git a/ats/evaluators.py b/ats/evaluators.py index f6e4e4d..c76515c 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -47,26 +47,26 @@ def evaluate_anomaly_detector(evaluated_timeseries_df, anomaly_labels, details=F def _calculate_model_scores(single_model_evaluation={},granularity='data_point'): - anomalies = list(single_model_evaluation['sample_1'].keys()) - samples_n = len(single_model_evaluation) - detections_per_anomaly = {} - avg_detections_per_anomaly = {} + dataset_anomalies = set() + for sample in single_model_evaluation.keys(): + sample_anomalies = set(single_model_evaluation[sample].keys()) + dataset_anomalies.update(sample_anomalies) - for anomaly in anomalies: - detections_per_anomaly[anomaly] = 0 + anomaly_scores = {} + for anomaly in dataset_anomalies: + anomaly_scores[anomaly] = 0 - for sample in single_model_evaluation.keys(): - for anomaly in single_model_evaluation[sample].keys(): - # TODO: evaluate_anomaly_detector and calculate_model_scores are redundant - if single_model_evaluation[sample][anomaly] and anomaly != 'false_positives': - detections_per_anomaly[anomaly] +=1 - elif anomaly == 'false_positives': - detections_per_anomaly[anomaly] +=single_model_evaluation[sample][anomaly] + for anomaly in dataset_anomalies: + for sample in single_model_evaluation.keys(): + if anomaly in single_model_evaluation[sample].keys(): + anomaly_scores[anomaly] += single_model_evaluation[sample][anomaly] - for anomaly,counts in detections_per_anomaly.items(): - avg_detections_per_anomaly[anomaly] = counts/samples_n if anomaly != 'false_positives' else counts + if granularity == 'series': + samples_n = len(single_model_evaluation) + for key in anomaly_scores.keys(): + anomaly_scores[key] /= samples_n - return avg_detections_per_anomaly + return anomaly_scores class Evaluator(): diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index e9a4cc0..c7d9f65 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -242,25 +242,61 @@ def test_get_model_output(self): def test_calculate_model_scores(self): single_model_evaluation = { 'sample_1': { - 'anomaly_1': True, - 'anomaly_2': False, - 'false_positives': 2 + 'anomaly_1': 0.5, + 'anomaly_2': 0.2, + 'false_positives': 0.6 + }, + 'sample_2': { + 'anomaly_1': 0.3, + 'anomaly_2': 0.4, + 'anomaly_3': 0.1, + 'false_positives': 0.2 + }, + } + model_scores = _calculate_model_scores(single_model_evaluation,granularity='data_point') + # model_scores: + # { 'anomaly_3': 0.1, + # 'anomaly_1': 0.8, + # 'false_positives': 0.8, + # 'anomaly_2': 0.6 + # } + self.assertEqual(len(model_scores),4) + self.assertIsInstance(model_scores,dict) + self.assertIn('anomaly_1',model_scores.keys()) + self.assertIn('anomaly_2',model_scores.keys()) + self.assertIn('anomaly_3',model_scores.keys()) + self.assertIn('false_positives',model_scores.keys()) + self.assertAlmostEqual(model_scores['anomaly_1'],0.8) + self.assertAlmostEqual(model_scores['anomaly_2'],0.6) + self.assertAlmostEqual(model_scores['anomaly_3'],0.1) + self.assertAlmostEqual(model_scores['false_positives'],0.8) + + def test_calculate_model_score_series_granularity(self): + single_model_evaluation = { + 'sample_1': { + 'anomaly_1': 1, }, 'sample_2': { - 'anomaly_1': True, - 'anomaly_2': True, 'false_positives': 1 }, + 'sample_3': { + 'anomaly_2': 1 + } } - model_scores = _calculate_model_scores(single_model_evaluation) + model_scores = _calculate_model_scores(single_model_evaluation,granularity='series') + # model_scores: + # { 'anomaly_1': 0.3333333333333333, + # 'false_positives': 0.3333333333333333, + # 'anomaly_2': 0.3333333333333333 + # } self.assertEqual(len(model_scores),3) self.assertIsInstance(model_scores,dict) self.assertIn('anomaly_1',model_scores.keys()) self.assertIn('anomaly_2',model_scores.keys()) self.assertIn('false_positives',model_scores.keys()) - self.assertAlmostEqual(model_scores['anomaly_1'],1.0) - self.assertAlmostEqual(model_scores['anomaly_2'],0.5) - self.assertAlmostEqual(model_scores['false_positives'],3) + self.assertAlmostEqual(model_scores['anomaly_1'],0.3333333333333333) + self.assertAlmostEqual(model_scores['anomaly_2'],0.3333333333333333) + self.assertAlmostEqual(model_scores['false_positives'],0.333333333333333) def test_evaluate(self): anomalies = ['spike_uv','step_uv'] From 9c35e0eee972930c43bf4b86cd4af73f451e5cdf Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Thu, 27 Nov 2025 23:03:58 +0100 Subject: [PATCH 07/10] Test the evaluation with 'data_point' granularity --- ats/evaluators.py | 6 +++--- ats/tests/test_evaluators.py | 27 +++++++++++++++------------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/ats/evaluators.py b/ats/evaluators.py index c76515c..a013be7 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -102,11 +102,11 @@ def evaluate(self,models={},granularity='data_point'): flagged_dataset = _get_model_output(dataset_copies[j],model) for i,sample_df in enumerate(flagged_dataset): if granularity == 'data_point': - single_model_evaluation[f'sample_{i+1}'] = _point_granularity_evaluation(sample_df,anomaly_labels) + single_model_evaluation[f'sample_{i+1}'] = _point_granularity_evaluation(sample_df,anomaly_labels_list[i]) if granularity == 'variable': - single_model_evaluation[f'sample_{i+1}'] = _variable_granularity_evaluation(sample_df,anomaly_labels) + single_model_evaluation[f'sample_{i+1}'] = _variable_granularity_evaluation(sample_df,anomaly_labels_list[i]) if granularity == 'series': - single_model_evaluation[f'sample_{i+1}'] = _series_granularity_evaluation(sample_df,anomaly_labels) + single_model_evaluation[f'sample_{i+1}'] = _series_granularity_evaluation(sample_df,anomaly_labels_list[i]) models_scores[model_name] = _calculate_model_scores(single_model_evaluation,granularity=granularity) j+=1 diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index c7d9f65..b0bca7b 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -298,11 +298,13 @@ def test_calculate_model_score_series_granularity(self): self.assertAlmostEqual(model_scores['anomaly_2'],0.3333333333333333) self.assertAlmostEqual(model_scores['false_positives'],0.333333333333333) - def test_evaluate(self): - anomalies = ['spike_uv','step_uv'] + def test_evaluate_point_granularity(self): + anomalies = ['step_uv'] + effects = [] + # series with 2880 data points series_generator = HumiTempTimeseriesGenerator() - series1 = series_generator.generate(anomalies=anomalies) - series2 = series_generator.generate(anomalies=anomalies) + series1 = series_generator.generate(anomalies=anomalies,effects=effects) + series2 = series_generator.generate(anomalies=anomalies,effects=effects) dataset = [series1,series2] evaluator = Evaluator(test_data=dataset) minmax1 = MinMaxAnomalyDetector() @@ -312,17 +314,18 @@ def test_evaluate(self): 'detector_2': minmax2, 'detector_3': minmax3 } - evaluation_results = evaluator.evaluate(models=models) + evaluation_results = evaluator.evaluate(models=models,granularity='data_point') # Evaluation_results: - # detector_1: {'step_uv': 1.0, 'spike_uv': 0.0, 'false_positives': 4} - # detector_2: {'step_uv': 1.0, 'spike_uv': 0.0, 'false_positives': 4} - # detector_3: {'step_uv': 1.0, 'spike_uv': 0.0, 'false_positives': 4} - + # detector_1: {'step_uv': 0.000694444444444444, 'false_positives': 0.000694444444444444} + # detector_2: {'step_uv': 0.000694444444444444, 'false_positives': 0.000694444444444444} + # detector_3: {'step_uv': 0.000694444444444444, 'false_positives': 0.000694444444444444} self.assertIsInstance(evaluation_results,dict) self.assertEqual(len(evaluation_results),3) - self.assertEqual(len(evaluation_results['detector_1']),3) - self.assertEqual(len(evaluation_results['detector_2']),3) - self.assertEqual(len(evaluation_results['detector_3']),3) + self.assertEqual(len(evaluation_results['detector_1']),2) + self.assertEqual(len(evaluation_results['detector_2']),2) + self.assertEqual(len(evaluation_results['detector_3']),2) + self.assertAlmostEqual(evaluation_results['detector_1']['step_uv'],0.000694444444444444) + self.assertAlmostEqual(evaluation_results['detector_1']['false_positives'],0.000694444444444444) def test_copy_dataset(self): series_generator = HumiTempTimeseriesGenerator() From 21a3c78f96c5136351c515c6d5eb6d07ffff1d00 Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Thu, 27 Nov 2025 23:12:31 +0100 Subject: [PATCH 08/10] Add test on evaluation with granularity= variable --- ats/tests/test_evaluators.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index b0bca7b..0c76349 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -327,6 +327,36 @@ def test_evaluate_point_granularity(self): self.assertAlmostEqual(evaluation_results['detector_1']['step_uv'],0.000694444444444444) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives'],0.000694444444444444) + def test_evaluate_variable_granularity(self): + anomalies = ['step_uv'] + effects = [] + # series with 2880 data points + series_generator = HumiTempTimeseriesGenerator() + series1 = series_generator.generate(anomalies=anomalies,effects=effects) + series2 = series_generator.generate(anomalies=anomalies,effects=effects) + dataset = [series1,series2] + evaluator = Evaluator(test_data=dataset) + minmax1 = MinMaxAnomalyDetector() + minmax2 = MinMaxAnomalyDetector() + minmax3 = MinMaxAnomalyDetector() + models={'detector_1': minmax1, + 'detector_2': minmax2, + 'detector_3': minmax3 + } + evaluation_results = evaluator.evaluate(models=models,granularity='variable') + # Evaluation_results: + # detector_1: {'step_uv': 0.000694444444444444, 'false_positives': 0.000694444444444444} + # detector_2: {'step_uv': 0.000694444444444444, 'false_positives': 0.000694444444444444} + # detector_3: {'step_uv': 0.000694444444444444, 'false_positives': 0.000694444444444444} + + self.assertIsInstance(evaluation_results,dict) + self.assertEqual(len(evaluation_results),3) + self.assertEqual(len(evaluation_results['detector_1']),2) + self.assertEqual(len(evaluation_results['detector_2']),2) + self.assertEqual(len(evaluation_results['detector_3']),2) + self.assertAlmostEqual(evaluation_results['detector_1']['step_uv'],0.000694444444444444) + self.assertAlmostEqual(evaluation_results['detector_1']['false_positives'],0.000694444444444444) + def test_copy_dataset(self): series_generator = HumiTempTimeseriesGenerator() series1 = series_generator.generate(effects=['noise']) From 098695a53eeaaaa0416d88fd3acadb5df8f2eedc Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Thu, 27 Nov 2025 23:26:34 +0100 Subject: [PATCH 09/10] Add test on "_series_granularity_evaluation()" --- ats/tests/test_evaluators.py | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index 0c76349..1f4054a 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -357,6 +357,49 @@ def test_evaluate_variable_granularity(self): self.assertAlmostEqual(evaluation_results['detector_1']['step_uv'],0.000694444444444444) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives'],0.000694444444444444) + def test_evaluate_series_granularity(self): + anomalies = ['step_uv'] + effects = [] + series_generator = HumiTempTimeseriesGenerator() + # series1 will be a true anomaly for the minmax + series1 = series_generator.generate(anomalies=anomalies,effects=effects) + # series2 will be a false positive for minmax (it sees always 2 anomalous data points for each variable) + series2 = series_generator.generate(anomalies=[],effects=effects) + dataset = [series1,series2] + evaluator = Evaluator(test_data=dataset) + minmax1 = MinMaxAnomalyDetector() + minmax2 = MinMaxAnomalyDetector() + minmax3 = MinMaxAnomalyDetector() + models={'detector_1': minmax1, + 'detector_2': minmax2, + 'detector_3': minmax3 + } + evaluation_results = evaluator.evaluate(models=models,granularity='series') + # Evaluation_results: + # detector_1: {'step_uv': 0.5, 'false_positives': 0.5} + # detector_2: {'step_uv': 0.5, 'false_positives': 0.5} + # detector_3: {'step_uv': 0.5, 'false_positives': 0.5} + + self.assertIsInstance(evaluation_results,dict) + self.assertEqual(len(evaluation_results),3) + self.assertEqual(len(evaluation_results['detector_1']),2) + self.assertEqual(len(evaluation_results['detector_2']),2) + self.assertEqual(len(evaluation_results['detector_3']),2) + self.assertAlmostEqual(evaluation_results['detector_1']['step_uv'],0.5) + self.assertAlmostEqual(evaluation_results['detector_1']['false_positives'],0.5) + + def test_raised_error_evaluation_series_granularity(self): + anomalies = ['step_uv','spike_uv'] + series_generator = HumiTempTimeseriesGenerator() + series = series_generator.generate(anomalies=anomalies) + dataset = [series] + minmax = MinMaxAnomalyDetector() + evaluator = Evaluator(test_data=dataset) + try: + evaluation_result = evaluator.evaluate(models={'detector':minmax},granularity='series') + except Exception as e: + self.assertIsInstance(e,ValueError) + def test_copy_dataset(self): series_generator = HumiTempTimeseriesGenerator() series1 = series_generator.generate(effects=['noise']) From 8c027f2d70a675de759af5eca9300dbbaded5925 Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Fri, 28 Nov 2025 12:04:15 +0100 Subject: [PATCH 10/10] Fix an error Now in the evaluation on a dataset with only anomalous series the false positives counts are set to zero --- ats/evaluators.py | 4 ++++ ats/tests/test_evaluators.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/ats/evaluators.py b/ats/evaluators.py index a013be7..56187b9 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -55,6 +55,8 @@ def _calculate_model_scores(single_model_evaluation={},granularity='data_point') anomaly_scores = {} for anomaly in dataset_anomalies: anomaly_scores[anomaly] = 0 + if 'false_positives' not in dataset_anomalies: + anomaly_scores['false_positives'] = 0.0 for anomaly in dataset_anomalies: for sample in single_model_evaluation.keys(): @@ -186,5 +188,7 @@ def _series_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df): one_series_evaluation_result['false_positives'] = 1 elif is_series_anomalous and anomalies: one_series_evaluation_result[anomalies[0]] = 1 + elif not is_series_anomalous and anomalies: + one_series_evaluation_result[anomalies[0]] = 0 return one_series_evaluation_result \ No newline at end of file diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index 1f4054a..cba8342 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -388,6 +388,36 @@ def test_evaluate_series_granularity(self): self.assertAlmostEqual(evaluation_results['detector_1']['step_uv'],0.5) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives'],0.5) + def test_series_granularity_eval_with_non_detected_anomalies(self): + effects = [] + series_generator = HumiTempTimeseriesGenerator() + # series1 will be a true anomaly for the minmax + series1 = series_generator.generate(anomalies=['step_uv'],effects=effects) + # series2 will be a false positive for minmax (it sees always 2 anomalous data points for each variable) + series2 = series_generator.generate(anomalies=['pattern_uv'],effects=effects) + dataset = [series1,series2] + evaluator = Evaluator(test_data=dataset) + minmax1 = MinMaxAnomalyDetector() + minmax2 = MinMaxAnomalyDetector() + minmax3 = MinMaxAnomalyDetector() + models={'detector_1': minmax1, + 'detector_2': minmax2, + 'detector_3': minmax3 + } + evaluation_results = evaluator.evaluate(models=models,granularity='series') + # Evaluation_results: + # detector_1: {'step_uv': 0.5, 'pattern_uv': 0.5, 'false_positives': 0.0} + # detector_2: {'step_uv': 0.5, 'pattern_uv': 0.5, 'false_positives': 0.0} + # detector_3: {'step_uv': 0.5, 'pattern_uv': 0.5, 'false_positives': 0.0} + self.assertIsInstance(evaluation_results,dict) + self.assertEqual(len(evaluation_results),3) + self.assertEqual(len(evaluation_results['detector_1']),3) + self.assertEqual(len(evaluation_results['detector_2']),3) + self.assertEqual(len(evaluation_results['detector_3']),3) + self.assertAlmostEqual(evaluation_results['detector_1']['step_uv'],0.5) + self.assertAlmostEqual(evaluation_results['detector_1']['pattern_uv'],0.5) + self.assertAlmostEqual(evaluation_results['detector_1']['false_positives'],0.0) + def test_raised_error_evaluation_series_granularity(self): anomalies = ['step_uv','spike_uv'] series_generator = HumiTempTimeseriesGenerator()