From 105e54f28ad19fbafe010d6597de3dd419915fea Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Sun, 7 Dec 2025 12:05:42 +0100 Subject: [PATCH 1/4] Change the key 'anomalies_count' of the evaluation result in 'true_positives_count' --- ats/evaluators.py | 16 +++++------ ats/tests/test_evaluators.py | 52 ++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/ats/evaluators.py b/ats/evaluators.py index 3131576..20276c9 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -54,14 +54,14 @@ def _calculate_model_scores(single_model_evaluation={}): false_positives_ratio = 0 anomalous_series_n = 0 for sample in single_model_evaluation.keys(): - anomalies_count += single_model_evaluation[sample]['anomalies_count'] + anomalies_count += single_model_evaluation[sample]['true_positives_count'] if single_model_evaluation[sample]['anomalies_ratio'] is not None: anomalies_ratio += single_model_evaluation[sample]['anomalies_ratio'] anomalous_series_n += 1 false_positives_count += single_model_evaluation[sample]['false_positives_count'] false_positives_ratio += single_model_evaluation[sample]['false_positives_ratio'] - model_scores['anomalies_count'] = anomalies_count + model_scores['true_positives_count'] = anomalies_count if anomalous_series_n: model_scores['anomalies_ratio'] = anomalies_ratio/anomalous_series_n else: @@ -72,8 +72,8 @@ def _calculate_model_scores(single_model_evaluation={}): def _get_breakdown_info(single_model_evaluation={}): for sample in single_model_evaluation.keys(): - if 'anomalies_count' in single_model_evaluation[sample].keys(): - del single_model_evaluation[sample]['anomalies_count'] + if 'true_positives_count' in single_model_evaluation[sample].keys(): + del single_model_evaluation[sample]['true_positives_count'] if 'anomalies_ratio' in single_model_evaluation[sample].keys(): del single_model_evaluation[sample]['anomalies_ratio'] if 'false_positives_count' in single_model_evaluation[sample].keys(): @@ -202,7 +202,7 @@ def _variable_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,bre total_inserted_anomalies_n *= variables_n one_series_evaluation_result['false_positives_count'] = false_positives_count one_series_evaluation_result['false_positives_ratio'] = false_positives_count/normalization_factor - one_series_evaluation_result['anomalies_count'] = total_detected_anomalies_n + one_series_evaluation_result['true_positives_count'] = total_detected_anomalies_n if total_inserted_anomalies_n: one_series_evaluation_result['anomalies_ratio'] = total_detected_anomalies_n/total_inserted_anomalies_n else: @@ -240,7 +240,7 @@ def _point_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,breakd one_series_evaluation_result['false_positives_count'] = false_positives_count one_series_evaluation_result['false_positives_ratio'] = false_positives_count/normalization_factor - one_series_evaluation_result['anomalies_count'] = total_detected_anomalies_n + one_series_evaluation_result['true_positives_count'] = total_detected_anomalies_n if total_inserted_anomalies_n: one_series_evaluation_result['anomalies_ratio'] = total_detected_anomalies_n/total_inserted_anomalies_n else: @@ -272,8 +272,8 @@ def _series_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,break break one_series_evaluation_result['false_positives_count'] = 1 if is_series_anomalous and not anomalies else 0 one_series_evaluation_result['false_positives_ratio'] = one_series_evaluation_result['false_positives_count'] - one_series_evaluation_result['anomalies_count'] = 1 if is_series_anomalous and anomalies else 0 - one_series_evaluation_result['anomalies_ratio'] = one_series_evaluation_result['anomalies_count'] if anomalies else None + one_series_evaluation_result['true_positives_count'] = 1 if is_series_anomalous and anomalies else 0 + one_series_evaluation_result['anomalies_ratio'] = one_series_evaluation_result['true_positives_count'] if anomalies else None if breakdown: return one_series_evaluation_result | breakdown_info diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index bf859ac..fcd3cea 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -264,24 +264,24 @@ def test_get_model_output(self): self.assertIn('humidity_anomaly',list(flagged_dataset[1].columns)) def test_calculate_model_scores(self): - single_model_evaluation = { 'sample_1': {'anomalies_count': 3, 'anomalies_ratio': 1.5, + single_model_evaluation = { 'sample_1': {'true_positives_count': 3, 'anomalies_ratio': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14}, - 'sample_2': {'anomalies_count': 3, 'anomalies_ratio': 1.5, + 'sample_2': {'true_positives_count': 3, 'anomalies_ratio': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14}, - 'sample_3': {'anomalies_count': 3, 'anomalies_ratio': 1.5, + 'sample_3': {'true_positives_count': 3, 'anomalies_ratio': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14} } model_scores = _calculate_model_scores(single_model_evaluation) self.assertIsInstance(model_scores,dict) - self.assertIn('anomalies_count',model_scores.keys()) + self.assertIn('true_positives_count',model_scores.keys()) self.assertIn('anomalies_ratio',model_scores.keys()) self.assertIn('false_positives_count',model_scores.keys()) self.assertIn('false_positives_ratio',model_scores.keys()) - self.assertAlmostEqual(model_scores['anomalies_count'],9) + self.assertAlmostEqual(model_scores['true_positives_count'],9) self.assertAlmostEqual(model_scores['anomalies_ratio'],1.5) self.assertAlmostEqual(model_scores['false_positives_count'],3) self.assertAlmostEqual(model_scores['false_positives_ratio'],0.14) @@ -297,7 +297,7 @@ def test_evaluate_point_granularity(self): } evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='point') - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],6) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],6) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],7/8) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],4) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],8/21) @@ -313,7 +313,7 @@ def test_evaluate_variable_granularity(self): } evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='variable') - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],7) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],7) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],25/48) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],5) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],31/126) @@ -329,7 +329,7 @@ def test_evaluate_series_granularity(self): } evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='series') - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],2) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],2) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],2/2) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],1/3) @@ -358,12 +358,12 @@ def test_variable_granularity_evaluation(self): evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='variable') self.assertIn('detector_1',evaluation_results.keys()) - self.assertIn('anomalies_count',evaluation_results['detector_1'].keys()) + self.assertIn('true_positives_count',evaluation_results['detector_1'].keys()) self.assertIn('anomalies_ratio',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_count',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_ratio',evaluation_results['detector_1'].keys()) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],4) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],4) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],4/6) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],0) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) @@ -371,7 +371,7 @@ def test_variable_granularity_evaluation(self): dataset1 = [self.series2] evaluator1 = Evaluator(test_data=dataset1) evaluation_results = evaluator1.evaluate(models=models,granularity='variable') - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],3) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],3) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],3/8) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],1/(7*2)) @@ -382,7 +382,7 @@ def test_variable_granularity_evaluation_with_breakdown(self): flagged_series = _get_model_output([formatted_series],minmax1) evaluation_results = _variable_granularity_evaluation(flagged_series[0],anomaly_labels,breakdown=True) - self.assertIn('anomalies_count',evaluation_results.keys()) + self.assertIn('true_positives_count',evaluation_results.keys()) self.assertIn('anomalies_ratio',evaluation_results.keys()) self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) @@ -414,12 +414,12 @@ def test_point_granularity_evaluation(self): evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='point') self.assertIn('detector_1',evaluation_results.keys()) - self.assertIn('anomalies_count',evaluation_results['detector_1'].keys()) + self.assertIn('true_positives_count',evaluation_results['detector_1'].keys()) self.assertIn('anomalies_ratio',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_count',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_ratio',evaluation_results['detector_1'].keys()) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],3) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],3) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],3/3) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],0) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) @@ -427,7 +427,7 @@ def test_point_granularity_evaluation(self): dataset1 = [self.series2] evaluator1 = Evaluator(test_data=dataset1) evaluation_results = evaluator1.evaluate(models=models,granularity='point') - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],3) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],3) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],3/4) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],1/7) @@ -438,7 +438,7 @@ def test_point_granularity_evaluation_with_breakdown(self): flagged_series = _get_model_output([formatted_series],minmax1) evaluation_results = _point_granularity_evaluation(flagged_series[0],anomaly_labels,breakdown=True) - self.assertIn('anomalies_count',evaluation_results.keys()) + self.assertIn('true_positives_count',evaluation_results.keys()) self.assertIn('anomalies_ratio',evaluation_results.keys()) self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) @@ -461,12 +461,12 @@ def test_series_granularity_evaluation(self): evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='series') self.assertIn('detector_1',evaluation_results.keys()) - self.assertIn('anomalies_count',evaluation_results['detector_1'].keys()) + self.assertIn('true_positives_count',evaluation_results['detector_1'].keys()) self.assertIn('anomalies_ratio',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_count',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_ratio',evaluation_results['detector_1'].keys()) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],1) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],0) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) @@ -474,7 +474,7 @@ def test_series_granularity_evaluation(self): dataset1 = [self.series3] evaluator1 = Evaluator(test_data=dataset1) evaluation_results = evaluator1.evaluate(models=models,granularity='series') - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],0) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],0) self.assertIsNone(evaluation_results['detector_1']['anomalies_ratio']) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],1) @@ -487,7 +487,7 @@ def test_series_granularity_evaluation_with_breakdown(self): flagged_series = _get_model_output([formatted_series],minmax1) evaluation_results = _series_granularity_evaluation(flagged_series[0],anomaly_labels,breakdown=True) - self.assertIn('anomalies_count',evaluation_results.keys()) + self.assertIn('true_positives_count',evaluation_results.keys()) self.assertIn('anomalies_ratio',evaluation_results.keys()) self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) @@ -504,12 +504,12 @@ def test_series_granularity_evaluation_with_breakdown(self): self.assertIsInstance(e,ValueError) def test_get_breakdown_info(self): - single_model_evaluation = { 'sample_1': {'anomalies_count': 3, 'anomalies_ratio': 1.5, + single_model_evaluation = { 'sample_1': {'true_positives_count': 3, 'anomalies_ratio': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, 'spike_anomaly_count': 1, 'spike_anomaly_ratio': 0.5}, - 'sample_2': {'anomalies_count': 3, 'anomalies_ratio': 1.5, + 'sample_2': {'true_positives_count': 3, 'anomalies_ratio': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, 'spike_anomaly_count': 1, @@ -517,7 +517,7 @@ def test_get_breakdown_info(self): 'step_anomaly_count': 2, 'step_anomaly_ratio': 2/3 }, - 'sample_3': {'anomalies_count': 3, 'anomalies_ratio': 1.5, + 'sample_3': {'true_positives_count': 3, 'anomalies_ratio': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, 'step_anomaly_count': 3, @@ -552,7 +552,7 @@ def test_variable_granularity_eval_with_breakdown(self): } evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='variable',breakdown=True) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],7) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],7) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],25/48) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],5) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],31/126) @@ -573,7 +573,7 @@ def test_point_granularity_eval_with_breakdown(self): } evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='point',breakdown=True) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],6) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],6) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],7/8) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],4) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],8/21) @@ -600,7 +600,7 @@ def test_series_granularity_eval_with_breakdown(self): } evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='series',breakdown=True) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_count'],3) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],3) self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],0) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) From beec504be6e3a80493e3c245119ab704fa0be9ef Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Sun, 7 Dec 2025 12:08:32 +0100 Subject: [PATCH 2/4] Change the key 'anomalies_ratio' of the evaluation result in 'true_positives_rate' --- ats/evaluators.py | 22 +++++++-------- ats/tests/test_evaluators.py | 52 ++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/ats/evaluators.py b/ats/evaluators.py index 20276c9..875396e 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -55,17 +55,17 @@ def _calculate_model_scores(single_model_evaluation={}): anomalous_series_n = 0 for sample in single_model_evaluation.keys(): anomalies_count += single_model_evaluation[sample]['true_positives_count'] - if single_model_evaluation[sample]['anomalies_ratio'] is not None: - anomalies_ratio += single_model_evaluation[sample]['anomalies_ratio'] + if single_model_evaluation[sample]['true_positives_rate'] is not None: + anomalies_ratio += single_model_evaluation[sample]['true_positives_rate'] anomalous_series_n += 1 false_positives_count += single_model_evaluation[sample]['false_positives_count'] false_positives_ratio += single_model_evaluation[sample]['false_positives_ratio'] model_scores['true_positives_count'] = anomalies_count if anomalous_series_n: - model_scores['anomalies_ratio'] = anomalies_ratio/anomalous_series_n + model_scores['true_positives_rate'] = anomalies_ratio/anomalous_series_n else: - model_scores['anomalies_ratio'] = None + model_scores['true_positives_rate'] = None model_scores['false_positives_count'] = false_positives_count model_scores['false_positives_ratio'] = false_positives_ratio/len(single_model_evaluation) return model_scores @@ -74,8 +74,8 @@ def _get_breakdown_info(single_model_evaluation={}): for sample in single_model_evaluation.keys(): if 'true_positives_count' in single_model_evaluation[sample].keys(): del single_model_evaluation[sample]['true_positives_count'] - if 'anomalies_ratio' in single_model_evaluation[sample].keys(): - del single_model_evaluation[sample]['anomalies_ratio'] + if 'true_positives_rate' in single_model_evaluation[sample].keys(): + del single_model_evaluation[sample]['true_positives_rate'] if 'false_positives_count' in single_model_evaluation[sample].keys(): del single_model_evaluation[sample]['false_positives_count'] if 'false_positives_ratio' in single_model_evaluation[sample].keys(): @@ -204,9 +204,9 @@ def _variable_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,bre one_series_evaluation_result['false_positives_ratio'] = false_positives_count/normalization_factor one_series_evaluation_result['true_positives_count'] = total_detected_anomalies_n if total_inserted_anomalies_n: - one_series_evaluation_result['anomalies_ratio'] = total_detected_anomalies_n/total_inserted_anomalies_n + one_series_evaluation_result['true_positives_rate'] = total_detected_anomalies_n/total_inserted_anomalies_n else: - one_series_evaluation_result['anomalies_ratio'] = None + one_series_evaluation_result['true_positives_rate'] = None if breakdown: return one_series_evaluation_result | breakdown_info else: @@ -242,9 +242,9 @@ def _point_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,breakd one_series_evaluation_result['false_positives_ratio'] = false_positives_count/normalization_factor one_series_evaluation_result['true_positives_count'] = total_detected_anomalies_n if total_inserted_anomalies_n: - one_series_evaluation_result['anomalies_ratio'] = total_detected_anomalies_n/total_inserted_anomalies_n + one_series_evaluation_result['true_positives_rate'] = total_detected_anomalies_n/total_inserted_anomalies_n else: - one_series_evaluation_result['anomalies_ratio'] = None + one_series_evaluation_result['true_positives_rate'] = None if breakdown: return one_series_evaluation_result | breakdown_info else: @@ -273,7 +273,7 @@ def _series_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,break one_series_evaluation_result['false_positives_count'] = 1 if is_series_anomalous and not anomalies else 0 one_series_evaluation_result['false_positives_ratio'] = one_series_evaluation_result['false_positives_count'] one_series_evaluation_result['true_positives_count'] = 1 if is_series_anomalous and anomalies else 0 - one_series_evaluation_result['anomalies_ratio'] = one_series_evaluation_result['true_positives_count'] if anomalies else None + one_series_evaluation_result['true_positives_rate'] = one_series_evaluation_result['true_positives_count'] if anomalies else None if breakdown: return one_series_evaluation_result | breakdown_info diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index fcd3cea..cd8251c 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -264,25 +264,25 @@ def test_get_model_output(self): self.assertIn('humidity_anomaly',list(flagged_dataset[1].columns)) def test_calculate_model_scores(self): - single_model_evaluation = { 'sample_1': {'true_positives_count': 3, 'anomalies_ratio': 1.5, + single_model_evaluation = { 'sample_1': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14}, - 'sample_2': {'true_positives_count': 3, 'anomalies_ratio': 1.5, + 'sample_2': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14}, - 'sample_3': {'true_positives_count': 3, 'anomalies_ratio': 1.5, + 'sample_3': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14} } model_scores = _calculate_model_scores(single_model_evaluation) self.assertIsInstance(model_scores,dict) self.assertIn('true_positives_count',model_scores.keys()) - self.assertIn('anomalies_ratio',model_scores.keys()) + self.assertIn('true_positives_rate',model_scores.keys()) self.assertIn('false_positives_count',model_scores.keys()) self.assertIn('false_positives_ratio',model_scores.keys()) self.assertAlmostEqual(model_scores['true_positives_count'],9) - self.assertAlmostEqual(model_scores['anomalies_ratio'],1.5) + self.assertAlmostEqual(model_scores['true_positives_rate'],1.5) self.assertAlmostEqual(model_scores['false_positives_count'],3) self.assertAlmostEqual(model_scores['false_positives_ratio'],0.14) @@ -298,7 +298,7 @@ def test_evaluate_point_granularity(self): evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='point') self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],6) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],7/8) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],7/8) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],4) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],8/21) @@ -314,7 +314,7 @@ def test_evaluate_variable_granularity(self): evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='variable') self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],7) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],25/48) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],25/48) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],5) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],31/126) @@ -330,7 +330,7 @@ def test_evaluate_series_granularity(self): evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='series') self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],2) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],2/2) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],2/2) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],1/3) @@ -359,12 +359,12 @@ def test_variable_granularity_evaluation(self): evaluation_results = evaluator.evaluate(models=models,granularity='variable') self.assertIn('detector_1',evaluation_results.keys()) self.assertIn('true_positives_count',evaluation_results['detector_1'].keys()) - self.assertIn('anomalies_ratio',evaluation_results['detector_1'].keys()) + self.assertIn('true_positives_rate',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_count',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_ratio',evaluation_results['detector_1'].keys()) self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],4) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],4/6) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],4/6) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],0) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) @@ -372,7 +372,7 @@ def test_variable_granularity_evaluation(self): evaluator1 = Evaluator(test_data=dataset1) evaluation_results = evaluator1.evaluate(models=models,granularity='variable') self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],3) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],3/8) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],3/8) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],1/(7*2)) @@ -383,7 +383,7 @@ def test_variable_granularity_evaluation_with_breakdown(self): evaluation_results = _variable_granularity_evaluation(flagged_series[0],anomaly_labels,breakdown=True) self.assertIn('true_positives_count',evaluation_results.keys()) - self.assertIn('anomalies_ratio',evaluation_results.keys()) + self.assertIn('true_positives_rate',evaluation_results.keys()) self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) @@ -415,12 +415,12 @@ def test_point_granularity_evaluation(self): evaluation_results = evaluator.evaluate(models=models,granularity='point') self.assertIn('detector_1',evaluation_results.keys()) self.assertIn('true_positives_count',evaluation_results['detector_1'].keys()) - self.assertIn('anomalies_ratio',evaluation_results['detector_1'].keys()) + self.assertIn('true_positives_rate',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_count',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_ratio',evaluation_results['detector_1'].keys()) self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],3) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],3/3) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],3/3) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],0) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) @@ -428,7 +428,7 @@ def test_point_granularity_evaluation(self): evaluator1 = Evaluator(test_data=dataset1) evaluation_results = evaluator1.evaluate(models=models,granularity='point') self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],3) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],3/4) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],3/4) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],1/7) @@ -439,7 +439,7 @@ def test_point_granularity_evaluation_with_breakdown(self): evaluation_results = _point_granularity_evaluation(flagged_series[0],anomaly_labels,breakdown=True) self.assertIn('true_positives_count',evaluation_results.keys()) - self.assertIn('anomalies_ratio',evaluation_results.keys()) + self.assertIn('true_positives_rate',evaluation_results.keys()) self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) @@ -462,12 +462,12 @@ def test_series_granularity_evaluation(self): evaluation_results = evaluator.evaluate(models=models,granularity='series') self.assertIn('detector_1',evaluation_results.keys()) self.assertIn('true_positives_count',evaluation_results['detector_1'].keys()) - self.assertIn('anomalies_ratio',evaluation_results['detector_1'].keys()) + self.assertIn('true_positives_rate',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_count',evaluation_results['detector_1'].keys()) self.assertIn('false_positives_ratio',evaluation_results['detector_1'].keys()) self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],1) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],1) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],0) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) @@ -475,7 +475,7 @@ def test_series_granularity_evaluation(self): evaluator1 = Evaluator(test_data=dataset1) evaluation_results = evaluator1.evaluate(models=models,granularity='series') self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],0) - self.assertIsNone(evaluation_results['detector_1']['anomalies_ratio']) + self.assertIsNone(evaluation_results['detector_1']['true_positives_rate']) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],1) @@ -488,7 +488,7 @@ def test_series_granularity_evaluation_with_breakdown(self): evaluation_results = _series_granularity_evaluation(flagged_series[0],anomaly_labels,breakdown=True) self.assertIn('true_positives_count',evaluation_results.keys()) - self.assertIn('anomalies_ratio',evaluation_results.keys()) + self.assertIn('true_positives_rate',evaluation_results.keys()) self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) self.assertIn('anomaly_1_anomaly_count',evaluation_results.keys()) @@ -504,12 +504,12 @@ def test_series_granularity_evaluation_with_breakdown(self): self.assertIsInstance(e,ValueError) def test_get_breakdown_info(self): - single_model_evaluation = { 'sample_1': {'true_positives_count': 3, 'anomalies_ratio': 1.5, + single_model_evaluation = { 'sample_1': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, 'spike_anomaly_count': 1, 'spike_anomaly_ratio': 0.5}, - 'sample_2': {'true_positives_count': 3, 'anomalies_ratio': 1.5, + 'sample_2': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, 'spike_anomaly_count': 1, @@ -517,7 +517,7 @@ def test_get_breakdown_info(self): 'step_anomaly_count': 2, 'step_anomaly_ratio': 2/3 }, - 'sample_3': {'true_positives_count': 3, 'anomalies_ratio': 1.5, + 'sample_3': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, 'step_anomaly_count': 3, @@ -553,7 +553,7 @@ def test_variable_granularity_eval_with_breakdown(self): evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='variable',breakdown=True) self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],7) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],25/48) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],25/48) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],5) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],31/126) @@ -574,7 +574,7 @@ def test_point_granularity_eval_with_breakdown(self): evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='point',breakdown=True) self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],6) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],7/8) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],7/8) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],4) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],8/21) @@ -601,7 +601,7 @@ def test_series_granularity_eval_with_breakdown(self): evaluator = Evaluator(test_data=dataset) evaluation_results = evaluator.evaluate(models=models,granularity='series',breakdown=True) self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_count'],3) - self.assertAlmostEqual(evaluation_results['detector_1']['anomalies_ratio'],1) + self.assertAlmostEqual(evaluation_results['detector_1']['true_positives_rate'],1) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],0) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) From 48047200149edff49db98d38614b2f9fc9207ef5 Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Sun, 7 Dec 2025 12:27:33 +0100 Subject: [PATCH 3/4] Change the key '_anomalies_ratio' of the breakdown info in '_true_positives_rate' --- ats/evaluators.py | 8 ++--- ats/tests/test_evaluators.py | 58 ++++++++++++++++++------------------ 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/ats/evaluators.py b/ats/evaluators.py index 875396e..3e1b02f 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -94,7 +94,7 @@ def _get_breakdown_info(single_model_evaluation={}): breakdown_info[key] = sample_evaluation[key] for key in breakdown_info.keys(): - if '_ratio' in key: + if '_rate' in key: breakdown_info[key] /= anomaly_series_count_by_type[key] return breakdown_info @@ -197,7 +197,7 @@ def _variable_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,bre if anomaly is not None: total_detected_anomalies_n += anomaly_count breakdown_info[anomaly + '_anomaly' + '_count'] = anomaly_count - breakdown_info[anomaly + '_anomaly' + '_ratio'] = anomaly_count/(frequency * variables_n) + breakdown_info[anomaly + '_true_positives_rate'] = anomaly_count/(frequency * variables_n) total_inserted_anomalies_n *= variables_n one_series_evaluation_result['false_positives_count'] = false_positives_count @@ -236,7 +236,7 @@ def _point_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,breakd if anomaly is not None: total_detected_anomalies_n += anomaly_count breakdown_info[anomaly + '_anomaly_count'] = anomaly_count - breakdown_info[anomaly + '_anomaly_ratio'] = anomaly_count/frequency + breakdown_info[anomaly + '_true_positives_rate'] = anomaly_count/frequency one_series_evaluation_result['false_positives_count'] = false_positives_count one_series_evaluation_result['false_positives_ratio'] = false_positives_count/normalization_factor @@ -268,7 +268,7 @@ def _series_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,break if anomalies: inserted_anomaly = anomalies[0] breakdown_info[inserted_anomaly + '_anomaly_count'] = 1 - breakdown_info[inserted_anomaly + '_anomaly_ratio'] = 1 + breakdown_info[inserted_anomaly + '_true_positives_rate'] = 1 break one_series_evaluation_result['false_positives_count'] = 1 if is_series_anomalous and not anomalies else 0 one_series_evaluation_result['false_positives_ratio'] = one_series_evaluation_result['false_positives_count'] diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index cd8251c..7bab664 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -388,23 +388,23 @@ def test_variable_granularity_evaluation_with_breakdown(self): self.assertIn('false_positives_ratio',evaluation_results.keys()) self.assertIn('anomaly_1_anomaly_count',evaluation_results.keys()) - self.assertIn('anomaly_1_anomaly_ratio',evaluation_results.keys()) + self.assertIn('anomaly_1_true_positives_rate',evaluation_results.keys()) self.assertIn('anomaly_2_anomaly_count',evaluation_results.keys()) - self.assertIn('anomaly_2_anomaly_ratio',evaluation_results.keys()) + self.assertIn('anomaly_2_true_positives_rate',evaluation_results.keys()) self.assertAlmostEqual(evaluation_results['anomaly_1_anomaly_count'],3) - self.assertAlmostEqual(evaluation_results['anomaly_1_anomaly_ratio'],3/4) + self.assertAlmostEqual(evaluation_results['anomaly_1_true_positives_rate'],3/4) self.assertAlmostEqual(evaluation_results['anomaly_2_anomaly_count'],1) - self.assertAlmostEqual(evaluation_results['anomaly_2_anomaly_ratio'],1/2) + self.assertAlmostEqual(evaluation_results['anomaly_2_true_positives_rate'],1/2) formatted_series1,anomaly_labels1 = _format_for_anomaly_detector(self.series3) flagged_series1 = _get_model_output([formatted_series1],minmax1) evaluation_results1 = _variable_granularity_evaluation(flagged_series1[0],anomaly_labels1,breakdown=True) self.assertNotIn('anomaly_1_anomaly_count',evaluation_results1.keys()) - self.assertNotIn('anomaly_1_anomaly_ratio',evaluation_results1.keys()) + self.assertNotIn('anomaly_1_true_positives_rate',evaluation_results1.keys()) self.assertNotIn('anomaly_2_anomaly_count',evaluation_results1.keys()) - self.assertNotIn('anomaly_2_anomaly_ratio',evaluation_results1.keys()) + self.assertNotIn('anomaly_2_true_positives_rate',evaluation_results1.keys()) def test_point_granularity_evaluation(self): dataset = [self.series1] @@ -444,14 +444,14 @@ def test_point_granularity_evaluation_with_breakdown(self): self.assertIn('false_positives_ratio',evaluation_results.keys()) self.assertIn('anomaly_1_anomaly_count',evaluation_results.keys()) - self.assertIn('anomaly_1_anomaly_ratio',evaluation_results.keys()) + self.assertIn('anomaly_1_true_positives_rate',evaluation_results.keys()) self.assertIn('anomaly_2_anomaly_count',evaluation_results.keys()) - self.assertIn('anomaly_2_anomaly_ratio',evaluation_results.keys()) + self.assertIn('anomaly_2_true_positives_rate',evaluation_results.keys()) self.assertAlmostEqual(evaluation_results['anomaly_1_anomaly_count'],2) - self.assertAlmostEqual(evaluation_results['anomaly_1_anomaly_ratio'],2/2) + self.assertAlmostEqual(evaluation_results['anomaly_1_true_positives_rate'],2/2) self.assertAlmostEqual(evaluation_results['anomaly_2_anomaly_count'],1) - self.assertAlmostEqual(evaluation_results['anomaly_2_anomaly_ratio'],1/1) + self.assertAlmostEqual(evaluation_results['anomaly_2_true_positives_rate'],1/1) def test_series_granularity_evaluation(self): dataset = [self.series1] @@ -492,9 +492,9 @@ def test_series_granularity_evaluation_with_breakdown(self): self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) self.assertIn('anomaly_1_anomaly_count',evaluation_results.keys()) - self.assertIn('anomaly_1_anomaly_ratio',evaluation_results.keys()) + self.assertIn('anomaly_1_true_positives_rate',evaluation_results.keys()) self.assertAlmostEqual(evaluation_results['anomaly_1_anomaly_count'],1) - self.assertAlmostEqual(evaluation_results['anomaly_1_anomaly_ratio'],1) + self.assertAlmostEqual(evaluation_results['anomaly_1_true_positives_rate'],1) formatted_series1,anomaly_labels1 = _format_for_anomaly_detector(self.series1) flagged_series1 = _get_model_output([formatted_series1],minmax1) @@ -508,38 +508,38 @@ def test_get_breakdown_info(self): 'false_positives_count': 1, 'false_positives_ratio': 0.14, 'spike_anomaly_count': 1, - 'spike_anomaly_ratio': 0.5}, + 'spike_true_positives_rate': 0.5}, 'sample_2': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, 'spike_anomaly_count': 1, - 'spike_anomaly_ratio': 0.5, + 'spike_true_positives_rate': 0.5, 'step_anomaly_count': 2, - 'step_anomaly_ratio': 2/3 + 'step_true_positives_rate': 2/3 }, 'sample_3': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, 'step_anomaly_count': 3, - 'step_anomaly_ratio': 1, + 'step_true_positives_rate': 1, 'pattern_anomaly_count': 2, - 'pattern_anomaly_ratio': 0.5 + 'pattern_true_positives_rate': 0.5 } } breakdown = _get_breakdown_info(single_model_evaluation) self.assertIn('spike_anomaly_count',breakdown.keys()) - self.assertIn('spike_anomaly_ratio',breakdown.keys()) + self.assertIn('spike_true_positives_rate',breakdown.keys()) self.assertIn('step_anomaly_count',breakdown.keys()) - self.assertIn('step_anomaly_ratio',breakdown.keys()) + self.assertIn('step_true_positives_rate',breakdown.keys()) self.assertIn('pattern_anomaly_count',breakdown.keys()) - self.assertIn('pattern_anomaly_ratio',breakdown.keys()) + self.assertIn('pattern_true_positives_rate',breakdown.keys()) self.assertAlmostEqual(breakdown['spike_anomaly_count'],2) - self.assertAlmostEqual(breakdown['spike_anomaly_ratio'],1/2) + self.assertAlmostEqual(breakdown['spike_true_positives_rate'],1/2) self.assertAlmostEqual(breakdown['step_anomaly_count'],5) - self.assertAlmostEqual(breakdown['step_anomaly_ratio'],5/6) + self.assertAlmostEqual(breakdown['step_true_positives_rate'],5/6) self.assertAlmostEqual(breakdown['pattern_anomaly_count'],2) - self.assertAlmostEqual(breakdown['pattern_anomaly_ratio'],0.5) + self.assertAlmostEqual(breakdown['pattern_true_positives_rate'],0.5) def test_variable_granularity_eval_with_breakdown(self): dataset = [self.series1, self.series2, self.series3] @@ -558,9 +558,9 @@ def test_variable_granularity_eval_with_breakdown(self): self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],31/126) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_anomaly_count'],5) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_anomaly_ratio'],13/24) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_true_positives_rate'],13/24) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_anomaly_count'],2) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_anomaly_ratio'],1/2) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_true_positives_rate'],1/2) def test_point_granularity_eval_with_breakdown(self): dataset = [self.series1, self.series2, self.series3] @@ -579,9 +579,9 @@ def test_point_granularity_eval_with_breakdown(self): self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],8/21) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_anomaly_count'],4) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_anomaly_ratio'],5/6) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_true_positives_rate'],5/6) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_anomaly_count'],2) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_anomaly_ratio'],1) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_true_positives_rate'],1) def test_series_granularity_eval_with_breakdown(self): series_1 = generate_timeseries_df(entries=3, variables=2) @@ -606,9 +606,9 @@ def test_series_granularity_eval_with_breakdown(self): self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_anomaly_count'],2) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_anomaly_ratio'],1) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_true_positives_rate'],1) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_anomaly_count'],1) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_anomaly_ratio'],1) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_true_positives_rate'],1) try: dataset = [self.series1, self.series2, self.series3] From 06bc317965584684bc92202fcb5c95ed747acd8d Mon Sep 17 00:00:00 2001 From: Agata Benvegna Date: Sun, 7 Dec 2025 12:30:30 +0100 Subject: [PATCH 4/4] Change the key '_anomalies_count' of the breakdown info in '_true_positives_count' --- ats/evaluators.py | 6 ++-- ats/tests/test_evaluators.py | 58 ++++++++++++++++++------------------ 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/ats/evaluators.py b/ats/evaluators.py index 3e1b02f..79a4846 100644 --- a/ats/evaluators.py +++ b/ats/evaluators.py @@ -196,7 +196,7 @@ def _variable_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,bre false_positives_count += flagged_timeseries_df.loc[timestamp,column] if anomaly is not None: total_detected_anomalies_n += anomaly_count - breakdown_info[anomaly + '_anomaly' + '_count'] = anomaly_count + breakdown_info[anomaly + '_true_positives_count'] = anomaly_count breakdown_info[anomaly + '_true_positives_rate'] = anomaly_count/(frequency * variables_n) total_inserted_anomalies_n *= variables_n @@ -235,7 +235,7 @@ def _point_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,breakd break if anomaly is not None: total_detected_anomalies_n += anomaly_count - breakdown_info[anomaly + '_anomaly_count'] = anomaly_count + breakdown_info[anomaly + '_true_positives_count'] = anomaly_count breakdown_info[anomaly + '_true_positives_rate'] = anomaly_count/frequency one_series_evaluation_result['false_positives_count'] = false_positives_count @@ -267,7 +267,7 @@ def _series_granularity_evaluation(flagged_timeseries_df,anomaly_labels_df,break is_series_anomalous = 1 if anomalies: inserted_anomaly = anomalies[0] - breakdown_info[inserted_anomaly + '_anomaly_count'] = 1 + breakdown_info[inserted_anomaly + '_true_positives_count'] = 1 breakdown_info[inserted_anomaly + '_true_positives_rate'] = 1 break one_series_evaluation_result['false_positives_count'] = 1 if is_series_anomalous and not anomalies else 0 diff --git a/ats/tests/test_evaluators.py b/ats/tests/test_evaluators.py index 7bab664..17480f5 100644 --- a/ats/tests/test_evaluators.py +++ b/ats/tests/test_evaluators.py @@ -387,23 +387,23 @@ def test_variable_granularity_evaluation_with_breakdown(self): self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) - self.assertIn('anomaly_1_anomaly_count',evaluation_results.keys()) + self.assertIn('anomaly_1_true_positives_count',evaluation_results.keys()) self.assertIn('anomaly_1_true_positives_rate',evaluation_results.keys()) - self.assertIn('anomaly_2_anomaly_count',evaluation_results.keys()) + self.assertIn('anomaly_2_true_positives_count',evaluation_results.keys()) self.assertIn('anomaly_2_true_positives_rate',evaluation_results.keys()) - self.assertAlmostEqual(evaluation_results['anomaly_1_anomaly_count'],3) + self.assertAlmostEqual(evaluation_results['anomaly_1_true_positives_count'],3) self.assertAlmostEqual(evaluation_results['anomaly_1_true_positives_rate'],3/4) - self.assertAlmostEqual(evaluation_results['anomaly_2_anomaly_count'],1) + self.assertAlmostEqual(evaluation_results['anomaly_2_true_positives_count'],1) self.assertAlmostEqual(evaluation_results['anomaly_2_true_positives_rate'],1/2) formatted_series1,anomaly_labels1 = _format_for_anomaly_detector(self.series3) flagged_series1 = _get_model_output([formatted_series1],minmax1) evaluation_results1 = _variable_granularity_evaluation(flagged_series1[0],anomaly_labels1,breakdown=True) - self.assertNotIn('anomaly_1_anomaly_count',evaluation_results1.keys()) + self.assertNotIn('anomaly_1_true_positives_count',evaluation_results1.keys()) self.assertNotIn('anomaly_1_true_positives_rate',evaluation_results1.keys()) - self.assertNotIn('anomaly_2_anomaly_count',evaluation_results1.keys()) + self.assertNotIn('anomaly_2_true_positives_count',evaluation_results1.keys()) self.assertNotIn('anomaly_2_true_positives_rate',evaluation_results1.keys()) def test_point_granularity_evaluation(self): @@ -443,14 +443,14 @@ def test_point_granularity_evaluation_with_breakdown(self): self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) - self.assertIn('anomaly_1_anomaly_count',evaluation_results.keys()) + self.assertIn('anomaly_1_true_positives_count',evaluation_results.keys()) self.assertIn('anomaly_1_true_positives_rate',evaluation_results.keys()) - self.assertIn('anomaly_2_anomaly_count',evaluation_results.keys()) + self.assertIn('anomaly_2_true_positives_count',evaluation_results.keys()) self.assertIn('anomaly_2_true_positives_rate',evaluation_results.keys()) - self.assertAlmostEqual(evaluation_results['anomaly_1_anomaly_count'],2) + self.assertAlmostEqual(evaluation_results['anomaly_1_true_positives_count'],2) self.assertAlmostEqual(evaluation_results['anomaly_1_true_positives_rate'],2/2) - self.assertAlmostEqual(evaluation_results['anomaly_2_anomaly_count'],1) + self.assertAlmostEqual(evaluation_results['anomaly_2_true_positives_count'],1) self.assertAlmostEqual(evaluation_results['anomaly_2_true_positives_rate'],1/1) def test_series_granularity_evaluation(self): @@ -491,9 +491,9 @@ def test_series_granularity_evaluation_with_breakdown(self): self.assertIn('true_positives_rate',evaluation_results.keys()) self.assertIn('false_positives_count',evaluation_results.keys()) self.assertIn('false_positives_ratio',evaluation_results.keys()) - self.assertIn('anomaly_1_anomaly_count',evaluation_results.keys()) + self.assertIn('anomaly_1_true_positives_count',evaluation_results.keys()) self.assertIn('anomaly_1_true_positives_rate',evaluation_results.keys()) - self.assertAlmostEqual(evaluation_results['anomaly_1_anomaly_count'],1) + self.assertAlmostEqual(evaluation_results['anomaly_1_true_positives_count'],1) self.assertAlmostEqual(evaluation_results['anomaly_1_true_positives_rate'],1) formatted_series1,anomaly_labels1 = _format_for_anomaly_detector(self.series1) @@ -507,38 +507,38 @@ def test_get_breakdown_info(self): single_model_evaluation = { 'sample_1': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, - 'spike_anomaly_count': 1, + 'spike_true_positives_count': 1, 'spike_true_positives_rate': 0.5}, 'sample_2': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, - 'spike_anomaly_count': 1, + 'spike_true_positives_count': 1, 'spike_true_positives_rate': 0.5, - 'step_anomaly_count': 2, + 'step_true_positives_count': 2, 'step_true_positives_rate': 2/3 }, 'sample_3': {'true_positives_count': 3, 'true_positives_rate': 1.5, 'false_positives_count': 1, 'false_positives_ratio': 0.14, - 'step_anomaly_count': 3, + 'step_true_positives_count': 3, 'step_true_positives_rate': 1, - 'pattern_anomaly_count': 2, + 'pattern_true_positives_count': 2, 'pattern_true_positives_rate': 0.5 } } breakdown = _get_breakdown_info(single_model_evaluation) - self.assertIn('spike_anomaly_count',breakdown.keys()) + self.assertIn('spike_true_positives_count',breakdown.keys()) self.assertIn('spike_true_positives_rate',breakdown.keys()) - self.assertIn('step_anomaly_count',breakdown.keys()) + self.assertIn('step_true_positives_count',breakdown.keys()) self.assertIn('step_true_positives_rate',breakdown.keys()) - self.assertIn('pattern_anomaly_count',breakdown.keys()) + self.assertIn('pattern_true_positives_count',breakdown.keys()) self.assertIn('pattern_true_positives_rate',breakdown.keys()) - self.assertAlmostEqual(breakdown['spike_anomaly_count'],2) + self.assertAlmostEqual(breakdown['spike_true_positives_count'],2) self.assertAlmostEqual(breakdown['spike_true_positives_rate'],1/2) - self.assertAlmostEqual(breakdown['step_anomaly_count'],5) + self.assertAlmostEqual(breakdown['step_true_positives_count'],5) self.assertAlmostEqual(breakdown['step_true_positives_rate'],5/6) - self.assertAlmostEqual(breakdown['pattern_anomaly_count'],2) + self.assertAlmostEqual(breakdown['pattern_true_positives_count'],2) self.assertAlmostEqual(breakdown['pattern_true_positives_rate'],0.5) def test_variable_granularity_eval_with_breakdown(self): @@ -557,9 +557,9 @@ def test_variable_granularity_eval_with_breakdown(self): self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],5) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],31/126) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_anomaly_count'],5) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_true_positives_count'],5) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_true_positives_rate'],13/24) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_anomaly_count'],2) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_true_positives_count'],2) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_true_positives_rate'],1/2) def test_point_granularity_eval_with_breakdown(self): @@ -578,9 +578,9 @@ def test_point_granularity_eval_with_breakdown(self): self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],4) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],8/21) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_anomaly_count'],4) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_true_positives_count'],4) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_true_positives_rate'],5/6) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_anomaly_count'],2) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_true_positives_count'],2) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_true_positives_rate'],1) def test_series_granularity_eval_with_breakdown(self): @@ -605,9 +605,9 @@ def test_series_granularity_eval_with_breakdown(self): self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_count'],0) self.assertAlmostEqual(evaluation_results['detector_1']['false_positives_ratio'],0) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_anomaly_count'],2) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_true_positives_count'],2) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_1_true_positives_rate'],1) - self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_anomaly_count'],1) + self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_true_positives_count'],1) self.assertAlmostEqual(evaluation_results['detector_1']['anomaly_2_true_positives_rate'],1) try: