From bd60db1363a9d50df3cd68c5b97c1db48efb5fda Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 6 Oct 2024 15:11:23 +0300 Subject: [PATCH 01/23] GroupedAlerts base class --- elementary/monitor/alerts/group_of_alerts.py | 64 ++++++++++++-------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/elementary/monitor/alerts/group_of_alerts.py b/elementary/monitor/alerts/group_of_alerts.py index 4cbbee88d..4383a5f74 100644 --- a/elementary/monitor/alerts/group_of_alerts.py +++ b/elementary/monitor/alerts/group_of_alerts.py @@ -17,7 +17,44 @@ class GroupingType(Enum): BY_TABLE = "table" -class GroupedByTableAlerts: +class GroupedAlert: + def __init__( + self, + alerts: List[Union[TestAlertModel, ModelAlertModel, SourceFreshnessAlertModel]], + ) -> None: + self.alerts = alerts + self.test_errors: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] + self.test_warnings: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] + self.test_failures: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] + self.model_errors: List[ModelAlertModel] = [] + self._sort_alerts() + + @property + def detected_at(self) -> datetime: + return min(alert.detected_at or datetime.max for alert in self.alerts) + + @property + def status(self) -> str: + if self.model_errors or self.test_errors: + return "error" + elif self.test_failures: + return "failure" + else: + return "warn" + + def _sort_alerts(self): + for alert in self.alerts: + if isinstance(alert, ModelAlertModel): + self.model_errors.append(alert) + elif alert.status == "error": + self.test_errors.append(alert) + elif alert.status == "warn": + self.test_warnings.append(alert) + else: + self.test_failures.append(alert) + + +class GroupedByTableAlerts(GroupedAlert): def __init__( self, alerts: List[Union[TestAlertModel, ModelAlertModel, SourceFreshnessAlertModel]], @@ -37,11 +74,6 @@ def model_unique_id(self) -> Optional[str]: def model(self) -> str: return get_shortened_model_name(self.model_unique_id) - @property - def detected_at(self) -> datetime: - # We return the minimum alert detected at time as the group detected at time - return min(alert.detected_at or datetime.max for alert in self.alerts) - @property def report_url(self) -> Optional[str]: return self.alerts[0].report_url @@ -70,28 +102,8 @@ def data(self) -> List[Dict]: def summary(self) -> str: return f"{self.model}: {len(self.alerts)} issues detected" - @property - def status(self) -> str: - if self.model_errors or self.test_errors: - return "error" - elif self.test_failures: - return "failure" - else: - return "warn" - def get_report_link(self) -> Optional[ReportLinkData]: if not self.model_errors: return get_model_test_runs_link(self.report_url, self.model_unique_id) return None - - def _sort_alerts(self): - for alert in self.alerts: - if isinstance(alert, ModelAlertModel): - self.model_errors.append(alert) - elif alert.status == "error": - self.test_errors.append(alert) - elif alert.status == "warn": - self.test_warnings.append(alert) - else: - self.test_failures.append(alert) From be005e6d1f3fa6f2035dedc350af0d3e5da9fa38 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 6 Oct 2024 15:29:42 +0300 Subject: [PATCH 02/23] splitted the grouped alert types into multiple modules --- .../monitor/alerts/grouped_alerts/__init__.py | 9 ++++ .../alerts/grouped_alerts/grouped_alert.py | 43 ++++++++++++++++++ .../grouped_by_table.py} | 45 +------------------ .../alerts/grouped_alerts/grouping_type.py | 6 +++ .../alerts/data_monitoring_alerts.py | 2 +- .../alerts/integrations/base_integration.py | 2 +- .../alerts/integrations/slack/slack.py | 2 +- .../alerts/integrations/teams/teams.py | 2 +- .../alerts/integrations/alerts_data_mock.py | 2 +- .../integrations/base_integration_mock.py | 2 +- 10 files changed, 65 insertions(+), 50 deletions(-) create mode 100644 elementary/monitor/alerts/grouped_alerts/__init__.py create mode 100644 elementary/monitor/alerts/grouped_alerts/grouped_alert.py rename elementary/monitor/alerts/{group_of_alerts.py => grouped_alerts/grouped_by_table.py} (62%) create mode 100644 elementary/monitor/alerts/grouped_alerts/grouping_type.py diff --git a/elementary/monitor/alerts/grouped_alerts/__init__.py b/elementary/monitor/alerts/grouped_alerts/__init__.py new file mode 100644 index 000000000..6af686482 --- /dev/null +++ b/elementary/monitor/alerts/grouped_alerts/__init__.py @@ -0,0 +1,9 @@ +from .grouped_alert import GroupedAlert +from .grouped_by_table import GroupedByTableAlerts +from .grouping_type import GroupingType + +__all__ = [ + "GroupedAlert", + "GroupedByTableAlerts", + "GroupingType", +] diff --git a/elementary/monitor/alerts/grouped_alerts/grouped_alert.py b/elementary/monitor/alerts/grouped_alerts/grouped_alert.py new file mode 100644 index 000000000..d510112c4 --- /dev/null +++ b/elementary/monitor/alerts/grouped_alerts/grouped_alert.py @@ -0,0 +1,43 @@ +from datetime import datetime +from typing import List, Union + +from elementary.monitor.alerts.model_alert import ModelAlertModel +from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel +from elementary.monitor.alerts.test_alert import TestAlertModel + + +class GroupedAlert: + def __init__( + self, + alerts: List[Union[TestAlertModel, ModelAlertModel, SourceFreshnessAlertModel]], + ) -> None: + self.alerts = alerts + self.test_errors: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] + self.test_warnings: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] + self.test_failures: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] + self.model_errors: List[ModelAlertModel] = [] + self._sort_alerts() + + @property + def detected_at(self) -> datetime: + return min(alert.detected_at or datetime.max for alert in self.alerts) + + @property + def status(self) -> str: + if self.model_errors or self.test_errors: + return "error" + elif self.test_failures: + return "failure" + else: + return "warn" + + def _sort_alerts(self): + for alert in self.alerts: + if isinstance(alert, ModelAlertModel): + self.model_errors.append(alert) + elif alert.status == "error": + self.test_errors.append(alert) + elif alert.status == "warn": + self.test_warnings.append(alert) + else: + self.test_failures.append(alert) diff --git a/elementary/monitor/alerts/group_of_alerts.py b/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py similarity index 62% rename from elementary/monitor/alerts/group_of_alerts.py rename to elementary/monitor/alerts/grouped_alerts/grouped_by_table.py index 4383a5f74..30059323b 100644 --- a/elementary/monitor/alerts/group_of_alerts.py +++ b/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py @@ -1,7 +1,6 @@ -from datetime import datetime -from enum import Enum from typing import Dict, List, Optional, Union +from elementary.monitor.alerts.grouped_alerts.grouped_alert import GroupedAlert from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -12,48 +11,6 @@ from elementary.utils.models import get_shortened_model_name -class GroupingType(Enum): - BY_ALERT = "alert" - BY_TABLE = "table" - - -class GroupedAlert: - def __init__( - self, - alerts: List[Union[TestAlertModel, ModelAlertModel, SourceFreshnessAlertModel]], - ) -> None: - self.alerts = alerts - self.test_errors: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] - self.test_warnings: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] - self.test_failures: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] - self.model_errors: List[ModelAlertModel] = [] - self._sort_alerts() - - @property - def detected_at(self) -> datetime: - return min(alert.detected_at or datetime.max for alert in self.alerts) - - @property - def status(self) -> str: - if self.model_errors or self.test_errors: - return "error" - elif self.test_failures: - return "failure" - else: - return "warn" - - def _sort_alerts(self): - for alert in self.alerts: - if isinstance(alert, ModelAlertModel): - self.model_errors.append(alert) - elif alert.status == "error": - self.test_errors.append(alert) - elif alert.status == "warn": - self.test_warnings.append(alert) - else: - self.test_failures.append(alert) - - class GroupedByTableAlerts(GroupedAlert): def __init__( self, diff --git a/elementary/monitor/alerts/grouped_alerts/grouping_type.py b/elementary/monitor/alerts/grouped_alerts/grouping_type.py new file mode 100644 index 000000000..11537b806 --- /dev/null +++ b/elementary/monitor/alerts/grouped_alerts/grouping_type.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class GroupingType(str, Enum): + BY_ALERT = "alert" + BY_TABLE = "table" diff --git a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py index 0494b3e7e..f8e33c485 100644 --- a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py +++ b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py @@ -6,7 +6,7 @@ from alive_progress import alive_it from elementary.config.config import Config -from elementary.monitor.alerts.group_of_alerts import GroupedByTableAlerts, GroupingType +from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts, GroupingType from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel diff --git a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py index ee3c089f8..6fca92ff1 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Union -from elementary.monitor.alerts.group_of_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index d81fbb8e0..8164e782a 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -9,7 +9,7 @@ from elementary.clients.slack.schema import SlackBlocksType, SlackMessageSchema from elementary.clients.slack.slack_message_builder import MessageColor from elementary.config.config import Config -from elementary.monitor.alerts.group_of_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel diff --git a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py index 03401e23c..7a9436fad 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py @@ -7,7 +7,7 @@ from elementary.clients.teams.client import TeamsClient from elementary.config.config import Config -from elementary.monitor.alerts.group_of_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel diff --git a/tests/mocks/data_monitoring/alerts/integrations/alerts_data_mock.py b/tests/mocks/data_monitoring/alerts/integrations/alerts_data_mock.py index 53c57c754..fa6284ade 100644 --- a/tests/mocks/data_monitoring/alerts/integrations/alerts_data_mock.py +++ b/tests/mocks/data_monitoring/alerts/integrations/alerts_data_mock.py @@ -1,4 +1,4 @@ -from elementary.monitor.alerts.group_of_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel diff --git a/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py b/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py index 598fdcc5b..fbc9bca25 100644 --- a/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py +++ b/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py @@ -1,6 +1,6 @@ from typing import Union -from elementary.monitor.alerts.group_of_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel From a3fac511d19ddde9f4708524bce4469628253288 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 6 Oct 2024 15:33:50 +0300 Subject: [PATCH 03/23] created all in one grouped alert type --- .../monitor/alerts/grouped_alerts/__init__.py | 2 ++ .../monitor/alerts/grouped_alerts/all_in_one.py | 7 +++++++ .../alerts/grouped_alerts/grouped_by_table.py | 15 +-------------- .../alerts/grouped_alerts/grouping_type.py | 1 + 4 files changed, 11 insertions(+), 14 deletions(-) create mode 100644 elementary/monitor/alerts/grouped_alerts/all_in_one.py diff --git a/elementary/monitor/alerts/grouped_alerts/__init__.py b/elementary/monitor/alerts/grouped_alerts/__init__.py index 6af686482..aedaaab6c 100644 --- a/elementary/monitor/alerts/grouped_alerts/__init__.py +++ b/elementary/monitor/alerts/grouped_alerts/__init__.py @@ -1,8 +1,10 @@ +from .all_in_one import AllInOneAlert from .grouped_alert import GroupedAlert from .grouped_by_table import GroupedByTableAlerts from .grouping_type import GroupingType __all__ = [ + "AllInOneAlert", "GroupedAlert", "GroupedByTableAlerts", "GroupingType", diff --git a/elementary/monitor/alerts/grouped_alerts/all_in_one.py b/elementary/monitor/alerts/grouped_alerts/all_in_one.py new file mode 100644 index 000000000..7a2bccaf9 --- /dev/null +++ b/elementary/monitor/alerts/grouped_alerts/all_in_one.py @@ -0,0 +1,7 @@ +from elementary.monitor.alerts.grouped_alerts.grouped_alert import GroupedAlert + + +class AllInOneAlert(GroupedAlert): + @property + def summary(self) -> str: + return f"{len(self.alerts)} issues detected" diff --git a/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py b/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py index 30059323b..915d49853 100644 --- a/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py +++ b/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py @@ -1,9 +1,7 @@ -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional from elementary.monitor.alerts.grouped_alerts.grouped_alert import GroupedAlert from elementary.monitor.alerts.model_alert import ModelAlertModel -from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel -from elementary.monitor.alerts.test_alert import TestAlertModel from elementary.monitor.data_monitoring.alerts.integrations.utils.report_link import ( ReportLinkData, get_model_test_runs_link, @@ -12,17 +10,6 @@ class GroupedByTableAlerts(GroupedAlert): - def __init__( - self, - alerts: List[Union[TestAlertModel, ModelAlertModel, SourceFreshnessAlertModel]], - ) -> None: - self.alerts = alerts - self.test_errors: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] - self.test_warnings: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] - self.test_failures: List[Union[TestAlertModel, SourceFreshnessAlertModel]] = [] - self.model_errors: List[ModelAlertModel] = [] - self._sort_alerts() - @property def model_unique_id(self) -> Optional[str]: return self.alerts[0].model_unique_id diff --git a/elementary/monitor/alerts/grouped_alerts/grouping_type.py b/elementary/monitor/alerts/grouped_alerts/grouping_type.py index 11537b806..99b62b6ef 100644 --- a/elementary/monitor/alerts/grouped_alerts/grouping_type.py +++ b/elementary/monitor/alerts/grouped_alerts/grouping_type.py @@ -4,3 +4,4 @@ class GroupingType(str, Enum): BY_ALERT = "alert" BY_TABLE = "table" + ALL_IN_ONE = "all_in_one" From ac97fe631de5cadaf7c4de3946bb311d511dff9b Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 6 Oct 2024 15:39:44 +0300 Subject: [PATCH 04/23] added group_all_alerts_threshold config key --- elementary/config/config.py | 7 ++++++- elementary/monitor/cli.py | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/elementary/config/config.py b/elementary/config/config.py index 35741ac1f..743c7005e 100644 --- a/elementary/config/config.py +++ b/elementary/config/config.py @@ -7,7 +7,7 @@ from google.auth.exceptions import DefaultCredentialsError # type: ignore[import] from elementary.exceptions.exceptions import InvalidArgumentsError -from elementary.monitor.alerts.group_of_alerts import GroupingType +from elementary.monitor.alerts.grouped_alerts import GroupingType from elementary.utils.ordered_yaml import OrderedYaml @@ -49,6 +49,7 @@ def __init__( slack_token: Optional[str] = None, slack_channel_name: Optional[str] = None, slack_group_alerts_by: Optional[str] = None, + group_all_alerts_threshold: Optional[int] = None, timezone: Optional[str] = None, aws_profile_name: Optional[str] = None, aws_region_name: Optional[str] = None, @@ -124,6 +125,10 @@ def __init__( slack_config.get("group_alerts_by"), GroupingType.BY_ALERT.value, ) + self.group_all_alerts_threshold = self._first_not_none( + group_all_alerts_threshold, + slack_config.get("group_all_alerts_threshold"), + ) teams_config = config.get(self._TEAMS, {}) self.teams_webhook = self._first_not_none( diff --git a/elementary/monitor/cli.py b/elementary/monitor/cli.py index c896977f5..78c7dcfea 100644 --- a/elementary/monitor/cli.py +++ b/elementary/monitor/cli.py @@ -200,6 +200,12 @@ def get_cli_properties() -> dict: default=None, help="DEPRECATED! - A slack webhook URL for sending alerts to a specific channel.", ) +@click.option( + "--group-all-alerts-threshold", + type=int, + default=100, + help="The threshold for grouping all alerts in a single message.", +) @click.option( "--timezone", "-tz", @@ -276,6 +282,7 @@ def monitor( deprecated_slack_webhook, slack_token, slack_channel_name, + group_all_alerts_threshold, timezone, config_dir, profiles_dir, @@ -322,6 +329,7 @@ def monitor( slack_webhook=slack_webhook, slack_token=slack_token, slack_channel_name=slack_channel_name, + group_all_alerts_threshold=group_all_alerts_threshold, timezone=timezone, env=env, slack_group_alerts_by=group_by, From 1566dc0c6f475a3be12f8b04d924f5fe41940e8d Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 6 Oct 2024 16:15:06 +0300 Subject: [PATCH 05/23] moved data and unified_meta properties to the base grouped alert class --- .../alerts/grouped_alerts/grouped_alert.py | 20 +++++++++++++++- .../alerts/grouped_alerts/grouped_by_table.py | 23 +------------------ 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/elementary/monitor/alerts/grouped_alerts/grouped_alert.py b/elementary/monitor/alerts/grouped_alerts/grouped_alert.py index d510112c4..f17242d0a 100644 --- a/elementary/monitor/alerts/grouped_alerts/grouped_alert.py +++ b/elementary/monitor/alerts/grouped_alerts/grouped_alert.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Union +from typing import Dict, List, Union from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel @@ -31,6 +31,24 @@ def status(self) -> str: else: return "warn" + @property + def data(self) -> List[Dict]: + return [alert.data for alert in self.alerts] + + @property + def unified_meta(self) -> Dict: + model_unified_meta = dict() + test_unified_meta = dict() + for alert in self.alerts: + alert_unified_meta = alert.unified_meta + if alert_unified_meta: + if isinstance(alert, ModelAlertModel): + model_unified_meta = alert_unified_meta + break + + test_unified_meta = alert_unified_meta + return model_unified_meta or test_unified_meta + def _sort_alerts(self): for alert in self.alerts: if isinstance(alert, ModelAlertModel): diff --git a/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py b/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py index 915d49853..565f65e86 100644 --- a/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py +++ b/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py @@ -1,7 +1,6 @@ -from typing import Dict, List, Optional +from typing import Optional from elementary.monitor.alerts.grouped_alerts.grouped_alert import GroupedAlert -from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.data_monitoring.alerts.integrations.utils.report_link import ( ReportLinkData, get_model_test_runs_link, @@ -22,26 +21,6 @@ def model(self) -> str: def report_url(self) -> Optional[str]: return self.alerts[0].report_url - @property - def unified_meta(self) -> Dict: - # If a model level unified meta is defined, we use is. - # Else we use one of the tests level unified metas. - model_unified_meta = dict() - test_unified_meta = dict() - for alert in self.alerts: - alert_unified_meta = alert.unified_meta - if alert_unified_meta: - if isinstance(alert, ModelAlertModel): - model_unified_meta = alert_unified_meta - break - - test_unified_meta = alert_unified_meta - return model_unified_meta or test_unified_meta - - @property - def data(self) -> List[Dict]: - return [alert.data for alert in self.alerts] - @property def summary(self) -> str: return f"{self.model}: {len(self.alerts)} issues detected" From b8efe335449b763ae91c33e074e6521962f890ec Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 6 Oct 2024 16:28:26 +0300 Subject: [PATCH 06/23] added abstract method for all-in-one alert template --- .../data_monitoring/alerts/data_monitoring_alerts.py | 10 +++++++--- .../alerts/integrations/base_integration.py | 9 ++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py index f8e33c485..df4a79a45 100644 --- a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py +++ b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py @@ -6,7 +6,11 @@ from alive_progress import alive_it from elementary.config.config import Config -from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts, GroupingType +from elementary.monitor.alerts.grouped_alerts import ( + GroupedAlert, + GroupedByTableAlerts, + GroupingType, +) from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -254,12 +258,12 @@ def _send_alerts( for alert in alerts_with_progress_bar: sent_successfully = self.alerts_integration.send_alert(alert=alert) if sent_successfully: - if isinstance(alert, GroupedByTableAlerts): + if isinstance(alert, GroupedAlert): sent_successfully_alerts.extend(alert.alerts) else: sent_successfully_alerts.append(alert) else: - if isinstance(alert, GroupedByTableAlerts): + if isinstance(alert, GroupedAlert): for grouped_alert in alert.alerts: logger.error( f"Could not send the alert - {grouped_alert.id}. Full alert: {json.dumps(grouped_alert.data)}" diff --git a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py index 6fca92ff1..4d36c27d8 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Union -from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import AllInOneAlert, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -22,6 +22,7 @@ def _get_alert_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], *args, **kwargs @@ -40,6 +41,8 @@ def _get_alert_template( return self._get_source_freshness_template(alert) elif isinstance(alert, GroupedByTableAlerts): return self._get_group_by_table_template(alert) + elif isinstance(alert, AllInOneAlert): + return self._get_all_in_one_template(alert) @abstractmethod def _get_dbt_test_template(self, alert: TestAlertModel, *args, **kwargs): @@ -69,6 +72,10 @@ def _get_group_by_table_template( ): raise NotImplementedError + @abstractmethod + def _get_all_in_one_template(self, alert: AllInOneAlert, *args, **kwargs): + raise NotImplementedError + @abstractmethod def _get_fallback_template( self, From cc5c8f174c0584faf93ed81293f80945c6a8dd55 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 6 Oct 2024 17:27:15 +0300 Subject: [PATCH 07/23] Slack integration - added all in one alert template --- .../alerts/integrations/slack/slack.py | 134 +++++++++++++++--- 1 file changed, 113 insertions(+), 21 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index 8164e782a..25bb83657 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -9,7 +9,11 @@ from elementary.clients.slack.schema import SlackBlocksType, SlackMessageSchema from elementary.clients.slack.slack_message_builder import MessageColor from elementary.config.config import Config -from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import ( + AllInOneAlert, + GroupedAlert, + GroupedByTableAlerts, +) from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -97,6 +101,7 @@ def _get_alert_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], *args, **kwargs, @@ -752,6 +757,26 @@ def _get_source_freshness_template( ), ) + def _get_alert_type_counters_block(self, alert: GroupedAlert) -> dict: + counters_text: List[str] = [] + if alert.model_errors: + counters_text.append(f":X: Model errors: {len(alert.model_errors)}") + if alert.test_failures: + if counters_text: + counters_text.append(" |") + counters_text.append( + f":small_red_triangle: Test failures: {len(alert.test_failures)}" + ) + if alert.test_warnings: + if counters_text: + counters_text.append(" |") + counters_text.append(f":warning: Test warnings: {len(alert.test_warnings)}") + if alert.test_errors: + if counters_text: + counters_text.append(" |") + counters_text.append(f":exclamation: Test errors: {len(alert.test_errors)}") + return self.message_builder.create_context_block(counters_text) + def _get_group_by_table_template( self, alert: GroupedByTableAlerts, *args, **kwargs ) -> SlackAlertMessageSchema: @@ -762,26 +787,9 @@ def _get_group_by_table_template( title_blocks = [ self.message_builder.create_header_block( f"{self._get_display_name(alert.status)}: {alert.summary}" - ) + ), + self._get_alert_type_counters_block(alert), ] - # summary of number of failed, errors, etc. - fields_summary: List[str] = [] - # summary of number of failed, errors, etc. - if alert.model_errors: - fields_summary.append(f":X: Model errors: {len(alert.model_errors)} |") - if alert.test_failures: - fields_summary.append( - f":small_red_triangle: Test failures: {len(alert.test_failures)} |" - ) - if alert.test_warnings: - fields_summary.append( - f":warning: Test warnings: {len(alert.test_warnings)} |" - ) - if alert.test_errors: - fields_summary.append( - f":exclamation: Test errors: {len(alert.test_errors)}" - ) - title_blocks.append(self.message_builder.create_context_block(fields_summary)) report_link = None # No report link when there is only model error @@ -873,6 +881,90 @@ def _get_group_by_table_template( title=title_blocks, preview=preview_blocks, details=details_blocks ) + def _get_all_in_one_template( + self, alert: AllInOneAlert, *args, **kwargs + ) -> SlackAlertMessageSchema: + self.message_builder.add_message_color(self._get_color(alert.status)) + + title_blocks = [ + self.message_builder.create_header_block( + f"{self._get_display_name(alert.status)}: {alert.summary}" + ), + self._get_alert_type_counters_block(alert), + ] + + details_blocks = [] + + if alert.model_errors: + details_blocks.append( + self.message_builder.create_text_section_block("*Model errors*") + ) + for model_error_alert in alert.model_errors: + text = f":X: {model_error_alert.summary}" + section = ( + self.message_builder.create_section_with_button( + text, + button_text="View Details", + url=model_error_alert.report_url, + ) + if model_error_alert.report_url + else self.message_builder.create_text_section_block(text) + ) + details_blocks.append(section) + + if alert.test_failures: + details_blocks.append( + self.message_builder.create_text_section_block("*Test failures*") + ) + for test_failure_alert in alert.test_failures: + text = f":small_red_triangle: {test_failure_alert.summary}" + section = ( + self.message_builder.create_section_with_button( + text, + button_text="View Details", + url=test_failure_alert.report_url, + ) + if test_failure_alert.report_url + else self.message_builder.create_text_section_block(text) + ) + details_blocks.append(section) + + if alert.test_warnings: + details_blocks.append( + self.message_builder.create_text_section_block("*Test warnings*") + ) + for test_warning_alert in alert.test_warnings: + text = f":warning: {test_warning_alert.summary}" + section = ( + self.message_builder.create_section_with_button( + text, + button_text="View Details", + url=test_warning_alert.report_url, + ) + if test_warning_alert.report_url + else self.message_builder.create_text_section_block(text) + ) + details_blocks.append(section) + + if alert.test_errors: + details_blocks.append( + self.message_builder.create_text_section_block("*Test errors*") + ) + for test_error_alert in alert.test_errors: + text = f":exclamation: {test_error_alert.summary}" + section = ( + self.message_builder.create_section_with_button( + text, + button_text="View Details", + url=test_error_alert.report_url, + ) + if test_error_alert.report_url + else self.message_builder.create_text_section_block(text) + ) + details_blocks.append(section) + + return SlackAlertMessageSchema(title=title_blocks, details=details_blocks) + @staticmethod def _get_model_error_block_header( model_error_alerts: List[ModelAlertModel], @@ -945,7 +1037,7 @@ def _fix_owners_and_subscribers( GroupedByTableAlerts, ], ): - if isinstance(alert, GroupedByTableAlerts): + if isinstance(alert, GroupedAlert): for grouped_alert in alert.alerts: grouped_alert.owners = self._parse_emails_to_ids(grouped_alert.owners) grouped_alert.subscribers = self._parse_emails_to_ids( From de78be67620bc8db06d6b1a1eb677d82309253b0 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 6 Oct 2024 17:30:16 +0300 Subject: [PATCH 08/23] Teams integration - added all in one alert template --- .../alerts/integrations/teams/teams.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py index 7a9436fad..67faadd86 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py @@ -8,6 +8,7 @@ from elementary.clients.teams.client import TeamsClient from elementary.config.config import Config from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts.all_in_one import AllInOneAlert from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -520,6 +521,69 @@ def _get_group_by_table_template( self._get_section("*Test errors*", f"{text}") ) + def _get_all_in_one_template(self, alert: AllInOneAlert, *args, **kwargs): + title = f"{self._get_display_name(alert.status)}: {alert.summary}" + subtitle = "" + + if alert.model_errors: + subtitle = ( + subtitle + + (" | " + f"😵 Model errors: {len(alert.model_errors)}") + if subtitle + else f"😵 Model errors: {len(alert.model_errors)}" + ) + if alert.test_failures: + subtitle = ( + subtitle + + (" | " + f"🔺 Test failures: {len(alert.test_failures)}") + if subtitle + else f"🔺 Test failures: {len(alert.test_failures)}" + ) + if alert.test_warnings: + subtitle = ( + subtitle + + (" | " + f"⚠ Test warnings: {len(alert.test_warnings)}") + if subtitle + else f"⚠ Test warnings: {len(alert.test_warnings)}" + ) + if alert.test_errors: + subtitle = ( + subtitle + (" | " + f"❗ Test errors: {len(alert.test_errors)}") + if subtitle + else f"❗ Test errors: {len(alert.test_errors)}" + ) + + self.message_builder.title(title) + self.message_builder.text(subtitle) + + if alert.model_errors: + rows = [alert.summary for alert in alert.model_errors] + text = "\n".join([f"😵 {row}" for row in rows]) + self.message_builder.addSection( + self._get_section("*Model errors*", f"{text}") + ) + + if alert.test_failures: + rows = [alert.summary for alert in alert.test_failures] + text = "\n".join([f"🔺 {row}" for row in rows]) + self.message_builder.addSection( + self._get_section("*Test failures*", f"{text}") + ) + + if alert.test_warnings: + rows = [alert.summary for alert in alert.test_warnings] + text = "\n".join([f"⚠ {row}" for row in rows]) + self.message_builder.addSection( + self._get_section("*Test warnings*", f"{text}") + ) + + if alert.test_errors: + rows = [alert.summary for alert in alert.test_errors] + text = "\n".join([f"❗ {row}" for row in rows]) + self.message_builder.addSection( + self._get_section("*Test errors*", f"{text}") + ) + def _get_fallback_template( self, alert: Union[ From fd24e51bdc48808b827db3cf3954d28555d20d1a Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 6 Oct 2024 17:58:12 +0300 Subject: [PATCH 09/23] added a compact version of the all-in-one-alert, to avoid the slack max length limit --- .../alerts/integrations/slack/slack.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index 25bb83657..2eeecd551 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -38,6 +38,8 @@ logger = get_logger(__name__) +COMPACT_SCHEMA_THRESHOLD = 500 + TABLE_FIELD = "table" COLUMN_FIELD = "column" DESCRIPTION_FIELD = "description" @@ -881,9 +883,50 @@ def _get_group_by_table_template( title=title_blocks, preview=preview_blocks, details=details_blocks ) + def _get_all_in_one_compact_template( + self, alert: AllInOneAlert + ) -> SlackAlertMessageSchema: + self.message_builder.add_message_color(self._get_color(alert.status)) + + title_blocks = [ + self.message_builder.create_header_block( + f"{self._get_display_name(alert.status)}: {alert.summary}" + ) + ] + + details_blocks = [] + if alert.model_errors: + details_blocks.append( + self.message_builder.create_text_section_block( + f":X: {len(alert.model_errors)} Model errors" + ) + ) + if alert.test_failures: + details_blocks.append( + self.message_builder.create_text_section_block( + f":small_red_triangle: {len(alert.test_failures)} Test failures" + ) + ) + if alert.test_warnings: + details_blocks.append( + self.message_builder.create_text_section_block( + f":warning: {len(alert.test_warnings)} Test warnings" + ) + ) + if alert.test_errors: + details_blocks.append( + self.message_builder.create_text_section_block( + f":exclamation: {len(alert.test_errors)} Test errors" + ) + ) + return SlackAlertMessageSchema(title=title_blocks, details=details_blocks) + def _get_all_in_one_template( self, alert: AllInOneAlert, *args, **kwargs ) -> SlackAlertMessageSchema: + if len(alert.alerts) >= COMPACT_SCHEMA_THRESHOLD: + return self._get_all_in_one_compact_template(alert) + self.message_builder.add_message_color(self._get_color(alert.status)) title_blocks = [ From 818886bb25eafed9148bf697ca3aded55c24e357 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 7 Oct 2024 13:57:13 +0300 Subject: [PATCH 10/23] using send_alerts method instead of send_alert --- .../alerts/data_monitoring_alerts.py | 5 ++-- .../alerts/integrations/base_integration.py | 25 ++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py index df4a79a45..b9473ae5e 100644 --- a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py +++ b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py @@ -255,8 +255,9 @@ def _send_alerts( alerts_with_progress_bar = alive_it(alerts, title="Sending alerts") sent_successfully_alerts = [] - for alert in alerts_with_progress_bar: - sent_successfully = self.alerts_integration.send_alert(alert=alert) + for alert, sent_successfully in self.alerts_integration.send_alerts( + alerts_with_progress_bar + ).items(): if sent_successfully: if isinstance(alert, GroupedAlert): sent_successfully_alerts.extend(alert.alerts) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py index 4d36c27d8..2dcbd3aad 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Union +from typing import Dict, List, Union from elementary.monitor.alerts.grouped_alerts import AllInOneAlert, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel @@ -104,6 +104,29 @@ def send_alert( ) -> bool: raise NotImplementedError + def send_alerts( + self, + alerts: List[ + Union[ + TestAlertModel, + ModelAlertModel, + SourceFreshnessAlertModel, + GroupedByTableAlerts, + ] + ], + *args, + **kwargs + ) -> Dict[ + Union[ + TestAlertModel, + ModelAlertModel, + SourceFreshnessAlertModel, + GroupedByTableAlerts, + ], + bool, + ]: + return {alert: self.send_alert(alert, *args, **kwargs) for alert in alerts} + @abstractmethod def send_test_message(self, *args, **kwargs) -> bool: raise NotImplementedError From 0b012c275734c9ca783f95ceb2af465be3215a28 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 7 Oct 2024 14:19:28 +0300 Subject: [PATCH 11/23] supporting unified alert only in slack integration --- .../alerts/integrations/base_integration.py | 9 +-- .../alerts/integrations/slack/slack.py | 42 +++++++++++- .../alerts/integrations/teams/teams.py | 64 ------------------- 3 files changed, 40 insertions(+), 75 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py index 2dcbd3aad..ef48d8ece 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Dict, List, Union -from elementary.monitor.alerts.grouped_alerts import AllInOneAlert, GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -22,7 +22,6 @@ def _get_alert_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, ], *args, **kwargs @@ -41,8 +40,6 @@ def _get_alert_template( return self._get_source_freshness_template(alert) elif isinstance(alert, GroupedByTableAlerts): return self._get_group_by_table_template(alert) - elif isinstance(alert, AllInOneAlert): - return self._get_all_in_one_template(alert) @abstractmethod def _get_dbt_test_template(self, alert: TestAlertModel, *args, **kwargs): @@ -72,10 +69,6 @@ def _get_group_by_table_template( ): raise NotImplementedError - @abstractmethod - def _get_all_in_one_template(self, alert: AllInOneAlert, *args, **kwargs): - raise NotImplementedError - @abstractmethod def _get_fallback_template( self, diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index 2eeecd551..c1b3ba0b7 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -110,9 +110,11 @@ def _get_alert_template( ) -> SlackMessageSchema: if self.config.is_slack_workflow: return SlackMessageSchema(text=json.dumps(alert.data, sort_keys=True)) - alert_schema: SlackAlertMessageSchema = super()._get_alert_template( - alert, *args, **kwargs - ) + + if isinstance(alert, AllInOneAlert): + alert_schema = self._get_all_in_one_template(alert) + else: + alert_schema = super()._get_alert_template(alert, *args, **kwargs) return self.message_builder.get_slack_message(alert_schema=alert_schema) def _get_dbt_test_template( @@ -1036,6 +1038,7 @@ def _get_fallback_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], *args, **kwargs, @@ -1078,6 +1081,7 @@ def _fix_owners_and_subscribers( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], ): if isinstance(alert, GroupedAlert): @@ -1097,6 +1101,7 @@ def send_alert( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], *args, **kwargs, @@ -1130,6 +1135,36 @@ def send_alert( self.message_builder.reset_slack_message() return sent_successfully + def send_alerts( + self, + alerts: List[ + TestAlertModel + | ModelAlertModel + | SourceFreshnessAlertModel + | GroupedByTableAlerts + ], + *args, + **kwargs, + ) -> Dict[ + TestAlertModel + | ModelAlertModel + | SourceFreshnessAlertModel + | GroupedByTableAlerts, + bool, + ]: + falttened_alerts = [] + for alert in alerts: + if isinstance(alert, GroupedByTableAlerts): + falttened_alerts.extend(alert.alerts) + else: + falttened_alerts.append(alert) + + if len(falttened_alerts) > self.config.group_all_alerts_threshold: + single_alert = AllInOneAlert(alerts=falttened_alerts) + sent_successfully = self.send_alert(single_alert, *args, **kwargs) + return {alert: sent_successfully for alert in alerts} + return super().send_alerts(alerts, *args, **kwargs) + def send_test_message(self, channel_name: str, *args, **kwargs) -> bool: test_message = self._get_test_message_template() return self.client.send_message(channel_name=channel_name, message=test_message) @@ -1141,6 +1176,7 @@ def _get_integration_params( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], *args, **kwargs, diff --git a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py index 67faadd86..7a9436fad 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py @@ -8,7 +8,6 @@ from elementary.clients.teams.client import TeamsClient from elementary.config.config import Config from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts -from elementary.monitor.alerts.grouped_alerts.all_in_one import AllInOneAlert from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -521,69 +520,6 @@ def _get_group_by_table_template( self._get_section("*Test errors*", f"{text}") ) - def _get_all_in_one_template(self, alert: AllInOneAlert, *args, **kwargs): - title = f"{self._get_display_name(alert.status)}: {alert.summary}" - subtitle = "" - - if alert.model_errors: - subtitle = ( - subtitle - + (" | " + f"😵 Model errors: {len(alert.model_errors)}") - if subtitle - else f"😵 Model errors: {len(alert.model_errors)}" - ) - if alert.test_failures: - subtitle = ( - subtitle - + (" | " + f"🔺 Test failures: {len(alert.test_failures)}") - if subtitle - else f"🔺 Test failures: {len(alert.test_failures)}" - ) - if alert.test_warnings: - subtitle = ( - subtitle - + (" | " + f"⚠ Test warnings: {len(alert.test_warnings)}") - if subtitle - else f"⚠ Test warnings: {len(alert.test_warnings)}" - ) - if alert.test_errors: - subtitle = ( - subtitle + (" | " + f"❗ Test errors: {len(alert.test_errors)}") - if subtitle - else f"❗ Test errors: {len(alert.test_errors)}" - ) - - self.message_builder.title(title) - self.message_builder.text(subtitle) - - if alert.model_errors: - rows = [alert.summary for alert in alert.model_errors] - text = "\n".join([f"😵 {row}" for row in rows]) - self.message_builder.addSection( - self._get_section("*Model errors*", f"{text}") - ) - - if alert.test_failures: - rows = [alert.summary for alert in alert.test_failures] - text = "\n".join([f"🔺 {row}" for row in rows]) - self.message_builder.addSection( - self._get_section("*Test failures*", f"{text}") - ) - - if alert.test_warnings: - rows = [alert.summary for alert in alert.test_warnings] - text = "\n".join([f"⚠ {row}" for row in rows]) - self.message_builder.addSection( - self._get_section("*Test warnings*", f"{text}") - ) - - if alert.test_errors: - rows = [alert.summary for alert in alert.test_errors] - text = "\n".join([f"❗ {row}" for row in rows]) - self.message_builder.addSection( - self._get_section("*Test errors*", f"{text}") - ) - def _get_fallback_template( self, alert: Union[ From 23f824a87df2264552c9a2c6d6a4cf95b4e50c7d Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Tue, 8 Oct 2024 12:34:17 +0300 Subject: [PATCH 12/23] moved the compact schema to class attribute --- .../data_monitoring/alerts/integrations/slack/slack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index c1b3ba0b7..8b0c7eb74 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -38,7 +38,6 @@ logger = get_logger(__name__) -COMPACT_SCHEMA_THRESHOLD = 500 TABLE_FIELD = "table" COLUMN_FIELD = "column" @@ -71,6 +70,8 @@ class SlackIntegration(BaseIntegration): + COMPACT_SCHEMA_THRESHOLD = 500 + def __init__( self, config: Config, @@ -926,7 +927,7 @@ def _get_all_in_one_compact_template( def _get_all_in_one_template( self, alert: AllInOneAlert, *args, **kwargs ) -> SlackAlertMessageSchema: - if len(alert.alerts) >= COMPACT_SCHEMA_THRESHOLD: + if len(alert.alerts) >= self.COMPACT_SCHEMA_THRESHOLD: return self._get_all_in_one_compact_template(alert) self.message_builder.add_message_color(self._get_color(alert.status)) From eaf4c4aba1e2529a4bf5b850b85447aca42a75d9 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Tue, 8 Oct 2024 12:35:26 +0300 Subject: [PATCH 13/23] changed send_alerts to return a generator --- .../alerts/data_monitoring_alerts.py | 2 +- .../alerts/integrations/base_integration.py | 23 +++++++----- .../alerts/integrations/slack/slack.py | 35 ++++++++++++------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py index b9473ae5e..74e27e4ed 100644 --- a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py +++ b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py @@ -257,7 +257,7 @@ def _send_alerts( sent_successfully_alerts = [] for alert, sent_successfully in self.alerts_integration.send_alerts( alerts_with_progress_bar - ).items(): + ): if sent_successfully: if isinstance(alert, GroupedAlert): sent_successfully_alerts.extend(alert.alerts) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py index ef48d8ece..26f56eddc 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Dict, List, Union +from typing import Generator, List, Tuple, Union from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel @@ -109,16 +109,21 @@ def send_alerts( ], *args, **kwargs - ) -> Dict[ - Union[ - TestAlertModel, - ModelAlertModel, - SourceFreshnessAlertModel, - GroupedByTableAlerts, + ) -> Generator[ + Tuple[ + Union[ + TestAlertModel, + ModelAlertModel, + SourceFreshnessAlertModel, + GroupedByTableAlerts, + ], + bool, ], - bool, + None, + None, ]: - return {alert: self.send_alert(alert, *args, **kwargs) for alert in alerts} + for alert in alerts: + yield alert, self.send_alert(alert) @abstractmethod def send_test_message(self, *args, **kwargs) -> bool: diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index 8b0c7eb74..639af8d9a 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -1,7 +1,7 @@ import json import re from datetime import datetime, timedelta -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Generator, List, Optional, Tuple, Union from slack_sdk.models.blocks import SectionBlock @@ -1139,19 +1139,27 @@ def send_alert( def send_alerts( self, alerts: List[ - TestAlertModel - | ModelAlertModel - | SourceFreshnessAlertModel - | GroupedByTableAlerts + Union[ + TestAlertModel, + ModelAlertModel, + SourceFreshnessAlertModel, + GroupedByTableAlerts, + ] ], *args, **kwargs, - ) -> Dict[ - TestAlertModel - | ModelAlertModel - | SourceFreshnessAlertModel - | GroupedByTableAlerts, - bool, + ) -> Generator[ + Tuple[ + Union[ + TestAlertModel, + ModelAlertModel, + SourceFreshnessAlertModel, + GroupedByTableAlerts, + ], + bool, + ], + None, + None, ]: falttened_alerts = [] for alert in alerts: @@ -1163,8 +1171,9 @@ def send_alerts( if len(falttened_alerts) > self.config.group_all_alerts_threshold: single_alert = AllInOneAlert(alerts=falttened_alerts) sent_successfully = self.send_alert(single_alert, *args, **kwargs) - return {alert: sent_successfully for alert in alerts} - return super().send_alerts(alerts, *args, **kwargs) + for alert in falttened_alerts: + yield alert, sent_successfully + yield from super().send_alerts(alerts, *args, **kwargs) def send_test_message(self, channel_name: str, *args, **kwargs) -> bool: test_message = self._get_test_message_template() From 3b8773781b55eebe8a1c806ed614b93e408fd5ca Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Tue, 8 Oct 2024 14:51:47 +0300 Subject: [PATCH 14/23] refactored al-in-one alert templates --- .../alerts/integrations/slack/slack.py | 238 ++++++++++-------- 1 file changed, 129 insertions(+), 109 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index 639af8d9a..225148e49 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -1,7 +1,7 @@ import json import re from datetime import datetime, timedelta -from typing import Any, Dict, Generator, List, Optional, Tuple, Union +from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union from slack_sdk.models.blocks import SectionBlock @@ -763,24 +763,24 @@ def _get_source_freshness_template( ) def _get_alert_type_counters_block(self, alert: GroupedAlert) -> dict: - counters_text: List[str] = [] + counters_texts: List[str] = [] if alert.model_errors: - counters_text.append(f":X: Model errors: {len(alert.model_errors)}") + counters_texts.append(f":X: Model errors: {len(alert.model_errors)}") if alert.test_failures: - if counters_text: - counters_text.append(" |") - counters_text.append( + counters_texts.append( f":small_red_triangle: Test failures: {len(alert.test_failures)}" ) if alert.test_warnings: - if counters_text: - counters_text.append(" |") - counters_text.append(f":warning: Test warnings: {len(alert.test_warnings)}") + counters_texts.append( + f":warning: Test warnings: {len(alert.test_warnings)}" + ) if alert.test_errors: - if counters_text: - counters_text.append(" |") - counters_text.append(f":exclamation: Test errors: {len(alert.test_errors)}") - return self.message_builder.create_context_block(counters_text) + counters_texts.append( + f":exclamation: Test errors: {len(alert.test_errors)}" + ) + return self.message_builder.create_context_block( + [" | ".join(counters_texts)] + ) def _get_group_by_table_template( self, alert: GroupedByTableAlerts, *args, **kwargs @@ -886,6 +886,58 @@ def _get_group_by_table_template( title=title_blocks, preview=preview_blocks, details=details_blocks ) + def _add_compact_all_in_one_sub_group_details_block( + self, + details_blocks: list, + alerts: Sequence[ + Union[ + TestAlertModel, + ModelAlertModel, + SourceFreshnessAlertModel, + GroupedByTableAlerts, + ], + ], + sub_title: str, + bullet_icon: str, + ) -> None: + if not alerts: + return + details_blocks.append( + self.message_builder.create_text_section_block( + f":{bullet_icon}: {len(alerts)} {sub_title}" + ) + ) + + def _get_compact_all_in_one_sub_group_details_block( + self, alert: AllInOneAlert + ) -> List[dict]: + details_blocks: List[dict] = [] + self._add_compact_all_in_one_sub_group_details_block( + details_blocks=details_blocks, + alerts=alert.model_errors, + sub_title="Model Errors", + bullet_icon="X", + ) + self._add_compact_all_in_one_sub_group_details_block( + details_blocks=details_blocks, + alerts=alert.test_failures, + sub_title="Test Failures", + bullet_icon="small_red_triangle", + ) + self._add_compact_all_in_one_sub_group_details_block( + details_blocks=details_blocks, + alerts=alert.test_warnings, + sub_title="Test Warnings", + bullet_icon="warning", + ) + self._add_compact_all_in_one_sub_group_details_block( + details_blocks=details_blocks, + alerts=alert.test_errors, + sub_title="Test Errors", + bullet_icon="exclamation", + ) + return details_blocks + def _get_all_in_one_compact_template( self, alert: AllInOneAlert ) -> SlackAlertMessageSchema: @@ -897,32 +949,71 @@ def _get_all_in_one_compact_template( ) ] - details_blocks = [] - if alert.model_errors: - details_blocks.append( - self.message_builder.create_text_section_block( - f":X: {len(alert.model_errors)} Model errors" - ) - ) - if alert.test_failures: - details_blocks.append( - self.message_builder.create_text_section_block( - f":small_red_triangle: {len(alert.test_failures)} Test failures" - ) - ) - if alert.test_warnings: - details_blocks.append( - self.message_builder.create_text_section_block( - f":warning: {len(alert.test_warnings)} Test warnings" - ) - ) - if alert.test_errors: - details_blocks.append( - self.message_builder.create_text_section_block( - f":exclamation: {len(alert.test_errors)} Test errors" + details_blocks = self._get_compact_all_in_one_sub_group_details_block(alert) + return SlackAlertMessageSchema(title=title_blocks, details=details_blocks) + + def _add_all_in_one_sub_group_details_block( + self, + details_blocks: list, + alerts: Sequence[ + Union[ + TestAlertModel, + ModelAlertModel, + SourceFreshnessAlertModel, + GroupedByTableAlerts, + ], + ], + sub_title: str, + bullet_icon: str, + ) -> None: + if not alerts: + return + + details_blocks.append( + self.message_builder.create_text_section_block(f"*{sub_title}*") + ) + for alert in alerts: + text = f":{bullet_icon}: {alert.summary}" + section = ( + self.message_builder.create_section_with_button( + text, + button_text="View Details", + url=alert.report_url, ) + if alert.report_url + else self.message_builder.create_text_section_block(text) ) - return SlackAlertMessageSchema(title=title_blocks, details=details_blocks) + details_blocks.append(section) + + def _get_all_in_one_sub_group_details_blocks( + self, alert: AllInOneAlert + ) -> List[dict]: + details_blocks: List[dict] = [] + self._add_all_in_one_sub_group_details_block( + details_blocks=details_blocks, + alerts=alert.model_errors, + sub_title="Model Errors", + bullet_icon="X", + ) + self._add_all_in_one_sub_group_details_block( + details_blocks=details_blocks, + alerts=alert.test_failures, + sub_title="Test Failures", + bullet_icon="small_red_triangle", + ) + self._add_all_in_one_sub_group_details_block( + details_blocks=details_blocks, + alerts=alert.test_warnings, + sub_title="Test Warnings", + bullet_icon="warning", + ) + self._add_all_in_one_sub_group_details_block( + details_blocks=details_blocks, + alerts=alert.test_errors, + sub_title="Test Errors", + bullet_icon="exclamation", + ) + return details_blocks def _get_all_in_one_template( self, alert: AllInOneAlert, *args, **kwargs @@ -931,84 +1022,13 @@ def _get_all_in_one_template( return self._get_all_in_one_compact_template(alert) self.message_builder.add_message_color(self._get_color(alert.status)) - title_blocks = [ self.message_builder.create_header_block( f"{self._get_display_name(alert.status)}: {alert.summary}" ), self._get_alert_type_counters_block(alert), ] - - details_blocks = [] - - if alert.model_errors: - details_blocks.append( - self.message_builder.create_text_section_block("*Model errors*") - ) - for model_error_alert in alert.model_errors: - text = f":X: {model_error_alert.summary}" - section = ( - self.message_builder.create_section_with_button( - text, - button_text="View Details", - url=model_error_alert.report_url, - ) - if model_error_alert.report_url - else self.message_builder.create_text_section_block(text) - ) - details_blocks.append(section) - - if alert.test_failures: - details_blocks.append( - self.message_builder.create_text_section_block("*Test failures*") - ) - for test_failure_alert in alert.test_failures: - text = f":small_red_triangle: {test_failure_alert.summary}" - section = ( - self.message_builder.create_section_with_button( - text, - button_text="View Details", - url=test_failure_alert.report_url, - ) - if test_failure_alert.report_url - else self.message_builder.create_text_section_block(text) - ) - details_blocks.append(section) - - if alert.test_warnings: - details_blocks.append( - self.message_builder.create_text_section_block("*Test warnings*") - ) - for test_warning_alert in alert.test_warnings: - text = f":warning: {test_warning_alert.summary}" - section = ( - self.message_builder.create_section_with_button( - text, - button_text="View Details", - url=test_warning_alert.report_url, - ) - if test_warning_alert.report_url - else self.message_builder.create_text_section_block(text) - ) - details_blocks.append(section) - - if alert.test_errors: - details_blocks.append( - self.message_builder.create_text_section_block("*Test errors*") - ) - for test_error_alert in alert.test_errors: - text = f":exclamation: {test_error_alert.summary}" - section = ( - self.message_builder.create_section_with_button( - text, - button_text="View Details", - url=test_error_alert.report_url, - ) - if test_error_alert.report_url - else self.message_builder.create_text_section_block(text) - ) - details_blocks.append(section) - + details_blocks = self._get_all_in_one_sub_group_details_blocks(alert) return SlackAlertMessageSchema(title=title_blocks, details=details_blocks) @staticmethod From c3c2b5dab572df769a963f26831170f0a6c7171b Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Tue, 8 Oct 2024 20:01:47 +0300 Subject: [PATCH 15/23] grouping alerts in the base integration --- .../alerts/grouped_alerts/grouped_alert.py | 4 ++ .../alerts/integrations/base_integration.py | 64 ++++++++++++++++-- .../alerts/integrations/slack/slack.py | 45 +------------ .../alerts/integrations/teams/teams.py | 67 ++++++++++++++++++- 4 files changed, 131 insertions(+), 49 deletions(-) diff --git a/elementary/monitor/alerts/grouped_alerts/grouped_alert.py b/elementary/monitor/alerts/grouped_alerts/grouped_alert.py index f17242d0a..561d8f6de 100644 --- a/elementary/monitor/alerts/grouped_alerts/grouped_alert.py +++ b/elementary/monitor/alerts/grouped_alerts/grouped_alert.py @@ -18,6 +18,10 @@ def __init__( self.model_errors: List[ModelAlertModel] = [] self._sort_alerts() + @property + def summary(self) -> str: + return f"{len(self.alerts)} issues detected" + @property def detected_at(self) -> datetime: return min(alert.detected_at or datetime.max for alert in self.alerts) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py index 26f56eddc..719aed7a8 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py @@ -1,13 +1,19 @@ from abc import ABC, abstractmethod -from typing import Generator, List, Tuple, Union +from typing import Generator, List, Sequence, Tuple, Union -from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import ( + AllInOneAlert, + GroupedAlert, + GroupedByTableAlerts, +) from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel class BaseIntegration(ABC): + GROUP_ALERTS_THRESHOLD = 100 + def __init__(self, *args, **kwargs) -> None: self.client = self._initial_client(*args, **kwargs) @@ -22,6 +28,7 @@ def _get_alert_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], *args, **kwargs @@ -40,6 +47,8 @@ def _get_alert_template( return self._get_source_freshness_template(alert) elif isinstance(alert, GroupedByTableAlerts): return self._get_group_by_table_template(alert) + elif isinstance(alert, AllInOneAlert): + return self._get_all_in_one_template(alert) @abstractmethod def _get_dbt_test_template(self, alert: TestAlertModel, *args, **kwargs): @@ -69,6 +78,10 @@ def _get_group_by_table_template( ): raise NotImplementedError + @abstractmethod + def _get_all_in_one_template(self, alert: AllInOneAlert, *args, **kwargs): + raise NotImplementedError + @abstractmethod def _get_fallback_template( self, @@ -91,15 +104,50 @@ def send_alert( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], *args, **kwargs ) -> bool: raise NotImplementedError + def _group_alerts( + self, + alerts: Sequence[ + Union[ + TestAlertModel, + ModelAlertModel, + SourceFreshnessAlertModel, + GroupedByTableAlerts, + ] + ], + ) -> Sequence[ + Union[ + TestAlertModel, + ModelAlertModel, + SourceFreshnessAlertModel, + GroupedByTableAlerts, + AllInOneAlert, + ] + ]: + flattened_alerts: List[ + Union[TestAlertModel, ModelAlertModel, SourceFreshnessAlertModel] + ] = [] + for alert in alerts: + if isinstance(alert, GroupedAlert): + flattened_alerts.extend(alert.alerts) + else: + flattened_alerts.append(alert) + + if len(flattened_alerts) > self.GROUP_ALERTS_THRESHOLD: + return [ + AllInOneAlert(alerts=flattened_alerts), + ] + return alerts + def send_alerts( self, - alerts: List[ + alerts: Sequence[ Union[ TestAlertModel, ModelAlertModel, @@ -115,15 +163,19 @@ def send_alerts( TestAlertModel, ModelAlertModel, SourceFreshnessAlertModel, - GroupedByTableAlerts, ], bool, ], None, None, ]: - for alert in alerts: - yield alert, self.send_alert(alert) + grouped_alerts = self._group_alerts(alerts) + for grouped_alert in grouped_alerts: + if isinstance(grouped_alert, GroupedAlert): + for alert in grouped_alert.alerts: + yield alert, self.send_alert(alert, *args, **kwargs) + else: + yield grouped_alert, self.send_alert(grouped_alert, *args, **kwargs) @abstractmethod def send_test_message(self, *args, **kwargs) -> bool: diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index 225148e49..f3a7d11fb 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -1,7 +1,7 @@ import json import re from datetime import datetime, timedelta -from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union +from typing import Any, Dict, List, Optional, Sequence, Union from slack_sdk.models.blocks import SectionBlock @@ -909,7 +909,7 @@ def _add_compact_all_in_one_sub_group_details_block( ) def _get_compact_all_in_one_sub_group_details_block( - self, alert: AllInOneAlert + self, alert: AllInOneAlert, *args, **kwargs ) -> List[dict]: details_blocks: List[dict] = [] self._add_compact_all_in_one_sub_group_details_block( @@ -986,7 +986,7 @@ def _add_all_in_one_sub_group_details_block( details_blocks.append(section) def _get_all_in_one_sub_group_details_blocks( - self, alert: AllInOneAlert + self, alert: AllInOneAlert, *args, **kwargs ) -> List[dict]: details_blocks: List[dict] = [] self._add_all_in_one_sub_group_details_block( @@ -1156,45 +1156,6 @@ def send_alert( self.message_builder.reset_slack_message() return sent_successfully - def send_alerts( - self, - alerts: List[ - Union[ - TestAlertModel, - ModelAlertModel, - SourceFreshnessAlertModel, - GroupedByTableAlerts, - ] - ], - *args, - **kwargs, - ) -> Generator[ - Tuple[ - Union[ - TestAlertModel, - ModelAlertModel, - SourceFreshnessAlertModel, - GroupedByTableAlerts, - ], - bool, - ], - None, - None, - ]: - falttened_alerts = [] - for alert in alerts: - if isinstance(alert, GroupedByTableAlerts): - falttened_alerts.extend(alert.alerts) - else: - falttened_alerts.append(alert) - - if len(falttened_alerts) > self.config.group_all_alerts_threshold: - single_alert = AllInOneAlert(alerts=falttened_alerts) - sent_successfully = self.send_alert(single_alert, *args, **kwargs) - for alert in falttened_alerts: - yield alert, sent_successfully - yield from super().send_alerts(alerts, *args, **kwargs) - def send_test_message(self, channel_name: str, *args, **kwargs) -> bool: test_message = self._get_test_message_template() return self.client.send_message(channel_name=channel_name, message=test_message) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py index 7a9436fad..42eb4f168 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py @@ -7,7 +7,7 @@ from elementary.clients.teams.client import TeamsClient from elementary.config.config import Config -from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import AllInOneAlert, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -520,6 +520,69 @@ def _get_group_by_table_template( self._get_section("*Test errors*", f"{text}") ) + def _get_all_in_one_template(self, alert: AllInOneAlert, *args, **kwargs): + title = f"{self._get_display_name(alert.status)}: {alert.summary}" + + subtitle = "" + if alert.model_errors: + subtitle = ( + subtitle + + (" | " + f"😵 Model errors: {len(alert.model_errors)}") + if subtitle + else f"😵 Model errors: {len(alert.model_errors)}" + ) + if alert.test_failures: + subtitle = ( + subtitle + + (" | " + f"🔺 Test failures: {len(alert.test_failures)}") + if subtitle + else f"🔺 Test failures: {len(alert.test_failures)}" + ) + if alert.test_warnings: + subtitle = ( + subtitle + + (" | " + f"⚠ Test warnings: {len(alert.test_warnings)}") + if subtitle + else f"⚠ Test warnings: {len(alert.test_warnings)}" + ) + if alert.test_errors: + subtitle = ( + subtitle + (" | " + f"❗ Test errors: {len(alert.test_errors)}") + if subtitle + else f"❗ Test errors: {len(alert.test_errors)}" + ) + + self.message_builder.title(title) + self.message_builder.text(subtitle) + + if alert.model_errors: + rows = [alert.summary for alert in alert.model_errors] + text = "\n".join([f"🔺 {row}" for row in rows]) + self.message_builder.addSection( + self._get_section("*Model errors*", f"{text}") + ) + + if alert.test_failures: + rows = [alert.summary for alert in alert.test_failures] + text = "\n".join([f"🔺 {row}" for row in rows]) + self.message_builder.addSection( + self._get_section("*Test failures*", f"{text}") + ) + + if alert.test_warnings: + rows = [alert.summary for alert in alert.test_warnings] + text = "\n".join([f"⚠ {row}" for row in rows]) + self.message_builder.addSection( + self._get_section("*Test warnings*", f"{text}") + ) + + if alert.test_errors: + rows = [alert.summary for alert in alert.test_errors] + text = "\n".join([f"❗ {row}" for row in rows]) + self.message_builder.addSection( + self._get_section("*Test errors*", f"{text}") + ) + def _get_fallback_template( self, alert: Union[ @@ -527,6 +590,7 @@ def _get_fallback_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], *args, **kwargs, @@ -550,6 +614,7 @@ def send_alert( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, + AllInOneAlert, ], *args, **kwargs, From 020c2c08cac52f799a2230dd415b276171914f6a Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Thu, 10 Oct 2024 10:43:54 +0300 Subject: [PATCH 16/23] changed config group alerts config key name --- elementary/config/config.py | 9 +++++---- elementary/monitor/cli.py | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/elementary/config/config.py b/elementary/config/config.py index 743c7005e..dd61f863c 100644 --- a/elementary/config/config.py +++ b/elementary/config/config.py @@ -49,7 +49,7 @@ def __init__( slack_token: Optional[str] = None, slack_channel_name: Optional[str] = None, slack_group_alerts_by: Optional[str] = None, - group_all_alerts_threshold: Optional[int] = None, + group_alerts_threshold: Optional[int] = None, timezone: Optional[str] = None, aws_profile_name: Optional[str] = None, aws_region_name: Optional[str] = None, @@ -125,9 +125,10 @@ def __init__( slack_config.get("group_alerts_by"), GroupingType.BY_ALERT.value, ) - self.group_all_alerts_threshold = self._first_not_none( - group_all_alerts_threshold, - slack_config.get("group_all_alerts_threshold"), + self.group_alerts_threshold = self._first_not_none( + group_alerts_threshold, + slack_config.get("group_alerts_threshold"), + 100, ) teams_config = config.get(self._TEAMS, {}) diff --git a/elementary/monitor/cli.py b/elementary/monitor/cli.py index 78c7dcfea..4f5207966 100644 --- a/elementary/monitor/cli.py +++ b/elementary/monitor/cli.py @@ -201,10 +201,10 @@ def get_cli_properties() -> dict: help="DEPRECATED! - A slack webhook URL for sending alerts to a specific channel.", ) @click.option( - "--group-all-alerts-threshold", + "--group-alerts-threshold", type=int, default=100, - help="The threshold for grouping all alerts in a single message.", + help="The threshold for all alerts in a single message.", ) @click.option( "--timezone", @@ -282,7 +282,7 @@ def monitor( deprecated_slack_webhook, slack_token, slack_channel_name, - group_all_alerts_threshold, + group_alerts_threshold, timezone, config_dir, profiles_dir, @@ -329,7 +329,7 @@ def monitor( slack_webhook=slack_webhook, slack_token=slack_token, slack_channel_name=slack_channel_name, - group_all_alerts_threshold=group_all_alerts_threshold, + group_alerts_threshold=group_alerts_threshold, timezone=timezone, env=env, slack_group_alerts_by=group_by, From 164ac8955f17870edb2ea05ea32c95a9fffc214e Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Thu, 10 Oct 2024 10:45:06 +0300 Subject: [PATCH 17/23] passing the threshold --- .../alerts/data_monitoring_alerts.py | 2 +- .../alerts/integrations/base_integration.py | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py index 74e27e4ed..454693ca9 100644 --- a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py +++ b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py @@ -256,7 +256,7 @@ def _send_alerts( alerts_with_progress_bar = alive_it(alerts, title="Sending alerts") sent_successfully_alerts = [] for alert, sent_successfully in self.alerts_integration.send_alerts( - alerts_with_progress_bar + alerts_with_progress_bar, self.config.group_alerts_threshold ): if sent_successfully: if isinstance(alert, GroupedAlert): diff --git a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py index 719aed7a8..8fa5adb28 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py @@ -9,11 +9,12 @@ from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel +from elementary.utils.log import get_logger +logger = get_logger(__name__) -class BaseIntegration(ABC): - GROUP_ALERTS_THRESHOLD = 100 +class BaseIntegration(ABC): def __init__(self, *args, **kwargs) -> None: self.client = self._initial_client(*args, **kwargs) @@ -31,7 +32,7 @@ def _get_alert_template( AllInOneAlert, ], *args, - **kwargs + **kwargs, ): if isinstance(alert, TestAlertModel): if alert.is_elementary_test: @@ -92,7 +93,7 @@ def _get_fallback_template( GroupedByTableAlerts, ], *args, - **kwargs + **kwargs, ): raise NotImplementedError @@ -107,7 +108,7 @@ def send_alert( AllInOneAlert, ], *args, - **kwargs + **kwargs, ) -> bool: raise NotImplementedError @@ -121,6 +122,7 @@ def _group_alerts( GroupedByTableAlerts, ] ], + threshold: int, ) -> Sequence[ Union[ TestAlertModel, @@ -139,7 +141,8 @@ def _group_alerts( else: flattened_alerts.append(alert) - if len(flattened_alerts) > self.GROUP_ALERTS_THRESHOLD: + if len(flattened_alerts) >= threshold: + logger.info(f"Grouping {len(flattened_alerts)} alerts into one") return [ AllInOneAlert(alerts=flattened_alerts), ] @@ -155,8 +158,9 @@ def send_alerts( GroupedByTableAlerts, ] ], + group_alerts_threshold: int, *args, - **kwargs + **kwargs, ) -> Generator[ Tuple[ Union[ @@ -169,11 +173,12 @@ def send_alerts( None, None, ]: - grouped_alerts = self._group_alerts(alerts) + grouped_alerts = self._group_alerts(alerts, group_alerts_threshold) for grouped_alert in grouped_alerts: if isinstance(grouped_alert, GroupedAlert): + sent_successfully = self.send_alert(grouped_alert, *args, **kwargs) for alert in grouped_alert.alerts: - yield alert, self.send_alert(alert, *args, **kwargs) + yield alert, sent_successfully else: yield grouped_alert, self.send_alert(grouped_alert, *args, **kwargs) From d26d8149db660c1a94c63ce2c0eed8ea9256ad35 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Thu, 10 Oct 2024 10:49:44 +0300 Subject: [PATCH 18/23] slack - reduced number of block to avoid the slack limit --- .../alerts/integrations/slack/slack.py | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index f3a7d11fb..f7d4a2cbc 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -952,7 +952,7 @@ def _get_all_in_one_compact_template( details_blocks = self._get_compact_all_in_one_sub_group_details_block(alert) return SlackAlertMessageSchema(title=title_blocks, details=details_blocks) - def _add_all_in_one_sub_group_details_block( + def _add_all_in_one_sub_group_details_section( self, details_blocks: list, alerts: Sequence[ @@ -969,45 +969,41 @@ def _add_all_in_one_sub_group_details_block( if not alerts: return - details_blocks.append( - self.message_builder.create_text_section_block(f"*{sub_title}*") - ) + section_text_rows = [f"*{sub_title}*"] for alert in alerts: text = f":{bullet_icon}: {alert.summary}" - section = ( - self.message_builder.create_section_with_button( - text, - button_text="View Details", - url=alert.report_url, - ) - if alert.report_url - else self.message_builder.create_text_section_block(text) - ) - details_blocks.append(section) + if alert.report_url: + text = " - ".join([text, f"<{alert.get_report_link()}|View Details>"]) + section_text_rows.append(text) + + section = self.message_builder.create_text_section_block( + "\n".join(section_text_rows) + ) + details_blocks.append(section) def _get_all_in_one_sub_group_details_blocks( self, alert: AllInOneAlert, *args, **kwargs ) -> List[dict]: details_blocks: List[dict] = [] - self._add_all_in_one_sub_group_details_block( + self._add_all_in_one_sub_group_details_section( details_blocks=details_blocks, alerts=alert.model_errors, sub_title="Model Errors", bullet_icon="X", ) - self._add_all_in_one_sub_group_details_block( + self._add_all_in_one_sub_group_details_section( details_blocks=details_blocks, alerts=alert.test_failures, sub_title="Test Failures", bullet_icon="small_red_triangle", ) - self._add_all_in_one_sub_group_details_block( + self._add_all_in_one_sub_group_details_section( details_blocks=details_blocks, alerts=alert.test_warnings, sub_title="Test Warnings", bullet_icon="warning", ) - self._add_all_in_one_sub_group_details_block( + self._add_all_in_one_sub_group_details_section( details_blocks=details_blocks, alerts=alert.test_errors, sub_title="Test Errors", From 3071e16823d624452b7663d3353ce2953983785e Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Thu, 10 Oct 2024 10:50:24 +0300 Subject: [PATCH 19/23] teams - change \n to
--- .../alerts/integrations/teams/teams.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py index 42eb4f168..6569f3aa6 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py @@ -501,21 +501,21 @@ def _get_group_by_table_template( if alert.test_failures: rows = [alert.concise_name for alert in alert.test_failures] - text = "\n".join([f"🔺 {row}" for row in rows]) + text = "
".join([f"🔺 {row}" for row in rows]) self.message_builder.addSection( self._get_section("*Test failures*", f"{text}") ) if alert.test_warnings: rows = [alert.concise_name for alert in alert.test_warnings] - text = "\n".join([f"⚠ {row}" for row in rows]) + text = "
".join([f"⚠ {row}" for row in rows]) self.message_builder.addSection( self._get_section("*Test warnings*", f"{text}") ) if alert.test_errors: rows = [alert.concise_name for alert in alert.test_errors] - text = "\n".join([f"❗ {row}" for row in rows]) + text = "
".join([f"❗ {row}" for row in rows]) self.message_builder.addSection( self._get_section("*Test errors*", f"{text}") ) @@ -557,28 +557,28 @@ def _get_all_in_one_template(self, alert: AllInOneAlert, *args, **kwargs): if alert.model_errors: rows = [alert.summary for alert in alert.model_errors] - text = "\n".join([f"🔺 {row}" for row in rows]) + text = "
".join([f"🔺 {row}" for row in rows]) self.message_builder.addSection( self._get_section("*Model errors*", f"{text}") ) if alert.test_failures: rows = [alert.summary for alert in alert.test_failures] - text = "\n".join([f"🔺 {row}" for row in rows]) + text = "
".join([f"🔺 {row}" for row in rows]) self.message_builder.addSection( self._get_section("*Test failures*", f"{text}") ) if alert.test_warnings: rows = [alert.summary for alert in alert.test_warnings] - text = "\n".join([f"⚠ {row}" for row in rows]) + text = "
".join([f"⚠ {row}" for row in rows]) self.message_builder.addSection( self._get_section("*Test warnings*", f"{text}") ) if alert.test_errors: rows = [alert.summary for alert in alert.test_errors] - text = "\n".join([f"❗ {row}" for row in rows]) + text = "
".join([f"❗ {row}" for row in rows]) self.message_builder.addSection( self._get_section("*Test errors*", f"{text}") ) From 6ec78e783abad0c265e17bb4cf827bf566a0d4c2 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Thu, 10 Oct 2024 11:24:51 +0300 Subject: [PATCH 20/23] using GroupedAlert directly --- .../monitor/alerts/grouped_alerts/__init__.py | 2 - .../alerts/grouped_alerts/all_in_one.py | 7 -- .../alerts/grouped_alerts/grouping_type.py | 1 - .../alerts/integrations/base_integration.py | 20 +++--- .../alerts/integrations/slack/slack.py | 64 ++++++++----------- .../alerts/integrations/teams/teams.py | 8 +-- .../integrations/base_integration_mock.py | 4 ++ 7 files changed, 44 insertions(+), 62 deletions(-) delete mode 100644 elementary/monitor/alerts/grouped_alerts/all_in_one.py diff --git a/elementary/monitor/alerts/grouped_alerts/__init__.py b/elementary/monitor/alerts/grouped_alerts/__init__.py index aedaaab6c..6af686482 100644 --- a/elementary/monitor/alerts/grouped_alerts/__init__.py +++ b/elementary/monitor/alerts/grouped_alerts/__init__.py @@ -1,10 +1,8 @@ -from .all_in_one import AllInOneAlert from .grouped_alert import GroupedAlert from .grouped_by_table import GroupedByTableAlerts from .grouping_type import GroupingType __all__ = [ - "AllInOneAlert", "GroupedAlert", "GroupedByTableAlerts", "GroupingType", diff --git a/elementary/monitor/alerts/grouped_alerts/all_in_one.py b/elementary/monitor/alerts/grouped_alerts/all_in_one.py deleted file mode 100644 index 7a2bccaf9..000000000 --- a/elementary/monitor/alerts/grouped_alerts/all_in_one.py +++ /dev/null @@ -1,7 +0,0 @@ -from elementary.monitor.alerts.grouped_alerts.grouped_alert import GroupedAlert - - -class AllInOneAlert(GroupedAlert): - @property - def summary(self) -> str: - return f"{len(self.alerts)} issues detected" diff --git a/elementary/monitor/alerts/grouped_alerts/grouping_type.py b/elementary/monitor/alerts/grouped_alerts/grouping_type.py index 99b62b6ef..11537b806 100644 --- a/elementary/monitor/alerts/grouped_alerts/grouping_type.py +++ b/elementary/monitor/alerts/grouped_alerts/grouping_type.py @@ -4,4 +4,3 @@ class GroupingType(str, Enum): BY_ALERT = "alert" BY_TABLE = "table" - ALL_IN_ONE = "all_in_one" diff --git a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py index 8fa5adb28..1355f046c 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py @@ -1,11 +1,7 @@ from abc import ABC, abstractmethod from typing import Generator, List, Sequence, Tuple, Union -from elementary.monitor.alerts.grouped_alerts import ( - AllInOneAlert, - GroupedAlert, - GroupedByTableAlerts, -) +from elementary.monitor.alerts.grouped_alerts import GroupedAlert, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -29,7 +25,7 @@ def _get_alert_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ], *args, **kwargs, @@ -48,8 +44,8 @@ def _get_alert_template( return self._get_source_freshness_template(alert) elif isinstance(alert, GroupedByTableAlerts): return self._get_group_by_table_template(alert) - elif isinstance(alert, AllInOneAlert): - return self._get_all_in_one_template(alert) + elif isinstance(alert, GroupedAlert): + return self._get_grouped_template(alert) @abstractmethod def _get_dbt_test_template(self, alert: TestAlertModel, *args, **kwargs): @@ -80,7 +76,7 @@ def _get_group_by_table_template( raise NotImplementedError @abstractmethod - def _get_all_in_one_template(self, alert: AllInOneAlert, *args, **kwargs): + def _get_grouped_template(self, alert: GroupedAlert, *args, **kwargs): raise NotImplementedError @abstractmethod @@ -105,7 +101,7 @@ def send_alert( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ], *args, **kwargs, @@ -129,7 +125,7 @@ def _group_alerts( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ] ]: flattened_alerts: List[ @@ -144,7 +140,7 @@ def _group_alerts( if len(flattened_alerts) >= threshold: logger.info(f"Grouping {len(flattened_alerts)} alerts into one") return [ - AllInOneAlert(alerts=flattened_alerts), + GroupedAlert(alerts=flattened_alerts), ] return alerts diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index f7d4a2cbc..ae95d3c16 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -9,11 +9,7 @@ from elementary.clients.slack.schema import SlackBlocksType, SlackMessageSchema from elementary.clients.slack.slack_message_builder import MessageColor from elementary.config.config import Config -from elementary.monitor.alerts.grouped_alerts import ( - AllInOneAlert, - GroupedAlert, - GroupedByTableAlerts, -) +from elementary.monitor.alerts.grouped_alerts import GroupedAlert, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -104,18 +100,14 @@ def _get_alert_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ], *args, **kwargs, ) -> SlackMessageSchema: if self.config.is_slack_workflow: return SlackMessageSchema(text=json.dumps(alert.data, sort_keys=True)) - - if isinstance(alert, AllInOneAlert): - alert_schema = self._get_all_in_one_template(alert) - else: - alert_schema = super()._get_alert_template(alert, *args, **kwargs) + alert_schema = super()._get_alert_template(alert, *args, **kwargs) return self.message_builder.get_slack_message(alert_schema=alert_schema) def _get_dbt_test_template( @@ -886,7 +878,7 @@ def _get_group_by_table_template( title=title_blocks, preview=preview_blocks, details=details_blocks ) - def _add_compact_all_in_one_sub_group_details_block( + def _add_compact_sub_group_details_block( self, details_blocks: list, alerts: Sequence[ @@ -908,29 +900,29 @@ def _add_compact_all_in_one_sub_group_details_block( ) ) - def _get_compact_all_in_one_sub_group_details_block( - self, alert: AllInOneAlert, *args, **kwargs + def _get_compact_sub_group_details_block( + self, alert: GroupedAlert, *args, **kwargs ) -> List[dict]: details_blocks: List[dict] = [] - self._add_compact_all_in_one_sub_group_details_block( + self._add_compact_sub_group_details_block( details_blocks=details_blocks, alerts=alert.model_errors, sub_title="Model Errors", bullet_icon="X", ) - self._add_compact_all_in_one_sub_group_details_block( + self._add_compact_sub_group_details_block( details_blocks=details_blocks, alerts=alert.test_failures, sub_title="Test Failures", bullet_icon="small_red_triangle", ) - self._add_compact_all_in_one_sub_group_details_block( + self._add_compact_sub_group_details_block( details_blocks=details_blocks, alerts=alert.test_warnings, sub_title="Test Warnings", bullet_icon="warning", ) - self._add_compact_all_in_one_sub_group_details_block( + self._add_compact_sub_group_details_block( details_blocks=details_blocks, alerts=alert.test_errors, sub_title="Test Errors", @@ -938,8 +930,8 @@ def _get_compact_all_in_one_sub_group_details_block( ) return details_blocks - def _get_all_in_one_compact_template( - self, alert: AllInOneAlert + def _get_grouped_compact_template( + self, alert: GroupedAlert ) -> SlackAlertMessageSchema: self.message_builder.add_message_color(self._get_color(alert.status)) @@ -949,10 +941,10 @@ def _get_all_in_one_compact_template( ) ] - details_blocks = self._get_compact_all_in_one_sub_group_details_block(alert) + details_blocks = self._get_compact_sub_group_details_block(alert) return SlackAlertMessageSchema(title=title_blocks, details=details_blocks) - def _add_all_in_one_sub_group_details_section( + def _add_sub_group_details_block( self, details_blocks: list, alerts: Sequence[ @@ -981,29 +973,29 @@ def _add_all_in_one_sub_group_details_section( ) details_blocks.append(section) - def _get_all_in_one_sub_group_details_blocks( - self, alert: AllInOneAlert, *args, **kwargs + def _get_sub_group_details_blocks( + self, alert: GroupedAlert, *args, **kwargs ) -> List[dict]: details_blocks: List[dict] = [] - self._add_all_in_one_sub_group_details_section( + self._add_sub_group_details_block( details_blocks=details_blocks, alerts=alert.model_errors, sub_title="Model Errors", bullet_icon="X", ) - self._add_all_in_one_sub_group_details_section( + self._add_sub_group_details_block( details_blocks=details_blocks, alerts=alert.test_failures, sub_title="Test Failures", bullet_icon="small_red_triangle", ) - self._add_all_in_one_sub_group_details_section( + self._add_sub_group_details_block( details_blocks=details_blocks, alerts=alert.test_warnings, sub_title="Test Warnings", bullet_icon="warning", ) - self._add_all_in_one_sub_group_details_section( + self._add_sub_group_details_block( details_blocks=details_blocks, alerts=alert.test_errors, sub_title="Test Errors", @@ -1011,11 +1003,11 @@ def _get_all_in_one_sub_group_details_blocks( ) return details_blocks - def _get_all_in_one_template( - self, alert: AllInOneAlert, *args, **kwargs + def _get_grouped_template( + self, alert: GroupedAlert, *args, **kwargs ) -> SlackAlertMessageSchema: if len(alert.alerts) >= self.COMPACT_SCHEMA_THRESHOLD: - return self._get_all_in_one_compact_template(alert) + return self._get_grouped_compact_template(alert) self.message_builder.add_message_color(self._get_color(alert.status)) title_blocks = [ @@ -1024,7 +1016,7 @@ def _get_all_in_one_template( ), self._get_alert_type_counters_block(alert), ] - details_blocks = self._get_all_in_one_sub_group_details_blocks(alert) + details_blocks = self._get_sub_group_details_blocks(alert) return SlackAlertMessageSchema(title=title_blocks, details=details_blocks) @staticmethod @@ -1055,7 +1047,7 @@ def _get_fallback_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ], *args, **kwargs, @@ -1098,7 +1090,7 @@ def _fix_owners_and_subscribers( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ], ): if isinstance(alert, GroupedAlert): @@ -1118,7 +1110,7 @@ def send_alert( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ], *args, **kwargs, @@ -1163,7 +1155,7 @@ def _get_integration_params( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ], *args, **kwargs, diff --git a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py index 6569f3aa6..2684da524 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py @@ -7,7 +7,7 @@ from elementary.clients.teams.client import TeamsClient from elementary.config.config import Config -from elementary.monitor.alerts.grouped_alerts import AllInOneAlert, GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts import GroupedAlert, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -520,7 +520,7 @@ def _get_group_by_table_template( self._get_section("*Test errors*", f"{text}") ) - def _get_all_in_one_template(self, alert: AllInOneAlert, *args, **kwargs): + def _get_grouped_template(self, alert: GroupedAlert, *args, **kwargs): title = f"{self._get_display_name(alert.status)}: {alert.summary}" subtitle = "" @@ -590,7 +590,7 @@ def _get_fallback_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ], *args, **kwargs, @@ -614,7 +614,7 @@ def send_alert( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - AllInOneAlert, + GroupedAlert, ], *args, **kwargs, diff --git a/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py b/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py index fbc9bca25..a99742f44 100644 --- a/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py +++ b/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py @@ -1,6 +1,7 @@ from typing import Union from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.grouped_alerts.grouped_alert import GroupedAlert from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -35,6 +36,9 @@ def _get_group_by_table_template( ): return "grouped_by_table" + def _get_grouped_template(self, alert: GroupedAlert, *args, **kwargs): + return "grouped" + def _get_fallback_template( self, alert: Union[ From d0e235d1bb7f2989eaa222840f140c86ec762841 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 14 Oct 2024 12:55:49 +0300 Subject: [PATCH 21/23] Refactor grouped alerts to alerts groups --- elementary/config/config.py | 2 +- .../__init__.py | 4 +- .../alerts_group.py} | 2 +- .../grouped_by_table.py | 4 +- .../grouping_type.py | 0 .../alerts/data_monitoring_alerts.py | 12 +++--- .../alerts/integrations/base_integration.py | 30 +++++++-------- .../alerts/integrations/slack/slack.py | 38 +++++++++---------- .../alerts/integrations/teams/teams.py | 8 ++-- .../integrations/base_integration_mock.py | 4 +- 10 files changed, 52 insertions(+), 52 deletions(-) rename elementary/monitor/alerts/{grouped_alerts => alerts_groups}/__init__.py (71%) rename elementary/monitor/alerts/{grouped_alerts/grouped_alert.py => alerts_groups/alerts_group.py} (99%) rename elementary/monitor/alerts/{grouped_alerts => alerts_groups}/grouped_by_table.py (87%) rename elementary/monitor/alerts/{grouped_alerts => alerts_groups}/grouping_type.py (100%) diff --git a/elementary/config/config.py b/elementary/config/config.py index dd61f863c..efbd48a35 100644 --- a/elementary/config/config.py +++ b/elementary/config/config.py @@ -7,7 +7,7 @@ from google.auth.exceptions import DefaultCredentialsError # type: ignore[import] from elementary.exceptions.exceptions import InvalidArgumentsError -from elementary.monitor.alerts.grouped_alerts import GroupingType +from elementary.monitor.alerts.alerts_groups import GroupingType from elementary.utils.ordered_yaml import OrderedYaml diff --git a/elementary/monitor/alerts/grouped_alerts/__init__.py b/elementary/monitor/alerts/alerts_groups/__init__.py similarity index 71% rename from elementary/monitor/alerts/grouped_alerts/__init__.py rename to elementary/monitor/alerts/alerts_groups/__init__.py index 6af686482..6fb953e51 100644 --- a/elementary/monitor/alerts/grouped_alerts/__init__.py +++ b/elementary/monitor/alerts/alerts_groups/__init__.py @@ -1,9 +1,9 @@ -from .grouped_alert import GroupedAlert +from .alerts_group import AlertsGroup from .grouped_by_table import GroupedByTableAlerts from .grouping_type import GroupingType __all__ = [ - "GroupedAlert", + "AlertsGroup", "GroupedByTableAlerts", "GroupingType", ] diff --git a/elementary/monitor/alerts/grouped_alerts/grouped_alert.py b/elementary/monitor/alerts/alerts_groups/alerts_group.py similarity index 99% rename from elementary/monitor/alerts/grouped_alerts/grouped_alert.py rename to elementary/monitor/alerts/alerts_groups/alerts_group.py index 561d8f6de..4e0c3a066 100644 --- a/elementary/monitor/alerts/grouped_alerts/grouped_alert.py +++ b/elementary/monitor/alerts/alerts_groups/alerts_group.py @@ -6,7 +6,7 @@ from elementary.monitor.alerts.test_alert import TestAlertModel -class GroupedAlert: +class AlertsGroup: def __init__( self, alerts: List[Union[TestAlertModel, ModelAlertModel, SourceFreshnessAlertModel]], diff --git a/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py b/elementary/monitor/alerts/alerts_groups/grouped_by_table.py similarity index 87% rename from elementary/monitor/alerts/grouped_alerts/grouped_by_table.py rename to elementary/monitor/alerts/alerts_groups/grouped_by_table.py index 565f65e86..726aba107 100644 --- a/elementary/monitor/alerts/grouped_alerts/grouped_by_table.py +++ b/elementary/monitor/alerts/alerts_groups/grouped_by_table.py @@ -1,6 +1,6 @@ from typing import Optional -from elementary.monitor.alerts.grouped_alerts.grouped_alert import GroupedAlert +from elementary.monitor.alerts.alerts_groups.alerts_group import AlertsGroup from elementary.monitor.data_monitoring.alerts.integrations.utils.report_link import ( ReportLinkData, get_model_test_runs_link, @@ -8,7 +8,7 @@ from elementary.utils.models import get_shortened_model_name -class GroupedByTableAlerts(GroupedAlert): +class GroupedByTableAlerts(AlertsGroup): @property def model_unique_id(self) -> Optional[str]: return self.alerts[0].model_unique_id diff --git a/elementary/monitor/alerts/grouped_alerts/grouping_type.py b/elementary/monitor/alerts/alerts_groups/grouping_type.py similarity index 100% rename from elementary/monitor/alerts/grouped_alerts/grouping_type.py rename to elementary/monitor/alerts/alerts_groups/grouping_type.py diff --git a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py index 454693ca9..29d943921 100644 --- a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py +++ b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py @@ -6,8 +6,8 @@ from alive_progress import alive_it from elementary.config.config import Config -from elementary.monitor.alerts.grouped_alerts import ( - GroupedAlert, +from elementary.monitor.alerts.alerts_groups import ( + AlertsGroup, GroupedByTableAlerts, GroupingType, ) @@ -259,15 +259,15 @@ def _send_alerts( alerts_with_progress_bar, self.config.group_alerts_threshold ): if sent_successfully: - if isinstance(alert, GroupedAlert): + if isinstance(alert, AlertsGroup): sent_successfully_alerts.extend(alert.alerts) else: sent_successfully_alerts.append(alert) else: - if isinstance(alert, GroupedAlert): - for grouped_alert in alert.alerts: + if isinstance(alert, AlertsGroup): + for inner_alert in alert.alerts: logger.error( - f"Could not send the alert - {grouped_alert.id}. Full alert: {json.dumps(grouped_alert.data)}" + f"Could not send the alert - {inner_alert.id}. Full alert: {json.dumps(inner_alert.data)}" ) else: logger.error( diff --git a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py index 1355f046c..aefa149d9 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/base_integration.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Generator, List, Sequence, Tuple, Union -from elementary.monitor.alerts.grouped_alerts import GroupedAlert, GroupedByTableAlerts +from elementary.monitor.alerts.alerts_groups import AlertsGroup, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -25,7 +25,7 @@ def _get_alert_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ], *args, **kwargs, @@ -44,8 +44,8 @@ def _get_alert_template( return self._get_source_freshness_template(alert) elif isinstance(alert, GroupedByTableAlerts): return self._get_group_by_table_template(alert) - elif isinstance(alert, GroupedAlert): - return self._get_grouped_template(alert) + elif isinstance(alert, AlertsGroup): + return self._get_alerts_group_template(alert) @abstractmethod def _get_dbt_test_template(self, alert: TestAlertModel, *args, **kwargs): @@ -76,7 +76,7 @@ def _get_group_by_table_template( raise NotImplementedError @abstractmethod - def _get_grouped_template(self, alert: GroupedAlert, *args, **kwargs): + def _get_alerts_group_template(self, alert: AlertsGroup, *args, **kwargs): raise NotImplementedError @abstractmethod @@ -101,7 +101,7 @@ def send_alert( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ], *args, **kwargs, @@ -125,14 +125,14 @@ def _group_alerts( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ] ]: flattened_alerts: List[ Union[TestAlertModel, ModelAlertModel, SourceFreshnessAlertModel] ] = [] for alert in alerts: - if isinstance(alert, GroupedAlert): + if isinstance(alert, AlertsGroup): flattened_alerts.extend(alert.alerts) else: flattened_alerts.append(alert) @@ -140,7 +140,7 @@ def _group_alerts( if len(flattened_alerts) >= threshold: logger.info(f"Grouping {len(flattened_alerts)} alerts into one") return [ - GroupedAlert(alerts=flattened_alerts), + AlertsGroup(alerts=flattened_alerts), ] return alerts @@ -170,13 +170,13 @@ def send_alerts( None, ]: grouped_alerts = self._group_alerts(alerts, group_alerts_threshold) - for grouped_alert in grouped_alerts: - if isinstance(grouped_alert, GroupedAlert): - sent_successfully = self.send_alert(grouped_alert, *args, **kwargs) - for alert in grouped_alert.alerts: - yield alert, sent_successfully + for alert in grouped_alerts: + if isinstance(alert, AlertsGroup): + sent_successfully = self.send_alert(alert, *args, **kwargs) + for inner_alert in alert.alerts: + yield inner_alert, sent_successfully else: - yield grouped_alert, self.send_alert(grouped_alert, *args, **kwargs) + yield alert, self.send_alert(alert, *args, **kwargs) @abstractmethod def send_test_message(self, *args, **kwargs) -> bool: diff --git a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py index ae95d3c16..a7a6bad7c 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/slack/slack.py @@ -9,7 +9,7 @@ from elementary.clients.slack.schema import SlackBlocksType, SlackMessageSchema from elementary.clients.slack.slack_message_builder import MessageColor from elementary.config.config import Config -from elementary.monitor.alerts.grouped_alerts import GroupedAlert, GroupedByTableAlerts +from elementary.monitor.alerts.alerts_groups import AlertsGroup, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -100,7 +100,7 @@ def _get_alert_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ], *args, **kwargs, @@ -754,7 +754,7 @@ def _get_source_freshness_template( ), ) - def _get_alert_type_counters_block(self, alert: GroupedAlert) -> dict: + def _get_alert_type_counters_block(self, alert: AlertsGroup) -> dict: counters_texts: List[str] = [] if alert.model_errors: counters_texts.append(f":X: Model errors: {len(alert.model_errors)}") @@ -901,7 +901,7 @@ def _add_compact_sub_group_details_block( ) def _get_compact_sub_group_details_block( - self, alert: GroupedAlert, *args, **kwargs + self, alert: AlertsGroup, *args, **kwargs ) -> List[dict]: details_blocks: List[dict] = [] self._add_compact_sub_group_details_block( @@ -930,8 +930,8 @@ def _get_compact_sub_group_details_block( ) return details_blocks - def _get_grouped_compact_template( - self, alert: GroupedAlert + def _get_alerts_group_compact_template( + self, alert: AlertsGroup ) -> SlackAlertMessageSchema: self.message_builder.add_message_color(self._get_color(alert.status)) @@ -974,7 +974,7 @@ def _add_sub_group_details_block( details_blocks.append(section) def _get_sub_group_details_blocks( - self, alert: GroupedAlert, *args, **kwargs + self, alert: AlertsGroup, *args, **kwargs ) -> List[dict]: details_blocks: List[dict] = [] self._add_sub_group_details_block( @@ -1003,11 +1003,11 @@ def _get_sub_group_details_blocks( ) return details_blocks - def _get_grouped_template( - self, alert: GroupedAlert, *args, **kwargs + def _get_alerts_group_template( + self, alert: AlertsGroup, *args, **kwargs ) -> SlackAlertMessageSchema: if len(alert.alerts) >= self.COMPACT_SCHEMA_THRESHOLD: - return self._get_grouped_compact_template(alert) + return self._get_alerts_group_compact_template(alert) self.message_builder.add_message_color(self._get_color(alert.status)) title_blocks = [ @@ -1047,7 +1047,7 @@ def _get_fallback_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ], *args, **kwargs, @@ -1090,14 +1090,14 @@ def _fix_owners_and_subscribers( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ], ): - if isinstance(alert, GroupedAlert): - for grouped_alert in alert.alerts: - grouped_alert.owners = self._parse_emails_to_ids(grouped_alert.owners) - grouped_alert.subscribers = self._parse_emails_to_ids( - grouped_alert.subscribers + if isinstance(alert, AlertsGroup): + for inner_alert in alert.alerts: + inner_alert.owners = self._parse_emails_to_ids(inner_alert.owners) + inner_alert.subscribers = self._parse_emails_to_ids( + inner_alert.subscribers ) else: alert.owners = self._parse_emails_to_ids(alert.owners) @@ -1110,7 +1110,7 @@ def send_alert( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ], *args, **kwargs, @@ -1155,7 +1155,7 @@ def _get_integration_params( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ], *args, **kwargs, diff --git a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py index 2684da524..db6339873 100644 --- a/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py +++ b/elementary/monitor/data_monitoring/alerts/integrations/teams/teams.py @@ -7,7 +7,7 @@ from elementary.clients.teams.client import TeamsClient from elementary.config.config import Config -from elementary.monitor.alerts.grouped_alerts import GroupedAlert, GroupedByTableAlerts +from elementary.monitor.alerts.alerts_groups import AlertsGroup, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -520,7 +520,7 @@ def _get_group_by_table_template( self._get_section("*Test errors*", f"{text}") ) - def _get_grouped_template(self, alert: GroupedAlert, *args, **kwargs): + def _get_alerts_group_template(self, alert: AlertsGroup, *args, **kwargs): title = f"{self._get_display_name(alert.status)}: {alert.summary}" subtitle = "" @@ -590,7 +590,7 @@ def _get_fallback_template( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ], *args, **kwargs, @@ -614,7 +614,7 @@ def send_alert( ModelAlertModel, SourceFreshnessAlertModel, GroupedByTableAlerts, - GroupedAlert, + AlertsGroup, ], *args, **kwargs, diff --git a/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py b/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py index a99742f44..99ec98b29 100644 --- a/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py +++ b/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py @@ -1,7 +1,7 @@ from typing import Union from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts -from elementary.monitor.alerts.grouped_alerts.grouped_alert import GroupedAlert +from elementary.monitor.alerts.grouped_alerts.grouped_alert import AlertsGroup from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel @@ -36,7 +36,7 @@ def _get_group_by_table_template( ): return "grouped_by_table" - def _get_grouped_template(self, alert: GroupedAlert, *args, **kwargs): + def _get_alerts_group_template(self, alert: AlertsGroup, *args, **kwargs): return "grouped" def _get_fallback_template( From a6248397e7811a70bd83eafba70048f981aab39f Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 14 Oct 2024 12:59:07 +0300 Subject: [PATCH 22/23] Move grouping_type module --- elementary/config/config.py | 2 +- elementary/monitor/alerts/alerts_groups/__init__.py | 2 -- .../monitor/alerts/{alerts_groups => }/grouping_type.py | 0 .../data_monitoring/alerts/data_monitoring_alerts.py | 7 ++----- 4 files changed, 3 insertions(+), 8 deletions(-) rename elementary/monitor/alerts/{alerts_groups => }/grouping_type.py (100%) diff --git a/elementary/config/config.py b/elementary/config/config.py index efbd48a35..8eaa6ceba 100644 --- a/elementary/config/config.py +++ b/elementary/config/config.py @@ -7,7 +7,7 @@ from google.auth.exceptions import DefaultCredentialsError # type: ignore[import] from elementary.exceptions.exceptions import InvalidArgumentsError -from elementary.monitor.alerts.alerts_groups import GroupingType +from elementary.monitor.alerts.grouping_type import GroupingType from elementary.utils.ordered_yaml import OrderedYaml diff --git a/elementary/monitor/alerts/alerts_groups/__init__.py b/elementary/monitor/alerts/alerts_groups/__init__.py index 6fb953e51..5e759d907 100644 --- a/elementary/monitor/alerts/alerts_groups/__init__.py +++ b/elementary/monitor/alerts/alerts_groups/__init__.py @@ -1,9 +1,7 @@ from .alerts_group import AlertsGroup from .grouped_by_table import GroupedByTableAlerts -from .grouping_type import GroupingType __all__ = [ "AlertsGroup", "GroupedByTableAlerts", - "GroupingType", ] diff --git a/elementary/monitor/alerts/alerts_groups/grouping_type.py b/elementary/monitor/alerts/grouping_type.py similarity index 100% rename from elementary/monitor/alerts/alerts_groups/grouping_type.py rename to elementary/monitor/alerts/grouping_type.py diff --git a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py index 29d943921..31b93e0e0 100644 --- a/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py +++ b/elementary/monitor/data_monitoring/alerts/data_monitoring_alerts.py @@ -6,11 +6,8 @@ from alive_progress import alive_it from elementary.config.config import Config -from elementary.monitor.alerts.alerts_groups import ( - AlertsGroup, - GroupedByTableAlerts, - GroupingType, -) +from elementary.monitor.alerts.alerts_groups import AlertsGroup, GroupedByTableAlerts +from elementary.monitor.alerts.grouping_type import GroupingType from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel From a074335b6229618a51896a4a743ea74876295e2d Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 14 Oct 2024 13:57:26 +0300 Subject: [PATCH 23/23] Add const for default group alerts threshold --- elementary/config/config.py | 4 +++- elementary/monitor/cli.py | 2 +- .../data_monitoring/alerts/integrations/alerts_data_mock.py | 2 +- .../alerts/integrations/base_integration_mock.py | 3 +-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/elementary/config/config.py b/elementary/config/config.py index 8eaa6ceba..6919ef433 100644 --- a/elementary/config/config.py +++ b/elementary/config/config.py @@ -35,6 +35,8 @@ class Config: DEFAULT_TARGET_PATH = os.getcwd() + "/edr_target" + DEFAULT_GROUP_ALERTS_THRESHOLD = 100 + def __init__( self, config_dir: str = DEFAULT_CONFIG_DIR, @@ -128,7 +130,7 @@ def __init__( self.group_alerts_threshold = self._first_not_none( group_alerts_threshold, slack_config.get("group_alerts_threshold"), - 100, + self.DEFAULT_GROUP_ALERTS_THRESHOLD, ) teams_config = config.get(self._TEAMS, {}) diff --git a/elementary/monitor/cli.py b/elementary/monitor/cli.py index 4f5207966..66c69c9c4 100644 --- a/elementary/monitor/cli.py +++ b/elementary/monitor/cli.py @@ -203,7 +203,7 @@ def get_cli_properties() -> dict: @click.option( "--group-alerts-threshold", type=int, - default=100, + default=Config.DEFAULT_GROUP_ALERTS_THRESHOLD, help="The threshold for all alerts in a single message.", ) @click.option( diff --git a/tests/mocks/data_monitoring/alerts/integrations/alerts_data_mock.py b/tests/mocks/data_monitoring/alerts/integrations/alerts_data_mock.py index fa6284ade..d0774d565 100644 --- a/tests/mocks/data_monitoring/alerts/integrations/alerts_data_mock.py +++ b/tests/mocks/data_monitoring/alerts/integrations/alerts_data_mock.py @@ -1,4 +1,4 @@ -from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts +from elementary.monitor.alerts.alerts_groups import GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel diff --git a/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py b/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py index 99ec98b29..3c656f4b0 100644 --- a/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py +++ b/tests/mocks/data_monitoring/alerts/integrations/base_integration_mock.py @@ -1,7 +1,6 @@ from typing import Union -from elementary.monitor.alerts.grouped_alerts import GroupedByTableAlerts -from elementary.monitor.alerts.grouped_alerts.grouped_alert import AlertsGroup +from elementary.monitor.alerts.alerts_groups import AlertsGroup, GroupedByTableAlerts from elementary.monitor.alerts.model_alert import ModelAlertModel from elementary.monitor.alerts.source_freshness_alert import SourceFreshnessAlertModel from elementary.monitor.alerts.test_alert import TestAlertModel