feat: track the processing of alerts (#586)

traas-stack · Aug 7, 2023 · b3be283 · b3be283
1 parent 186c17e
commit b3be283
Show file tree

Hide file tree

Showing 34 changed files with 1,384 additions and 329 deletions.
diff --git a/...mmon/common-service/src/main/java/io/holoinsight/server/common/service/FuseProtector.java b/...mmon/common-service/src/main/java/io/holoinsight/server/common/service/FuseProtector.java
diff --git a/.../common-service/src/test/java/io/holoinsight/server/common/service/FuseProtectorTest.java b/.../common-service/src/test/java/io/holoinsight/server/common/service/FuseProtectorTest.java
diff --git a/...n-flyway/src/main/resources/db/migration/V15__230718_CREATE_alert_notify_record_TABLE.sql b/...n-flyway/src/main/resources/db/migration/V15__230718_CREATE_alert_notify_record_TABLE.sql
@@ -0,0 +1,22 @@
+CREATE TABLE `alert_notify_record` (
+                                       `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT '主键id',
+                                       `history_id` BIGINT DEFAULT NULL COMMENT '历史id',
+                                       `history_detail_id` BIGINT DEFAULT NULL COMMENT '历史详情id',
+                                       `unique_id` VARCHAR ( 64 ) DEFAULT NULL,
+                                       `rule_name` VARCHAR ( 255 ) DEFAULT NULL COMMENT '告警规则名称',
+                                       `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP comment '创建时间',
+                                       `gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP comment '修改时间',
+                                       `notify_error_time` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '告警异常时间',
+                                       `is_success` TINYINT NOT NULL COMMENT '告警是否成功',
+                                       `notify_channel` VARCHAR ( 255 ) DEFAULT NULL COMMENT '通知渠道',
+                                       `notify_error_node` LONGTEXT DEFAULT NULL COMMENT '通知异常节点',
+                                       `notify_user` TEXT DEFAULT NULL COMMENT '通知人信息',
+                                       `tenant` VARCHAR ( 64 ) DEFAULT NULL COMMENT '租户',
+                                       `extra` LONGTEXT DEFAULT NULL COMMENT '额外信息',
+                                       `trigger_result` LONGTEXT DEFAULT NULL COMMENT '告警计算信息',
+                                       `env_type` VARCHAR ( 255 ) DEFAULT NULL COMMENT '运行环境',
+                                       `workspace` VARCHAR ( 64 ) DEFAULT NULL,
+                                       PRIMARY KEY ( `id` ),
+                                       INDEX ( `history_detail_id` ),
+                                       INDEX ( `history_id` ),
+                                       INDEX ( `unique_id` )) ENGINE = INNODB COMMENT = '告警通知记录';
diff --git a/...lert/src/main/java/io/holoinsight/server/home/alert/model/compute/ComputeTaskPackage.java b/...lert/src/main/java/io/holoinsight/server/home/alert/model/compute/ComputeTaskPackage.java
@@ -3,6 +3,8 @@
  */
 package io.holoinsight.server.home.alert.model.compute;
 
+import io.holoinsight.server.home.dal.model.AlertNotifyRecord;
+import io.holoinsight.server.home.facade.AlertNotifyRecordDTO;
 import io.holoinsight.server.home.facade.InspectConfig;
 import lombok.Data;
 
@@ -34,4 +36,9 @@ public class ComputeTaskPackage {
 
   // 后续可以考虑聚合数据源查询
 
+  /**
+   * 告警通知记录
+   */
+  public AlertNotifyRecordDTO alertNotifyRecord;
+
 }
diff --git a/...me/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotify.java b/...me/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotify.java
@@ -4,6 +4,9 @@
 package io.holoinsight.server.home.alert.model.event;
 
 import io.holoinsight.server.common.AddressUtil;
+import io.holoinsight.server.common.J;
+import io.holoinsight.server.home.alert.service.event.RecordSucOrFailNotify;
+import io.holoinsight.server.home.facade.AlertNotifyRecordDTO;
 import io.holoinsight.server.home.facade.InspectConfig;
 import io.holoinsight.server.home.facade.PqlRule;
 import io.holoinsight.server.home.facade.TemplateValue;
@@ -23,6 +26,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * @author wangsiyuan
@@ -94,32 +98,53 @@ public class AlertNotify {
 
   private List<String> logSample;
 
-  public static AlertNotify eventInfoConvert(EventInfo eventInfo, InspectConfig inspectConfig) {
+  private AlertNotifyRecordDTO alertNotifyRecord;
+
+  private boolean alertRecord;
+
+  public static AlertNotify eventInfoConvert(EventInfo eventInfo, InspectConfig inspectConfig,
+      List<AlertNotifyRecordDTO> alertNotifyRecordDTOS) {
     AlertNotify alertNotify = new AlertNotify();
-    BeanUtils.copyProperties(inspectConfig, alertNotify);
-    BeanUtils.copyProperties(eventInfo, alertNotify);
-    if (!eventInfo.getIsRecover()) {
-      alertNotify.setIsPql(inspectConfig.getIsPql() != null && inspectConfig.getIsPql());
-      Map<Trigger, List<NotifyDataInfo>> notifyDataInfoMap = new HashMap<>();
-      eventInfo.getAlarmTriggerResults().forEach((trigger, resultList) -> {
-        List<NotifyDataInfo> notifyDataInfos = new ArrayList<>();
-        resultList.forEach(result -> {
-          NotifyDataInfo notifyDataInfo = new NotifyDataInfo();
-          notifyDataInfo.setMetric(result.getMetric());
-          notifyDataInfo.setTags(result.getTags());
-          notifyDataInfo.setCurrentValue(result.getCurrentValue());
-          notifyDataInfo.setTriggerContent(result.getTriggerContent());
-          notifyDataInfos.add(notifyDataInfo);
+    try {
+      BeanUtils.copyProperties(inspectConfig, alertNotify);
+      BeanUtils.copyProperties(eventInfo, alertNotify);
+      if (!eventInfo.getIsRecover()) {
+        alertNotify.setIsPql(inspectConfig.getIsPql() != null && inspectConfig.getIsPql());
+        Map<Trigger, List<NotifyDataInfo>> notifyDataInfoMap = new HashMap<>();
+        eventInfo.getAlarmTriggerResults().forEach((trigger, resultList) -> {
+          List<NotifyDataInfo> notifyDataInfos = new ArrayList<>();
+          resultList.forEach(result -> {
+            NotifyDataInfo notifyDataInfo = new NotifyDataInfo();
+            notifyDataInfo.setMetric(result.getMetric());
+            notifyDataInfo.setTags(result.getTags());
+            notifyDataInfo.setCurrentValue(result.getCurrentValue());
+            notifyDataInfo.setTriggerContent(result.getTriggerContent());
+            notifyDataInfos.add(notifyDataInfo);
+          });
+          notifyDataInfoMap.put(trigger, notifyDataInfos);
         });
-        notifyDataInfoMap.put(trigger, notifyDataInfos);
-      });
-      alertNotify.setNotifyDataInfos(notifyDataInfoMap);
-      alertNotify.setAggregationNum(notifyDataInfoMap.size());
-      alertNotify.setEnvType(eventInfo.getEnvType());
-      // 对于平台消费侧，可能需要知道完整的告警规则
-      alertNotify.setRuleConfig(inspectConfig);
-      tryFixAlertLevel(alertNotify, eventInfo.getAlarmTriggerResults());
-      alertNotify.setAlertServer(AddressUtil.getHostAddress());
+        // add alert trigger result for record
+        if (!CollectionUtils.isEmpty(alertNotifyRecordDTOS)) {
+          for (AlertNotifyRecordDTO alertNotifyRecord : alertNotifyRecordDTOS) {
+            if (Objects.equals(alertNotifyRecord.getUniqueId(), alertNotify.getUniqueId())) {
+              alertNotifyRecord.setTriggerResult(J.toJson(notifyDataInfoMap.values()));
+              alertNotify.setAlertNotifyRecord(alertNotifyRecord);
+              break;
+            }
+          }
+        }
+        alertNotify.setNotifyDataInfos(notifyDataInfoMap);
+        alertNotify.setAggregationNum(notifyDataInfoMap.size());
+        alertNotify.setEnvType(eventInfo.getEnvType());
+        // 对于平台消费侧，可能需要知道完整的告警规则
+        alertNotify.setRuleConfig(inspectConfig);
+        tryFixAlertLevel(alertNotify, eventInfo.getAlarmTriggerResults());
+        alertNotify.setAlertServer(AddressUtil.getHostAddress());
+      }
+    } catch (Exception e) {
+      RecordSucOrFailNotify.alertNotifyProcess("event convert alert notify exception" + e,
+          "alert task compute", "event convert alert notify", alertNotify.getAlertNotifyRecord());
+      throw e;
     }
     return alertNotify;
   }