feat: alert recover notify (#801)

traas-stack · Feb 20, 2024 · 9b0857e · 9b0857e
1 parent 81c0b2d
commit 9b0857e
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 47 deletions.
diff --git a/...me/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotify.java b/...me/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotify.java
@@ -60,6 +60,8 @@ public class AlertNotify {
 
   private Boolean isRecover;
 
+  private boolean recoverNotify;
+
   private List<SubscriptionInfo> subscriptionInfos;
 
   private List<UserInfo> userInfos;
@@ -110,8 +112,11 @@ public static AlertNotify eventInfoConvert(EventInfo eventInfo, InspectConfig in
     try {
       BeanUtils.copyProperties(inspectConfig, alertNotify);
       BeanUtils.copyProperties(eventInfo, alertNotify);
+      if (inspectConfig.getRecover() != null) {
+        alertNotify.setRecoverNotify(inspectConfig.getRecover());
+      }
+      alertNotify.setIsPql(inspectConfig.getIsPql() != null && inspectConfig.getIsPql());
       if (!eventInfo.getIsRecover()) {
-        alertNotify.setIsPql(inspectConfig.getIsPql() != null && inspectConfig.getIsPql());
         Map<Trigger, List<NotifyDataInfo>> notifyDataInfoMap = new HashMap<>();
         eventInfo.getAlarmTriggerResults().forEach((trigger, resultList) -> {
           List<NotifyDataInfo> notifyDataInfos = new ArrayList<>();
@@ -137,13 +142,13 @@ public static AlertNotify eventInfoConvert(EventInfo eventInfo, InspectConfig in
         }
         alertNotify.setNotifyDataInfos(notifyDataInfoMap);
         alertNotify.setAggregationNum(notifyDataInfoMap.size());
-        alertNotify.setEnvType(eventInfo.getEnvType());
-        // 对于平台消费侧，可能需要知道完整的告警规则
-        alertNotify.setRuleConfig(inspectConfig);
-        tryFixAlertLevel(alertNotify, eventInfo.getAlarmTriggerResults());
-        alertNotify.setAlertServer(AddressUtil.getLocalHostName());
-        alertNotify.setAlertIp(AddressUtil.getHostAddress());
       }
+      alertNotify.setEnvType(eventInfo.getEnvType());
+      // 对于平台消费侧，可能需要知道完整的告警规则
+      alertNotify.setRuleConfig(inspectConfig);
+      tryFixAlertLevel(alertNotify, eventInfo.getAlarmTriggerResults());
+      alertNotify.setAlertServer(AddressUtil.getLocalHostName());
+      alertNotify.setAlertIp(AddressUtil.getHostAddress());
     } catch (Exception e) {
       RecordSucOrFailNotify.alertNotifyProcessFail("event convert alert notify exception" + e,
           "alert task compute", "event convert alert notify", alertNotify.getAlertNotifyRecord());
@@ -182,4 +187,18 @@ public static List<TemplateValue> convertAlertNotify(AlertNotify alertNotify) {
     templateValues.add(templateValue);
     return templateValues;
   }
+
+  public boolean notifyRecover() {
+    if (isRecover == null) {
+      return false;
+    }
+    return isRecover && recoverNotify;
+  }
+
+  public boolean nonNotifyRecover() {
+    if (isRecover == null) {
+      return false;
+    }
+    return isRecover && !recoverNotify;
+  }
 }
diff --git a/...-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotifyRequest.java b/...-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotifyRequest.java
@@ -79,4 +79,6 @@ public class AlertNotifyRequest {
 
   private AlertNotifyRecordDTO alertNotifyRecord;
 
+  private boolean notifyRecover;
+
 }
diff --git a/.../home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/AlertNotifyHandler.java b/.../home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/AlertNotifyHandler.java
@@ -57,7 +57,7 @@ public void handle(List<AlertNotify> alertNotifies) {
     for (AlertNotify alertNotify : alertNotifies) {
       try {
         LOGGER.info("{} alert notify handle begin.", alertNotify.getTraceId());
-        if (alertNotify.getIsRecover() != null && alertNotify.getIsRecover()) {
+        if (alertNotify.nonNotifyRecover()) {
           LOGGER.info("{} alert notify is recover.", alertNotify.getTraceId());
           continue;
         }
@@ -121,6 +121,7 @@ private void sendPlugin(AlertNotify alertNotify, AlertNotifyRecordLatch recordLa
     alertNotifyRequest.setWorkspace(getWorkspace(alertNotify));
     alertNotifyRequest.setLogAnalysis(alertNotify.getLogAnalysis());
     alertNotifyRequest.setLogSample(alertNotify.getLogSample());
+    alertNotifyRequest.setNotifyRecover(alertNotify.notifyRecover());
 
     alertNotifyRequest.setAlertNotifyRecord(alertNotify.getAlertNotifyRecord());
 

diff --git a/...home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GatewayService.java b/...home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GatewayService.java
@@ -73,14 +73,14 @@ public abstract class GatewayService {
 
   public boolean sendAlertNotifyV3(AlertNotifyRequest notify, AlertNotifyRecordLatch recordLatch) {
     String traceId = notify.getTraceId();
-    LOGGER.info("{} receive_alarm_notify_request at {}", traceId,
-        this.environmentProperties.getDeploymentSite());
+    LOGGER.info("{} receive_alarm_notify_request at {}, recover notify {}", traceId,
+        this.environmentProperties.getDeploymentSite(), notify.isNotifyRecover());
 
     NotifyChain defaultNotifyChain = new NotifyChain(this.scheduleQueue);
 
     String tenant = notify.getTenant();
     String type = defaultNotifyChain.name;
-    if (CollectionUtils.isEmpty(notify.getNotifyDataInfos())) {
+    if (!notify.isNotifyRecover() && CollectionUtils.isEmpty(notify.getNotifyDataInfos())) {
       LOGGER.info("{} notify data info is empty.", traceId);
       RecordSucOrFailNotify.alertNotifyProcessFail(traceId + ": notify data info is empty ",
           GATEWAY, "check notify data info", notify.getAlertNotifyRecord());
@@ -183,22 +183,28 @@ public boolean sendAlertNotifyV3(AlertNotifyRequest notify, AlertNotifyRecordLat
    * @return
    */
   private List<Map<String, Object>> convertToInput(AlertNotifyRequest notifyRequest) {
+    List<Map<String, Object>> res = new ArrayList<>();
     Map<Trigger, List<NotifyDataInfo>> notifyDataInfoMap = notifyRequest.getNotifyDataInfos();
     if (CollectionUtils.isEmpty(notifyDataInfoMap)) {
-      return Collections.emptyList();
-    }
-    List<Map<String, Object>> res = new ArrayList<>();
-    for (Map.Entry<Trigger, List<NotifyDataInfo>> entry : notifyDataInfoMap.entrySet()) {
-      for (NotifyDataInfo notifyDataInfo : entry.getValue()) {
-        Map<String, Object> item = new HashMap<>();
-        item.put("alarmTime", notifyRequest.getAlarmTime());
-        item.put("alarmLevel", notifyRequest.getAlarmLevel());
-        item.put("alarmName", notifyRequest.getRuleName());
-        item.put("tenant", notifyRequest.getTenant());
-        item.put("triggerContent", notifyDataInfo.getTriggerContent());
-        item.put("currentValue", notifyDataInfo.getCurrentValue());
-        item.put("tags", notifyDataInfo.getTags());
-        res.add(item);
+      Map<String, Object> item = new HashMap<>();
+      item.put("alarmTime", notifyRequest.getAlarmTime());
+      item.put("alarmLevel", notifyRequest.getAlarmLevel());
+      item.put("alarmName", notifyRequest.getRuleName());
+      item.put("tenant", notifyRequest.getTenant());
+      res.add(item);
+    } else {
+      for (Map.Entry<Trigger, List<NotifyDataInfo>> entry : notifyDataInfoMap.entrySet()) {
+        for (NotifyDataInfo notifyDataInfo : entry.getValue()) {
+          Map<String, Object> item = new HashMap<>();
+          item.put("alarmTime", notifyRequest.getAlarmTime());
+          item.put("alarmLevel", notifyRequest.getAlarmLevel());
+          item.put("alarmName", notifyRequest.getRuleName());
+          item.put("tenant", notifyRequest.getTenant());
+          item.put("triggerContent", notifyDataInfo.getTriggerContent());
+          item.put("currentValue", notifyDataInfo.getCurrentValue());
+          item.put("tags", notifyDataInfo.getTags());
+          res.add(item);
+        }
       }
     }
 

diff --git a/...e-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GetSubscriptionHandler.java b/...e-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GetSubscriptionHandler.java
@@ -104,7 +104,8 @@ public void handle(List<AlertNotify> alertNotifies) {
               alertNotify.getUniqueId(), G.get().toJson(alertSubscribeList));
 
           InspectConfig inspectConfig = alertNotify.getRuleConfig();
-          if (keepSilence(inspectConfig.getAlertSilenceConfig(), alertNotify.getAlarmTime())) {
+          if (keepSilence(alertNotify.notifyRecover(), inspectConfig.getAlertSilenceConfig(),
+              alertNotify.getAlarmTime())) {
             LOGGER.info("{} keep silence {}.", alertNotify.getTraceId(),
                 J.toJson(inspectConfig.getAlertSilenceConfig()));
             return;
@@ -230,7 +231,11 @@ public void handle(List<AlertNotify> alertNotifies) {
     }
   }
 
-  private boolean keepSilence(AlertSilenceConfig alertSilenceConfig, Long alarmTime) {
+  private boolean keepSilence(boolean notifyRecover, AlertSilenceConfig alertSilenceConfig,
+      Long alarmTime) {
+    if (notifyRecover) {
+      return false;
+    }
     return alertSilenceConfig != null
         && !StringUtils.equals(alertSilenceConfig.getSilenceMode(), "default")
         && !alertSilenceConfig.needShoot(alarmTime);
@@ -246,9 +251,12 @@ private void handleBlock(List<AlertNotify> alertNotifies) {
       AlertNotify alertNotify = iterator.next();
 
       if (alertNotify.getIsRecover()) {
-        iterator.remove();
-        LOGGER.info("{} alert rule {} has recovered.", alertNotify.getTraceId(),
-            alertNotify.getUniqueId());
+        if (!alertNotify.isRecoverNotify()) {
+          // 告警屏蔽对告警恢复通知无效
+          iterator.remove();
+          LOGGER.info("{} alert rule {} has recovered.", alertNotify.getTraceId(),
+              alertNotify.getUniqueId());
+        }
         continue;
       }
 
@@ -337,6 +345,9 @@ private void handleBlock(List<AlertNotify> alertNotifies) {
 
   private void addGlobalWebhook(AlertNotify alertNotify,
       Map<String, List<AlarmWebhook>> alertWebhookMap) {
+    if (alertNotify.getIsRecover()) {
+      return;
+    }
     List<WebhookInfo> webhookInfos = new ArrayList<>();
     List<AlarmWebhook> alertWebhookList = alertWebhookMap.get(alertNotify.getTenant());
     if (!CollectionUtils.isEmpty(alertWebhookList)) {

diff --git a/...ome-alert/src/main/java/io/holoinsight/server/home/alert/service/converter/DoConvert.java b/...ome-alert/src/main/java/io/holoinsight/server/home/alert/service/converter/DoConvert.java
@@ -119,11 +119,4 @@ public static AlarmHistory alertHistoryConverter(AlertNotify alertNotify) {
     return alarmHistory;
   }
 
-  public static AlertNotify eventInfoConverter(AlertmanagerWebhook alertmanagerWebhook) {
-    AlertNotify alertNotify = new AlertNotify();
-    BeanUtils.copyProperties(alertmanagerWebhook, alertNotify);
-    // alarmNotify.setMembers(alertmanagerWebhook.getSucribeInfoListOrBuilderList().stream().map(AlarmWebhook.SubcribeInfoOrBuilder::getNoticeId).collect(Collectors.toList()));
-    return alertNotify;
-  }
-
 }
diff --git a/...ome/home-facade/src/main/java/io/holoinsight/server/home/facade/NotificationTemplate.java b/...ome/home-facade/src/main/java/io/holoinsight/server/home/facade/NotificationTemplate.java
@@ -214,7 +214,9 @@ private String getValue(TemplateValue templateValue, AlertTemplateField field,
 
   private String buildTagValue(String alarmTags, boolean tagMarkdown) {
     if (tagMarkdown) {
-      if (alarmTags.startsWith("[")) {
+      if (StringUtils.isEmpty(alarmTags)) {
+        return StringUtils.EMPTY;
+      } else if (alarmTags.startsWith("[")) {
         return alarmTags;
       }
       Map<String /* tagk */, String /* alias */> reverseTagMap = reverseTagMap();
@@ -293,7 +295,7 @@ public boolean parseText() {
         Matcher matcher = pattern.matcher(fieldName);
         if (matcher.find()) {
           fieldName = matcher.group(1); // group(1) corresponds to (.*?), which is the content
-                                        // within ${}
+          // within ${}
         }
         if (fieldName.startsWith("ALERT_SCOPE.") || fieldName.startsWith("alarmTags.")) {
           tagMap.put(alias, fieldName.split("\\.", 2)[1]);

diff --git a/...a/io/holoinsight/server/home/web/security/custom/AlarmDingDingRobotFacadeImplChecker.java b/...a/io/holoinsight/server/home/web/security/custom/AlarmDingDingRobotFacadeImplChecker.java
@@ -197,10 +197,8 @@ private boolean checkUserIds(String extra) {
   boolean checkIdExists(Long id, String tenant, String workspace) {
     QueryWrapper<AlarmDingDingRobot> queryWrapper = new QueryWrapper<>();
     queryWrapper.eq("id", id);
-    queryWrapper.eq("tenant", tenant);
-    if (StringUtils.isNotEmpty(workspace)) {
-      queryWrapper.eq("workspace", workspace);
-    }
+    this.requestContextAdapter.queryWrapperTenantAdapt(queryWrapper, tenant, workspace);
+
     List<AlarmDingDingRobot> exist = this.alarmDingDingRobotMapper.selectList(queryWrapper);
     if (CollectionUtils.isEmpty(exist)) {
       log.error("fail to check id for no existed {} {} {}", id, tenant, workspace);

diff --git a/...in/java/io/holoinsight/server/home/web/security/custom/AlarmHistoryFacadeImplChecker.java b/...in/java/io/holoinsight/server/home/web/security/custom/AlarmHistoryFacadeImplChecker.java
@@ -6,6 +6,7 @@
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.google.common.reflect.TypeToken;
 import io.holoinsight.server.common.J;
+import io.holoinsight.server.home.common.service.RequestContextAdapter;
 import io.holoinsight.server.home.common.util.scope.MonitorScope;
 import io.holoinsight.server.home.common.util.scope.RequestContext;
 import io.holoinsight.server.home.dal.mapper.AlarmHistoryMapper;
@@ -16,6 +17,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.aopalliance.intercept.MethodInvocation;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
@@ -32,6 +34,8 @@ public class AlarmHistoryFacadeImplChecker extends AbstractResourceChecker {
 
   @Resource
   private AlarmHistoryMapper alarmHistoryMapper;
+  @Autowired
+  private RequestContextAdapter requestContextAdapter;
 
   @Override
   public boolean check(LevelAuthorizationMetaData levelAuthMetaData,
@@ -63,10 +67,8 @@ private boolean checkParameters(String methodName, List<String> parameters, Stri
   boolean checkIdExists(Long id, String tenant, String workspace) {
     QueryWrapper<AlarmHistory> queryWrapper = new QueryWrapper<>();
     queryWrapper.eq("id", id);
-    queryWrapper.eq("tenant", tenant);
-    if (StringUtils.isNotEmpty(workspace)) {
-      queryWrapper.eq("workspace", workspace);
-    }
+    this.requestContextAdapter.queryWrapperTenantAdapt(queryWrapper, tenant, workspace);
+
     List<AlarmHistory> exist = this.alarmHistoryMapper.selectList(queryWrapper);
     if (CollectionUtils.isEmpty(exist)) {
       log.error("fail to check id for no existed {} {} {}", id, tenant, workspace);