From 9b0857e208a9ce59abab61ce9ebd72dbd63da045 Mon Sep 17 00:00:00 2001 From: masaimu Date: Tue, 20 Feb 2024 18:27:25 +0800 Subject: [PATCH] feat: alert recover notify (#801) --- .../home/alert/model/event/AlertNotify.java | 33 +++++++++++---- .../alert/model/event/AlertNotifyRequest.java | 2 + .../home/alert/plugin/AlertNotifyHandler.java | 3 +- .../home/alert/plugin/GatewayService.java | 40 +++++++++++-------- .../alert/plugin/GetSubscriptionHandler.java | 21 +++++++--- .../alert/service/converter/DoConvert.java | 7 ---- .../home/facade/NotificationTemplate.java | 6 ++- .../AlarmDingDingRobotFacadeImplChecker.java | 6 +-- .../custom/AlarmHistoryFacadeImplChecker.java | 10 +++-- 9 files changed, 81 insertions(+), 47 deletions(-) diff --git a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotify.java b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotify.java index 26265e78e..7ee15c9dc 100644 --- a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotify.java +++ b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotify.java @@ -60,6 +60,8 @@ public class AlertNotify { private Boolean isRecover; + private boolean recoverNotify; + private List subscriptionInfos; private List userInfos; @@ -110,8 +112,11 @@ public static AlertNotify eventInfoConvert(EventInfo eventInfo, InspectConfig in try { BeanUtils.copyProperties(inspectConfig, alertNotify); BeanUtils.copyProperties(eventInfo, alertNotify); + if (inspectConfig.getRecover() != null) { + alertNotify.setRecoverNotify(inspectConfig.getRecover()); + } + alertNotify.setIsPql(inspectConfig.getIsPql() != null && inspectConfig.getIsPql()); if (!eventInfo.getIsRecover()) { - alertNotify.setIsPql(inspectConfig.getIsPql() != null && inspectConfig.getIsPql()); Map> notifyDataInfoMap = new HashMap<>(); eventInfo.getAlarmTriggerResults().forEach((trigger, resultList) -> { List notifyDataInfos = new ArrayList<>(); @@ -137,13 +142,13 @@ public static AlertNotify eventInfoConvert(EventInfo eventInfo, InspectConfig in } alertNotify.setNotifyDataInfos(notifyDataInfoMap); alertNotify.setAggregationNum(notifyDataInfoMap.size()); - alertNotify.setEnvType(eventInfo.getEnvType()); - // 对于平台消费侧，可能需要知道完整的告警规则 - alertNotify.setRuleConfig(inspectConfig); - tryFixAlertLevel(alertNotify, eventInfo.getAlarmTriggerResults()); - alertNotify.setAlertServer(AddressUtil.getLocalHostName()); - alertNotify.setAlertIp(AddressUtil.getHostAddress()); } + alertNotify.setEnvType(eventInfo.getEnvType()); + // 对于平台消费侧，可能需要知道完整的告警规则 + alertNotify.setRuleConfig(inspectConfig); + tryFixAlertLevel(alertNotify, eventInfo.getAlarmTriggerResults()); + alertNotify.setAlertServer(AddressUtil.getLocalHostName()); + alertNotify.setAlertIp(AddressUtil.getHostAddress()); } catch (Exception e) { RecordSucOrFailNotify.alertNotifyProcessFail("event convert alert notify exception" + e, "alert task compute", "event convert alert notify", alertNotify.getAlertNotifyRecord()); @@ -182,4 +187,18 @@ public static List convertAlertNotify(AlertNotify alertNotify) { templateValues.add(templateValue); return templateValues; } + + public boolean notifyRecover() { + if (isRecover == null) { + return false; + } + return isRecover && recoverNotify; + } + + public boolean nonNotifyRecover() { + if (isRecover == null) { + return false; + } + return isRecover && !recoverNotify; + } } diff --git a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotifyRequest.java b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotifyRequest.java index 450a341e5..b967d608f 100644 --- a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotifyRequest.java +++ b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/model/event/AlertNotifyRequest.java @@ -79,4 +79,6 @@ public class AlertNotifyRequest { private AlertNotifyRecordDTO alertNotifyRecord; + private boolean notifyRecover; + } diff --git a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/AlertNotifyHandler.java b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/AlertNotifyHandler.java index 64b4821a8..29a7476b2 100644 --- a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/AlertNotifyHandler.java +++ b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/AlertNotifyHandler.java @@ -57,7 +57,7 @@ public void handle(List alertNotifies) { for (AlertNotify alertNotify : alertNotifies) { try { LOGGER.info("{} alert notify handle begin.", alertNotify.getTraceId()); - if (alertNotify.getIsRecover() != null && alertNotify.getIsRecover()) { + if (alertNotify.nonNotifyRecover()) { LOGGER.info("{} alert notify is recover.", alertNotify.getTraceId()); continue; } @@ -121,6 +121,7 @@ private void sendPlugin(AlertNotify alertNotify, AlertNotifyRecordLatch recordLa alertNotifyRequest.setWorkspace(getWorkspace(alertNotify)); alertNotifyRequest.setLogAnalysis(alertNotify.getLogAnalysis()); alertNotifyRequest.setLogSample(alertNotify.getLogSample()); + alertNotifyRequest.setNotifyRecover(alertNotify.notifyRecover()); alertNotifyRequest.setAlertNotifyRecord(alertNotify.getAlertNotifyRecord()); diff --git a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GatewayService.java b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GatewayService.java index 0abe5747b..ee2f89179 100644 --- a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GatewayService.java +++ b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GatewayService.java @@ -73,14 +73,14 @@ public abstract class GatewayService { public boolean sendAlertNotifyV3(AlertNotifyRequest notify, AlertNotifyRecordLatch recordLatch) { String traceId = notify.getTraceId(); - LOGGER.info("{} receive_alarm_notify_request at {}", traceId, - this.environmentProperties.getDeploymentSite()); + LOGGER.info("{} receive_alarm_notify_request at {}, recover notify {}", traceId, + this.environmentProperties.getDeploymentSite(), notify.isNotifyRecover()); NotifyChain defaultNotifyChain = new NotifyChain(this.scheduleQueue); String tenant = notify.getTenant(); String type = defaultNotifyChain.name; - if (CollectionUtils.isEmpty(notify.getNotifyDataInfos())) { + if (!notify.isNotifyRecover() && CollectionUtils.isEmpty(notify.getNotifyDataInfos())) { LOGGER.info("{} notify data info is empty.", traceId); RecordSucOrFailNotify.alertNotifyProcessFail(traceId + ": notify data info is empty ", GATEWAY, "check notify data info", notify.getAlertNotifyRecord()); @@ -183,22 +183,28 @@ public boolean sendAlertNotifyV3(AlertNotifyRequest notify, AlertNotifyRecordLat * @return */ private List> convertToInput(AlertNotifyRequest notifyRequest) { + List> res = new ArrayList<>(); Map> notifyDataInfoMap = notifyRequest.getNotifyDataInfos(); if (CollectionUtils.isEmpty(notifyDataInfoMap)) { - return Collections.emptyList(); - } - List> res = new ArrayList<>(); - for (Map.Entry> entry : notifyDataInfoMap.entrySet()) { - for (NotifyDataInfo notifyDataInfo : entry.getValue()) { - Map item = new HashMap<>(); - item.put("alarmTime", notifyRequest.getAlarmTime()); - item.put("alarmLevel", notifyRequest.getAlarmLevel()); - item.put("alarmName", notifyRequest.getRuleName()); - item.put("tenant", notifyRequest.getTenant()); - item.put("triggerContent", notifyDataInfo.getTriggerContent()); - item.put("currentValue", notifyDataInfo.getCurrentValue()); - item.put("tags", notifyDataInfo.getTags()); - res.add(item); + Map item = new HashMap<>(); + item.put("alarmTime", notifyRequest.getAlarmTime()); + item.put("alarmLevel", notifyRequest.getAlarmLevel()); + item.put("alarmName", notifyRequest.getRuleName()); + item.put("tenant", notifyRequest.getTenant()); + res.add(item); + } else { + for (Map.Entry> entry : notifyDataInfoMap.entrySet()) { + for (NotifyDataInfo notifyDataInfo : entry.getValue()) { + Map item = new HashMap<>(); + item.put("alarmTime", notifyRequest.getAlarmTime()); + item.put("alarmLevel", notifyRequest.getAlarmLevel()); + item.put("alarmName", notifyRequest.getRuleName()); + item.put("tenant", notifyRequest.getTenant()); + item.put("triggerContent", notifyDataInfo.getTriggerContent()); + item.put("currentValue", notifyDataInfo.getCurrentValue()); + item.put("tags", notifyDataInfo.getTags()); + res.add(item); + } } } diff --git a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GetSubscriptionHandler.java b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GetSubscriptionHandler.java index 9d1356ac0..28b147d3e 100644 --- a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GetSubscriptionHandler.java +++ b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/plugin/GetSubscriptionHandler.java @@ -104,7 +104,8 @@ public void handle(List alertNotifies) { alertNotify.getUniqueId(), G.get().toJson(alertSubscribeList)); InspectConfig inspectConfig = alertNotify.getRuleConfig(); - if (keepSilence(inspectConfig.getAlertSilenceConfig(), alertNotify.getAlarmTime())) { + if (keepSilence(alertNotify.notifyRecover(), inspectConfig.getAlertSilenceConfig(), + alertNotify.getAlarmTime())) { LOGGER.info("{} keep silence {}.", alertNotify.getTraceId(), J.toJson(inspectConfig.getAlertSilenceConfig())); return; @@ -230,7 +231,11 @@ public void handle(List alertNotifies) { } } - private boolean keepSilence(AlertSilenceConfig alertSilenceConfig, Long alarmTime) { + private boolean keepSilence(boolean notifyRecover, AlertSilenceConfig alertSilenceConfig, + Long alarmTime) { + if (notifyRecover) { + return false; + } return alertSilenceConfig != null && !StringUtils.equals(alertSilenceConfig.getSilenceMode(), "default") && !alertSilenceConfig.needShoot(alarmTime); @@ -246,9 +251,12 @@ private void handleBlock(List alertNotifies) { AlertNotify alertNotify = iterator.next(); if (alertNotify.getIsRecover()) { - iterator.remove(); - LOGGER.info("{} alert rule {} has recovered.", alertNotify.getTraceId(), - alertNotify.getUniqueId()); + if (!alertNotify.isRecoverNotify()) { + // 告警屏蔽对告警恢复通知无效 + iterator.remove(); + LOGGER.info("{} alert rule {} has recovered.", alertNotify.getTraceId(), + alertNotify.getUniqueId()); + } continue; } @@ -337,6 +345,9 @@ private void handleBlock(List alertNotifies) { private void addGlobalWebhook(AlertNotify alertNotify, Map> alertWebhookMap) { + if (alertNotify.getIsRecover()) { + return; + } List webhookInfos = new ArrayList<>(); List alertWebhookList = alertWebhookMap.get(alertNotify.getTenant()); if (!CollectionUtils.isEmpty(alertWebhookList)) { diff --git a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/service/converter/DoConvert.java b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/service/converter/DoConvert.java index 1c3b167a9..f60380cb0 100644 --- a/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/service/converter/DoConvert.java +++ b/server/home/home-alert/src/main/java/io/holoinsight/server/home/alert/service/converter/DoConvert.java @@ -119,11 +119,4 @@ public static AlarmHistory alertHistoryConverter(AlertNotify alertNotify) { return alarmHistory; } - public static AlertNotify eventInfoConverter(AlertmanagerWebhook alertmanagerWebhook) { - AlertNotify alertNotify = new AlertNotify(); - BeanUtils.copyProperties(alertmanagerWebhook, alertNotify); - // alarmNotify.setMembers(alertmanagerWebhook.getSucribeInfoListOrBuilderList().stream().map(AlarmWebhook.SubcribeInfoOrBuilder::getNoticeId).collect(Collectors.toList())); - return alertNotify; - } - } diff --git a/server/home/home-facade/src/main/java/io/holoinsight/server/home/facade/NotificationTemplate.java b/server/home/home-facade/src/main/java/io/holoinsight/server/home/facade/NotificationTemplate.java index 601ce8690..639d77b17 100644 --- a/server/home/home-facade/src/main/java/io/holoinsight/server/home/facade/NotificationTemplate.java +++ b/server/home/home-facade/src/main/java/io/holoinsight/server/home/facade/NotificationTemplate.java @@ -214,7 +214,9 @@ private String getValue(TemplateValue templateValue, AlertTemplateField field, private String buildTagValue(String alarmTags, boolean tagMarkdown) { if (tagMarkdown) { - if (alarmTags.startsWith("[")) { + if (StringUtils.isEmpty(alarmTags)) { + return StringUtils.EMPTY; + } else if (alarmTags.startsWith("[")) { return alarmTags; } Map reverseTagMap = reverseTagMap(); @@ -293,7 +295,7 @@ public boolean parseText() { Matcher matcher = pattern.matcher(fieldName); if (matcher.find()) { fieldName = matcher.group(1); // group(1) corresponds to (.*?), which is the content - // within ${} + // within ${} } if (fieldName.startsWith("ALERT_SCOPE.") || fieldName.startsWith("alarmTags.")) { tagMap.put(alias, fieldName.split("\\.", 2)[1]); diff --git a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/security/custom/AlarmDingDingRobotFacadeImplChecker.java b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/security/custom/AlarmDingDingRobotFacadeImplChecker.java index fff89078d..2fdacf6df 100644 --- a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/security/custom/AlarmDingDingRobotFacadeImplChecker.java +++ b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/security/custom/AlarmDingDingRobotFacadeImplChecker.java @@ -197,10 +197,8 @@ private boolean checkUserIds(String extra) { boolean checkIdExists(Long id, String tenant, String workspace) { QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.eq("id", id); - queryWrapper.eq("tenant", tenant); - if (StringUtils.isNotEmpty(workspace)) { - queryWrapper.eq("workspace", workspace); - } + this.requestContextAdapter.queryWrapperTenantAdapt(queryWrapper, tenant, workspace); + List exist = this.alarmDingDingRobotMapper.selectList(queryWrapper); if (CollectionUtils.isEmpty(exist)) { log.error("fail to check id for no existed {} {} {}", id, tenant, workspace); diff --git a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/security/custom/AlarmHistoryFacadeImplChecker.java b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/security/custom/AlarmHistoryFacadeImplChecker.java index b84db5493..2f88d0409 100644 --- a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/security/custom/AlarmHistoryFacadeImplChecker.java +++ b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/security/custom/AlarmHistoryFacadeImplChecker.java @@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.google.common.reflect.TypeToken; import io.holoinsight.server.common.J; +import io.holoinsight.server.home.common.service.RequestContextAdapter; import io.holoinsight.server.home.common.util.scope.MonitorScope; import io.holoinsight.server.home.common.util.scope.RequestContext; import io.holoinsight.server.home.dal.mapper.AlarmHistoryMapper; @@ -16,6 +17,7 @@ import lombok.extern.slf4j.Slf4j; import org.aopalliance.intercept.MethodInvocation; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -32,6 +34,8 @@ public class AlarmHistoryFacadeImplChecker extends AbstractResourceChecker { @Resource private AlarmHistoryMapper alarmHistoryMapper; + @Autowired + private RequestContextAdapter requestContextAdapter; @Override public boolean check(LevelAuthorizationMetaData levelAuthMetaData, @@ -63,10 +67,8 @@ private boolean checkParameters(String methodName, List parameters, Stri boolean checkIdExists(Long id, String tenant, String workspace) { QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.eq("id", id); - queryWrapper.eq("tenant", tenant); - if (StringUtils.isNotEmpty(workspace)) { - queryWrapper.eq("workspace", workspace); - } + this.requestContextAdapter.queryWrapperTenantAdapt(queryWrapper, tenant, workspace); + List exist = this.alarmHistoryMapper.selectList(queryWrapper); if (CollectionUtils.isEmpty(exist)) { log.error("fail to check id for no existed {} {} {}", id, tenant, workspace);