Don't send notification on pending-resolved alert transitions

gimlet-io · Nov 24, 2023 · a50b1d3 · a50b1d3
1 parent e9c18df
commit a50b1d3
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 10 deletions.
diff --git a/pkg/dashboard/alert/alertStateManager.go b/pkg/dashboard/alert/alertStateManager.go
@@ -184,17 +184,20 @@ func (a AlertStateManager) TrackPod(pod *api.Pod, repoName string, envName strin
 	for _, nonResolvedAlert := range nonResolvedAlerts {
 		t := ThresholdByType(a.thresholds, nonResolvedAlert.Type)
 		if t != nil && t.Resolved(dbPod) {
+			previousState := nonResolvedAlert.Status
 			nonResolvedAlert.SetResolved()
 			err := a.store.UpdateAlertState(nonResolvedAlert)
 			if err != nil {
 				logrus.Errorf("couldn't set resolved state for alerts: %s", err)
 			}
 
 			apiAlert := api.NewAlert(nonResolvedAlert, t.Text())
-			a.notifManager.Broadcast(&notifications.AlertMessage{
-				Alert:       *apiAlert,
-				ImChannelId: pod.ImChannelId,
-			})
+			if previousState == model.FIRING { // don't notify people about pending then resolved alerts
+				a.notifManager.Broadcast(&notifications.AlertMessage{
+					Alert:       *apiAlert,
+					ImChannelId: pod.ImChannelId,
+				})
+			}
 			a.broadcast(apiAlert, streaming.AlertResolvedEventString)
 		}
 	}
@@ -217,17 +220,20 @@ func (a AlertStateManager) DeletePod(podName string) error {
 	}
 
 	for _, nonResolvedAlert := range nonResolvedAlerts {
+		previousState := nonResolvedAlert.Status
 		nonResolvedAlert.SetResolved()
 		err := a.store.UpdateAlertState(nonResolvedAlert)
 		if err != nil {
 			logrus.Errorf("couldn't set resolved state for alerts: %s", err)
 		}
 
 		apiAlert := api.NewAlert(nonResolvedAlert, "")
-		a.notifManager.Broadcast(&notifications.AlertMessage{
-			Alert:       *apiAlert,
-			ImChannelId: nonResolvedAlert.ImChannelId,
-		})
+		if previousState == model.FIRING { // don't notify people about pending then resolved alerts
+			a.notifManager.Broadcast(&notifications.AlertMessage{
+				Alert:       *apiAlert,
+				ImChannelId: nonResolvedAlert.ImChannelId,
+			})
+		}
 		a.broadcast(apiAlert, streaming.AlertResolvedEventString)
 	}
 

diff --git a/pkg/dashboard/notifications/alertMessage.go b/pkg/dashboard/notifications/alertMessage.go
@@ -2,6 +2,7 @@ package notifications
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/bwmarrin/discordgo"
 	"github.com/gimlet-io/gimlet-cli/pkg/dashboard/api"
@@ -20,7 +21,7 @@ func (am *AlertMessage) AsSlackMessage() (*slackMessage, error) {
 		Attachments: []Attachment{},
 	}
 
-	msg.Text = fmt.Sprintf("Pod %s %s", am.Alert.ObjectName, am.Alert.Status)
+	msg.Text = fmt.Sprintf("%s ALERT %s for pod %s", strings.Title(am.Alert.Type), strings.ToUpper(am.Alert.Status), am.Alert.ObjectName)
 	if am.Alert.Status == model.RESOLVED {
 		msg.Attachments = append(msg.Attachments,
 			Attachment{

diff --git a/pkg/dashboard/notifications/gitopsDeployMessage.go b/pkg/dashboard/notifications/gitopsDeployMessage.go
@@ -56,7 +56,7 @@ func (gm *gitopsDeployMessage) AsSlackMessage() (*slackMessage, error) {
 		if gm.event.TriggeredBy == "policy" {
 			msg.Text = fmt.Sprintf("Policy based rollout of %s on %s", gm.event.Manifest.App, gm.event.Artifact.Version.RepositoryName)
 		} else {
-			msg.Text = fmt.Sprintf("%s is rolling out %s on %s", gm.event.TriggeredBy, gm.event.Manifest.App, gm.event.Artifact.Version.RepositoryName)
+			msg.Text = fmt.Sprintf("%s is rolling out %s on %s", gm.event.TriggeredBy, gm.event.Manifest.App, gm.event.Manifest.Env)
 		}
 		msg.Blocks = append(msg.Blocks,
 			Block{