From 51812eec0344429989880de3930664458a5d86b1 Mon Sep 17 00:00:00 2001
From: Steve Simpson <steve.simpson@grafana.com>
Date: Wed, 27 Nov 2024 16:04:51 +0100
Subject: [PATCH] Add timeout option for webhook notifier.

---
 config/notifiers.go                      |  3 ++
 notify/webhook/webhook.go                |  6 +++
 test/with_api_v2/acceptance/send_test.go | 52 ++++++++++++++++++++++++
 3 files changed, 61 insertions(+)

diff --git a/config/notifiers.go b/config/notifiers.go
index fe28ca05c4..3ff062c1ad 100644
--- a/config/notifiers.go
+++ b/config/notifiers.go
@@ -535,6 +535,9 @@ type WebhookConfig struct {
 	// Alerts exceeding this threshold will be truncated. Setting this to 0
 	// allows an unlimited number of alerts.
 	MaxAlerts uint64 `yaml:"max_alerts" json:"max_alerts"`
+
+	// Timeout is the maximum time allowed to invoke the webhook.
+	Timeout *time.Duration `yaml:"timeout" json:"timeout"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
diff --git a/notify/webhook/webhook.go b/notify/webhook/webhook.go
index 1b5546ddbe..8df6eebd3f 100644
--- a/notify/webhook/webhook.go
+++ b/notify/webhook/webhook.go
@@ -112,6 +112,12 @@ func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, er
 		url = strings.TrimSpace(string(content))
 	}
 
+	if n.conf.Timeout != nil {
+		postCtx, cancel := context.WithTimeoutCause(ctx, *n.conf.Timeout, fmt.Errorf("configured webhook timeout (%s) reached", *n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
 	resp, err := notify.PostJSON(ctx, n.client, url, &buf)
 	if err != nil {
 		return true, notify.RedactURL(err)
diff --git a/test/with_api_v2/acceptance/send_test.go b/test/with_api_v2/acceptance/send_test.go
index 4e13d37ca5..41c44e89ab 100644
--- a/test/with_api_v2/acceptance/send_test.go
+++ b/test/with_api_v2/acceptance/send_test.go
@@ -464,3 +464,55 @@ receivers:
 
 	t.Log(co.Check())
 }
+
+func TestWebhookTimeout(t *testing.T) {
+	t.Parallel()
+
+	conf := `
+route:
+  receiver: "default"
+  group_by: [alertname]
+  group_wait:      1s
+  group_interval:  10s
+  repeat_interval: 1m
+
+receivers:
+- name: "default"
+  webhook_configs:
+  - url: 'http://%s'
+    timeout: 100ms
+`
+
+	at := NewAcceptanceTest(t, &AcceptanceOpts{
+		Tolerance: 150 * time.Millisecond,
+	})
+
+	co := at.Collector("webhook")
+	wh := NewWebhook(t, co)
+
+	count := 0
+
+	wh.Func = func(ts float64) bool {
+		// Make the first webhook request slow enough to hit
+		// the webhook timeout, but not so slow as to hit the
+		// dispatcher timeout.
+		if count < 1 {
+			time.Sleep(500 * time.Millisecond)
+		}
+		count++
+		return false
+	}
+
+	am := at.AlertmanagerCluster(fmt.Sprintf(conf, wh.Address()), 1)
+
+	am.Push(At(1), Alert("alertname", "test1"))
+
+	// First alert will be considered a failure due to timeout, and retried.
+	co.Want(Between(2, 3), Alert("alertname", "test1").Active(1))
+	// Second attempt will not be delayed, so successful.
+	co.Want(Between(2.5, 3.5), Alert("alertname", "test1").Active(1))
+
+	at.Run()
+
+	t.Log(co.Check())
+}