Update limit for GardenerFailureRateTooHighOrMissing to prevent false…

… alarms (#1023) * Update limit for GardenerFailureRateTooHighOrMissing * Fix parentheses typo
m-lab · Dec 8, 2023 · a713d6d · a713d6d
1 parent 983e373
commit a713d6d
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/config/federation/prometheus/alerts.yml b/config/federation/prometheus/alerts.yml
@@ -879,9 +879,8 @@ groups:
 # GardenerFailureRateTooHigh fires when the number of failed Gardener jobs
 # in the last day rises above 1%.
   - alert: GardenerFailureRateTooHighOrMissing
-    expr: (sum(rate(gardener_jobs_total{status!="success"}[1d])) by (experiment, datatype) /
-      sum(rate(gardener_jobs_total[1d])) by (experiment, datatype)) > 0.01
-    for: 10m
+    expr: sum(increase(gardener_jobs_total{status!="success"}[1d])) by (experiment, datatype) > 3
+    for: 24h
     labels:
       repo: dev-tracker
       severity: ticket