feat(notifier): add selfstate controller

cmd/notifier/config.go

@@ -10,6 +10,7 @@ import (
 	"github.com/moira-alert/moira/cmd"
 	"github.com/moira-alert/moira/notifier"
 	"github.com/moira-alert/moira/notifier/selfstate"
+	"github.com/moira-alert/moira/notifier/selfstate/controller"
 	"github.com/moira-alert/moira/notifier/selfstate/heartbeat"
 	"github.com/moira-alert/moira/notifier/selfstate/monitor"
 )
@@ -178,10 +179,18 @@ type monitorConfig struct {
 	UserCfg  userMonitorConfig  `yaml:"user"`
 }
 
+// controllerConfig defines the selfstate controller configuration.
+type controllerConfig struct {
+	Enabled        bool             `yaml:"enabled"`
+	HearbeatersCfg heartbeatsConfig `yaml:"heartbeaters"`
+	CheckInterval  string           `yaml:"check_interval"`
+}
+
 // selfstateConfig defines the configuration of the selfstate worker.
 type selfstateConfig struct {
-	Enabled    bool          `yaml:"enabled"`
-	MonitorCfg monitorConfig `yaml:"monitor"`
+	Enabled       bool             `yaml:"enabled"`
+	MonitorCfg    monitorConfig    `yaml:"monitor"`
+	ControllerCfg controllerConfig `yaml:"controller"`
 }
 
 func (cfg *selfstateConfig) getSettings() selfstate.Config {
@@ -206,6 +215,11 @@ func (cfg *selfstateConfig) getSettings() selfstate.Config {
 				},
 			},
 		},
+		ControllerCfg: controller.ControllerConfig{
+			Enabled:         cfg.ControllerCfg.Enabled,
+			HeartbeatersCfg: cfg.ControllerCfg.HearbeatersCfg.getSettings(),
+			CheckInterval:   to.Duration(cfg.ControllerCfg.CheckInterval),
+		},
 	}
 }
 
@@ -279,6 +293,10 @@ func getDefault() config {
 						},
 					},
 				},
+				ControllerCfg: controllerConfig{
+					Enabled:       false,
+					CheckInterval: "10s",
+				},
 			},
 			FrontURI:                      "http://localhost",
 			Timezone:                      "UTC",

cmd/notifier/main.go

@@ -118,11 +118,12 @@ func main() {
 	}
 
 	selfstateCfg := config.Notifier.Selfstate.getSettings()
+	heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(telemetry.Metrics)
 
 	// Start moira selfstate checker
 	if selfstateCfg.Enabled {
 		logger.Info().Msg("Selfstate enabled")
-		selfstateWorker, err := selfstate.NewSelfstateWorker(selfstateCfg, logger, database, sender, systemClock)
+		selfstateWorker, err := selfstate.NewSelfstateWorker(selfstateCfg, logger, database, sender, systemClock, heartbeatMetrics)
 		if err != nil {
 			logger.Fatal().
 				Error(err).

generate_mocks.sh

@@ -20,6 +20,7 @@ mockgen -destination=mock/metric_source/source.go  -package=mock_metric_source g
 mockgen -destination=mock/metric_source/fetch_result.go -package=mock_metric_source github.com/moira-alert/moira/metric_source FetchResult
 mockgen -destination=mock/heartbeat/heartbeat.go -package=mock_heartbeat github.com/moira-alert/moira/notifier/selfstate/heartbeat Heartbeater
 mockgen -destination=mock/monitor/monitor.go -package=mock_monitor github.com/moira-alert/moira/notifier/selfstate/monitor Monitor
+mockgen -destination=mock/controller/controller.go -package=mock_controller github.com/moira-alert/moira/notifier/selfstate/controller Controller
 mockgen -destination=mock/clock/clock.go -package=mock_clock github.com/moira-alert/moira Clock
 mockgen -destination=mock/notifier/mattermost/client.go -package=mock_mattermost github.com/moira-alert/moira/senders/mattermost Client
 

local/api.yml

@@ -59,11 +59,17 @@ web:
       label: MS Teams
     - type: mattermost
       label: Mattermost
+    - type: webhook
+      label: Webhook
   feature_flags:
     is_plotting_available: true
     is_plotting_default_on: true
     is_subscription_to_all_tags_available: true
     is_readonly_enabled: false
+  emergency_contact_types:
+    - heartbeat_local_checker
+    - heartbeat_notifier
+    - heartbeat_database
 notification_history:
   ttl: 48h
   query_limit: 10000

local/notifier.yml

@@ -45,13 +45,13 @@ notifier:
             need_turn_off_notifier: true
             alert:
               name: Database Problems
-            redis_disconect_delay: 60s
+            redis_disconnect_delay: 10s
           notifier:
             enabled: true
             need_turn_off_notifier: true
             alert:
               name: Notifier Problems
-        notice_interval: 30s
+        notice_interval: 20s
         check_interval: 5s
         contacts:
             - type: webhook
@@ -64,14 +64,29 @@ notifier:
             need_turn_off_notifier: true
             alert:
               name: Database Problems
-            redis_disconect_delay: 60s
+            redis_disconnect_delay: 10s
           notifier:
             enabled: true
             need_turn_off_notifier: true
             alert:
               name: Notifier Problems
-        notice_interval: 10s
+        notice_interval: 20s
         check_interval: 5s
+    controller:
+      enabled: true
+      heartbeaters:
+          database:
+            enabled: true
+            need_turn_off_notifier: true
+            redis_disconnect_delay: 10s
+          local_checker:
+            enabled: true
+            need_turn_off_notifier: true
+            last_check_delay: 10s
+          notifier:
+            enabled: true
+            need_turn_off_notifier: true
+      check_interval: 5s
   front_uri: http://localhost
   timezone: UTC
   date_time_format: "15:04 02.01.2006"

notifier/selfstate/config.go

@@ -1,6 +1,7 @@
 package selfstate
 
 import (
+	"github.com/moira-alert/moira/notifier/selfstate/controller"
 	"github.com/moira-alert/moira/notifier/selfstate/monitor"
 )
 
@@ -12,6 +13,7 @@ type MonitorConfig struct {
 
 // Config sets the configuration of the selfstate worker.
 type Config struct {
-	Enabled    bool
-	MonitorCfg MonitorConfig
+	Enabled       bool
+	MonitorCfg    MonitorConfig
+	ControllerCfg controller.ControllerConfig
 }