diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index ed2f3b902..d82c419e5 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -10,6 +10,7 @@ import ( "github.com/moira-alert/moira/cmd" "github.com/moira-alert/moira/notifier" "github.com/moira-alert/moira/notifier/selfstate" + "github.com/moira-alert/moira/notifier/selfstate/controller" "github.com/moira-alert/moira/notifier/selfstate/heartbeat" "github.com/moira-alert/moira/notifier/selfstate/monitor" ) @@ -178,10 +179,18 @@ type monitorConfig struct { UserCfg userMonitorConfig `yaml:"user"` } +// controllerConfig defines the selfstate controller configuration. +type controllerConfig struct { + Enabled bool `yaml:"enabled"` + HearbeatersCfg heartbeatsConfig `yaml:"heartbeaters"` + CheckInterval string `yaml:"check_interval"` +} + // selfstateConfig defines the configuration of the selfstate worker. type selfstateConfig struct { - Enabled bool `yaml:"enabled"` - MonitorCfg monitorConfig `yaml:"monitor"` + Enabled bool `yaml:"enabled"` + MonitorCfg monitorConfig `yaml:"monitor"` + ControllerCfg controllerConfig `yaml:"controller"` } func (cfg *selfstateConfig) getSettings() selfstate.Config { @@ -206,6 +215,11 @@ func (cfg *selfstateConfig) getSettings() selfstate.Config { }, }, }, + ControllerCfg: controller.ControllerConfig{ + Enabled: cfg.ControllerCfg.Enabled, + HeartbeatersCfg: cfg.ControllerCfg.HearbeatersCfg.getSettings(), + CheckInterval: to.Duration(cfg.ControllerCfg.CheckInterval), + }, } } @@ -279,6 +293,10 @@ func getDefault() config { }, }, }, + ControllerCfg: controllerConfig{ + Enabled: false, + CheckInterval: "10s", + }, }, FrontURI: "http://localhost", Timezone: "UTC", diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index 9a3ab5e59..905b3aec4 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -118,11 +118,12 @@ func main() { } selfstateCfg := config.Notifier.Selfstate.getSettings() + heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(telemetry.Metrics) // Start moira selfstate checker if selfstateCfg.Enabled { logger.Info().Msg("Selfstate enabled") - selfstateWorker, err := selfstate.NewSelfstateWorker(selfstateCfg, logger, database, sender, systemClock) + selfstateWorker, err := selfstate.NewSelfstateWorker(selfstateCfg, logger, database, sender, systemClock, heartbeatMetrics) if err != nil { logger.Fatal(). Error(err). diff --git a/generate_mocks.sh b/generate_mocks.sh index 7074c02cb..4a457bf44 100755 --- a/generate_mocks.sh +++ b/generate_mocks.sh @@ -20,6 +20,7 @@ mockgen -destination=mock/metric_source/source.go -package=mock_metric_source g mockgen -destination=mock/metric_source/fetch_result.go -package=mock_metric_source github.com/moira-alert/moira/metric_source FetchResult mockgen -destination=mock/heartbeat/heartbeat.go -package=mock_heartbeat github.com/moira-alert/moira/notifier/selfstate/heartbeat Heartbeater mockgen -destination=mock/monitor/monitor.go -package=mock_monitor github.com/moira-alert/moira/notifier/selfstate/monitor Monitor +mockgen -destination=mock/controller/controller.go -package=mock_controller github.com/moira-alert/moira/notifier/selfstate/controller Controller mockgen -destination=mock/clock/clock.go -package=mock_clock github.com/moira-alert/moira Clock mockgen -destination=mock/notifier/mattermost/client.go -package=mock_mattermost github.com/moira-alert/moira/senders/mattermost Client diff --git a/local/api.yml b/local/api.yml index 631240d4e..28d8f0e21 100644 --- a/local/api.yml +++ b/local/api.yml @@ -59,11 +59,17 @@ web: label: MS Teams - type: mattermost label: Mattermost + - type: webhook + label: Webhook feature_flags: is_plotting_available: true is_plotting_default_on: true is_subscription_to_all_tags_available: true is_readonly_enabled: false + emergency_contact_types: + - heartbeat_local_checker + - heartbeat_notifier + - heartbeat_database notification_history: ttl: 48h query_limit: 10000 diff --git a/local/notifier.yml b/local/notifier.yml index e7097b4cd..151f7df4e 100644 --- a/local/notifier.yml +++ b/local/notifier.yml @@ -45,13 +45,13 @@ notifier: need_turn_off_notifier: true alert: name: Database Problems - redis_disconect_delay: 60s + redis_disconnect_delay: 10s notifier: enabled: true need_turn_off_notifier: true alert: name: Notifier Problems - notice_interval: 30s + notice_interval: 20s check_interval: 5s contacts: - type: webhook @@ -64,14 +64,29 @@ notifier: need_turn_off_notifier: true alert: name: Database Problems - redis_disconect_delay: 60s + redis_disconnect_delay: 10s notifier: enabled: true need_turn_off_notifier: true alert: name: Notifier Problems - notice_interval: 10s + notice_interval: 20s check_interval: 5s + controller: + enabled: true + heartbeaters: + database: + enabled: true + need_turn_off_notifier: true + redis_disconnect_delay: 10s + local_checker: + enabled: true + need_turn_off_notifier: true + last_check_delay: 10s + notifier: + enabled: true + need_turn_off_notifier: true + check_interval: 5s front_uri: http://localhost timezone: UTC date_time_format: "15:04 02.01.2006" diff --git a/mock/controller/controller.go b/mock/controller/controller.go new file mode 100644 index 000000000..16c632522 --- /dev/null +++ b/mock/controller/controller.go @@ -0,0 +1,65 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/moira-alert/moira/notifier/selfstate/controller (interfaces: Controller) +// +// Generated by this command: +// +// mockgen -destination=mock/controller/controller.go -package=mock_controller github.com/moira-alert/moira/notifier/selfstate/controller Controller +// + +// Package mock_controller is a generated GoMock package. +package mock_controller + +import ( + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockController is a mock of Controller interface. +type MockController struct { + ctrl *gomock.Controller + recorder *MockControllerMockRecorder +} + +// MockControllerMockRecorder is the mock recorder for MockController. +type MockControllerMockRecorder struct { + mock *MockController +} + +// NewMockController creates a new mock instance. +func NewMockController(ctrl *gomock.Controller) *MockController { + mock := &MockController{ctrl: ctrl} + mock.recorder = &MockControllerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockController) EXPECT() *MockControllerMockRecorder { + return m.recorder +} + +// Start mocks base method. +func (m *MockController) Start() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Start") +} + +// Start indicates an expected call of Start. +func (mr *MockControllerMockRecorder) Start() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockController)(nil).Start)) +} + +// Stop mocks base method. +func (m *MockController) Stop() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop") + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *MockControllerMockRecorder) Stop() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockController)(nil).Stop)) +} diff --git a/notifier/selfstate/config.go b/notifier/selfstate/config.go index 095bb62f3..5fb5bd4e3 100644 --- a/notifier/selfstate/config.go +++ b/notifier/selfstate/config.go @@ -1,6 +1,7 @@ package selfstate import ( + "github.com/moira-alert/moira/notifier/selfstate/controller" "github.com/moira-alert/moira/notifier/selfstate/monitor" ) @@ -12,6 +13,7 @@ type MonitorConfig struct { // Config sets the configuration of the selfstate worker. type Config struct { - Enabled bool - MonitorCfg MonitorConfig + Enabled bool + MonitorCfg MonitorConfig + ControllerCfg controller.ControllerConfig } diff --git a/notifier/selfstate/controller/controller.go b/notifier/selfstate/controller/controller.go new file mode 100644 index 000000000..7b0f60ebd --- /dev/null +++ b/notifier/selfstate/controller/controller.go @@ -0,0 +1,213 @@ +package controller + +import ( + "fmt" + "time" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" + "github.com/moira-alert/moira/metrics" + "github.com/moira-alert/moira/notifier/selfstate/heartbeat" + w "github.com/moira-alert/moira/worker" + "gopkg.in/tomb.v2" +) + +// Verify that controller matches the Controller interface. +var _ Controller = (*controller)(nil) + +const ( + name = "Moira Selfstate Controller" + lockName = "moira-selfstate-controller" + lockTTL = 15 * time.Second +) + +// Controller interface that defines the methods of the selfstate controller. +type Controller interface { + Start() + Stop() error +} + +// ControllerConfig defines the selfstate controller configuration. +type ControllerConfig struct { + Enabled bool + HeartbeatersCfg heartbeat.HeartbeatersConfig `validate:"required_if=Enabled true"` + CheckInterval time.Duration `validate:"required_if=Enabled true,gte=0"` +} + +type controller struct { + cfg ControllerConfig + tomb tomb.Tomb + logger moira.Logger + database moira.Database + heartbeaters []heartbeat.Heartbeater + heartbeatMetrics *metrics.HeartBeatMetrics +} + +// NewController is a function to create a new selfstate controller. +func NewController( + cfg ControllerConfig, + logger moira.Logger, + database moira.Database, + clock moira.Clock, + heartbeatMetrics *metrics.HeartBeatMetrics, +) (*controller, error) { + if err := moira.ValidateStruct(cfg); err != nil { + return nil, fmt.Errorf("controller configuration error: %w", err) + } + + heartbeaters := createHeartbeaters(cfg.HeartbeatersCfg, logger, database, clock) + + return &controller{ + cfg: cfg, + logger: logger, + database: database, + heartbeaters: heartbeaters, + heartbeatMetrics: heartbeatMetrics, + }, nil +} + +func createHeartbeaters( //nolint:gocyclo + heartbeatersCfg heartbeat.HeartbeatersConfig, + logger moira.Logger, + database moira.Database, + clock moira.Clock, +) []heartbeat.Heartbeater { + heartbeaters := make([]heartbeat.Heartbeater, 0) + + if heartbeatersCfg.DatabaseCfg.Enabled && heartbeatersCfg.DatabaseCfg.NeedTurnOffNotifier { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) + databaseHeartbeater, err := heartbeat.NewDatabaseHeartbeater(heartbeatersCfg.DatabaseCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(databaseHeartbeater.Type())). + Msg("Failed to create a new database heartbeater") + } else { + heartbeaters = append(heartbeaters, databaseHeartbeater) + } + } + + if heartbeatersCfg.FilterCfg.Enabled && heartbeatersCfg.FilterCfg.NeedTurnOffNotifier { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) + filterHeartbeater, err := heartbeat.NewFilterHeartbeater(heartbeatersCfg.FilterCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(filterHeartbeater.Type())). + Msg("Failed to create a new filter heartbeater") + } else { + heartbeaters = append(heartbeaters, filterHeartbeater) + } + } + + if heartbeatersCfg.LocalCheckerCfg.Enabled && heartbeatersCfg.LocalCheckerCfg.NeedTurnOffNotifier { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) + localCheckerHeartbeater, err := heartbeat.NewLocalCheckerHeartbeater(heartbeatersCfg.LocalCheckerCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(localCheckerHeartbeater.Type())). + Msg("Failed to create a new local checker heartbeater") + } else { + heartbeaters = append(heartbeaters, localCheckerHeartbeater) + } + } + + if heartbeatersCfg.RemoteCheckerCfg.Enabled && heartbeatersCfg.RemoteCheckerCfg.NeedTurnOffNotifier { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) + remoteCheckerHeartbeater, err := heartbeat.NewRemoteCheckerHeartbeater(heartbeatersCfg.RemoteCheckerCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(remoteCheckerHeartbeater.Type())). + Msg("Failed to create a new remote checker heartbeater") + } else { + heartbeaters = append(heartbeaters, remoteCheckerHeartbeater) + } + } + + if heartbeatersCfg.NotifierCfg.Enabled && heartbeatersCfg.NotifierCfg.NeedTurnOffNotifier { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) + notifierHeartbeater, err := heartbeat.NewNotifierHeartbeater(heartbeatersCfg.NotifierCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(notifierHeartbeater.Type())). + Msg("Failed to create a new notifier heartbeater") + } else { + heartbeaters = append(heartbeaters, notifierHeartbeater) + } + } + + return heartbeaters +} + +// Start is the method to start the selfstate controller. +func (c *controller) Start() { + c.tomb.Go(func() error { + w.NewWorker( + name, + c.logger, + c.database.NewLock(lockName, lockTTL), + c.selfstateCheck, + ).Run(nil) + return nil + }) +} + +func (c *controller) selfstateCheck(stop <-chan struct{}) error { + c.logger.Info().Msg(fmt.Sprintf("%s started", name)) + + checkTicker := time.NewTicker(c.cfg.CheckInterval) + defer checkTicker.Stop() + + for { + select { + case <-stop: + c.logger.Info().Msg(fmt.Sprintf("%s stopped", name)) + return nil + case <-checkTicker.C: + c.logger.Debug().Msg(fmt.Sprintf("%s selfstate check", name)) + + c.checkHeartbeats() + } + } +} + +func (c *controller) checkHeartbeats() { + for _, heartbeater := range c.heartbeaters { + heartbeatState, err := heartbeater.Check() + if err != nil { + c.logger.Error(). + Error(err). + String("name", name). + String("heartbeater", string(heartbeater.Type())). + Msg("Heartbeat check failed") + } + + if heartbeater.Type() == datatypes.HeartbeatNotifier { + if heartbeatState == heartbeat.StateError { + c.heartbeatMetrics.MarkNotifierIsAlive(false) + } else { + c.heartbeatMetrics.MarkNotifierIsAlive(true) + } + } + + if heartbeatState == heartbeat.StateError { + if err = c.database.SetNotifierState(moira.SelfStateERROR); err != nil { + c.logger.Error(). + Error(err). + String("name", name). + String("heartbeater", string(heartbeater.Type())). + Msg("Failed to set notifier state to error") + } + break + } + } +} + +// Stop is a method to stop the selfstate controller. +func (c *controller) Stop() error { + c.tomb.Kill(nil) + return c.tomb.Wait() +} diff --git a/notifier/selfstate/controller/controller_test.go b/notifier/selfstate/controller/controller_test.go new file mode 100644 index 000000000..edebf0b88 --- /dev/null +++ b/notifier/selfstate/controller/controller_test.go @@ -0,0 +1,337 @@ +package controller + +import ( + "errors" + "testing" + "time" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" + logging "github.com/moira-alert/moira/logging/zerolog_adapter" + "github.com/moira-alert/moira/metrics" + mock_clock "github.com/moira-alert/moira/mock/clock" + mock_heartbeat "github.com/moira-alert/moira/mock/heartbeat" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + "github.com/moira-alert/moira/notifier/selfstate/heartbeat" + . "github.com/smartystreets/goconvey/convey" + "go.uber.org/mock/gomock" +) + +func TestNewController(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + mockClock := mock_clock.NewMockClock(mockCtrl) + logger, _ := logging.GetLogger("Test") + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + dummySource := metrics.NewDummyRegistry() + heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummySource) + + Convey("Test NewController", t, func() { + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + + Convey("With disabled config", func() { + cfg := ControllerConfig{} + + c, err := NewController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics) + So(err, ShouldBeNil) + So(c, ShouldResemble, &controller{ + cfg: cfg, + logger: logger, + database: mockDatabase, + heartbeaters: make([]heartbeat.Heartbeater, 0), + heartbeatMetrics: heartbeatMetrics, + }) + }) + + Convey("With just enabled config", func() { + cfg := ControllerConfig{ + Enabled: true, + } + + c, err := NewController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics) + So(err, ShouldNotBeNil) + So(c, ShouldBeNil) + }) + + Convey("With just enabled config and filled heartbeaters", func() { + cfg := ControllerConfig{ + Enabled: true, + HeartbeatersCfg: heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + }, + } + + c, err := NewController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics) + So(err, ShouldNotBeNil) + So(c, ShouldBeNil) + }) + + Convey("With enabled config, filled heartbeaters and set check interval", func() { + cfg := ControllerConfig{ + Enabled: true, + HeartbeatersCfg: heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + }, + CheckInterval: time.Minute, + } + + _, err := NewController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics) + So(err, ShouldBeNil) + }) + }) +} + +func TestCreateHeartbeaters(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + mockClock := mock_clock.NewMockClock(mockCtrl) + mockLogger := mock_moira_alert.NewMockLogger(mockCtrl) + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test createHeartbeaters", t, func() { + Convey("Without any heartbeater", func() { + hbCfg := heartbeat.HeartbeatersConfig{} + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldBeEmpty) + }) + + Convey("With just enabled database heartbeater", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldBeEmpty) + }) + + Convey("With database heartbeater", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + RedisDisconnectDelay: time.Minute, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 1) + }) + + Convey("With database and filter heartbeaters", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + RedisDisconnectDelay: time.Minute, + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + MetricReceivedDelay: time.Minute, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 2) + }) + + Convey("With database, filter and local checker heartbeaters", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + RedisDisconnectDelay: time.Minute, + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + MetricReceivedDelay: time.Minute, + }, + LocalCheckerCfg: heartbeat.LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + LocalCheckDelay: time.Minute, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 3) + }) + + Convey("With database, filter, local checker and remote checker heartbeaters", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + RedisDisconnectDelay: time.Minute, + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + MetricReceivedDelay: time.Minute, + }, + LocalCheckerCfg: heartbeat.LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + LocalCheckDelay: time.Minute, + }, + RemoteCheckerCfg: heartbeat.RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + RemoteCheckDelay: time.Minute, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 4) + }) + + Convey("With database, filter, local checker, remote checker and notifier heartbeaters", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + RedisDisconnectDelay: time.Minute, + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + MetricReceivedDelay: time.Minute, + }, + LocalCheckerCfg: heartbeat.LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + LocalCheckDelay: time.Minute, + }, + RemoteCheckerCfg: heartbeat.RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + RemoteCheckDelay: time.Minute, + }, + NotifierCfg: heartbeat.NotifierHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + NeedTurnOffNotifier: true, + }, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 5) + }) + }) +} + +func TestCheckHeartbeats(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockClock := mock_clock.NewMockClock(mockCtrl) + logger, _ := logging.GetLogger("Test") + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + dummySource := metrics.NewDummyRegistry() + heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummySource) + + Convey("Test checkHeartbeats", t, func() { + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + + databaseHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl) + localCheckerHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl) + notifierHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl) + + c := &controller{ + heartbeaters: []heartbeat.Heartbeater{databaseHeartbeater, localCheckerHeartbeater, notifierHeartbeater}, + logger: logger, + database: mockDatabase, + heartbeatMetrics: heartbeatMetrics, + } + + Convey("Without error heartbeat states", func() { + databaseHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil) + databaseHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatDatabase) + + localCheckerHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil) + localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker) + + notifierHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil) + notifierHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatNotifier) + + c.checkHeartbeats() + }) + + Convey("With heartbeat error state", func() { + databaseHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil) + databaseHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatDatabase) + + localCheckerHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil) + localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker) + + mockDatabase.EXPECT().SetNotifierState(moira.SelfStateERROR).Return(nil) + + c.checkHeartbeats() + }) + + Convey("With heartbeat error state and error while set notifier state", func() { + dbErr := errors.New("test database error") + databaseHeartbeater.EXPECT().Check().Return(heartbeat.StateOK, nil) + databaseHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatDatabase) + + localCheckerHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil) + localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker) + + mockDatabase.EXPECT().SetNotifierState(moira.SelfStateERROR).Return(dbErr) + localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker) + + c.checkHeartbeats() + }) + }) +} diff --git a/notifier/selfstate/selfstate.go b/notifier/selfstate/selfstate.go index 7a7d4f985..966677097 100644 --- a/notifier/selfstate/selfstate.go +++ b/notifier/selfstate/selfstate.go @@ -4,7 +4,9 @@ import ( "errors" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/metrics" "github.com/moira-alert/moira/notifier" + "github.com/moira-alert/moira/notifier/selfstate/controller" "github.com/moira-alert/moira/notifier/selfstate/monitor" ) @@ -18,7 +20,8 @@ type SelfstateWorker interface { } type selfstateWorker struct { - monitors []monitor.Monitor + monitors []monitor.Monitor + controller controller.Controller } // NewSelfstateWorker is a method to create a new selfstate worker. @@ -28,11 +31,14 @@ func NewSelfstateWorker( database moira.Database, notifier notifier.Notifier, clock moira.Clock, + heartbeatMetrics *metrics.HeartBeatMetrics, ) (*selfstateWorker, error) { monitors := createMonitors(cfg.MonitorCfg, logger, database, clock, notifier) + controller := createController(cfg.ControllerCfg, logger, database, clock, heartbeatMetrics) return &selfstateWorker{ - monitors: monitors, + monitors: monitors, + controller: controller, }, nil } @@ -85,11 +91,37 @@ func createMonitors( return monitors } +func createController( + controllerCfg controller.ControllerConfig, + logger moira.Logger, + database moira.Database, + clock moira.Clock, + heartbeatMetrics *metrics.HeartBeatMetrics, +) controller.Controller { + var c controller.Controller + var err error + + if controllerCfg.Enabled { + c, err = controller.NewController(controllerCfg, logger, database, clock, heartbeatMetrics) + if err != nil { + logger.Error(). + Error(err). + Msg("Failed to create a new controller") + } + } + + return c +} + // Start is a method to start a selfstate worker. func (selfstateWorker *selfstateWorker) Start() { for _, monitor := range selfstateWorker.monitors { monitor.Start() } + + if selfstateWorker.controller != nil { + selfstateWorker.controller.Start() + } } // Stop is a method for stopping a selfstate worker. @@ -102,5 +134,11 @@ func (selfstateWorker *selfstateWorker) Stop() error { } } + if selfstateWorker.controller != nil { + if err := selfstateWorker.controller.Stop(); err != nil { + stopErrors = append(stopErrors, err) + } + } + return errors.Join(stopErrors...) } diff --git a/notifier/selfstate/selfstate_test.go b/notifier/selfstate/selfstate_test.go index 9058b4168..85c420ad2 100644 --- a/notifier/selfstate/selfstate_test.go +++ b/notifier/selfstate/selfstate_test.go @@ -8,10 +8,13 @@ import ( "go.uber.org/mock/gomock" logging "github.com/moira-alert/moira/logging/zerolog_adapter" + "github.com/moira-alert/moira/metrics" mock_clock "github.com/moira-alert/moira/mock/clock" + mock_controller "github.com/moira-alert/moira/mock/controller" mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" mock_monitor "github.com/moira-alert/moira/mock/monitor" mock_notifier "github.com/moira-alert/moira/mock/notifier" + "github.com/moira-alert/moira/notifier/selfstate/controller" "github.com/moira-alert/moira/notifier/selfstate/heartbeat" "github.com/moira-alert/moira/notifier/selfstate/monitor" . "github.com/smartystreets/goconvey/convey" @@ -25,14 +28,17 @@ func TestNewSelfstateWorker(t *testing.T) { mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) logger, _ := logging.GetLogger("Test") mockNotifier := mock_notifier.NewMockNotifier(mockCtrl) + dummyRegistry := metrics.NewDummyRegistry() + heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) cfg := Config{ - Enabled: true, - MonitorCfg: MonitorConfig{}, + Enabled: true, + MonitorCfg: MonitorConfig{}, + ControllerCfg: controller.ControllerConfig{}, } Convey("Test NewSelfstateWorker", t, func() { - worker, err := NewSelfstateWorker(cfg, logger, mockDatabase, mockNotifier, mockClock) + worker, err := NewSelfstateWorker(cfg, logger, mockDatabase, mockNotifier, mockClock, heartbeatMetrics) So(err, ShouldBeNil) So(worker.monitors, ShouldHaveLength, 0) }) @@ -122,20 +128,62 @@ func TestCreateMonitors(t *testing.T) { }) } +func TestCreateController(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockClock := mock_clock.NewMockClock(mockCtrl) + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + logger, _ := logging.GetLogger("Test") + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + dummyRegistry := metrics.NewDummyRegistry() + heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) + + Convey("Test createController", t, func() { + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + + Convey("With disabled controller", func() { + controller := createController(controller.ControllerConfig{}, logger, mockDatabase, mockClock, heartbeatMetrics) + So(controller, ShouldBeNil) + }) + + Convey("With enabled controller", func() { + cfg := controller.ControllerConfig{ + Enabled: true, + HeartbeatersCfg: heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + }, + CheckInterval: time.Minute, + } + + controller := createController(cfg, logger, mockDatabase, mockClock, heartbeatMetrics) + So(controller, ShouldNotBeNil) + }) + }) +} + func TestStart(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mockUserMonitor := mock_monitor.NewMockMonitor(mockCtrl) mockAdminMonitor := mock_monitor.NewMockMonitor(mockCtrl) + mockController := mock_controller.NewMockController(mockCtrl) worker := &selfstateWorker{ - monitors: []monitor.Monitor{mockUserMonitor, mockAdminMonitor}, + monitors: []monitor.Monitor{mockUserMonitor, mockAdminMonitor}, + controller: mockController, } Convey("Test Start", t, func() { mockUserMonitor.EXPECT().Start() mockAdminMonitor.EXPECT().Start() + mockController.EXPECT().Start() worker.Start() }) @@ -147,18 +195,22 @@ func TestStop(t *testing.T) { mockUserMonitor := mock_monitor.NewMockMonitor(mockCtrl) mockAdminMonitor := mock_monitor.NewMockMonitor(mockCtrl) + mockController := mock_controller.NewMockController(mockCtrl) userMonitorErr := errors.New("test user monitor error") adminMonitorErr := errors.New("test admin monitor error") + controllerErr := errors.New("test controller error") worker := &selfstateWorker{ - monitors: []monitor.Monitor{mockUserMonitor, mockAdminMonitor}, + monitors: []monitor.Monitor{mockUserMonitor, mockAdminMonitor}, + controller: mockController, } Convey("Test Stop", t, func() { Convey("Without any errors", func() { mockUserMonitor.EXPECT().Stop().Return(nil) mockAdminMonitor.EXPECT().Stop().Return(nil) + mockController.EXPECT().Stop().Return(nil) err := worker.Stop() So(err, ShouldBeNil) @@ -167,6 +219,7 @@ func TestStop(t *testing.T) { Convey("With user monitor error", func() { mockUserMonitor.EXPECT().Stop().Return(userMonitorErr) mockAdminMonitor.EXPECT().Stop().Return(nil) + mockController.EXPECT().Stop().Return(nil) err := worker.Stop() So(err, ShouldResemble, errors.Join(userMonitorErr)) @@ -175,17 +228,37 @@ func TestStop(t *testing.T) { Convey("With admin monitor error", func() { mockUserMonitor.EXPECT().Stop().Return(nil) mockAdminMonitor.EXPECT().Stop().Return(adminMonitorErr) + mockController.EXPECT().Stop().Return(nil) err := worker.Stop() So(err, ShouldResemble, errors.Join(adminMonitorErr)) }) + Convey("With controller error", func() { + mockUserMonitor.EXPECT().Stop().Return(nil) + mockAdminMonitor.EXPECT().Stop().Return(nil) + mockController.EXPECT().Stop().Return(controllerErr) + + err := worker.Stop() + So(err, ShouldResemble, errors.Join(controllerErr)) + }) + Convey("With admin and user monitor errors", func() { mockUserMonitor.EXPECT().Stop().Return(userMonitorErr) mockAdminMonitor.EXPECT().Stop().Return(adminMonitorErr) + mockController.EXPECT().Stop().Return(nil) err := worker.Stop() So(err, ShouldResemble, errors.Join(userMonitorErr, adminMonitorErr)) }) + + Convey("With admin, user monitors and controller errors", func() { + mockUserMonitor.EXPECT().Stop().Return(userMonitorErr) + mockAdminMonitor.EXPECT().Stop().Return(adminMonitorErr) + mockController.EXPECT().Stop().Return(controllerErr) + + err := worker.Stop() + So(err, ShouldResemble, errors.Join(userMonitorErr, adminMonitorErr, controllerErr)) + }) }) }