diff --git a/go.mod b/go.mod index b309808bd..cc875b256 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( require github.com/prometheus/common v0.37.0 require ( - github.com/go-playground/validator/v10 v10.4.1 + github.com/go-playground/validator/v10 v10.22.1 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/mattermost/mattermost/server/public v0.1.1 github.com/mitchellh/mapstructure v1.5.0 @@ -173,19 +173,20 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a // indirect github.com/fatih/color v1.16.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/spec v0.20.9 // indirect github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-playground/locales v0.13.0 // indirect - github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-plugin v1.6.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/leodido/go-urn v1.2.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect diff --git a/go.sum b/go.sum index a013bebd9..30410d294 100644 --- a/go.sum +++ b/go.sum @@ -238,6 +238,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= @@ -279,14 +281,18 @@ github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= +github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= @@ -527,8 +533,9 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80 h1:KVyDGUXjVOdHQt24wIgY4ZdGFXHtQHLWw0L/MAK3Kb0= diff --git a/mock/heartbeat/heartbeat.go b/mock/heartbeat/heartbeat.go index f0a6f6acf..694bee3c1 100644 --- a/mock/heartbeat/heartbeat.go +++ b/mock/heartbeat/heartbeat.go @@ -12,6 +12,8 @@ package mock_heartbeat import ( reflect "reflect" + datatypes "github.com/moira-alert/moira/datatypes" + heartbeat "github.com/moira-alert/moira/notifier/selfstate/heartbeat" gomock "go.uber.org/mock/gomock" ) @@ -38,60 +40,45 @@ func (m *MockHeartbeater) EXPECT() *MockHeartbeaterMockRecorder { return m.recorder } -// Check mocks base method. -func (m *MockHeartbeater) Check(arg0 int64) (int64, bool, error) { +// AlertSettings mocks base method. +func (m *MockHeartbeater) AlertSettings() heartbeat.AlertConfig { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Check", arg0) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// Check indicates an expected call of Check. -func (mr *MockHeartbeaterMockRecorder) Check(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockHeartbeater)(nil).Check), arg0) -} - -// GetErrorMessage mocks base method. -func (m *MockHeartbeater) GetErrorMessage() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetErrorMessage") - ret0, _ := ret[0].(string) + ret := m.ctrl.Call(m, "AlertSettings") + ret0, _ := ret[0].(heartbeat.AlertConfig) return ret0 } -// GetErrorMessage indicates an expected call of GetErrorMessage. -func (mr *MockHeartbeaterMockRecorder) GetErrorMessage() *gomock.Call { +// AlertSettings indicates an expected call of AlertSettings. +func (mr *MockHeartbeaterMockRecorder) AlertSettings() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetErrorMessage", reflect.TypeOf((*MockHeartbeater)(nil).GetErrorMessage)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AlertSettings", reflect.TypeOf((*MockHeartbeater)(nil).AlertSettings)) } -// NeedToCheckOthers mocks base method. -func (m *MockHeartbeater) NeedToCheckOthers() bool { +// Check mocks base method. +func (m *MockHeartbeater) Check() (heartbeat.State, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NeedToCheckOthers") - ret0, _ := ret[0].(bool) - return ret0 + ret := m.ctrl.Call(m, "Check") + ret0, _ := ret[0].(heartbeat.State) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// NeedToCheckOthers indicates an expected call of NeedToCheckOthers. -func (mr *MockHeartbeaterMockRecorder) NeedToCheckOthers() *gomock.Call { +// Check indicates an expected call of Check. +func (mr *MockHeartbeaterMockRecorder) Check() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NeedToCheckOthers", reflect.TypeOf((*MockHeartbeater)(nil).NeedToCheckOthers)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockHeartbeater)(nil).Check)) } -// NeedTurnOffNotifier mocks base method. -func (m *MockHeartbeater) NeedTurnOffNotifier() bool { +// Type mocks base method. +func (m *MockHeartbeater) Type() datatypes.HeartbeatType { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NeedTurnOffNotifier") - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "Type") + ret0, _ := ret[0].(datatypes.HeartbeatType) return ret0 } -// NeedTurnOffNotifier indicates an expected call of NeedTurnOffNotifier. -func (mr *MockHeartbeaterMockRecorder) NeedTurnOffNotifier() *gomock.Call { +// Type indicates an expected call of Type. +func (mr *MockHeartbeaterMockRecorder) Type() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NeedTurnOffNotifier", reflect.TypeOf((*MockHeartbeater)(nil).NeedTurnOffNotifier)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Type", reflect.TypeOf((*MockHeartbeater)(nil).Type)) } diff --git a/notifier/selfstate/heartbeat/database.go b/notifier/selfstate/heartbeat/database.go index 5b8cd551f..69ab7eed7 100644 --- a/notifier/selfstate/heartbeat/database.go +++ b/notifier/selfstate/heartbeat/database.go @@ -1,52 +1,64 @@ package heartbeat import ( + "fmt" "time" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" ) -type databaseHeartbeat struct{ heartbeat } +// Verify that databaseHeartbeater matches the Heartbeater interface. +var _ Heartbeater = (*databaseHeartbeater)(nil) -func GetDatabase(delay int64, logger moira.Logger, database moira.Database) Heartbeater { - if delay > 0 { - return &databaseHeartbeat{heartbeat{ - logger: logger, - database: database, - delay: delay, - lastSuccessfulCheck: time.Now().Unix(), - }} - } - return nil +// DatabaseHeartbeaterConfig structure describing the databaseHeartbeater configuration. +type DatabaseHeartbeaterConfig struct { + HeartbeaterBaseConfig + + RedisDisconnectDelay time.Duration `validate:"required_if=Enabled true,gte=0"` } -func (check *databaseHeartbeat) Check(nowTS int64) (int64, bool, error) { - _, err := check.database.GetChecksUpdatesCount() - if err == nil { - check.lastSuccessfulCheck = nowTS - return 0, false, nil - } +type databaseHeartbeater struct { + *heartbeaterBase - if check.lastSuccessfulCheck < nowTS-check.delay { - check.logger.Error(). - String("error", check.GetErrorMessage()). - Int64("time_since_successful_check", nowTS-check.heartbeat.lastSuccessfulCheck). - Msg("Send message") + cfg DatabaseHeartbeaterConfig +} - return nowTS - check.lastSuccessfulCheck, true, nil +// NewDatabaseHeartbeater is a function that creates a new databaseHeartbeater. +func NewDatabaseHeartbeater(cfg DatabaseHeartbeaterConfig, base *heartbeaterBase) (*databaseHeartbeater, error) { + if err := moira.ValidateStruct(cfg); err != nil { + return nil, fmt.Errorf("database heartbeater configuration error: %w", err) } - return 0, false, nil + return &databaseHeartbeater{ + heartbeaterBase: base, + cfg: cfg, + }, nil } -func (databaseHeartbeat) NeedTurnOffNotifier() bool { - return true +// Check is a function that checks if the database is working correctly. +func (heartbeater *databaseHeartbeater) Check() (State, error) { + now := heartbeater.clock.NowUTC() + + _, err := heartbeater.database.GetChecksUpdatesCount() + if err == nil { + heartbeater.lastSuccessfulCheck = now + return StateOK, nil + } + + if now.Sub(heartbeater.lastSuccessfulCheck) > heartbeater.cfg.RedisDisconnectDelay { + return StateError, nil + } + + return StateOK, err } -func (databaseHeartbeat) NeedToCheckOthers() bool { - return false +// Type is a function that returns the current heartbeat type. +func (databaseHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HeartbeatDatabase } -func (databaseHeartbeat) GetErrorMessage() string { - return "Redis disconnected" +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater databaseHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/database_test.go b/notifier/selfstate/heartbeat/database_test.go index a5b8b8a5a..66d4277f6 100644 --- a/notifier/selfstate/heartbeat/database_test.go +++ b/notifier/selfstate/heartbeat/database_test.go @@ -5,69 +5,148 @@ import ( "testing" "time" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" - - logging "github.com/moira-alert/moira/logging/zerolog_adapter" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestDatabaseHeartbeat(t *testing.T) { - Convey("Test database heartbeat", t, func() { - now := time.Now().Unix() - err := errors.New("test database error") - check := createRedisDelayTest(t) - database := check.database.(*mock_moira_alert.MockDatabase) +const ( + defaultRedisDisconnectDelay = time.Minute +) + +func TestNewDatabaseHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + validationErr := validator.ValidationErrors{} - Convey("Checking the created heartbeat database", func() { - expected := &databaseHeartbeat{heartbeat{database: check.database, logger: check.logger, delay: 1, lastSuccessfulCheck: now}} + Convey("Test NewDatabaseHeartbeater", t, func() { + Convey("With too low redis disconnect delay", func() { + cfg := DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: -1, + } - So(GetDatabase(0, check.logger, check.database), ShouldBeNil) - So(GetDatabase(1, check.logger, check.database), ShouldResemble, expected) + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(databaseHeartbeater, ShouldBeNil) }) - Convey("Test update lastSuccessfulCheck", func() { - now += 1000 - database.EXPECT().GetChecksUpdatesCount().Return(int64(1), nil) + Convey("Without redis disconnect delay", func() { + cfg := DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(databaseHeartbeater, ShouldBeNil) }) - Convey("Database error handling test", func() { - database.EXPECT().GetChecksUpdatesCount().Return(int64(1), err) + Convey("With correct database heartbeater config", func() { + cfg := DatabaseHeartbeaterConfig{ + RedisDisconnectDelay: 1, + } - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + expected := &databaseHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } + + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(databaseHeartbeater, ShouldResemble, expected) }) + }) +} - Convey("Check for notification", func() { - check.lastSuccessfulCheck = now - check.delay - 1 +func TestDatabaseHeartbeaterCheck(t *testing.T) { + database, clock, testTime, heartbeaterBase := heartbeaterHelper(t) - database.EXPECT().GetChecksUpdatesCount().Return(int64(0), err) + cfg := DatabaseHeartbeaterConfig{ + RedisDisconnectDelay: defaultRedisDisconnectDelay, + } - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, now-check.lastSuccessfulCheck) + databaseHeartbeater, _ := NewDatabaseHeartbeater(cfg, heartbeaterBase) + + var ( + testErr = errors.New("test error") + checkUpdates int64 + ) + + Convey("Test databaseHeartbeater.Check", t, func() { + Convey("With nil error in GetCheckUpdatedCount", func() { + database.EXPECT().GetChecksUpdatesCount().Return(checkUpdates, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := databaseHeartbeater.Check() + So(state, ShouldResemble, StateOK) + So(err, ShouldBeNil) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - So(check.NeedTurnOffNotifier(), ShouldBeTrue) - So(check.NeedToCheckOthers(), ShouldBeFalse) + Convey("With too much time elapsed since the last successful check", func() { + heartbeaterBase.lastSuccessfulCheck = testTime.Add(-10 * defaultRedisDisconnectDelay) + defer func() { + heartbeaterBase.lastSuccessfulCheck = testTime + }() + + database.EXPECT().GetChecksUpdatesCount().Return(checkUpdates, testErr) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := databaseHeartbeater.Check() + So(state, ShouldResemble, StateError) + So(err, ShouldBeNil) }) + + Convey("With only error from GetChecksUpdateCount", func() { + database.EXPECT().GetChecksUpdatesCount().Return(checkUpdates, testErr) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := databaseHeartbeater.Check() + So(state, ShouldResemble, StateOK) + So(err, ShouldResemble, testErr) + }) + }) +} + +func TestDatabaseHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test databaseHeartbeater.Type", t, func() { + cfg := DatabaseHeartbeaterConfig{ + RedisDisconnectDelay: defaultRedisDisconnectDelay, + } + + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + databaseHeartbeaterType := databaseHeartbeater.Type() + So(databaseHeartbeaterType, ShouldResemble, datatypes.HeartbeatDatabase) }) } -func createRedisDelayTest(t *testing.T) *databaseHeartbeat { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("CheckDelay") +func TestDatabaseHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test databaseHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } - return GetDatabase(10, logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*databaseHeartbeat) + cfg := DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + RedisDisconnectDelay: defaultRedisDisconnectDelay, + } + + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := databaseHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) } diff --git a/notifier/selfstate/heartbeat/filter.go b/notifier/selfstate/heartbeat/filter.go index 80f000e35..15a222b2b 100644 --- a/notifier/selfstate/heartbeat/filter.go +++ b/notifier/selfstate/heartbeat/filter.go @@ -1,70 +1,78 @@ package heartbeat import ( + "fmt" "time" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" ) -type filter struct { - heartbeat - count int64 - firstCheckWasSuccessful bool +var ( + localClusterKey = moira.DefaultLocalCluster + + // Verify that filterHeartbeater matches the Heartbeater interface. + _ Heartbeater = (*filterHeartbeater)(nil) +) + +// FilterHeartbeaterConfig structure describing the filterHeartbeater configuration. +type FilterHeartbeaterConfig struct { + HeartbeaterBaseConfig + + MetricReceivedDelay time.Duration `validate:"required_if=Enabled true,gte=0"` } -func GetFilter(delay int64, logger moira.Logger, database moira.Database) Heartbeater { - if delay > 0 { - return &filter{ - heartbeat: heartbeat{ - logger: logger, - database: database, - delay: delay, - lastSuccessfulCheck: time.Now().Unix(), - }, - firstCheckWasSuccessful: false, - } +type filterHeartbeater struct { + *heartbeaterBase + + cfg FilterHeartbeaterConfig + lastMetricsCount int64 +} + +// NewFilterHeartbeater is a function that creates a new filterHeartbeater. +func NewFilterHeartbeater(cfg FilterHeartbeaterConfig, base *heartbeaterBase) (*filterHeartbeater, error) { + if err := moira.ValidateStruct(cfg); err != nil { + return nil, fmt.Errorf("filter heartheater configuration error: %w", err) } - return nil + + return &filterHeartbeater{ + heartbeaterBase: base, + cfg: cfg, + }, nil } -func (check *filter) Check(nowTS int64) (int64, bool, error) { - defaultLocalCluster := moira.DefaultLocalCluster - triggersCount, err := check.database.GetTriggersToCheckCount(defaultLocalCluster) +// Check is a function that checks that filters accept metrics and that their number of metrics is not constant. +func (heartbeater *filterHeartbeater) Check() (State, error) { + triggersCount, err := heartbeater.database.GetTriggersToCheckCount(localClusterKey) if err != nil { - return 0, false, err + return StateError, err } - metricsCount, err := check.database.GetMetricsUpdatesCount() + metricsCount, err := heartbeater.database.GetMetricsUpdatesCount() if err != nil { - return 0, false, err - } - if check.count != metricsCount || triggersCount == 0 { - check.count = metricsCount - check.lastSuccessfulCheck = nowTS - return 0, false, nil + return StateError, err } - if check.lastSuccessfulCheck < nowTS-check.heartbeat.delay { - check.logger.Error(). - String("error", check.GetErrorMessage()). - Int64("time_since_successful_check", nowTS-check.heartbeat.lastSuccessfulCheck). - Msg("Send message") + now := heartbeater.clock.NowUTC() + if heartbeater.lastMetricsCount != metricsCount || triggersCount == 0 { + heartbeater.lastMetricsCount = metricsCount + heartbeater.lastSuccessfulCheck = now + return StateOK, nil + } - check.firstCheckWasSuccessful = true - return nowTS - check.heartbeat.lastSuccessfulCheck, true, nil + if now.Sub(heartbeater.lastSuccessfulCheck) > heartbeater.cfg.MetricReceivedDelay { + return StateError, nil } - return 0, false, nil -} -// NeedTurnOffNotifier: turn off notifications if at least once the filter check was successful. -func (check filter) NeedTurnOffNotifier() bool { - return check.firstCheckWasSuccessful + return StateOK, nil } -func (check filter) NeedToCheckOthers() bool { - return true +// Type is a function that returns the current heartbeat type. +func (filterHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HeartbeatFilter } -func (filter) GetErrorMessage() string { - return "Moira-Filter does not receive metrics" +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater filterHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/filter_test.go b/notifier/selfstate/heartbeat/filter_test.go index 678c443d9..52f29ebde 100644 --- a/notifier/selfstate/heartbeat/filter_test.go +++ b/notifier/selfstate/heartbeat/filter_test.go @@ -5,92 +5,190 @@ import ( "testing" "time" - "github.com/moira-alert/moira" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestFilter(t *testing.T) { - Convey("Test filter heartbeat", t, func() { - err := errors.New("test filter error") - now := time.Now().Unix() - check, mockCtrl := createFilterTest(t) - defer mockCtrl.Finish() - database := check.database.(*mock_moira_alert.MockDatabase) - defaultLocalCluster := moira.MakeClusterKey(moira.GraphiteLocal, moira.DefaultCluster) - - Convey("Checking the created filter", func() { - expected := &filter{ - heartbeat: heartbeat{ - database: check.database, - logger: check.logger, - delay: 1, - lastSuccessfulCheck: now, +const ( + defaultMetricReceivedDelay = time.Minute +) + +func TestNewFilterHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + validationErr := validator.ValidationErrors{} + + Convey("Test NewFilterHeartbeater", t, func() { + Convey("With too low metric received delay", func() { + cfg := FilterHeartbeaterConfig{ + MetricReceivedDelay: -1, + } + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(filterHeartbeater, ShouldBeNil) + }) + + Convey("Without metric received delay", func() { + cfg := FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, }, } - So(GetFilter(0, check.logger, check.database), ShouldBeNil) - So(GetFilter(1, check.logger, check.database), ShouldResemble, expected) + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(filterHeartbeater, ShouldBeNil) }) - Convey("Filter error handling test", func() { - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), err) + Convey("With correct filter heartbeater config", func() { + cfg := FilterHeartbeaterConfig{ + MetricReceivedDelay: 1, + } - value, needSend, errActual := check.Check(now) - So(errActual, ShouldEqual, err) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + expected := &filterHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(filterHeartbeater, ShouldResemble, expected) }) + }) +} + +func TestFilterHeartbeaterCheck(t *testing.T) { + database, clock, testTime, heartbeaterBase := heartbeaterHelper(t) + + cfg := FilterHeartbeaterConfig{ + MetricReceivedDelay: defaultMetricReceivedDelay, + } - Convey("Test update lastSuccessfulCheck", func() { - now += 1000 - database.EXPECT().GetMetricsUpdatesCount().Return(int64(1), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + filterHeartbeater, _ := NewFilterHeartbeater(cfg, heartbeaterBase) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + var ( + testErr = errors.New("test error") + triggersToCheckCount, metricsUpdatesCount int64 = 10, 10 + ) + + Convey("Test filterHeartbeater.Check", t, func() { + Convey("With GetTriggersToCheckCount error", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, testErr) + + state, err := filterHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("Check for notification", func() { - check.lastSuccessfulCheck = now - check.delay - 1 + Convey("With GetMetricsUpdatesCount error", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, testErr) + + state, err := filterHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) + }) - database.EXPECT().GetMetricsUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + Convey("With last metrics count not equal current metrics count", func() { + defer func() { + filterHeartbeater.lastMetricsCount = 0 + }() - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, now-check.lastSuccessfulCheck) + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := filterHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(filterHeartbeater.lastMetricsCount, ShouldResemble, metricsUpdatesCount) }) - Convey("Exit without action", func() { - database.EXPECT().GetMetricsUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + Convey("With zero triggers to check count", func() { + defer func() { + filterHeartbeater.lastMetricsCount = 0 + }() + + var zeroTriggersToCheckCount int64 + + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(zeroTriggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + state, err := filterHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(filterHeartbeater.lastMetricsCount, ShouldResemble, metricsUpdatesCount) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - // TODO(litleleprikon): seems that this test checks nothing. Seems that NeedToCheckOthers and NeedTurnOffNotifier do not work. - So(check.NeedToCheckOthers(), ShouldBeTrue) + filterHeartbeater.lastMetricsCount = metricsUpdatesCount + + Convey("With too much time elapsed since the last successful check", func() { + filterHeartbeater.lastSuccessfulCheck = testTime.Add(-10 * defaultMetricReceivedDelay) + defer func() { + filterHeartbeater.lastSuccessfulCheck = testTime + }() + + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - So(check.NeedTurnOffNotifier(), ShouldBeFalse) + state, err := filterHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateError) }) + + Convey("With short time elapsed since the last successful check", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := filterHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + }) + }) +} + +func TestFilterHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test filterHeartbeater.Type", t, func() { + cfg := FilterHeartbeaterConfig{ + MetricReceivedDelay: defaultMetricReceivedDelay, + } + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + filterHeartbeaterType := filterHeartbeater.Type() + So(filterHeartbeaterType, ShouldResemble, datatypes.HeartbeatFilter) }) } -func createFilterTest(t *testing.T) (*filter, *gomock.Controller) { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("MetricDelay") +func TestFilterHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) - return GetFilter(60, logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*filter), mockCtrl + Convey("Test filterHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } + + cfg := FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + MetricReceivedDelay: defaultMetricReceivedDelay, + } + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := filterHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) } diff --git a/notifier/selfstate/heartbeat/heartbeat.go b/notifier/selfstate/heartbeat/heartbeat.go index 244de1bcd..cf96cdc5b 100644 --- a/notifier/selfstate/heartbeat/heartbeat.go +++ b/notifier/selfstate/heartbeat/heartbeat.go @@ -1,21 +1,71 @@ package heartbeat import ( + "time" + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" +) + +// State characterises the state of the heartbeat. +type State string + +const ( + StateOK State = "heartbeat_state_ok" + StateError State = "heartbeat_state_error" ) +// IsDegraded checks if the condition is still degraded. +func (State) IsDegraded(newState State) bool { + return newState == StateError +} + +// IsRecovered checks if the condition has recovered. +func (lastState State) IsRecovered(newState State) bool { + return lastState == StateError && newState == StateOK +} + // Heartbeater is the interface for simplified events verification. type Heartbeater interface { - Check(int64) (int64, bool, error) - NeedTurnOffNotifier() bool - NeedToCheckOthers() bool - GetErrorMessage() string + Check() (State, error) + AlertSettings() AlertConfig + Type() datatypes.HeartbeatType } -// heartbeat basic structure for Heartbeater. -type heartbeat struct { +// HeartbeaterBaseConfig contains common fields for all heartbeaters. +type HeartbeaterBaseConfig struct { + Enabled bool + NeedTurnOffNotifier bool + + AlertCfg AlertConfig +} + +// AlertConfig contains the configuration of the alerts that heartbeater sends out. +type AlertConfig struct { + Name string + Desc string +} + +// HeartbeatBase is basic structure for Heartbeater. +type heartbeaterBase struct { logger moira.Logger database moira.Database + clock moira.Clock + + lastSuccessfulCheck time.Time +} + +// NewHeartbeaterBase function that creates a base for heartbeater. +func NewHeartbeaterBase( + logger moira.Logger, + database moira.Database, + clock moira.Clock, +) *heartbeaterBase { + return &heartbeaterBase{ + logger: logger, + database: database, + clock: clock, - delay, lastSuccessfulCheck int64 + lastSuccessfulCheck: clock.NowUTC(), + } } diff --git a/notifier/selfstate/heartbeat/heartbeat_test.go b/notifier/selfstate/heartbeat/heartbeat_test.go new file mode 100644 index 000000000..5dd2961e2 --- /dev/null +++ b/notifier/selfstate/heartbeat/heartbeat_test.go @@ -0,0 +1,150 @@ +package heartbeat + +import ( + "testing" + "time" + + "github.com/moira-alert/moira" + logging "github.com/moira-alert/moira/logging/zerolog_adapter" + mock_clock "github.com/moira-alert/moira/mock/clock" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + . "github.com/smartystreets/goconvey/convey" + "go.uber.org/mock/gomock" +) + +func heartbeaterHelper(t *testing.T) (*mock_moira_alert.MockDatabase, *mock_clock.MockClock, time.Time, *heartbeaterBase) { + t.Helper() + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + logger, _ := logging.GetLogger("Test") + database := mock_moira_alert.NewMockDatabase(mockCtrl) + clock := mock_clock.NewMockClock(mockCtrl) + + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + clock.EXPECT().NowUTC().Return(testTime) + heartbeaterBase := NewHeartbeaterBase(logger, database, clock) + + return database, clock, testTime, heartbeaterBase +} + +func TestStateIsDegradated(t *testing.T) { + Convey("Test state.IsDegraded", t, func() { + Convey("With continue degraded", func() { + lastState := StateError + newState := StateError + + degradated := lastState.IsDegraded(newState) + So(degradated, ShouldBeTrue) + }) + + Convey("With degraded state", func() { + lastState := StateOK + newState := StateError + + degradated := lastState.IsDegraded(newState) + So(degradated, ShouldBeTrue) + }) + + Convey("Without degraded state", func() { + lastState := StateError + newState := StateOK + + degradated := lastState.IsDegraded(newState) + So(degradated, ShouldBeFalse) + }) + + Convey("With continue recovered", func() { + lastState := StateOK + newState := StateOK + + degradated := lastState.IsDegraded(newState) + So(degradated, ShouldBeFalse) + }) + }) +} + +func TestStateIsRecovered(t *testing.T) { + Convey("Test state.IsRecovered", t, func() { + Convey("With recovered state", func() { + lastState := StateError + newState := StateOK + + recovered := lastState.IsRecovered(newState) + So(recovered, ShouldBeTrue) + }) + + Convey("Without recovered state", func() { + lastState := StateOK + newState := StateError + + recovered := lastState.IsRecovered(newState) + So(recovered, ShouldBeFalse) + }) + + Convey("With continue recovered", func() { + lastState := StateOK + newState := StateOK + + recovered := lastState.IsRecovered(newState) + So(recovered, ShouldBeFalse) + }) + + Convey("With continue degraded", func() { + lastState := StateError + newState := StateError + + recovered := lastState.IsRecovered(newState) + So(recovered, ShouldBeFalse) + }) + }) +} + +func TestNewHeartbeaterBase(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + logger, _ := logging.GetLogger("Test") + database := mock_moira_alert.NewMockDatabase(mockCtrl) + clock := mock_clock.NewMockClock(mockCtrl) + + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test NewHeartbeaterBase", t, func() { + clock.EXPECT().NowUTC().Return(testTime) + + expected := &heartbeaterBase{ + logger: logger, + database: database, + clock: clock, + + lastSuccessfulCheck: testTime, + } + + heartbeaterBase := NewHeartbeaterBase(logger, database, clock) + So(heartbeaterBase, ShouldResemble, expected) + }) +} + +func TestValidateHeartbeaterBaseConfig(t *testing.T) { + Convey("Test validation heartbeaterBaseConfig", t, func() { + Convey("With disabled config", func() { + hbCfg := HeartbeaterBaseConfig{} + err := moira.ValidateStruct(hbCfg) + So(err, ShouldBeNil) + }) + + Convey("With enabled config, added and filled alert config", func() { + hbCfg := HeartbeaterBaseConfig{ + Enabled: true, + AlertCfg: AlertConfig{ + Name: "test name", + }, + } + err := moira.ValidateStruct(hbCfg) + So(err, ShouldBeNil) + }) + }) +} diff --git a/notifier/selfstate/heartbeat/local_checker.go b/notifier/selfstate/heartbeat/local_checker.go index 7c4c88cfd..9a80b6ba1 100644 --- a/notifier/selfstate/heartbeat/local_checker.go +++ b/notifier/selfstate/heartbeat/local_checker.go @@ -1,62 +1,74 @@ package heartbeat import ( + "fmt" "time" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" ) -type localChecker struct { - heartbeat - count int64 +// Verify that localCheckerHeartbeater matches the Heartbeater interface. +var _ Heartbeater = (*localCheckerHeartbeater)(nil) + +// LocalCheckerHeartbeaterConfig structure describing the localCheckerHeartbeater configuration. +type LocalCheckerHeartbeaterConfig struct { + HeartbeaterBaseConfig + + LocalCheckDelay time.Duration `validate:"required_if=Enabled true,gte=0"` +} + +type localCheckerHeartbeater struct { + *heartbeaterBase + + cfg LocalCheckerHeartbeaterConfig + lastChecksCount int64 } -func GetLocalChecker(delay int64, logger moira.Logger, database moira.Database) Heartbeater { - if delay > 0 { - return &localChecker{heartbeat: heartbeat{ - logger: logger, - database: database, - delay: delay, - lastSuccessfulCheck: time.Now().Unix(), - }} +// NewLocalCheckerHeartbeater is a function that creates a new localCheckerHeartbeater. +func NewLocalCheckerHeartbeater(cfg LocalCheckerHeartbeaterConfig, base *heartbeaterBase) (*localCheckerHeartbeater, error) { + if err := moira.ValidateStruct(cfg); err != nil { + return nil, fmt.Errorf("local checker heartbeater configuration error: %w", err) } - return nil + + return &localCheckerHeartbeater{ + heartbeaterBase: base, + cfg: cfg, + }, nil } -func (check *localChecker) Check(nowTS int64) (int64, bool, error) { - defaultLocalCluster := moira.DefaultLocalCluster - triggersCount, err := check.database.GetTriggersToCheckCount(defaultLocalCluster) +// Check is a function that checks that the local checker checks triggers and the number of triggers is not constant. +func (heartbeater *localCheckerHeartbeater) Check() (State, error) { + triggersCount, err := heartbeater.database.GetTriggersToCheckCount(localClusterKey) if err != nil { - return 0, false, err + return StateError, err } - checksCount, _ := check.database.GetChecksUpdatesCount() - if check.count != checksCount || triggersCount == 0 { - check.count = checksCount - check.lastSuccessfulCheck = nowTS - return 0, false, nil + checksCount, err := heartbeater.database.GetChecksUpdatesCount() + if err != nil { + return StateError, err } - if check.lastSuccessfulCheck < nowTS-check.delay { - check.logger.Error(). - String("error", check.GetErrorMessage()). - Int64("time_since_successful_check", nowTS-check.heartbeat.lastSuccessfulCheck). - Msg("Send message") - - return nowTS - check.lastSuccessfulCheck, true, nil + now := heartbeater.clock.NowUTC() + if heartbeater.lastChecksCount != checksCount || triggersCount == 0 { + heartbeater.lastChecksCount = checksCount + heartbeater.lastSuccessfulCheck = now + return StateOK, nil } - return 0, false, nil -} + if now.Sub(heartbeater.lastSuccessfulCheck) > heartbeater.cfg.LocalCheckDelay { + return StateError, nil + } -func (localChecker) NeedToCheckOthers() bool { - return true + return StateOK, nil } -func (check localChecker) NeedTurnOffNotifier() bool { - return false +// Type is a function that returns the current heartbeat type. +func (localCheckerHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HeartbeatLocalChecker } -func (localChecker) GetErrorMessage() string { - return "Moira-Checker does not check triggers" +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater localCheckerHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/local_checker_test.go b/notifier/selfstate/heartbeat/local_checker_test.go index 7c638907a..cfe720674 100644 --- a/notifier/selfstate/heartbeat/local_checker_test.go +++ b/notifier/selfstate/heartbeat/local_checker_test.go @@ -5,84 +5,190 @@ import ( "testing" "time" - "github.com/moira-alert/moira" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestCheckDelay_Check(t *testing.T) { - defaultLocalCluster := moira.MakeClusterKey(moira.GraphiteLocal, moira.DefaultCluster) - Convey("Test local checker heartbeat", t, func() { - err := errors.New("test error localChecker") - now := time.Now().Unix() - check, mockCtrl := createGraphiteLocalCheckerTest(t) - defer mockCtrl.Finish() - database := check.database.(*mock_moira_alert.MockDatabase) - - Convey("Test creation localChecker", func() { - expected := &localChecker{heartbeat: heartbeat{database: check.database, logger: check.logger, delay: 1, lastSuccessfulCheck: now}} - So(GetLocalChecker(0, check.logger, check.database), ShouldBeNil) - So(GetLocalChecker(1, check.logger, check.database), ShouldResemble, expected) +const ( + defaultLocalCheckDelay = time.Minute +) + +func TestNewLocalCheckerHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + validationErr := validator.ValidationErrors{} + + Convey("Test NewLocalCheckerHeartbeater", t, func() { + Convey("With too low local check delay", func() { + cfg := LocalCheckerHeartbeaterConfig{ + LocalCheckDelay: -1, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(localCheckerHeartbeater, ShouldBeNil) + }) + + Convey("Without local check delay", func() { + cfg := LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(localCheckerHeartbeater, ShouldBeNil) + }) + + Convey("With correct local checker heartbeater config", func() { + cfg := LocalCheckerHeartbeaterConfig{ + LocalCheckDelay: 1, + } + + expected := &localCheckerHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(localCheckerHeartbeater, ShouldResemble, expected) + }) + }) +} + +func TestLocalCheckerHeartbeaterCheck(t *testing.T) { + database, clock, testTime, heartbeaterBase := heartbeaterHelper(t) + + cfg := LocalCheckerHeartbeaterConfig{ + LocalCheckDelay: defaultMetricReceivedDelay, + } + + localCheckerHeartbeater, _ := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + + var ( + testErr = errors.New("test error") + triggersToCheckCount, checksUpdatesCount int64 = 10, 10 + ) + + Convey("Test localCheckerHeartbeater.Check", t, func() { + Convey("With GetTriggersToCheckCount error", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, testErr) + + state, err := localCheckerHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("GraphiteLocalChecker error handling test", func() { - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), err) + Convey("With GetChecksUpdatesCount error", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, testErr) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldEqual, err) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + state, err := localCheckerHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("Test update lastSuccessfulCheck", func() { - now += 1000 - database.EXPECT().GetChecksUpdatesCount().Return(int64(1), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + Convey("With last checks count not equal current checks count", func() { + defer func() { + localCheckerHeartbeater.lastChecksCount = 0 + }() - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := localCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(localCheckerHeartbeater.lastChecksCount, ShouldResemble, checksUpdatesCount) }) - Convey("Test get notification", func() { - check.lastSuccessfulCheck = now - check.delay - 1 - database.EXPECT().GetChecksUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + Convey("With zero triggers to check count", func() { + defer func() { + localCheckerHeartbeater.lastChecksCount = 0 + }() + + var zeroTriggersToCheckCount int64 - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, now-check.lastSuccessfulCheck) + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(zeroTriggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := localCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(localCheckerHeartbeater.lastChecksCount, ShouldResemble, checksUpdatesCount) }) - Convey("Exit without action", func() { - database.EXPECT().GetChecksUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + localCheckerHeartbeater.lastChecksCount = checksUpdatesCount + + Convey("With too much time elapsed since the last successful check", func() { + localCheckerHeartbeater.lastSuccessfulCheck = testTime.Add(-10 * defaultLocalCheckDelay) + defer func() { + localCheckerHeartbeater.lastSuccessfulCheck = testTime + }() - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := localCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateError) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - // TODO(litleleprikon): seems that this test checks nothing. Seems that NeedToCheckOthers and NeedTurnOffNotifier do not work. - needCheck := check.NeedToCheckOthers() - So(needCheck, ShouldBeTrue) + Convey("With short time elapsed since the last successful check", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - So(check.NeedTurnOffNotifier(), ShouldBeFalse) + state, err := localCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) }) }) } -func createGraphiteLocalCheckerTest(t *testing.T) (*localChecker, *gomock.Controller) { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("CheckDelay") +func TestLocalCheckerHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test localCheckerHeartbeater.Type", t, func() { + cfg := LocalCheckerHeartbeaterConfig{ + LocalCheckDelay: defaultLocalCheckDelay, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + localCheckerHeartbeaterType := localCheckerHeartbeater.Type() + So(localCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatLocalChecker) + }) +} + +func TestLocalCheckerHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) - return GetLocalChecker(120, logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*localChecker), mockCtrl + Convey("Test localCheckerHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } + + cfg := LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + LocalCheckDelay: defaultLocalCheckDelay, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := localCheckerHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) } diff --git a/notifier/selfstate/heartbeat/notifier.go b/notifier/selfstate/heartbeat/notifier.go index 7b6877be4..b9f604b91 100644 --- a/notifier/selfstate/heartbeat/notifier.go +++ b/notifier/selfstate/heartbeat/notifier.go @@ -1,56 +1,55 @@ package heartbeat import ( - "fmt" - - "github.com/moira-alert/moira/metrics" - "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" ) -type notifier struct { - db moira.Database - log moira.Logger - metrics *metrics.HeartBeatMetrics -} +// Verify that notifierHeartbeater matches the Heartbeater interface. +var _ Heartbeater = (*notifierHeartbeater)(nil) -func GetNotifier(logger moira.Logger, database moira.Database, metrics *metrics.HeartBeatMetrics) Heartbeater { - return ¬ifier{ - db: database, - log: logger, - metrics: metrics, - } +// NotifierHeartbeaterConfig structure describing the notifierHeartbeater configuration. +type NotifierHeartbeaterConfig struct { + HeartbeaterBaseConfig } -func (check notifier) Check(int64) (int64, bool, error) { - state, _ := check.db.GetNotifierState() - if state != moira.SelfStateOK { - check.metrics.MarkNotifierIsAlive(false) +type notifierHeartbeater struct { + *heartbeaterBase + + cfg NotifierHeartbeaterConfig +} - check.log.Error(). - String("error", check.GetErrorMessage()). - Msg("Notifier is not healthy") +// NewNotifierHeartbeater is a function that creates a new notifierHeartbeater. +func NewNotifierHeartbeater( + cfg NotifierHeartbeaterConfig, + base *heartbeaterBase, +) (*notifierHeartbeater, error) { + return ¬ifierHeartbeater{ + cfg: cfg, + heartbeaterBase: base, + }, nil +} - return 0, true, nil +// Check is a function that returns the state of the notifier. +func (heartbeater *notifierHeartbeater) Check() (State, error) { + notifierState, err := heartbeater.database.GetNotifierState() + if err != nil { + return StateError, err } - check.metrics.MarkNotifierIsAlive(true) - - check.log.Debug(). - String("state", state). - Msg("Notifier is healthy") - return 0, false, nil -} + if notifierState != moira.SelfStateOK { + return StateError, nil + } -func (notifier) NeedTurnOffNotifier() bool { - return false + return StateOK, nil } -func (notifier) NeedToCheckOthers() bool { - return true +// Type is a function that returns the current heartbeat type. +func (notifierHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HeartbeatNotifier } -func (check notifier) GetErrorMessage() string { - state, _ := check.db.GetNotifierState() - return fmt.Sprintf("Moira-Notifier does not send messages. State: %v", state) +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater notifierHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/notifier_test.go b/notifier/selfstate/heartbeat/notifier_test.go index 3d4976035..a0a76bbc2 100644 --- a/notifier/selfstate/heartbeat/notifier_test.go +++ b/notifier/selfstate/heartbeat/notifier_test.go @@ -1,53 +1,103 @@ package heartbeat import ( + "errors" "testing" - "time" - - "github.com/moira-alert/moira/metrics" "github.com/moira-alert/moira" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + "github.com/moira-alert/moira/datatypes" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestNotifierState(t *testing.T) { - Convey("Test notifier delay heartbeat", t, func() { - now := time.Now().Unix() - check := createNotifierStateTest(t) +func TestNewNotifierHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test NewNotifierHeartbeater", t, func() { + Convey("With correct local checker heartbeater config", func() { + cfg := NotifierHeartbeaterConfig{} + + expected := ¬ifierHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } + + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(notifierHeartbeater, ShouldResemble, expected) + }) + }) +} + +func TestNotifierHeartbeaterCheck(t *testing.T) { + database, _, _, heartbeaterBase := heartbeaterHelper(t) + + cfg := NotifierHeartbeaterConfig{} + + notifierHeartbeater, _ := NewNotifierHeartbeater(cfg, heartbeaterBase) + + testErr := errors.New("test error") - Convey("Test get notifier delay", func() { - check.db.(*mock_moira_alert.MockDatabase).EXPECT().GetNotifierState().Return(moira.SelfStateOK, nil) + Convey("Test notifierHeartbeater.Check", t, func() { + Convey("With GetNotifierState error", func() { + database.EXPECT().GetNotifierState().Return(string(moira.SelfStateOK), testErr) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + state, err := notifierHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("Test get notification", func() { - check.db.(*mock_moira_alert.MockDatabase).EXPECT().GetNotifierState().Return(moira.SelfStateERROR, nil).Times(2) + Convey("With notifier state equals error", func() { + database.EXPECT().GetNotifierState().Return(moira.SelfStateERROR, nil) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, 0) + state, err := notifierHeartbeater.Check() + So(err, ShouldResemble, nil) + So(state, ShouldResemble, StateError) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - So(check.NeedTurnOffNotifier(), ShouldBeFalse) - So(check.NeedToCheckOthers(), ShouldBeTrue) + Convey("With notifier state equals ok", func() { + database.EXPECT().GetNotifierState().Return(moira.SelfStateOK, nil) + + state, err := notifierHeartbeater.Check() + So(err, ShouldResemble, nil) + So(state, ShouldResemble, StateOK) }) }) } -func createNotifierStateTest(t *testing.T) *notifier { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("MetricDelay") - metric := metrics.ConfigureHeartBeatMetrics(metrics.NewDummyRegistry()) +func TestNotifierHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test notifierHeartbeater.Type", t, func() { + cfg := NotifierHeartbeaterConfig{} + + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + notifierHeartbeaterType := notifierHeartbeater.Type() + So(notifierHeartbeaterType, ShouldResemble, datatypes.HeartbeatNotifier) + }) +} - return GetNotifier(logger, mock_moira_alert.NewMockDatabase(mockCtrl), metric).(*notifier) +func TestNotifierHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test notifierHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } + + cfg := NotifierHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + } + + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := notifierHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) } diff --git a/notifier/selfstate/heartbeat/remote_checker.go b/notifier/selfstate/heartbeat/remote_checker.go index 4b0e76113..1253184a8 100644 --- a/notifier/selfstate/heartbeat/remote_checker.go +++ b/notifier/selfstate/heartbeat/remote_checker.go @@ -1,60 +1,78 @@ package heartbeat import ( + "fmt" "time" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" ) -type remoteChecker struct { - heartbeat - count int64 +var ( + remoteClusterKey = moira.DefaultGraphiteRemoteCluster + + // Verify that remoteCheckerHeartbeater matches the Heartbeater interface. + _ Heartbeater = (*remoteCheckerHeartbeater)(nil) +) + +// RemoteCheckerHeartbeaterConfig structure describing the remoteCheckerHeartbeater configuration. +type RemoteCheckerHeartbeaterConfig struct { + HeartbeaterBaseConfig + + RemoteCheckDelay time.Duration `validate:"required_if=Enabled true,gte=0"` +} + +type remoteCheckerHeartbeater struct { + *heartbeaterBase + + cfg RemoteCheckerHeartbeaterConfig + lastRemoteChecksCount int64 } -func GetRemoteChecker(delay int64, logger moira.Logger, database moira.Database) Heartbeater { - if delay > 0 { - return &remoteChecker{heartbeat: heartbeat{ - logger: logger, - database: database, - delay: delay, - lastSuccessfulCheck: time.Now().Unix(), - }} +// NewRemoteCheckerHeartbeater is a function that creates a new remoteCheckerHeartbeater. +func NewRemoteCheckerHeartbeater(cfg RemoteCheckerHeartbeaterConfig, base *heartbeaterBase) (*remoteCheckerHeartbeater, error) { + if err := moira.ValidateStruct(cfg); err != nil { + return nil, fmt.Errorf("remote checker heartbeater configuration error: %w", err) } - return nil + + return &remoteCheckerHeartbeater{ + heartbeaterBase: base, + cfg: cfg, + }, nil } -func (check *remoteChecker) Check(nowTS int64) (int64, bool, error) { - defaultRemoteCluster := moira.DefaultGraphiteRemoteCluster - triggerCount, err := check.database.GetTriggersToCheckCount(defaultRemoteCluster) +// Check is a function that checks that the remote checker checks triggers and the number of triggers is not constant. +func (heartbeater remoteCheckerHeartbeater) Check() (State, error) { + triggersCount, err := heartbeater.database.GetTriggersToCheckCount(remoteClusterKey) + if err != nil { + return StateError, err + } + + remoteChecksCount, err := heartbeater.database.GetRemoteChecksUpdatesCount() if err != nil { - return 0, false, err + return StateError, err } - remoteTriggersCount, _ := check.database.GetRemoteChecksUpdatesCount() - if check.count != remoteTriggersCount || triggerCount == 0 { - check.count = remoteTriggersCount - check.lastSuccessfulCheck = nowTS - return 0, false, nil + now := heartbeater.clock.NowUTC() + if heartbeater.lastRemoteChecksCount != remoteChecksCount || triggersCount == 0 { + heartbeater.lastRemoteChecksCount = remoteChecksCount + heartbeater.lastSuccessfulCheck = now + return StateOK, nil } - if check.lastSuccessfulCheck < nowTS-check.delay { - check.logger.Error(). - String("error", check.GetErrorMessage()). - Int64("time_since_successful_check", nowTS-check.heartbeat.lastSuccessfulCheck). - Msg("Send message") - return nowTS - check.lastSuccessfulCheck, true, nil + if now.Sub(heartbeater.lastSuccessfulCheck) > heartbeater.cfg.RemoteCheckDelay { + return StateError, nil } - return 0, false, nil -} -func (check remoteChecker) NeedTurnOffNotifier() bool { - return false + return StateOK, nil } -func (remoteChecker) NeedToCheckOthers() bool { - return true +// Type is a function that returns the current heartbeat type. +func (remoteCheckerHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HeartbeatRemoteChecker } -func (remoteChecker) GetErrorMessage() string { - return "Moira-Remote-Checker does not check remote triggers" +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater remoteCheckerHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/remote_checker_test.go b/notifier/selfstate/heartbeat/remote_checker_test.go index a48ee30a8..6d7cf02e6 100644 --- a/notifier/selfstate/heartbeat/remote_checker_test.go +++ b/notifier/selfstate/heartbeat/remote_checker_test.go @@ -5,84 +5,190 @@ import ( "testing" "time" - "github.com/moira-alert/moira" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestGraphiteRemoteChecker(t *testing.T) { - defaultRemoteCluster := moira.DefaultGraphiteRemoteCluster +const ( + defaultRemoteCheckDelay = time.Minute +) + +func TestNewRemoteCheckerHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + validationErr := validator.ValidationErrors{} + + Convey("Test NewRemoteCheckerHeartbeater", t, func() { + Convey("With too low remote check delay", func() { + cfg := RemoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: -1, + } + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(remoteCheckerHeartbeater, ShouldBeNil) + }) + + Convey("Without remote check delay", func() { + cfg := RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(remoteCheckerHeartbeater, ShouldBeNil) + }) + + Convey("With correct remote checker heartbeater config", func() { + cfg := RemoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: 1, + } + + expected := &remoteCheckerHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(remoteCheckerHeartbeater, ShouldResemble, expected) + }) + }) +} + +func TestRemoteCheckerHeartbeaterCheck(t *testing.T) { + database, clock, testTime, heartbeaterBase := heartbeaterHelper(t) + + cfg := RemoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: defaultRemoteCheckDelay, + } - Convey("Test remote checker heartbeat", t, func() { - err := errors.New("test error remoteChecker") - now := time.Now().Unix() - check, mockCtrl := createGraphiteRemoteCheckerTest(t) - defer mockCtrl.Finish() - database := check.database.(*mock_moira_alert.MockDatabase) + remoteCheckerHeartbeater, _ := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) - Convey("Checking the created graphite remote checker", func() { - expected := &remoteChecker{heartbeat: heartbeat{database: check.database, logger: check.logger, delay: 1, lastSuccessfulCheck: now}} - So(GetRemoteChecker(0, check.logger, check.database), ShouldBeNil) - So(GetRemoteChecker(1, check.logger, check.database), ShouldResemble, expected) + var ( + testErr = errors.New("test error") + triggersToCheckCount, remoteChecksUpdatesCount int64 = 10, 10 + ) + + Convey("Test remoteCheckerHeartbeater.Check", t, func() { + Convey("With GetTriggersToCheckCount error", func() { + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, testErr) + + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("GraphiteRemoteChecker error handling test", func() { - database.EXPECT().GetTriggersToCheckCount(defaultRemoteCluster).Return(int64(0), err) + Convey("With GetRemoteChecksUpdatesCount error", func() { + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, testErr) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldEqual, err) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("Test update lastSuccessfulCheck", func() { - now += 1000 - database.EXPECT().GetRemoteChecksUpdatesCount().Return(int64(1), nil) - database.EXPECT().GetTriggersToCheckCount(defaultRemoteCluster).Return(int64(1), nil) + Convey("With last remote checks count not equal current remote checks count", func() { + defer func() { + remoteCheckerHeartbeater.lastRemoteChecksCount = 0 + }() + + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(remoteCheckerHeartbeater.lastRemoteChecksCount, ShouldResemble, remoteChecksUpdatesCount) }) - Convey("Check for notification", func() { - check.lastSuccessfulCheck = now - check.delay - 1 + Convey("With zero triggers to check count", func() { + defer func() { + remoteCheckerHeartbeater.lastRemoteChecksCount = 0 + }() - database.EXPECT().GetRemoteChecksUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultRemoteCluster).Return(int64(1), nil) + var zeroTriggersToCheckCount int64 - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, now-check.lastSuccessfulCheck) + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(zeroTriggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(remoteCheckerHeartbeater.lastRemoteChecksCount, ShouldResemble, remoteChecksUpdatesCount) }) - Convey("Exit without action", func() { - database.EXPECT().GetRemoteChecksUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultRemoteCluster).Return(int64(1), nil) + remoteCheckerHeartbeater.lastRemoteChecksCount = remoteChecksUpdatesCount + + Convey("With too much time elapsed since the last successful check", func() { + remoteCheckerHeartbeater.lastSuccessfulCheck = testTime.Add(-10 * defaultRemoteCheckDelay) + defer func() { + remoteCheckerHeartbeater.lastSuccessfulCheck = testTime + }() - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateError) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - // TODO(litleleprikon): seems that this test checks nothing. Seems that NeedToCheckOthers and NeedTurnOffNotifier do not work. - So(check.NeedToCheckOthers(), ShouldBeTrue) - So(check.NeedTurnOffNotifier(), ShouldBeFalse) + Convey("With short time elapsed since the last successful check", func() { + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) }) }) } -func createGraphiteRemoteCheckerTest(t *testing.T) (*remoteChecker, *gomock.Controller) { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("MetricDelay") +func TestRemoteCheckerHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test remoteCheckerHeartbeater.Type", t, func() { + cfg := RemoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: defaultRemoteCheckDelay, + } + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) - return GetRemoteChecker(120, logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*remoteChecker), mockCtrl + remoteCheckerHeartbeaterType := remoteCheckerHeartbeater.Type() + So(remoteCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatRemoteChecker) + }) +} + +func TestRemoteCheckerHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test remoteCheckerHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } + + cfg := RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + RemoteCheckDelay: defaultRemoteCheckDelay, + } + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := remoteCheckerHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) }