From 567971e69b9231e5ecee95026e0c4d105924e877 Mon Sep 17 00:00:00 2001 From: almostinf Date: Thu, 31 Oct 2024 18:58:18 +0300 Subject: [PATCH 1/8] implement heartbeaters --- go.mod | 10 +- go.sum | 15 +- notifier/selfstate/heartbeat/database.go | 80 +++--- notifier/selfstate/heartbeat/database_test.go | 180 ++++++++++---- notifier/selfstate/heartbeat/filter.go | 103 ++++---- notifier/selfstate/heartbeat/filter_test.go | 235 +++++++++++++----- notifier/selfstate/heartbeat/heartbeat.go | 64 ++++- notifier/selfstate/heartbeat/local_checker.go | 92 ++++--- .../selfstate/heartbeat/local_checker_test.go | 233 ++++++++++++----- notifier/selfstate/heartbeat/notifier.go | 71 +++--- notifier/selfstate/heartbeat/notifier_test.go | 128 +++++++--- .../selfstate/heartbeat/remote_checker.go | 97 +++++--- .../heartbeat/remote_checker_test.go | 229 +++++++++++++---- 13 files changed, 1109 insertions(+), 428 deletions(-) diff --git a/go.mod b/go.mod index 91857a6eb..e1fcbeb76 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,9 @@ require ( require github.com/prometheus/common v0.37.0 require ( - github.com/golang/mock v1.6.0 + github.com/elliotchance/sshtunnel v1.6.1 + github.com/go-playground/validator v9.31.0+incompatible + github.com/go-playground/validator/v10 v10.4.1 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/mattermost/mattermost/server/public v0.1.1 github.com/mitchellh/mapstructure v1.5.0 @@ -157,7 +159,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.24.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/image v0.13.0 // indirect golang.org/x/net v0.26.0 // indirect @@ -184,12 +186,15 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/spec v0.20.9 // indirect github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-plugin v1.6.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/leodido/go-urn v1.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -202,6 +207,7 @@ require ( golang.org/x/tools v0.22.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/grpc v1.62.0 // indirect + gopkg.in/go-playground/assert.v1 v1.2.1 // indirect ) // Have to exclude version that is incorectly retracted by authors. diff --git a/go.sum b/go.sum index d95d23b39..b5e77e67d 100644 --- a/go.sum +++ b/go.sum @@ -562,6 +562,8 @@ github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc= +github.com/elliotchance/sshtunnel v1.6.1 h1:uunvEbhtzDqEyl58E1qC7j2sDFXhtEcj0sEsc33e/Gw= +github.com/elliotchance/sshtunnel v1.6.1/go.mod h1:gRPilFGawrilzilJ+4ySFZxu/qoNZN++GQQ1HVFrVJk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -637,9 +639,15 @@ github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA= +github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= @@ -680,7 +688,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -899,6 +906,7 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -1255,6 +1263,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -1543,6 +1552,8 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1906,6 +1917,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= diff --git a/notifier/selfstate/heartbeat/database.go b/notifier/selfstate/heartbeat/database.go index 5b8cd551f..632a01a59 100644 --- a/notifier/selfstate/heartbeat/database.go +++ b/notifier/selfstate/heartbeat/database.go @@ -1,52 +1,74 @@ package heartbeat import ( + "fmt" "time" - "github.com/moira-alert/moira" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" ) -type databaseHeartbeat struct{ heartbeat } +// Verify that databaseHeartbeater matches the Heartbeater interface. +var _ Heartbeater = (*databaseHeartbeater)(nil) -func GetDatabase(delay int64, logger moira.Logger, database moira.Database) Heartbeater { - if delay > 0 { - return &databaseHeartbeat{heartbeat{ - logger: logger, - database: database, - delay: delay, - lastSuccessfulCheck: time.Now().Unix(), - }} +// DatabaseHeartbeaterConfig structure describing the databaseHeartbeater configuration. +type DatabaseHeartbeaterConfig struct { + HeartbeaterBaseConfig + + RedisDisconnectDelay time.Duration `validate:"required,gt=0"` +} + +func (cfg DatabaseHeartbeaterConfig) validate() error { + validator := validator.New() + return validator.Struct(cfg) +} + +type databaseHeartbeater struct { + *heartbeaterBase + + cfg DatabaseHeartbeaterConfig +} + +// NewDatabaseHeartbeater is a function that creates a new databaseHeartbeater. +func NewDatabaseHeartbeater(cfg DatabaseHeartbeaterConfig, base *heartbeaterBase) (*databaseHeartbeater, error) { + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("database heartbeater configuration error: %w", err) } - return nil + + return &databaseHeartbeater{ + heartbeaterBase: base, + cfg: cfg, + }, nil } -func (check *databaseHeartbeat) Check(nowTS int64) (int64, bool, error) { - _, err := check.database.GetChecksUpdatesCount() +// Check is a function that checks if the database is working correctly. +func (heartbeater *databaseHeartbeater) Check() (State, error) { + now := heartbeater.clock.NowUTC() + + _, err := heartbeater.database.GetChecksUpdatesCount() if err == nil { - check.lastSuccessfulCheck = nowTS - return 0, false, nil + heartbeater.lastSuccessfulCheck = now + return StateOK, nil } - if check.lastSuccessfulCheck < nowTS-check.delay { - check.logger.Error(). - String("error", check.GetErrorMessage()). - Int64("time_since_successful_check", nowTS-check.heartbeat.lastSuccessfulCheck). - Msg("Send message") - - return nowTS - check.lastSuccessfulCheck, true, nil + if now.Sub(heartbeater.lastSuccessfulCheck) > heartbeater.cfg.RedisDisconnectDelay { + return StateError, nil } - return 0, false, nil + return StateOK, err } -func (databaseHeartbeat) NeedTurnOffNotifier() bool { - return true +// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. +func (heartbeater databaseHeartbeater) NeedTurnOffNotifier() bool { + return heartbeater.cfg.NeedTurnOffNotifier } -func (databaseHeartbeat) NeedToCheckOthers() bool { - return false +// Type is a function that returns the current heartbeat type. +func (databaseHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HearbeatTypeNotSet } -func (databaseHeartbeat) GetErrorMessage() string { - return "Redis disconnected" +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater databaseHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/database_test.go b/notifier/selfstate/heartbeat/database_test.go index a5b8b8a5a..c4c36e873 100644 --- a/notifier/selfstate/heartbeat/database_test.go +++ b/notifier/selfstate/heartbeat/database_test.go @@ -5,69 +5,163 @@ import ( "testing" "time" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" - - logging "github.com/moira-alert/moira/logging/zerolog_adapter" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestDatabaseHeartbeat(t *testing.T) { - Convey("Test database heartbeat", t, func() { - now := time.Now().Unix() - err := errors.New("test database error") - check := createRedisDelayTest(t) - database := check.database.(*mock_moira_alert.MockDatabase) +const ( + defaultRedisDisconnectDelay = time.Minute +) + +func TestNewDatabaseHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + validationErr := validator.ValidationErrors{} - Convey("Checking the created heartbeat database", func() { - expected := &databaseHeartbeat{heartbeat{database: check.database, logger: check.logger, delay: 1, lastSuccessfulCheck: now}} + Convey("Test NewDatabaseHeartbeater", t, func() { + Convey("With too low redis disconnect delay", func() { + cfg := DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: -1, + } - So(GetDatabase(0, check.logger, check.database), ShouldBeNil) - So(GetDatabase(1, check.logger, check.database), ShouldResemble, expected) + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(databaseHeartbeater, ShouldBeNil) }) - Convey("Test update lastSuccessfulCheck", func() { - now += 1000 - database.EXPECT().GetChecksUpdatesCount().Return(int64(1), nil) + Convey("Without redis disconnect delay", func() { + cfg := DatabaseHeartbeaterConfig{} + + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(databaseHeartbeater, ShouldBeNil) + }) + + Convey("With correct database heartbeater config", func() { + cfg := DatabaseHeartbeaterConfig{ + RedisDisconnectDelay: 1, + } + + expected := &databaseHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(databaseHeartbeater, ShouldResemble, expected) }) + }) +} + +func TestDatabaseHeartbeaterCheck(t *testing.T) { + database, clock, testTime, heartbeaterBase := heartbeaterHelper(t) + + cfg := DatabaseHeartbeaterConfig{ + RedisDisconnectDelay: defaultRedisDisconnectDelay, + } - Convey("Database error handling test", func() { - database.EXPECT().GetChecksUpdatesCount().Return(int64(1), err) + databaseHeartbeater, _ := NewDatabaseHeartbeater(cfg, heartbeaterBase) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + var ( + testErr = errors.New("test error") + checkUpdates int64 + ) + + Convey("Test databaseHeartbeater.Check", t, func() { + Convey("With nil error in GetCheckUpdatedCount", func() { + database.EXPECT().GetChecksUpdatesCount().Return(checkUpdates, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := databaseHeartbeater.Check() + So(state, ShouldResemble, StateOK) + So(err, ShouldBeNil) }) - Convey("Check for notification", func() { - check.lastSuccessfulCheck = now - check.delay - 1 + Convey("With too much time elapsed since the last successful check", func() { + heartbeaterBase.lastSuccessfulCheck = testTime.Add(-10 * defaultRedisDisconnectDelay) + defer func() { + heartbeaterBase.lastSuccessfulCheck = testTime + }() - database.EXPECT().GetChecksUpdatesCount().Return(int64(0), err) + database.EXPECT().GetChecksUpdatesCount().Return(checkUpdates, testErr) + clock.EXPECT().NowUTC().Return(testTime) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, now-check.lastSuccessfulCheck) + state, err := databaseHeartbeater.Check() + So(state, ShouldResemble, StateError) + So(err, ShouldBeNil) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - So(check.NeedTurnOffNotifier(), ShouldBeTrue) - So(check.NeedToCheckOthers(), ShouldBeFalse) + Convey("With only error from GetChecksUpdateCount", func() { + database.EXPECT().GetChecksUpdatesCount().Return(checkUpdates, testErr) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := databaseHeartbeater.Check() + So(state, ShouldResemble, StateOK) + So(err, ShouldResemble, testErr) }) }) } -func createRedisDelayTest(t *testing.T) *databaseHeartbeat { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("CheckDelay") +func TestDatabaseHeartbeaterNeedTurnOffNotifier(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test databaseHeartbeater.TurnOffNotifier", t, func() { + cfg := DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + NeedTurnOffNotifier: true, + }, + RedisDisconnectDelay: defaultRedisDisconnectDelay, + } - return GetDatabase(10, logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*databaseHeartbeat) + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + needTurnOffNotifier := databaseHeartbeater.NeedTurnOffNotifier() + So(needTurnOffNotifier, ShouldBeTrue) + }) +} + +func TestDatabaseHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test databaseHeartbeater.Type", t, func() { + cfg := DatabaseHeartbeaterConfig{ + RedisDisconnectDelay: defaultRedisDisconnectDelay, + } + + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + databaseHeartbeaterType := databaseHeartbeater.Type() + So(databaseHeartbeaterType, ShouldResemble, datatypes.HearbeatTypeNotSet) + }) +} + +func TestDatabaseHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test databaseHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } + + cfg := DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + RedisDisconnectDelay: defaultRedisDisconnectDelay, + } + + databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := databaseHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) } diff --git a/notifier/selfstate/heartbeat/filter.go b/notifier/selfstate/heartbeat/filter.go index 80f000e35..2dbfd2bc5 100644 --- a/notifier/selfstate/heartbeat/filter.go +++ b/notifier/selfstate/heartbeat/filter.go @@ -1,70 +1,89 @@ package heartbeat import ( + "fmt" "time" + "github.com/go-playground/validator/v10" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" ) -type filter struct { - heartbeat - count int64 - firstCheckWasSuccessful bool +var ( + localClusterKey = moira.DefaultLocalCluster + + // Verify that filterHeartbeater matches the Heartbeater interface. + _ Heartbeater = (*filterHeartbeater)(nil) +) + +// FilterHeartbeaterConfig structure describing the filterHeartbeater configuration. +type FilterHeartbeaterConfig struct { + HeartbeaterBaseConfig + + MetricReceivedDelay time.Duration `validate:"required,gt=0"` } -func GetFilter(delay int64, logger moira.Logger, database moira.Database) Heartbeater { - if delay > 0 { - return &filter{ - heartbeat: heartbeat{ - logger: logger, - database: database, - delay: delay, - lastSuccessfulCheck: time.Now().Unix(), - }, - firstCheckWasSuccessful: false, - } +func (cfg FilterHeartbeaterConfig) validate() error { + validator := validator.New() + return validator.Struct(cfg) +} + +type filterHeartbeater struct { + *heartbeaterBase + + cfg FilterHeartbeaterConfig + lastMetricsCount int64 +} + +// NewFilterHeartbeater is a function that creates a new filterHeartbeater. +func NewFilterHeartbeater(cfg FilterHeartbeaterConfig, base *heartbeaterBase) (*filterHeartbeater, error) { + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("filter heartheater configuration error: %w", err) } - return nil + + return &filterHeartbeater{ + heartbeaterBase: base, + cfg: cfg, + }, nil } -func (check *filter) Check(nowTS int64) (int64, bool, error) { - defaultLocalCluster := moira.DefaultLocalCluster - triggersCount, err := check.database.GetTriggersToCheckCount(defaultLocalCluster) +// Check is a function that checks that filters accept metrics and that their number of metrics is not constant. +func (heartbeater *filterHeartbeater) Check() (State, error) { + triggersCount, err := heartbeater.database.GetTriggersToCheckCount(localClusterKey) if err != nil { - return 0, false, err + return StateError, err } - metricsCount, err := check.database.GetMetricsUpdatesCount() + metricsCount, err := heartbeater.database.GetMetricsUpdatesCount() if err != nil { - return 0, false, err - } - if check.count != metricsCount || triggersCount == 0 { - check.count = metricsCount - check.lastSuccessfulCheck = nowTS - return 0, false, nil + return StateError, err } - if check.lastSuccessfulCheck < nowTS-check.heartbeat.delay { - check.logger.Error(). - String("error", check.GetErrorMessage()). - Int64("time_since_successful_check", nowTS-check.heartbeat.lastSuccessfulCheck). - Msg("Send message") + now := heartbeater.clock.NowUTC() + if heartbeater.lastMetricsCount != metricsCount || triggersCount == 0 { + heartbeater.lastMetricsCount = metricsCount + heartbeater.lastSuccessfulCheck = now + return StateOK, nil + } - check.firstCheckWasSuccessful = true - return nowTS - check.heartbeat.lastSuccessfulCheck, true, nil + if now.Sub(heartbeater.lastSuccessfulCheck) > heartbeater.cfg.MetricReceivedDelay { + return StateError, nil } - return 0, false, nil + + return StateOK, nil } -// NeedTurnOffNotifier: turn off notifications if at least once the filter check was successful. -func (check filter) NeedTurnOffNotifier() bool { - return check.firstCheckWasSuccessful +// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. +func (heartbeater filterHeartbeater) NeedTurnOffNotifier() bool { + return heartbeater.cfg.NeedTurnOffNotifier } -func (check filter) NeedToCheckOthers() bool { - return true +// Type is a function that returns the current heartbeat type. +func (filterHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HearbeatTypeNotSet } -func (filter) GetErrorMessage() string { - return "Moira-Filter does not receive metrics" +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater filterHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/filter_test.go b/notifier/selfstate/heartbeat/filter_test.go index 678c443d9..3c6b6e0a6 100644 --- a/notifier/selfstate/heartbeat/filter_test.go +++ b/notifier/selfstate/heartbeat/filter_test.go @@ -5,92 +5,205 @@ import ( "testing" "time" - "github.com/moira-alert/moira" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestFilter(t *testing.T) { - Convey("Test filter heartbeat", t, func() { - err := errors.New("test filter error") - now := time.Now().Unix() - check, mockCtrl := createFilterTest(t) - defer mockCtrl.Finish() - database := check.database.(*mock_moira_alert.MockDatabase) - defaultLocalCluster := moira.MakeClusterKey(moira.GraphiteLocal, moira.DefaultCluster) - - Convey("Checking the created filter", func() { - expected := &filter{ - heartbeat: heartbeat{ - database: check.database, - logger: check.logger, - delay: 1, - lastSuccessfulCheck: now, - }, +const ( + defaultMetricReceivedDelay = time.Minute +) + +func TestNewFilterHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + validationErr := validator.ValidationErrors{} + + Convey("Test NewFilterHeartbeater", t, func() { + Convey("With too low metric received delay", func() { + cfg := FilterHeartbeaterConfig{ + MetricReceivedDelay: -1, + } + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(filterHeartbeater, ShouldBeNil) + }) + + Convey("Without metric received delay", func() { + cfg := FilterHeartbeaterConfig{} + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(filterHeartbeater, ShouldBeNil) + }) + + Convey("With correct filter heartbeater config", func() { + cfg := FilterHeartbeaterConfig{ + MetricReceivedDelay: 1, } - So(GetFilter(0, check.logger, check.database), ShouldBeNil) - So(GetFilter(1, check.logger, check.database), ShouldResemble, expected) + expected := &filterHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(filterHeartbeater, ShouldResemble, expected) + }) + }) +} + +func TestFilterHeartbeaterCheck(t *testing.T) { + database, clock, testTime, heartbeaterBase := heartbeaterHelper(t) + + cfg := FilterHeartbeaterConfig{ + MetricReceivedDelay: defaultMetricReceivedDelay, + } + + filterHeartbeater, _ := NewFilterHeartbeater(cfg, heartbeaterBase) + + var ( + testErr = errors.New("test error") + triggersToCheckCount, metricsUpdatesCount int64 = 10, 10 + ) + + Convey("Test filterHeartbeater.Check", t, func() { + Convey("With GetTriggersToCheckCount error", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, testErr) + + state, err := filterHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("Filter error handling test", func() { - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), err) + Convey("With GetMetricsUpdatesCount error", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, testErr) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldEqual, err) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + state, err := filterHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("Test update lastSuccessfulCheck", func() { - now += 1000 - database.EXPECT().GetMetricsUpdatesCount().Return(int64(1), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + Convey("With last metrics count not equal current metrics count", func() { + defer func() { + filterHeartbeater.lastMetricsCount = 0 + }() + + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + state, err := filterHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(filterHeartbeater.lastMetricsCount, ShouldResemble, metricsUpdatesCount) }) - Convey("Check for notification", func() { - check.lastSuccessfulCheck = now - check.delay - 1 + Convey("With zero triggers to check count", func() { + defer func() { + filterHeartbeater.lastMetricsCount = 0 + }() + + var zeroTriggersToCheckCount int64 - database.EXPECT().GetMetricsUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(zeroTriggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, now-check.lastSuccessfulCheck) + state, err := filterHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(filterHeartbeater.lastMetricsCount, ShouldResemble, metricsUpdatesCount) }) - Convey("Exit without action", func() { - database.EXPECT().GetMetricsUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + filterHeartbeater.lastMetricsCount = metricsUpdatesCount - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + Convey("With too much time elapsed since the last successful check", func() { + filterHeartbeater.lastSuccessfulCheck = testTime.Add(-10 * defaultMetricReceivedDelay) + defer func() { + filterHeartbeater.lastSuccessfulCheck = testTime + }() + + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := filterHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateError) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - // TODO(litleleprikon): seems that this test checks nothing. Seems that NeedToCheckOthers and NeedTurnOffNotifier do not work. - So(check.NeedToCheckOthers(), ShouldBeTrue) + Convey("With short time elapsed since the last successful check", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetMetricsUpdatesCount().Return(metricsUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - So(check.NeedTurnOffNotifier(), ShouldBeFalse) + state, err := filterHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) }) }) } -func createFilterTest(t *testing.T) (*filter, *gomock.Controller) { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("MetricDelay") +func TestFilterHeartbeaterNeedTurnOffNotifier(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test filterHeartbeater.TurnOffNotifier", t, func() { + cfg := FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + NeedTurnOffNotifier: true, + }, + MetricReceivedDelay: defaultMetricReceivedDelay, + } + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + needTurnOffNotifier := filterHeartbeater.NeedTurnOffNotifier() + So(needTurnOffNotifier, ShouldBeTrue) + }) +} + +func TestFilterHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) - return GetFilter(60, logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*filter), mockCtrl + Convey("Test filterHeartbeater.Type", t, func() { + cfg := FilterHeartbeaterConfig{ + MetricReceivedDelay: defaultMetricReceivedDelay, + } + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + filterHeartbeaterType := filterHeartbeater.Type() + So(filterHeartbeaterType, ShouldResemble, datatypes.HearbeatTypeNotSet) + }) +} + +func TestFilterHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test filterHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } + + cfg := FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + MetricReceivedDelay: defaultMetricReceivedDelay, + } + + filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := filterHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) } diff --git a/notifier/selfstate/heartbeat/heartbeat.go b/notifier/selfstate/heartbeat/heartbeat.go index 244de1bcd..ef8d72852 100644 --- a/notifier/selfstate/heartbeat/heartbeat.go +++ b/notifier/selfstate/heartbeat/heartbeat.go @@ -1,21 +1,73 @@ package heartbeat import ( + "time" + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" +) + +// State characterises the state of the heartbeat. +type State string + +const ( + StateOK State = "heartbeat_state_ok" + StateError State = "heartbeat_state_error" ) +// IsDegraded checks if the condition has degraded. +func (lastState State) IsDegraded(newState State) bool { + return lastState == StateOK && newState == StateError +} + +// IsRecovered checks if the condition has recovered. +func (lastState State) IsRecovered(newState State) bool { + return lastState == StateError && newState == StateOK +} + // Heartbeater is the interface for simplified events verification. type Heartbeater interface { - Check(int64) (int64, bool, error) + Check() (State, error) NeedTurnOffNotifier() bool - NeedToCheckOthers() bool - GetErrorMessage() string + AlertSettings() AlertConfig + Type() datatypes.HeartbeatType } -// heartbeat basic structure for Heartbeater. -type heartbeat struct { +// HeartbeaterBaseConfig contains common fields for all heartbeaters. +type HeartbeaterBaseConfig struct { + Enabled bool + NeedTurnOffNotifier bool + NeedToCheckOthers bool + + AlertCfg AlertConfig `validate:"required_if=Enabled true"` +} + +// AlertConfig contains the configuration of the alerts that heartbeater sends out. +type AlertConfig struct { + Name string `validate:"required_if=Enabled true"` + Desc string +} + +// HeartbeatBase is basic structure for Heartbeater. +type heartbeaterBase struct { logger moira.Logger database moira.Database + clock moira.Clock + + lastSuccessfulCheck time.Time +} + +// NewHeartbeaterBase function that creates a base for heartbeater. +func NewHeartbeaterBase( + logger moira.Logger, + database moira.Database, + clock moira.Clock, +) *heartbeaterBase { + return &heartbeaterBase{ + logger: logger, + database: database, + clock: clock, - delay, lastSuccessfulCheck int64 + lastSuccessfulCheck: clock.NowUTC(), + } } diff --git a/notifier/selfstate/heartbeat/local_checker.go b/notifier/selfstate/heartbeat/local_checker.go index 7c4c88cfd..0eb7d208d 100644 --- a/notifier/selfstate/heartbeat/local_checker.go +++ b/notifier/selfstate/heartbeat/local_checker.go @@ -1,62 +1,84 @@ package heartbeat import ( + "fmt" "time" - "github.com/moira-alert/moira" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" ) -type localChecker struct { - heartbeat - count int64 +// Verify that localCheckerHeartbeater matches the Heartbeater interface. +var _ Heartbeater = (*localCheckerHeartbeater)(nil) + +// LocalCheckerHeartbeaterConfig structure describing the localCheckerHeartbeater configuration. +type LocalCheckerHeartbeaterConfig struct { + HeartbeaterBaseConfig + + LocalCheckDelay time.Duration `validate:"required,gt=0"` +} + +func (cfg LocalCheckerHeartbeaterConfig) validate() error { + validator := validator.New() + return validator.Struct(cfg) +} + +type localCheckerHeartbeater struct { + *heartbeaterBase + + cfg LocalCheckerHeartbeaterConfig + lastChecksCount int64 } -func GetLocalChecker(delay int64, logger moira.Logger, database moira.Database) Heartbeater { - if delay > 0 { - return &localChecker{heartbeat: heartbeat{ - logger: logger, - database: database, - delay: delay, - lastSuccessfulCheck: time.Now().Unix(), - }} +// NewLocalCheckerHeartbeater is a function that creates a new localCheckerHeartbeater. +func NewLocalCheckerHeartbeater(cfg LocalCheckerHeartbeaterConfig, base *heartbeaterBase) (*localCheckerHeartbeater, error) { + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("local checker heartbeater configuration error: %w", err) } - return nil + + return &localCheckerHeartbeater{ + heartbeaterBase: base, + cfg: cfg, + }, nil } -func (check *localChecker) Check(nowTS int64) (int64, bool, error) { - defaultLocalCluster := moira.DefaultLocalCluster - triggersCount, err := check.database.GetTriggersToCheckCount(defaultLocalCluster) +// Check is a function that checks that the local checker checks triggers and the number of triggers is not constant. +func (heartbeater *localCheckerHeartbeater) Check() (State, error) { + triggersCount, err := heartbeater.database.GetTriggersToCheckCount(localClusterKey) if err != nil { - return 0, false, err + return StateError, err } - checksCount, _ := check.database.GetChecksUpdatesCount() - if check.count != checksCount || triggersCount == 0 { - check.count = checksCount - check.lastSuccessfulCheck = nowTS - return 0, false, nil + checksCount, err := heartbeater.database.GetChecksUpdatesCount() + if err != nil { + return StateError, err } - if check.lastSuccessfulCheck < nowTS-check.delay { - check.logger.Error(). - String("error", check.GetErrorMessage()). - Int64("time_since_successful_check", nowTS-check.heartbeat.lastSuccessfulCheck). - Msg("Send message") + now := heartbeater.clock.NowUTC() + if heartbeater.lastChecksCount != checksCount || triggersCount == 0 { + heartbeater.lastChecksCount = checksCount + heartbeater.lastSuccessfulCheck = now + return StateOK, nil + } - return nowTS - check.lastSuccessfulCheck, true, nil + if now.Sub(heartbeater.lastSuccessfulCheck) > heartbeater.cfg.LocalCheckDelay { + return StateError, nil } - return 0, false, nil + return StateOK, nil } -func (localChecker) NeedToCheckOthers() bool { - return true +// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. +func (heartbeater localCheckerHeartbeater) NeedTurnOffNotifier() bool { + return heartbeater.cfg.NeedTurnOffNotifier } -func (check localChecker) NeedTurnOffNotifier() bool { - return false +// Type is a function that returns the current heartbeat type. +func (localCheckerHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HearbeatTypeNotSet } -func (localChecker) GetErrorMessage() string { - return "Moira-Checker does not check triggers" +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater localCheckerHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/local_checker_test.go b/notifier/selfstate/heartbeat/local_checker_test.go index 7c638907a..42c1cd4b7 100644 --- a/notifier/selfstate/heartbeat/local_checker_test.go +++ b/notifier/selfstate/heartbeat/local_checker_test.go @@ -5,84 +5,205 @@ import ( "testing" "time" - "github.com/moira-alert/moira" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestCheckDelay_Check(t *testing.T) { - defaultLocalCluster := moira.MakeClusterKey(moira.GraphiteLocal, moira.DefaultCluster) - Convey("Test local checker heartbeat", t, func() { - err := errors.New("test error localChecker") - now := time.Now().Unix() - check, mockCtrl := createGraphiteLocalCheckerTest(t) - defer mockCtrl.Finish() - database := check.database.(*mock_moira_alert.MockDatabase) - - Convey("Test creation localChecker", func() { - expected := &localChecker{heartbeat: heartbeat{database: check.database, logger: check.logger, delay: 1, lastSuccessfulCheck: now}} - So(GetLocalChecker(0, check.logger, check.database), ShouldBeNil) - So(GetLocalChecker(1, check.logger, check.database), ShouldResemble, expected) +const ( + defaultLocalCheckDelay = time.Minute +) + +func TestNewLocalCheckerHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + validationErr := validator.ValidationErrors{} + + Convey("Test NewLocalCheckerHeartbeater", t, func() { + Convey("With too low local check delay", func() { + cfg := LocalCheckerHeartbeaterConfig{ + LocalCheckDelay: -1, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(localCheckerHeartbeater, ShouldBeNil) + }) + + Convey("Without local check delay", func() { + cfg := LocalCheckerHeartbeaterConfig{} + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(localCheckerHeartbeater, ShouldBeNil) + }) + + Convey("With correct local checker heartbeater config", func() { + cfg := LocalCheckerHeartbeaterConfig{ + LocalCheckDelay: 1, + } + + expected := &localCheckerHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(localCheckerHeartbeater, ShouldResemble, expected) + }) + }) +} + +func TestLocalCheckerHeartbeaterCheck(t *testing.T) { + database, clock, testTime, heartbeaterBase := heartbeaterHelper(t) + + cfg := LocalCheckerHeartbeaterConfig{ + LocalCheckDelay: defaultMetricReceivedDelay, + } + + localCheckerHeartbeater, _ := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + + var ( + testErr = errors.New("test error") + triggersToCheckCount, checksUpdatesCount int64 = 10, 10 + ) + + Convey("Test localCheckerHeartbeater.Check", t, func() { + Convey("With GetTriggersToCheckCount error", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, testErr) + + state, err := localCheckerHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("GraphiteLocalChecker error handling test", func() { - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), err) + Convey("With GetChecksUpdatesCount error", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, testErr) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldEqual, err) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + state, err := localCheckerHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("Test update lastSuccessfulCheck", func() { - now += 1000 - database.EXPECT().GetChecksUpdatesCount().Return(int64(1), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + Convey("With last checks count not equal current checks count", func() { + defer func() { + localCheckerHeartbeater.lastChecksCount = 0 + }() - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := localCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(localCheckerHeartbeater.lastChecksCount, ShouldResemble, checksUpdatesCount) }) - Convey("Test get notification", func() { - check.lastSuccessfulCheck = now - check.delay - 1 - database.EXPECT().GetChecksUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + Convey("With zero triggers to check count", func() { + defer func() { + localCheckerHeartbeater.lastChecksCount = 0 + }() + + var zeroTriggersToCheckCount int64 + + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(zeroTriggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, now-check.lastSuccessfulCheck) + state, err := localCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(localCheckerHeartbeater.lastChecksCount, ShouldResemble, checksUpdatesCount) }) - Convey("Exit without action", func() { - database.EXPECT().GetChecksUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil) + localCheckerHeartbeater.lastChecksCount = checksUpdatesCount - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + Convey("With too much time elapsed since the last successful check", func() { + localCheckerHeartbeater.lastSuccessfulCheck = testTime.Add(-10 * defaultLocalCheckDelay) + defer func() { + localCheckerHeartbeater.lastSuccessfulCheck = testTime + }() + + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := localCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateError) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - // TODO(litleleprikon): seems that this test checks nothing. Seems that NeedToCheckOthers and NeedTurnOffNotifier do not work. - needCheck := check.NeedToCheckOthers() - So(needCheck, ShouldBeTrue) + Convey("With short time elapsed since the last successful check", func() { + database.EXPECT().GetTriggersToCheckCount(localClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetChecksUpdatesCount().Return(checksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - So(check.NeedTurnOffNotifier(), ShouldBeFalse) + state, err := localCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) }) }) } -func createGraphiteLocalCheckerTest(t *testing.T) (*localChecker, *gomock.Controller) { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("CheckDelay") +func TestLocalCheckerHeartbeaterNeedTurnOffNotifier(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test localCheckerHeartbeater.TurnOffNotifier", t, func() { + cfg := LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + NeedTurnOffNotifier: true, + }, + LocalCheckDelay: defaultLocalCheckDelay, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + needTurnOffNotifier := localCheckerHeartbeater.NeedTurnOffNotifier() + So(needTurnOffNotifier, ShouldBeTrue) + }) +} + +func TestLocalCheckerHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) - return GetLocalChecker(120, logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*localChecker), mockCtrl + Convey("Test localCheckerHeartbeater.Type", t, func() { + cfg := LocalCheckerHeartbeaterConfig{ + LocalCheckDelay: defaultLocalCheckDelay, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + localCheckerHeartbeaterType := localCheckerHeartbeater.Type() + So(localCheckerHeartbeaterType, ShouldResemble, datatypes.HearbeatTypeNotSet) + }) +} + +func TestLocalCheckerHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test localCheckerHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } + + cfg := LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + LocalCheckDelay: defaultLocalCheckDelay, + } + + localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := localCheckerHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) } diff --git a/notifier/selfstate/heartbeat/notifier.go b/notifier/selfstate/heartbeat/notifier.go index 7b6877be4..9d26e45c5 100644 --- a/notifier/selfstate/heartbeat/notifier.go +++ b/notifier/selfstate/heartbeat/notifier.go @@ -1,56 +1,57 @@ package heartbeat import ( - "fmt" - - "github.com/moira-alert/moira/metrics" - "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" ) -type notifier struct { - db moira.Database - log moira.Logger - metrics *metrics.HeartBeatMetrics -} +// Verify that notifierHeartbeater matches the Heartbeater interface. +var _ Heartbeater = (*notifierHeartbeater)(nil) -func GetNotifier(logger moira.Logger, database moira.Database, metrics *metrics.HeartBeatMetrics) Heartbeater { - return ¬ifier{ - db: database, - log: logger, - metrics: metrics, - } +// NotifierHeartbeaterConfig structure describing the notifierHeartbeater configuration. +type NotifierHeartbeaterConfig struct { + HeartbeaterBaseConfig } -func (check notifier) Check(int64) (int64, bool, error) { - state, _ := check.db.GetNotifierState() - if state != moira.SelfStateOK { - check.metrics.MarkNotifierIsAlive(false) +type notifierHeartbeater struct { + *heartbeaterBase - check.log.Error(). - String("error", check.GetErrorMessage()). - Msg("Notifier is not healthy") + cfg NotifierHeartbeaterConfig +} + +// NewNotifierHeartbeater is a function that creates a new notifierHeartbeater. +func NewNotifierHeartbeater(cfg NotifierHeartbeaterConfig, base *heartbeaterBase) (*notifierHeartbeater, error) { + return ¬ifierHeartbeater{ + heartbeaterBase: base, + cfg: cfg, + }, nil +} - return 0, true, nil +// Check is a function that returns the state of the notifier. +func (heartbeater *notifierHeartbeater) Check() (State, error) { + notifierState, err := heartbeater.database.GetNotifierState() + if err != nil { + return StateError, err } - check.metrics.MarkNotifierIsAlive(true) - check.log.Debug(). - String("state", state). - Msg("Notifier is healthy") + if notifierState != moira.SelfStateOK { + return StateError, nil + } - return 0, false, nil + return StateOK, nil } -func (notifier) NeedTurnOffNotifier() bool { - return false +// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. +func (heartbeater *notifierHeartbeater) NeedTurnOffNotifier() bool { + return heartbeater.cfg.NeedTurnOffNotifier } -func (notifier) NeedToCheckOthers() bool { - return true +// Type is a function that returns the current heartbeat type. +func (notifierHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HeartbeatNotifierOff } -func (check notifier) GetErrorMessage() string { - state, _ := check.db.GetNotifierState() - return fmt.Sprintf("Moira-Notifier does not send messages. State: %v", state) +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater notifierHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/notifier_test.go b/notifier/selfstate/heartbeat/notifier_test.go index 3d4976035..9b1767eb5 100644 --- a/notifier/selfstate/heartbeat/notifier_test.go +++ b/notifier/selfstate/heartbeat/notifier_test.go @@ -1,53 +1,121 @@ package heartbeat import ( + "errors" "testing" - "time" - - "github.com/moira-alert/moira/metrics" "github.com/moira-alert/moira" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + "github.com/moira-alert/moira/datatypes" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestNotifierState(t *testing.T) { - Convey("Test notifier delay heartbeat", t, func() { - now := time.Now().Unix() - check := createNotifierStateTest(t) +func TestNewNotifierHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test NewNotifierHeartbeater", t, func() { + Convey("With correct local checker heartbeater config", func() { + cfg := NotifierHeartbeaterConfig{} + + expected := ¬ifierHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } + + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(notifierHeartbeater, ShouldResemble, expected) + }) + }) +} + +func TestNotifierHeartbeaterCheck(t *testing.T) { + database, _, _, heartbeaterBase := heartbeaterHelper(t) + + cfg := NotifierHeartbeaterConfig{} + + notifierHeartbeater, _ := NewNotifierHeartbeater(cfg, heartbeaterBase) + + testErr := errors.New("test error") - Convey("Test get notifier delay", func() { - check.db.(*mock_moira_alert.MockDatabase).EXPECT().GetNotifierState().Return(moira.SelfStateOK, nil) + Convey("Test notifierHeartbeater.Check", t, func() { + Convey("With GetNotifierState error", func() { + database.EXPECT().GetNotifierState().Return(string(moira.SelfStateOK), testErr) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + state, err := notifierHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("Test get notification", func() { - check.db.(*mock_moira_alert.MockDatabase).EXPECT().GetNotifierState().Return(moira.SelfStateERROR, nil).Times(2) + Convey("With notifier state equals error", func() { + database.EXPECT().GetNotifierState().Return(moira.SelfStateERROR, nil) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, 0) + state, err := notifierHeartbeater.Check() + So(err, ShouldResemble, nil) + So(state, ShouldResemble, StateError) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - So(check.NeedTurnOffNotifier(), ShouldBeFalse) - So(check.NeedToCheckOthers(), ShouldBeTrue) + Convey("With notifier state equals ok", func() { + database.EXPECT().GetNotifierState().Return(moira.SelfStateOK, nil) + + state, err := notifierHeartbeater.Check() + So(err, ShouldResemble, nil) + So(state, ShouldResemble, StateOK) }) }) } -func createNotifierStateTest(t *testing.T) *notifier { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("MetricDelay") - metric := metrics.ConfigureHeartBeatMetrics(metrics.NewDummyRegistry()) +func TestNotifierHeartbeaterNeedTurnOffNotifier(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test notifierHeartbeater.TurnOffNotifier", t, func() { + cfg := NotifierHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + NeedTurnOffNotifier: true, + }, + } + + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + needTurnOffNotifier := notifierHeartbeater.NeedTurnOffNotifier() + So(needTurnOffNotifier, ShouldBeTrue) + }) +} + +func TestNotifierHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test notifierHeartbeater.Type", t, func() { + cfg := NotifierHeartbeaterConfig{} + + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + notifierHeartbeaterType := notifierHeartbeater.Type() + So(notifierHeartbeaterType, ShouldResemble, datatypes.HeartbeatNotifierOff) + }) +} + +func TestNotifierHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test notifierHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } - return GetNotifier(logger, mock_moira_alert.NewMockDatabase(mockCtrl), metric).(*notifier) + cfg := NotifierHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + } + + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := notifierHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) } diff --git a/notifier/selfstate/heartbeat/remote_checker.go b/notifier/selfstate/heartbeat/remote_checker.go index 4b0e76113..c87d7d18c 100644 --- a/notifier/selfstate/heartbeat/remote_checker.go +++ b/notifier/selfstate/heartbeat/remote_checker.go @@ -1,60 +1,89 @@ package heartbeat import ( + "fmt" "time" + "github.com/go-playground/validator/v10" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" ) -type remoteChecker struct { - heartbeat - count int64 +var ( + remoteClusterKey = moira.DefaultGraphiteRemoteCluster + + // Verify that remoteCheckerHeartbeater matches the Heartbeater interface. + _ Heartbeater = (*remoteCheckerHeartbeater)(nil) +) + +// RemoteCheckerHeartbeaterConfig structure describing the remoteCheckerHeartbeater configuration. +type RemoteCheckerHeartbeaterConfig struct { + HeartbeaterBaseConfig + + RemoteCheckDelay time.Duration `validate:"required,gt=0"` +} + +func (cfg RemoteCheckerHeartbeaterConfig) validate() error { + validator := validator.New() + return validator.Struct(cfg) +} + +type remoteCheckerHeartbeater struct { + *heartbeaterBase + + cfg RemoteCheckerHeartbeaterConfig + lastRemoteChecksCount int64 } -func GetRemoteChecker(delay int64, logger moira.Logger, database moira.Database) Heartbeater { - if delay > 0 { - return &remoteChecker{heartbeat: heartbeat{ - logger: logger, - database: database, - delay: delay, - lastSuccessfulCheck: time.Now().Unix(), - }} +// NewRemoteCheckerHeartbeater is a function that creates a new remoteCheckerHeartbeater. +func NewRemoteCheckerHeartbeater(cfg RemoteCheckerHeartbeaterConfig, base *heartbeaterBase) (*remoteCheckerHeartbeater, error) { + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("remote checker heartbeater configuration error: %w", err) } - return nil + + return &remoteCheckerHeartbeater{ + heartbeaterBase: base, + cfg: cfg, + }, nil } -func (check *remoteChecker) Check(nowTS int64) (int64, bool, error) { - defaultRemoteCluster := moira.DefaultGraphiteRemoteCluster - triggerCount, err := check.database.GetTriggersToCheckCount(defaultRemoteCluster) +// Check is a function that checks that the remote checker checks triggers and the number of triggers is not constant. +func (heartbeater remoteCheckerHeartbeater) Check() (State, error) { + triggersCount, err := heartbeater.database.GetTriggersToCheckCount(remoteClusterKey) if err != nil { - return 0, false, err + return StateError, err } - remoteTriggersCount, _ := check.database.GetRemoteChecksUpdatesCount() - if check.count != remoteTriggersCount || triggerCount == 0 { - check.count = remoteTriggersCount - check.lastSuccessfulCheck = nowTS - return 0, false, nil + remoteChecksCount, err := heartbeater.database.GetRemoteChecksUpdatesCount() + if err != nil { + return StateError, err } - if check.lastSuccessfulCheck < nowTS-check.delay { - check.logger.Error(). - String("error", check.GetErrorMessage()). - Int64("time_since_successful_check", nowTS-check.heartbeat.lastSuccessfulCheck). - Msg("Send message") - return nowTS - check.lastSuccessfulCheck, true, nil + now := heartbeater.clock.NowUTC() + if heartbeater.lastRemoteChecksCount != remoteChecksCount || triggersCount == 0 { + heartbeater.lastRemoteChecksCount = remoteChecksCount + heartbeater.lastSuccessfulCheck = now + return StateOK, nil } - return 0, false, nil + + if now.Sub(heartbeater.lastSuccessfulCheck) > heartbeater.cfg.RemoteCheckDelay { + return StateError, nil + } + + return StateOK, nil } -func (check remoteChecker) NeedTurnOffNotifier() bool { - return false +// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. +func (heartbeater remoteCheckerHeartbeater) NeedTurnOffNotifier() bool { + return heartbeater.cfg.NeedTurnOffNotifier } -func (remoteChecker) NeedToCheckOthers() bool { - return true +// Type is a function that returns the current heartbeat type. +func (remoteCheckerHeartbeater) Type() datatypes.HeartbeatType { + return datatypes.HearbeatTypeNotSet } -func (remoteChecker) GetErrorMessage() string { - return "Moira-Remote-Checker does not check remote triggers" +// AlertSettings is a function that returns the current settings for alerts. +func (heartbeater remoteCheckerHeartbeater) AlertSettings() AlertConfig { + return heartbeater.cfg.AlertCfg } diff --git a/notifier/selfstate/heartbeat/remote_checker_test.go b/notifier/selfstate/heartbeat/remote_checker_test.go index a48ee30a8..c46ab1bae 100644 --- a/notifier/selfstate/heartbeat/remote_checker_test.go +++ b/notifier/selfstate/heartbeat/remote_checker_test.go @@ -5,84 +5,205 @@ import ( "testing" "time" - "github.com/moira-alert/moira" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira/datatypes" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" ) -func TestGraphiteRemoteChecker(t *testing.T) { - defaultRemoteCluster := moira.DefaultGraphiteRemoteCluster +const ( + defaultRemoteCheckDelay = time.Minute +) + +func TestNewRemoteCheckerHeartbeater(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + validationErr := validator.ValidationErrors{} + + Convey("Test NewRemoteCheckerHeartbeater", t, func() { + Convey("With too low remote check delay", func() { + cfg := RemoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: -1, + } + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(remoteCheckerHeartbeater, ShouldBeNil) + }) + + Convey("Without remote check delay", func() { + cfg := RemoteCheckerHeartbeaterConfig{} + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(errors.As(err, &validationErr), ShouldBeTrue) + So(remoteCheckerHeartbeater, ShouldBeNil) + }) + + Convey("With correct remote checker heartbeater config", func() { + cfg := RemoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: 1, + } - Convey("Test remote checker heartbeat", t, func() { - err := errors.New("test error remoteChecker") - now := time.Now().Unix() - check, mockCtrl := createGraphiteRemoteCheckerTest(t) - defer mockCtrl.Finish() - database := check.database.(*mock_moira_alert.MockDatabase) + expected := &remoteCheckerHeartbeater{ + heartbeaterBase: heartbeaterBase, + cfg: cfg, + } - Convey("Checking the created graphite remote checker", func() { - expected := &remoteChecker{heartbeat: heartbeat{database: check.database, logger: check.logger, delay: 1, lastSuccessfulCheck: now}} - So(GetRemoteChecker(0, check.logger, check.database), ShouldBeNil) - So(GetRemoteChecker(1, check.logger, check.database), ShouldResemble, expected) + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + So(remoteCheckerHeartbeater, ShouldResemble, expected) }) + }) +} + +func TestRemoteCheckerHeartbeaterCheck(t *testing.T) { + database, clock, testTime, heartbeaterBase := heartbeaterHelper(t) + + cfg := RemoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: defaultRemoteCheckDelay, + } + + remoteCheckerHeartbeater, _ := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) - Convey("GraphiteRemoteChecker error handling test", func() { - database.EXPECT().GetTriggersToCheckCount(defaultRemoteCluster).Return(int64(0), err) + var ( + testErr = errors.New("test error") + triggersToCheckCount, remoteChecksUpdatesCount int64 = 10, 10 + ) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldEqual, err) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + Convey("Test remoteCheckerHeartbeater.Check", t, func() { + Convey("With GetTriggersToCheckCount error", func() { + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, testErr) + + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) }) - Convey("Test update lastSuccessfulCheck", func() { - now += 1000 - database.EXPECT().GetRemoteChecksUpdatesCount().Return(int64(1), nil) - database.EXPECT().GetTriggersToCheckCount(defaultRemoteCluster).Return(int64(1), nil) + Convey("With GetRemoteChecksUpdatesCount error", func() { + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, testErr) + + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldResemble, testErr) + So(state, ShouldResemble, StateError) + }) + + Convey("With last remote checks count not equal current remote checks count", func() { + defer func() { + remoteCheckerHeartbeater.lastRemoteChecksCount = 0 + }() + + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) - So(check.lastSuccessfulCheck, ShouldResemble, now) + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(remoteCheckerHeartbeater.lastRemoteChecksCount, ShouldResemble, remoteChecksUpdatesCount) }) - Convey("Check for notification", func() { - check.lastSuccessfulCheck = now - check.delay - 1 + Convey("With zero triggers to check count", func() { + defer func() { + remoteCheckerHeartbeater.lastRemoteChecksCount = 0 + }() - database.EXPECT().GetRemoteChecksUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultRemoteCluster).Return(int64(1), nil) + var zeroTriggersToCheckCount int64 - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeTrue) - So(value, ShouldEqual, now-check.lastSuccessfulCheck) + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(zeroTriggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) + So(remoteCheckerHeartbeater.lastRemoteChecksCount, ShouldResemble, remoteChecksUpdatesCount) }) - Convey("Exit without action", func() { - database.EXPECT().GetRemoteChecksUpdatesCount().Return(int64(0), nil) - database.EXPECT().GetTriggersToCheckCount(defaultRemoteCluster).Return(int64(1), nil) + remoteCheckerHeartbeater.lastRemoteChecksCount = remoteChecksUpdatesCount + + Convey("With too much time elapsed since the last successful check", func() { + remoteCheckerHeartbeater.lastSuccessfulCheck = testTime.Add(-10 * defaultRemoteCheckDelay) + defer func() { + remoteCheckerHeartbeater.lastSuccessfulCheck = testTime + }() + + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) - value, needSend, errActual := check.Check(now) - So(errActual, ShouldBeNil) - So(needSend, ShouldBeFalse) - So(value, ShouldEqual, 0) + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateError) }) - Convey("Test NeedToCheckOthers and NeedTurnOffNotifier", func() { - // TODO(litleleprikon): seems that this test checks nothing. Seems that NeedToCheckOthers and NeedTurnOffNotifier do not work. - So(check.NeedToCheckOthers(), ShouldBeTrue) - So(check.NeedTurnOffNotifier(), ShouldBeFalse) + Convey("With short time elapsed since the last successful check", func() { + database.EXPECT().GetTriggersToCheckCount(remoteClusterKey).Return(triggersToCheckCount, nil) + database.EXPECT().GetRemoteChecksUpdatesCount().Return(remoteChecksUpdatesCount, nil) + clock.EXPECT().NowUTC().Return(testTime) + + state, err := remoteCheckerHeartbeater.Check() + So(err, ShouldBeNil) + So(state, ShouldResemble, StateOK) }) }) } -func createGraphiteRemoteCheckerTest(t *testing.T) (*remoteChecker, *gomock.Controller) { - mockCtrl := gomock.NewController(t) - logger, _ := logging.GetLogger("MetricDelay") +func TestRemoteCheckerHeartbeaterNeedTurnOffNotifier(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test remoteCheckerHeartbeater.TurnOffNotifier", t, func() { + cfg := RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + NeedTurnOffNotifier: true, + }, + RemoteCheckDelay: defaultRemoteCheckDelay, + } + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + needTurnOffNotifier := remoteCheckerHeartbeater.NeedTurnOffNotifier() + So(needTurnOffNotifier, ShouldBeTrue) + }) +} + +func TestRemoteCheckerHeartbeaterType(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) - return GetRemoteChecker(120, logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*remoteChecker), mockCtrl + Convey("Test remoteCheckerHeartbeater.Type", t, func() { + cfg := RemoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: defaultRemoteCheckDelay, + } + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + remoteCheckerHeartbeaterType := remoteCheckerHeartbeater.Type() + So(remoteCheckerHeartbeaterType, ShouldResemble, datatypes.HearbeatTypeNotSet) + }) +} + +func TestRemoteCheckerHeartbeaterAlertSettings(t *testing.T) { + _, _, _, heartbeaterBase := heartbeaterHelper(t) + + Convey("Test remoteCheckerHeartbeater.AlertSettings", t, func() { + alertCfg := AlertConfig{ + Name: "test name", + Desc: "test desc", + } + + cfg := RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + AlertCfg: alertCfg, + }, + RemoteCheckDelay: defaultRemoteCheckDelay, + } + + remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) + So(err, ShouldBeNil) + + alertSettings := remoteCheckerHeartbeater.AlertSettings() + So(alertSettings, ShouldResemble, alertCfg) + }) } From 2d8ec971c688ddbc5902ba7f1bdc5afa66ef40e0 Mon Sep 17 00:00:00 2001 From: almostinf Date: Thu, 31 Oct 2024 18:58:39 +0300 Subject: [PATCH 2/8] add heartbeaters test --- .../selfstate/heartbeat/heartbeat_test.go | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 notifier/selfstate/heartbeat/heartbeat_test.go diff --git a/notifier/selfstate/heartbeat/heartbeat_test.go b/notifier/selfstate/heartbeat/heartbeat_test.go new file mode 100644 index 000000000..daec4cff8 --- /dev/null +++ b/notifier/selfstate/heartbeat/heartbeat_test.go @@ -0,0 +1,96 @@ +package heartbeat + +import ( + "testing" + "time" + + logging "github.com/moira-alert/moira/logging/zerolog_adapter" + mock_clock "github.com/moira-alert/moira/mock/clock" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + . "github.com/smartystreets/goconvey/convey" + "go.uber.org/mock/gomock" +) + +func heartbeaterHelper(t *testing.T) (*mock_moira_alert.MockDatabase, *mock_clock.MockClock, time.Time, *heartbeaterBase) { + t.Helper() + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + logger, _ := logging.GetLogger("Test") + database := mock_moira_alert.NewMockDatabase(mockCtrl) + clock := mock_clock.NewMockClock(mockCtrl) + + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + clock.EXPECT().NowUTC().Return(testTime) + heartbeaterBase := NewHeartbeaterBase(logger, database, clock) + + return database, clock, testTime, heartbeaterBase +} + +func TestStateIsDegradated(t *testing.T) { + Convey("Test state.IsDegradated", t, func() { + Convey("With degradated state", func() { + lastState := StateOK + newState := StateError + + degradated := lastState.IsDegraded(newState) + So(degradated, ShouldBeTrue) + }) + + Convey("Without degradated state", func() { + lastState := StateError + newState := StateOK + + degradated := lastState.IsDegraded(newState) + So(degradated, ShouldBeFalse) + }) + }) +} + +func TestStateIsRecovered(t *testing.T) { + Convey("Test state.IsRecovered", t, func() { + Convey("With recovered state", func() { + lastState := StateError + newState := StateOK + + recovered := lastState.IsRecovered(newState) + So(recovered, ShouldBeTrue) + }) + + Convey("Without recovered state", func() { + lastState := StateOK + newState := StateError + + recovered := lastState.IsRecovered(newState) + So(recovered, ShouldBeFalse) + }) + }) +} + +func TestNewHeartbeaterBase(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + logger, _ := logging.GetLogger("Test") + database := mock_moira_alert.NewMockDatabase(mockCtrl) + clock := mock_clock.NewMockClock(mockCtrl) + + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test NewHeartbeaterBase", t, func() { + clock.EXPECT().NowUTC().Return(testTime) + + expected := &heartbeaterBase{ + logger: logger, + database: database, + clock: clock, + + lastSuccessfulCheck: testTime, + } + + heartbeaterBase := NewHeartbeaterBase(logger, database, clock) + So(heartbeaterBase, ShouldResemble, expected) + }) +} From e9ef82cbec123598ba080b64401e7438621e55e3 Mon Sep 17 00:00:00 2001 From: almostinf Date: Fri, 1 Nov 2024 15:24:21 +0300 Subject: [PATCH 3/8] add validate heartbeater base config tests --- .../selfstate/heartbeat/heartbeat_test.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/notifier/selfstate/heartbeat/heartbeat_test.go b/notifier/selfstate/heartbeat/heartbeat_test.go index daec4cff8..539c44587 100644 --- a/notifier/selfstate/heartbeat/heartbeat_test.go +++ b/notifier/selfstate/heartbeat/heartbeat_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/moira-alert/moira" logging "github.com/moira-alert/moira/logging/zerolog_adapter" mock_clock "github.com/moira-alert/moira/mock/clock" mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" @@ -94,3 +95,41 @@ func TestNewHeartbeaterBase(t *testing.T) { So(heartbeaterBase, ShouldResemble, expected) }) } + +func TestValidateHeartbeaterBaseConfig(t *testing.T) { + Convey("Test validation heartbeaterBaseConfig", t, func() { + Convey("With disabled config", func() { + hbCfg := HeartbeaterBaseConfig{} + err := moira.ValidateStruct(hbCfg) + So(err, ShouldBeNil) + }) + + Convey("With just enabled config", func() { + hbCfg := HeartbeaterBaseConfig{ + Enabled: true, + } + err := moira.ValidateStruct(hbCfg) + So(err, ShouldNotBeNil) + }) + + Convey("With enabled config and added alert config", func() { + hbCfg := HeartbeaterBaseConfig{ + Enabled: true, + AlertCfg: AlertConfig{}, + } + err := moira.ValidateStruct(hbCfg) + So(err, ShouldNotBeNil) + }) + + Convey("With enabled config, added and filled alert config", func() { + hbCfg := HeartbeaterBaseConfig{ + Enabled: true, + AlertCfg: AlertConfig{ + Name: "test name", + }, + } + err := moira.ValidateStruct(hbCfg) + So(err, ShouldBeNil) + }) + }) +} From ad85a9771b44718054328b097758c527665f3413 Mon Sep 17 00:00:00 2001 From: almostinf Date: Tue, 5 Nov 2024 18:03:10 +0300 Subject: [PATCH 4/8] fix heartbeats --- go.mod | 9 +-- go.sum | 17 ++++-- notifier/selfstate/heartbeat/database.go | 7 +-- notifier/selfstate/heartbeat/database_test.go | 27 ++------- notifier/selfstate/heartbeat/filter.go | 9 +-- notifier/selfstate/heartbeat/filter_test.go | 27 ++------- notifier/selfstate/heartbeat/heartbeat.go | 7 +-- .../selfstate/heartbeat/heartbeat_test.go | 55 ++++++++++++------- notifier/selfstate/heartbeat/local_checker.go | 9 +-- .../selfstate/heartbeat/local_checker_test.go | 27 ++------- notifier/selfstate/heartbeat/notifier.go | 7 +-- notifier/selfstate/heartbeat/notifier_test.go | 20 +------ .../selfstate/heartbeat/remote_checker.go | 9 +-- .../heartbeat/remote_checker_test.go | 27 ++------- 14 files changed, 88 insertions(+), 169 deletions(-) diff --git a/go.mod b/go.mod index b309808bd..cc875b256 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( require github.com/prometheus/common v0.37.0 require ( - github.com/go-playground/validator/v10 v10.4.1 + github.com/go-playground/validator/v10 v10.22.1 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/mattermost/mattermost/server/public v0.1.1 github.com/mitchellh/mapstructure v1.5.0 @@ -173,19 +173,20 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a // indirect github.com/fatih/color v1.16.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/spec v0.20.9 // indirect github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-playground/locales v0.13.0 // indirect - github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-plugin v1.6.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/leodido/go-urn v1.2.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect diff --git a/go.sum b/go.sum index a013bebd9..30410d294 100644 --- a/go.sum +++ b/go.sum @@ -238,6 +238,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= @@ -279,14 +281,18 @@ github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= +github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= @@ -527,8 +533,9 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80 h1:KVyDGUXjVOdHQt24wIgY4ZdGFXHtQHLWw0L/MAK3Kb0= diff --git a/notifier/selfstate/heartbeat/database.go b/notifier/selfstate/heartbeat/database.go index 47b466ffa..592325ef1 100644 --- a/notifier/selfstate/heartbeat/database.go +++ b/notifier/selfstate/heartbeat/database.go @@ -53,14 +53,9 @@ func (heartbeater *databaseHeartbeater) Check() (State, error) { return StateOK, err } -// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. -func (heartbeater databaseHeartbeater) NeedTurnOffNotifier() bool { - return heartbeater.cfg.NeedTurnOffNotifier -} - // Type is a function that returns the current heartbeat type. func (databaseHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatTypeNotSet + return datatypes.HeartbeatDatabase } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/database_test.go b/notifier/selfstate/heartbeat/database_test.go index 99312d892..66d4277f6 100644 --- a/notifier/selfstate/heartbeat/database_test.go +++ b/notifier/selfstate/heartbeat/database_test.go @@ -34,7 +34,11 @@ func TestNewDatabaseHeartbeater(t *testing.T) { }) Convey("Without redis disconnect delay", func() { - cfg := DatabaseHeartbeaterConfig{} + cfg := DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) So(errors.As(err, &validationErr), ShouldBeTrue) @@ -107,25 +111,6 @@ func TestDatabaseHeartbeaterCheck(t *testing.T) { }) } -func TestDatabaseHeartbeaterNeedTurnOffNotifier(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) - - Convey("Test databaseHeartbeater.TurnOffNotifier", t, func() { - cfg := DatabaseHeartbeaterConfig{ - HeartbeaterBaseConfig: HeartbeaterBaseConfig{ - NeedTurnOffNotifier: true, - }, - RedisDisconnectDelay: defaultRedisDisconnectDelay, - } - - databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) - So(err, ShouldBeNil) - - needTurnOffNotifier := databaseHeartbeater.NeedTurnOffNotifier() - So(needTurnOffNotifier, ShouldBeTrue) - }) -} - func TestDatabaseHeartbeaterType(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) @@ -138,7 +123,7 @@ func TestDatabaseHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) databaseHeartbeaterType := databaseHeartbeater.Type() - So(databaseHeartbeaterType, ShouldResemble, datatypes.HeartbeatTypeNotSet) + So(databaseHeartbeaterType, ShouldResemble, datatypes.HeartbeatDatabase) }) } diff --git a/notifier/selfstate/heartbeat/filter.go b/notifier/selfstate/heartbeat/filter.go index 73ff3fcc3..15a222b2b 100644 --- a/notifier/selfstate/heartbeat/filter.go +++ b/notifier/selfstate/heartbeat/filter.go @@ -19,7 +19,7 @@ var ( type FilterHeartbeaterConfig struct { HeartbeaterBaseConfig - MetricReceivedDelay time.Duration `validate:"required,gt=0"` + MetricReceivedDelay time.Duration `validate:"required_if=Enabled true,gte=0"` } type filterHeartbeater struct { @@ -67,14 +67,9 @@ func (heartbeater *filterHeartbeater) Check() (State, error) { return StateOK, nil } -// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. -func (heartbeater filterHeartbeater) NeedTurnOffNotifier() bool { - return heartbeater.cfg.NeedTurnOffNotifier -} - // Type is a function that returns the current heartbeat type. func (filterHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatTypeNotSet + return datatypes.HeartbeatFilter } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/filter_test.go b/notifier/selfstate/heartbeat/filter_test.go index fe54dc894..52f29ebde 100644 --- a/notifier/selfstate/heartbeat/filter_test.go +++ b/notifier/selfstate/heartbeat/filter_test.go @@ -32,7 +32,11 @@ func TestNewFilterHeartbeater(t *testing.T) { }) Convey("Without metric received delay", func() { - cfg := FilterHeartbeaterConfig{} + cfg := FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) So(errors.As(err, &validationErr), ShouldBeTrue) @@ -149,25 +153,6 @@ func TestFilterHeartbeaterCheck(t *testing.T) { }) } -func TestFilterHeartbeaterNeedTurnOffNotifier(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) - - Convey("Test filterHeartbeater.TurnOffNotifier", t, func() { - cfg := FilterHeartbeaterConfig{ - HeartbeaterBaseConfig: HeartbeaterBaseConfig{ - NeedTurnOffNotifier: true, - }, - MetricReceivedDelay: defaultMetricReceivedDelay, - } - - filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) - So(err, ShouldBeNil) - - needTurnOffNotifier := filterHeartbeater.NeedTurnOffNotifier() - So(needTurnOffNotifier, ShouldBeTrue) - }) -} - func TestFilterHeartbeaterType(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) @@ -180,7 +165,7 @@ func TestFilterHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) filterHeartbeaterType := filterHeartbeater.Type() - So(filterHeartbeaterType, ShouldResemble, datatypes.HeartbeatTypeNotSet) + So(filterHeartbeaterType, ShouldResemble, datatypes.HeartbeatFilter) }) } diff --git a/notifier/selfstate/heartbeat/heartbeat.go b/notifier/selfstate/heartbeat/heartbeat.go index ef8d72852..3a85f4094 100644 --- a/notifier/selfstate/heartbeat/heartbeat.go +++ b/notifier/selfstate/heartbeat/heartbeat.go @@ -17,7 +17,7 @@ const ( // IsDegraded checks if the condition has degraded. func (lastState State) IsDegraded(newState State) bool { - return lastState == StateOK && newState == StateError + return newState == StateError } // IsRecovered checks if the condition has recovered. @@ -28,7 +28,6 @@ func (lastState State) IsRecovered(newState State) bool { // Heartbeater is the interface for simplified events verification. type Heartbeater interface { Check() (State, error) - NeedTurnOffNotifier() bool AlertSettings() AlertConfig Type() datatypes.HeartbeatType } @@ -39,12 +38,12 @@ type HeartbeaterBaseConfig struct { NeedTurnOffNotifier bool NeedToCheckOthers bool - AlertCfg AlertConfig `validate:"required_if=Enabled true"` + AlertCfg AlertConfig } // AlertConfig contains the configuration of the alerts that heartbeater sends out. type AlertConfig struct { - Name string `validate:"required_if=Enabled true"` + Name string Desc string } diff --git a/notifier/selfstate/heartbeat/heartbeat_test.go b/notifier/selfstate/heartbeat/heartbeat_test.go index 539c44587..5dd2961e2 100644 --- a/notifier/selfstate/heartbeat/heartbeat_test.go +++ b/notifier/selfstate/heartbeat/heartbeat_test.go @@ -31,8 +31,16 @@ func heartbeaterHelper(t *testing.T) (*mock_moira_alert.MockDatabase, *mock_cloc } func TestStateIsDegradated(t *testing.T) { - Convey("Test state.IsDegradated", t, func() { - Convey("With degradated state", func() { + Convey("Test state.IsDegraded", t, func() { + Convey("With continue degraded", func() { + lastState := StateError + newState := StateError + + degradated := lastState.IsDegraded(newState) + So(degradated, ShouldBeTrue) + }) + + Convey("With degraded state", func() { lastState := StateOK newState := StateError @@ -40,13 +48,21 @@ func TestStateIsDegradated(t *testing.T) { So(degradated, ShouldBeTrue) }) - Convey("Without degradated state", func() { + Convey("Without degraded state", func() { lastState := StateError newState := StateOK degradated := lastState.IsDegraded(newState) So(degradated, ShouldBeFalse) }) + + Convey("With continue recovered", func() { + lastState := StateOK + newState := StateOK + + degradated := lastState.IsDegraded(newState) + So(degradated, ShouldBeFalse) + }) }) } @@ -67,6 +83,22 @@ func TestStateIsRecovered(t *testing.T) { recovered := lastState.IsRecovered(newState) So(recovered, ShouldBeFalse) }) + + Convey("With continue recovered", func() { + lastState := StateOK + newState := StateOK + + recovered := lastState.IsRecovered(newState) + So(recovered, ShouldBeFalse) + }) + + Convey("With continue degraded", func() { + lastState := StateError + newState := StateError + + recovered := lastState.IsRecovered(newState) + So(recovered, ShouldBeFalse) + }) }) } @@ -104,23 +136,6 @@ func TestValidateHeartbeaterBaseConfig(t *testing.T) { So(err, ShouldBeNil) }) - Convey("With just enabled config", func() { - hbCfg := HeartbeaterBaseConfig{ - Enabled: true, - } - err := moira.ValidateStruct(hbCfg) - So(err, ShouldNotBeNil) - }) - - Convey("With enabled config and added alert config", func() { - hbCfg := HeartbeaterBaseConfig{ - Enabled: true, - AlertCfg: AlertConfig{}, - } - err := moira.ValidateStruct(hbCfg) - So(err, ShouldNotBeNil) - }) - Convey("With enabled config, added and filled alert config", func() { hbCfg := HeartbeaterBaseConfig{ Enabled: true, diff --git a/notifier/selfstate/heartbeat/local_checker.go b/notifier/selfstate/heartbeat/local_checker.go index 4ea80c113..9a80b6ba1 100644 --- a/notifier/selfstate/heartbeat/local_checker.go +++ b/notifier/selfstate/heartbeat/local_checker.go @@ -15,7 +15,7 @@ var _ Heartbeater = (*localCheckerHeartbeater)(nil) type LocalCheckerHeartbeaterConfig struct { HeartbeaterBaseConfig - LocalCheckDelay time.Duration `validate:"required,gt=0"` + LocalCheckDelay time.Duration `validate:"required_if=Enabled true,gte=0"` } type localCheckerHeartbeater struct { @@ -63,14 +63,9 @@ func (heartbeater *localCheckerHeartbeater) Check() (State, error) { return StateOK, nil } -// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. -func (heartbeater localCheckerHeartbeater) NeedTurnOffNotifier() bool { - return heartbeater.cfg.NeedTurnOffNotifier -} - // Type is a function that returns the current heartbeat type. func (localCheckerHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatTypeNotSet + return datatypes.HeartbeatLocalChecker } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/local_checker_test.go b/notifier/selfstate/heartbeat/local_checker_test.go index 50eae03e8..cfe720674 100644 --- a/notifier/selfstate/heartbeat/local_checker_test.go +++ b/notifier/selfstate/heartbeat/local_checker_test.go @@ -32,7 +32,11 @@ func TestNewLocalCheckerHeartbeater(t *testing.T) { }) Convey("Without local check delay", func() { - cfg := LocalCheckerHeartbeaterConfig{} + cfg := LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) So(errors.As(err, &validationErr), ShouldBeTrue) @@ -149,25 +153,6 @@ func TestLocalCheckerHeartbeaterCheck(t *testing.T) { }) } -func TestLocalCheckerHeartbeaterNeedTurnOffNotifier(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) - - Convey("Test localCheckerHeartbeater.TurnOffNotifier", t, func() { - cfg := LocalCheckerHeartbeaterConfig{ - HeartbeaterBaseConfig: HeartbeaterBaseConfig{ - NeedTurnOffNotifier: true, - }, - LocalCheckDelay: defaultLocalCheckDelay, - } - - localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) - So(err, ShouldBeNil) - - needTurnOffNotifier := localCheckerHeartbeater.NeedTurnOffNotifier() - So(needTurnOffNotifier, ShouldBeTrue) - }) -} - func TestLocalCheckerHeartbeaterType(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) @@ -180,7 +165,7 @@ func TestLocalCheckerHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) localCheckerHeartbeaterType := localCheckerHeartbeater.Type() - So(localCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatTypeNotSet) + So(localCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatLocalChecker) }) } diff --git a/notifier/selfstate/heartbeat/notifier.go b/notifier/selfstate/heartbeat/notifier.go index 9d26e45c5..6d92a5ef6 100644 --- a/notifier/selfstate/heartbeat/notifier.go +++ b/notifier/selfstate/heartbeat/notifier.go @@ -41,14 +41,9 @@ func (heartbeater *notifierHeartbeater) Check() (State, error) { return StateOK, nil } -// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. -func (heartbeater *notifierHeartbeater) NeedTurnOffNotifier() bool { - return heartbeater.cfg.NeedTurnOffNotifier -} - // Type is a function that returns the current heartbeat type. func (notifierHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatNotifierOff + return datatypes.HeartbeatNotifier } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/notifier_test.go b/notifier/selfstate/heartbeat/notifier_test.go index 9b1767eb5..a0a76bbc2 100644 --- a/notifier/selfstate/heartbeat/notifier_test.go +++ b/notifier/selfstate/heartbeat/notifier_test.go @@ -65,24 +65,6 @@ func TestNotifierHeartbeaterCheck(t *testing.T) { }) } -func TestNotifierHeartbeaterNeedTurnOffNotifier(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) - - Convey("Test notifierHeartbeater.TurnOffNotifier", t, func() { - cfg := NotifierHeartbeaterConfig{ - HeartbeaterBaseConfig: HeartbeaterBaseConfig{ - NeedTurnOffNotifier: true, - }, - } - - notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) - So(err, ShouldBeNil) - - needTurnOffNotifier := notifierHeartbeater.NeedTurnOffNotifier() - So(needTurnOffNotifier, ShouldBeTrue) - }) -} - func TestNotifierHeartbeaterType(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) @@ -93,7 +75,7 @@ func TestNotifierHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) notifierHeartbeaterType := notifierHeartbeater.Type() - So(notifierHeartbeaterType, ShouldResemble, datatypes.HeartbeatNotifierOff) + So(notifierHeartbeaterType, ShouldResemble, datatypes.HeartbeatNotifier) }) } diff --git a/notifier/selfstate/heartbeat/remote_checker.go b/notifier/selfstate/heartbeat/remote_checker.go index 096342378..1253184a8 100644 --- a/notifier/selfstate/heartbeat/remote_checker.go +++ b/notifier/selfstate/heartbeat/remote_checker.go @@ -19,7 +19,7 @@ var ( type RemoteCheckerHeartbeaterConfig struct { HeartbeaterBaseConfig - RemoteCheckDelay time.Duration `validate:"required,gt=0"` + RemoteCheckDelay time.Duration `validate:"required_if=Enabled true,gte=0"` } type remoteCheckerHeartbeater struct { @@ -67,14 +67,9 @@ func (heartbeater remoteCheckerHeartbeater) Check() (State, error) { return StateOK, nil } -// NeedTurnOffNotifier is a function that checks to see if the notifier needs to be turned off. -func (heartbeater remoteCheckerHeartbeater) NeedTurnOffNotifier() bool { - return heartbeater.cfg.NeedTurnOffNotifier -} - // Type is a function that returns the current heartbeat type. func (remoteCheckerHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatTypeNotSet + return datatypes.HeartbeatRemoteChecker } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/remote_checker_test.go b/notifier/selfstate/heartbeat/remote_checker_test.go index aac4dbaf9..6d7cf02e6 100644 --- a/notifier/selfstate/heartbeat/remote_checker_test.go +++ b/notifier/selfstate/heartbeat/remote_checker_test.go @@ -32,7 +32,11 @@ func TestNewRemoteCheckerHeartbeater(t *testing.T) { }) Convey("Without remote check delay", func() { - cfg := RemoteCheckerHeartbeaterConfig{} + cfg := RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) So(errors.As(err, &validationErr), ShouldBeTrue) @@ -149,25 +153,6 @@ func TestRemoteCheckerHeartbeaterCheck(t *testing.T) { }) } -func TestRemoteCheckerHeartbeaterNeedTurnOffNotifier(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) - - Convey("Test remoteCheckerHeartbeater.TurnOffNotifier", t, func() { - cfg := RemoteCheckerHeartbeaterConfig{ - HeartbeaterBaseConfig: HeartbeaterBaseConfig{ - NeedTurnOffNotifier: true, - }, - RemoteCheckDelay: defaultRemoteCheckDelay, - } - - remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) - So(err, ShouldBeNil) - - needTurnOffNotifier := remoteCheckerHeartbeater.NeedTurnOffNotifier() - So(needTurnOffNotifier, ShouldBeTrue) - }) -} - func TestRemoteCheckerHeartbeaterType(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) @@ -180,7 +165,7 @@ func TestRemoteCheckerHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) remoteCheckerHeartbeaterType := remoteCheckerHeartbeater.Type() - So(remoteCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatTypeNotSet) + So(remoteCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatRemoteChecker) }) } From d42af769eee2a486428d0c0a9633e4826abd0f6d Mon Sep 17 00:00:00 2001 From: almostinf Date: Tue, 5 Nov 2024 18:03:59 +0300 Subject: [PATCH 5/8] regenerate mocks --- mock/heartbeat/heartbeat.go | 63 +++++++++++++++---------------------- 1 file changed, 25 insertions(+), 38 deletions(-) diff --git a/mock/heartbeat/heartbeat.go b/mock/heartbeat/heartbeat.go index f0a6f6acf..694bee3c1 100644 --- a/mock/heartbeat/heartbeat.go +++ b/mock/heartbeat/heartbeat.go @@ -12,6 +12,8 @@ package mock_heartbeat import ( reflect "reflect" + datatypes "github.com/moira-alert/moira/datatypes" + heartbeat "github.com/moira-alert/moira/notifier/selfstate/heartbeat" gomock "go.uber.org/mock/gomock" ) @@ -38,60 +40,45 @@ func (m *MockHeartbeater) EXPECT() *MockHeartbeaterMockRecorder { return m.recorder } -// Check mocks base method. -func (m *MockHeartbeater) Check(arg0 int64) (int64, bool, error) { +// AlertSettings mocks base method. +func (m *MockHeartbeater) AlertSettings() heartbeat.AlertConfig { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Check", arg0) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// Check indicates an expected call of Check. -func (mr *MockHeartbeaterMockRecorder) Check(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockHeartbeater)(nil).Check), arg0) -} - -// GetErrorMessage mocks base method. -func (m *MockHeartbeater) GetErrorMessage() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetErrorMessage") - ret0, _ := ret[0].(string) + ret := m.ctrl.Call(m, "AlertSettings") + ret0, _ := ret[0].(heartbeat.AlertConfig) return ret0 } -// GetErrorMessage indicates an expected call of GetErrorMessage. -func (mr *MockHeartbeaterMockRecorder) GetErrorMessage() *gomock.Call { +// AlertSettings indicates an expected call of AlertSettings. +func (mr *MockHeartbeaterMockRecorder) AlertSettings() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetErrorMessage", reflect.TypeOf((*MockHeartbeater)(nil).GetErrorMessage)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AlertSettings", reflect.TypeOf((*MockHeartbeater)(nil).AlertSettings)) } -// NeedToCheckOthers mocks base method. -func (m *MockHeartbeater) NeedToCheckOthers() bool { +// Check mocks base method. +func (m *MockHeartbeater) Check() (heartbeat.State, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NeedToCheckOthers") - ret0, _ := ret[0].(bool) - return ret0 + ret := m.ctrl.Call(m, "Check") + ret0, _ := ret[0].(heartbeat.State) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// NeedToCheckOthers indicates an expected call of NeedToCheckOthers. -func (mr *MockHeartbeaterMockRecorder) NeedToCheckOthers() *gomock.Call { +// Check indicates an expected call of Check. +func (mr *MockHeartbeaterMockRecorder) Check() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NeedToCheckOthers", reflect.TypeOf((*MockHeartbeater)(nil).NeedToCheckOthers)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockHeartbeater)(nil).Check)) } -// NeedTurnOffNotifier mocks base method. -func (m *MockHeartbeater) NeedTurnOffNotifier() bool { +// Type mocks base method. +func (m *MockHeartbeater) Type() datatypes.HeartbeatType { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NeedTurnOffNotifier") - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "Type") + ret0, _ := ret[0].(datatypes.HeartbeatType) return ret0 } -// NeedTurnOffNotifier indicates an expected call of NeedTurnOffNotifier. -func (mr *MockHeartbeaterMockRecorder) NeedTurnOffNotifier() *gomock.Call { +// Type indicates an expected call of Type. +func (mr *MockHeartbeaterMockRecorder) Type() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NeedTurnOffNotifier", reflect.TypeOf((*MockHeartbeater)(nil).NeedTurnOffNotifier)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Type", reflect.TypeOf((*MockHeartbeater)(nil).Type)) } From 0f76bdf22e95d9120bc4525e35e01ffe83734eea Mon Sep 17 00:00:00 2001 From: almostinf Date: Tue, 5 Nov 2024 18:11:50 +0300 Subject: [PATCH 6/8] refactor heartbeats --- notifier/selfstate/heartbeat/database.go | 2 +- notifier/selfstate/heartbeat/heartbeat.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/notifier/selfstate/heartbeat/database.go b/notifier/selfstate/heartbeat/database.go index 592325ef1..69ab7eed7 100644 --- a/notifier/selfstate/heartbeat/database.go +++ b/notifier/selfstate/heartbeat/database.go @@ -15,7 +15,7 @@ var _ Heartbeater = (*databaseHeartbeater)(nil) type DatabaseHeartbeaterConfig struct { HeartbeaterBaseConfig - RedisDisconnectDelay time.Duration `validate:"required,gt=0"` + RedisDisconnectDelay time.Duration `validate:"required_if=Enabled true,gte=0"` } type databaseHeartbeater struct { diff --git a/notifier/selfstate/heartbeat/heartbeat.go b/notifier/selfstate/heartbeat/heartbeat.go index 3a85f4094..b7fbedab3 100644 --- a/notifier/selfstate/heartbeat/heartbeat.go +++ b/notifier/selfstate/heartbeat/heartbeat.go @@ -15,8 +15,8 @@ const ( StateError State = "heartbeat_state_error" ) -// IsDegraded checks if the condition has degraded. -func (lastState State) IsDegraded(newState State) bool { +// IsDegraded checks if the condition is still degraded. +func (State) IsDegraded(newState State) bool { return newState == StateError } From 729bacf93d2899583b266c7204a7c3c9b7fad84c Mon Sep 17 00:00:00 2001 From: almostinf Date: Wed, 6 Nov 2024 15:38:20 +0300 Subject: [PATCH 7/8] add heartbeat metrics to notifier heartbeater --- notifier/selfstate/heartbeat/heartbeat.go | 1 - notifier/selfstate/heartbeat/notifier.go | 17 +++++++++++--- notifier/selfstate/heartbeat/notifier_test.go | 22 +++++++++++++++---- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/notifier/selfstate/heartbeat/heartbeat.go b/notifier/selfstate/heartbeat/heartbeat.go index b7fbedab3..cf96cdc5b 100644 --- a/notifier/selfstate/heartbeat/heartbeat.go +++ b/notifier/selfstate/heartbeat/heartbeat.go @@ -36,7 +36,6 @@ type Heartbeater interface { type HeartbeaterBaseConfig struct { Enabled bool NeedTurnOffNotifier bool - NeedToCheckOthers bool AlertCfg AlertConfig } diff --git a/notifier/selfstate/heartbeat/notifier.go b/notifier/selfstate/heartbeat/notifier.go index 6d92a5ef6..3340ffa43 100644 --- a/notifier/selfstate/heartbeat/notifier.go +++ b/notifier/selfstate/heartbeat/notifier.go @@ -3,6 +3,7 @@ package heartbeat import ( "github.com/moira-alert/moira" "github.com/moira-alert/moira/datatypes" + "github.com/moira-alert/moira/metrics" ) // Verify that notifierHeartbeater matches the Heartbeater interface. @@ -16,14 +17,20 @@ type NotifierHeartbeaterConfig struct { type notifierHeartbeater struct { *heartbeaterBase - cfg NotifierHeartbeaterConfig + metrics *metrics.HeartBeatMetrics + cfg NotifierHeartbeaterConfig } // NewNotifierHeartbeater is a function that creates a new notifierHeartbeater. -func NewNotifierHeartbeater(cfg NotifierHeartbeaterConfig, base *heartbeaterBase) (*notifierHeartbeater, error) { +func NewNotifierHeartbeater( + cfg NotifierHeartbeaterConfig, + base *heartbeaterBase, + metrics *metrics.HeartBeatMetrics, +) (*notifierHeartbeater, error) { return ¬ifierHeartbeater{ - heartbeaterBase: base, cfg: cfg, + heartbeaterBase: base, + metrics: metrics, }, nil } @@ -31,13 +38,17 @@ func NewNotifierHeartbeater(cfg NotifierHeartbeaterConfig, base *heartbeaterBase func (heartbeater *notifierHeartbeater) Check() (State, error) { notifierState, err := heartbeater.database.GetNotifierState() if err != nil { + heartbeater.metrics.MarkNotifierIsAlive(false) return StateError, err } if notifierState != moira.SelfStateOK { + heartbeater.metrics.MarkNotifierIsAlive(false) return StateError, nil } + heartbeater.metrics.MarkNotifierIsAlive(true) + return StateOK, nil } diff --git a/notifier/selfstate/heartbeat/notifier_test.go b/notifier/selfstate/heartbeat/notifier_test.go index a0a76bbc2..22ef32563 100644 --- a/notifier/selfstate/heartbeat/notifier_test.go +++ b/notifier/selfstate/heartbeat/notifier_test.go @@ -6,6 +6,7 @@ import ( "github.com/moira-alert/moira" "github.com/moira-alert/moira/datatypes" + "github.com/moira-alert/moira/metrics" . "github.com/smartystreets/goconvey/convey" ) @@ -13,6 +14,9 @@ import ( func TestNewNotifierHeartbeater(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) + dummyRegistry := metrics.NewDummyRegistry() + heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) + Convey("Test NewNotifierHeartbeater", t, func() { Convey("With correct local checker heartbeater config", func() { cfg := NotifierHeartbeaterConfig{} @@ -20,9 +24,10 @@ func TestNewNotifierHeartbeater(t *testing.T) { expected := ¬ifierHeartbeater{ heartbeaterBase: heartbeaterBase, cfg: cfg, + metrics: heartbeatMetrics, } - notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase, heartbeatMetrics) So(err, ShouldBeNil) So(notifierHeartbeater, ShouldResemble, expected) }) @@ -32,9 +37,12 @@ func TestNewNotifierHeartbeater(t *testing.T) { func TestNotifierHeartbeaterCheck(t *testing.T) { database, _, _, heartbeaterBase := heartbeaterHelper(t) + dummyRegistry := metrics.NewDummyRegistry() + heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) + cfg := NotifierHeartbeaterConfig{} - notifierHeartbeater, _ := NewNotifierHeartbeater(cfg, heartbeaterBase) + notifierHeartbeater, _ := NewNotifierHeartbeater(cfg, heartbeaterBase, heartbeatMetrics) testErr := errors.New("test error") @@ -68,10 +76,13 @@ func TestNotifierHeartbeaterCheck(t *testing.T) { func TestNotifierHeartbeaterType(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) + dummyRegistry := metrics.NewDummyRegistry() + heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) + Convey("Test notifierHeartbeater.Type", t, func() { cfg := NotifierHeartbeaterConfig{} - notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase, heartbeatMetrics) So(err, ShouldBeNil) notifierHeartbeaterType := notifierHeartbeater.Type() @@ -82,6 +93,9 @@ func TestNotifierHeartbeaterType(t *testing.T) { func TestNotifierHeartbeaterAlertSettings(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) + dummyRegistry := metrics.NewDummyRegistry() + heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) + Convey("Test notifierHeartbeater.AlertSettings", t, func() { alertCfg := AlertConfig{ Name: "test name", @@ -94,7 +108,7 @@ func TestNotifierHeartbeaterAlertSettings(t *testing.T) { }, } - notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase, heartbeatMetrics) So(err, ShouldBeNil) alertSettings := notifierHeartbeater.AlertSettings() From 877228ae4eb0ad2116fe7122fa80f6055f092244 Mon Sep 17 00:00:00 2001 From: almostinf Date: Wed, 6 Nov 2024 16:53:28 +0300 Subject: [PATCH 8/8] return previous logic --- notifier/selfstate/heartbeat/notifier.go | 10 +-------- notifier/selfstate/heartbeat/notifier_test.go | 22 ++++--------------- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/notifier/selfstate/heartbeat/notifier.go b/notifier/selfstate/heartbeat/notifier.go index 3340ffa43..b9f604b91 100644 --- a/notifier/selfstate/heartbeat/notifier.go +++ b/notifier/selfstate/heartbeat/notifier.go @@ -3,7 +3,6 @@ package heartbeat import ( "github.com/moira-alert/moira" "github.com/moira-alert/moira/datatypes" - "github.com/moira-alert/moira/metrics" ) // Verify that notifierHeartbeater matches the Heartbeater interface. @@ -17,20 +16,17 @@ type NotifierHeartbeaterConfig struct { type notifierHeartbeater struct { *heartbeaterBase - metrics *metrics.HeartBeatMetrics - cfg NotifierHeartbeaterConfig + cfg NotifierHeartbeaterConfig } // NewNotifierHeartbeater is a function that creates a new notifierHeartbeater. func NewNotifierHeartbeater( cfg NotifierHeartbeaterConfig, base *heartbeaterBase, - metrics *metrics.HeartBeatMetrics, ) (*notifierHeartbeater, error) { return ¬ifierHeartbeater{ cfg: cfg, heartbeaterBase: base, - metrics: metrics, }, nil } @@ -38,17 +34,13 @@ func NewNotifierHeartbeater( func (heartbeater *notifierHeartbeater) Check() (State, error) { notifierState, err := heartbeater.database.GetNotifierState() if err != nil { - heartbeater.metrics.MarkNotifierIsAlive(false) return StateError, err } if notifierState != moira.SelfStateOK { - heartbeater.metrics.MarkNotifierIsAlive(false) return StateError, nil } - heartbeater.metrics.MarkNotifierIsAlive(true) - return StateOK, nil } diff --git a/notifier/selfstate/heartbeat/notifier_test.go b/notifier/selfstate/heartbeat/notifier_test.go index 22ef32563..a0a76bbc2 100644 --- a/notifier/selfstate/heartbeat/notifier_test.go +++ b/notifier/selfstate/heartbeat/notifier_test.go @@ -6,7 +6,6 @@ import ( "github.com/moira-alert/moira" "github.com/moira-alert/moira/datatypes" - "github.com/moira-alert/moira/metrics" . "github.com/smartystreets/goconvey/convey" ) @@ -14,9 +13,6 @@ import ( func TestNewNotifierHeartbeater(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) - dummyRegistry := metrics.NewDummyRegistry() - heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) - Convey("Test NewNotifierHeartbeater", t, func() { Convey("With correct local checker heartbeater config", func() { cfg := NotifierHeartbeaterConfig{} @@ -24,10 +20,9 @@ func TestNewNotifierHeartbeater(t *testing.T) { expected := ¬ifierHeartbeater{ heartbeaterBase: heartbeaterBase, cfg: cfg, - metrics: heartbeatMetrics, } - notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase, heartbeatMetrics) + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) So(err, ShouldBeNil) So(notifierHeartbeater, ShouldResemble, expected) }) @@ -37,12 +32,9 @@ func TestNewNotifierHeartbeater(t *testing.T) { func TestNotifierHeartbeaterCheck(t *testing.T) { database, _, _, heartbeaterBase := heartbeaterHelper(t) - dummyRegistry := metrics.NewDummyRegistry() - heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) - cfg := NotifierHeartbeaterConfig{} - notifierHeartbeater, _ := NewNotifierHeartbeater(cfg, heartbeaterBase, heartbeatMetrics) + notifierHeartbeater, _ := NewNotifierHeartbeater(cfg, heartbeaterBase) testErr := errors.New("test error") @@ -76,13 +68,10 @@ func TestNotifierHeartbeaterCheck(t *testing.T) { func TestNotifierHeartbeaterType(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) - dummyRegistry := metrics.NewDummyRegistry() - heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) - Convey("Test notifierHeartbeater.Type", t, func() { cfg := NotifierHeartbeaterConfig{} - notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase, heartbeatMetrics) + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) So(err, ShouldBeNil) notifierHeartbeaterType := notifierHeartbeater.Type() @@ -93,9 +82,6 @@ func TestNotifierHeartbeaterType(t *testing.T) { func TestNotifierHeartbeaterAlertSettings(t *testing.T) { _, _, _, heartbeaterBase := heartbeaterHelper(t) - dummyRegistry := metrics.NewDummyRegistry() - heartbeatMetrics := metrics.ConfigureHeartBeatMetrics(dummyRegistry) - Convey("Test notifierHeartbeater.AlertSettings", t, func() { alertCfg := AlertConfig{ Name: "test name", @@ -108,7 +94,7 @@ func TestNotifierHeartbeaterAlertSettings(t *testing.T) { }, } - notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase, heartbeatMetrics) + notifierHeartbeater, err := NewNotifierHeartbeater(cfg, heartbeaterBase) So(err, ShouldBeNil) alertSettings := notifierHeartbeater.AlertSettings()