diff --git a/checker/check.go b/checker/check.go index 03aa5283e..e874e2391 100644 --- a/checker/check.go +++ b/checker/check.go @@ -170,27 +170,6 @@ func (triggerChecker *TriggerChecker) handleFetchError(checkData moira.CheckData ) } -func (triggerChecker *TriggerChecker) reactOnSourceUnavailableError(checkData moira.CheckData, err error) moira.CheckData { - timeSinceLastSuccessfulCheck := checkData.Timestamp - checkData.LastSuccessfulCheckTimestamp - if timeSinceLastSuccessfulCheck >= triggerChecker.ttl { - checkData.State = moira.StateEXCEPTION - checkData.Message = fmt.Sprintf("Remote server unavailable. Trigger is not checked for %d seconds", timeSinceLastSuccessfulCheck) - - var comparingErr error - checkData, comparingErr = triggerChecker.compareTriggerStates(checkData) - if comparingErr != nil { - triggerChecker.logger.Error(). - Error(comparingErr). - String(moira.LogFieldNameTriggerID, triggerChecker.triggerID). - Msg("cannot compare trigger states") - } - } - - logTriggerCheckException(triggerChecker.logger, triggerChecker.triggerID, err) - - return checkData -} - // handleUndefinedError is a function that check error with undefined type. func (triggerChecker *TriggerChecker) handleUndefinedError(checkData moira.CheckData, err error) error { triggerChecker.metrics.CheckError.Mark(1) diff --git a/checker/check_test.go b/checker/check_test.go index 994d983ea..cc6a880ba 100644 --- a/checker/check_test.go +++ b/checker/check_test.go @@ -2066,7 +2066,7 @@ func TestTriggerChecker_handleFetchError(t *testing.T) { Metric: triggerChecker.trigger.Name, }, true, - ).Return(nil).Times(2) + ).Return(nil).Times(2) // this is strange... why 2? dataBase.EXPECT().SetTriggerLastCheck( triggerChecker.triggerID, &expectedCheckData, @@ -2076,6 +2076,120 @@ func TestTriggerChecker_handleFetchError(t *testing.T) { err := triggerChecker.handleFetchError(newCheckData(triggerChecker.lastCheck, triggerChecker.until), givenErr) So(err, ShouldBeNil) }) + + Convey("time since last successful check < triggerChecker.ttl", func() { + triggerChecker.ttl = 30 + triggerChecker.lastCheck.LastSuccessfulCheckTimestamp = triggerChecker.until - 15 + + expectedCheckData := moira.CheckData{ + Score: 0, + Metrics: triggerChecker.lastCheck.Metrics, + State: triggerChecker.lastCheck.State, + Timestamp: triggerChecker.until, + EventTimestamp: triggerChecker.until, + LastSuccessfulCheckTimestamp: triggerChecker.lastCheck.LastSuccessfulCheckTimestamp, + Message: "", + MetricsToTargetRelation: map[string]string{}, + } + + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &expectedCheckData, + triggerChecker.trigger.ClusterKey(), + ).Return(nil).Times(1) + + err := triggerChecker.handleFetchError(newCheckData(triggerChecker.lastCheck, triggerChecker.until), givenErr) + So(err, ShouldBeNil) + }) + }) + + Convey("with bad functions, problems in expressions, etc", func() { + errorsList := []error{ + local.ErrorUnknownFunction(errors.New("unknown func \"dance\"")), + local.ErrorEvalExpression(errors.New("eval expr"), "badExpr(dancing.mops"), + remote.ErrRemoteTriggerResponse{ + InternalError: errors.New("user write bad target"), + Target: "bad target", + }, + } + + for i, givenErr := range errorsList { + Convey(fmt.Sprintf("Case %v: %T", i+1, givenErr), func() { + expectedCheckData := moira.CheckData{ + Score: int64(100_000), + Metrics: triggerChecker.lastCheck.Metrics, + State: moira.StateEXCEPTION, + Timestamp: triggerChecker.until, + EventTimestamp: triggerChecker.until, + LastSuccessfulCheckTimestamp: triggerChecker.lastCheck.LastSuccessfulCheckTimestamp, + Message: givenErr.Error(), + MetricsToTargetRelation: map[string]string{}, + } + + dataBase.EXPECT().PushNotificationEvent( + &moira.NotificationEvent{ + IsTriggerEvent: true, + TriggerID: triggerChecker.triggerID, + State: moira.StateEXCEPTION, + OldState: triggerChecker.lastCheck.State, + Timestamp: triggerChecker.until, + Metric: triggerChecker.trigger.Name, + }, + true, + ).Return(nil).Times(1) + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &expectedCheckData, + triggerChecker.trigger.ClusterKey(), + ).Return(nil).Times(1) + + err := triggerChecker.handleFetchError(newCheckData(triggerChecker.lastCheck, triggerChecker.until), givenErr) + So(err, ShouldBeNil) + }) + } + }) + + Convey("with undefined err", func() { + givenErr := errors.New("some undefined error") + + checkMetrics, err := metrics.ConfigureCheckerMetrics( + metrics.NewDummyRegistry(), + []moira.ClusterKey{moira.DefaultLocalCluster}, + ).GetCheckMetricsBySource(moira.DefaultLocalCluster) + So(err, ShouldBeNil) + + triggerChecker.metrics = checkMetrics + + expectedCheckData := moira.CheckData{ + Score: int64(100_000), + Metrics: triggerChecker.lastCheck.Metrics, + State: moira.StateEXCEPTION, + Timestamp: triggerChecker.until, + EventTimestamp: triggerChecker.until, + LastSuccessfulCheckTimestamp: triggerChecker.lastCheck.LastSuccessfulCheckTimestamp, + Message: givenErr.Error(), + MetricsToTargetRelation: map[string]string{}, + } + + dataBase.EXPECT().PushNotificationEvent( + &moira.NotificationEvent{ + IsTriggerEvent: true, + TriggerID: triggerChecker.triggerID, + State: moira.StateEXCEPTION, + OldState: triggerChecker.lastCheck.State, + Timestamp: triggerChecker.until, + Metric: triggerChecker.trigger.Name, + }, + true, + ).Return(nil).Times(1) + dataBase.EXPECT().SetTriggerLastCheck( + triggerChecker.triggerID, + &expectedCheckData, + triggerChecker.trigger.ClusterKey(), + ).Return(nil).Times(1) + + err = triggerChecker.handleFetchError(newCheckData(triggerChecker.lastCheck, triggerChecker.until), givenErr) + So(err, ShouldBeNil) }) }) } diff --git a/metric_source/local/errors.go b/metric_source/local/errors.go index b9605024f..c4f43656a 100644 --- a/metric_source/local/errors.go +++ b/metric_source/local/errors.go @@ -58,6 +58,13 @@ type ErrEvalExpr struct { target string } +func ErrorEvalExpression(err error, target string) ErrEvalExpr { + return ErrEvalExpr{ + internalError: err, + target: target, + } +} + // Error is implementation of golang error interface for ErrEvalExpr struct. func (err ErrEvalExpr) Error() string { return fmt.Sprintf("failed to evaluate target '%s': %s", err.target, err.internalError.Error()) diff --git a/metric_source/local/eval.go b/metric_source/local/eval.go index 4b429d189..5b4dacf47 100644 --- a/metric_source/local/eval.go +++ b/metric_source/local/eval.go @@ -95,10 +95,7 @@ func (eval *evaluator) Eval( } else if isErrUnknownFunction(err) { err = ErrorUnknownFunction(err) } else { - err = ErrEvalExpr{ - target: exp.ToString(), - internalError: err, - } + err = ErrorEvalExpression(err, exp.ToString()) } }