diff --git a/checker/worker/metric_events.go b/checker/worker/metric_events.go index 67c686115..c7a5d44e5 100644 --- a/checker/worker/metric_events.go +++ b/checker/worker/metric_events.go @@ -84,7 +84,9 @@ func (manager *WorkerManager) handleMetricEvent(pattern string) error { start := time.Now() defer manager.Metrics.MetricEventsHandleTime.UpdateSince(start) - manager.lastData = time.Now().UTC().Unix() + now := time.Now().UTC().Unix() + manager.lastData = now + triggerIds, err := manager.Database.GetPatternTriggerIDs(pattern) if err != nil { return err diff --git a/checker/worker/scheduler.go b/checker/worker/scheduler.go index 77eb14b98..c1b5f46c5 100644 --- a/checker/worker/scheduler.go +++ b/checker/worker/scheduler.go @@ -84,7 +84,7 @@ func (ch *scheduler) triggerScheduler(stop <-chan struct{}) error { func (ch *scheduler) scheduleTriggersToCheck() error { err := ch.validateSource() if err != nil { - ch.manager.Logger.Info(). + ch.manager.Logger.Warning(). Error(err). String("cluster_key", ch.clusterKey.String()). Msg("Source is invalid. Stop scheduling trigger checks") diff --git a/checker/worker/trigger_handler.go b/checker/worker/trigger_handler.go index 1b2e617ac..817b6e23b 100644 --- a/checker/worker/trigger_handler.go +++ b/checker/worker/trigger_handler.go @@ -47,7 +47,6 @@ func (manager *WorkerManager) handleTrigger(triggerID string, metrics *metrics.C func (manager *WorkerManager) handleTriggerInLock(triggerID string, metrics *metrics.CheckMetrics) error { acquired, err := manager.Database.SetTriggerCheckLock(triggerID) - defer manager.Database.DeleteTriggerCheckLock(triggerID) //nolint if err != nil { return err @@ -57,6 +56,8 @@ func (manager *WorkerManager) handleTriggerInLock(triggerID string, metrics *met return nil } + defer manager.Database.DeleteTriggerCheckLock(triggerID) //nolint + startedAt := time.Now() defer metrics.TriggersCheckTime.UpdateSince(startedAt) diff --git a/checker/worker/worker_manager.go b/checker/worker/worker_manager.go index 02cec4676..2b1a64b8a 100644 --- a/checker/worker/worker_manager.go +++ b/checker/worker/worker_manager.go @@ -87,11 +87,11 @@ func (manager *WorkerManager) makeSourceValidator(clusterKey moira.ClusterKey) ( func (manager *WorkerManager) validateGraphiteLocal() error { now := time.Now().UTC().Unix() - if manager.lastData+manager.Config.StopCheckingIntervalSeconds < now { + if manager.lastData+manager.Config.StopCheckingIntervalSeconds > now { return nil } - return fmt.Errorf("graphite local source invalid: no metrics for %d second", manager.Config.StopCheckingIntervalSeconds) + return fmt.Errorf("graphite local source invalid: no metrics for %d second", now-manager.lastData) } func (manager *WorkerManager) startCheckerWorker(w *scheduler) error {