diff --git a/Dockerfile b/Dockerfile index ede3cc9..d93bfa1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN GOOS=darwin GARCH=amd64 go build \ -o /dist/promutil \ -a \ -ldflags "-s -w -extldflags \"-fno-PIC -static\" -X github.com/kadaan/promutil/version.Version=$VERSION -X github.com/kadaan/promutil/version.Revision=$REVISION -X github.com/kadaan/promutil/version.Branch=$BRANCH -X github.com/kadaan/promutil/version.BuildUser=$USER -X github.com/kadaan/promutil/version.BuildHost=$HOST -X github.com/kadaan/promutil/version.BuildDate=$BUILD_DATE" \ - -tags 'osusergo netgo' \ + -tags 'osusergo netgo jsoniter' \ -installsuffix netgo && \ tar -czf "/archives/promutil_darwin.tar.gz" -C "/dist" . @@ -39,7 +39,7 @@ RUN GOOS=linux GARCH=amd64 go build \ -o /dist/promutil \ -a \ -ldflags "-s -w -X github.com/kadaan/promutil/version.Version=$VERSION -X github.com/kadaan/promutil/version.Revision=$REVISION -X github.com/kadaan/promutil/version.Branch=$BRANCH -X github.com/kadaan/promutil/version.BuildUser=$USER -X github.com/kadaan/promutil/version.BuildHost=$HOST -X github.com/kadaan/promutil/version.BuildDate=$BUILD_DATE" \ - -tags 'osusergo netgo static_build' \ + -tags 'osusergo netgo jsoniter static_build' \ -installsuffix netgo && \ tar -czf "/archives/promutil_linux.tar.gz" -C "/dist" . diff --git a/cmd/web.go b/cmd/web.go index 31fedd1..9942d76 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -2,6 +2,7 @@ package cmd import ( "github.com/kadaan/promutil/config" + "github.com/kadaan/promutil/lib/block" "github.com/kadaan/promutil/lib/command" "github.com/kadaan/promutil/lib/web" ) @@ -17,5 +18,6 @@ func init() { fb.ListenAddress(&cfg.ListenAddress, "the listen address") fb.SampleInterval(&cfg.SampleInterval, "interval at which samples will be taken within a range") fb.Host(&cfg.Host, "remote prometheus host") + fb.Parallelism(&cfg.Parallelism, block.MaxParallelism, "parallelism for backfill") }) } diff --git a/config/common.go b/config/common.go index fb3c7bb..2e97b8e 100644 --- a/config/common.go +++ b/config/common.go @@ -36,6 +36,7 @@ var ( defaultRuleGroupFilters = []*regexp.Regexp{regexp.MustCompile(".+")} defaultRuleNameFilters = []*regexp.Regexp{regexp.MustCompile(".+")} yamlFileExtensions = []string{"yml", "yaml"} + defaultListenAddress = ListenAddress{Host: "", Port: 8080} ) func NewFlagBuilder(cmd *cobra.Command) FlagBuilder { @@ -241,6 +242,6 @@ func (fb *flagBuilder) Matchers(dest *map[string][]*labels.Matcher, usage string func (fb *flagBuilder) ListenAddress(dest *ListenAddress, usage string) Flag { return fb.newFlag(listenAddressKey, func(flagSet *pflag.FlagSet) { - flagSet.Var(NewListenAddressValue(dest, ListenAddress{Host: "", Port: 8080}), listenAddressKey, usage) + flagSet.Var(NewListenAddressValue(dest, defaultListenAddress), listenAddressKey, usage) }) } diff --git a/config/web.go b/config/web.go index 2ba3226..28973f6 100644 --- a/config/web.go +++ b/config/web.go @@ -5,13 +5,10 @@ import ( "time" ) -const ( - DefaultListenAddress = ":8080" -) - // WebConfig represents the configuration of the web command. type WebConfig struct { ListenAddress ListenAddress Host *url.URL SampleInterval time.Duration + Parallelism uint8 } diff --git a/lib/block/planner.go b/lib/block/planner.go index f07db19..f88526d 100644 --- a/lib/block/planner.go +++ b/lib/block/planner.go @@ -146,12 +146,12 @@ func (p *planner[V]) Plan(transform func(int64, int64, int64) []PlanEntry[V]) [] stepDuration := int64(p.config.SampleInterval() / (time.Millisecond / time.Nanosecond)) for ; blockStart <= endInMs; blockStart = blockStart + p.config.BlockDuration() { blockEnd := blockStart + p.config.BlockDuration() - 1 - currStart := p.max(blockStart/int64(time.Second/time.Millisecond), p.config.StartTime().Unix()) + currStart := common.MaxInt64(blockStart/int64(time.Second/time.Millisecond), p.config.StartTime().Unix()) startWithAlignment := p.evalTimestamp(time.Unix(currStart, 0).UTC().UnixNano(), stepDuration) for startWithAlignment.Unix() < currStart { startWithAlignment = startWithAlignment.Add(p.config.SampleInterval()) } - end := time.Unix(p.min(blockEnd/int64(time.Second/time.Millisecond), p.config.EndTime().Unix()), 0).UTC() + end := time.Unix(common.MinInt64(blockEnd/int64(time.Second/time.Millisecond), p.config.EndTime().Unix()), 0).UTC() if end.Equal(startWithAlignment) || end.Before(startWithAlignment) { break } @@ -181,20 +181,6 @@ func (p *planner[V]) planBlock(blockStart time.Time, blockEnd time.Time, stepDur return plan } -func (p *planner[V]) max(x, y int64) int64 { - if x > y { - return x - } - return y -} - -func (p *planner[V]) min(x, y int64) int64 { - if x < y { - return x - } - return y -} - func (p *planner[V]) evalTimestamp(startTime int64, stepDuration int64) time.Time { var ( offset = stepDuration @@ -437,7 +423,7 @@ func (p *plannedBlockWriter[V]) Run() error { go func() { select { - case <-s.C: + case <-s.C(): klog.V(0).Infof("Stopping producer, consumers, and db") cancel() } diff --git a/lib/command/command.go b/lib/command/command.go index 4eacd91..f6ad583 100644 --- a/lib/command/command.go +++ b/lib/command/command.go @@ -10,9 +10,13 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" + "io" "k8s.io/klog/v2" "log" "os" + "runtime" + "runtime/pprof" + "runtime/trace" ) var ( @@ -22,6 +26,7 @@ var ( type RootCommand interface { Execute() addCommand(cmd *cobra.Command) + startProfiler() (io.Closer, error) } func NewRootCommand(short string, long string) RootCommand { @@ -42,15 +47,68 @@ func NewRootCommand(short string, long string) RootCommand { r.addVersionCommand(r.cmd) r.addCompletionCommand(r.cmd) cobra.OnInitialize(r.initConfig) + r.cmd.PersistentFlags().StringVar(&r.cpuProfile, "cpuProfile", "", "Cpu profile result file") + r.cmd.PersistentFlags().StringVar(&r.memoryProfile, "memoryProfile", "", "Memory profile result file") + r.cmd.PersistentFlags().StringVar(&r.traceProfile, "traceProfile", "", "Trace profile result file") r.cmd.PersistentFlags().CountVarP(&r.verbosity, "verbose", "v", "enables verbose logging (multiple times increases verbosity)") r.cmd.PersistentFlags().StringVar(&r.cfgFile, "config", "", "config file (default is ."+version.Name+".config)") return r } type rootCommand struct { - verbosity int - cfgFile string - cmd *cobra.Command + cpuProfile string + memoryProfile string + traceProfile string + verbosity int + cfgFile string + cmd *cobra.Command +} + +func (r *rootCommand) startProfiler() (io.Closer, error) { + p := profiler{} + if r.traceProfile != "" { + f, err := os.Create(r.traceProfile) + if err != nil { + return &p, errors.Wrap(err, "could not create Trace profile") + } + p.fileClosers = append(p.fileClosers, f) + if err = trace.Start(f); err != nil { + return &p, errors.Wrap(err, "could not start Trace profile") + } + p.methodClosers = append(p.methodClosers, func() error { + trace.Stop() + return nil + }) + } + if r.cpuProfile != "" { + f, err := os.Create(r.cpuProfile) + if err != nil { + return &p, errors.Wrap(err, "could not create CPU profile") + } + p.fileClosers = append(p.fileClosers, f) + if err = pprof.StartCPUProfile(f); err != nil { + return &p, errors.Wrap(err, "could not start CPU profile") + } + p.methodClosers = append(p.methodClosers, func() error { + pprof.StopCPUProfile() + return nil + }) + } + if r.memoryProfile != "" { + f, err := os.Create(r.memoryProfile) + if err != nil { + return &p, errors.Wrap(err, "could not create memory profile") + } + p.fileClosers = append(p.fileClosers, f) + p.methodClosers = append(p.methodClosers, func() error { + runtime.GC() + if err = pprof.WriteHeapProfile(f); err != nil { + return errors.Wrap(err, "could not write memory profile") + } + return nil + }) + } + return &p, nil } func (r *rootCommand) addCommand(cmd *cobra.Command) { @@ -188,7 +246,17 @@ func NewCommand[C any](root RootCommand, use string, short string, long string, Short: short, Long: long, RunE: func(cmd *cobra.Command, args []string) error { - if err := task.Run(cfg); err != nil { + p, err := root.startProfiler() + defer func(profiler io.Closer) { + errP := profiler.Close() + if errP != nil { + klog.Errorf("failed to close profiler: %w", errP) + } + }(p) + if err != nil { + return errors.Wrap(err, "failed to start profiler") + } + if err = task.Run(cfg); err != nil { return errors.Wrap(err, "%s failed", use) } return nil @@ -204,3 +272,26 @@ func NewCommand[C any](root RootCommand, use string, short string, long string, type Task[C any] interface { Run(cfg *C) error } + +type profiler struct { + fileClosers []io.Closer + methodClosers []func() error +} + +func (p *profiler) Close() error { + var errs []error + for _, methodCloser := range p.methodClosers { + if err := methodCloser(); err != nil { + errs = append(errs, err) + } + } + for _, fileCloser := range p.fileClosers { + if err := fileCloser.Close(); err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return errors.NewMulti(errs, "failed to close profiler") + } + return nil +} diff --git a/lib/common/canceller.go b/lib/common/canceller.go index 5a834d9..67069b8 100644 --- a/lib/common/canceller.go +++ b/lib/common/canceller.go @@ -3,22 +3,35 @@ package common import "sync" type Canceller interface { + C() chan struct{} + Cancelled() bool Cancel() } type canceller struct { - C chan struct{} - once sync.Once + cancelled bool + c chan struct{} + once sync.Once } func NewCanceller() *canceller { return &canceller{ - C: make(chan struct{}), + cancelled: false, + c: make(chan struct{}), } } +func (s *canceller) C() chan struct{} { + return s.c +} + +func (s *canceller) Cancelled() bool { + return s.cancelled +} + func (s *canceller) Cancel() { s.once.Do(func() { - close(s.C) + s.cancelled = true + close(s.c) }) } diff --git a/lib/common/common.go b/lib/common/common.go index 825fbed..a21d79d 100644 --- a/lib/common/common.go +++ b/lib/common/common.go @@ -101,3 +101,31 @@ func JoinUrl(base *url.URL, paths ...string) (*url.URL, error) { p := path.Join(paths...) return url.Parse(fmt.Sprintf("%s/%s", strings.TrimRight(s, "/"), strings.TrimLeft(p, "/"))) } + +func MaxInt64(x int64, y int64) int64 { + if x > y { + return x + } + return y +} + +func MinInt64(x int64, y int64) int64 { + if x < y { + return x + } + return y +} + +func MaxUInt8(x uint8, y uint8) uint8 { + if x > y { + return x + } + return y +} + +func MinTime(x time.Time, y time.Time) time.Time { + if x.After(y) { + return y + } + return x +} diff --git a/lib/remote/queryable.go b/lib/remote/queryable.go index 2313c8c..cad034c 100644 --- a/lib/remote/queryable.go +++ b/lib/remote/queryable.go @@ -7,6 +7,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/kadaan/promutil/lib/common" "github.com/kadaan/promutil/lib/errors" + "github.com/kadaan/tracerr" "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" @@ -14,28 +15,72 @@ import ( "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" - "math" + "io" "net/url" "sort" + "sync" "time" - //"time" ) -func NewQueryable(address *url.URL) (Queryable, error) { +const ( + maxChunkDuration = 30 * time.Minute +) + +func NewQueryable(address *url.URL, parallelism uint8) (Queryable, error) { client, err := api.NewClient(api.Config{ Address: address.String(), }) if err != nil { return nil, errors.Wrap(err, "failed to create queryable provider") } + ctx, cancel := context.WithCancel(context.Background()) + var cg sync.WaitGroup + promApi := v1.NewAPI(client) + inputChan := make(chan plan) + for i := uint8(0); i < common.MaxUInt8(parallelism, uint8(1)); i++ { + qr := querier{ + ctx: ctx, + cg: &cg, + promApi: promApi, + input: inputChan, + stopOnce: &sync.Once{}, + } + cg.Add(1) + go qr.run() + } return &queryable{ - client: client, + cg: &cg, + inputChan: inputChan, + cancel: cancel, + }, nil +} + +type queryable struct { + cg *sync.WaitGroup + inputChan chan plan + cancel context.CancelFunc +} + +func (q *queryable) Close() error { + q.cancel() + q.cg.Wait() + return nil +} + +func (q *queryable) QueryFuncProvider(minTimestamp time.Time, maxTimestamp time.Time, step time.Duration) (QueryFuncProvider, error) { + return &queryFuncProvider{ + minTimestamp: minTimestamp, + maxTimestamp: maxTimestamp, + step: step, + inputChan: q.inputChan, + queryResultCache: map[string]map[time.Duration][]*queryCacheEntry{}, }, nil } type RangeQueryFunc func(ctx context.Context, qs string, start time.Time, end time.Time, interval time.Duration) (promql.Matrix, error) type Queryable interface { + io.Closer QueryFuncProvider(minTimestamp time.Time, maxTimestamp time.Time, step time.Duration) (QueryFuncProvider, error) } @@ -50,15 +95,14 @@ type queryCacheEntry struct { query string matrix *promql.Matrix interval time.Duration - stats []seriesStats cached bool } type queryFuncProvider struct { - promApi v1.API minTimestamp time.Time maxTimestamp time.Time step time.Duration + inputChan chan plan queryResultCache map[string]map[time.Duration][]*queryCacheEntry } @@ -132,17 +176,17 @@ func (q *queryFuncProvider) InstantQueryFunc(allowArbitraryQueries bool) rules.Q func (q *queryFuncProvider) RangeQueryFunc() RangeQueryFunc { return func(ctx context.Context, qs string, start time.Time, end time.Time, interval time.Duration) (promql.Matrix, error) { - result, err := q.query(ctx, qs, start, end, interval, true, true) + r, err := q.query(ctx, qs, start, end, interval, true, true) if err != nil { return nil, err } - if result.minTimestamp.Equal(start) && result.maxTimestamp.Equal(end) { - return *result.matrix, nil + if r.minTimestamp.Equal(start) && r.maxTimestamp.Equal(end) { + return *r.matrix, nil } minTs := start.UnixMilli() maxTs := end.UnixMilli() var matrix promql.Matrix - for _, series := range *result.matrix { + for _, series := range *r.matrix { var points []promql.Point for _, point := range series.Points { if point.T >= minTs && point.T <= maxTs { @@ -160,50 +204,63 @@ func (q *queryFuncProvider) RangeQueryFunc() RangeQueryFunc { } } -type seriesStats struct { - MinTimestamp int64 - MaxTimestamp int64 +type plan struct { + ctx context.Context + wg *sync.WaitGroup + queryRange v1.Range + queryExpr string + output chan<- result + stopOnce *sync.Once + canceller common.Canceller } -func (q *queryFuncProvider) query(ctx context.Context, query string, start time.Time, end time.Time, interval time.Duration, addToCache bool, allowArbitraryQueries bool) (*queryCacheEntry, error) { - if queryCache, ok := q.queryResultCache[query]; ok { - if intervalCache, ok2 := queryCache[interval]; ok2 { - for _, entry := range intervalCache { - if start.Before(entry.minTimestamp) || - start.After(entry.maxTimestamp) || - end.Before(entry.minTimestamp) || - end.After(entry.maxTimestamp) { - // Skip because the request query is not a subset of this query - } else { - return entry, nil +type result struct { + cancelled bool + wg *sync.WaitGroup + err tracerr.Error + series []*promql.Series +} + +type querier struct { + ctx context.Context + cg *sync.WaitGroup + promApi v1.API + input <-chan plan + stopOnce *sync.Once +} + +func (q *querier) run() { + for { + select { + case <-q.ctx.Done(): + q.stopOnce.Do(func() { + q.cg.Done() + }) + return + case p, ok := <-q.input: + if !ok { + p.stopOnce.Do(func() { + q.cg.Done() + }) + return + } + if p.canceller.Cancelled() { + p.output <- result{ + cancelled: true, + wg: p.wg, } + continue } + q.execute(p) } } +} - if !allowArbitraryQueries { - var matrix promql.Matrix - entry := &queryCacheEntry{ - minTimestamp: start, - maxTimestamp: end, - interval: interval, - query: query, - matrix: &matrix, - cached: false, - } - return entry, nil - } - - rng := v1.Range{ - Start: start, - End: end, - Step: interval, - } - +func (q *querier) execute(p plan) { var value *model.Value const attempts = 5 err := backoff.Retry(func() error { - r, _, e := q.promApi.QueryRange(ctx, query, rng) + r, _, e := q.promApi.QueryRange(p.ctx, p.queryExpr, p.queryRange) if e != nil { return e } @@ -211,36 +268,24 @@ func (q *queryFuncProvider) query(ctx context.Context, query string, start time. return nil }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), attempts)) if err != nil { - return nil, errors.Wrap(err, "failed to query '%s' from %d to %d after %d attempts", query, rng.Start, rng.End, attempts) + p.output <- result{ + wg: p.wg, + err: errors.Wrap(err, "failed to query '%s' from %d to %d after %d attempts", p.queryExpr, p.queryRange.Start, p.queryRange.End, attempts), + } + return } - var stats []seriesStats - var matrix promql.Matrix + switch v := (*value).(type) { case model.Matrix: + var series []*promql.Series for _, ss := range v { - stat := seriesStats{ - MinTimestamp: math.MaxInt64, - MaxTimestamp: math.MinInt64, - } var points []promql.Point for _, s := range ss.Values { - ts := common.TimeMilliseconds(s.Timestamp) - if ts < stat.MinTimestamp { - stat.MinTimestamp = ts - } - if ts > stat.MaxTimestamp { - stat.MaxTimestamp = ts - } points = append(points, promql.Point{ - T: ts, + T: common.TimeMilliseconds(s.Timestamp), V: float64(s.Value), }) } - for _, s := range matrix { - sort.Slice(s.Points, func(i, j int) bool { - return s.Points[i].T-s.Points[j].T < 0 - }) - } var metric labels.Labels for k, v := range ss.Metric { metric = append(metric, labels.Label{ @@ -249,26 +294,135 @@ func (q *queryFuncProvider) query(ctx context.Context, query string, start time. }) } sort.Sort(metric) - sort.Slice(points, func(i, j int) bool { - return points[i].T-points[j].T < 0 - }) - matrix = append(matrix, promql.Series{ + series = append(series, &promql.Series{ Metric: metric, Points: points, }) - stats = append(stats, stat) + } + p.output <- result{ + wg: p.wg, + series: series, } default: - return nil, errors.New("query range result is not a matrix") + p.output <- result{ + wg: p.wg, + err: errors.New("query range result is not a matrix"), + } + } +} + +func (q *queryFuncProvider) query(ctx context.Context, query string, start time.Time, end time.Time, interval time.Duration, addToCache bool, allowArbitraryQueries bool) (*queryCacheEntry, error) { + if queryCache, ok := q.queryResultCache[query]; ok { + if intervalCache, ok2 := queryCache[interval]; ok2 { + for _, entry := range intervalCache { + if start.Before(entry.minTimestamp) || + start.After(entry.maxTimestamp) || + end.Before(entry.minTimestamp) || + end.After(entry.maxTimestamp) { + // Skip because the request query is not a subset of this query + } else { + return entry, nil + } + } + } + } + + if !allowArbitraryQueries { + var matrix promql.Matrix + entry := &queryCacheEntry{ + minTimestamp: start, + maxTimestamp: end, + interval: interval, + query: query, + matrix: &matrix, + cached: false, + } + return entry, nil + } + + pCtx, cancel := context.WithCancel(ctx) + canceller := common.NewCanceller() + outputChan := make(chan result) + defer close(outputChan) + + var wg sync.WaitGroup + var err tracerr.Error + metricSeriesMap := make(map[uint64]*promql.Series) + go func(canceller common.Canceller) { + for { + select { + case s, ok := <-outputChan: + if !ok { + return + } + if s.err != nil { + err = s.err + canceller.Cancel() + } else if !s.cancelled { + for _, ss := range s.series { + metricHash := ss.Metric.Hash() + if series, exists := metricSeriesMap[metricHash]; !exists { + metricSeriesMap[metricHash] = ss + } else { + series.Points = append(series.Points, ss.Points...) + } + } + } + s.wg.Done() + } + } + }(canceller) + + chunkStart := start + for ; !chunkStart.After(end); chunkStart = chunkStart.Add(maxChunkDuration) { + chunkEnd := common.MinTime(chunkStart.Add(maxChunkDuration).Add(-1*time.Nanosecond), end) + wg.Add(1) + q.inputChan <- plan{ + canceller: canceller, + ctx: pCtx, + wg: &wg, + queryRange: v1.Range{ + Start: chunkStart, + End: chunkEnd, + Step: interval, + }, + queryExpr: query, + output: outputChan, + stopOnce: &sync.Once{}, + } + } + + go func() { + select { + case <-ctx.Done(): + canceller.Cancel() + cancel() + case <-canceller.C(): + cancel() + } + }() + + wg.Wait() + cancel() + + if err != nil { + return nil, err + } + + var matrix promql.Matrix + for _, s := range metricSeriesMap { + sort.Slice(s.Points, func(i, j int) bool { + return s.Points[i].T-s.Points[j].T < 0 + }) + matrix = append(matrix, *s) } entry := &queryCacheEntry{ - minTimestamp: rng.Start, - maxTimestamp: rng.End, + minTimestamp: start, + maxTimestamp: end, interval: interval, query: query, matrix: &matrix, - stats: stats, cached: false, } @@ -277,26 +431,11 @@ func (q *queryFuncProvider) query(ctx context.Context, query string, start time. if _, exists := q.queryResultCache[query]; !exists { q.queryResultCache[query] = make(map[time.Duration][]*queryCacheEntry) } - if _, exists := q.queryResultCache[query][rng.Step]; !exists { - q.queryResultCache[query][rng.Step] = make([]*queryCacheEntry, 0) + if _, exists := q.queryResultCache[query][interval]; !exists { + q.queryResultCache[query][interval] = make([]*queryCacheEntry, 0) } - q.queryResultCache[query][rng.Step] = append(q.queryResultCache[query][rng.Step], entry) + q.queryResultCache[query][interval] = append(q.queryResultCache[query][interval], entry) } return entry, nil } - -type queryable struct { - client api.Client -} - -func (q *queryable) QueryFuncProvider(minTimestamp time.Time, maxTimestamp time.Time, step time.Duration) (QueryFuncProvider, error) { - promApi := v1.NewAPI(q.client) - return &queryFuncProvider{ - promApi: promApi, - minTimestamp: minTimestamp, - maxTimestamp: maxTimestamp, - step: step, - queryResultCache: map[string]map[time.Duration][]*queryCacheEntry{}, - }, nil -} diff --git a/lib/web/alertTester.go b/lib/web/alertTester.go index ae99461..d84d612 100644 --- a/lib/web/alertTester.go +++ b/lib/web/alertTester.go @@ -1,12 +1,12 @@ package web import ( - "bytes" "context" "fmt" "github.com/cespare/xxhash/v2" "github.com/gin-gonic/gin" kitLog "github.com/go-kit/kit/log" + jsoniter "github.com/json-iterator/go" "github.com/kadaan/promutil/config" "github.com/kadaan/promutil/lib/common" "github.com/kadaan/promutil/lib/errors" @@ -18,11 +18,15 @@ import ( "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/util/stats" + htmlTemplate "html/template" + "math" "net/http" "net/url" + "sort" + "strconv" "strings" - textTemplate "text/template" "time" + "unsafe" ) const ( @@ -30,52 +34,112 @@ const ( alertTestingRoute = "/alerts_testing" alertRuleTestingRoute = "/alert-rule-testing" alertForStateMetricName = "ALERTS_FOR_STATE" - alertHtmlSnippet = `name: {{ .Alert }} -expr: {{ .Expr }} -for: {{ .For }} -{{- if .Labels }} -labels: - {{- range $key, $value := .Labels }} - {{ $key }}: {{ $value }} - {{- end }} -{{- end }} -{{- if .Annotations }} -annotations: - {{- range $key, $value := .Annotations }} - {{ $key }}: {{ $value }} - {{- end }} -{{- end }}` + // alertHtmlSnippet = `name: {{ .Alert }} + //expr: {{ .Expr }} + //for: {{ .For }} + //{{- if .Labels }} + //labels: + // {{- range $key, $value := .Labels }} + // {{ $key }}: {{ $value }} + // {{- end }} + //{{- end }} + //{{- if .Annotations }} + //annotations: + // {{- range $key, $value := .Annotations }} + // {{ $key }}: {{ $value }} + // {{- end }} + //{{- end }}` ) -type timestamp int64 +func init() { + jsoniter.RegisterTypeEncoderFunc("float64", marshalValueJSON, marshalValueJSONIsEmpty) + jsoniter.RegisterTypeEncoderFunc("time.Time", marshalTimeJSON, marshalTimeJSONIsEmpty) + jsoniter.RegisterTypeEncoderFunc("promql.Point", marshalPointJSON, marshalPointJSONIsEmpty) +} + +func marshalPointJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) { + p := *((*promql.Point)(ptr)) + stream.WriteArrayStart() + marshalTimestamp(p.T, stream) + stream.WriteMore() + marshalValue(p.V, stream) + stream.WriteArrayEnd() +} -func (t timestamp) MarshalJSON() ([]byte, error) { - buffer := bytes.Buffer{} - ts := int64(t) - if ts < 0 { - buffer.WriteString("-") - ts = -ts +func marshalPointJSONIsEmpty(_ unsafe.Pointer) bool { + return false +} + +func marshalTimeJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) { + p := *((*time.Time)(ptr)) + if p.IsZero() { + stream.WriteNil() + } else { + marshalTimestamp(p.UnixMilli(), stream) } - buffer.WriteString(fmt.Sprintf("%d", ts/1000)) - fraction := ts % 1000 +} + +func marshalTimeJSONIsEmpty(_ unsafe.Pointer) bool { + return false +} + +func marshalTimestamp(t int64, stream *jsoniter.Stream) { + // Write out the timestamp as a float divided by 1000. + // This is ~3x faster than converting to a float. + if t < 0 { + stream.WriteRaw(`-`) + t = -t + } + stream.WriteInt64(t / 1000) + fraction := t % 1000 if fraction != 0 { - buffer.WriteString(".") + stream.WriteRaw(`.`) if fraction < 100 { - buffer.WriteString("0") + stream.WriteRaw(`0`) } if fraction < 10 { - buffer.WriteString("0") + stream.WriteRaw(`0`) + } + stream.WriteInt64(fraction) + } +} + +func marshalValueJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) { + p := *((*float64)(ptr)) + marshalValue(p, stream) +} + +func marshalValueJSONIsEmpty(_ unsafe.Pointer) bool { + return false +} + +func marshalValue(v float64, stream *jsoniter.Stream) { + if math.IsNaN(v) { + stream.WriteString("NaN") + } else { + stream.WriteRaw(`"`) + // Taken from https://github.com/json-iterator/go/blob/master/stream_float.go#L71 as a workaround + // to https://github.com/json-iterator/go/issues/365 (jsoniter, to follow json standard, doesn't allow inf/nan). + buf := stream.Buffer() + abs := math.Abs(v) + valueFmt := byte('f') + // Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right. + if abs != 0 { + if abs < 1e-6 || abs >= 1e21 { + valueFmt = 'e' + } } - buffer.WriteString(fmt.Sprintf("%d", fraction)) + buf = strconv.AppendFloat(buf, v, valueFmt, -1, 64) + stream.SetBuffer(buf) + stream.WriteRaw(`"`) } - return buffer.Bytes(), nil } type alertsTestResult struct { IsError bool `json:"isError"` Errors []string `json:"errors"` - Start timestamp `json:"start"` - End timestamp `json:"end"` + Start time.Time `json:"start"` + End time.Time `json:"end"` Step int64 `json:"step"` AlertStateToRowClass map[rules.AlertState]string `json:"alertStateToRowClass"` AlertStateToName map[rules.AlertState]string `json:"alertStateToName"` @@ -109,12 +173,20 @@ func newAlertsTestResult() alertsTestResult { } type ruleResult struct { - Group string `json:"group"` - Name string `json:"name"` + Definition *alertDefinition `json:"definition"` Alerts *[]rules.Alert `json:"alerts"` MatrixResult *queryData `json:"matrixResult"` ExprQueryResult *queryDataWithExpr `json:"exprQueryResult"` - HTMLSnippet string `json:"htmlSnippet"` +} + +type alertDefinition struct { + Group string `json:"group"` + Name string `json:"name"` + Expr string `json:"expr"` + ExprTableUrl string `json:"exprTableUrl"` + For string `json:"for"` + Labels []map[string]string `json:"labels"` + Annotations []map[string]string `json:"annotations"` } type queryData struct { @@ -130,21 +202,21 @@ type queryDataWithExpr struct { } type alertTester struct { - templateExecutor TemplateExecutor - queryable remote.Queryable - config *config.WebConfig - alertHtmlSnippetTemplate *textTemplate.Template + templateExecutor TemplateExecutor + queryable remote.Queryable + config *config.WebConfig + //alertHtmlSnippetTemplate *textTemplate.Template } func NewAlertTester(config *config.WebConfig) (Route, error) { - alertHtmlSnippetTemplate := textTemplate.New("alertHtmlSnippet") - var err error - if alertHtmlSnippetTemplate, err = alertHtmlSnippetTemplate.Parse(alertHtmlSnippet); err != nil { - return nil, errors.Wrap(err, "failed to parse alert html snippet") - } + //alertHtmlSnippetTemplate := textTemplate.New("alertHtmlSnippet") + //var err error + //if alertHtmlSnippetTemplate, err = alertHtmlSnippetTemplate.Parse(alertHtmlSnippet); err != nil { + // return nil, errors.Wrap(err, "failed to parse alert html snippet") + //} return &alertTester{ - config: config, - alertHtmlSnippetTemplate: alertHtmlSnippetTemplate, + config: config, + //alertHtmlSnippetTemplate: alertHtmlSnippetTemplate, }, nil } @@ -180,15 +252,15 @@ func (t *alertTester) alertsTesting(requestContext *gin.Context) { if cfg, err := t.parseAlertsTestingBody(requestContext.Request); err != nil { result.addErrors(err) } else { - result.Start = timestamp(cfg.Start.UnixMilli()) - result.End = timestamp(cfg.End.UnixMilli()) + result.Start = time.UnixMilli(cfg.Start.UnixMilli()) + result.End = time.UnixMilli(cfg.End.UnixMilli()) result.Step = cfg.Step.Milliseconds() for _, group := range cfg.RuleGroups.Groups { for _, rule := range group.Rules { if rule.Alert.Value == "" { continue } - htmlSnippet, alerts, matrixResult, exprQueryResult, errA := t.evaluateAlertRule( + alertDefinition, alerts, matrixResult, exprQueryResult, errA := t.evaluateAlertRule( requestContext.Request.Context(), t.queryable, cfg.Start, @@ -197,10 +269,8 @@ func (t *alertTester) alertsTesting(requestContext *gin.Context) { group, rule) r := ruleResult{ - Group: group.Name, - Name: rule.Alert.Value, + Definition: alertDefinition, Alerts: alerts, - HTMLSnippet: htmlSnippet, MatrixResult: matrixResult, ExprQueryResult: exprQueryResult, } @@ -247,12 +317,6 @@ func (t *alertTester) parseAlertsTestingBody(r *http.Request) (*alertsTestingCon } } - // For safety, limit the number of returned points per timeseries. - // This is sufficient for 60s resolution for a week or 1h resolution for a year. - if end.Sub(start)/step > 11000 { - return nil, errors.New("failed to parse alert testing request: exceeded maximum resolution of 11,000 points") - } - configStringUnescaped, err := url.QueryUnescape(configString) if err != nil { return nil, errors.Wrap(err, "failed to parse alert testing request: could not unescape rule config") @@ -271,10 +335,10 @@ func (t *alertTester) parseAlertsTestingBody(r *http.Request) (*alertsTestingCon }, nil } -func (t *alertTester) evaluateAlertRule(ctx context.Context, queryable remote.Queryable, minTimestamp time.Time, maxTimestamp time.Time, step time.Duration, group rulefmt.RuleGroup, rule rulefmt.RuleNode) (string, *[]rules.Alert, *queryData, *queryDataWithExpr, error) { +func (t *alertTester) evaluateAlertRule(ctx context.Context, queryable remote.Queryable, minTimestamp time.Time, maxTimestamp time.Time, step time.Duration, group rulefmt.RuleGroup, rule rulefmt.RuleNode) (*alertDefinition, *[]rules.Alert, *queryData, *queryDataWithExpr, error) { expr, err := parser.ParseExpr(rule.Expr.Value) if err != nil { - return "", nil, nil, nil, errors.Wrap(err, "failed to parse the expression %q", rule.Expr) + return nil, nil, nil, nil, errors.Wrap(err, "failed to parse the expression %q", rule.Expr) } interval := time.Duration(group.Interval) if interval <= 0 { @@ -284,8 +348,8 @@ func (t *alertTester) evaluateAlertRule(ctx context.Context, queryable remote.Qu rule.Alert.Value, expr, time.Duration(rule.For), - labels.FromMap(rule.Labels), - labels.FromMap(rule.Annotations), + labels.Labels{}, + labels.Labels{}, labels.Labels{}, "", true, @@ -294,7 +358,7 @@ func (t *alertTester) evaluateAlertRule(ctx context.Context, queryable remote.Qu provider, err := queryable.QueryFuncProvider(minTimestamp, maxTimestamp, interval) if err != nil { - return "", nil, nil, nil, errors.Wrap(err, "failed to create queryable") + return nil, nil, nil, nil, errors.Wrap(err, "failed to create queryable") } maxSamples := int((maxTimestamp.UnixMilli() - minTimestamp.UnixMilli()) / step.Milliseconds()) @@ -306,12 +370,12 @@ func (t *alertTester) evaluateAlertRule(ctx context.Context, queryable remote.Qu maxTimestamp, interval) if err != nil { - return "", nil, nil, nil, errors.Wrap(err, "failed to query %s from %d to %d", rule.Expr.Value, minTimestamp, maxTimestamp) + return nil, nil, nil, nil, errors.Wrap(err, "failed to query %s from %d to %d", rule.Expr.Value, minTimestamp, maxTimestamp) } queryMatrix = common.DownsampleMatrix(queryMatrix, maxSamples, true) - activeAlertsByLabels := make(map[uint64][]*rules.Alert) + importantAlertTimestampSet := make(map[time.Time]interface{}) queryFunc := provider.InstantQueryFunc(false) seriesHashMap := make(map[uint64]*promql.Series) @@ -330,7 +394,7 @@ func (t *alertTester) evaluateAlertRule(ctx context.Context, queryable remote.Qu nil, group.Limit) if errA != nil { - return "", nil, nil, nil, errors.Wrap(errA, "failed to evaluate rule %s at %d", rule.Expr.Value, ts) + return nil, nil, nil, nil, errors.Wrap(errA, "failed to evaluate rule %s at %d", rule.Expr.Value, ts) } for _, smpl := range vec { series, ok := seriesHashMap[smpl.Metric.Hash()] @@ -340,6 +404,67 @@ func (t *alertTester) evaluateAlertRule(ctx context.Context, queryable remote.Qu } series.Points = append(series.Points, smpl.Point) } + alertingRule.ForEachActiveAlert(func(activeAlert *rules.Alert) { + if !activeAlert.ActiveAt.IsZero() { + importantAlertTimestampSet[activeAlert.ActiveAt] = nil + } + if !activeAlert.FiredAt.IsZero() { + importantAlertTimestampSet[activeAlert.FiredAt] = nil + } + if !activeAlert.ResolvedAt.IsZero() { + importantAlertTimestampSet[activeAlert.ResolvedAt] = nil + } + }) + } + + var matrix promql.Matrix + for _, series := range seriesHashMap { + if series.Metric.Get(labels.MetricName) == alertForStateMetricName { + continue + } + p := 0 + for p < len(matrix) { + if matrix[p].Metric.Hash() >= series.Metric.Hash() { + break + } + p++ + } + matrix = append(matrix[:p], append(promql.Matrix{*series}, matrix[p:]...)...) + } + + matrix = common.DownsampleMatrix(matrix, maxSamples, false) + + var importantAlertTimestamps []time.Time + for ts := range importantAlertTimestampSet { + importantAlertTimestamps = append(importantAlertTimestamps, ts) + } + sort.Slice(importantAlertTimestamps, func(i, j int) bool { + return importantAlertTimestamps[i].Before(importantAlertTimestamps[j]) + }) + + activeAlertsByLabels := make(map[uint64][]*rules.Alert) + alertQueryFunc := provider.InstantQueryFunc(true) + alertingRule = rules.NewAlertingRule( + rule.Alert.Value, + expr, + time.Duration(rule.For), + labels.FromMap(rule.Labels), + labels.FromMap(rule.Annotations), + labels.Labels{}, + "", + true, + kitLog.NewNopLogger(), + ) + for _, ts := range importantAlertTimestamps { + _, errA := alertingRule.Eval( + ctx, + ts, + alertQueryFunc, + nil, + group.Limit) + if errA != nil { + return nil, nil, nil, nil, errors.Wrap(errA, "failed to evaluate rule %s at %d", rule.Expr.Value, ts) + } alertingRule.ForEachActiveAlert(func(activeAlert *rules.Alert) { aaHash := t.activeAlertHash(activeAlert) if existingAlerts, exists := activeAlertsByLabels[aaHash]; !exists { @@ -377,68 +502,17 @@ func (t *alertTester) evaluateAlertRule(ctx context.Context, queryable remote.Qu }) } - var matrix promql.Matrix - for _, series := range seriesHashMap { - if series.Metric.Get(labels.MetricName) == alertForStateMetricName { - continue - } - p := 0 - for p < len(matrix) { - if matrix[p].Metric.Hash() >= series.Metric.Hash() { - break - } - p++ - } - matrix = append(matrix[:p], append(promql.Matrix{*series}, matrix[p:]...)...) - } - - matrix = common.DownsampleMatrix(matrix, maxSamples, false) - - htmlSnippet, err := t.htmlSnippetWithoutLinks(alertingRule) - if err != nil { - return "", nil, nil, nil, err - } - var activeAlertList []rules.Alert - alertQueryFunc := provider.InstantQueryFunc(true) for _, activeAlertsByLabel := range activeAlertsByLabels { for _, activeAlert := range activeAlertsByLabel { - var ts time.Time - if activeAlert.State == rules.StatePending { - ts = activeAlert.ActiveAt - } else { - ts = activeAlert.FiredAt - } - alertingRule = rules.NewAlertingRule( - rule.Alert.Value, - expr, - time.Duration(rule.For), - labels.FromMap(rule.Labels), - labels.FromMap(rule.Annotations), - labels.Labels{}, - "", - true, - kitLog.NewNopLogger(), - ) - _, errA := alertingRule.Eval( - ctx, - ts, - alertQueryFunc, - nil, - group.Limit) - if errA != nil { - return "", nil, nil, nil, errors.Wrap(errA, "failed to evaluate rule %s at %d", rule.Expr.Value, ts) - } - alertingRule.ForEachActiveAlert(func(alert *rules.Alert) { - if alert.ActiveAt == ts && activeAlert.Labels.Hash() == alert.Labels.Hash() { - (*activeAlert).Annotations = (*alert).Annotations - } - }) activeAlertList = append(activeAlertList, *activeAlert) } } + sort.Slice(activeAlertList, func(i, j int) bool { + return activeAlertList[i].ActiveAt.Before(activeAlertList[j].ActiveAt) + }) - return htmlSnippet, + return t.toAlertDefinition(group, alertingRule, minTimestamp, maxTimestamp), &activeAlertList, &queryData{ Result: matrix, @@ -459,26 +533,32 @@ func (t *alertTester) activeAlertHash(alert *rules.Alert) uint64 { return xxhash.Sum64(buf) } -func (t *alertTester) htmlSnippetWithoutLinks(r *rules.AlertingRule) (string, error) { - lbls := make(map[string]string, len(r.Labels())) - for _, l := range r.Labels() { - lbls[l.Name] = l.Value +func (t *alertTester) toAlertDefinition(group rulefmt.RuleGroup, rule *rules.AlertingRule, startTime time.Time, endTime time.Time) *alertDefinition { + lbls := make([]map[string]string, len(rule.Labels())) + for i, l := range rule.Labels() { + lbls[i] = map[string]string{"name": l.Name, "value": l.Value} } - annotations := make(map[string]string, len(r.Annotations())) - for _, l := range r.Annotations() { - annotations[l.Name] = l.Value + annotations := make([]map[string]string, len(rule.Annotations())) + for i, a := range rule.Annotations() { + annotations[i] = map[string]string{"name": a.Name, "value": a.Value} } - ar := rulefmt.Rule{ - Alert: r.Name(), - Expr: r.Query().String(), - For: model.Duration(r.HoldDuration()), - Labels: lbls, - Annotations: annotations, + return &alertDefinition{ + Group: group.Name, + Name: rule.Name(), + Expr: rule.Query().String(), + ExprTableUrl: t.graphLinkForExpression(rule.Query().String(), startTime, endTime), + For: model.Duration(rule.HoldDuration()).String(), + Labels: lbls, + Annotations: annotations, } +} - var tpl bytes.Buffer - if err := t.alertHtmlSnippetTemplate.Execute(&tpl, ar); err != nil { - return "", errors.Wrap(err, "failed to execute alert html snippet template") - } - return tpl.String(), nil +func (t *alertTester) graphLinkForExpression(expr string, startTime time.Time, endTime time.Time) string { + escapedExpression := url.QueryEscape(expr) + return fmt.Sprintf("%s/graph?g0.expr=%s&g0.tab=0&g0.range_input=%s&g0.end_input=%s&g0.moment_input=%s", + t.config.Host.String(), + escapedExpression, + model.Duration(endTime.Sub(startTime)).String(), + htmlTemplate.HTMLEscapeString(startTime.Format("2006-01-02 15:04:05")), + htmlTemplate.HTMLEscapeString(endTime.Format("2006-01-02 15:04:05"))) } diff --git a/lib/web/server.go b/lib/web/server.go index 4fa633f..17c62e2 100644 --- a/lib/web/server.go +++ b/lib/web/server.go @@ -65,16 +65,18 @@ type server struct { config config.WebConfig httpServer http.Server routes []Route + queryable remote.Queryable } func (s *server) createServer() error { - queryable, err := remote.NewQueryable(s.config.Host) + queryable, err := remote.NewQueryable(s.config.Host, s.config.Parallelism) if err != nil { return err } options := s.newOptions() tmplExecutor := NewTemplateExecutor(options) router := s.createRouter(tmplExecutor, queryable) + s.queryable = queryable s.httpServer = http.Server{ Addr: s.config.ListenAddress.String(), Handler: router, @@ -108,6 +110,7 @@ func (s *server) Stop() { cancel() s.state = stopped }() + s.queryable.Close() if err := s.httpServer.Shutdown(ctx); err != nil { log.Panicf("Server shutdown failed:%s", err) } diff --git a/lib/web/ui/static/assets/css/alerts.css b/lib/web/ui/static/assets/css/alerts.css index a3f8378..f08c26e 100644 --- a/lib/web/ui/static/assets/css/alerts.css +++ b/lib/web/ui/static/assets/css/alerts.css @@ -26,4 +26,3 @@ div.show-annotations button { div.show-annotations.is-checked { color: #286090; } - diff --git a/lib/web/ui/static/assets/css/alertsTest.css b/lib/web/ui/static/assets/css/alertsTest.css index 501393f..486bb39 100644 --- a/lib/web/ui/static/assets/css/alertsTest.css +++ b/lib/web/ui/static/assets/css/alertsTest.css @@ -48,7 +48,16 @@ div.show-hide-all button { } #ruleTextArea { + height: 350px; min-width: 100px; + flex: 1 1 auto; +} + +#ruleTestInfo { + height: 100%; + background: white; + overflow: auto; + flex: 0 5 28.8%; } #dev_end,#inc_end,#dec_range,#inc_range,#range_input,#evaluate { @@ -64,7 +73,7 @@ div.show-hide-all button { } .prometheus_input_group .input { - width: 100px; + width: 70px; height: 30px; padding: 6px 12px; font-size: 14px; @@ -81,7 +90,7 @@ div.show-hide-all button { } .prometheus_input_group .date_input { - width: 200px; + width: 150px; } .alert_annotations_list { @@ -100,10 +109,145 @@ div.show-hide-all button { border-style: none !important; } -.alert_when_list_key { +.alert_value { + display: flex; + flex-direction: row; + justify-content: right; + gap: 5px; +} + +.alert_when_list { + display: flex; + flex-direction: row; + justify-content: space-between; + gap: 5px; +} + +.alert_when_list_key, .alert_value_key, .alert_range_info_key { font-weight: bold; } -.alert_when_list_key, .alert_when_list_value { +.alert_when_list_key, .alert_when_list_value, .alert_value_key, .alert_value_value, .alert_range_info_key, .alert_range_info_value { font-size: 75%; + white-space: nowrap; +} + +.rickshaw_annotation_timeline .annotation.active .content { + width: min-content; + border-width: 1px !important; + border: #000; + border-style: solid; +} + +.alert_popup { + display: flex; + flex-direction: column; + min-width: 250px; +} + +.alert_popup_header { + flex: 0 1 auto; + display: flex; + flex-direction: row; + align-items: center; + justify-content: space-between; + margin-bottom: 5px; } + +.alert_popup_name, .alert_popup_status { + flex: 0 1 auto; + margin-right: 10px; +} + +.alert_popup_value { + flex: 1 1 auto; +} + +.alert_popup_body { + flex: 1 1 auto; + display: flex; + flex-direction: row; + flex-wrap: wrap; +} + +.alert_popup_label { + margin: 2px; +} + +.rickshaw_annotation_timeline .annotation:hover .content { + display: block; + z-index: 50; + width: min-content; + border-width: 1px !important; + border: #000; + border-style: solid; +} + +.rickshaw_annotation_timeline .annotation.active, .rickshaw_annotation_timeline .annotation:hover { + cursor: pointer; +} + +.alert_range_info { + flex: 1 1 auto; + display: flex; + flex-direction: row; + justify-content: end; + margin-right: 40px; +} + +.alert_range_info_element { + display: flex; + flex-direction: row; + flex: 0 1 auto; +} + +.alert_range_info_key { + flex: 0 1 auto; + margin-left: 20px; + margin-right: 5px; +} + +.alert_range_info_value { + flex: 1 1 auto; +} + +#result_control_wrapper { + display: flex; + flex-direction: row; + flex-wrap: wrap-reverse; + align-items: center; + white-space: nowrap; + row-gap: 10px; + width: 100%; +} + +.result_controls { + flex: 9999 1 auto; +} + +.alert_definition { + display: block; + padding: 9.5px; + margin: 0 0 10px; + font-size: 13px; + line-height: 1.42857143; + color: #333; + background-color: #f5f5f5; + border: 1px solid #ccc; + border-radius: 4px; + font-family: Menlo,Monaco,Consolas,"Courier New",monospace; + overflow-x: scroll; + white-space: nowrap; +} + +.alert_definition_subelement { + margin-left: 1.5em; +} + +.alert_definition_group { + width: fit-content; +} + +.table-fixed { + table-layout: fixed; +} \ No newline at end of file diff --git a/lib/web/ui/static/assets/css/graph.css b/lib/web/ui/static/assets/css/graph.css index f3c37ca..7ac1235 100644 --- a/lib/web/ui/static/assets/css/graph.css +++ b/lib/web/ui/static/assets/css/graph.css @@ -68,7 +68,7 @@ div.page-options { .legend { display: inline-block; vertical-align: top; - margin: 0 0 0 60px; + margin: 10px 0 0 60px; } .graph_area { @@ -160,3 +160,12 @@ input[name="end_input"], input[name="range_input"] { background-color: #222222; border-radius: 0; } + +.slider { + margin-top: 10px; +} + +.slider, .timeline { + margin-left: 60px; + margin-right: 40px; +} diff --git a/lib/web/ui/static/assets/js/alert_testing/graph_template.handlebar b/lib/web/ui/static/assets/js/alert_testing/graph_template.handlebar index 05ec807..1e2cfaa 100644 --- a/lib/web/ui/static/assets/js/alert_testing/graph_template.handlebar +++ b/lib/web/ui/static/assets/js/alert_testing/graph_template.handlebar @@ -1,115 +1,138 @@
{{ruleName}} ({{activeAlertsLength}} active) | +{{definition.name}} ({{activeAlertsLength}} active) | |||||||||||||||||||||||||||||||||||||||||||||||||||
-
-
+
+
- name: {{definition.name}}
+ expr: {{definition.expr}}
+ {{#definition.labels.length}}
+
+ labels:
+ {{#definition.labels}}
+
+ {{/definition.labels.length}}
+ {{#definition.annotations.length}}
+ {{name}}: {{value}}
+ {{/definition.labels}}
+
+ annotations:
+ {{#definition.annotations}}
+
+ {{/definition.annotations.length}}
{{name}}: {{value}}
+ {{/definition.annotations}}
+
|