From 3cda2e57679e92dedc0feb9f028a80839ac201bb Mon Sep 17 00:00:00 2001
From: Danil Tarasov <87192879+almostinf@users.noreply.github.com>
Date: Thu, 30 May 2024 10:58:53 +0300
Subject: [PATCH] perf(checker): add fetching metrics values in pipe (#1029)

---
 database/redis/metric.go      |  38 +++++++++---
 database/redis/metric_test.go | 106 ++++++++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+), 8 deletions(-)

diff --git a/database/redis/metric.go b/database/redis/metric.go
index 0e12806a6..dee87307c 100644
--- a/database/redis/metric.go
+++ b/database/redis/metric.go
@@ -50,25 +50,47 @@ func (connector *DbConnector) GetPatterns() ([]string, error) {
 // GetMetricsValues gets metrics values for given interval.
 func (connector *DbConnector) GetMetricsValues(metrics []string, from int64, until int64) (map[string][]*moira.MetricValue, error) {
 	c := *connector.client
-	resultByMetrics := make([]*redis.ZSliceCmd, 0, len(metrics))
+	ctx := connector.context
+
+	pipe := c.TxPipeline()
 
 	for _, metric := range metrics {
-		rng := &redis.ZRangeBy{Min: strconv.FormatInt(from, 10), Max: strconv.FormatInt(until, 10)}
-		result := c.ZRangeByScoreWithScores(connector.context, metricDataKey(metric), rng)
-		resultByMetrics = append(resultByMetrics, result)
+		rng := &redis.ZRangeBy{
+			Min: strconv.FormatInt(from, 10),
+			Max: strconv.FormatInt(until, 10),
+		}
+		pipe.ZRangeByScoreWithScores(connector.context, metricDataKey(metric), rng)
 	}
 
-	res := make(map[string][]*moira.MetricValue, len(resultByMetrics))
+	cmds, err := pipe.Exec(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to Exec in get metrics values: %w", err)
+	}
+
+	resultByMetrics := make([]*redis.ZSliceCmd, 0, len(metrics))
+
+	for _, cmd := range cmds {
+		res, ok := cmd.(*redis.ZSliceCmd)
+		if !ok {
+			return nil, fmt.Errorf("failed to convert cmd response to *ZSliceCmd in get metrics values")
+		}
+
+		resultByMetrics = append(resultByMetrics, res)
+	}
+
+	result := make(map[string][]*moira.MetricValue, len(resultByMetrics))
 
 	for i, resultByMetric := range resultByMetrics {
 		metric := metrics[i]
 		metricsValues, err := reply.MetricValues(resultByMetric)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to convert ZSliceCmd to metric values in get metrics values: %w", err)
 		}
-		res[metric] = metricsValues
+
+		result[metric] = metricsValues
 	}
-	return res, nil
+
+	return result, nil
 }
 
 // GetMetricRetention gets given metric retention, if retention is empty then return default retention value(60).
diff --git a/database/redis/metric_test.go b/database/redis/metric_test.go
index a6e96e1cb..64a2fa6d5 100644
--- a/database/redis/metric_test.go
+++ b/database/redis/metric_test.go
@@ -3,6 +3,7 @@ package redis
 import (
 	"errors"
 	"fmt"
+	"math/rand"
 	"strconv"
 	"testing"
 	"time"
@@ -22,6 +23,111 @@ const (
 	fromInf = "-inf"
 )
 
+func saveMetricsWithRandomValues(
+	connector *DbConnector,
+	metricNames []string,
+	metricPattern string,
+	pointNum int,
+) error {
+	for i := 0; i < pointNum; i++ {
+		matchedMetrics := make(map[string]*moira.MatchedMetric, len(metricNames))
+
+		for _, metricName := range metricNames {
+			matchedMetric := &moira.MatchedMetric{
+				Metric:             metricName,
+				Patterns:           []string{metricPattern},
+				Value:              rand.Float64(),
+				Timestamp:          int64(i),
+				RetentionTimestamp: int64(i),
+				Retention:          rand.Int(),
+			}
+
+			matchedMetrics[metricName] = matchedMetric
+		}
+
+		if err := connector.SaveMetrics(matchedMetrics); err != nil {
+			return fmt.Errorf("failed to save matched metrics: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// randStringBytesMaskImpr function, which is needed for fast generation of random strings.
+func randStringBytesMaskImpr(n, letterIdxBits, letterIdxMax int, letterIdxMask int64, letterBytes []byte) string {
+	b := make([]byte, n)
+	// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
+	for i, cache, remain := n-1, rand.Int63(), letterIdxMax; i >= 0; {
+		if remain == 0 {
+			cache, remain = rand.Int63(), letterIdxMax
+		}
+		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
+			b[i] = letterBytes[idx]
+			i--
+		}
+		cache >>= letterIdxBits
+		remain--
+	}
+
+	return string(b)
+}
+
+func BenchmarkGetMetricsValues(b *testing.B) {
+	const (
+		metricNameLen = 20
+		metricPattern = "random-pattern"
+		letterBytes   = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+		letterIdxBits = 6                    // 6 bits to represent a letter index
+		letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
+		letterIdxMax  = 63 / letterIdxBits   // number of letter indices fitting in 63 bits
+		pointsNum     = 10
+	)
+
+	testcases := map[string]struct {
+		pointsNum  int
+		metricsNum int
+	}{
+		"100-metrics": {
+			pointsNum:  pointsNum,
+			metricsNum: 100,
+		},
+		"1000-metrics": {
+			pointsNum:  pointsNum,
+			metricsNum: 1000,
+		},
+		"1500-metrics": {
+			pointsNum:  pointsNum,
+			metricsNum: 1500,
+		},
+	}
+
+	logger, _ := logging.GetLogger("database")
+	database := NewTestDatabase(logger)
+	database.Flush()
+
+	for testname, testcase := range testcases {
+		b.Run(testname, func(b *testing.B) {
+			randomMetrics := make([]string, 0, testcase.metricsNum)
+			for i := 0; i < testcase.metricsNum; i++ {
+				randomMetrics = append(randomMetrics, randStringBytesMaskImpr(metricNameLen, letterIdxBits, letterIdxMax, letterIdxMask, []byte(letterBytes)))
+			}
+
+			database.Flush()
+			if err := saveMetricsWithRandomValues(database, randomMetrics, metricPattern, testcase.pointsNum); err != nil {
+				b.Fatalf("failed to save metrics with random values: %s", err.Error())
+			}
+
+			b.ResetTimer()
+
+			for i := 0; i < b.N; i++ {
+				if _, err := database.GetMetricsValues(randomMetrics, 0, int64(testcase.pointsNum)); err != nil {
+					b.Fatalf("failed to get metrics values: %s", err.Error())
+				}
+			}
+		})
+	}
+}
+
 func TestMetricsStoring(t *testing.T) {
 	logger, _ := logging.GetLogger("dataBase")
 	dataBase := NewTestDatabase(logger)