add metrics generation behavior config

hasura · Sep 26, 2024 · c8babc9 · c8babc9
1 parent fe139f0
commit c8babc9
Show file tree

Hide file tree

Showing 12 changed files with 6,606 additions and 86 deletions.
diff --git a/configuration/update.go b/configuration/update.go
@@ -39,7 +39,7 @@ type updateCommand struct {
 
 func introspectSchema(ctx context.Context, args *UpdateArguments) error {
 	start := time.Now()
-	slog.Info("introspect metrics metadata...", slog.String("dir", args.Dir))
+	slog.Info("introspecting metadata", slog.String("dir", args.Dir))
 	originalConfig, err := metadata.ReadConfiguration(args.Dir)
 	if err != nil {
 		if !os.IsNotExist(err) {
@@ -62,6 +62,11 @@ func introspectSchema(ctx context.Context, args *UpdateArguments) error {
 	}
 
 	if originalConfig.Generator.Metrics.Enabled {
+		slog.Info("introspecting metrics",
+			slog.String("behavior", string(originalConfig.Generator.Metrics.Behavior)),
+			slog.Any("include", originalConfig.Generator.Metrics.Include),
+			slog.Any("exclude", originalConfig.Generator.Metrics.Exclude),
+		)
 		for _, el := range originalConfig.Generator.Metrics.ExcludeLabels {
 			if len(el.Labels) == 0 {
 				continue
@@ -97,6 +102,15 @@ func (uc *updateCommand) updateMetricsMetadata(ctx context.Context) error {
 	}
 
 	newMetrics := map[string]metadata.MetricInfo{}
+	if uc.Config.Generator.Metrics.Behavior == metadata.MetricsGenerationMerge {
+		for key, metric := range uc.Config.Metadata.Metrics {
+			if (len(uc.Include) > 0 && !validateRegularExpressions(uc.Include, key)) || validateRegularExpressions(uc.Exclude, key) {
+				continue
+			}
+			newMetrics[key] = metric
+		}
+	}
+
 	for key, info := range metricsInfo {
 		if len(info) == 0 {
 			continue
@@ -117,7 +131,6 @@ func (uc *updateCommand) updateMetricsMetadata(ctx context.Context) error {
 			Labels:      labels,
 		}
 	}
-
 	uc.Config.Metadata.Metrics = newMetrics
 	return nil
 }
@@ -185,7 +198,7 @@ func (uc *updateCommand) writeConfigFile() error {
 	var buf bytes.Buffer
 	writer := bufio.NewWriter(&buf)
 
-	_, _ = writer.WriteString("# yaml-language-server: $schema=../../jsonschema/configuration.json\n")
+	_, _ = writer.WriteString("# yaml-language-server: $schema=https://raw.githubusercontent.com/hasura/ndc-prometheus/main/jsonschema/configuration.json\n")
 	encoder := yaml.NewEncoder(writer)
 	encoder.SetIndent(2)
 	if err := encoder.Encode(uc.Config); err != nil {
@@ -203,6 +216,7 @@ var defaultConfiguration = metadata.Configuration{
 	Generator: metadata.GeneratorSettings{
 		Metrics: metadata.MetricsGeneratorSettings{
 			Enabled:       true,
+			Behavior:      metadata.MetricsGenerationMerge,
 			Include:       []string{},
 			Exclude:       []string{},
 			ExcludeLabels: []metadata.ExcludeLabelsSetting{},

diff --git a/connector-definition/configuration.yaml b/connector-definition/configuration.yaml
@@ -5,6 +5,7 @@ connection_settings:
 generator:
   metrics:
     enabled: true
+    behavior: merge
     include: []
     exclude: []
     exclude_labels: []

diff --git a/connector/metadata/configuration.go b/connector/metadata/configuration.go
@@ -16,9 +16,19 @@ type Configuration struct {
 	Metadata           Metadata              `json:"metadata" yaml:"metadata"`
 }
 
+// MetricsGenerationBehavior the behavior of metrics generation
+type MetricsGenerationBehavior string
+
+const (
+	MetricsGenerationMerge   = "merge"
+	MetricsGenerationReplace = "replace"
+)
+
 // MetricsGeneratorSettings contain settings for the metrics generation
 type MetricsGeneratorSettings struct {
-	Enabled bool `json:"enabled" yaml:"enabled"`
+	// Enable the metrics generation
+	Enabled  bool                      `json:"enabled" yaml:"enabled"`
+	Behavior MetricsGenerationBehavior `json:"behavior" yaml:"behavior" jsonschema:"enum=merge,enum=replace"`
 	// Include metrics with regular expression matching. Include all metrics by default
 	Include []string `json:"include" yaml:"include"`
 	// Exclude metrics with regular expression matching.

diff --git a/jsonschema/configuration.json b/jsonschema/configuration.json
@@ -256,6 +256,13 @@
         "enabled": {
           "type": "boolean"
         },
+        "behavior": {
+          "type": "string",
+          "enum": [
+            "merge",
+            "replace"
+          ]
+        },
         "include": {
           "items": {
             "type": "string"
@@ -283,6 +290,7 @@
       "type": "object",
       "required": [
         "enabled",
+        "behavior",
         "include",
         "exclude",
         "exclude_labels",

diff --git a/tests/configuration/configuration.yaml b/tests/configuration/configuration.yaml
@@ -11,6 +11,7 @@ connection_settings:
 generator:
   metrics:
     enabled: true
+    behavior: merge
     include: []
     exclude:
       - ^prometheus_+
@@ -23,6 +24,20 @@ generator:
     start_at: 2024-09-01T00:00:00Z
 metadata:
   metrics:
+    ndc_prometheus_query_total:
+      type: counter
+      description: Total number of query requests
+      labels:
+        collection: {}
+        http_status: {}
+        instance: {}
+        job: {}
+        otel_scope_name: {}
+        status: {}
+    ndc_prometheus_query_total_time:
+      type: histogram
+      description: Total time taken to plan and execute a query, in seconds
+      labels: {}
     net_conntrack_dialer_conn_attempted_total:
       type: counter
       description: Total number of connections attempted by the given dialer a given name.
@@ -48,6 +63,13 @@ metadata:
         instance: {}
         job: {}
         reason: {}
+    otel_scope_info:
+      type: gauge
+      description: Instrumentation Scope metadata
+      labels:
+        instance: {}
+        job: {}
+        otel_scope_name: {}
     process_cpu_seconds_total:
       type: counter
       description: Total user and system CPU time spent in seconds.