From a0980c87d1f2c5603afff3f5cd5116bd26a2ad5a Mon Sep 17 00:00:00 2001 From: Julian van den Berkmortel Date: Tue, 28 Jun 2022 23:31:49 +0200 Subject: [PATCH] Add metric for pod tolerations (Ref: #1744) --- docs/pod-metrics.md | 1 + internal/store/pod.go | 57 ++++++++++++++++++++++++++++++++++++++ internal/store/pod_test.go | 43 +++++++++++++++++++++++++++- pkg/app/server_test.go | 2 ++ 4 files changed, 102 insertions(+), 1 deletion(-) diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index 67b79e80ef..06a545920a 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -47,6 +47,7 @@ | kube_pod_status_reason | Gauge | The pod status reasons | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<Evicted\|NodeAffinity\|NodeLost\|Shutdown\|UnexpectedAdmissionError>
`uid`=<pod-uid> | EXPERIMENTAL | - | | kube_pod_status_scheduled_time | Gauge | Unix timestamp when pod moved into scheduled status | seconds |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | | kube_pod_status_unschedulable | Gauge | Describes the unschedulable status for the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_tolerations | Gauge | Information about the pod tolerations | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid>
`key`=<toleration-key>
`operator`=<toleration-operator>
`value`=<toleration-value>
`effect`=<toleration-effect> `toleration_seconds`=<toleration-seconds> | EXPERIMENTAL | - | ## Useful metrics queries diff --git a/internal/store/pod.go b/internal/store/pod.go index 5cba91fb32..e593c67815 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -85,6 +85,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat createPodStatusScheduledFamilyGenerator(), createPodStatusScheduledTimeFamilyGenerator(), createPodStatusUnschedulableFamilyGenerator(), + createPodTolerationsFamilyGenerator(), createPodNodeSelectorsFamilyGenerator(), } } @@ -1403,6 +1404,62 @@ func createPodStatusUnschedulableFamilyGenerator() generator.FamilyGenerator { ) } +func createPodTolerationsFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGenerator( + "kube_pod_tolerations", + "Information about the pod tolerations", + metric.Gauge, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + var ms []*metric.Metric + + for _, t := range p.Spec.Tolerations { + var labelKeys []string + var labelValues []string + + if t.Key != "" { + labelKeys = append(labelKeys, "key") + labelValues = append(labelValues, t.Key) + } + + if t.Operator != "" { + labelKeys = append(labelKeys, "operator") + labelValues = append(labelValues, string(t.Operator)) + } + + if t.Value != "" { + labelKeys = append(labelKeys, "value") + labelValues = append(labelValues, t.Value) + } + + if t.Effect != "" { + labelKeys = append(labelKeys, "effect") + labelValues = append(labelValues, string(t.Effect)) + } + + if t.TolerationSeconds != nil { + labelKeys = append(labelKeys, "toleration_seconds") + labelValues = append(labelValues, strconv.FormatInt(*t.TolerationSeconds, 10)) + } + + if len(labelKeys) == 0 { + continue + } + + ms = append(ms, &metric.Metric{ + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + func createPodNodeSelectorsFamilyGenerator() generator.FamilyGenerator { return *generator.NewOptInFamilyGenerator( "kube_pod_nodeselectors", diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index bd1531fbee..9e58232f2d 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -1926,6 +1926,47 @@ func TestPodStore(t *testing.T) { "kube_pod_annotations", }, }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + }, + Spec: v1.PodSpec{ + Tolerations: []v1.Toleration{ + { + Key: "key1", + Operator: v1.TolerationOpEqual, + Value: "value1", + Effect: v1.TaintEffectNoSchedule, + }, + { + Key: "key2", + Operator: v1.TolerationOpExists, + }, + { + Key: "key3", + Operator: v1.TolerationOpEqual, + Value: "value3", + }, + { + // an empty toleration to ensure that an empty toleration does not result in a metric + }, + }, + }, + }, + Want: ` + # HELP kube_pod_tolerations Information about the pod tolerations + # TYPE kube_pod_tolerations gauge + kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key1",operator="Equal",value="value1",effect="NoSchedule"} 1 + kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key2",operator="Exists"} 1 + kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key3",operator="Equal",value="value3"} 1 + `, + MetricNames: []string{ + "kube_pod_tolerations", + }, + }, } for i, c := range cases { @@ -2018,7 +2059,7 @@ func BenchmarkPodStore(b *testing.B) { }, } - expectedFamilies := 45 + expectedFamilies := 46 for n := 0; n < b.N; n++ { families := f(pod) if len(families) != expectedFamilies { diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go index 2bf2550b5f..71d77861d7 100644 --- a/pkg/app/server_test.go +++ b/pkg/app/server_test.go @@ -234,6 +234,7 @@ func TestFullScrapeCycle(t *testing.T) { # HELP kube_pod_status_scheduled Describes the status of the scheduling process for the pod. # HELP kube_pod_status_scheduled_time Unix timestamp when pod moved into scheduled status # HELP kube_pod_status_unschedulable Describes the unschedulable status for the pod. +# HELP kube_pod_tolerations Information about the pod tolerations # TYPE kube_pod_annotations gauge # TYPE kube_pod_completion_time gauge # TYPE kube_pod_container_info gauge @@ -278,6 +279,7 @@ func TestFullScrapeCycle(t *testing.T) { # TYPE kube_pod_status_scheduled gauge # TYPE kube_pod_status_scheduled_time gauge # TYPE kube_pod_status_unschedulable gauge +# TYPE kube_pod_tolerations gauge kube_pod_annotations{namespace="default",pod="pod0",uid="abc-0"} 1 kube_pod_container_info{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",image_spec="k8s.gcr.io/hyperkube2_spec",image="k8s.gcr.io/hyperkube2",image_id="docker://sha256:bbb",container_id="docker://cd456"} 1 kube_pod_container_info{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",image_spec="k8s.gcr.io/hyperkube3_spec",image="k8s.gcr.io/hyperkube3",image_id="docker://sha256:ccc",container_id="docker://ef789"} 1