diff --git a/docs/README.md b/docs/README.md index 6e5e48e074..a6f6840f30 100644 --- a/docs/README.md +++ b/docs/README.md @@ -65,6 +65,7 @@ See each file for specific documentation about the exposed metrics: - [ClusterRole Metrics](clusterrole-metrics.md) - [ClusterRoleBinding Metrics](clusterrolebinding-metrics.md) +- [EndpointSlice Metrics](endpointslice-metrics.md) - [IngressClass Metrics](ingressclass-metrics.md) - [Role Metrics](role-metrics.md) - [RoleBinding Metrics](rolebinding-metrics.md) diff --git a/docs/endpointslice-metrics.md b/docs/endpointslice-metrics.md new file mode 100644 index 0000000000..f2327a5265 --- /dev/null +++ b/docs/endpointslice-metrics.md @@ -0,0 +1,10 @@ +# Endpoint Metrics + +| Metric name| Metric type | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------- | +| kube_endpointslice_annotations | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace>
`annotation_ENDPOINTSLICE_ANNOTATION`=<ENDPOINTSLICE_ANNOTATION> | EXPERIMENTAL | +| kube_endpointslice_info | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace> | EXPERIMENTAL | +| kube_endpointslice_ports | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace>
`port_name`=<endpointslice-port-name>
`port_protocol`=<endpointslice-port-protocol>
`port_number`=<endpointslice-port-number> | EXPERIMENTAL | +| kube_endpointslice_endpoints | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace>
`ready`=<endpointslice-ready>
`serving`=<endpointslice-serving>
`terminating`=<endpointslice-terminating>
`hostname`=<endpointslice-hostname>
`targetref_kind`=<endpointslice-targetref-kind>
`targetref_name`=<endpointslice-targetref-name>
`targetref_namespace`=<endpointslice-targetref-namespace>
`nodename`=<endpointslice-nodename>
`endpoint_zone`=<endpointslice-zone> | EXPERIMENTAL | +| kube_endpointslice_labels | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace>
`label_ENDPOINTSLICE_LABEL`=<ENDPOINTSLICE_LABEL> | EXPERIMENTAL | +| kube_endpointslice_created | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace> | EXPERIMENTAL | diff --git a/examples/autosharding/cluster-role.yaml b/examples/autosharding/cluster-role.yaml index ac5436f7f1..e20db08235 100644 --- a/examples/autosharding/cluster-role.yaml +++ b/examples/autosharding/cluster-role.yaml @@ -77,6 +77,13 @@ rules: verbs: - list - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch - apiGroups: - storage.k8s.io resources: diff --git a/examples/standard/cluster-role.yaml b/examples/standard/cluster-role.yaml index ac5436f7f1..e20db08235 100644 --- a/examples/standard/cluster-role.yaml +++ b/examples/standard/cluster-role.yaml @@ -77,6 +77,13 @@ rules: verbs: - list - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch - apiGroups: - storage.k8s.io resources: diff --git a/internal/store/builder.go b/internal/store/builder.go index df00ed496d..9f58933e6f 100644 --- a/internal/store/builder.go +++ b/internal/store/builder.go @@ -32,6 +32,7 @@ import ( certv1 "k8s.io/api/certificates/v1" coordinationv1 "k8s.io/api/coordination/v1" v1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" networkingv1 "k8s.io/api/networking/v1" policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -293,6 +294,7 @@ var availableStores = map[string]func(f *Builder) []cache.Store{ "daemonsets": func(b *Builder) []cache.Store { return b.buildDaemonSetStores() }, "deployments": func(b *Builder) []cache.Store { return b.buildDeploymentStores() }, "endpoints": func(b *Builder) []cache.Store { return b.buildEndpointsStores() }, + "endpointslices": func(b *Builder) []cache.Store { return b.buildEndpointSlicesStores() }, "horizontalpodautoscalers": func(b *Builder) []cache.Store { return b.buildHPAStores() }, "ingresses": func(b *Builder) []cache.Store { return b.buildIngressStores() }, "ingressclasses": func(b *Builder) []cache.Store { return b.buildIngressClassStores() }, @@ -355,6 +357,10 @@ func (b *Builder) buildEndpointsStores() []cache.Store { return b.buildStoresFunc(endpointMetricFamilies(b.allowAnnotationsList["endpoints"], b.allowLabelsList["endpoints"]), &v1.Endpoints{}, createEndpointsListWatch, b.useAPIServerCache) } +func (b *Builder) buildEndpointSlicesStores() []cache.Store { + return b.buildStoresFunc(endpointSliceMetricFamilies(b.allowAnnotationsList["endpointslices"], b.allowLabelsList["endpointslices"]), &discoveryv1.EndpointSlice{}, createEndpointSliceListWatch, b.useAPIServerCache) +} + func (b *Builder) buildHPAStores() []cache.Store { return b.buildStoresFunc(hpaMetricFamilies(b.allowAnnotationsList["horizontalpodautoscalers"], b.allowLabelsList["horizontalpodautoscalers"]), &autoscaling.HorizontalPodAutoscaler{}, createHPAListWatch, b.useAPIServerCache) } diff --git a/internal/store/endpointslice.go b/internal/store/endpointslice.go new file mode 100644 index 0000000000..0b01284468 --- /dev/null +++ b/internal/store/endpointslice.go @@ -0,0 +1,234 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + "strconv" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +var ( + descEndpointSliceAnnotationsName = "kube_endpointslice_annotations" + descEndpointSliceAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descEndpointSliceLabelsName = "kube_endpointslice_labels" + descEndpointSliceLabelsHelp = "Kubernetes labels converted to Prometheus labels." + descEndpointSliceLabelsDefaultLabels = []string{"endpointslice"} +) + +func endpointSliceMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_endpointslice_info", + "Information about endpointslice.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(s *discoveryv1.EndpointSlice) *metric.Family { + + m := metric.Metric{ + LabelKeys: []string{"addresstype"}, + LabelValues: []string{string(s.AddressType)}, + Value: 1, + } + return &metric.Family{Metrics: []*metric.Metric{&m}} + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_endpointslice_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(s *discoveryv1.EndpointSlice) *metric.Family { + ms := []*metric.Metric{} + if !s.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + Value: float64(s.CreationTimestamp.Unix()), + }) + } + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_endpointslice_endpoints", + "Endpoints attached to the endpointslice.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(e *discoveryv1.EndpointSlice) *metric.Family { + m := []*metric.Metric{} + for _, ep := range e.Endpoints { + var ( + labelKeys, + labelValues []string + ) + + if ep.Conditions.Ready != nil { + labelKeys = append(labelKeys, "ready") + labelValues = append(labelValues, strconv.FormatBool(*ep.Conditions.Ready)) + } + if ep.Conditions.Serving != nil { + labelKeys = append(labelKeys, "serving") + labelValues = append(labelValues, strconv.FormatBool(*ep.Conditions.Serving)) + } + if ep.Conditions.Terminating != nil { + labelKeys = append(labelKeys, "terminating") + labelValues = append(labelValues, strconv.FormatBool(*ep.Conditions.Terminating)) + } + + if ep.Hostname != nil { + labelKeys = append(labelKeys, "hostname") + labelValues = append(labelValues, *ep.Hostname) + } + + if ep.TargetRef != nil { + if ep.TargetRef.Kind != "" { + labelKeys = append(labelKeys, "targetref_kind") + labelValues = append(labelValues, ep.TargetRef.Kind) + } + if ep.TargetRef.Name != "" { + labelKeys = append(labelKeys, "targetref_name") + labelValues = append(labelValues, ep.TargetRef.Name) + } + if ep.TargetRef.Namespace != "" { + labelKeys = append(labelKeys, "targetref_namespace") + labelValues = append(labelValues, ep.TargetRef.Namespace) + } + } + + if ep.NodeName != nil { + labelKeys = append(labelKeys, "endpoint_nodename") + labelValues = append(labelValues, *ep.NodeName) + } + + if ep.Zone != nil { + labelKeys = append(labelKeys, "endpoint_zone") + labelValues = append(labelValues, *ep.Zone) + } + labelKeys = append(labelKeys, "address") + for _, address := range ep.Addresses { + newlabelValues := make([]string, len(labelValues)) + copy(newlabelValues, labelValues) + m = append(m, &metric.Metric{ + LabelKeys: labelKeys, + LabelValues: append(newlabelValues, address), + Value: 1, + }) + } + } + return &metric.Family{ + Metrics: m, + } + }), + ), + + *generator.NewFamilyGeneratorWithStability( + "kube_endpointslice_ports", + "Ports attached to the endpointslice.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(e *discoveryv1.EndpointSlice) *metric.Family { + m := []*metric.Metric{} + for _, port := range e.Ports { + m = append(m, &metric.Metric{ + LabelValues: []string{*port.Name, string(*port.Protocol), strconv.FormatInt(int64(*port.Port), 10)}, + LabelKeys: []string{"port_name", "port_protocol", "port_number"}, + Value: 1, + }) + } + return &metric.Family{ + Metrics: m, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descEndpointSliceAnnotationsName, + descEndpointSliceAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(s *discoveryv1.EndpointSlice) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", s.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descEndpointSliceLabelsName, + descEndpointSliceLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(s *discoveryv1.EndpointSlice) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", s.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + } +} + +func wrapEndpointSliceFunc(f func(*discoveryv1.EndpointSlice) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + endpointSlice := obj.(*discoveryv1.EndpointSlice) + + metricFamily := f(endpointSlice) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descEndpointSliceLabelsDefaultLabels, []string{endpointSlice.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} + +func createEndpointSliceListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + return kubeClient.DiscoveryV1().EndpointSlices(ns).List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + return kubeClient.DiscoveryV1().EndpointSlices(ns).Watch(context.TODO(), opts) + }, + } +} diff --git a/internal/store/endpointslice_test.go b/internal/store/endpointslice_test.go new file mode 100644 index 0000000000..cca8638cd9 --- /dev/null +++ b/internal/store/endpointslice_test.go @@ -0,0 +1,161 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestEndpointSliceStore(t *testing.T) { + startTime := 1501569018 + metav1StartTime := metav1.Unix(int64(startTime), 0) + portname := "http" + portnumber := int32(80) + portprotocol := corev1.Protocol("TCP") + nodename := "node" + hostname := "host" + zone := "west" + ready := true + terminating := false + addresses := []string{"10.0.0.1", "192.168.1.10"} + + cases := []generateMetricsTestCase{ + { + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_endpointslice-info", + }, + AddressType: "IPv4", + }, + Want: ` + # HELP kube_endpointslice_info Information about endpointslice. + # TYPE kube_endpointslice_info gauge + kube_endpointslice_info{endpointslice="test_endpointslice-info",addresstype="IPv4"} 1 + `, + MetricNames: []string{ + "kube_endpointslice_info", + }, + }, + { + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_kube_endpointslice-created", + CreationTimestamp: metav1StartTime, + }, + AddressType: "IPv4", + }, + Want: ` + # HELP kube_endpointslice_created Unix creation timestamp + # TYPE kube_endpointslice_created gauge + kube_endpointslice_created{endpointslice="test_kube_endpointslice-created"} 1.501569018e+09 + `, + MetricNames: []string{ + "kube_endpointslice_created", + }, + }, + { + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_endpointslice-ports", + }, + AddressType: "IPv4", + Ports: []discoveryv1.EndpointPort{ + {Name: &portname, + Port: &portnumber, + Protocol: &portprotocol, + }, + }, + }, + Want: ` + # HELP kube_endpointslice_ports Ports attached to the endpointslice. + # TYPE kube_endpointslice_ports gauge + kube_endpointslice_ports{endpointslice="test_endpointslice-ports",port_name="http",port_protocol="TCP",port_number="80"} 1 + `, + MetricNames: []string{ + "kube_endpointslice_ports", + }, + }, + { + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_endpointslice-endpoints", + }, + AddressType: "IPv4", + Endpoints: []discoveryv1.Endpoint{ + { + NodeName: &nodename, + Conditions: discoveryv1.EndpointConditions{ + Ready: &ready, + Terminating: &terminating, + }, + Hostname: &hostname, + Zone: &zone, + Addresses: addresses, + }, + }, + }, + Want: ` + # HELP kube_endpointslice_endpoints Endpoints attached to the endpointslice. + # TYPE kube_endpointslice_endpoints gauge + kube_endpointslice_endpoints{address="10.0.0.1",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="host",ready="true",terminating="false"} 1 + kube_endpointslice_endpoints{address="192.168.1.10",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="host",ready="true",terminating="false"} 1 + `, + + MetricNames: []string{ + "kube_endpointslice_endpoints", + }, + }, + { + AllowAnnotationsList: []string{ + "foo", + }, + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_endpointslice-labels", + Annotations: map[string]string{ + "foo": "baz", + }, + Labels: map[string]string{ + "foo": "bar", + }, + }, + AddressType: "IPv4", + }, + Want: ` + # HELP kube_endpointslice_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_endpointslice_labels Kubernetes labels converted to Prometheus labels. + # TYPE kube_endpointslice_annotations gauge + # TYPE kube_endpointslice_labels gauge + kube_endpointslice_annotations{endpointslice="test_endpointslice-labels",annotation_foo="baz"} 1 + kube_endpointslice_labels{endpointslice="test_endpointslice-labels"} 1 + `, + MetricNames: []string{ + "kube_endpointslice_annotations", "kube_endpointslice_labels", + }, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(endpointSliceMetricFamilies(c.AllowAnnotationsList, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(endpointSliceMetricFamilies(c.AllowAnnotationsList, nil)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} diff --git a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet index 2275c7b584..d562d02c3b 100644 --- a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet +++ b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet @@ -114,6 +114,13 @@ ], verbs: ['list', 'watch'], }, + { + apiGroups: ['discovery.k8s.io'], + resources: [ + 'endpointslices', + ], + verbs: ['list', 'watch'], + }, { apiGroups: ['storage.k8s.io'], resources: [ diff --git a/tests/e2e/main_test.go b/tests/e2e/main_test.go index 4f9d08e86e..d0880a3261 100644 --- a/tests/e2e/main_test.go +++ b/tests/e2e/main_test.go @@ -256,6 +256,7 @@ func TestDefaultCollectorMetricsAvailable(t *testing.T) { nonDefaultResources := map[string]bool{ "clusterrole": true, "clusterrolebinding": true, + "endpointslice": true, "ingressclass": true, "role": true, "rolebinding": true, diff --git a/tests/manifests/endpointslice.yaml b/tests/manifests/endpointslice.yaml new file mode 100644 index 0000000000..078e67f2bb --- /dev/null +++ b/tests/manifests/endpointslice.yaml @@ -0,0 +1,13 @@ +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice +metadata: + name: example-endpointslice + namespace: default +addressType: IPv4 +endpoints: +- addresses: + - 10.0.0.2 + conditions: + ready: true + serving: true + terminating: false