From f7af83cae333541bde3baf10a80bccf033821f3d Mon Sep 17 00:00:00 2001 From: Pawel Szkamruk Date: Tue, 1 Oct 2024 13:46:07 +0200 Subject: [PATCH 1/6] k8s attributes procesor for cluster receiver to ingest events into index from annotation --- ...ver_events_into_index_from_annotation.yaml | 12 ++++++++ docs/advanced-configuration.md | 1 + .../configmap-cluster-receiver.yaml | 1 + .../v1.29/controller_manager_metrics.yaml | 2 +- .../expected/v1.29/coredns_metrics.yaml | 2 +- .../templates/config/_common.tpl | 30 +++++++++++++++++++ .../_otel-k8s-cluster-receiver-config.tpl | 7 +++++ 7 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 .chloggen/ingest_cluster_receiver_events_into_index_from_annotation.yaml diff --git a/.chloggen/ingest_cluster_receiver_events_into_index_from_annotation.yaml b/.chloggen/ingest_cluster_receiver_events_into_index_from_annotation.yaml new file mode 100644 index 0000000000..b256805161 --- /dev/null +++ b/.chloggen/ingest_cluster_receiver_events_into_index_from_annotation.yaml @@ -0,0 +1,12 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: 'enhancement' +# The name of the component, or a single word describing the area of concern, (e.g. agent, clusterReceiver, gateway, operator, chart, other) +component: clusterReceiver +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: configure k8s attributes processor for cluster receiver to ingest events into index defined in namespace annotation +# One or more tracking issues related to the change +issues: [1481] +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/docs/advanced-configuration.md b/docs/advanced-configuration.md index 1414f28efb..ae30e35700 100644 --- a/docs/advanced-configuration.md +++ b/docs/advanced-configuration.md @@ -559,6 +559,7 @@ Manage Splunk OTel Collector Logging with these supported annotations. * Use `splunk.com/index` annotation on pod and/or namespace to tell which Splunk platform indexes to ingest to. Pod annotation will take precedence over namespace annotation when both are annotated. For example, the following command will make logs from `kube-system` namespace to be sent to `k8s_events` index: `kubectl annotate namespace kube-system splunk.com/index=k8s_events` + **Please Note:** Cluster receiver supports only namespace index annotations, pod index annotations are not supported. * Use `splunk.com/metricsIndex` annotation on pod and/or namespace to tell which Splunk platform metric indexes to ingest to. Pod annotation will take precedence over namespace annotation when both are annotated. * Filter logs using pod and/or namespace annotation * If `logsCollection.containers.useSplunkIncludeAnnotation` is `false` (default: false), set `splunk.com/exclude` annotation to `true` on pod and/or namespace to exclude its logs from ingested. diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml index 70f92e9f92..507378eb1a 100644 --- a/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml +++ b/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml @@ -130,6 +130,7 @@ data: - resourcedetection - resource - transform/add_sourcetype + - k8sattributes/clusterReceiver receivers: - k8sobjects metrics: diff --git a/functional_tests/testdata_histogram/expected/v1.29/controller_manager_metrics.yaml b/functional_tests/testdata_histogram/expected/v1.29/controller_manager_metrics.yaml index cf64577368..5504487d8b 100644 --- a/functional_tests/testdata_histogram/expected/v1.29/controller_manager_metrics.yaml +++ b/functional_tests/testdata_histogram/expected/v1.29/controller_manager_metrics.yaml @@ -4883,4 +4883,4 @@ resourceMetrics: name: node_ipam_controller_cidrset_allocation_tries_per_request scope: name: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver - version: v0.110.0 + version: v0.109.0 diff --git a/functional_tests/testdata_histogram/expected/v1.29/coredns_metrics.yaml b/functional_tests/testdata_histogram/expected/v1.29/coredns_metrics.yaml index a379eb94dd..82b43fc357 100644 --- a/functional_tests/testdata_histogram/expected/v1.29/coredns_metrics.yaml +++ b/functional_tests/testdata_histogram/expected/v1.29/coredns_metrics.yaml @@ -388,4 +388,4 @@ resourceMetrics: name: coredns_kubernetes_rest_client_rate_limiter_duration_seconds scope: name: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver - version: v0.110.0 + version: v0.109.0 diff --git a/helm-charts/splunk-otel-collector/templates/config/_common.tpl b/helm-charts/splunk-otel-collector/templates/config/_common.tpl index a3baf5a894..782469f7cf 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_common.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_common.tpl @@ -211,6 +211,36 @@ k8sattributes: {{- include "splunk-otel-collector.addExtraLabels" . | nindent 6 }} {{- end }} {{- end }} +{{- define "splunk-otel-collector.k8sClusterReceiverAttributesProcessor" -}} +k8sattributes/clusterReceiver: + pod_association: + - sources: + - from: resource_attribute + name: k8s.namespace.name + - sources: + - from: resource_attribute + name: k8s.node.name + extract: + metadata: + - k8s.namespace.name + - k8s.node.name + - k8s.pod.name + - k8s.pod.uid + - container.id + - container.image.name + - container.image.tag + {{- if eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true"}} + annotations: + - key: splunk.com/sourcetype + from: pod + - key: splunk.com/index + tag_name: com.splunk.index + from: namespace + - key: splunk.com/index + tag_name: com.splunk.index + from: pod + {{- end}} +{{- end }} {{/* Common config for K8s attributes processor adding k8s metadata to metrics resource attributes. diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl index ede63ac5e6..4ea5548675 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl @@ -117,6 +117,10 @@ processors: - set(resource.attributes["com.splunk.sourcetype"], Concat(["kube:object:", attributes["k8s.resource.name"]], "")) {{- end }} + {{- if and $clusterReceiver.eventsEnabled (eq (include "splunk-otel-collector.logsEnabled" .) "true") }} + {{- include "splunk-otel-collector.k8sClusterReceiverAttributesProcessor" . | nindent 2 }} + {{- end }} + # Resource attributes specific to the collector itself. resource/add_collector_k8s: attributes: @@ -290,6 +294,7 @@ service: {{- if .Values.environment }} - resource/add_environment {{- end }} + - k8sattributes/clusterReceiver exporters: {{- if (eq (include "splunk-otel-collector.o11yLogsEnabled" .) "true") }} - splunk_hec/o11y @@ -312,6 +317,7 @@ service: {{- if .Values.environment }} - resource/add_environment {{- end }} + - k8sattributes/clusterReceiver exporters: {{- if (eq (include "splunk-otel-collector.o11yLogsEnabled" .) "true") }} - splunk_hec/o11y @@ -333,6 +339,7 @@ service: {{- if .Values.clusterName }} - resource/add_event_k8s {{- end }} + - k8sattributes/clusterReceiver exporters: - signalfx {{- end }} From c586d3bbef342599a13f51676ae94d199e1ec926 Mon Sep 17 00:00:00 2001 From: Pawel Szkamruk Date: Thu, 21 Nov 2024 21:23:28 +0100 Subject: [PATCH 2/6] update to the version from main --- .../expected/v1.29/controller_manager_metrics.yaml | 2 +- .../testdata_histogram/expected/v1.29/coredns_metrics.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/functional_tests/testdata_histogram/expected/v1.29/controller_manager_metrics.yaml b/functional_tests/testdata_histogram/expected/v1.29/controller_manager_metrics.yaml index 5504487d8b..cf64577368 100644 --- a/functional_tests/testdata_histogram/expected/v1.29/controller_manager_metrics.yaml +++ b/functional_tests/testdata_histogram/expected/v1.29/controller_manager_metrics.yaml @@ -4883,4 +4883,4 @@ resourceMetrics: name: node_ipam_controller_cidrset_allocation_tries_per_request scope: name: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver - version: v0.109.0 + version: v0.110.0 diff --git a/functional_tests/testdata_histogram/expected/v1.29/coredns_metrics.yaml b/functional_tests/testdata_histogram/expected/v1.29/coredns_metrics.yaml index 82b43fc357..a379eb94dd 100644 --- a/functional_tests/testdata_histogram/expected/v1.29/coredns_metrics.yaml +++ b/functional_tests/testdata_histogram/expected/v1.29/coredns_metrics.yaml @@ -388,4 +388,4 @@ resourceMetrics: name: coredns_kubernetes_rest_client_rate_limiter_duration_seconds scope: name: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver - version: v0.109.0 + version: v0.110.0 From 7567c6c2a1c57281e9ed9e1f149c4ed642e4b1d2 Mon Sep 17 00:00:00 2001 From: Pawel Szkamruk Date: Thu, 21 Nov 2024 21:27:51 +0100 Subject: [PATCH 3/6] pre-commit --- .../rendered_manifests/deployment-cluster-receiver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml index 27dce4bb8a..c7cf41029a 100644 --- a/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml +++ b/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml @@ -31,7 +31,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: 0720e7cbfe34090213f911bb21f5fa7057da4b7e7c8d8da08211166ff4efcb38 + checksum/config: 31db8ab8c088da32db35a94a62cad5b0fa5e6e31275613d75a98747d74a49a12 spec: serviceAccountName: default-splunk-otel-collector nodeSelector: From 7345d03bcc7b760cfb512a068061e26f1111475f Mon Sep 17 00:00:00 2001 From: Pawel Szkamruk Date: Fri, 22 Nov 2024 10:47:13 +0100 Subject: [PATCH 4/6] pre-commit --- .../rendered_manifests/deployment-cluster-receiver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml index 590541ae6c..f22cccbf29 100644 --- a/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml +++ b/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml @@ -31,7 +31,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: c9ed91daa196038bd9df714e679f9472c009405ca19d21f51f4d397b8e2066b4 + checksum/config: 004d88a7822945a4e1314128a9ea760fb99c20fb8761d3884bab0646200cfe22 spec: serviceAccountName: default-splunk-otel-collector nodeSelector: From 424d75abf6c674fb239270bb74c7a0e7b78f868e Mon Sep 17 00:00:00 2001 From: Pawel Szkamruk Date: Mon, 25 Nov 2024 14:38:47 +0100 Subject: [PATCH 5/6] changes after review - if statement to import k8sAttributesProcessor all logs pipelines --- .../templates/config/_otel-k8s-cluster-receiver-config.tpl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl index 4ea5548675..530d9358f9 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl @@ -117,7 +117,11 @@ processors: - set(resource.attributes["com.splunk.sourcetype"], Concat(["kube:object:", attributes["k8s.resource.name"]], "")) {{- end }} - {{- if and $clusterReceiver.eventsEnabled (eq (include "splunk-otel-collector.logsEnabled" .) "true") }} + {{- if or + (and $clusterReceiver.eventsEnabled (eq (include "splunk-otel-collector.logsEnabled" .) "true")) + (and (eq (include "splunk-otel-collector.objectsEnabled" .) "true") (eq (include "splunk-otel-collector.logsEnabled" .) "true")) + (eq (include "splunk-otel-collector.o11yInfraMonEventsEnabled" .) "true") + }} {{- include "splunk-otel-collector.k8sClusterReceiverAttributesProcessor" . | nindent 2 }} {{- end }} From 9ec37318a3167ad3573d5de83924793167973242 Mon Sep 17 00:00:00 2001 From: Pawel Szkamruk Date: Mon, 25 Nov 2024 15:26:26 +0100 Subject: [PATCH 6/6] pre-commit + make reder --- .../configmap-cluster-receiver.yaml | 17 +++++++++++++++++ .../deployment-cluster-receiver.yaml | 2 +- .../_otel-k8s-cluster-receiver-config.tpl | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml index a2290d9f18..54d483318f 100644 --- a/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml +++ b/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml @@ -36,6 +36,23 @@ data: processors: batch: send_batch_max_size: 32768 + k8sattributes/clusterReceiver: + extract: + metadata: + - k8s.namespace.name + - k8s.node.name + - k8s.pod.name + - k8s.pod.uid + - container.id + - container.image.name + - container.image.tag + pod_association: + - sources: + - from: resource_attribute + name: k8s.namespace.name + - sources: + - from: resource_attribute + name: k8s.node.name memory_limiter: check_interval: 2s limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB} diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml index 5287b6168f..0ce37d81fd 100644 --- a/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml +++ b/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml @@ -31,7 +31,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: b30b3bb56ee1f2c16bfa9aeff8933007d6f4e388850220def3c3da90815b3916 + checksum/config: b7c6c8b6f15da89afa7eff5ec8989a631bb3004f90d82d3bf6b7f36a94c08813 spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl index 530d9358f9..974927a7da 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl @@ -120,7 +120,7 @@ processors: {{- if or (and $clusterReceiver.eventsEnabled (eq (include "splunk-otel-collector.logsEnabled" .) "true")) (and (eq (include "splunk-otel-collector.objectsEnabled" .) "true") (eq (include "splunk-otel-collector.logsEnabled" .) "true")) - (eq (include "splunk-otel-collector.o11yInfraMonEventsEnabled" .) "true") + (eq (include "splunk-otel-collector.o11yInfraMonEventsEnabled" .) "true") }} {{- include "splunk-otel-collector.k8sClusterReceiverAttributesProcessor" . | nindent 2 }} {{- end }}