k8s attributes procesor for cluster receiver to ingest events into in…

…dex from annotation
signalfx · Nov 18, 2024 · 860451b · 860451b
1 parent e2b277d
commit 860451b
Show file tree

Hide file tree

Showing 35 changed files with 790 additions and 4 deletions.
diff --git a/.chloggen/ingest_cluster_receiver_events_into_index_from_annotation.yaml b/.chloggen/ingest_cluster_receiver_events_into_index_from_annotation.yaml
@@ -5,7 +5,7 @@ component: clusterReceiver
 # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
 note: configure k8s attributes processor for cluster receiver to ingest events into index defined in namespace annotation
 # One or more tracking issues related to the change
-issues: [14]
+issues: [1481]
 # (Optional) One or more lines of additional information to render under the primary note.
 # These lines will be padded with 2 spaces and then inserted directly into the document.
 # Use pipe (|) for multiline entries.

diff --git a/docs/advanced-configuration.md b/docs/advanced-configuration.md
@@ -559,6 +559,7 @@ Manage Splunk OTel Collector Logging with these supported annotations.
 
 * Use `splunk.com/index` annotation on pod and/or namespace to tell which Splunk platform indexes to ingest to. Pod annotation will take precedence over namespace annotation when both are annotated.
   For example, the following command will make logs from `kube-system` namespace to be sent to `k8s_events` index: `kubectl annotate namespace kube-system splunk.com/index=k8s_events`
+  **Please Note:** Cluster receiver supports only namespace index annotations, pod index annotations are not supported.
 * Use `splunk.com/metricsIndex` annotation on pod and/or namespace to tell which Splunk platform metric indexes to ingest to. Pod annotation will take precedence over namespace annotation when both are annotated.
 * Filter logs using pod and/or namespace annotation
   * If `logsCollection.containers.useSplunkIncludeAnnotation` is `false` (default: false), set `splunk.com/exclude` annotation to `true` on pod and/or namespace to exclude its logs from ingested.

diff --git a/examples/add-filter-processor/rendered_manifests/configmap-cluster-receiver.yaml b/examples/add-filter-processor/rendered_manifests/configmap-cluster-receiver.yaml
@@ -30,6 +30,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/add-kafkametrics-receiver/rendered_manifests/configmap-cluster-receiver.yaml b/examples/add-kafkametrics-receiver/rendered_manifests/configmap-cluster-receiver.yaml
@@ -30,6 +30,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/add-receiver-creator/rendered_manifests/configmap-cluster-receiver.yaml b/examples/add-receiver-creator/rendered_manifests/configmap-cluster-receiver.yaml
@@ -30,6 +30,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/add-sampler/rendered_manifests/configmap-cluster-receiver.yaml b/examples/add-sampler/rendered_manifests/configmap-cluster-receiver.yaml
@@ -30,6 +30,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/autodetect-istio/rendered_manifests/configmap-cluster-receiver.yaml b/examples/autodetect-istio/rendered_manifests/configmap-cluster-receiver.yaml
@@ -30,6 +30,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/collector-all-modes/rendered_manifests/configmap-cluster-receiver.yaml b/examples/collector-all-modes/rendered_manifests/configmap-cluster-receiver.yaml
@@ -30,6 +30,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml
@@ -36,6 +36,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
@@ -130,6 +156,7 @@ data:
           - resourcedetection
           - resource
           - transform/add_sourcetype
+          - k8sattributes/clusterReceiver
           receivers:
           - k8sobjects
         metrics:

diff --git a/examples/controlplane-histogram-metrics/rendered_manifests/configmap-cluster-receiver.yaml b/examples/controlplane-histogram-metrics/rendered_manifests/configmap-cluster-receiver.yaml
@@ -30,6 +30,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/crio-logging/rendered_manifests/configmap-cluster-receiver.yaml b/examples/crio-logging/rendered_manifests/configmap-cluster-receiver.yaml
@@ -30,6 +30,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/default/rendered_manifests/configmap-cluster-receiver.yaml b/examples/default/rendered_manifests/configmap-cluster-receiver.yaml
@@ -30,6 +30,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/disable-persistence-queue-traces/rendered_manifests/configmap-cluster-receiver.yaml b/examples/disable-persistence-queue-traces/rendered_manifests/configmap-cluster-receiver.yaml
@@ -47,6 +47,32 @@ data:
     processors:
       batch:
         send_batch_max_size: 32768
+      k8sattributes/clusterReceiver:
+        extract:
+          annotations:
+          - from: pod
+            key: splunk.com/sourcetype
+          - from: namespace
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          - from: pod
+            key: splunk.com/index
+            tag_name: com.splunk.index
+          metadata:
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.pod.name
+          - k8s.pod.uid
+          - container.id
+          - container.image.name
+          - container.image.tag
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.node.name
       k8sattributes/metrics:
         extract:
           annotations: