capacity.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: capacity
spec:
  groups:
    - name: capacity.rules
      rules:
        - expr: |
            sum(sum(kube_pod_container_resource_requests{resource="cpu", unit="core"}) by(pod, node) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (node)
          record: capacity:node:cpu:requests
        - expr: |
            sum(sum(kube_pod_container_resource_limits{resource="cpu", unit="core"}) by(pod, node) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (node)
          record: capacity:node:cpu:limits
        - expr: |
            kube_node_status_allocatable{resource="cpu", unit="core"} * on(node) group_left(role) kube_node_role
          record: capacity:node:cpu:available
        - expr: |
            label_replace(instance:node_cpu:rate:sum, "node", "$1", "instance", "(.+)")
          record: capacity:node:cpu:usage
        - expr: |
            sum(sum(kube_pod_container_resource_requests{resource="memory", unit="bytes"}) by(pod, node) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (node)
          record: capacity:node:memory:requests
        - expr: |
            sum(sum(kube_pod_container_resource_limits{resource="memory", unit="bytes"}) by(pod, node) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (node)
          record: capacity:node:memory:limits
        - expr: |
            kube_node_status_allocatable{resource="memory", unit="byte"} * on(node) group_left(role) kube_node_role
          record: capacity:node:memory:available
        - expr: |
            sum(sum(container_memory_working_set_bytes) by (pod, node) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (node)
          record: capacity:node:memory:usage

        - expr: |
            sum(sum(kube_pod_container_resource_requests{resource="cpu", unit="core"}) by(pod, namespace) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (namespace)
          record: capacity:namespace:cpu:requests
        - expr: |
            sum(sum(kube_pod_container_resource_limits{resource="cpu", unit="core"}) by(pod, namespace) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (namespace)
          record: capacity:namespace:cpu:limits
        - expr: |
            sum(sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate) by (pod, namespace) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (namespace)
          record: capacity:namespace:cpu:usage
        - expr: |
            sum(sum(kube_pod_container_resource_requests{resource="memory", unit="bytes"}) by (pod, namespace) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (namespace)
          record: capacity:namespace:memory:requests
        - expr: |
            sum(sum(kube_pod_container_resource_limits{resource="memory", unit="bytes"}) by(pod, namespace) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (namespace)
          record: capacity:namespace:memory:limits
        - expr: |
            sum(sum(container_memory_working_set_bytes) by (pod, namespace) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (namespace)
          record: capacity:namespace:memory:usage

        - expr: |
            sum(sum(sum(container_memory_working_set_bytes) by (pod, node) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (node) * on(node) group_left(role) kube_node_role) by (role)
          record: capacity:role:memory:usage
        - expr: |
            sum(sum(sum(kube_pod_container_resource_requests{resource="memory", unit="bytes"}) by(pod, node) * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (node) * on(node) group_left(role) kube_node_role) by (role)
          record: capacity:role:memory:requests

    - name: capacity-pods.alerts
      rules:
        - alert: NodePodsCapacityHigh
          annotations:
            message: >-
              Number of pods on node {{ $labels.node }} is high
          expr: |
             (kube_node_status_allocatable{resource="pods"} - on(node) sum(kube_pod_info * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (node)) < 30
          for: 8h
          labels:
            type: capacity
            source: user
            severity: high
        - alert: NodePodsCapacityCritical
          annotations:
            message: >-
              Number of pods on node {{ $labels.node }} is critical
          expr: |
             (kube_node_status_allocatable{resource="pods"} - on(node) sum(kube_pod_info * on(pod) group_left max(kube_pod_status_phase{phase="Running"}) by (pod)) by (node)) < 10
          for: 2h
          labels:
            type: capacity
            source: user
            severity: critical
    - name: capacity-cpu.alerts
      rules:
        - alert: NodeCpuRequestsWarning
          annotations:
            message: >-
              CPU requests on node {{ $labels.node }} are increasing
          expr: |
             (capacity:node:cpu:requests / on(node) capacity:node:cpu:available) > 0.8
          for: 8h
          labels:
            type: capacity
            source: user
            severity: warning
        - alert: NodeCpuRequestsHigh
          annotations:
            message: >-
              CPU requests on node {{ $labels.node }} are high
          expr: |
             (capacity:node:cpu:requests / on(node) capacity:node:cpu:available) > 0.9
          for: 2h
          labels:
            type: capacity
            source: user
            severity: high
        - alert: NodeCpuRequestsCritical
          annotations:
            message: >-
              CPU requests on node {{ $labels.node }} are critical
          expr: |
             (capacity:node:cpu:requests / on(node) capacity:node:cpu:available) > 0.95
          for: 1h
          labels:
            type: capacity
            source: user
            severity: critical

        - alert: NodeCpuUsageWarning
          annotations:
            message: >-
              CPU usage on node {{ $labels.node }} is increasing
          expr: |
             (capacity:node:cpu:usage / on(node) capacity:node:cpu:available) > 0.7
          for: 8h
          labels:
            type: capacity
            source: user
            severity: warning
        - alert: NodeCpuUsageHigh
          annotations:
            message: >-
              CPU usage on node {{ $labels.node }} is high
          expr: |
             (capacity:node:cpu:usage / on(node) capacity:node:cpu:available) > 0.8
          for: 4h
          labels:
            type: capacity
            source: user
            severity: high
        - alert: NodeCpuUsageCritical
          annotations:
            message: >-
              CPU usage on node {{ $labels.node }} is critical
          expr: |
             (capacity:node:cpu:usage / on(node) capacity:node:cpu:available) > 0.9
          for: 2h
          labels:
            type: capacity
            source: user
            severity: critical


    - name: capacity-memory.alerts
      rules:
        - alert: NodeMemoryRequestsWarning
          annotations:
            message: >-
              Memory requests on node {{ $labels.node }} are increasing
          expr: |
             (capacity:node:memory:requests / on(node) capacity:node:memory:available) > 0.85
          for: 8h
          labels:
            type: capacity
            source: user
            severity: warning
        - alert: NodeMemoryRequestsHigh
          annotations:
            message: >-
              Memory requests on node {{ $labels.node }} are high
          expr: |
             (capacity:node:memory:requests / on(node) capacity:node:memory:available) > 0.9
          for: 2h
          labels:
            type: capacity
            source: user
            severity: high
        - alert: NodeMemoryRequestsCritical
          annotations:
            message: >-
              Memory requests on node {{ $labels.node }} are critical
          expr: |
             (capacity:node:memory:requests / on(node) capacity:node:memory:available) > 0.95
          for: 1h
          labels:
            type: capacity
            source: user
            severity: critical

        - alert: NodeMemoryUsageWarning
          annotations:
            message: >-
              Memory usage on node {{ $labels.node }} is increasing
          expr: |
             (capacity:node:memory:usage / on(node) capacity:node:memory:available) > 0.75
          for: 8h
          labels:
            type: capacity
            source: user
            severity: warning
        - alert: NodeMemoryUsageHigh
          annotations:
            message: >-
              Memory usage on node {{ $labels.node }} is high
          expr: |
             (capacity:node:memory:usage / on(node) capacity:node:memory:available) > 0.85
          for: 4h
          labels:
            type: capacity
            source: user
            severity: high
        - alert: NodeMemoryUsageCritical
          annotations:
            message: >-
              Memory usage on node {{ $labels.node }} is critical
          expr: |
             (capacity:node:memory:usage / on(node) capacity:node:memory:available) > 0.9
          for: 2h
          labels:
            type: capacity
            source: user
            severity: critical

        - alert: ClusterAvailabilityNotice
          expr: |
            (capacity:role:memory:requests{role="worker"} / sum(capacity:node:memory:available{role="worker"}) by (role)) >= 0.7
          for: 12h
          annotations:
            message: >-
              The cluster memory requests are increasing. This will impact cluster availability.
          labels:
            type: capacity
            source: user
            severity: info
        - alert: ClusterAvailabilityNotice2
          expr: |
            (capacity:role:memory:requests{role="worker"} / sum(capacity:node:memory:available{role="worker"}) by (role)) >= 0.8
          for: 8h
          annotations:
            message: >-
              The cluster memory requests are too high. Cluster availability can be impacted.
          labels:
            type: capacity
            source: user
            severity: warning
        - alert: ClusterAvailabilityWarning
          expr: |
            (capacity:role:memory:usage{role="worker"} / sum(capacity:node:memory:available{role="worker"}) by (role)) > 0.6
          for: 8h
          annotations:
            message: >-
              The cluster memory usage is increasing. Soon there will be not enough resources to meet the availability requirements.
          labels:
            type: capacity
            source: user
            severity: warning
        - alert: ClusterAvailabilityDegraded
          expr: |
            (capacity:role:memory:usage{role="worker"} / sum(capacity:node:memory:available{role="worker"}) by (role)) > 0.66
          for: 12h
          annotations:
            message: >-
              The cluster availability is degraded. There is not enough memory to meet the availability requirements.
          labels:
            type: capacity
            source: user
            severity: high
        - alert: ClusterAvailabilityCritical
          expr: |
            (capacity:role:memory:usage{role="worker"} / sum(capacity:node:memory:available{role="worker"}) by (role)) >= 0.85
          for: 4h
          annotations:
            message: >-
              The cluster availability is critical. Reduce memory usage or add additional resources.
          labels:
            type: capacity
            source: user
            severity: critical


    - name: capacity-storage.alerts
      rules:
        - alert: PvcStorageHigh
          annotations:
            message: >-
              Storage on PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} is above 90%
          expr: |
             (1 - (kubelet_volume_stats_available_bytes / on(persistentvolumeclaim, namespace) kubelet_volume_stats_capacity_bytes)) > 0.90
          for: 30m
          labels:
            type: capacity
            source: user
            severity: high
        - alert: PvcStorageCritical
          annotations:
            message: >-
              Storage on PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} is critical (> 95%)
          expr: |
             (1 - (kubelet_volume_stats_available_bytes / on(persistentvolumeclaim, namespace) kubelet_volume_stats_capacity_bytes)) > 0.95
          for: 10m
          labels:
            type: capacity
            source: user
            severity: critical
        - alert: PvcPredictionWarning
          for: 2h
          labels:
            type: capacity
            source: user
            severity: warning
          annotations:
            message: >-
              Storage on PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} will be full in 24 hours
          expr: |
            (1 - (predict_linear(kubelet_volume_stats_available_bytes[5h], 3600 * 24) / on(persistentvolumeclaim, namespace, node) max_over_time(kubelet_volume_stats_capacity_bytes[5h]))) >= 1
        - alert: PvcPredictionWarning2
          for: 1h
          labels:
            type: capacity
            source: user
            severity: warning
          annotations:
            message: >-
              Storage on PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} will be full in 12 hours
          expr: |
            (1 - (predict_linear(kubelet_volume_stats_available_bytes[3h], 3600 * 12) / on(persistentvolumeclaim, namespace, node) max_over_time(kubelet_volume_stats_capacity_bytes[3h]))) >= 1
        - alert: PvcPredictionWarning3
          for: 1h
          labels:
            type: capacity
            source: user
            severity: warning
          annotations:
            message: >-
              Storage on PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} will be full in 6 hours
          expr: |
            (1 - (predict_linear(kubelet_volume_stats_available_bytes[2h], 3600 * 6) / on(persistentvolumeclaim, namespace, node) max_over_time(kubelet_volume_stats_capacity_bytes[2h]))) >= 1
        - alert: PvcPredictionHigh
          for: 30m
          labels:
            type: capacity
            source: user
            severity: high
          annotations:
            message: >-
              Storage on PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} will be full in 3 hours
          expr: |
            (1 - (predict_linear(kubelet_volume_stats_available_bytes[1h], 3600 * 3) / on(persistentvolumeclaim, namespace, node) max_over_time(kubelet_volume_stats_capacity_bytes[1h]))) >= 1
        - alert: PvcPredictionCritical
          for: 15m
          labels:
            type: capacity
            source: user
            severity: critical
          annotations:
            message: >-
              Storage on PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} will be full in 1 hour
          expr: |
            (1 - (predict_linear(kubelet_volume_stats_available_bytes[15m], 3600 * 1) / on(persistentvolumeclaim, namespace, node) max_over_time(kubelet_volume_stats_capacity_bytes[15m]))) >= 1