diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md
index 8f5dfff3f4..fcea2027d6 100644
--- a/docs/pod-metrics.md
+++ b/docs/pod-metrics.md
@@ -21,6 +21,7 @@
| kube_pod_container_status_terminated | Gauge | Describes whether the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - |
| kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - |
| kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - |
+| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - |
| kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - |
| kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name>
`uid`=<pod-uid> | STABLE | - |
| kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - |
diff --git a/internal/store/pod.go b/internal/store/pod.go
index e593c67815..774fabaa00 100644
--- a/internal/store/pod.go
+++ b/internal/store/pod.go
@@ -47,6 +47,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
createPodContainerResourceRequestsFamilyGenerator(),
createPodContainerStateStartedFamilyGenerator(),
createPodContainerStatusLastTerminatedReasonFamilyGenerator(),
+ createPodContainerStatusLastTerminatedExitCodeFamilyGenerator(),
createPodContainerStatusReadyFamilyGenerator(),
createPodContainerStatusRestartsTotalFamilyGenerator(),
createPodContainerStatusRunningFamilyGenerator(),
@@ -335,6 +336,31 @@ func createPodContainerStatusLastTerminatedReasonFamilyGenerator() generator.Fam
)
}
+func createPodContainerStatusLastTerminatedExitCodeFamilyGenerator() generator.FamilyGenerator {
+ return *generator.NewFamilyGenerator(
+ "kube_pod_container_status_last_terminated_exitcode",
+ "Describes the exit code for the last container in terminated state.",
+ metric.Gauge,
+ "",
+ wrapPodFunc(func(p *v1.Pod) *metric.Family {
+ ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses))
+ for _, cs := range p.Status.ContainerStatuses {
+ if cs.LastTerminationState.Terminated != nil {
+ ms = append(ms, &metric.Metric{
+ LabelKeys: []string{"container"},
+ LabelValues: []string{cs.Name},
+ Value: float64(cs.LastTerminationState.Terminated.ExitCode),
+ })
+ }
+ }
+
+ return &metric.Family{
+ Metrics: ms,
+ }
+ }),
+ )
+}
+
func createPodContainerStatusReadyFamilyGenerator() generator.FamilyGenerator {
return *generator.NewFamilyGenerator(
"kube_pod_container_status_ready",
diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go
index 9e58232f2d..e08a7cfb78 100644
--- a/internal/store/pod_test.go
+++ b/internal/store/pod_test.go
@@ -573,7 +573,8 @@ func TestPodStore(t *testing.T) {
StartedAt: metav1.Time{
Time: time.Unix(1501777018, 0),
},
- Reason: "OOMKilled",
+ Reason: "OOMKilled",
+ ExitCode: 137,
},
},
},
@@ -648,12 +649,14 @@ func TestPodStore(t *testing.T) {
},
},
Want: `
+ # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state.
# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
+ # TYPE kube_pod_container_status_last_terminated_exitcode gauge
# TYPE kube_pod_container_status_last_terminated_reason gauge
# TYPE kube_pod_container_status_running gauge
# TYPE kube_pod_container_status_terminated gauge
@@ -678,6 +681,10 @@ func TestPodStore(t *testing.T) {
"kube_pod_container_status_last_terminated_reason",
"kube_pod_container_status_last_terminated_reason",
"kube_pod_container_status_last_terminated_reason",
+ "kube_pod_container_status_last_terminated_exitcode",
+ "kube_pod_container_status_last_terminated_exitcode",
+ "kube_pod_container_status_last_terminated_exitcode",
+ "kube_pod_container_status_last_terminated_exitcode",
},
},
{
@@ -709,7 +716,8 @@ func TestPodStore(t *testing.T) {
},
LastTerminationState: v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
- Reason: "OOMKilled",
+ Reason: "OOMKilled",
+ ExitCode: 137,
},
},
},
@@ -718,6 +726,7 @@ func TestPodStore(t *testing.T) {
},
Want: `
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
+ # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state.
# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
@@ -725,6 +734,7 @@ func TestPodStore(t *testing.T) {
# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
# TYPE kube_pod_container_status_last_terminated_reason gauge
+ # TYPE kube_pod_container_status_last_terminated_exitcode gauge
# TYPE kube_pod_container_status_running gauge
# TYPE kube_pod_container_status_terminated gauge
# TYPE kube_pod_container_status_terminated_reason gauge
@@ -736,9 +746,11 @@ func TestPodStore(t *testing.T) {
kube_pod_container_status_terminated{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0
kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled",uid="uid6"} 1
+ kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 137
`,
MetricNames: []string{
"kube_pod_container_status_last_terminated_reason",
+ "kube_pod_container_status_last_terminated_exitcode",
"kube_pod_container_status_running",
"kube_pod_container_state_started",
"kube_pod_container_status_terminated",
@@ -774,7 +786,8 @@ func TestPodStore(t *testing.T) {
},
LastTerminationState: v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
- Reason: "DeadlineExceeded",
+ Reason: "DeadlineExceeded",
+ ExitCode: 143,
},
},
},
@@ -782,6 +795,7 @@ func TestPodStore(t *testing.T) {
},
},
Want: `
+ # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
@@ -789,6 +803,7 @@ func TestPodStore(t *testing.T) {
# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
+ # TYPE kube_pod_container_status_last_terminated_exitcode gauge
# TYPE kube_pod_container_status_last_terminated_reason gauge
# TYPE kube_pod_container_status_running gauge
# TYPE kube_pod_container_state_started gauge
@@ -797,6 +812,7 @@ func TestPodStore(t *testing.T) {
# TYPE kube_pod_container_status_waiting gauge
# TYPE kube_pod_container_status_waiting_reason gauge
kube_pod_container_state_started{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501777018e+09
+ kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 143
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="DeadlineExceeded",uid="uid7"} 1
kube_pod_container_status_running{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1
kube_pod_container_status_terminated{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0
@@ -809,6 +825,7 @@ func TestPodStore(t *testing.T) {
"kube_pod_container_status_terminated_reason",
"kube_pod_container_status_waiting",
"kube_pod_container_status_last_terminated_reason",
+ "kube_pod_container_status_last_terminated_exitcode",
},
},
{
@@ -2019,7 +2036,8 @@ func BenchmarkPodStore(b *testing.B) {
},
LastTerminationState: v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
- Reason: "OOMKilled",
+ Reason: "OOMKilled",
+ ExitCode: 137,
},
},
},
@@ -2035,7 +2053,8 @@ func BenchmarkPodStore(b *testing.B) {
},
LastTerminationState: v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
- Reason: "OOMKilled",
+ Reason: "OOMKilled",
+ ExitCode: 137,
},
},
},
@@ -2051,7 +2070,8 @@ func BenchmarkPodStore(b *testing.B) {
},
LastTerminationState: v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
- Reason: "OOMKilled",
+ Reason: "OOMKilled",
+ ExitCode: 137,
},
},
},
@@ -2059,7 +2079,7 @@ func BenchmarkPodStore(b *testing.B) {
},
}
- expectedFamilies := 46
+ expectedFamilies := 47
for n := 0; n < b.N; n++ {
families := f(pod)
if len(families) != expectedFamilies {
diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go
index 71d77861d7..e1012dc5af 100644
--- a/pkg/app/server_test.go
+++ b/pkg/app/server_test.go
@@ -196,6 +196,7 @@ func TestFullScrapeCycle(t *testing.T) {
# HELP kube_pod_container_resource_limits The number of requested limit resource by a container.
# HELP kube_pod_container_resource_requests The number of requested request resource by a container.
# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
+# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
# HELP kube_pod_container_status_ready Describes whether the containers readiness check succeeded.
# HELP kube_pod_container_status_restarts_total The number of container restarts per container.
@@ -241,6 +242,7 @@ func TestFullScrapeCycle(t *testing.T) {
# TYPE kube_pod_container_resource_limits gauge
# TYPE kube_pod_container_resource_requests gauge
# TYPE kube_pod_container_state_started gauge
+# TYPE kube_pod_container_status_last_terminated_exitcode gauge
# TYPE kube_pod_container_status_last_terminated_reason gauge
# TYPE kube_pod_container_status_ready gauge
# TYPE kube_pod_container_status_restarts_total counter
@@ -297,6 +299,7 @@ kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",
kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="storage",unit="byte"} 4e+08
kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="cpu",unit="core"} 0.3
kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08
+kube_pod_container_status_last_terminated_exitcode{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 137
kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",reason="OOMKilled"} 1
kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0
kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0
@@ -794,7 +797,8 @@ func pod(client *fake.Clientset, index int) error {
},
LastTerminationState: v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
- Reason: "OOMKilled",
+ Reason: "OOMKilled",
+ ExitCode: 137,
},
},
},