From 8a579dba6df37ed9a13b3b6b0949bf89ebe25dbe Mon Sep 17 00:00:00 2001 From: Indrranil Pawar Date: Tue, 17 Dec 2024 14:56:52 +0530 Subject: [PATCH 1/3] [Feature] Add Karpenter Do Not Disrupt policy for Karpenter 1.x migration This policy adds the karpenter.sh/do-not-disrupt label to pods that have the deprecated karpenter.sh/do-not-evict label to ensure smooth upgrades from Karpenter 0.x to 1.x versions. Fixes #1191 Signed-off-by: Indrranil Pawar --- .../add-karpenter-donot-disrupt.yaml | 35 +++++++++++++++++++ .../artifacthub-pkg.yml | 33 +++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 karpenter/add-karpenter-donot-disrupt/add-karpenter-donot-disrupt.yaml create mode 100644 karpenter/add-karpenter-donot-disrupt/artifacthub-pkg.yml diff --git a/karpenter/add-karpenter-donot-disrupt/add-karpenter-donot-disrupt.yaml b/karpenter/add-karpenter-donot-disrupt/add-karpenter-donot-disrupt.yaml new file mode 100644 index 000000000..c38e57512 --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/add-karpenter-donot-disrupt.yaml @@ -0,0 +1,35 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-karpenter-donot-disrupt + annotations: + policies.kyverno.io/title: Add Karpenter Do Not Disrupt + policies.kyverno.io/category: Karpenter, EKS Best Practices + policies.kyverno.io/severity: medium + policies.kyverno.io/subject: Pod + kyverno.io/kyverno-version: 1.7.1 + policies.kyverno.io/minversion: 1.6.0 + kyverno.io/kubernetes-version: "1.23" + policies.kyverno.io/description: >- + This policy assists in migrating from Karpenter 0.x to 1.x by adding the + karpenter.sh/do-not-disrupt label to pods that have the deprecated + karpenter.sh/do-not-evict label. This ensures smooth upgrades and + maintains the intended protection against pod disruption. +spec: + rules: + - name: add-donot-disrupt-label + match: + any: + - resources: + kinds: + - Pod + preconditions: + all: + - key: '{{request.object.metadata.labels."karpenter.sh/do-not-evict"}}' + operator: Equals + value: "true" + mutate: + patchStrategicMerge: + metadata: + labels: + "karpenter.sh/do-not-disrupt": "true" diff --git a/karpenter/add-karpenter-donot-disrupt/artifacthub-pkg.yml b/karpenter/add-karpenter-donot-disrupt/artifacthub-pkg.yml new file mode 100644 index 000000000..73b3e7aa8 --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/artifacthub-pkg.yml @@ -0,0 +1,33 @@ +name: add-karpenter-donot-disrupt +version: 1.0.0 +displayName: Add Karpenter Do Not Disrupt Label +createdAt: "2024-12-17T00:00:00Z" +description: | + This policy assists in migrating from Karpenter 0.x to 1.x by adding the + karpenter.sh/do-not-disrupt label to pods that have the deprecated + karpenter.sh/do-not-evict label. This ensures smooth upgrades and + maintains the intended protection against pod disruption during Karpenter + node termination. +install: | + ```sh + kubectl apply -f https://raw.githubusercontent.com/kyverno/policies/main/karpenter/add-karpenter-donot-disrupt/add-karpenter-donot-disrupt.yaml + ``` +keywords: + - kyverno + - karpenter +readme: | + # Add Karpenter Do Not Disrupt Label + + This policy facilitates the migration from Karpenter 0.x to 1.x by automatically + adding the new karpenter.sh/do-not-disrupt label to pods that use the deprecated + karpenter.sh/do-not-evict label. + + When a pod is created with the old karpenter.sh/do-not-evict label, this policy + automatically adds the new karpenter.sh/do-not-disrupt label to ensure the pod + remains protected during node termination operations in Karpenter 1.x. + + Reference for this change: https://karpenter.sh/v1.0/upgrading/v1beta1-migration/ +annotations: + kyverno.io/category: Pod + kyverno.io/kubernetes-version: "1.23" + kyverno.io/subject: "Pod, Migration" From 884c3a4873ebabce6d3c4995788c7e82a0c898ed Mon Sep 17 00:00:00 2001 From: Indrranil Pawar Date: Tue, 17 Dec 2024 16:10:50 +0530 Subject: [PATCH 2/3] [Feature] Add tests for Karpenter Do Not Disrupt policy Added chainsaw and kyverno test files to verify policy behavior for: - Label mutation from do-not-evict to do-not-disrupt - Proper handling of pods with and without the original label - Policy readiness verification Signed-off-by: Indrranil Pawar --- .../.chainsaw-test/chainsaw-test.yaml | 28 +++++++++++++++++++ .../.chainsaw-test/patched03.yaml | 11 ++++++++ .../.chainsaw-test/patched04.yaml | 10 +++++++ .../.chainsaw-test/policy-ready.yaml | 9 ++++++ .../.chainsaw-test/resource-others.yaml | 21 ++++++++++++++ .../.kyverno-test/kyverno-test.yaml | 23 +++++++++++++++ .../.kyverno-test/patched01.yaml | 11 ++++++++ .../.kyverno-test/patched02.yaml | 8 ++++++ .../.kyverno-test/resource.yaml | 19 +++++++++++++ 9 files changed, 140 insertions(+) create mode 100644 karpenter/add-karpenter-donot-disrupt/.chainsaw-test/chainsaw-test.yaml create mode 100644 karpenter/add-karpenter-donot-disrupt/.chainsaw-test/patched03.yaml create mode 100644 karpenter/add-karpenter-donot-disrupt/.chainsaw-test/patched04.yaml create mode 100644 karpenter/add-karpenter-donot-disrupt/.chainsaw-test/policy-ready.yaml create mode 100644 karpenter/add-karpenter-donot-disrupt/.chainsaw-test/resource-others.yaml create mode 100644 karpenter/add-karpenter-donot-disrupt/.kyverno-test/kyverno-test.yaml create mode 100644 karpenter/add-karpenter-donot-disrupt/.kyverno-test/patched01.yaml create mode 100644 karpenter/add-karpenter-donot-disrupt/.kyverno-test/patched02.yaml create mode 100644 karpenter/add-karpenter-donot-disrupt/.kyverno-test/resource.yaml diff --git a/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/chainsaw-test.yaml b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/chainsaw-test.yaml new file mode 100644 index 000000000..a3a77be87 --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/chainsaw-test.yaml @@ -0,0 +1,28 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: add-karpenter-donot-disrupt +spec: + steps: + - name: step-01 + try: + - apply: + file: ../add-karpenter-donot-disrupt.yaml + - assert: + file: policy-ready.yaml + - name: step-02 + try: + - apply: + file: ../.kyverno-test/resource.yaml + - apply: + file: resource-others.yaml + - name: step-03 + try: + - assert: + file: ../.kyverno-test/patched01.yaml + - assert: + file: ../.kyverno-test/patched02.yaml + - assert: + file: patched03.yaml + - assert: + file: patched04.yaml diff --git a/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/patched03.yaml b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/patched03.yaml new file mode 100644 index 000000000..1f235c4e5 --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/patched03.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-1 + labels: + karpenter.sh/do-not-evict: "true" + karpenter.sh/do-not-disrupt: "true" +spec: + containers: + - name: nginx + image: nginx:1.14.2 diff --git a/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/patched04.yaml b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/patched04.yaml new file mode 100644 index 000000000..990e7ebf5 --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/patched04.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-2 + labels: + karpenter.sh/do-not-evict: "false" +spec: + containers: + - name: nginx + image: nginx:1.14.2 diff --git a/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/policy-ready.yaml b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/policy-ready.yaml new file mode 100644 index 000000000..de050740d --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/policy-ready.yaml @@ -0,0 +1,9 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-karpenter-donot-disrupt +status: + conditions: + - reason: Succeeded + status: "True" + type: Ready diff --git a/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/resource-others.yaml b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/resource-others.yaml new file mode 100644 index 000000000..e37481128 --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/.chainsaw-test/resource-others.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-1 + labels: + karpenter.sh/do-not-evict: "true" +spec: + containers: + - name: nginx + image: nginx:1.14.2 +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-2 + labels: + karpenter.sh/do-not-evict: "false" +spec: + containers: + - name: nginx + image: nginx:1.14.2 diff --git a/karpenter/add-karpenter-donot-disrupt/.kyverno-test/kyverno-test.yaml b/karpenter/add-karpenter-donot-disrupt/.kyverno-test/kyverno-test.yaml new file mode 100644 index 000000000..ab4db8c9f --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/.kyverno-test/kyverno-test.yaml @@ -0,0 +1,23 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: add-karpenter-donot-disrupt +policies: + - ../add-karpenter-donot-disrupt.yaml +resources: + - resource.yaml +results: + - kind: Pod + patchedResource: patched01.yaml + policy: add-karpenter-donot-disrupt + resources: + - test-pod-with-evict + result: pass + rule: add-donot-disrupt-label + - kind: Pod + patchedResource: patched02.yaml + policy: add-karpenter-donot-disrupt + resources: + - test-pod-without-evict + result: skip + rule: add-donot-disrupt-label diff --git a/karpenter/add-karpenter-donot-disrupt/.kyverno-test/patched01.yaml b/karpenter/add-karpenter-donot-disrupt/.kyverno-test/patched01.yaml new file mode 100644 index 000000000..ea4f6970e --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/.kyverno-test/patched01.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-evict + labels: + karpenter.sh/do-not-evict: "true" + karpenter.sh/do-not-disrupt: "true" +spec: + containers: + - name: nginx + image: nginx:1.14.2 diff --git a/karpenter/add-karpenter-donot-disrupt/.kyverno-test/patched02.yaml b/karpenter/add-karpenter-donot-disrupt/.kyverno-test/patched02.yaml new file mode 100644 index 000000000..836891034 --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/.kyverno-test/patched02.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-without-evict +spec: + containers: + - name: nginx + image: nginx:1.14.2 diff --git a/karpenter/add-karpenter-donot-disrupt/.kyverno-test/resource.yaml b/karpenter/add-karpenter-donot-disrupt/.kyverno-test/resource.yaml new file mode 100644 index 000000000..8c501f713 --- /dev/null +++ b/karpenter/add-karpenter-donot-disrupt/.kyverno-test/resource.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-evict + labels: + karpenter.sh/do-not-evict: "true" +spec: + containers: + - name: nginx + image: nginx:1.14.2 +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-without-evict +spec: + containers: + - name: nginx + image: nginx:1.14.2 From 59cd794b0121280d141645fb55ccfff6c61df5e7 Mon Sep 17 00:00:00 2001 From: Indrranil Pawar Date: Tue, 17 Dec 2024 17:00:55 +0530 Subject: [PATCH 3/3] adjust preconditions to properly skip unlabeled pods Signed-off-by: Indrranil Pawar --- .../add-karpenter-donot-disrupt.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/karpenter/add-karpenter-donot-disrupt/add-karpenter-donot-disrupt.yaml b/karpenter/add-karpenter-donot-disrupt/add-karpenter-donot-disrupt.yaml index c38e57512..56d702179 100644 --- a/karpenter/add-karpenter-donot-disrupt/add-karpenter-donot-disrupt.yaml +++ b/karpenter/add-karpenter-donot-disrupt/add-karpenter-donot-disrupt.yaml @@ -19,17 +19,16 @@ spec: rules: - name: add-donot-disrupt-label match: - any: - - resources: - kinds: - - Pod + resources: + kinds: + - Pod preconditions: all: - - key: '{{request.object.metadata.labels."karpenter.sh/do-not-evict"}}' - operator: Equals - value: "true" + - key: '{{ request.object.metadata.labels."karpenter.sh/do-not-evict" || '''' }}' + operator: NotEquals + value: "" mutate: patchStrategicMerge: metadata: labels: - "karpenter.sh/do-not-disrupt": "true" + karpenter.sh/do-not-disrupt: "true"