Add CI test case for kube-prometheus-stack upgrade

.ci/chart_test.sh

@@ -52,6 +52,11 @@ if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then
     # produce messages with old version of pulsar and consume with new version
     ci::test_pulsar_producer_consumer "produce"
     test_action="consume"
+
+    if [[ "$(ci::helm_values_for_deployment | yq .kube-prometheus-stack.enabled)" == "true" ]]; then
+        echo "Upgrade Prometheus Operator CRDs before upgrading the deployment"
+        ${PULSAR_HOME}/scripts/kube-prometheus-stack/upgrade_prometheus_operator_crds.sh
+    fi
 fi
 
 PULSAR_CHART_VERSION="local"

.ci/clusters/values-prometheus-grafana.yaml

@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+kube-prometheus-stack:
+  enabled: true
+  prometheus:
+    enabled: true
+  grafana:
+    enabled: true
+    adminPassword: pulsar-ci-admin
+  alertmanager:
+    enabled: false
+  prometheus-node-exporter:
+    enabled: true

.github/workflows/pulsar-helm-chart-ci.yaml

@@ -199,6 +199,15 @@ jobs:
               values_file: .ci/clusters/values-psp.yaml
               shortname: psp
               type: upgrade
+          - k8sVersion:
+              version: "1.21.14"
+              kind_image_tag: v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
+            testScenario:
+              name: "Upgrade kube-prometheus-stack"
+              values_file: .ci/clusters/values-prometheus-grafana.yaml
+              shortname: prometheus-grafana
+              type: upgrade
+              upgradeFromVersion: 3.2.0
           - k8sVersion:
               version: "1.21.14"
               kind_image_tag: v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
@@ -270,7 +279,7 @@ jobs:
               ;;
           esac
           if [[ "${{ matrix.testScenario.type || 'install' }}" == "upgrade" ]]; then
-            export UPGRADE_FROM_VERSION=latest
+            export UPGRADE_FROM_VERSION="${{ matrix.testScenario.upgradeFromVersion || 'latest' }}"
           fi
           .ci/chart_test.sh ${{ matrix.testScenario.values_file }}