Add testing of upgrading of latest released chart version to CI (#432)

* Add upgrade test to CI - install latest released chart version and then upgrade to the local chart version * Add metallb for loadbalancer support so that "--wait" will work * Improve script logging * produce messages with old version of pulsar and consume with new version * Add 10 second delay * Remove duplication * Improve logging * Fix quotes * Before trying produce-consume test, verify that the Admin API is available * Add 2 more upgrade scenarios: TLS & PSP
apache · Jan 16, 2024 · a8c7745 · a8c7745
1 parent f20756a
commit a8c7745
Show file tree

Hide file tree

Showing 5 changed files with 242 additions and 88 deletions.
diff --git a/.ci/chart_test.sh b/.ci/chart_test.sh
@@ -38,14 +38,39 @@ if [[ "x${SYMMETRIC}" == "xtrue" ]]; then
     extra_opts="-s"
 fi
 
-# install pulsar chart
-ci::install_pulsar_chart ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}
+install_type="install"
+test_action="produce-consume"
+if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then
+    # install older version of pulsar chart
+    PULSAR_CHART_VERSION="$UPGRADE_FROM_VERSION"
+    ci::install_pulsar_chart install ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}    
+    install_type="upgrade"
+    echo "Wait 10 seconds"
+    sleep 10
+    # test that we can access the admin api
+    ci::test_pulsar_admin_api_access
+    # produce messages with old version of pulsar and consume with new version
+    ci::test_pulsar_producer_consumer "produce"
+    test_action="consume"
+fi
+
+PULSAR_CHART_VERSION="local"
+# install (or upgrade) pulsar chart
+ci::install_pulsar_chart ${install_type} ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} ${extra_opts}
+
+echo "Wait 10 seconds"
+sleep 10
+
+# check pulsar environment
+ci::check_pulsar_environment
 
+# test that we can access the admin api
+ci::test_pulsar_admin_api_access
 # test producer/consumer
-ci::test_pulsar_producer_consumer
+ci::test_pulsar_producer_consumer "${test_action}"
 
 if [[ "x${FUNCTION}" == "xtrue" ]]; then
-    # install cert manager
+    # test functions
     ci::test_pulsar_function
 fi
 

diff --git a/.ci/clusters/values-basic.yaml → .ci/clusters/values-upgrade.yaml b/.ci/clusters/values-basic.yaml → .ci/clusters/values-upgrade.yaml
diff --git a/.ci/helm.sh b/.ci/helm.sh
@@ -20,13 +20,15 @@
 BINDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 PULSAR_HOME="$(cd "${BINDIR}/.." && pwd)"
 CHARTS_HOME=${PULSAR_HOME}
+PULSAR_CHART_LOCAL=${CHARTS_HOME}/charts/pulsar
+PULSAR_CHART_VERSION=${PULSAR_CHART_VERSION:-"local"}
 OUTPUT_BIN=${CHARTS_HOME}/output/bin
 KIND_BIN=$OUTPUT_BIN/kind
 HELM=${OUTPUT_BIN}/helm
 KUBECTL=${OUTPUT_BIN}/kubectl
 NAMESPACE=pulsar
 CLUSTER=pulsar-ci
-CLUSTER_ID=$(uuidgen)
+: ${CLUSTER_ID:=$(uuidgen)}
 K8S_LOGS_DIR="${K8S_LOGS_DIR:-/tmp/k8s-logs}"
 export PATH="$OUTPUT_BIN:$PATH"
 
@@ -100,68 +102,110 @@ function ci::collect_k8s_logs() {
 }
 
 function ci::install_pulsar_chart() {
-    local common_value_file=$1
-    local value_file=$2
-    local extra_opts=$3
-
-    echo "Installing the pulsar chart"
-    ${KUBECTL} create namespace ${NAMESPACE}
-    ci::install_cert_manager
-    echo ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} ${extra_opts}
-    ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} ${extra_opts}
-    sleep 10
-
-    echo ${HELM} dependency update ${CHARTS_HOME}/charts/pulsar
-    ${HELM} dependency update ${CHARTS_HOME}/charts/pulsar
-    echo ${HELM} install --set initialize=true --values ${common_value_file} --values ${value_file} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
-    ${HELM} template --values ${common_value_file} --values ${value_file} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
-    ${HELM} install --set initialize=true --values ${common_value_file} --values ${value_file} --namespace=${NAMESPACE} ${CLUSTER} ${CHARTS_HOME}/charts/pulsar
-
-    echo "wait until broker is alive"
-    WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
-    counter=1
-    while [[ ${WC} -lt 1 ]]; do
-      ((counter++))
-      echo ${WC};
-      sleep 15
-      ${KUBECTL} get pods,jobs -n ${NAMESPACE}
-      ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
-      if [[ $((counter % 20)) -eq 0 ]]; then
-        ci::print_pod_logs
-        if [[ $counter -gt 100 ]]; then
-          echo >&2 "Timeout waiting..."
-          exit 1
-        fi
-      fi
-      WC=$(${KUBECTL} get pods -n ${NAMESPACE} | grep ${CLUSTER}-broker | wc -l)
-      if [[ ${WC} -gt 1 ]]; then
-        ${KUBECTL} describe pod -n ${NAMESPACE} pulsar-ci-broker-0
-        ${KUBECTL} logs -n ${NAMESPACE} pulsar-ci-broker-0
+    local install_type=$1
+    local common_value_file=$2
+    local value_file=$3
+    local extra_opts=$4
+    local install_args
+
+    if [[ "${install_type}" == "install" ]]; then
+      echo "Installing the pulsar chart"
+      ${KUBECTL} create namespace ${NAMESPACE}
+      ci::install_cert_manager
+      echo ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} ${extra_opts}
+      ${CHARTS_HOME}/scripts/pulsar/prepare_helm_release.sh -k ${CLUSTER} -n ${NAMESPACE} ${extra_opts}
+      sleep 10
+
+      # install metallb for loadbalancer support
+      # following instructions from https://kind.sigs.k8s.io/docs/user/loadbalancer/
+      ${KUBECTL} apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
+      # wait until metallb is ready
+      ${KUBECTL} wait --namespace metallb-system \
+                --for=condition=ready pod \
+                --selector=app=metallb \
+                --timeout=90s
+      # configure metallb
+      ${KUBECTL} apply -f ${BINDIR}/metallb/metallb-config.yaml
+
+      install_args=""
+    else 
+      install_args="--wait --wait-for-jobs --timeout 300s --debug"
+    fi
+
+    CHART_ARGS=""
+    if [[ "${PULSAR_CHART_VERSION}" == "local" ]]; then
+      set -x
+      ${HELM} dependency update ${PULSAR_CHART_LOCAL}
+      set +x
+      CHART_ARGS="${PULSAR_CHART_LOCAL}"
+    else
+      set -x
+      ${HELM} repo add apache https://pulsar.apache.org/charts
+      set +x
+      CHART_ARGS="apache/pulsar --dependency-update"
+      if [[ "${PULSAR_CHART_VERSION}" != "latest" ]]; then
+        CHART_ARGS="${CHART_ARGS} --version ${PULSAR_CHART_VERSION}"
       fi
-      WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
-    done
-    timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
-    timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+    fi
+    set -x
+    ${HELM} template --values ${common_value_file} --values ${value_file} ${CLUSTER} ${CHART_ARGS}
+    ${HELM} ${install_type} --values ${common_value_file} --values ${value_file} --namespace=${NAMESPACE} ${CLUSTER} ${CHART_ARGS} ${install_args}
+    set +x
 
-    WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
-    counter=1
-    while [[ ${WC} -lt 1 ]]; do
-      ((counter++))
-      echo ${WC};
-      sleep 15
-      ${KUBECTL} get pods,jobs -n ${NAMESPACE}
-      ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
-      if [[ $((counter % 8)) -eq 0 ]]; then
-        ci::print_pod_logs
-        if [[ $counter -gt 16 ]]; then
-          echo >&2 "Timeout waiting..."
-          exit 1
+    if [[ "${install_type}" == "install" ]]; then
+      echo "wait until broker is alive"
+      WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
+      counter=1
+      while [[ ${WC} -lt 1 ]]; do
+        ((counter++))
+        echo ${WC};
+        sleep 15
+        ${KUBECTL} get pods,jobs -n ${NAMESPACE}
+        ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
+        if [[ $((counter % 20)) -eq 0 ]]; then
+          ci::print_pod_logs
+          if [[ $counter -gt 100 ]]; then
+            echo >&2 "Timeout waiting..."
+            exit 1
+          fi
         fi
-      fi
+        WC=$(${KUBECTL} get pods -n ${NAMESPACE} | grep ${CLUSTER}-broker | wc -l)
+        if [[ ${WC} -gt 1 ]]; then
+          ${KUBECTL} describe pod -n ${NAMESPACE} pulsar-ci-broker-0
+          ${KUBECTL} logs -n ${NAMESPACE} pulsar-ci-broker-0
+        fi
+        WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-broker | wc -l)
+      done
+      timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+
       WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
-    done
-    timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
-    # ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -L http://pulsar-ci-proxy:8080/status.html)" == "OK" ]; do sleep 3; done'
+      counter=1
+      while [[ ${WC} -lt 1 ]]; do
+        ((counter++))
+        echo ${WC};
+        sleep 15
+        ${KUBECTL} get pods,jobs -n ${NAMESPACE}
+        ${KUBECTL} get events --sort-by=.lastTimestamp -A | tail -n 30 || true
+        if [[ $((counter % 8)) -eq 0 ]]; then
+          ci::print_pod_logs
+          if [[ $counter -gt 16 ]]; then
+            echo >&2 "Timeout waiting..."
+            exit 1
+          fi
+        fi
+        WC=$(${KUBECTL} get pods -n ${NAMESPACE} --field-selector=status.phase=Running | grep ${CLUSTER}-proxy | wc -l)
+      done
+      timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      echo "Install complete"
+    else
+      echo "wait until broker is alive"
+      timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      timeout 120s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until [ "$(curl -s -L http://pulsar-ci-broker:8080/status.html)" == "OK" ]; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      echo "wait until proxy is alive"
+      timeout 300s ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done' || { echo >&2 "Timeout waiting..."; ci::print_pod_logs; exit 1; }
+      echo "Upgrade complete"
+    fi
 }
 
 helm_values_cached=""
@@ -173,28 +217,66 @@ function ci::helm_values_for_deployment() {
     printf "%s" "${helm_values_cached}"
 }
 
-function ci::test_pulsar_producer_consumer() {
-    sleep 120
+function ci::check_pulsar_environment() {
+    echo "Wait until pulsar-ci-broker is ready"
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
+    echo "Wait until pulsar-ci-proxy is ready"
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
+    echo "bookie-0 disk usage"
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
+    echo "bookie-0 bookkeeper.conf"
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- cat conf/bookkeeper.conf
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants create pulsar-ci
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin namespaces create pulsar-ci/test
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create pulsar-ci/test/test-topic
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test-topic
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m "test-message" pulsar-ci/test/test-topic
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test-topic
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test2 pulsar-ci/test/test-topic
+    echo "bookie-0 bookies list (rw)"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw | grep ListBookiesCommand
+    echo "bookie-0 bookies list (ro)"
+    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro | grep ListBookiesCommand
+}
+
+# function to retry a given commend 3 times with a backoff of 10 seconds in between
+function ci::retry() {
+  local n=1
+  local max=3
+  local delay=10
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "::warning::Command failed. Attempt $n/$max:"
+        sleep $delay
+      else
+        fail "::error::The command has failed after $n attempts."
+      fi
+    }
+  done
+}
+
+function ci::test_pulsar_admin_api_access() {
+  ci::retry ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants list
+}
+
+function ci::test_pulsar_producer_consumer() {
+    action="${1:-"produce-consume"}"
+    echo "Testing with ${action}"
     if [[ "$(ci::helm_values_for_deployment | yq .tls.proxy.enabled)" == "true" ]]; then
       PROXY_URL="pulsar+ssl://pulsar-ci-proxy:6651"
     else
       PROXY_URL="pulsar://pulsar-ci-proxy:6650"
     fi
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" produce -m "test-message2" pulsar-ci/test/test-topic
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" consume -s test2 pulsar-ci/test/test-topic
+    set -x
+    if [[ "${action}" == "produce" || "${action}" == "produce-consume" ]]; then
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin tenants create pulsar-ci
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin namespaces create pulsar-ci/test
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client produce -m "test-message" pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin topics create-subscription -s test2 pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" produce -m "test-message2" pulsar-ci/test/test-topic
+    fi
+    if [[ "${action}" == "consume" || "${action}" == "produce-consume" ]]; then
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client consume -s test pulsar-ci/test/test-topic
+      ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-client --url "${PROXY_URL}" consume -s test2 pulsar-ci/test/test-topic
+    fi
+    set +x
 }
 
 function ci::wait_function_running() {
@@ -219,14 +301,7 @@ function ci::wait_message_processed() {
 }
 
 function ci::test_pulsar_function() {
-    sleep 120
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-broker; do sleep 3; done'
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bash -c 'until nslookup pulsar-ci-proxy; do sleep 3; done'
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-bookie-0 -- df -h
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -rw
-    ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/bookkeeper shell listbookies -ro
     ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-toolset-0 -- bin/pulsar-admin functions create --tenant pulsar-ci --namespace test --name test-function --inputs "pulsar-ci/test/test_input" --output "pulsar-ci/test/test_output" --parallelism 1 --classname org.apache.pulsar.functions.api.examples.ExclamationFunction --jar /pulsar/examples/api-examples.jar
-
     # wait until the function is running
     # TODO: re-enable function test
     # ci::wait_function_running

diff --git a/.ci/metallb/metallb-config.yaml b/.ci/metallb/metallb-config.yaml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: example
+  namespace: metallb-system
+spec:
+  addresses:
+  - 172.19.255.200-172.19.255.250
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: empty
+  namespace: metallb-system