Merge pull request #24731 from def-/pr-mzfg

ci: Upload heap profiles profiles continuously

ci/plugins/cloudtest/hooks/post-command

@@ -9,8 +9,7 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-# -x to debug mysterious failures without any logging
-set -euxo pipefail
+set -euo pipefail
 
 . misc/shlib/shlib.bash
 . test/cloudtest/config.bash

ci/plugins/mzcompose/hooks/command

@@ -70,14 +70,59 @@ if [ -n "${CI_COVERAGE_ENABLED:-}" ]; then
     mkdir -p coverage/
     chmod 777 coverage/
     # Not all tests contain all of these containers:
-    mzcompose cp sqllogictest:/usr/local/bin/sqllogictest coverage/ || true
-    mzcompose cp sqllogictest:/usr/local/bin/clusterd coverage/ || true
-    mzcompose cp materialized:/usr/local/bin/environmentd coverage/ || true
-    mzcompose cp materialized:/usr/local/bin/clusterd coverage/ || true
-    mzcompose cp testdrive:/usr/local/bin/testdrive coverage/ || true
-    mzcompose cp balancerd:/usr/local/bin/balancerd coverage/ || true
+    mzcompose --mz-quiet cp sqllogictest:/usr/local/bin/sqllogictest coverage/ || true
+    mzcompose --mz-quiet cp sqllogictest:/usr/local/bin/clusterd coverage/ || true
+    mzcompose --mz-quiet cp materialized:/usr/local/bin/environmentd coverage/ || true
+    mzcompose --mz-quiet cp materialized:/usr/local/bin/clusterd coverage/ || true
+    mzcompose --mz-quiet cp testdrive:/usr/local/bin/testdrive coverage/ || true
+    mzcompose --mz-quiet cp balancerd:/usr/local/bin/balancerd coverage/ || true
 fi
 
+upload_artifact() {
+    cservice="$1"
+    artifact="$2"
+    # For materialized
+    if mzcompose --mz-quiet exec "$cservice" curl --silent http://127.0.0.1:6878/prof/heap > "$artifact"; then
+        if [ -s "$artifact" ]; then
+            buildkite-agent artifact upload --log-level error "$artifact"
+            i=0
+            mzcompose --mz-quiet exec "$cservice" ps aux | grep "/usr/local/bin/clusterd" | grep -v grep | sed -e "s/.* --internal-http-listen-addr=\(\/tmp\/[^ ]*\) .*/\1/" | while read -r socket; do
+                artifact_socket=$(echo "$artifact" | sed -e "s#prof/#prof/clusterd$i-#")
+                if mzcompose --mz-quiet exec "$cservice" curl --silent --unix-socket "$socket" http:/prof/heap > "$artifact_socket"; then
+                    if [ -s "$artifact_socket" ]; then
+                        buildkite-agent artifact upload --log-level error "$artifact_socket"
+                    fi
+                fi
+                ((i++))
+            done
+        else
+            # For clusterd
+            if mzcompose --mz-quiet exec "$cservice" curl --silent http://127.0.0.1:6878/heap > "$artifact"; then
+                if [ -s "$artifact" ]; then
+                    buildkite-agent artifact upload --log-level error "$artifact"
+                fi
+            fi
+        fi
+    fi
+}
+
+rm -rf prof
+mkdir prof
+(while true; do
+    sleep 10
+    for cservice in $(mzcompose --mz-quiet ps --services); do
+        artifact=$(date "+prof/$cservice-%Y-%m-%d_%H:%M:%S.pb.gz")
+        upload_artifact "$cservice" "$artifact" &
+    done
+done
+) &
+
 ci_uncollapsed_heading ":docker: Running \`bin/mzcompose --find \"$BUILDKITE_PLUGIN_MZCOMPOSE_COMPOSITION\" run ${run_args[*]}\`"
 
 mzcompose run "${run_args[@]}" |& tee run.log
+RESULT=$?
+if [ "$RESULT" != "0" ]; then
+    # Give the logs some time to log panics, otherwise they might be missing later
+    sleep 10
+fi
+exit $RESULT

ci/plugins/mzcompose/hooks/post-command

@@ -9,8 +9,7 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-# -x to debug mysterious failures without any logging
-set -euxo pipefail
+set -euo pipefail
 
 . misc/shlib/shlib.bash