Merge pull request #29990 from def-/pr-kafka-matrix-time

kafka-matrix: Bump timeout, new versions, parallelize
MaterializeInc · Oct 15, 2024 · aadace9 · aadace9
2 parents 1686281 + cd7df8f
commit aadace9
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 15 deletions.
diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml
@@ -187,9 +187,10 @@ steps:
     key: kafka
     steps:
       - id: kafka-matrix
-        label: Kafka smoke test against previous Kafka versions
+        label: Kafka smoke test against previous Kafka versions %N
         depends_on: build-aarch64
         timeout_in_minutes: 120
+        parallelism: 2
         agents:
           queue: hetzner-aarch64-4cpu-8gb
         plugins:

diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py
@@ -29,7 +29,7 @@
 say = ui.speaker("C> ")
 
 
-DEFAULT_CONFLUENT_PLATFORM_VERSION = "7.6.0"
+DEFAULT_CONFLUENT_PLATFORM_VERSION = "7.7.0"
 
 DEFAULT_MZ_VOLUMES = [
     "mzdata:/mzdata",

diff --git a/misc/python/materialize/mzcompose/services/redpanda.py b/misc/python/materialize/mzcompose/services/redpanda.py
@@ -13,7 +13,7 @@
     ServiceConfig,
 )
 
-REDPANDA_VERSION = "v24.2.2"
+REDPANDA_VERSION = "v24.2.7"
 
 
 class Redpanda(Service):

diff --git a/test/kafka-matrix/mzcompose.py b/test/kafka-matrix/mzcompose.py
@@ -12,6 +12,8 @@
 working with Materialize.
 """
 
+from materialize import buildkite
+from materialize.mzcompose import DEFAULT_CONFLUENT_PLATFORM_VERSION
 from materialize.mzcompose.composition import Composition
 from materialize.mzcompose.services.kafka import Kafka
 from materialize.mzcompose.services.localstack import Localstack
@@ -25,25 +27,30 @@
     "v22.3.25",
     "v23.1.21",
     "v23.2.29",
-    "v23.3.18",
-    "v24.1.9",
+    "v23.3.21",
+    "v24.1.17",
     REDPANDA_VERSION,
+    "latest",
 ]
 
 CONFLUENT_PLATFORM_VERSIONS = [
-    "7.0.14",
-    "7.1.12",
-    "7.2.10",
-    "7.3.8",
-    "7.4.4",
-    "7.5.4",
-    "7.6.0",
+    "7.0.16",
+    "7.1.14",
+    "7.2.12",
+    "7.3.10",
+    "7.4.7",
+    "7.5.6",
+    "7.6.3",
+    DEFAULT_CONFLUENT_PLATFORM_VERSION,
     "latest",
 ]
 
 SERVICES = [
     Materialized(),
-    Testdrive(volumes_extra=["../testdrive:/workdir/testdrive"], default_timeout="60s"),
+    # Occasional timeouts in CI with 60s timeout
+    Testdrive(
+        volumes_extra=["../testdrive:/workdir/testdrive"], default_timeout="120s"
+    ),
     Redpanda(),
     Zookeeper(),
     Kafka(),
@@ -63,14 +70,24 @@
 def workflow_default(c: Composition) -> None:
     c.up("localstack")
 
-    for redpanda_version in REDPANDA_VERSIONS:
+    redpanda_versions = buildkite.shard_list(REDPANDA_VERSIONS, lambda v: v)
+    print(
+        f"Redpanda versions in shard with index {buildkite.get_parallelism_index()}: {redpanda_versions}"
+    )
+
+    for redpanda_version in redpanda_versions:
         print(f"--- Testing Redpanda {redpanda_version}")
         with c.override(Redpanda(version=redpanda_version)):
             c.down(destroy_volumes=True)
             c.up("redpanda", "materialized")
             c.run_testdrive_files(*TD_CMD)
 
-    for confluent_version in CONFLUENT_PLATFORM_VERSIONS:
+    confluent_versions = buildkite.shard_list(CONFLUENT_PLATFORM_VERSIONS, lambda v: v)
+    print(
+        f"Confluent Platform versions in shard with index {buildkite.get_parallelism_index()}: {confluent_versions}"
+    )
+
+    for confluent_version in confluent_versions:
         print(f"--- Testing Confluent Platform {confluent_version}")
         with c.override(
             Zookeeper(tag=confluent_version),