Merge branch 'pr-pw-0dt' of github.com:def-/materialize into 0dt-tests

MaterializeInc · Jul 17, 2024 · 300c87d · 300c87d
2 parents 26ce88b + 7e62732
commit 300c87d
Show file tree

Hide file tree

Showing 30 changed files with 441 additions and 220 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -116,6 +116,7 @@
 /src/sql/src/plan/expr.rs           @MaterializeInc/adapter @MaterializeInc/cluster
 # The lowering of HirRelationExpr to MirRelationExpr is part of the cluster
 # layer, despite being located in the `sql` crate.
+/src/sql/src/plan/lowering          @MaterializeInc/cluster
 /src/sql/src/plan/lowering.rs       @MaterializeInc/cluster
 /src/sql-lexer                      @MaterializeInc/adapter
 /src/sql-parser                     @MaterializeInc/adapter

diff --git a/LICENSE b/LICENSE
@@ -13,7 +13,7 @@ Business Source License 1.1
 
 Licensor:                  Materialize, Inc.
 
-Licensed Work:             Materialize Version 20240716
+Licensed Work:             Materialize Version 20240717
                            The Licensed Work is © 2024 Materialize, Inc.
 
 Additional Use Grant:      Within a single installation of Materialize, you
@@ -32,7 +32,7 @@ Additional Use Grant:      Within a single installation of Materialize, you
                            whose definitions are controlled by such third
                            parties.
 
-Change Date:               July 16, 2028
+Change Date:               July 17, 2028
 
 Change License:            Apache License, Version 2.0
 

diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml
@@ -1408,7 +1408,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: parallel-workload
-              args: [--runtime=1500, --scenario=kill, --threads=16]
+              args: [--runtime=1500, --scenario=kill, --threads=8]
 
       - id: parallel-workload-backup-restore
         label: "Parallel Workload (backup & restore)"
@@ -1423,6 +1423,18 @@ steps:
               composition: parallel-workload
               args: [--runtime=1500, --scenario=backup-restore, --naughty-identifiers, --threads=16]
 
+      - id: parallel-workload-0dt
+        label: "Parallel Workload (0dt deploy)"
+        depends_on: build-aarch64
+        artifact_paths: [parallel-workload-queries.log.zst]
+        timeout_in_minutes: 60
+        agents:
+          queue: linux-aarch64-large
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: parallel-workload
+              args: [--runtime=1500, --scenario=0dt-deploy, --threads=16]
+
   - id: incident-70
     label: "Test for incident 70"
     depends_on: build-aarch64

diff --git a/misc/python/materialize/cli/ci_annotate_errors.py b/misc/python/materialize/cli/ci_annotate_errors.py
@@ -146,6 +146,9 @@
     # For 0dt upgrades
     | halting\ process:\ unable\ to\ confirm\ leadership
     | halting\ process:\ fenced\ out\ old\ deployment;\ rebooting\ as\ leader
+    | parallel-workload-.*\ halting\ process:\ this\ deployment\ has\ been\ fenced\ out
+    | parallel-workload-.*\ fenced\ by\ new\ catalog\ upper
+    | parallel-workload-.*\ fenced\ by\ new\ catalog\ epoch
     )
     """,
     re.VERBOSE | re.MULTILINE,

diff --git a/misc/python/materialize/data_ingest/executor.py b/misc/python/materialize/data_ingest/executor.py
@@ -8,6 +8,7 @@
 # by the Apache License, Version 2.0.
 
 import json
+import random
 import time
 from typing import Any
 
@@ -62,7 +63,13 @@ def __init__(
     def reconnect(self) -> None:
         self.mz_conn = pg8000.connect(
             host="localhost",
-            port=self.ports["materialized"],
+            port=self.ports[
+                (
+                    random.choice(["materialized", "materialized2"])
+                    if "materialized2" in self.ports
+                    else "materialized"
+                )
+            ],
             user="materialize",
             database=self.database,
         )

diff --git a/misc/python/materialize/data_ingest/workload.py b/misc/python/materialize/data_ingest/workload.py
@@ -84,6 +84,23 @@ def __init__(self, composition: Composition | None) -> None:
             self.cycle.append(RestartMz(composition, probability=0.1))
 
 
+class SingleSensorUpdating0dtDeploy(Workload):
+    def __init__(self, composition: Composition | None) -> None:
+        self.cycle = [
+            TransactionDef(
+                [
+                    Upsert(
+                        keyspace=Keyspace.SINGLE_VALUE,
+                        count=Records.ONE,
+                        record_size=RecordSize.SMALL,
+                    ),
+                ]
+            ),
+        ]
+        if composition:
+            self.cycle.append(ZeroDowntimeDeploy(composition, probability=0.1))
+
+
 class DeleteDataAtEndOfDay(Workload):
     def __init__(self, composition: Composition | None) -> None:
         insert = Insert(
@@ -139,6 +156,35 @@ def __init__(self, composition: Composition | None) -> None:
             self.cycle.append(RestartMz(composition, probability=0.1))
 
 
+class DeleteDataAtEndOfDay0dtDeploys(Workload):
+    def __init__(self, composition: Composition | None) -> None:
+        insert = Insert(
+            count=Records.SOME,
+            record_size=RecordSize.SMALL,
+        )
+        insert_phase = TransactionDef(
+            size=TransactionSize.HUGE,
+            operations=[insert],
+        )
+        # Delete all records in a single transaction
+        delete_phase = TransactionDef(
+            [
+                Delete(
+                    number_of_records=Records.ALL,
+                    record_size=RecordSize.SMALL,
+                    num=insert.max_key(),
+                )
+            ]
+        )
+        self.cycle = [
+            insert_phase,
+            delete_phase,
+        ]
+
+        if composition:
+            self.cycle.append(ZeroDowntimeDeploy(composition, probability=0.1))
+
+
 # TODO: Implement
 # class ProgressivelyEnrichRecords(Workload):
 #    def __init__(self) -> None:

diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py
@@ -59,6 +59,7 @@
     "compute_dataflow_max_inflight_bytes": "134217728",  # 128 MiB
     "compute_hydration_concurrency": 2,
     "disk_cluster_replicas_default": "true",
+    "enable_0dt_deployment": "true",
     "enable_alter_swap": "true",
     "enable_assert_not_null": "true",
     "enable_columnation_lgalloc": "true",

diff --git a/misc/python/materialize/output_consistency/input_data/operations/all_operations_provider.py b/misc/python/materialize/output_consistency/input_data/operations/all_operations_provider.py
@@ -45,6 +45,9 @@
 from materialize.output_consistency.input_data.operations.number_operations_provider import (
     NUMERIC_OPERATION_TYPES,
 )
+from materialize.output_consistency.input_data.operations.pg_operations_provider import (
+    PG_OPERATIONS,
+)
 from materialize.output_consistency.input_data.operations.range_operations_provider import (
     RANGE_OPERATION_TYPES,
 )
@@ -54,9 +57,6 @@
 from materialize.output_consistency.input_data.operations.set_operations_provider import (
     SET_OPERATION_TYPES,
 )
-from materialize.output_consistency.input_data.operations.special_operations_provider import (
-    SPECIAL_OPERATION_TYPES,
-)
 from materialize.output_consistency.input_data.operations.string_operations_provider import (
     STRING_OPERATION_TYPES,
 )
@@ -88,6 +88,6 @@
         RANGE_OPERATION_TYPES,
         UUID_OPERATION_TYPES,
         RECORD_OPERATION_TYPES,
-        SPECIAL_OPERATION_TYPES,
+        PG_OPERATIONS,
     )
 )
diff --git a/...operations/special_operations_provider.py → ...data/operations/pg_operations_provider.py b/...operations/special_operations_provider.py → ...data/operations/pg_operations_provider.py
@@ -6,9 +6,13 @@
 # As of the Change Date specified in that file, in accordance with
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
+from materialize.mz_version import MzVersion
 from materialize.output_consistency.input_data.params.any_operation_param import (
     AnyOperationParam,
 )
+from materialize.output_consistency.input_data.params.number_operation_param import (
+    NumericOperationParam,
+)
 from materialize.output_consistency.input_data.return_specs.string_return_spec import (
     StringReturnTypeSpec,
 )
@@ -17,12 +21,21 @@
     DbOperationOrFunction,
 )
 
-SPECIAL_OPERATION_TYPES: list[DbOperationOrFunction] = []
+PG_OPERATIONS: list[DbOperationOrFunction] = []
 
-SPECIAL_OPERATION_TYPES.append(
+PG_OPERATIONS.append(
     DbFunction(
         "pg_typeof",
         [AnyOperationParam()],
         StringReturnTypeSpec(),
     )
 )
+
+PG_OPERATIONS.append(
+    DbFunction(
+        "pg_size_pretty",
+        [NumericOperationParam()],
+        StringReturnTypeSpec(),
+        since_mz_version=MzVersion.parse_mz("v0.109.0"),
+    )
+)
diff --git a/misc/python/materialize/output_consistency/input_data/types/number_types_provider.py b/misc/python/materialize/output_consistency/input_data/types/number_types_provider.py
@@ -73,6 +73,8 @@ def resolve_return_type_spec(
 REAL_TYPE_IDENTIFIER = "REAL"
 DOUBLE_TYPE_IDENTIFIER = "DOUBLE"
 
+DECIMAL_TYPE_IDENTIFIERS = {DECIMAL_39_0_TYPE_IDENTIFIER, DECIMAL_39_8_TYPE_IDENTIFIER}
+
 INT2_TYPE = NumberDataType(
     INT2_TYPE_IDENTIFIER,
     "INT2",