Merge pull request #24195 from def-/pr-parallel-workload-drop

parallel-workload: Fix locking for dropping replica & Ignore 404 in toggle-persist-txn scenario

misc/python/materialize/parallel_workload/action.py

@@ -1029,10 +1029,13 @@ def run(self, exe: Executor) -> bool:
             if len(cluster.replicas) <= 1:
                 return False
             replica = self.rng.choice(cluster.replicas)
-        with replica.lock:
+
+        with cluster.lock, replica.lock:
             # Was dropped while we were acquiring lock
             if replica not in cluster.replicas:
                 return False
+            if cluster not in exe.db.clusters:
+                return False
             # Avoid "has no replicas available to service request" error
             if len(cluster.replicas) <= 1:
                 return False
@@ -1623,9 +1626,10 @@ def run(self, exe: Executor) -> bool:
                     raise
             except QueryError as e:
                 # expected, see #20465
-                if exe.db.scenario != Scenario.Kill or (
-                    "404: no object was found at the path" not in e.msg
-                ):
+                if exe.db.scenario not in (
+                    Scenario.Kill,
+                    Scenario.TogglePersistTxn,
+                ) or ("404: no object was found at the path" not in e.msg):
                     raise e
         return True