From 1a408033daf458f1ceebbe14a560355a1a2c0a70 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 26 Feb 2024 20:27:41 -0800 Subject: [PATCH] [SPARK-47181][CORE][TESTS] Fix `MasterSuite` to validate the number of registered workers ### What changes were proposed in this pull request? This PR aims to fix `MasterSuite` to validate the number of registered workers during `SPARK-46881: scheduling with workerSelectionPolicy *` tests. ### Why are the changes needed? To fix a flakiness. - https://github.com/apache/spark/actions/runs/8042308713/job/21962794853#step:10:17224 ``` [info] - SPARK-46881: scheduling with workerSelectionPolicy - CORES_FREE_DESC (false) *** FAILED *** (178 milliseconds) [info] List("10004") did not equal List("10005") (MasterSuite.scala:728) [info] Analysis: [info] List(0: "10004" -> "10005") ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #45274 from dongjoon-hyun/SPARK-47181. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../scala/org/apache/spark/deploy/master/MasterSuite.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala index 0db58ae0c834e..9992c2020f27e 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala @@ -705,7 +705,7 @@ class MasterSuite extends SparkFunSuite // Use different core and memory values to simplify the tests MockWorker.counter.set(10000) - (1 to 5).map { idx => + (1 to 5).foreach { idx => val worker = new MockWorker(master.self, conf) worker.rpcEnv.setupEndpoint(s"worker-$idx", worker) val workerReg = RegisterWorker( @@ -718,7 +718,9 @@ class MasterSuite extends SparkFunSuite "http://localhost:8080", RpcAddress("localhost", 10000)) master.self.send(workerReg) - worker + eventually(timeout(10.seconds)) { + assert(master.self.askSync[MasterStateResponse](RequestMasterState).workers.size === idx) + } } // An application with two executors