Use hash join when writing sparkey (#5402)

spotify · Jul 8, 2024 · c915a53 · c915a53
1 parent 9282396
commit c915a53
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/scio-extra/src/main/scala/com/spotify/scio/extra/sparkey/SparkeyIO.scala b/scio-extra/src/main/scala/com/spotify/scio/extra/sparkey/SparkeyIO.scala
@@ -124,7 +124,7 @@ object SparkeyIO {
 
       // write files to temporary locations
       val tempShardUris = shards
-        .rightOuterJoin(allShards)
+        .hashFullOuterJoin(allShards)
         .map { case (shard, (xs, _)) =>
           // use a temp uri so that if a bundle fails retries will not fail
           val tempUri = SparkeyUri(s"$tempPath/${UUID.randomUUID}")