From 96424a720469dea023381dabb520d62c9f4acc94 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sat, 14 Mar 2026 12:53:07 -0400 Subject: [PATCH 01/16] Coalesce broadcast exchange batches before broadcasting CometBroadcastExchangeExec previously broadcast an Array[ChunkedByteBuffer] with one entry per source partition. Each consumer partition independently deserialized all entries, creating a separate compression codec and Arrow IPC reader per entry. For broadcasts with many source partitions, this produced large per-task overhead in the hash join build-side collection. Decode and concatenate all broadcast batches into a single ChunkedByteBuffer on the driver using VectorSchemaRootAppender before broadcasting. Falls back to per-batch serialization if dictionary-encoded vectors are present. --- .../apache/spark/sql/comet/util/Utils.scala | 73 ++++++++++++++++++- .../comet/CometBroadcastExchangeExec.scala | 5 +- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index 6eaa9cad44a..297c1e8aa66 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -32,7 +32,9 @@ import org.apache.arrow.vector.dictionary.DictionaryProvider import org.apache.arrow.vector.ipc.ArrowStreamWriter import org.apache.arrow.vector.types._ import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema} +import org.apache.arrow.vector.util.VectorSchemaRootAppender import org.apache.spark.{SparkEnv, SparkException} +import org.apache.spark.internal.Logging import org.apache.spark.io.CompressionCodec import org.apache.spark.sql.comet.execution.arrow.ArrowReaderIterator import org.apache.spark.sql.types._ @@ -43,7 +45,7 @@ import org.apache.comet.Constants.COMET_CONF_DIR_ENV import org.apache.comet.shims.CometTypeShim import org.apache.comet.vector.CometVector -object Utils extends CometTypeShim { +object Utils extends CometTypeShim with Logging { def getConfPath(confFileName: String): String = { sys.env .get(COMET_CONF_DIR_ENV) @@ -252,6 +254,75 @@ object Utils extends CometTypeShim { new ArrowReaderIterator(Channels.newChannel(ins), source) } + /** + * Coalesces many small ChunkedByteBuffers (one per source partition) into a single + * ChunkedByteBuffer containing one Arrow IPC stream with one record batch. This avoids each + * consumer partition having to deserialize N separate streams. + */ + def coalesceBroadcastBatches(input: Iterator[ChunkedByteBuffer]): Array[ChunkedByteBuffer] = { + val decoded = input.flatMap(decodeBatches(_, "broadcast-coalesce")).toArray + if (decoded.isEmpty) { + return Array.empty + } + + try { + var hasDictionary = false + val sourceRoots = decoded.map { batch => + val (fieldVectors, providerOpt) = getBatchFieldVectors(batch) + if (providerOpt.isDefined) { + hasDictionary = true + } + new VectorSchemaRoot(fieldVectors.asJava) + } + + // Fall back to per-batch serialization if any batch has dictionary-encoded vectors, + // since merging dictionaries across batches is not supported. + if (hasDictionary) { + logInfo( + s"Broadcast coalesce falling back to per-batch serialization due to " + + s"dictionary-encoded vectors (${decoded.length} batches)") + return decoded.flatMap { batch => + serializeBatches(Iterator(batch)).map(_._2) + } + } + + val allocator = org.apache.comet.CometArrowAllocator + .newChildAllocator("broadcast-coalesce", 0, Long.MaxValue) + try { + val schema = sourceRoots.head.getSchema + val targetRoot = VectorSchemaRoot.create(schema, allocator) + try { + VectorSchemaRootAppender.append(targetRoot, sourceRoots: _*) + + val expectedRows = decoded.map(_.numRows().toLong).sum + assert( + targetRoot.getRowCount.toLong == expectedRows, + s"Row count mismatch after coalesce: ${targetRoot.getRowCount} != $expectedRows") + + logInfo( + s"Coalesced ${decoded.length} broadcast batches into 1 " + + s"($expectedRows rows)") + + val codec = CompressionCodec.createCodec(SparkEnv.get.conf) + val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) + val out = new DataOutputStream(codec.compressedOutputStream(cbbos)) + val writer = new ArrowStreamWriter(targetRoot, null, Channels.newChannel(out)) + writer.start() + writer.writeBatch() + writer.close() + + Array(cbbos.toChunkedByteBuffer) + } finally { + targetRoot.close() + } + } finally { + allocator.close() + } + } finally { + decoded.foreach(_.close()) + } + } + def getBatchFieldVectors( batch: ColumnarBatch): (Seq[FieldVector], Option[DictionaryProvider]) = { var provider: Option[DictionaryProvider] = None diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala index f40e05ea0cf..ea4e8b1405e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala @@ -155,7 +155,10 @@ case class CometBroadcastExchangeExec( val beforeBuild = System.nanoTime() longMetric("collectTime") += NANOSECONDS.toMillis(beforeBuild - beforeCollect) - val batches = input.toArray + // Coalesce many small per-partition buffers into a single buffer so each + // consumer partition only deserializes one Arrow IPC stream. + // May produce multiple buffers if dictionary-encoded vectors are present. + val batches = Utils.coalesceBroadcastBatches(input) val dataSize = batches.map(_.size).sum From a51bc8b1709617b3b3ab4985851fafbcb5cb1230 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sat, 14 Mar 2026 13:05:33 -0400 Subject: [PATCH 02/16] Fix scalastyle. --- .../src/main/scala/org/apache/spark/sql/comet/util/Utils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index 297c1e8aa66..28943f5ad17 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -279,7 +279,7 @@ object Utils extends CometTypeShim with Logging { // since merging dictionaries across batches is not supported. if (hasDictionary) { logInfo( - s"Broadcast coalesce falling back to per-batch serialization due to " + + "Broadcast coalesce falling back to per-batch serialization due to " + s"dictionary-encoded vectors (${decoded.length} batches)") return decoded.flatMap { batch => serializeBatches(Iterator(batch)).map(_._2) From 43d3920c9350f96121bcb099f396c179d15f5a18 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sun, 15 Mar 2026 16:06:31 -0400 Subject: [PATCH 03/16] Add test, change implementation. --- .../apache/spark/sql/comet/util/Utils.scala | 92 +++++++++---------- .../comet/CometBroadcastExchangeExec.scala | 1 - .../apache/comet/exec/CometJoinSuite.scala | 55 +++++++++++ 3 files changed, 101 insertions(+), 47 deletions(-) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index 28943f5ad17..525d7c17599 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -29,7 +29,7 @@ import org.apache.arrow.c.CDataDictionaryProvider import org.apache.arrow.vector.{BigIntVector, BitVector, DateDayVector, DecimalVector, FieldVector, FixedSizeBinaryVector, Float4Vector, Float8Vector, IntVector, NullVector, SmallIntVector, TimeStampMicroTZVector, TimeStampMicroVector, TinyIntVector, ValueVector, VarBinaryVector, VarCharVector, VectorSchemaRoot} import org.apache.arrow.vector.complex.{ListVector, MapVector, StructVector} import org.apache.arrow.vector.dictionary.DictionaryProvider -import org.apache.arrow.vector.ipc.ArrowStreamWriter +import org.apache.arrow.vector.ipc.{ArrowStreamReader, ArrowStreamWriter} import org.apache.arrow.vector.types._ import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema} import org.apache.arrow.vector.util.VectorSchemaRootAppender @@ -260,66 +260,66 @@ object Utils extends CometTypeShim with Logging { * consumer partition having to deserialize N separate streams. */ def coalesceBroadcastBatches(input: Iterator[ChunkedByteBuffer]): Array[ChunkedByteBuffer] = { - val decoded = input.flatMap(decodeBatches(_, "broadcast-coalesce")).toArray - if (decoded.isEmpty) { + val buffers = input.filterNot(_.size == 0).toArray + if (buffers.isEmpty) { return Array.empty } + val allocator = org.apache.comet.CometArrowAllocator + .newChildAllocator("broadcast-coalesce", 0, Long.MaxValue) try { - var hasDictionary = false - val sourceRoots = decoded.map { batch => - val (fieldVectors, providerOpt) = getBatchFieldVectors(batch) - if (providerOpt.isDefined) { - hasDictionary = true + var targetRoot: VectorSchemaRoot = null + var totalRows = 0L + var batchCount = 0 + + try { + for (bytes <- buffers) { + val codec = CompressionCodec.createCodec(SparkEnv.get.conf) + val cbbis = bytes.toInputStream() + val ins = new DataInputStream(codec.compressedInputStream(cbbis)) + val channel = Channels.newChannel(ins) + val reader = new ArrowStreamReader(channel, allocator) + try { + while (reader.loadNextBatch()) { + val sourceRoot = reader.getVectorSchemaRoot + if (targetRoot == null) { + targetRoot = VectorSchemaRoot.create(sourceRoot.getSchema, allocator) + } + VectorSchemaRootAppender.append(targetRoot, sourceRoot) + totalRows += sourceRoot.getRowCount + batchCount += 1 + } + } finally { + reader.close() + } } - new VectorSchemaRoot(fieldVectors.asJava) - } - // Fall back to per-batch serialization if any batch has dictionary-encoded vectors, - // since merging dictionaries across batches is not supported. - if (hasDictionary) { - logInfo( - "Broadcast coalesce falling back to per-batch serialization due to " + - s"dictionary-encoded vectors (${decoded.length} batches)") - return decoded.flatMap { batch => - serializeBatches(Iterator(batch)).map(_._2) + if (targetRoot == null) { + return Array.empty } - } - val allocator = org.apache.comet.CometArrowAllocator - .newChildAllocator("broadcast-coalesce", 0, Long.MaxValue) - try { - val schema = sourceRoots.head.getSchema - val targetRoot = VectorSchemaRoot.create(schema, allocator) - try { - VectorSchemaRootAppender.append(targetRoot, sourceRoots: _*) + assert( + targetRoot.getRowCount.toLong == totalRows, + s"Row count mismatch after coalesce: ${targetRoot.getRowCount} != $totalRows") - val expectedRows = decoded.map(_.numRows().toLong).sum - assert( - targetRoot.getRowCount.toLong == expectedRows, - s"Row count mismatch after coalesce: ${targetRoot.getRowCount} != $expectedRows") + logInfo(s"Coalesced $batchCount broadcast batches into 1 ($totalRows rows)") - logInfo( - s"Coalesced ${decoded.length} broadcast batches into 1 " + - s"($expectedRows rows)") + val outCodec = CompressionCodec.createCodec(SparkEnv.get.conf) + val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) + val out = new DataOutputStream(outCodec.compressedOutputStream(cbbos)) + val writer = new ArrowStreamWriter(targetRoot, null, Channels.newChannel(out)) + writer.start() + writer.writeBatch() + writer.close() - val codec = CompressionCodec.createCodec(SparkEnv.get.conf) - val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) - val out = new DataOutputStream(codec.compressedOutputStream(cbbos)) - val writer = new ArrowStreamWriter(targetRoot, null, Channels.newChannel(out)) - writer.start() - writer.writeBatch() - writer.close() - - Array(cbbos.toChunkedByteBuffer) - } finally { + Array(cbbos.toChunkedByteBuffer) + } finally { + if (targetRoot != null) { targetRoot.close() } - } finally { - allocator.close() } } finally { - decoded.foreach(_.close()) + allocator.close() } } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala index ea4e8b1405e..291006f356b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala @@ -157,7 +157,6 @@ case class CometBroadcastExchangeExec( // Coalesce many small per-partition buffers into a single buffer so each // consumer partition only deserializes one Arrow IPC stream. - // May produce multiple buffers if dictionary-encoded vectors are present. val batches = Utils.coalesceBroadcastBatches(input) val dataSize = batches.map(_.size).sum diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index d5a8387be70..8de66b8d2f9 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -383,4 +383,59 @@ class CometJoinSuite extends CometTestBase { """.stripMargin)) } } + + test("Broadcast hash join build-side batch coalescing") { + // Use many shuffle partitions to produce many small broadcast batches, + // then verify that coalescing reduces the build-side batch count to 1 per task. + val numPartitions = 512 + withSQLConf( + CometConf.COMET_BATCH_SIZE.key -> "100", + SQLConf.PREFER_SORTMERGEJOIN.key -> "false", + "spark.sql.join.forceApplyShuffledHashJoin" -> "true", + SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + SQLConf.SHUFFLE_PARTITIONS.key -> numPartitions.toString) { + withParquetTable((0 until 10000).map(i => (i, i % 5)), "tbl_a") { + withParquetTable((0 until 10000).map(i => (i % 10, i + 2)), "tbl_b") { + // Force a shuffle on tbl_a before broadcast so the broadcast source has + // numPartitions partitions, not just the number of parquet files. + val query = + s"""SELECT /*+ BROADCAST(a) */ * + |FROM (SELECT /*+ REPARTITION($numPartitions) */ * FROM tbl_a) a + |JOIN tbl_b ON a._2 = tbl_b._1""".stripMargin + + // First verify correctness + val df = sql(query) + checkSparkAnswerAndOperator( + df, + Seq(classOf[CometBroadcastExchangeExec], classOf[CometBroadcastHashJoinExec])) + + // Run again and check metrics on the executed plan + val df2 = sql(query) + df2.collect() + + val joins = collect(df2.queryExecution.executedPlan) { + case j: CometBroadcastHashJoinExec => j + } + assert(joins.nonEmpty, "Expected CometBroadcastHashJoinExec in plan") + + val join = joins.head + val buildBatches = join.metrics("build_input_batches").value + val buildRows = join.metrics("build_input_rows").value + + // Without coalescing, build_input_batches would be ~numPartitions per task, + // totaling ~numPartitions * numPartitions across all tasks. + // With coalescing, each task gets 1 batch, so total ≈ numPartitions. + // scalastyle:off println + println(s"Build-side metrics: batches=$buildBatches, rows=$buildRows") + // scalastyle:on println + assert( + buildBatches <= numPartitions, + s"Expected at most $numPartitions build batches (1 per task), got $buildBatches. " + + "Broadcast batch coalescing may not be working.") + } + } + } + } } From 80f117648c0130ceb8163e5201980d87b7d44eb9 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 10:56:24 -0400 Subject: [PATCH 04/16] Fix VectorSchemaRoot allocation. --- .../apache/spark/sql/comet/util/Utils.scala | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index 525d7c17599..951a3f352cf 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -256,8 +256,13 @@ object Utils extends CometTypeShim with Logging { /** * Coalesces many small ChunkedByteBuffers (one per source partition) into a single - * ChunkedByteBuffer containing one Arrow IPC stream with one record batch. This avoids each - * consumer partition having to deserialize N separate streams. + * ChunkedByteBuffer. Without coalescing, each consumer task in a broadcast hash join + * deserializes N separate Arrow IPC streams (one per source partition), which dominates + * build-side time when partition counts are high (e.g. 200+ partitions in TPC-H Q18). + * + * We decode and append all source batches into one VectorSchemaRoot on the driver, + * then re-serialize once via ArrowStreamWriter. This is done on the driver (not per-task) + * so the cost is paid once rather than once per consumer partition. */ def coalesceBroadcastBatches(input: Iterator[ChunkedByteBuffer]): Array[ChunkedByteBuffer] = { val buffers = input.filterNot(_.size == 0).toArray @@ -280,10 +285,18 @@ object Utils extends CometTypeShim with Logging { val channel = Channels.newChannel(ins) val reader = new ArrowStreamReader(channel, allocator) try { + // Dictionary-encoded columns would be unsafe to coalesce because each + // partition can have a different dictionary, and appending index vectors + // would silently mix indices from incompatible dictionaries. + // DataFusion decodes dictionaries during execution, so this shouldn't happen. + assert( + reader.getDictionaryVectors.isEmpty, + "Cannot coalesce dictionary-encoded broadcast batches") while (reader.loadNextBatch()) { val sourceRoot = reader.getVectorSchemaRoot if (targetRoot == null) { targetRoot = VectorSchemaRoot.create(sourceRoot.getSchema, allocator) + targetRoot.allocateNew() } VectorSchemaRootAppender.append(targetRoot, sourceRoot) totalRows += sourceRoot.getRowCount @@ -308,9 +321,12 @@ object Utils extends CometTypeShim with Logging { val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) val out = new DataOutputStream(outCodec.compressedOutputStream(cbbos)) val writer = new ArrowStreamWriter(targetRoot, null, Channels.newChannel(out)) - writer.start() - writer.writeBatch() - writer.close() + try { + writer.start() + writer.writeBatch() + } finally { + writer.close() + } Array(cbbos.toChunkedByteBuffer) } finally { From c2da31e903692225533a6ea4fa64764f5ad738f3 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 10:59:40 -0400 Subject: [PATCH 05/16] Spotless. --- .../main/scala/org/apache/spark/sql/comet/util/Utils.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index 951a3f352cf..be4e09e205a 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -260,9 +260,9 @@ object Utils extends CometTypeShim with Logging { * deserializes N separate Arrow IPC streams (one per source partition), which dominates * build-side time when partition counts are high (e.g. 200+ partitions in TPC-H Q18). * - * We decode and append all source batches into one VectorSchemaRoot on the driver, - * then re-serialize once via ArrowStreamWriter. This is done on the driver (not per-task) - * so the cost is paid once rather than once per consumer partition. + * We decode and append all source batches into one VectorSchemaRoot on the driver, then + * re-serialize once via ArrowStreamWriter. This is done on the driver (not per-task) so the + * cost is paid once rather than once per consumer partition. */ def coalesceBroadcastBatches(input: Iterator[ChunkedByteBuffer]): Array[ChunkedByteBuffer] = { val buffers = input.filterNot(_.size == 0).toArray From bfd3f6b1db93fdf50793fc269735b285ecca32ed Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 11:05:37 -0400 Subject: [PATCH 06/16] Spotless. --- .../src/main/scala/org/apache/spark/sql/comet/util/Utils.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index be4e09e205a..84138b2b7b7 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -320,6 +320,7 @@ object Utils extends CometTypeShim with Logging { val outCodec = CompressionCodec.createCodec(SparkEnv.get.conf) val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) val out = new DataOutputStream(outCodec.compressedOutputStream(cbbos)) + // null provider is safe here — we assert no dictionary-encoded columns above val writer = new ArrowStreamWriter(targetRoot, null, Channels.newChannel(out)) try { writer.start() From 12ded1d18c78b0554b5187db329e0419984d0ae7 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 11:29:44 -0400 Subject: [PATCH 07/16] Add metrics, spotless. --- .../org/apache/spark/sql/comet/util/Utils.scala | 14 ++++++++------ .../sql/comet/CometBroadcastExchangeExec.scala | 12 ++++++++++-- .../org/apache/comet/exec/CometJoinSuite.scala | 17 ++++++++++++++--- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index 84138b2b7b7..a3c97f15b3e 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -26,7 +26,7 @@ import java.nio.channels.Channels import scala.jdk.CollectionConverters._ import org.apache.arrow.c.CDataDictionaryProvider -import org.apache.arrow.vector.{BigIntVector, BitVector, DateDayVector, DecimalVector, FieldVector, FixedSizeBinaryVector, Float4Vector, Float8Vector, IntVector, NullVector, SmallIntVector, TimeStampMicroTZVector, TimeStampMicroVector, TinyIntVector, ValueVector, VarBinaryVector, VarCharVector, VectorSchemaRoot} +import org.apache.arrow.vector._ import org.apache.arrow.vector.complex.{ListVector, MapVector, StructVector} import org.apache.arrow.vector.dictionary.DictionaryProvider import org.apache.arrow.vector.ipc.{ArrowStreamReader, ArrowStreamWriter} @@ -234,6 +234,7 @@ object Utils extends CometTypeShim with Logging { /** * Decodes the byte arrays back to ColumnarBatchs and put them into buffer. + * * @param bytes * the serialized batches * @param source @@ -264,10 +265,11 @@ object Utils extends CometTypeShim with Logging { * re-serialize once via ArrowStreamWriter. This is done on the driver (not per-task) so the * cost is paid once rather than once per consumer partition. */ - def coalesceBroadcastBatches(input: Iterator[ChunkedByteBuffer]): Array[ChunkedByteBuffer] = { + def coalesceBroadcastBatches( + input: Iterator[ChunkedByteBuffer]): (Array[ChunkedByteBuffer], Long, Long) = { val buffers = input.filterNot(_.size == 0).toArray if (buffers.isEmpty) { - return Array.empty + return (Array.empty, 0L, 0L) } val allocator = org.apache.comet.CometArrowAllocator @@ -308,7 +310,7 @@ object Utils extends CometTypeShim with Logging { } if (targetRoot == null) { - return Array.empty + return (Array.empty, 0L, 0L) } assert( @@ -320,7 +322,7 @@ object Utils extends CometTypeShim with Logging { val outCodec = CompressionCodec.createCodec(SparkEnv.get.conf) val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) val out = new DataOutputStream(outCodec.compressedOutputStream(cbbos)) - // null provider is safe here — we assert no dictionary-encoded columns above + // null provider is safe here because we assert no dictionary-encoded columns above val writer = new ArrowStreamWriter(targetRoot, null, Channels.newChannel(out)) try { writer.start() @@ -329,7 +331,7 @@ object Utils extends CometTypeShim with Logging { writer.close() } - Array(cbbos.toChunkedByteBuffer) + (Array(cbbos.toChunkedByteBuffer), batchCount.toLong, totalRows) } finally { if (targetRoot != null) { targetRoot.close() diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala index 291006f356b..303a0a17a66 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala @@ -77,7 +77,13 @@ case class CometBroadcastExchangeExec( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "collectTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to collect"), "buildTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to build"), - "broadcastTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to broadcast")) + "broadcastTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to broadcast"), + "numCoalescedBatches" -> SQLMetrics.createMetric( + sparkContext, + "number of coalesced batches for broadcast"), + "numCoalescedRows" -> SQLMetrics.createMetric( + sparkContext, + "number of coalesced rows for broadcast")) override def doCanonicalize(): SparkPlan = { CometBroadcastExchangeExec(null, null, mode, child.canonicalized) @@ -157,7 +163,9 @@ case class CometBroadcastExchangeExec( // Coalesce many small per-partition buffers into a single buffer so each // consumer partition only deserializes one Arrow IPC stream. - val batches = Utils.coalesceBroadcastBatches(input) + val (batches, coalescedBatches, coalescedRows) = Utils.coalesceBroadcastBatches(input) + longMetric("numCoalescedBatches") += coalescedBatches + longMetric("numCoalescedRows") += coalescedRows val dataSize = batches.map(_.size).sum diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index 8de66b8d2f9..fe23f4870df 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -427,13 +427,24 @@ class CometJoinSuite extends CometTestBase { // Without coalescing, build_input_batches would be ~numPartitions per task, // totaling ~numPartitions * numPartitions across all tasks. // With coalescing, each task gets 1 batch, so total ≈ numPartitions. - // scalastyle:off println - println(s"Build-side metrics: batches=$buildBatches, rows=$buildRows") - // scalastyle:on println assert( buildBatches <= numPartitions, s"Expected at most $numPartitions build batches (1 per task), got $buildBatches. " + "Broadcast batch coalescing may not be working.") + + val broadcasts = collect(df2.queryExecution.executedPlan) { + case b: CometBroadcastExchangeExec => b + } + assert(broadcasts.nonEmpty, "Expected CometBroadcastExchangeExec in plan") + + val broadcast = broadcasts.head + val coalescedBatches = broadcast.metrics("numCoalescedBatches").value + val coalescedRows = broadcast.metrics("numCoalescedRows").value + + assert( + coalescedBatches >= numPartitions, + s"Expected at least $numPartitions coalesced batches, got $coalescedBatches") + assert(coalescedRows == 10000, s"Expected 10000 coalesced rows, got $coalescedRows") } } } From ba9fad8b9fb09ba36dd6e5a8255bcec0314ce356 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 11:55:47 -0400 Subject: [PATCH 08/16] Fix unused variable in test. --- .../src/test/scala/org/apache/comet/exec/CometJoinSuite.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index fe23f4870df..d29363774f3 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -444,7 +444,9 @@ class CometJoinSuite extends CometTestBase { assert( coalescedBatches >= numPartitions, s"Expected at least $numPartitions coalesced batches, got $coalescedBatches") - assert(coalescedRows == 10000, s"Expected 10000 coalesced rows, got $coalescedRows") + assert( + coalescedRows == buildRows, + s"Expected $buildRows coalesced rows, got $coalescedRows") } } } From ebe85fa96e5ef977e43e1677a4f7bdb7870e6242 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 12:05:21 -0400 Subject: [PATCH 09/16] Fix test. --- .../apache/comet/exec/CometJoinSuite.scala | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index d29363774f3..60a658ebde5 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf class CometJoinSuite extends CometTestBase { + import testImplicits._ override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit @@ -359,27 +360,27 @@ class CometJoinSuite extends CometTestBase { checkSparkAnswer(left.join(right, ($"left.N" === $"right.N") && ($"right.N" =!= 3), "full")) checkSparkAnswer(sql(""" - |SELECT l.a, count(*) - |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N) - |GROUP BY l.a + |SELECT l.a, count(*) + |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N) + |GROUP BY l.a """.stripMargin)) checkSparkAnswer(sql(""" - |SELECT r.N, count(*) - |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N) - |GROUP BY r.N + |SELECT r.N, count(*) + |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N) + |GROUP BY r.N """.stripMargin)) checkSparkAnswer(sql(""" - |SELECT l.N, count(*) - |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a) - |GROUP BY l.N + |SELECT l.N, count(*) + |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a) + |GROUP BY l.N """.stripMargin)) checkSparkAnswer(sql(""" - |SELECT r.a, count(*) - |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a) - |GROUP BY r.a + |SELECT r.a, count(*) + |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a) + |GROUP BY r.a """.stripMargin)) } } @@ -422,7 +423,6 @@ class CometJoinSuite extends CometTestBase { val join = joins.head val buildBatches = join.metrics("build_input_batches").value - val buildRows = join.metrics("build_input_rows").value // Without coalescing, build_input_batches would be ~numPartitions per task, // totaling ~numPartitions * numPartitions across all tasks. @@ -444,9 +444,7 @@ class CometJoinSuite extends CometTestBase { assert( coalescedBatches >= numPartitions, s"Expected at least $numPartitions coalesced batches, got $coalescedBatches") - assert( - coalescedRows == buildRows, - s"Expected $buildRows coalesced rows, got $coalescedRows") + assert(coalescedRows == 10000, s"Expected 10000 coalesced rows, got $coalescedRows") } } } From af4876b5190a24e62fa45fa391e97689336a4700 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 12:53:51 -0400 Subject: [PATCH 10/16] Fix test "Comet native metrics: BroadcastHashJoin". --- .../src/test/scala/org/apache/comet/exec/CometExecSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 0bf9bbc95be..3001aeb6e50 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo, He import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateMode, BloomFilterAggregate} import org.apache.spark.sql.comet._ import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometShuffleExchangeExec} -import org.apache.spark.sql.execution.{CollectLimitExec, ProjectExec, SparkPlan, SQLExecution, UnionExec} +import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, BroadcastQueryStageExec} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec} @@ -712,7 +712,7 @@ class CometExecSuite extends CometTestBase { assert(metrics.contains("build_time")) assert(metrics("build_time").value > 1L) assert(metrics.contains("build_input_batches")) - assert(metrics("build_input_batches").value == 25L) + assert(metrics("build_input_batches").value == 5L) assert(metrics.contains("build_mem_used")) assert(metrics("build_mem_used").value > 1L) assert(metrics.contains("build_input_rows")) From 5ce757c58653f72b0043266249e50fbc5735b6e6 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 13:36:57 -0400 Subject: [PATCH 11/16] Add metrics checking to CometExecSuite "CometBroadcastExchangeExec". --- .../src/test/scala/org/apache/comet/exec/CometExecSuite.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 3001aeb6e50..938bb8511ce 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -474,6 +474,10 @@ class CometExecSuite extends CometTestBase { val expected = (0 until numParts).flatMap(_ => (0 until 5).map(i => i + 1)).sorted assert(rowContents === expected) + + val metrics = nativeBroadcast.metrics + assert(metrics("numCoalescedBatches").value == 5L) + assert(metrics("numCoalescedRows").value == 5L) } } } From d7326815cdb47e0637774503d0db8e35c95c461c Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 13:53:05 -0400 Subject: [PATCH 12/16] Add metrics checking to CometExecSuite "CometBroadcastExchangeExec: empty broadcast". --- .../src/test/scala/org/apache/comet/exec/CometExecSuite.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 938bb8511ce..aff18162657 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -497,6 +497,10 @@ class CometExecSuite extends CometTestBase { }.get.asInstanceOf[CometBroadcastExchangeExec] val rows = nativeBroadcast.executeCollect() assert(rows.isEmpty) + + val metrics = nativeBroadcast.metrics + assert(metrics("numCoalescedBatches").value == 0L) + assert(metrics("numCoalescedRows").value == 0L) } } } From 128bfdb061b8f8b4ec2d5f41b6d8211c7dda0276 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 14:00:10 -0400 Subject: [PATCH 13/16] Fallback and emit a warning if we hit a dictionary-encoded column. --- .../apache/spark/sql/comet/util/Utils.scala | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index a3c97f15b3e..b1fdc71fde8 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -287,13 +287,21 @@ object Utils extends CometTypeShim with Logging { val channel = Channels.newChannel(ins) val reader = new ArrowStreamReader(channel, allocator) try { - // Dictionary-encoded columns would be unsafe to coalesce because each + // Comet decodes dictionaries during execution, so this shouldn't happen. + // If it does, fall back to the original uncoalesced buffers because each // partition can have a different dictionary, and appending index vectors // would silently mix indices from incompatible dictionaries. - // DataFusion decodes dictionaries during execution, so this shouldn't happen. - assert( - reader.getDictionaryVectors.isEmpty, - "Cannot coalesce dictionary-encoded broadcast batches") + if (!reader.getDictionaryVectors.isEmpty) { + logWarning( + "Unexpected dictionary-encoded column during BroadcastExchange coalescing; " + + "skipping coalesce") + reader.close() + if (targetRoot != null) { + targetRoot.close() + targetRoot = null + } + return (buffers, 0L, 0L) + } while (reader.loadNextBatch()) { val sourceRoot = reader.getVectorSchemaRoot if (targetRoot == null) { @@ -322,7 +330,7 @@ object Utils extends CometTypeShim with Logging { val outCodec = CompressionCodec.createCodec(SparkEnv.get.conf) val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) val out = new DataOutputStream(outCodec.compressedOutputStream(cbbos)) - // null provider is safe here because we assert no dictionary-encoded columns above + // null provider is safe here because we check for dictionary-encoded columns above val writer = new ArrowStreamWriter(targetRoot, null, Channels.newChannel(out)) try { writer.start() From 79f1ec31fe24f8a93188e4b1f7f841edf96ea92a Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 16:37:18 -0400 Subject: [PATCH 14/16] Update docs. --- .../apache/spark/sql/comet/util/Utils.scala | 21 ++++++++++++------- .../comet/CometBroadcastExchangeExec.scala | 7 +++++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index b1fdc71fde8..ae83f7a23dd 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -256,14 +256,21 @@ object Utils extends CometTypeShim with Logging { } /** - * Coalesces many small ChunkedByteBuffers (one per source partition) into a single - * ChunkedByteBuffer. Without coalescing, each consumer task in a broadcast hash join - * deserializes N separate Arrow IPC streams (one per source partition), which dominates - * build-side time when partition counts are high (e.g. 200+ partitions in TPC-H Q18). + * Coalesces many small Arrow IPC batches into a single batch for broadcasting. * - * We decode and append all source batches into one VectorSchemaRoot on the driver, then - * re-serialize once via ArrowStreamWriter. This is done on the driver (not per-task) so the - * cost is paid once rather than once per consumer partition. + * Why this is necessary: The broadcast exchange collects shuffle output by calling + * getByteArrayRdd, which serializes each ColumnarBatch independently into its own + * ChunkedByteBuffer. The shuffle reader (CometBlockStoreShuffleReader) produces one + * ColumnarBatch per shuffle block, and there is one block per writer task per output partition. + * So with W writer tasks and P output partitions, the broadcast collects up to W * P tiny + * batches. For example, with 400 writer tasks and 500 partitions, 1M rows would arrive as ~200K + * batches of ~5 rows each. + * + * Without coalescing, every consumer task in the broadcast join would independently deserialize + * all of these tiny Arrow IPC streams, paying per-stream overhead (schema parsing, buffer + * allocation) for each one. With coalescing, we decode and append all batches into one + * VectorSchemaRoot on the driver, then re-serialize once. Each consumer task then deserializes + * a single Arrow IPC stream. */ def coalesceBroadcastBatches( input: Iterator[ChunkedByteBuffer]): (Array[ChunkedByteBuffer], Long, Long) = { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala index 303a0a17a66..4a323e575a8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala @@ -161,8 +161,11 @@ case class CometBroadcastExchangeExec( val beforeBuild = System.nanoTime() longMetric("collectTime") += NANOSECONDS.toMillis(beforeBuild - beforeCollect) - // Coalesce many small per-partition buffers into a single buffer so each - // consumer partition only deserializes one Arrow IPC stream. + // Coalesce the many small per-shuffle-block buffers into a single buffer. + // Without this, each consumer task deserializes one Arrow IPC stream per + // shuffle block (one per writer task per partition), which is very expensive + // when there are hundreds of writer tasks and partitions. See the scaladoc + // on coalesceBroadcastBatches for details. val (batches, coalescedBatches, coalescedRows) = Utils.coalesceBroadcastBatches(input) longMetric("numCoalescedBatches") += coalescedBatches longMetric("numCoalescedRows") += coalescedRows From 3fdd14baeef4c5adf2d564f467ebb2f36380a9ff Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 16:47:30 -0400 Subject: [PATCH 15/16] Address PR feedback. --- .../apache/spark/sql/comet/util/Utils.scala | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index ae83f7a23dd..78f2e81c7c6 100644 --- a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -286,13 +286,13 @@ object Utils extends CometTypeShim with Logging { var totalRows = 0L var batchCount = 0 + val codec = CompressionCodec.createCodec(SparkEnv.get.conf) try { for (bytes <- buffers) { - val codec = CompressionCodec.createCodec(SparkEnv.get.conf) - val cbbis = bytes.toInputStream() - val ins = new DataInputStream(codec.compressedInputStream(cbbis)) - val channel = Channels.newChannel(ins) - val reader = new ArrowStreamReader(channel, allocator) + val compressedInputStream = + new DataInputStream(codec.compressedInputStream(bytes.toInputStream())) + val reader = + new ArrowStreamReader(Channels.newChannel(compressedInputStream), allocator) try { // Comet decodes dictionaries during execution, so this shouldn't happen. // If it does, fall back to the original uncoalesced buffers because each @@ -334,11 +334,11 @@ object Utils extends CometTypeShim with Logging { logInfo(s"Coalesced $batchCount broadcast batches into 1 ($totalRows rows)") - val outCodec = CompressionCodec.createCodec(SparkEnv.get.conf) - val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) - val out = new DataOutputStream(outCodec.compressedOutputStream(cbbos)) - // null provider is safe here because we check for dictionary-encoded columns above - val writer = new ArrowStreamWriter(targetRoot, null, Channels.newChannel(out)) + val outputStream = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) + val compressedOutputStream = + new DataOutputStream(codec.compressedOutputStream(outputStream)) + val writer = + new ArrowStreamWriter(targetRoot, null, Channels.newChannel(compressedOutputStream)) try { writer.start() writer.writeBatch() @@ -346,7 +346,7 @@ object Utils extends CometTypeShim with Logging { writer.close() } - (Array(cbbos.toChunkedByteBuffer), batchCount.toLong, totalRows) + (Array(outputStream.toChunkedByteBuffer), batchCount.toLong, totalRows) } finally { if (targetRoot != null) { targetRoot.close() From 3e6b404d2d3c17cd4a4830b53b3bc9a4d27135e8 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 16 Mar 2026 17:18:03 -0400 Subject: [PATCH 16/16] Speed up CometJoinSuite "Broadcast hash join build-side batch coalescing". --- .../org/apache/comet/exec/CometJoinSuite.scala | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index 60a658ebde5..49fbe10c307 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -406,18 +406,12 @@ class CometJoinSuite extends CometTestBase { |FROM (SELECT /*+ REPARTITION($numPartitions) */ * FROM tbl_a) a |JOIN tbl_b ON a._2 = tbl_b._1""".stripMargin - // First verify correctness - val df = sql(query) - checkSparkAnswerAndOperator( - df, + val (_, cometPlan) = checkSparkAnswerAndOperator( + sql(query), Seq(classOf[CometBroadcastExchangeExec], classOf[CometBroadcastHashJoinExec])) - // Run again and check metrics on the executed plan - val df2 = sql(query) - df2.collect() - - val joins = collect(df2.queryExecution.executedPlan) { - case j: CometBroadcastHashJoinExec => j + val joins = collect(cometPlan) { case j: CometBroadcastHashJoinExec => + j } assert(joins.nonEmpty, "Expected CometBroadcastHashJoinExec in plan") @@ -432,8 +426,8 @@ class CometJoinSuite extends CometTestBase { s"Expected at most $numPartitions build batches (1 per task), got $buildBatches. " + "Broadcast batch coalescing may not be working.") - val broadcasts = collect(df2.queryExecution.executedPlan) { - case b: CometBroadcastExchangeExec => b + val broadcasts = collect(cometPlan) { case b: CometBroadcastExchangeExec => + b } assert(broadcasts.nonEmpty, "Expected CometBroadcastExchangeExec in plan")