Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ case class CometBroadcastExchangeExec(
val beforeBuild = System.nanoTime()
longMetric("collectTime") += NANOSECONDS.toMillis(beforeBuild - beforeCollect)

// Coalesce the many small per-shuffle-block buffers into a single buffer.
// Coalesce the many small per-shuffle-block buffers, usually into a single buffer.
// Without this, each consumer task deserializes one Arrow IPC stream per
// shuffle block (one per writer task per partition), which is very expensive
// when there are hundreds of writer tasks and partitions. See the scaladoc
Expand Down
119 changes: 91 additions & 28 deletions spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvid
import org.apache.arrow.vector.ipc.{ArrowStreamReader, ArrowStreamWriter}
import org.apache.arrow.vector.types._
import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema}
import org.apache.arrow.vector.util.VectorSchemaRootAppender
import org.apache.arrow.vector.util.{OversizedAllocationException, VectorSchemaRootAppender}
import org.apache.spark.{SparkEnv, SparkException}
import org.apache.spark.internal.Logging
import org.apache.spark.io.CompressionCodec
Expand Down Expand Up @@ -360,7 +360,15 @@ object Utils extends CometTypeShim with Logging {
}

/**
* Coalesces many small Arrow IPC batches into a single batch for broadcasting.
* The most bytes of values a string or binary vector of a coalesced broadcast batch holds.
* `VectorAppender` grows a data buffer by doubling it, and Arrow refuses a buffer of 2^31
* bytes, so 2^30 is as far as an appended vector can grow (#7192).
*/
private val MaxCoalescedValueBytes = 1L << 30

/**
* Coalesces many small Arrow IPC batches into as few batches as Arrow can hold for
* broadcasting, usually one.
*
* Why this is necessary: The broadcast exchange collects shuffle output by calling
* getByteArrayRdd, which serializes each ColumnarBatch independently into its own
Expand All @@ -375,9 +383,16 @@ object Utils extends CometTypeShim with Logging {
* allocation) for each one. With coalescing, we decode and append all batches into one
* VectorSchemaRoot on the driver, then re-serialize once. Each consumer task then deserializes
* a single Arrow IPC stream.
*
* A broadcast whose string or binary values do not fit one vector is written as several
* batches: a batch that would take a vector past `maxValueBytes` starts the next one.
*
* @param maxValueBytes
* the most bytes of values a string or binary vector of one coalesced batch may hold
*/
def coalesceBroadcastBatches(
input: Iterator[ChunkedByteBuffer]): (Array[ChunkedByteBuffer], Long, Long) = {
input: Iterator[ChunkedByteBuffer],
maxValueBytes: Long = MaxCoalescedValueBytes): (Array[ChunkedByteBuffer], Long, Long) = {
val buffers = input.filterNot(_.size == 0).toArray
if (buffers.isEmpty) {
return (Array.empty, 0L, 0L)
Expand All @@ -386,11 +401,45 @@ object Utils extends CometTypeShim with Logging {
val allocator = org.apache.comet.CometArrowAllocator
.newChildAllocator("broadcast-coalesce", 0, Long.MaxValue)
try {
var schema: Schema = null
var targetRoot: VectorSchemaRoot = null
// The rows appended to targetRoot, which VSRAppender does not count without columns.
var targetRows = 0L
var totalRows = 0L
var batchCount = 0
val coalesced = scala.collection.mutable.ArrayBuffer.empty[ChunkedByteBuffer]

val codec = CompressionCodec.createCodec(SparkEnv.get.conf)

// Writes targetRoot as one Arrow IPC stream, and releases it.
def writeTarget(): Unit = {
if (targetRoot.getSchema.getFields.isEmpty) {
// VSRAppender does not update rowCount with no columns
targetRoot.setRowCount(targetRows.toInt)
}

assert(
targetRoot.getRowCount.toLong == targetRows,
s"Row count mismatch after coalesce: ${targetRoot.getRowCount} != $targetRows")

val outputStream = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate)
val compressedOutputStream =
new DataOutputStream(codec.compressedOutputStream(outputStream))
val writer =
new ArrowStreamWriter(targetRoot, null, Channels.newChannel(compressedOutputStream))
try {
writer.start()
writer.writeBatch()
} finally {
writer.close()
}

coalesced += outputStream.toChunkedByteBuffer
targetRoot.close()
targetRoot = null
targetRows = 0L
}

try {
for (bytes <- buffers) {
val compressedInputStream =
Expand All @@ -415,15 +464,23 @@ object Utils extends CometTypeShim with Logging {
}
while (reader.loadNextBatch()) {
val sourceRoot = reader.getVectorSchemaRoot
if (targetRows > 0 && exceedsValueBytes(targetRoot, sourceRoot, maxValueBytes)) {
writeTarget()
}
if (targetRoot == null) {
targetRoot = VectorSchemaRoot.create(sourceRoot.getSchema, allocator)
// Every coalesced batch takes the first one's schema, so a batch that does not
// match it still leaves the broadcast uncoalesced.
if (schema == null) schema = sourceRoot.getSchema
targetRoot = VectorSchemaRoot.create(schema, allocator)
targetRoot.allocateNew()
}
try {
// Appending needs offsets that start at zero, which serializeBatches writes.
VectorSchemaRootAppender.append(targetRoot, sourceRoot)
} catch {
case e: IllegalArgumentException =>
// Arrow throws OversizedAllocationException for a batch that is too large to
// append even after the split above.
case e @ (_: IllegalArgumentException | _: OversizedAllocationException) =>
logWarning(
"Arrow batches cannot be appended during BroadcastExchange coalescing; " +
"skipping coalesce",
Expand All @@ -432,6 +489,7 @@ object Utils extends CometTypeShim with Logging {
targetRoot = null
return (buffers, 0L, 0L)
}
targetRows += sourceRoot.getRowCount
totalRows += sourceRoot.getRowCount
batchCount += 1
}
Expand All @@ -443,31 +501,12 @@ object Utils extends CometTypeShim with Logging {
if (targetRoot == null) {
return (Array.empty, 0L, 0L)
}
writeTarget()

if (targetRoot.getSchema.getFields.isEmpty) {
// VSRAppender does not update rowCount with no columns
targetRoot.setRowCount(totalRows.toInt)
}

assert(
targetRoot.getRowCount.toLong == totalRows,
s"Row count mismatch after coalesce: ${targetRoot.getRowCount} != $totalRows")

logInfo(s"Coalesced $batchCount broadcast batches into 1 ($totalRows rows)")

val outputStream = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate)
val compressedOutputStream =
new DataOutputStream(codec.compressedOutputStream(outputStream))
val writer =
new ArrowStreamWriter(targetRoot, null, Channels.newChannel(compressedOutputStream))
try {
writer.start()
writer.writeBatch()
} finally {
writer.close()
}
logInfo(
s"Coalesced $batchCount broadcast batches into ${coalesced.length} ($totalRows rows)")

(Array(outputStream.toChunkedByteBuffer), batchCount.toLong, totalRows)
(coalesced.toArray, batchCount.toLong, totalRows)
} finally {
if (targetRoot != null) {
targetRoot.close()
Expand All @@ -478,6 +517,30 @@ object Utils extends CometTypeShim with Logging {
}
}

/**
* Whether appending `source` to `target` would take a string or binary vector of `target`, at
* any depth, past `maxValueBytes` of values.
*/
private def exceedsValueBytes(
target: VectorSchemaRoot,
source: VectorSchemaRoot,
maxValueBytes: Long): Boolean = {
def exceeds(t: FieldVector, s: FieldVector): Boolean = {
val valueBytes = (t, s) match {
case (tv: BaseVariableWidthVector, sv: BaseVariableWidthVector) =>
tv.sizeOfValueBuffer.toLong + sv.sizeOfValueBuffer
case _ => 0L
}
valueBytes > maxValueBytes ||
t.getChildrenFromFields.asScala.zip(s.getChildrenFromFields.asScala).exists {
case (tc, sc) => exceeds(tc, sc)
}
}
target.getFieldVectors.asScala.zip(source.getFieldVectors.asScala).exists { case (t, s) =>
exceeds(t, s)
}
}

/**
* Whether every column in `batch` is an Arrow-backed `CometVector`, so [[getBatchFieldVectors]]
* can hand out its vectors directly. Callers that may receive batches from a plan they did not
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import scala.jdk.CollectionConverters._
import org.apache.arrow.c.CDataDictionaryProvider
import org.apache.arrow.memory.BufferAllocator
import org.apache.arrow.vector.{BaseLargeVariableWidthVector, BitVectorHelper, IntVector, LargeVarBinaryVector, LargeVarCharVector, ValueVector, VarBinaryVector, VarCharVector, VariableWidthFieldVector}
import org.apache.arrow.vector.complex.StructVector
import org.apache.arrow.vector.dictionary.{Dictionary, DictionaryProvider}
import org.apache.arrow.vector.ipc.message.ArrowFieldNode
import org.apache.arrow.vector.types.pojo.{ArrowType, DictionaryEncoding, Field, FieldType}
Expand Down Expand Up @@ -343,6 +344,47 @@ class UtilsSuite extends CometTestBase {
}
}

test("coalesceBroadcastBatches starts a new batch before a string vector passes the cap") {
// VectorAppender cannot grow a string or binary vector past 1 GiB of values (#7192), so a
// batch that would take one past the cap starts the next coalesced batch. A small cap shows
// it, and only the strings nested in the struct reach it.
val numBatches = 4
val vectors = (0 until numBatches).map { b =>
(
fill(new VarCharVector("s", CometArrowAllocator), Seq(s"$b", null)),
structOfStrings("st", Seq(s"abc$b", s"def$b")))
}
try {
val batches = vectors.map { case (s, st) =>
new ColumnarBatch(Array(cometVector(s), cometVector(st)), 2)
}
val bufs = Utils.serializeBatches(batches.iterator).map(_._2).toSeq.iterator
// Each batch holds 8 bytes of nested values, so two fit under the cap and a third does not.
val (coalesced, batchCount, totalRows) =
Utils.coalesceBroadcastBatches(bufs, maxValueBytes = 20L)
assert(batchCount == numBatches)
assert(totalRows == 2L * numBatches)

val decoded = coalesced.toSeq.map { buf =>
val it = Utils.decodeBatches(buf, "test")
val out = it.next()
val values = (
stringValues(out.column(0), out.numRows()),
stringValues(out.column(1).getChild(0), out.numRows()))
assert(!it.hasNext)
values
}
assert(decoded.map(_._1.length) == Seq(4, 4))
assert(decoded.flatMap(_._1) == (0 until numBatches).flatMap(b => Seq(s"$b", null)))
assert(decoded.flatMap(_._2) == (0 until numBatches).flatMap(b => Seq(s"abc$b", s"def$b")))
} finally {
vectors.foreach { case (s, st) =>
s.close()
st.close()
}
}
}

test("serializeBatches releases the vectors it made when a later column fails") {
// A materialized constant and a 32-bit copy of a large-offset column are both made before the
// last column is refused. Nothing outside serializeBatches ever holds them, so it has to
Expand Down Expand Up @@ -418,6 +460,18 @@ class UtilsSuite extends CometTestBase {
vector
}

/** A struct column `name` whose one child, `c`, holds `values`. */
private def structOfStrings(name: String, values: Seq[String]): StructVector = {
val struct = StructVector.empty(name, CometArrowAllocator)
val child =
struct.addOrGet("c", FieldType.nullable(ArrowType.Utf8.INSTANCE), classOf[VarCharVector])
struct.allocateNew()
fill(child, values)
values.indices.foreach(struct.setIndexDefined)
struct.setValueCount(values.length)
struct
}

/**
* Loads `values` into `vector` the way an imported slice holds them: the data buffer starts
* with the bytes of `skipped`, which no value covers, so the offsets start past zero.
Expand Down
Loading