From ccbf5b7eac80dbe79fd4c8e116e38da9a088782b Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 27 Aug 2026 14:01:20 +0800 Subject: [PATCH 01/15] fix: make CometDiskBlockWriter spill registry per-task instead of executor-global The static currentWriters list in CometDiskBlockWriter was shared by all shuffle tasks in the executor, so a task under memory pressure could spill other tasks' writers, count their memory as freed for itself, and retain writers of failed tasks forever. It also allowed an ABBA deadlock between the writer monitor and the registry lock when two tasks spilled concurrently, and unsynchronized cross-thread reads could throw ConcurrentModificationException. Replace the static registry with a task-owned list created by CometBypassMergeSortShuffleWriter and passed to its partition writers, and delete the dead spillingWriters path (nothing ever added to it). All access to a writer is now confined to its own task's thread. --- .github/workflows/pr_build_linux.yml | 1 + .github/workflows/pr_build_macos.yml | 1 + .../CometBypassMergeSortShuffleWriter.java | 8 +- .../shuffle/CometDiskBlockWriter.java | 17 +- .../shuffle/CometDiskBlockWriterSuite.scala | 155 ++++++++++++++++++ 5 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index b7c88254c93..a317d5f5cc8 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -328,6 +328,7 @@ jobs: org.apache.comet.exec.CometNativeColumnarToRowSuite org.apache.comet.exec.CometNativeShuffleSuite org.apache.spark.sql.comet.execution.shuffle.CometNativeShuffleInputRDDSuite + org.apache.spark.sql.comet.execution.shuffle.CometDiskBlockWriterSuite org.apache.comet.exec.CometShuffleEncryptionSuite org.apache.comet.exec.CometShuffleManagerSuite org.apache.comet.exec.CometAsyncShuffleSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index 8210f91b7f9..cf107afdb01 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -144,6 +144,7 @@ jobs: org.apache.comet.exec.CometNativeColumnarToRowSuite org.apache.comet.exec.CometNativeShuffleSuite org.apache.spark.sql.comet.execution.shuffle.CometNativeShuffleInputRDDSuite + org.apache.spark.sql.comet.execution.shuffle.CometDiskBlockWriterSuite org.apache.comet.exec.CometShuffleEncryptionSuite org.apache.comet.exec.CometShuffleManagerSuite org.apache.comet.exec.CometAsyncShuffleSuite diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java index 07da649f4ae..8a510b9128f 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.channels.FileChannel; +import java.util.LinkedList; import java.util.Optional; import javax.annotation.Nullable; @@ -176,6 +177,10 @@ public void write(Iterator> records) throws IOException { CometShuffleExternalSorter.MAXIMUM_PAGE_SIZE_BYTES, memoryManager.pageSizeBytes())); + // This task's disk writers. Under memory pressure a writer spills its sibling writers in + // this list, never those of other tasks. + final LinkedList taskWriters = new LinkedList<>(); + // Allocate the disk writers, and open the files that we'll be writing to for (int i = 0; i < numPartitions; i++) { final Tuple2 tempShuffleBlockIdPlusFile = @@ -190,7 +195,8 @@ public void write(Iterator> records) throws IOException { schema, writeMetrics, conf, - tracingEnabled); + tracingEnabled, + taskWriters); if (partitionChecksums.length > 0) { writer.setChecksum(partitionChecksums[i]); writer.setChecksumAlgo(checksumAlgorithm); diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java index 504de9ac4a3..c4a68d50285 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java @@ -62,11 +62,11 @@ public final class CometDiskBlockWriter { private static final Logger logger = LoggerFactory.getLogger(CometDiskBlockWriter.class); private static final ClassTag OBJECT_CLASS_TAG = ClassTag$.MODULE$.Object(); - /** List of all `NativeDiskBlockArrowIPCWriter`s of same shuffle task. */ - private static final LinkedList currentWriters = new LinkedList<>(); - - /** List of `ArrowIPCWriter`s which are spilling. */ - private final LinkedList spillingWriters = new LinkedList<>(); + /** + * All writers of the same shuffle task. Spilling under memory pressure only ever touches this + * task's writers, never those of other tasks. + */ + private final LinkedList currentWriters; private final TaskContext taskContext; @@ -131,8 +131,10 @@ public long getEncodeNanos() { StructType schema, ShuffleWriteMetricsReporter writeMetrics, SparkConf conf, - boolean tracingEnabled) { + boolean tracingEnabled, + LinkedList taskWriters) { this.nativeLib = new Native(); + this.currentWriters = taskWriters; this.allocator = allocator; this.taskContext = taskContext; this.serializer = serializer; @@ -275,9 +277,6 @@ long getActiveMemoryUsage() { } void freeMemory() { - for (ArrowIPCWriter writer : spillingWriters) { - writer.freeMemory(); - } activeWriter.freeMemory(); } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala new file mode 100644 index 00000000000..eaef56470d3 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet.execution.shuffle + +import java.io.File +import java.util.{LinkedList => JLinkedList, Properties} + +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.spark.{SparkConf, TaskContextImpl} +import org.apache.spark.executor.{ShuffleWriteMetrics, TaskMetrics} +import org.apache.spark.memory.{TaskMemoryManager, TestMemoryManager} +import org.apache.spark.shuffle.comet.CometShuffleMemoryAllocator +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.UnsafeProjection +import org.apache.spark.sql.execution.UnsafeRowSerializer +import org.apache.spark.sql.types.{BinaryType, StructField, StructType} +import org.apache.spark.util.Utils + +class CometDiskBlockWriterSuite extends AnyFunSuite { + + private val schema = StructType(Seq(StructField("a", BinaryType))) + private val pageSize: Long = 256 * 1024 + + private def newTaskContext(tmm: TaskMemoryManager, taskAttemptId: Long): TaskContextImpl = { + new TaskContextImpl( + stageId = 0, + stageAttemptNumber = 0, + partitionId = taskAttemptId.toInt, + numPartitions = 2, + taskAttemptId = taskAttemptId, + attemptNumber = 0, + taskMemoryManager = tmm, + localProperties = new Properties, + metricsSystem = null, + taskMetrics = TaskMetrics.empty, + cpus = 1, + resources = Map.empty) + } + + test("memory pressure spills only writers of the requesting task") { + val conf = new SparkConf() + .set("spark.memory.offHeap.enabled", "true") + .set("spark.memory.offHeap.size", "1g") + val memoryManager = new TestMemoryManager(conf) + // Enough memory for task B to buffer 12 pages and task A to buffer 10 pages; task A's + // 11th page allocation must fail so that it goes through the spilling path. + memoryManager.limit(22 * pageSize) + + val tmmA = new TaskMemoryManager(memoryManager, 0L) + val tmmB = new TaskMemoryManager(memoryManager, 1L) + val taskContextA = newTaskContext(tmmA, 0L) + val taskContextB = newTaskContext(tmmB, 1L) + val allocatorA = CometShuffleMemoryAllocator.getInstance(conf, tmmA, pageSize) + val allocatorB = CometShuffleMemoryAllocator.getInstance(conf, tmmB, pageSize) + + val tempDir = Utils.createTempDir() + try { + val writersA = new JLinkedList[CometDiskBlockWriter]() + val writersB = new JLinkedList[CometDiskBlockWriter]() + val serializer = new UnsafeRowSerializer(1).newInstance() + val fileA = new File(tempDir, "taskA-partition0") + val fileB = new File(tempDir, "taskB-partition0") + val writerA = new CometDiskBlockWriter( + fileA, + allocatorA, + taskContextA, + serializer, + schema, + new ShuffleWriteMetrics, + conf, + false, + writersA) + val writerB = new CometDiskBlockWriter( + fileB, + allocatorB, + taskContextB, + serializer, + schema, + new ShuffleWriteMetrics, + conf, + false, + writersB) + + val toUnsafe = UnsafeProjection.create(schema) + def insertOne(writer: CometDiskBlockWriter): Unit = { + writer.insertRow(toUnsafe(InternalRow(new Array[Byte](1024))), 0) + } + + // Task B buffers 12 pages worth of rows on its own thread and then idles, holding the + // buffered rows in memory (as a concurrently running task would). + var rowsB = 0L + val threadB = new Thread(() => { + while (allocatorB.getUsed < 12 * pageSize) { + insertOne(writerB) + rowsB += 1 + } + }) + threadB.start() + threadB.join() + assert(allocatorB.getUsed == 12 * pageSize) + + // Task A buffers 10 pages, then keeps inserting: its 11th page allocation fails, which + // must spill task A's own writers and leave task B's writer untouched. + var rowsA = 0L + while (allocatorA.getUsed < 10 * pageSize) { + insertOne(writerA) + rowsA += 1 + } + (0 until 400).foreach { _ => + insertOne(writerA) + rowsA += 1 + } + + // Task A resolved its memory pressure by spilling its own data... + assert(taskContextA.taskMetrics.diskBytesSpilled > 0) + assert(writerA.getOutputRecords > 0) + // ... and task B's buffered rows were not spilled, not written out, and not charged. + assert(allocatorB.getUsed == 12 * pageSize) + assert(writerB.getActiveMemoryUsage == 12 * pageSize) + assert(writerB.getOutputRecords == 0) + assert(taskContextB.taskMetrics.diskBytesSpilled == 0) + assert(fileB.length() == 0) + + val segmentA = writerA.close() + val segmentB = writerB.close() + assert(writerA.getOutputRecords == rowsA) + assert(writerB.getOutputRecords == rowsB) + assert(segmentA.length > 0) + assert(segmentB.length > 0) + assert(writersA.isEmpty && writersB.isEmpty) + } finally { + Utils.deleteRecursively(tempDir) + tmmA.cleanUpAllAllocatedMemory() + tmmB.cleanUpAllAllocatedMemory() + } + } +} From 3cee056cd1ded0240892ef53ebfef515fd6c1e4e Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 27 Aug 2026 15:21:26 +0800 Subject: [PATCH 02/15] address review: wait for shared on-heap pool memory instead of failing In on-heap mode all tasks share one CometBoundedShuffleMemoryAllocator, so with per-task spill registries a task whose own writers hold nothing spillable failed with SparkOutOfMemoryError when other tasks held the pool. Preserve allocation progress by letting the post-spill retry in SpillWriter.initialCurrentPage wait (bounded by a timeout) for other tasks to free pool memory; free() notifies waiters. This point is only reached after the task has spilled everything it owns, so waiting tasks hold no pool memory and the tasks holding it can always progress and free it. The off-heap allocator keeps its non-waiting behavior since Spark's memory manager already arbitrates between tasks. --- .../CometBoundedShuffleMemoryAllocator.java | 33 +++++++ .../CometShuffleMemoryAllocatorTrait.java | 11 +++ .../comet/execution/shuffle/SpillWriter.java | 6 +- .../shuffle/CometDiskBlockWriterSuite.scala | 92 +++++++++++++++++++ 4 files changed, 140 insertions(+), 2 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java index dae55c04bfe..13ffdd57437 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java @@ -57,6 +57,9 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory private final long totalMemory; private long allocatedMemory = 0L; + /** How long `allocateBlocking` waits for other tasks to free memory before giving up. */ + private static final long MEMORY_WAIT_TIMEOUT_MS = 60_000L; + /** The number of bits used to address the page table. */ private static final int PAGE_NUMBER_BITS = 13; @@ -112,6 +115,34 @@ public synchronized MemoryBlock allocate(long required) { return allocateMemoryBlock(size); } + /** + * Like {@link #allocate(long)}, but waits for other tasks of this shared pool to free memory + * before giving up. Callers must only use this after spilling their own buffered data, so a + * waiting task holds no pool memory itself and the tasks still holding memory can always progress + * and eventually free it. + */ + @Override + public synchronized MemoryBlock allocateBlocking(long required) { + long size = Math.max(pageSize, required); + long deadline = System.currentTimeMillis() + MEMORY_WAIT_TIMEOUT_MS; + while (true) { + try { + return allocateMemoryBlock(size); + } catch (SparkOutOfMemoryError e) { + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw e; + } + try { + wait(remaining); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw e; + } + } + } + } + private synchronized MemoryBlock allocateMemoryBlock(long required) { if (required > TaskMemoryManager.MAXIMUM_PAGE_SIZE_BYTES) { throw new TooLargePageException(required); @@ -160,6 +191,8 @@ public synchronized long free(MemoryBlock block) { block.pageNumber = MemoryBlock.FREED_IN_TMM_PAGE_NUMBER; allocator.free(block); + // Wake up tasks waiting in `allocateBlocking`. + notifyAll(); return blockSize; } diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java index 36fa9d2ff48..a4fcd5c4c28 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java @@ -33,6 +33,17 @@ protected CometShuffleMemoryAllocatorTrait( public abstract MemoryBlock allocate(long required); + /** + * Like {@link #allocate(long)}, but may wait for memory freed by other tasks before giving up. + * Callers must only use this after spilling their own buffered data, so that a waiting task holds + * no memory that other tasks could be waiting for. The default implementation does not wait: + * `CometUnifiedShuffleMemoryAllocator` delegates to Spark's memory manager, which already + * arbitrates memory between tasks. + */ + public MemoryBlock allocateBlocking(long required) { + return allocate(required); + } + public abstract long free(MemoryBlock block); public abstract long getOffsetInPage(long pagePlusOffsetAddress); diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java index 47e85383bb4..8d9331b586c 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java @@ -165,11 +165,13 @@ public boolean acquireNewPageIfNecessary(int required) { return true; } - /** Allocates initial memory page */ + /** Allocates initial memory page. Only called after `spill()` has flushed this writer's data. */ public void initialCurrentPage(int required) { assert (currentPage == null); try { - currentPage = allocator.allocate(required); + // This writer has already spilled its own data, so on a shared pool it may wait for other + // tasks to free memory instead of failing right away. + currentPage = allocator.allocateBlocking(required); } catch (SparkOutOfMemoryError e) { logger.error("Unable to acquire {} bytes of memory", required); throw e; diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index eaef56470d3..e1c68a9c177 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -152,4 +152,96 @@ class CometDiskBlockWriterSuite extends AnyFunSuite { tmmB.cleanUpAllAllocatedMemory() } } + + test("on-heap shared pool: a task with nothing to spill waits for other tasks to free memory") { + // On-heap mode uses one executor-wide CometBoundedShuffleMemoryAllocator. A task whose own + // writers hold nothing spillable must wait for other tasks to free pool memory rather than + // fail with SparkOutOfMemoryError. + val conf = new SparkConf().set("spark.comet.memoryOverhead", "1") // 1 MiB shared pool + resetOnHeapAllocatorSingleton() + val memoryManager = new TestMemoryManager(conf) + val tmmA = new TaskMemoryManager(memoryManager, 0L) + val tmmB = new TaskMemoryManager(memoryManager, 1L) + val taskContextA = newTaskContext(tmmA, 0L) + val taskContextB = newTaskContext(tmmB, 1L) + // Both tasks get the same singleton allocator in on-heap mode. + val allocatorA = CometShuffleMemoryAllocator.getInstance(conf, tmmA, pageSize) + val allocatorB = CometShuffleMemoryAllocator.getInstance(conf, tmmB, pageSize) + assert(allocatorA eq allocatorB) + + val tempDir = Utils.createTempDir() + try { + val serializer = new UnsafeRowSerializer(1).newInstance() + val fileA = new File(tempDir, "onheap-taskA") + val fileB = new File(tempDir, "onheap-taskB") + val writerA = new CometDiskBlockWriter( + fileA, + allocatorA, + taskContextA, + serializer, + schema, + new ShuffleWriteMetrics, + conf, + false, + new JLinkedList[CometDiskBlockWriter]()) + val writerB = new CometDiskBlockWriter( + fileB, + allocatorB, + taskContextB, + serializer, + schema, + new ShuffleWriteMetrics, + conf, + false, + new JLinkedList[CometDiskBlockWriter]()) + + val toUnsafe = UnsafeProjection.create(schema) + def insertOne(writer: CometDiskBlockWriter): Unit = { + writer.insertRow(toUnsafe(InternalRow(new Array[Byte](1024))), 0) + } + + // Task B fills the whole 1 MiB pool (4 pages) and idles. + var rowsB = 0L + while (writerB.getActiveMemoryUsage < 4 * pageSize) { + insertOne(writerB) + rowsB += 1 + } + + // Task B finishes a bit later on its own thread, freeing the pool. + val threadB = new Thread(() => { + Thread.sleep(500) + writerB.close() + }) + threadB.start() + + // Task A's first insert finds the pool exhausted and has nothing of its own to spill; it + // must wait for task B to finish instead of throwing SparkOutOfMemoryError. + var rowsA = 0L + (0 until 10).foreach { _ => + insertOne(writerA) + rowsA += 1 + } + threadB.join() + + val segmentA = writerA.close() + assert(writerA.getOutputRecords == rowsA) + assert(writerB.getOutputRecords == rowsB) + assert(segmentA.length > 0) + // Neither task spilled: A waited instead of stealing B's memory, and B was never touched. + assert(taskContextA.taskMetrics.diskBytesSpilled == 0) + assert(taskContextB.taskMetrics.diskBytesSpilled == 0) + } finally { + Utils.deleteRecursively(tempDir) + tmmA.cleanUpAllAllocatedMemory() + tmmB.cleanUpAllAllocatedMemory() + resetOnHeapAllocatorSingleton() + } + } + + /** Clears the CometShuffleMemoryAllocator singleton so this suite controls its pool size. */ + private def resetOnHeapAllocatorSingleton(): Unit = { + val field = classOf[CometShuffleMemoryAllocator].getDeclaredField("INSTANCE") + field.setAccessible(true) + field.set(null, null) + } } From 4c12797e0d8e3ab5e2cc4d97b454561574a3797d Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 27 Aug 2026 16:56:53 +0800 Subject: [PATCH 03/15] address review: wait for shared pool memory without a deadline Any finite deadline fails some workload: a task holding the shared on-heap pool longer than the cap still made waiters fail with SparkOutOfMemoryError. Wait until memory is freed instead, mirroring how Spark's unified memory manager blocks a task until memory becomes available. Liveness is preserved because a waiting task has already spilled everything it owns, so pool memory is only held by tasks that can progress and free it; interrupting the task (e.g. task kill) still aborts the wait. Log once when a task starts waiting so long waits are diagnosable. --- .../CometBoundedShuffleMemoryAllocator.java | 29 +++++++++++-------- .../CometShuffleMemoryAllocatorTrait.java | 10 +++---- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java index 13ffdd57437..bbfa1abc0c8 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java @@ -22,6 +22,9 @@ import java.io.IOException; import java.util.BitSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.spark.SparkConf; import org.apache.spark.memory.MemoryConsumer; import org.apache.spark.memory.MemoryMode; @@ -51,15 +54,15 @@ * Columnar Shuffle and execution apart from Spark's on-heap memory configuration. */ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemoryAllocatorTrait { + private static final Logger logger = + LoggerFactory.getLogger(CometBoundedShuffleMemoryAllocator.class); + private final UnsafeMemoryAllocator allocator = new UnsafeMemoryAllocator(); private final long pageSize; private final long totalMemory; private long allocatedMemory = 0L; - /** How long `allocateBlocking` waits for other tasks to free memory before giving up. */ - private static final long MEMORY_WAIT_TIMEOUT_MS = 60_000L; - /** The number of bits used to address the page table. */ private static final int PAGE_NUMBER_BITS = 13; @@ -116,25 +119,27 @@ public synchronized MemoryBlock allocate(long required) { } /** - * Like {@link #allocate(long)}, but waits for other tasks of this shared pool to free memory - * before giving up. Callers must only use this after spilling their own buffered data, so a - * waiting task holds no pool memory itself and the tasks still holding memory can always progress - * and eventually free it. + * Like {@link #allocate(long)}, but waits for other tasks of this shared pool to free memory, + * mirroring how Spark's unified memory manager blocks a task until memory becomes available. + * Callers must only use this after spilling their own buffered data, so a waiting task holds no + * pool memory itself and the tasks still holding memory can always progress and eventually free + * it. Interrupting the task (e.g. task kill) aborts the wait. */ @Override public synchronized MemoryBlock allocateBlocking(long required) { long size = Math.max(pageSize, required); - long deadline = System.currentTimeMillis() + MEMORY_WAIT_TIMEOUT_MS; + boolean logged = false; while (true) { try { return allocateMemoryBlock(size); } catch (SparkOutOfMemoryError e) { - long remaining = deadline - System.currentTimeMillis(); - if (remaining <= 0) { - throw e; + if (!logged) { + logger.warn( + "Waiting for other tasks to free up {} bytes of Comet shuffle pool memory", size); + logged = true; } try { - wait(remaining); + wait(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw e; diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java index a4fcd5c4c28..8b2cf3bd0f2 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java @@ -34,11 +34,11 @@ protected CometShuffleMemoryAllocatorTrait( public abstract MemoryBlock allocate(long required); /** - * Like {@link #allocate(long)}, but may wait for memory freed by other tasks before giving up. - * Callers must only use this after spilling their own buffered data, so that a waiting task holds - * no memory that other tasks could be waiting for. The default implementation does not wait: - * `CometUnifiedShuffleMemoryAllocator` delegates to Spark's memory manager, which already - * arbitrates memory between tasks. + * Like {@link #allocate(long)}, but may wait for memory freed by other tasks instead of failing + * immediately. Callers must only use this after spilling their own buffered data, so that a + * waiting task holds no memory that other tasks could be waiting for. The default implementation + * does not wait: `CometUnifiedShuffleMemoryAllocator` delegates to Spark's memory manager, which + * already arbitrates memory between tasks. */ public MemoryBlock allocateBlocking(long required) { return allocate(required); From b8fadf587123cc28f957d8aee40f7fb83d69dafd Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 27 Aug 2026 17:47:32 +0800 Subject: [PATCH 04/15] address review: fail fast on unsatisfiable waits and keep task kills non-fatal The unbounded wait could hang when it can never succeed: the sort-based path retains its pointer array through an empty spill(), so a waiting task may itself hold pool memory, and two such tasks can block each other forever; a single request larger than the pool waited on an empty pool. Track the pool memory retained per thread and give up before waiting when the request cannot fit next to the requester's own retained memory, or when all allocated memory is retained by threads that are themselves blocked waiting (nothing can be freed anymore). Also stop rethrowing SparkOutOfMemoryError when the wait is interrupted: it extends OutOfMemoryError, so Spark's killed-task handler does not match it and an intentional task kill was reported as ExceptionFailure. Throw a RuntimeException wrapping the InterruptedException instead so kills are classified as TaskKilled. --- .../CometBoundedShuffleMemoryAllocator.java | 84 ++++++++-- .../shuffle/CometDiskBlockWriterSuite.scala | 153 ++++++++++++++++-- 2 files changed, 210 insertions(+), 27 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java index bbfa1abc0c8..55e9fb032bb 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.BitSet; +import java.util.HashMap; +import java.util.HashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,6 +74,15 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory private final MemoryBlock[] pageTable = new MemoryBlock[PAGE_TABLE_SIZE]; private final BitSet allocatedPages = new BitSet(PAGE_TABLE_SIZE); + /** The thread that allocated each page, used to decide whether a blocked wait can succeed. */ + private final Thread[] pageOwners = new Thread[PAGE_TABLE_SIZE]; + + /** Pool memory currently retained by each thread. */ + private final HashMap retainedMemory = new HashMap<>(); + + /** Threads currently blocked in {@link #allocateBlocking(long)}. */ + private final HashSet waitingThreads = new HashSet<>(); + private static final int OFFSET_BITS = 51; private static final long MASK_LONG_LOWER_51_BITS = 0x7FFFFFFFFFFFFL; @@ -121,33 +132,66 @@ public synchronized MemoryBlock allocate(long required) { /** * Like {@link #allocate(long)}, but waits for other tasks of this shared pool to free memory, * mirroring how Spark's unified memory manager blocks a task until memory becomes available. - * Callers must only use this after spilling their own buffered data, so a waiting task holds no - * pool memory itself and the tasks still holding memory can always progress and eventually free - * it. Interrupting the task (e.g. task kill) aborts the wait. + * Callers must only use this after spilling their own buffered data. The wait fails fast when it + * can never succeed: when the request does not fit next to the memory this thread itself still + * retains (e.g. the sorter's pointer array), or when all allocated memory is retained by threads + * that are themselves blocked here. Interrupting the task (e.g. task kill) aborts the wait. */ @Override public synchronized MemoryBlock allocateBlocking(long required) { long size = Math.max(pageSize, required); + Thread self = Thread.currentThread(); boolean logged = false; - while (true) { - try { - return allocateMemoryBlock(size); - } catch (SparkOutOfMemoryError e) { - if (!logged) { - logger.warn( - "Waiting for other tasks to free up {} bytes of Comet shuffle pool memory", size); - logged = true; - } + try { + while (true) { try { - wait(); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw e; + return allocateMemoryBlock(size); + } catch (SparkOutOfMemoryError e) { + if (waitingThreads.add(self)) { + // Wake existing waiters so they re-evaluate the deadlock check against the enlarged + // waiting set. + notifyAll(); + } + // This thread cannot free what it retains while it waits, so a request that does not + // fit next to its own retained memory can never be satisfied. + if (size > totalMemory - retainedMemory.getOrDefault(self, 0L)) { + throw e; + } + // The allocation just failed, so the request does not fit in the unallocated pool. If + // all allocated memory is retained by blocked threads, nothing can be freed anymore + // and waiting would deadlock. + if (allocatedMemory <= retainedByWaitingThreads()) { + throw e; + } + if (!logged) { + logger.warn( + "Waiting for other tasks to free up {} bytes of Comet shuffle pool memory", size); + logged = true; + } + try { + wait(); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + // Not an allocation failure: stay non-fatal so that an intentional task kill is + // classified as TaskKilled rather than ExceptionFailure. + throw new RuntimeException( + "Interrupted while waiting for Comet shuffle pool memory", ie); + } } } + } finally { + waitingThreads.remove(self); } } + private long retainedByWaitingThreads() { + long retained = 0; + for (Thread thread : waitingThreads) { + retained += retainedMemory.getOrDefault(thread, 0L); + } + return retained; + } + private synchronized MemoryBlock allocateMemoryBlock(long required) { if (required > TaskMemoryManager.MAXIMUM_PAGE_SIZE_BYTES) { throw new TooLargePageException(required); @@ -178,6 +222,8 @@ private synchronized MemoryBlock allocateMemoryBlock(long required) { block.pageNumber = pageNumber; pageTable[pageNumber] = block; allocatedPages.set(pageNumber); + pageOwners[pageNumber] = Thread.currentThread(); + retainedMemory.merge(Thread.currentThread(), got, Long::sum); return block; } @@ -191,6 +237,12 @@ public synchronized long free(MemoryBlock block) { long blockSize = block.size(); allocatedMemory -= blockSize; + Thread owner = pageOwners[block.pageNumber]; + pageOwners[block.pageNumber] = null; + if (owner != null) { + retainedMemory.computeIfPresent(owner, (t, v) -> v - blockSize <= 0 ? null : v - blockSize); + } + pageTable[block.pageNumber] = null; allocatedPages.clear(block.pageNumber); block.pageNumber = MemoryBlock.FREED_IN_TMM_PAGE_NUMBER; diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index e1c68a9c177..98f2c53d3c1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -21,20 +21,25 @@ package org.apache.spark.sql.comet.execution.shuffle import java.io.File import java.util.{LinkedList => JLinkedList, Properties} +import java.util.concurrent.CountDownLatch +import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits} import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.time.{Seconds, Span} import org.apache.spark.{SparkConf, TaskContextImpl} import org.apache.spark.executor.{ShuffleWriteMetrics, TaskMetrics} -import org.apache.spark.memory.{TaskMemoryManager, TestMemoryManager} -import org.apache.spark.shuffle.comet.CometShuffleMemoryAllocator +import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager} +import org.apache.spark.shuffle.comet.{CometBoundedShuffleMemoryAllocator, CometShuffleMemoryAllocator, CometShuffleMemoryAllocatorTrait} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.UnsafeProjection import org.apache.spark.sql.execution.UnsafeRowSerializer import org.apache.spark.sql.types.{BinaryType, StructField, StructType} import org.apache.spark.util.Utils -class CometDiskBlockWriterSuite extends AnyFunSuite { +class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { + + private implicit val signaler: Signaler = ThreadSignaler private val schema = StructType(Seq(StructField("a", BinaryType))) private val pageSize: Long = 256 * 1024 @@ -200,19 +205,22 @@ class CometDiskBlockWriterSuite extends AnyFunSuite { writer.insertRow(toUnsafe(InternalRow(new Array[Byte](1024))), 0) } - // Task B fills the whole 1 MiB pool (4 pages) and idles. + // Task B fills the whole 1 MiB pool (4 pages) on its own thread, idles for a while, and + // then finishes, freeing the pool. var rowsB = 0L - while (writerB.getActiveMemoryUsage < 4 * pageSize) { - insertOne(writerB) - rowsB += 1 - } - - // Task B finishes a bit later on its own thread, freeing the pool. + var segmentBLength = 0L + val poolFilled = new CountDownLatch(1) val threadB = new Thread(() => { + while (writerB.getActiveMemoryUsage < 4 * pageSize) { + insertOne(writerB) + rowsB += 1 + } + poolFilled.countDown() Thread.sleep(500) - writerB.close() + segmentBLength = writerB.close().length }) threadB.start() + poolFilled.await() // Task A's first insert finds the pool exhausted and has nothing of its own to spill; it // must wait for task B to finish instead of throwing SparkOutOfMemoryError. @@ -227,6 +235,7 @@ class CometDiskBlockWriterSuite extends AnyFunSuite { assert(writerA.getOutputRecords == rowsA) assert(writerB.getOutputRecords == rowsB) assert(segmentA.length > 0) + assert(segmentBLength > 0) // Neither task spilled: A waited instead of stealing B's memory, and B was never touched. assert(taskContextA.taskMetrics.diskBytesSpilled == 0) assert(taskContextB.taskMetrics.diskBytesSpilled == 0) @@ -238,6 +247,128 @@ class CometDiskBlockWriterSuite extends AnyFunSuite { } } + test("on-heap shared pool: unsatisfiable allocations fail fast instead of waiting") { + val conf = new SparkConf().set("spark.comet.memoryOverhead", "1") // 1 MiB shared pool + resetOnHeapAllocatorSingleton() + val memoryManager = new TestMemoryManager(conf) + val tmm = new TaskMemoryManager(memoryManager, 0L) + val taskContext = newTaskContext(tmm, 0L) + val allocator = CometShuffleMemoryAllocator.getInstance(conf, tmm, pageSize) + val tempDir = Utils.createTempDir() + try { + val writer = newWriter(new File(tempDir, "unsatisfiable"), allocator, taskContext, conf) + val toUnsafe = UnsafeProjection.create(schema) + failAfter(Span(60, Seconds)) { + // A row larger than the whole pool can never be satisfied. + intercept[SparkOutOfMemoryError] { + writer.insertRow(toUnsafe(InternalRow(new Array[Byte](2 * 1024 * 1024))), 0) + } + // A request that does not fit next to memory this thread itself retains (like the + // sorter's pointer array, which survives an empty spill) can never be satisfied either. + val bounded = allocator.asInstanceOf[CometBoundedShuffleMemoryAllocator] + val retained = bounded.allocateArray(pageSize / 8) // retain one page worth of longs + try { + intercept[SparkOutOfMemoryError] { + writer.insertRow(toUnsafe(InternalRow(new Array[Byte](900 * 1024))), 0) + } + } finally { + bounded.freeArray(retained) + } + } + writer.freeMemory() + } finally { + Utils.deleteRecursively(tempDir) + tmm.cleanUpAllAllocatedMemory() + resetOnHeapAllocatorSingleton() + } + } + + test("on-heap shared pool: interrupting a waiting task does not throw a fatal error") { + val conf = new SparkConf().set("spark.comet.memoryOverhead", "1") // 1 MiB shared pool + resetOnHeapAllocatorSingleton() + val memoryManager = new TestMemoryManager(conf) + val tmmHolder = new TaskMemoryManager(memoryManager, 0L) + val tmmWaiter = new TaskMemoryManager(memoryManager, 1L) + val taskContextHolder = newTaskContext(tmmHolder, 0L) + val taskContextWaiter = newTaskContext(tmmWaiter, 1L) + val allocatorHolder = CometShuffleMemoryAllocator.getInstance(conf, tmmHolder, pageSize) + val allocatorWaiter = CometShuffleMemoryAllocator.getInstance(conf, tmmWaiter, pageSize) + val tempDir = Utils.createTempDir() + try { + val writerHolder = + newWriter(new File(tempDir, "holder"), allocatorHolder, taskContextHolder, conf) + val writerWaiter = + newWriter(new File(tempDir, "waiter"), allocatorWaiter, taskContextWaiter, conf) + val toUnsafe = UnsafeProjection.create(schema) + def insertOne(writer: CometDiskBlockWriter): Unit = { + writer.insertRow(toUnsafe(InternalRow(new Array[Byte](1024))), 0) + } + + failAfter(Span(60, Seconds)) { + // The holder fills the whole pool on its own thread and keeps it until released. + val poolFilled = new CountDownLatch(1) + val release = new CountDownLatch(1) + val holderThread = new Thread(() => { + while (writerHolder.getActiveMemoryUsage < 4 * pageSize) { + insertOne(writerHolder) + } + poolFilled.countDown() + release.await() + writerHolder.freeMemory() + }) + holderThread.start() + poolFilled.await() + + // The waiter blocks on its first insert, and is then killed via interrupt. + @volatile var thrown: Throwable = null + val waiterThread = new Thread(() => { + try insertOne(writerWaiter) + catch { case t: Throwable => thrown = t } + }) + waiterThread.start() + while (waiterThread.getState != Thread.State.WAITING) { + Thread.sleep(10) + } + waiterThread.interrupt() + waiterThread.join() + + // The interruption must not surface as a fatal error such as SparkOutOfMemoryError, + // otherwise an intentional task kill is reported as ExceptionFailure instead of + // TaskKilled. + assert(thrown != null) + assert(!thrown.isInstanceOf[OutOfMemoryError]) + assert(thrown.isInstanceOf[RuntimeException]) + assert(thrown.getCause.isInstanceOf[InterruptedException]) + + release.countDown() + holderThread.join() + writerWaiter.freeMemory() + } + } finally { + Utils.deleteRecursively(tempDir) + tmmHolder.cleanUpAllAllocatedMemory() + tmmWaiter.cleanUpAllAllocatedMemory() + resetOnHeapAllocatorSingleton() + } + } + + private def newWriter( + file: File, + allocator: CometShuffleMemoryAllocatorTrait, + taskContext: TaskContextImpl, + conf: SparkConf): CometDiskBlockWriter = { + new CometDiskBlockWriter( + file, + allocator, + taskContext, + new UnsafeRowSerializer(1).newInstance(), + schema, + new ShuffleWriteMetrics, + conf, + false, + new JLinkedList[CometDiskBlockWriter]()) + } + /** Clears the CometShuffleMemoryAllocator singleton so this suite controls its pool size. */ private def resetOnHeapAllocatorSingleton(): Unit = { val field = classOf[CometShuffleMemoryAllocator].getDeclaredField("INSTANCE") From de5d00a9bbd0ef33f246944026e7717ec87dca6e Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 27 Aug 2026 21:33:17 +0800 Subject: [PATCH 05/15] address review: no false deadlock while another waiter can proceed The deadlock check counted notified-but-not-yet-resumed waiters as permanently blocked: after a holder freed the pool, a large waiter that resumed first could declare deadlock even though a smaller waiter, still registered as waiting, could satisfy its request from the free pool, finish, and release memory for the large one. Track each waiter's pending request size and only declare deadlock when, in addition to all allocated memory being retained by blocked waiters, none of their requests fits in the free pool. A waiter whose request fits will proceed once it reacquires the monitor and eventually free what it retains, so the wait remains live. --- .../CometBoundedShuffleMemoryAllocator.java | 31 +++++++--- .../shuffle/CometDiskBlockWriterSuite.scala | 61 +++++++++++++++++++ 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java index 55e9fb032bb..336635eb57e 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.BitSet; import java.util.HashMap; -import java.util.HashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,8 +79,8 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory /** Pool memory currently retained by each thread. */ private final HashMap retainedMemory = new HashMap<>(); - /** Threads currently blocked in {@link #allocateBlocking(long)}. */ - private final HashSet waitingThreads = new HashSet<>(); + /** Threads currently blocked in {@link #allocateBlocking(long)} and their request sizes. */ + private final HashMap waitingThreads = new HashMap<>(); private static final int OFFSET_BITS = 51; private static final long MASK_LONG_LOWER_51_BITS = 0x7FFFFFFFFFFFFL; @@ -135,7 +134,8 @@ public synchronized MemoryBlock allocate(long required) { * Callers must only use this after spilling their own buffered data. The wait fails fast when it * can never succeed: when the request does not fit next to the memory this thread itself still * retains (e.g. the sorter's pointer array), or when all allocated memory is retained by threads - * that are themselves blocked here. Interrupting the task (e.g. task kill) aborts the wait. + * that are themselves blocked here and none of their requests fits in the free pool. Interrupting + * the task (e.g. task kill) aborts the wait. */ @Override public synchronized MemoryBlock allocateBlocking(long required) { @@ -147,7 +147,7 @@ public synchronized MemoryBlock allocateBlocking(long required) { try { return allocateMemoryBlock(size); } catch (SparkOutOfMemoryError e) { - if (waitingThreads.add(self)) { + if (waitingThreads.put(self, size) == null) { // Wake existing waiters so they re-evaluate the deadlock check against the enlarged // waiting set. notifyAll(); @@ -157,10 +157,11 @@ public synchronized MemoryBlock allocateBlocking(long required) { if (size > totalMemory - retainedMemory.getOrDefault(self, 0L)) { throw e; } - // The allocation just failed, so the request does not fit in the unallocated pool. If - // all allocated memory is retained by blocked threads, nothing can be freed anymore - // and waiting would deadlock. - if (allocatedMemory <= retainedByWaitingThreads()) { + // The allocation just failed, so the request does not fit in the unallocated pool. + // Waiting can only succeed while some thread can still free memory: either a thread + // outside the waiting set retains pool memory, or another waiter's request fits in the + // free pool, in which case that waiter can proceed and eventually free what it retains. + if (allocatedMemory <= retainedByWaitingThreads() && !anyWaiterCanProceed()) { throw e; } if (!logged) { @@ -186,12 +187,22 @@ public synchronized MemoryBlock allocateBlocking(long required) { private long retainedByWaitingThreads() { long retained = 0; - for (Thread thread : waitingThreads) { + for (Thread thread : waitingThreads.keySet()) { retained += retainedMemory.getOrDefault(thread, 0L); } return retained; } + private boolean anyWaiterCanProceed() { + long free = totalMemory - allocatedMemory; + for (long requested : waitingThreads.values()) { + if (requested <= free) { + return true; + } + } + return false; + } + private synchronized MemoryBlock allocateMemoryBlock(long required) { if (required > TaskMemoryManager.MAXIMUM_PAGE_SIZE_BYTES) { throw new TooLargePageException(required); diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 98f2c53d3c1..195f99833d0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -352,6 +352,67 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { } } + test("on-heap shared pool: no false deadlock while another waiter can proceed") { + // Two waiters each retain a small pointer array while a holder owns most of the pool. When + // the holder frees, the large waiter's request still does not fit, but the small waiter's + // does: the small waiter must be allowed to proceed and finish, unblocking the large one, + // instead of either waiter being declared deadlocked. + val conf = new SparkConf().set("spark.comet.memoryOverhead", "1") // 1 MiB shared pool + resetOnHeapAllocatorSingleton() + val memoryManager = new TestMemoryManager(conf) + val tmm = new TaskMemoryManager(memoryManager, 0L) + val bounded = CometShuffleMemoryAllocator + .getInstance(conf, tmm, pageSize) + .asInstanceOf[CometBoundedShuffleMemoryAllocator] + try { + failAfter(Span(60, Seconds)) { + val holderBlock = bounded.allocate(900 * 1024) + + @volatile var largeError: Throwable = null + @volatile var smallError: Throwable = null + val arraysAllocated = new CountDownLatch(2) + val largeWaiter = new Thread(() => { + val array = bounded.allocateArray(4096) // retains 32768 bytes + arraysAllocated.countDown() + try { + bounded.free(bounded.allocateBlocking(999448)) + } catch { + case t: Throwable => largeError = t + } finally { + bounded.freeArray(array) + } + }) + val smallWaiter = new Thread(() => { + val array = bounded.allocateArray(4096) // retains 32768 bytes + arraysAllocated.countDown() + try { + bounded.free(bounded.allocateBlocking(262144)) + } catch { + case t: Throwable => smallError = t + } finally { + bounded.freeArray(array) + } + }) + largeWaiter.start() + smallWaiter.start() + arraysAllocated.await() + while (largeWaiter.getState != Thread.State.WAITING || + smallWaiter.getState != Thread.State.WAITING) { + Thread.sleep(10) + } + + bounded.free(holderBlock) + largeWaiter.join() + smallWaiter.join() + assert(largeError == null) + assert(smallError == null) + } + } finally { + tmm.cleanUpAllAllocatedMemory() + resetOnHeapAllocatorSingleton() + } + } + private def newWriter( file: File, allocator: CometShuffleMemoryAllocatorTrait, From cab20e38fa99fdbe13ef116fcd5dc22ec26443c6 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 28 Aug 2026 00:03:15 +0800 Subject: [PATCH 06/15] address review: log periodically while waiting for pool memory A task blocked waiting for the shared on-heap pool logged a single warning and then went silent, so a long stall (e.g. a holder stuck on I/O) was invisible to operators. Use a timed wait and re-log every 30 seconds with the elapsed time, requested size, free bytes, and number of waiting threads. The wait itself remains unbounded (a fixed deadline was shown in review to fail workloads whose holder legitimately outlives the cap); the existing fail-fast checks still cover the provably-unsatisfiable cases. Also harden the suite's thread-state polls to accept TIMED_WAITING now that the allocator uses a timed wait. --- .../CometBoundedShuffleMemoryAllocator.java | 28 +++++++++++++++---- .../shuffle/CometDiskBlockWriterSuite.scala | 11 ++++++-- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java index 336635eb57e..9b611bc5326 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java @@ -64,6 +64,9 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory private final long totalMemory; private long allocatedMemory = 0L; + /** How often a thread blocked in {@link #allocateBlocking(long)} logs that it is waiting. */ + private static final long WAIT_LOG_INTERVAL_MS = 30_000L; + /** The number of bits used to address the page table. */ private static final int PAGE_NUMBER_BITS = 13; @@ -141,7 +144,8 @@ public synchronized MemoryBlock allocate(long required) { public synchronized MemoryBlock allocateBlocking(long required) { long size = Math.max(pageSize, required); Thread self = Thread.currentThread(); - boolean logged = false; + long waitStart = 0; + long lastLog = 0; try { while (true) { try { @@ -164,17 +168,31 @@ public synchronized MemoryBlock allocateBlocking(long required) { if (allocatedMemory <= retainedByWaitingThreads() && !anyWaiterCanProceed()) { throw e; } - if (!logged) { + // Log when the wait starts and periodically while it lasts, so a stalled task is + // visible in the executor logs. + long now = System.currentTimeMillis(); + if (waitStart == 0) { + waitStart = now; + lastLog = now; logger.warn( "Waiting for other tasks to free up {} bytes of Comet shuffle pool memory", size); - logged = true; + } else if (now - lastLog >= WAIT_LOG_INTERVAL_MS) { + lastLog = now; + logger.warn( + "Still waiting ({} ms so far) for {} bytes of Comet shuffle pool memory; " + + "{} bytes free, {} thread(s) waiting", + now - waitStart, + size, + totalMemory - allocatedMemory, + waitingThreads.size()); } try { - wait(); + wait(WAIT_LOG_INTERVAL_MS); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); // Not an allocation failure: stay non-fatal so that an intentional task kill is - // classified as TaskKilled rather than ExceptionFailure. + // classified as TaskKilled rather than ExceptionFailure (Spark's killed-task handler + // only matches `InterruptedException | NonFatal(_)`). throw new RuntimeException( "Interrupted while waiting for Comet shuffle pool memory", ie); } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 195f99833d0..196ac843a6e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -326,7 +326,7 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { catch { case t: Throwable => thrown = t } }) waiterThread.start() - while (waiterThread.getState != Thread.State.WAITING) { + while (!isBlockedInWait(waiterThread)) { Thread.sleep(10) } waiterThread.interrupt() @@ -396,8 +396,7 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { largeWaiter.start() smallWaiter.start() arraysAllocated.await() - while (largeWaiter.getState != Thread.State.WAITING || - smallWaiter.getState != Thread.State.WAITING) { + while (!isBlockedInWait(largeWaiter) || !isBlockedInWait(smallWaiter)) { Thread.sleep(10) } @@ -430,6 +429,12 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { new JLinkedList[CometDiskBlockWriter]()) } + /** Whether the thread is parked in `Object.wait` (the allocator uses a timed wait). */ + private def isBlockedInWait(thread: Thread): Boolean = { + val state = thread.getState + state == Thread.State.WAITING || state == Thread.State.TIMED_WAITING + } + /** Clears the CometShuffleMemoryAllocator singleton so this suite controls its pool size. */ private def resetOnHeapAllocatorSingleton(): Unit = { val field = classOf[CometShuffleMemoryAllocator].getDeclaredField("INSTANCE") From a21f3f19ee6782fcb118fd00649dd85b0d0b2eb6 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 28 Aug 2026 01:51:01 +0800 Subject: [PATCH 07/15] address review: reclaim allocations when the SpillSorter constructor fails SpillSorter's constructor allocates the one-entry ShuffleInMemorySorter array and then the real pointer array. When the second allocation failed, the first was orphaned: the writer is never handed to Spark, so no cleanup path could ever free it. Pre-existing, the orphan was only a tiny pool leak; with blocking allocation it also broke the wait's progress condition, since the orphaned bytes look like memory a live holder could still free, turning a formerly prompt SparkOutOfMemoryError for a later full-pool request into an unbounded wait. Free the partially-constructed sorter's allocations before rethrowing, so every allocated byte again has an owner with a cleanup path - the invariant the blocking wait relies on. --- .../spark/shuffle/sort/SpillSorter.java | 17 ++++++- .../shuffle/CometDiskBlockWriterSuite.scala | 45 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/sort/SpillSorter.java b/spark/src/main/java/org/apache/spark/shuffle/sort/SpillSorter.java index 36b50e6202a..77c5ff27fdf 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/sort/SpillSorter.java +++ b/spark/src/main/java/org/apache/spark/shuffle/sort/SpillSorter.java @@ -131,8 +131,21 @@ public SpillSorter( + "https://github.com/apache/arrow-datafusion-comet?tab=readme-ov-file#enable-comet-shuffle", e); } - sorterArray = allocator.allocateArray(initialSize); - this.inMemSorter.expandPointerArray(sorterArray); + try { + sorterArray = allocator.allocateArray(initialSize); + this.inMemSorter.expandPointerArray(sorterArray); + } catch (Throwable t) { + // This writer is never handed to Spark when its constructor fails, so nothing else could + // reclaim what was allocated so far; free it here to keep the shared pool leak-free. When + // we get here `expandPointerArray` has not adopted `sorterArray`, so the sorter still owns + // only its initial array. + if (sorterArray != null) { + allocator.freeArray(sorterArray); + sorterArray = null; + } + this.inMemSorter.free(); + throw t; + } this.allocatedPages = new LinkedList<>(); diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 196ac843a6e..df2d33ea63d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -31,10 +31,12 @@ import org.apache.spark.{SparkConf, TaskContextImpl} import org.apache.spark.executor.{ShuffleWriteMetrics, TaskMetrics} import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager} import org.apache.spark.shuffle.comet.{CometBoundedShuffleMemoryAllocator, CometShuffleMemoryAllocator, CometShuffleMemoryAllocatorTrait} +import org.apache.spark.shuffle.sort.SpillSorter import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.UnsafeProjection import org.apache.spark.sql.execution.UnsafeRowSerializer import org.apache.spark.sql.types.{BinaryType, StructField, StructType} +import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.spark.util.Utils class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { @@ -429,6 +431,49 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { new JLinkedList[CometDiskBlockWriter]()) } + test("on-heap shared pool: a failed SpillSorter constructor does not leak pool memory") { + // The unsafe sorter's constructor first allocates a one-entry array (8 bytes) inside + // ShuffleInMemorySorter and then its real pointer array. If the second allocation fails, the + // first must be reclaimed: the writer is never handed to Spark, so no cleanup path would + // ever free it, and the orphaned bytes would make later full-pool allocations impossible. + val conf = new SparkConf().set("spark.comet.memoryOverhead", "1") // 1 MiB shared pool + resetOnHeapAllocatorSingleton() + val memoryManager = new TestMemoryManager(conf) + val tmm = new TaskMemoryManager(memoryManager, 0L) + val taskContext = newTaskContext(tmm, 0L) + val bounded = CometShuffleMemoryAllocator + .getInstance(conf, tmm, pageSize) + .asInstanceOf[CometBoundedShuffleMemoryAllocator] + try { + // A healthy task holds most of the pool, leaving room for the one-entry array but not for + // the 4096-entry pointer array. + val holderBlock = bounded.allocate(1008 * 1024) + intercept[SparkOutOfMemoryError] { + new SpillSorter( + bounded, + 4096, + schema, + UnsafeAlignedOffset.getUaoSize(), + 1.0, + "zstd", + 1, + "adler32", + new Array[Long](0), + new ShuffleWriteMetrics, + taskContext, + new JLinkedList[SpillInfo](), + () => ()) + } + bounded.free(holderBlock) + // With the constructor cleanup the pool is empty again, so a full-pool allocation + // succeeds; a leaked constructor allocation would make it fail forever. + bounded.free(bounded.allocate(1024 * 1024)) + } finally { + tmm.cleanUpAllAllocatedMemory() + resetOnHeapAllocatorSingleton() + } + } + /** Whether the thread is parked in `Object.wait` (the allocator uses a timed wait). */ private def isBlockedInWait(thread: Thread): Boolean = { val state = thread.getState From a1a4a84ef319eb41154f13e9bae78c69a7c8eddd Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 28 Aug 2026 04:24:26 +0800 Subject: [PATCH 08/15] address review: free buffered pages when write() fails with a fatal error Spark's ShuffleWriteProcessor only calls stop(false) when the shuffle writer throws an Exception, so a fatal error such as SparkOutOfMemoryError (e.g. from the blocking allocator's fail-fast checks) skipped cleanup and orphaned the task's buffered pages. Those pages live in the executor-shared bounded pool that Spark's task-memory cleanup does not own, so on-heap they starved other tasks' allocations - and with blocking allocation, an unrelated waiter could wait on them indefinitely. Catch Throwable in CometBypassMergeSortShuffleWriter.write() and free all partition writers' memory before rethrowing (freeMemory is a no-op when stop(false) runs afterwards on Exception paths). CometUnsafeShuffleWriter already handles generic throwables via its success/finally pattern. --- .../CometBypassMergeSortShuffleWriter.java | 16 +++- .../shuffle/CometDiskBlockWriterSuite.scala | 90 ++++++++++++++++++- 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java index 8a510b9128f..d833667cd1e 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java @@ -259,7 +259,21 @@ public void write(Iterator> records) throws IOException { // TODO: We probably can move checksum generation here when concatenating partition files partitionLengths = writePartitionedData(mapOutputWriter); mapStatus = MapStatusHelper.apply(blockManager.shuffleServerId(), partitionLengths, mapId); - } catch (Exception e) { + } catch (Throwable e) { + // Spark only calls stop(false) when write() throws an Exception; a fatal error such as + // SparkOutOfMemoryError skips it, and the buffered pages are invisible to Spark's + // task-memory cleanup (in on-heap mode they live in an executor-shared pool). Free them + // here so they cannot starve other tasks' allocations. + if (partitionWriters != null) { + for (CometDiskBlockWriter writer : partitionWriters) { + try { + writer.freeMemory(); + } catch (Exception e2) { + logger.error("Failed to free memory of partition writer", e2); + e.addSuppressed(e2); + } + } + } try { mapOutputWriter.abort(e); } catch (Exception e2) { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index df2d33ea63d..936da527ae5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -27,14 +27,17 @@ import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.time.{Seconds, Span} -import org.apache.spark.{SparkConf, TaskContextImpl} +import org.apache.spark.{Partitioner, SparkConf, SparkContext, SparkEnv, TaskContextImpl} import org.apache.spark.executor.{ShuffleWriteMetrics, TaskMetrics} import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager} +import org.apache.spark.shuffle.api.{ShuffleExecutorComponents, ShuffleMapOutputWriter, ShufflePartitionWriter} +import org.apache.spark.shuffle.api.metadata.MapOutputCommitMessage import org.apache.spark.shuffle.comet.{CometBoundedShuffleMemoryAllocator, CometShuffleMemoryAllocator, CometShuffleMemoryAllocatorTrait} import org.apache.spark.shuffle.sort.SpillSorter import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.UnsafeProjection +import org.apache.spark.sql.catalyst.expressions.{UnsafeProjection, UnsafeRow} import org.apache.spark.sql.execution.UnsafeRowSerializer +import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types.{BinaryType, StructField, StructType} import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.spark.util.Utils @@ -431,6 +434,89 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { new JLinkedList[CometDiskBlockWriter]()) } + test("on-heap shared pool: a fatal error during write() frees the task's buffered pages") { + // Spark's ShuffleWriteProcessor only calls stop(false) when write() throws an Exception, so + // a fatal error such as SparkOutOfMemoryError skips it. The buffered pages live in the + // executor-shared bounded pool where Spark's task-memory cleanup cannot see them, so + // write() itself must free them on the way out or they starve other tasks forever. + val conf = new SparkConf() + .setMaster("local[1]") + .setAppName("CometDiskBlockWriterSuite") + .set("spark.comet.memoryOverhead", "1") // 1 MiB shared pool + .set("spark.buffer.pageSize", "256k") + resetOnHeapAllocatorSingleton() + val sc = new SparkContext(conf) + val memoryManager = new TestMemoryManager(conf) + val tmm = new TaskMemoryManager(memoryManager, 0L) + try { + val taskContext = newTaskContext(tmm, 0L) + val partitioner = new Partitioner { + override def numPartitions: Int = 3 + override def getPartition(key: Any): Int = key.asInstanceOf[Int] % 3 + } + val dep = new CometShuffleDependency[Int, UnsafeRow, UnsafeRow]( + _rdd = sc.parallelize(Seq.empty[(Int, UnsafeRow)], 1), + partitioner = partitioner, + serializer = new UnsafeRowSerializer(1), + schema = Some(schema), + decodeTime = new SQLMetric("nsTiming")) + val components = new ShuffleExecutorComponents { + override def initializeExecutor( + appId: String, + execId: String, + extraConfigs: java.util.Map[String, String]): Unit = {} + override def createMapOutputWriter( + shuffleId: Int, + mapTaskId: Long, + numPartitions: Int): ShuffleMapOutputWriter = new ShuffleMapOutputWriter { + override def getPartitionWriter(reducePartitionId: Int): ShufflePartitionWriter = + throw new UnsupportedOperationException + override def commitAllPartitions(checksums: Array[Long]): MapOutputCommitMessage = + throw new UnsupportedOperationException + override def abort(error: Throwable): Unit = {} + } + } + val writer = new CometBypassMergeSortShuffleWriter[Int, UnsafeRow]( + SparkEnv.get.blockManager, + tmm, + taskContext, + new CometBypassMergeSortShuffleHandle[Int, UnsafeRow](0, dep), + 0L, + conf, + taskContext.taskMetrics.shuffleWriteMetrics, + components, + null) + + // Buffer three pages worth of rows, then hit a fatal error mid-write. + val toUnsafe = UnsafeProjection.create(schema) + val rows: Iterator[Product2[Int, UnsafeRow]] = + (0 until 700).iterator.map { i => + (i % 3, toUnsafe(InternalRow(new Array[Byte](1024)))) + } ++ new Iterator[Product2[Int, UnsafeRow]] { + override def hasNext: Boolean = true + override def next(): Product2[Int, UnsafeRow] = { + throw new SparkOutOfMemoryError( + "UNABLE_TO_ACQUIRE_MEMORY", + java.util.Map.of("requestedBytes", "1", "receivedBytes", "0")) + } + } + intercept[SparkOutOfMemoryError] { + writer.write(rows) + } + + // All pages buffered by the failed task were reclaimed, so a full-pool allocation + // succeeds; without the reclaim the orphaned pages would make it fail forever. + val bounded = CometShuffleMemoryAllocator + .getInstance(conf, tmm, pageSize) + .asInstanceOf[CometBoundedShuffleMemoryAllocator] + bounded.free(bounded.allocate(1024 * 1024)) + } finally { + sc.stop() + tmm.cleanUpAllAllocatedMemory() + resetOnHeapAllocatorSingleton() + } + } + test("on-heap shared pool: a failed SpillSorter constructor does not leak pool memory") { // The unsafe sorter's constructor first allocates a one-entry array (8 bytes) inside // ShuffleInMemorySorter and then its real pointer array. If the second allocation fails, the From 844b30867c6ee3b80bf1bdc845f5f960e14f2978 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 28 Aug 2026 14:30:29 +0800 Subject: [PATCH 09/15] address review: reclaim the unsafe writer's pre-write allocation at task completion The unsafe writer's sorter allocates its pointer array at construction, before Spark has evaluated the shuffle input iterator and called write(). A fatal error during iterator evaluation (e.g. SparkOutOfMemoryError from an eager input sort) therefore skipped every existing cleanup path: the writer's success/finally scope never starts, ShuffleWriteProcessor only calls stop(false) for exceptions, and Spark's task-memory cleanup does not own Comet's pool. The orphaned allocation could then starve other tasks' blocking waits indefinitely. Register a TaskCompletionListener at writer construction that runs sorter.cleanupResources() if the sorter is still alive; listeners run in Task.run's finally regardless of failure type, and cleanupResources() is idempotent so this is a no-op whenever write()/stop() already cleaned up. --- .../shuffle/CometUnsafeShuffleWriter.java | 15 +++ .../shuffle/CometDiskBlockWriterSuite.scala | 93 +++++++++++++++---- 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java index 7997f60532a..f19acac0ff8 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java @@ -71,6 +71,7 @@ import org.apache.spark.storage.BlockManager; import org.apache.spark.storage.TimeTrackingOutputStream; import org.apache.spark.unsafe.Platform; +import org.apache.spark.util.TaskCompletionListener; import org.apache.spark.util.Utils; import com.google.common.annotations.VisibleForTesting; @@ -177,6 +178,20 @@ public CometUnsafeShuffleWriter( (int) (long) sparkConf.get(package$.MODULE$.SHUFFLE_FILE_BUFFER_SIZE()) * 1024; this.tracingEnabled = (boolean) CometConf.COMET_TRACING_ENABLED().get(); this.encodeTimeMetric = encodeTimeMetric; + // The sorter allocates its pointer array at construction time, before Spark has evaluated + // the shuffle input iterator and called write(). A fatal error in between (Spark only calls + // stop() for exceptions) would orphan that allocation, which Spark's task-memory cleanup + // does not own, so reclaim it at task completion. cleanupResources() is idempotent, making + // this a no-op whenever write()/stop() already cleaned up. + if (taskContext != null) { + taskContext.addTaskCompletionListener( + (TaskCompletionListener) + context -> { + if (sorter != null) { + sorter.cleanupResources(); + } + }); + } open(); } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 936da527ae5..545050c9c66 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -460,22 +460,6 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { serializer = new UnsafeRowSerializer(1), schema = Some(schema), decodeTime = new SQLMetric("nsTiming")) - val components = new ShuffleExecutorComponents { - override def initializeExecutor( - appId: String, - execId: String, - extraConfigs: java.util.Map[String, String]): Unit = {} - override def createMapOutputWriter( - shuffleId: Int, - mapTaskId: Long, - numPartitions: Int): ShuffleMapOutputWriter = new ShuffleMapOutputWriter { - override def getPartitionWriter(reducePartitionId: Int): ShufflePartitionWriter = - throw new UnsupportedOperationException - override def commitAllPartitions(checksums: Array[Long]): MapOutputCommitMessage = - throw new UnsupportedOperationException - override def abort(error: Throwable): Unit = {} - } - } val writer = new CometBypassMergeSortShuffleWriter[Int, UnsafeRow]( SparkEnv.get.blockManager, tmm, @@ -484,7 +468,7 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { 0L, conf, taskContext.taskMetrics.shuffleWriteMetrics, - components, + newShuffleExecutorComponents(), null) // Buffer three pages worth of rows, then hit a fatal error mid-write. @@ -560,6 +544,81 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { } } + test("on-heap shared pool: task completion reclaims the unsafe writer's pre-write allocation") { + // The unsafe writer's sorter allocates its pointer array at construction, before Spark has + // evaluated the shuffle input iterator and called write(). When iterator evaluation fails + // with a fatal error, neither write() cleanup nor stop(false) runs; the task-completion + // listener must reclaim the allocation so other tasks are not starved by it. + val conf = new SparkConf() + .setMaster("local[1]") + .setAppName("CometDiskBlockWriterSuite") + .set("spark.comet.memoryOverhead", "1") // 1 MiB shared pool + .set("spark.buffer.pageSize", "256k") + resetOnHeapAllocatorSingleton() + val sc = new SparkContext(conf) + val memoryManager = new TestMemoryManager(conf) + val tmm = new TaskMemoryManager(memoryManager, 0L) + try { + val taskContext = newTaskContext(tmm, 0L) + val partitioner = new Partitioner { + override def numPartitions: Int = 4 + override def getPartition(key: Any): Int = key.asInstanceOf[Int] % 4 + } + val dep = new CometShuffleDependency[Int, UnsafeRow, UnsafeRow]( + _rdd = sc.parallelize(Seq.empty[(Int, UnsafeRow)], 1), + partitioner = partitioner, + serializer = new UnsafeRowSerializer(1), + schema = Some(schema), + decodeTime = new SQLMetric("nsTiming")) + // Construct the writer exactly as Spark does before evaluating the input iterator; its + // sorter allocates the pointer array from the shared pool. + new CometUnsafeShuffleWriter[Int, UnsafeRow]( + SparkEnv.get.blockManager, + tmm, + new CometSerializedShuffleHandle[Int, UnsafeRow](0, dep), + 0L, + taskContext, + conf, + taskContext.taskMetrics.shuffleWriteMetrics, + newShuffleExecutorComponents(), + null) + val bounded = CometShuffleMemoryAllocator + .getInstance(conf, tmm, pageSize) + .asInstanceOf[CometBoundedShuffleMemoryAllocator] + // The pre-write pointer array occupies the pool... + intercept[SparkOutOfMemoryError] { + bounded.allocate(1024 * 1024) + } + // ... and after the task completes without write() or stop() ever running, the + // allocation is reclaimed instead of starving other tasks forever. + taskContext.markTaskCompleted(None) + bounded.free(bounded.allocate(1024 * 1024)) + } finally { + sc.stop() + tmm.cleanUpAllAllocatedMemory() + resetOnHeapAllocatorSingleton() + } + } + + private def newShuffleExecutorComponents(): ShuffleExecutorComponents = { + new ShuffleExecutorComponents { + override def initializeExecutor( + appId: String, + execId: String, + extraConfigs: java.util.Map[String, String]): Unit = {} + override def createMapOutputWriter( + shuffleId: Int, + mapTaskId: Long, + numPartitions: Int): ShuffleMapOutputWriter = new ShuffleMapOutputWriter { + override def getPartitionWriter(reducePartitionId: Int): ShufflePartitionWriter = + throw new UnsupportedOperationException + override def commitAllPartitions(checksums: Array[Long]): MapOutputCommitMessage = + throw new UnsupportedOperationException + override def abort(error: Throwable): Unit = {} + } + } + } + /** Whether the thread is parked in `Object.wait` (the allocator uses a timed wait). */ private def isBlockedInWait(thread: Thread): Boolean = { val state = thread.getState From 4b592b9c21fc5896fea04c023459788926e3fe70 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 28 Aug 2026 17:13:46 +0800 Subject: [PATCH 10/15] address review: cover late SpillSorter constructor failures too The constructor cleanup only wrapped the pointer-array allocation, but serializeSchema (and the Native library load) run after the array is adopted and can still throw - e.g. schema serialization rejects a nested field whose parquet.field.id exceeds the 32-bit range. The enclosing sorter field is never assigned in that case, so not even the unsafe writer's task-completion listener could reclaim the adopted 32 KiB array. Widen the cleanup to the rest of the constructor and track whether the sorter adopted the array, so exactly one owner frees it: before adoption the array is freed directly (the sorter still owns only its initial one-entry array), after adoption inMemSorter.free() releases it, avoiding a double free that would trip TaskMemoryManager assertions in off-heap mode. Also skip null entries in the bypass writer's failure cleanup, in case a mid-loop writer construction failure left the tail of the array unassigned. --- .../spark/shuffle/sort/SpillSorter.java | 20 +++++++------ .../CometBypassMergeSortShuffleWriter.java | 3 ++ .../shuffle/CometDiskBlockWriterSuite.scala | 30 ++++++++++++++++++- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/sort/SpillSorter.java b/spark/src/main/java/org/apache/spark/shuffle/sort/SpillSorter.java index 77c5ff27fdf..2bff92f9fa2 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/sort/SpillSorter.java +++ b/spark/src/main/java/org/apache/spark/shuffle/sort/SpillSorter.java @@ -131,26 +131,28 @@ public SpillSorter( + "https://github.com/apache/arrow-datafusion-comet?tab=readme-ov-file#enable-comet-shuffle", e); } + boolean adopted = false; try { sorterArray = allocator.allocateArray(initialSize); this.inMemSorter.expandPointerArray(sorterArray); + adopted = true; + + this.allocatedPages = new LinkedList<>(); + + this.nativeLib = new Native(); + this.dataTypes = serializeSchema(schema); } catch (Throwable t) { // This writer is never handed to Spark when its constructor fails, so nothing else could - // reclaim what was allocated so far; free it here to keep the shared pool leak-free. When - // we get here `expandPointerArray` has not adopted `sorterArray`, so the sorter still owns - // only its initial array. - if (sorterArray != null) { + // reclaim what was allocated so far; free it here to keep the shared pool leak-free. + // Before adoption the sorter still owns only its initial one-entry array; after adoption + // it owns `sorterArray`, and `free()` releases whichever it holds. + if (!adopted && sorterArray != null) { allocator.freeArray(sorterArray); sorterArray = null; } this.inMemSorter.free(); throw t; } - - this.allocatedPages = new LinkedList<>(); - - this.nativeLib = new Native(); - this.dataTypes = serializeSchema(schema); } /** Frees allocated memory pages of this writer */ diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java index d833667cd1e..80c3e2b7f15 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java @@ -266,6 +266,9 @@ public void write(Iterator> records) throws IOException { // here so they cannot starve other tasks' allocations. if (partitionWriters != null) { for (CometDiskBlockWriter writer : partitionWriters) { + if (writer == null) { + continue; + } try { writer.freeMemory(); } catch (Exception e2) { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 545050c9c66..6f9630f4cf4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -38,7 +38,7 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{UnsafeProjection, UnsafeRow} import org.apache.spark.sql.execution.UnsafeRowSerializer import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.types.{BinaryType, StructField, StructType} +import org.apache.spark.sql.types.{BinaryType, IntegerType, MetadataBuilder, StructField, StructType} import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.spark.util.Utils @@ -538,6 +538,34 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { // With the constructor cleanup the pool is empty again, so a full-pool allocation // succeeds; a leaked constructor allocation would make it fail forever. bounded.free(bounded.allocate(1024 * 1024)) + + // A failure after the pointer array is adopted (here: schema serialization rejecting an + // out-of-range parquet.field.id) must free the adopted array as well. The enclosing + // sorter field is never assigned in this case, so not even the unsafe writer's + // task-completion listener could see the allocation. + val badField = StructField( + "a", + IntegerType, + nullable = true, + new MetadataBuilder().putLong("parquet.field.id", 2147483648L).build()) + val badSchema = StructType(Seq(StructField("s", StructType(Seq(badField))))) + intercept[IllegalArgumentException] { + new SpillSorter( + bounded, + 4096, + badSchema, + UnsafeAlignedOffset.getUaoSize(), + 1.0, + "zstd", + 1, + "adler32", + new Array[Long](0), + new ShuffleWriteMetrics, + taskContext, + new JLinkedList[SpillInfo](), + () => ()) + } + bounded.free(bounded.allocate(1024 * 1024)) } finally { tmm.cleanUpAllAllocatedMemory() resetOnHeapAllocatorSingleton() From 3ca39cf33e8430608b939758e9b48f6a1447e76b Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 28 Aug 2026 18:27:09 +0800 Subject: [PATCH 11/15] address review: allocate the unsafe writer's sorter lazily in write() Spark evaluates the shuffle input iterator between constructing the shuffle writer and calling write(), and that evaluation can block on Spark's execution-memory pool (e.g. Comet's eager round-robin input sort). Because the writer's constructor allocated the sorter's pointer array up front, a task blocked in that window retained Comet pool memory while holding no place in the Comet allocator's waiting set, so a second task blocked in the Comet pool saw it as a live holder and waited forever: task A waits on Spark memory held by B, task B waits on Comet memory held by A, and neither can reach cleanup. Move open() from the constructor into write(), so no Comet pool memory is retained across blocking input construction and the cross-pool cycle cannot form. Everything that uses the sorter already runs inside write(), and stop(), peak-memory accounting, and the task-completion listener all null-guard it. The listener stays as a backstop for fatal errors during write() itself. --- .../shuffle/CometUnsafeShuffleWriter.java | 14 ++++--- .../shuffle/CometDiskBlockWriterSuite.scala | 38 +++++++++++++------ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java index f19acac0ff8..c44c3b2401e 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java @@ -178,10 +178,9 @@ public CometUnsafeShuffleWriter( (int) (long) sparkConf.get(package$.MODULE$.SHUFFLE_FILE_BUFFER_SIZE()) * 1024; this.tracingEnabled = (boolean) CometConf.COMET_TRACING_ENABLED().get(); this.encodeTimeMetric = encodeTimeMetric; - // The sorter allocates its pointer array at construction time, before Spark has evaluated - // the shuffle input iterator and called write(). A fatal error in between (Spark only calls - // stop() for exceptions) would orphan that allocation, which Spark's task-memory cleanup - // does not own, so reclaim it at task completion. cleanupResources() is idempotent, making + // Reclaim the sorter's memory at task completion if no other cleanup path ran first: Spark + // only calls stop() for exceptions, so a fatal error could otherwise orphan an allocation + // that Spark's task-memory cleanup does not own. cleanupResources() is idempotent, making // this a no-op whenever write()/stop() already cleaned up. if (taskContext != null) { taskContext.addTaskCompletionListener( @@ -192,7 +191,6 @@ public CometUnsafeShuffleWriter( } }); } - open(); } private static OutputStream openStreamUnchecked(ShufflePartitionWriter writer) { @@ -227,6 +225,12 @@ public void write(Iterator> records) throws IOException { @Override public void write(scala.collection.Iterator> records) throws IOException { + // Allocate the sorter (and its pointer array) only now: Spark evaluates the shuffle input + // iterator between constructing this writer and calling write(), and that evaluation can + // block on Spark's execution-memory pool (e.g. an eager input sort). Allocating earlier + // would retain Comet pool memory across that wait, which can form a cross-pool cycle with + // a task waiting on the Comet pool. + open(); // Keep track of success so we know if we encountered an exception // We do this rather than a standard try/catch/re-throw to handle // generic throwables. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 6f9630f4cf4..9472e8925b6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -572,11 +572,12 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { } } - test("on-heap shared pool: task completion reclaims the unsafe writer's pre-write allocation") { - // The unsafe writer's sorter allocates its pointer array at construction, before Spark has - // evaluated the shuffle input iterator and called write(). When iterator evaluation fails - // with a fatal error, neither write() cleanup nor stop(false) runs; the task-completion - // listener must reclaim the allocation so other tasks are not starved by it. + test("on-heap shared pool: the unsafe writer allocates nothing before write()") { + // Spark evaluates the shuffle input iterator between constructing the writer and calling + // write(), and that evaluation can block on Spark's execution-memory pool (e.g. an eager + // input sort). The writer must not retain Comet pool memory across that window, or two + // tasks can deadlock across the two pools; and when write() fails, even with a fatal + // error, everything it allocated must be reclaimed. val conf = new SparkConf() .setMaster("local[1]") .setAppName("CometDiskBlockWriterSuite") @@ -598,9 +599,8 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { serializer = new UnsafeRowSerializer(1), schema = Some(schema), decodeTime = new SQLMetric("nsTiming")) - // Construct the writer exactly as Spark does before evaluating the input iterator; its - // sorter allocates the pointer array from the shared pool. - new CometUnsafeShuffleWriter[Int, UnsafeRow]( + // Construct the writer exactly as Spark does before evaluating the input iterator. + val writer = new CometUnsafeShuffleWriter[Int, UnsafeRow]( SparkEnv.get.blockManager, tmm, new CometSerializedShuffleHandle[Int, UnsafeRow](0, dep), @@ -613,12 +613,26 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { val bounded = CometShuffleMemoryAllocator .getInstance(conf, tmm, pageSize) .asInstanceOf[CometBoundedShuffleMemoryAllocator] - // The pre-write pointer array occupies the pool... + // Construction must not have taken anything from the shared pool. + bounded.free(bounded.allocate(1024 * 1024)) + + // A fatal error from the record iterator mid-write must not leak the sorter's pages or + // pointer array either, and the task-completion listener stays a no-op afterwards. + val toUnsafe = UnsafeProjection.create(schema) + val rows: Iterator[Product2[Int, UnsafeRow]] = + (0 until 100).iterator.map { i => + (i % 4, toUnsafe(InternalRow(new Array[Byte](1024)))) + } ++ new Iterator[Product2[Int, UnsafeRow]] { + override def hasNext: Boolean = true + override def next(): Product2[Int, UnsafeRow] = { + throw new SparkOutOfMemoryError( + "UNABLE_TO_ACQUIRE_MEMORY", + java.util.Map.of("requestedBytes", "1", "receivedBytes", "0")) + } + } intercept[SparkOutOfMemoryError] { - bounded.allocate(1024 * 1024) + writer.write(rows) } - // ... and after the task completes without write() or stop() ever running, the - // allocation is reclaimed instead of starving other tasks forever. taskContext.markTaskCompleted(None) bounded.free(bounded.allocate(1024 * 1024)) } finally { From ab5c033ad98d99bcf667e87d1206f0edb26919a2 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 28 Aug 2026 20:55:13 +0800 Subject: [PATCH 12/15] address review: unwind waiters whose holders are blocked on Spark memory Deferring the sorter allocation to write() still reserved the pointer array before the first records.hasNext(), where lazy input (a sort or partial aggregate materializing its child) can block on Spark execution memory - and mid-write hasNext() calls can block the same way, so no placement of the allocation removes the cross-pool cycle entirely: a task blocked in Spark's pool retains Comet memory, while a Comet waiter holds Spark-side input pages the first task needs. Two changes: allocate the sorter only after the input has produced its first record, so the common lazy-materialization window retains nothing; and make a Comet waiter unwind (after at least one full wait interval) when the memory it waits for is retained by threads parked in an untimed wait inside Spark's execution-memory pool, detected from their stack frames (ExecutionMemoryPool / UnifiedMemoryManager). That is precise for the cycle's dependency: sleeping, latch-parked, or computing holders do not match and keep the wait alive, matching the previously-reviewed healthy-holder scenarios. Unwinding raises the managed SparkOutOfMemoryError, as base did, so the task releases its memory and tasks in the other pool can progress. --- .../CometBoundedShuffleMemoryAllocator.java | 42 +++++++++- .../shuffle/CometUnsafeShuffleWriter.java | 18 ++-- .../shuffle/CometDiskBlockWriterSuite.scala | 82 ++++++++++++++++++- 3 files changed, 133 insertions(+), 9 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java index 9b611bc5326..474e8b889a4 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java @@ -64,8 +64,11 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory private final long totalMemory; private long allocatedMemory = 0L; - /** How often a thread blocked in {@link #allocateBlocking(long)} logs that it is waiting. */ - private static final long WAIT_LOG_INTERVAL_MS = 30_000L; + /** + * How often a thread blocked in {@link #allocateBlocking(long)} wakes up to re-check progress and + * log that it is waiting. Not final so that tests can shorten it. + */ + private static long WAIT_LOG_INTERVAL_MS = 30_000L; /** The number of bits used to address the page table. */ private static final int PAGE_NUMBER_BITS = 13; @@ -168,6 +171,17 @@ public synchronized MemoryBlock allocateBlocking(long required) { if (allocatedMemory <= retainedByWaitingThreads() && !anyWaiterCanProceed()) { throw e; } + // A thread parked waiting for Spark execution memory cannot free its pool memory + // either, and Spark's pool may in turn be waiting for a task blocked here: a cycle + // across the two pools that no amount of waiting resolves. Unwind (after at least one + // full wait interval, to let transient states settle) so the failed task releases its + // memory and the tasks in the other pool can progress. + if (waitStart != 0 + && allocatedMemory + <= retainedByWaitingThreads() + retainedByThreadsBlockedOnSparkMemory() + && !anyWaiterCanProceed()) { + throw e; + } // Log when the wait starts and periodically while it lasts, so a stalled task is // visible in the executor logs. long now = System.currentTimeMillis(); @@ -221,6 +235,30 @@ private boolean anyWaiterCanProceed() { return false; } + /** + * Pool memory retained by threads that are parked in an untimed wait inside Spark's + * execution-memory pool. Such a thread frees nothing until Spark memory is released, which may + * itself depend on a task blocked in this pool. Sleeping, latch-parked, or computing owners + * deliberately do not match: only a stack currently inside Spark's memory arbitration does. + */ + private long retainedByThreadsBlockedOnSparkMemory() { + long retained = 0; + for (java.util.Map.Entry entry : retainedMemory.entrySet()) { + Thread thread = entry.getKey(); + if (!waitingThreads.containsKey(thread) && thread.getState() == Thread.State.WAITING) { + for (StackTraceElement frame : thread.getStackTrace()) { + String className = frame.getClassName(); + if (className.equals("org.apache.spark.memory.ExecutionMemoryPool") + || className.equals("org.apache.spark.memory.UnifiedMemoryManager")) { + retained += entry.getValue(); + break; + } + } + } + } + return retained; + } + private synchronized MemoryBlock allocateMemoryBlock(long required) { if (required > TaskMemoryManager.MAXIMUM_PAGE_SIZE_BYTES) { throw new TooLargePageException(required); diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java index c44c3b2401e..d031a616928 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java @@ -225,12 +225,6 @@ public void write(Iterator> records) throws IOException { @Override public void write(scala.collection.Iterator> records) throws IOException { - // Allocate the sorter (and its pointer array) only now: Spark evaluates the shuffle input - // iterator between constructing this writer and calling write(), and that evaluation can - // block on Spark's execution-memory pool (e.g. an eager input sort). Allocating earlier - // would retain Comet pool memory across that wait, which can form a cross-pool cycle with - // a task waiting on the Comet pool. - open(); // Keep track of success so we know if we encountered an exception // We do this rather than a standard try/catch/re-throw to handle // generic throwables. @@ -241,8 +235,20 @@ public void write(scala.collection.Iterator> records) throws IOEx String offheapMemKey = "thread_" + nativeLib.getRustThreadId() + "_comet_jvm_shuffle"; try { while (records.hasNext()) { + // Allocate the sorter (and its pointer array) only after the input produced a record: + // both the evaluation of the input iterator (between constructing this writer and + // calling write()) and hasNext() on lazy input (e.g. a sort materializing its child) + // can block on Spark's execution-memory pool, and retaining Comet pool memory across + // that wait can form a cross-pool cycle with a task waiting on the Comet pool. + if (sorter == null) { + open(); + } insertRecordIntoSorter(records.next()); } + if (sorter == null) { + // Empty input: the output path below still needs the sorter. + open(); + } if (tracingEnabled) { nativeLib.logMemoryUsage(offheapMemKey, this.allocator.getUsed()); } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 9472e8925b6..8257520e57e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -29,7 +29,7 @@ import org.scalatest.time.{Seconds, Span} import org.apache.spark.{Partitioner, SparkConf, SparkContext, SparkEnv, TaskContextImpl} import org.apache.spark.executor.{ShuffleWriteMetrics, TaskMetrics} -import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager} +import org.apache.spark.memory.{MemoryConsumer, MemoryMode, SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager, UnifiedMemoryManager} import org.apache.spark.shuffle.api.{ShuffleExecutorComponents, ShuffleMapOutputWriter, ShufflePartitionWriter} import org.apache.spark.shuffle.api.metadata.MapOutputCommitMessage import org.apache.spark.shuffle.comet.{CometBoundedShuffleMemoryAllocator, CometShuffleMemoryAllocator, CometShuffleMemoryAllocatorTrait} @@ -572,6 +572,76 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { } } + test("on-heap shared pool: a waiter unwinds when the holder is blocked on Spark memory") { + // Cross-pool cycle: a task blocked in Spark's execution-memory pool retains Comet pool + // memory, while another task waits in the Comet pool for memory that only the blocked task + // could free. Spark's pool may in turn only be freed by the Comet waiter, so no amount of + // waiting resolves it; the Comet waiter must unwind so its task can release memory. + val conf = new SparkConf() + .set("spark.comet.memoryOverhead", "1") // 1 MiB Comet pool + .set("spark.testing.memory", (8 * 1024 * 1024).toString) + .set("spark.testing.reservedMemory", "0") + .set("spark.memory.fraction", "1.0") + resetOnHeapAllocatorSingleton() + val unified = UnifiedMemoryManager(conf, numCores = 2) + val tmmSparkHog = new TaskMemoryManager(unified, 0L) + val tmmHolder = new TaskMemoryManager(unified, 1L) + val bounded = CometShuffleMemoryAllocator + .getInstance(conf, new TaskMemoryManager(new TestMemoryManager(conf), 2L), pageSize) + .asInstanceOf[CometBoundedShuffleMemoryAllocator] + def newConsumer(tmm: TaskMemoryManager): MemoryConsumer = + new MemoryConsumer(tmm, 1024 * 1024, MemoryMode.ON_HEAP) { + override def spill(size: Long, trigger: MemoryConsumer): Long = 0 + } + val oldInterval = setAllocatorWaitInterval(100) + try { + failAfter(Span(60, Seconds)) { + // Another task owns the whole Spark execution pool. + val sparkHog = newConsumer(tmmSparkHog) + assert(sparkHog.acquireMemory(8 * 1024 * 1024) == 8 * 1024 * 1024) + + // The holder retains Comet pool memory and then blocks acquiring Spark memory. + val holderReleased = new CountDownLatch(1) + val holderThread = new Thread(() => { + val cometBlock = bounded.allocate(500 * 1024) + val holderConsumer = newConsumer(tmmHolder) + try { + val got = holderConsumer.acquireMemory(1024 * 1024) // blocks until sparkHog frees + holderConsumer.freeMemory(got) + } finally { + bounded.free(cometBlock) + holderReleased.countDown() + } + }) + holderThread.start() + def parkedOnSparkMemory: Boolean = + holderThread.getState == Thread.State.WAITING && + holderThread.getStackTrace + .exists(_.getClassName == "org.apache.spark.memory.ExecutionMemoryPool") + while (!parkedOnSparkMemory) { + Thread.sleep(10) + } + + // The Comet waiter cannot fit next to the blocked holder's memory and must unwind + // instead of waiting for a release that can never come. + intercept[SparkOutOfMemoryError] { + bounded.allocateBlocking(800 * 1024) + } + + // Once Spark memory frees, the holder resumes and releases its Comet memory. + sparkHog.freeMemory(8 * 1024 * 1024) + holderReleased.await() + holderThread.join() + bounded.free(bounded.allocate(1024 * 1024)) + } + } finally { + setAllocatorWaitInterval(oldInterval) + tmmSparkHog.cleanUpAllAllocatedMemory() + tmmHolder.cleanUpAllAllocatedMemory() + resetOnHeapAllocatorSingleton() + } + } + test("on-heap shared pool: the unsafe writer allocates nothing before write()") { // Spark evaluates the shuffle input iterator between constructing the writer and calling // write(), and that evaluation can block on Spark's execution-memory pool (e.g. an eager @@ -667,6 +737,16 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { state == Thread.State.WAITING || state == Thread.State.TIMED_WAITING } + /** Shortens the allocator's wait/re-check interval for tests; returns the previous value. */ + private def setAllocatorWaitInterval(ms: Long): Long = { + val field = + classOf[CometBoundedShuffleMemoryAllocator].getDeclaredField("WAIT_LOG_INTERVAL_MS") + field.setAccessible(true) + val previous = field.getLong(null) + field.setLong(null, ms) + previous + } + /** Clears the CometShuffleMemoryAllocator singleton so this suite controls its pool size. */ private def resetOnHeapAllocatorSingleton(): Unit = { val field = classOf[CometShuffleMemoryAllocator].getDeclaredField("INSTANCE") From d9bf677f05d05eba08ee6a9931ec85af1b0dc1db Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 29 Aug 2026 00:23:26 +0800 Subject: [PATCH 13/15] address review: bound the pool wait instead of detecting blocked holders Review produced schedules where the stack-based Spark-memory detector fails in both directions: a holder blocked on a cache lock whose producer waits for Spark memory has no recognizable frame (transitive cycles evade any frame check), and a holder parked on Spark memory that an independently progressing task would free can be misclassified as stuck. No detector local to this allocator can classify holders reliably, because the resources they depend on are outside its view. Replace the detector with an overall bound on the wait, configurable via spark.comet.shuffle.jvm.memoryWaitTimeout (internal, default 5 minutes). The precise fail-fast checks stay; on timeout the waiter throws the managed SparkOutOfMemoryError as the pre-PR code did, so its task releases memory, resources outside the pool can progress, and Spark's task retry can recover. The default is generous enough that every healthy slow-holder scenario raised in review completes, while any undetectable cycle now unwinds instead of hanging the executor. --- .../CometBoundedShuffleMemoryAllocator.java | 74 ++++++++----------- .../scala/org/apache/comet/CometConf.scala | 12 +++ .../shuffle/CometDiskBlockWriterSuite.scala | 20 ++--- 3 files changed, 47 insertions(+), 59 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java index 474e8b889a4..7a50e2fa842 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java @@ -36,6 +36,7 @@ import org.apache.spark.unsafe.memory.MemoryBlock; import org.apache.spark.unsafe.memory.UnsafeMemoryAllocator; +import org.apache.comet.CometConf$; import org.apache.comet.CometSparkSessionExtensions$; /** @@ -64,11 +65,11 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory private final long totalMemory; private long allocatedMemory = 0L; - /** - * How often a thread blocked in {@link #allocateBlocking(long)} wakes up to re-check progress and - * log that it is waiting. Not final so that tests can shorten it. - */ - private static long WAIT_LOG_INTERVAL_MS = 30_000L; + /** How often a thread blocked in {@link #allocateBlocking(long)} logs that it is waiting. */ + private static final long WAIT_LOG_INTERVAL_MS = 30_000L; + + /** How long {@link #allocateBlocking(long)} waits in total before giving up. */ + private final long memoryWaitTimeoutMs; /** The number of bits used to address the page table. */ private static final int PAGE_NUMBER_BITS = 13; @@ -97,6 +98,8 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory this.pageSize = pageSize; this.totalMemory = CometSparkSessionExtensions$.MODULE$.getCometShuffleMemorySize(conf, SQLConf.get()); + this.memoryWaitTimeoutMs = + (long) CometConf$.MODULE$.COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT().get(); } private synchronized long _acquireMemory(long size) { @@ -140,8 +143,11 @@ public synchronized MemoryBlock allocate(long required) { * Callers must only use this after spilling their own buffered data. The wait fails fast when it * can never succeed: when the request does not fit next to the memory this thread itself still * retains (e.g. the sorter's pointer array), or when all allocated memory is retained by threads - * that are themselves blocked here and none of their requests fits in the free pool. Interrupting - * the task (e.g. task kill) aborts the wait. + * that are themselves blocked here and none of their requests fits in the free pool. Because the + * holders it depends on may in turn be blocked on resources outside this pool that only a task + * waiting here can release, the wait is also bounded by + * `spark.comet.shuffle.jvm.memoryWaitTimeout`, after which the managed allocation error is thrown + * and Spark's task retry can recover. Interrupting the task (e.g. task kill) aborts the wait. */ @Override public synchronized MemoryBlock allocateBlocking(long required) { @@ -171,25 +177,25 @@ public synchronized MemoryBlock allocateBlocking(long required) { if (allocatedMemory <= retainedByWaitingThreads() && !anyWaiterCanProceed()) { throw e; } - // A thread parked waiting for Spark execution memory cannot free its pool memory - // either, and Spark's pool may in turn be waiting for a task blocked here: a cycle - // across the two pools that no amount of waiting resolves. Unwind (after at least one - // full wait interval, to let transient states settle) so the failed task releases its - // memory and the tasks in the other pool can progress. - if (waitStart != 0 - && allocatedMemory - <= retainedByWaitingThreads() + retainedByThreadsBlockedOnSparkMemory() - && !anyWaiterCanProceed()) { - throw e; - } - // Log when the wait starts and periodically while it lasts, so a stalled task is - // visible in the executor logs. + // The holders this wait depends on may themselves be blocked on resources outside + // this pool (Spark execution memory, locks, I/O) that only a task waiting here can + // release - a cycle this allocator cannot observe. Bound the wait so such cycles + // unwind with the managed allocation error instead of hanging the executor; Spark's + // task retry can then recover. long now = System.currentTimeMillis(); if (waitStart == 0) { waitStart = now; lastLog = now; logger.warn( "Waiting for other tasks to free up {} bytes of Comet shuffle pool memory", size); + } else if (now - waitStart >= memoryWaitTimeoutMs) { + logger.warn( + "Giving up after waiting {} ms for {} bytes of Comet shuffle pool memory " + + "(see {})", + now - waitStart, + size, + CometConf$.MODULE$.COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT().key()); + throw e; } else if (now - lastLog >= WAIT_LOG_INTERVAL_MS) { lastLog = now; logger.warn( @@ -201,7 +207,9 @@ public synchronized MemoryBlock allocateBlocking(long required) { waitingThreads.size()); } try { - wait(WAIT_LOG_INTERVAL_MS); + wait( + Math.max( + 1L, Math.min(WAIT_LOG_INTERVAL_MS, memoryWaitTimeoutMs - (now - waitStart)))); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); // Not an allocation failure: stay non-fatal so that an intentional task kill is @@ -235,30 +243,6 @@ private boolean anyWaiterCanProceed() { return false; } - /** - * Pool memory retained by threads that are parked in an untimed wait inside Spark's - * execution-memory pool. Such a thread frees nothing until Spark memory is released, which may - * itself depend on a task blocked in this pool. Sleeping, latch-parked, or computing owners - * deliberately do not match: only a stack currently inside Spark's memory arbitration does. - */ - private long retainedByThreadsBlockedOnSparkMemory() { - long retained = 0; - for (java.util.Map.Entry entry : retainedMemory.entrySet()) { - Thread thread = entry.getKey(); - if (!waitingThreads.containsKey(thread) && thread.getState() == Thread.State.WAITING) { - for (StackTraceElement frame : thread.getStackTrace()) { - String className = frame.getClassName(); - if (className.equals("org.apache.spark.memory.ExecutionMemoryPool") - || className.equals("org.apache.spark.memory.UnifiedMemoryManager")) { - retained += entry.getValue(); - break; - } - } - } - } - return retained; - } - private synchronized MemoryBlock allocateMemoryBlock(long required) { if (required > TaskMemoryManager.MAXIMUM_PAGE_SIZE_BYTES) { throw new TooLargePageException(required); diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index d8fe5b69890..0217041eaeb 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -526,6 +526,18 @@ object CometConf extends ShimCometConf { .intConf .createWithDefault(Int.MaxValue) + val COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT: ConfigEntry[Long] = + conf("spark.comet.shuffle.jvm.memoryWaitTimeout") + .category(CATEGORY_SHUFFLE) + .doc( + "How long a Comet JVM (columnar) shuffle task running in on-heap mode waits for other " + + "tasks to free shared shuffle pool memory before failing with an out-of-memory error " + + "(Spark may then retry the task). The wait ends earlier when it provably cannot " + + "succeed. This is an internal config for testing purpose or advanced tuning.") + .internal() + .timeConf(TimeUnit.MILLISECONDS) + .createWithDefault(TimeUnit.MINUTES.toMillis(5)) + val COMET_SHUFFLE_JVM_MEMORY_FACTOR: ConfigEntry[Double] = conf("spark.comet.shuffle.jvm.memoryFactor") .withAlternative("spark.comet.columnar.shuffle.memory.factor") diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 8257520e57e..06ac43150ed 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{UnsafeProjection, UnsafeRow} import org.apache.spark.sql.execution.UnsafeRowSerializer import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{BinaryType, IntegerType, MetadataBuilder, StructField, StructType} import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.spark.util.Utils @@ -575,14 +576,16 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { test("on-heap shared pool: a waiter unwinds when the holder is blocked on Spark memory") { // Cross-pool cycle: a task blocked in Spark's execution-memory pool retains Comet pool // memory, while another task waits in the Comet pool for memory that only the blocked task - // could free. Spark's pool may in turn only be freed by the Comet waiter, so no amount of - // waiting resolves it; the Comet waiter must unwind so its task can release memory. + // could free. Spark's pool may in turn only be freed by the Comet waiter, and dependencies + // outside this pool are invisible to the allocator, so the wait is bounded: the Comet + // waiter must unwind at the timeout so its task can release memory. val conf = new SparkConf() .set("spark.comet.memoryOverhead", "1") // 1 MiB Comet pool .set("spark.testing.memory", (8 * 1024 * 1024).toString) .set("spark.testing.reservedMemory", "0") .set("spark.memory.fraction", "1.0") resetOnHeapAllocatorSingleton() + SQLConf.get.setConfString("spark.comet.shuffle.jvm.memoryWaitTimeout", "1s") val unified = UnifiedMemoryManager(conf, numCores = 2) val tmmSparkHog = new TaskMemoryManager(unified, 0L) val tmmHolder = new TaskMemoryManager(unified, 1L) @@ -593,7 +596,6 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { new MemoryConsumer(tmm, 1024 * 1024, MemoryMode.ON_HEAP) { override def spill(size: Long, trigger: MemoryConsumer): Long = 0 } - val oldInterval = setAllocatorWaitInterval(100) try { failAfter(Span(60, Seconds)) { // Another task owns the whole Spark execution pool. @@ -635,7 +637,7 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { bounded.free(bounded.allocate(1024 * 1024)) } } finally { - setAllocatorWaitInterval(oldInterval) + SQLConf.get.unsetConf("spark.comet.shuffle.jvm.memoryWaitTimeout") tmmSparkHog.cleanUpAllAllocatedMemory() tmmHolder.cleanUpAllAllocatedMemory() resetOnHeapAllocatorSingleton() @@ -737,16 +739,6 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { state == Thread.State.WAITING || state == Thread.State.TIMED_WAITING } - /** Shortens the allocator's wait/re-check interval for tests; returns the previous value. */ - private def setAllocatorWaitInterval(ms: Long): Long = { - val field = - classOf[CometBoundedShuffleMemoryAllocator].getDeclaredField("WAIT_LOG_INTERVAL_MS") - field.setAccessible(true) - val previous = field.getLong(null) - field.setLong(null, ms) - previous - } - /** Clears the CometShuffleMemoryAllocator singleton so this suite controls its pool size. */ private def resetOnHeapAllocatorSingleton(): Unit = { val field = classOf[CometShuffleMemoryAllocator].getDeclaredField("INSTANCE") From acb71a1bdccd43c2333efa8aa138c55c47a7be54 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 1 Sep 2026 18:59:10 +0800 Subject: [PATCH 14/15] Address shuffle spill review feedback --- .../CometBoundedShuffleMemoryAllocator.java | 7 +- .../CometBypassMergeSortShuffleWriter.java | 34 +++- .../shuffle/CometDiskBlockWriter.java | 16 +- .../shuffle/CometUnsafeShuffleWriter.java | 2 +- .../shuffle/CometDiskBlockWriterSuite.scala | 186 +++++++++++++----- 5 files changed, 172 insertions(+), 73 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java index 7a50e2fa842..20a0cea6b4b 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java @@ -68,9 +68,6 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory /** How often a thread blocked in {@link #allocateBlocking(long)} logs that it is waiting. */ private static final long WAIT_LOG_INTERVAL_MS = 30_000L; - /** How long {@link #allocateBlocking(long)} waits in total before giving up. */ - private final long memoryWaitTimeoutMs; - /** The number of bits used to address the page table. */ private static final int PAGE_NUMBER_BITS = 13; @@ -98,8 +95,6 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory this.pageSize = pageSize; this.totalMemory = CometSparkSessionExtensions$.MODULE$.getCometShuffleMemorySize(conf, SQLConf.get()); - this.memoryWaitTimeoutMs = - (long) CometConf$.MODULE$.COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT().get(); } private synchronized long _acquireMemory(long size) { @@ -151,6 +146,8 @@ public synchronized MemoryBlock allocate(long required) { */ @Override public synchronized MemoryBlock allocateBlocking(long required) { + long memoryWaitTimeoutMs = + (long) CometConf$.MODULE$.COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT().get(); long size = Math.max(pageSize, required); Thread self = Thread.currentThread(); long waitStart = 0; diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java index 80c3e2b7f15..5e788b424fd 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java @@ -263,18 +263,32 @@ public void write(Iterator> records) throws IOException { // Spark only calls stop(false) when write() throws an Exception; a fatal error such as // SparkOutOfMemoryError skips it, and the buffered pages are invisible to Spark's // task-memory cleanup (in on-heap mode they live in an executor-shared pool). Free them - // here so they cannot starve other tasks' allocations. + // and delete their temp files here so they cannot starve other tasks' allocations or leak + // disk space. if (partitionWriters != null) { - for (CometDiskBlockWriter writer : partitionWriters) { - if (writer == null) { - continue; - } - try { - writer.freeMemory(); - } catch (Exception e2) { - logger.error("Failed to free memory of partition writer", e2); - e.addSuppressed(e2); + try { + for (CometDiskBlockWriter writer : partitionWriters) { + if (writer == null) { + continue; + } + try { + writer.freeMemory(); + } catch (Exception e2) { + logger.error("Failed to free memory of partition writer", e2); + e.addSuppressed(e2); + } + try { + File file = writer.getFile(); + if (file.exists() && !file.delete()) { + logger.error("Error while deleting file {}", file.getAbsolutePath()); + } + } catch (Exception e2) { + logger.error("Failed to delete file of partition writer", e2); + e.addSuppressed(e2); + } } + } finally { + partitionWriters = null; } } try { diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java index c4a68d50285..b378ce33f37 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java @@ -376,6 +376,11 @@ long doSpilling(boolean isLast) throws IOException { protected void spill(int required) throws IOException { // Cannot allocate enough memory, spill and try again synchronized (currentWriters) { + // initialCurrentPage() requires this writer to have released its current page, even when + // spilling a larger sibling would free enough memory for the allocation on its own. + long totalFreed = getActiveMemoryUsage(); + CometDiskBlockWriter.this.doSpill(); + // Spill from the largest writer first to maximize the amount of memory we can // acquire Collections.sort( @@ -389,17 +394,18 @@ public int compare(CometDiskBlockWriter lhs, CometDiskBlockWriter rhs) { } }); - long totalFreed = 0; for (CometDiskBlockWriter writer : currentWriters) { + if (totalFreed >= required) { + break; + } + if (writer == CometDiskBlockWriter.this) { + continue; + } long used = writer.getActiveMemoryUsage(); writer.doSpill(); totalFreed += used; - - if (totalFreed >= required) { - break; - } } } } diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java index d031a616928..078a4c34219 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java @@ -272,7 +272,7 @@ public void write(scala.collection.Iterator> records) throws IOEx } } } - if (tracingEnabled) { + if (tracingEnabled && this.allocator != null) { nativeLib.logMemoryUsage(offheapMemKey, this.allocator.getUsed()); } } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 06ac43150ed..439dd069f9c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -27,7 +27,7 @@ import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.time.{Seconds, Span} -import org.apache.spark.{Partitioner, SparkConf, SparkContext, SparkEnv, TaskContextImpl} +import org.apache.spark.{Partitioner, SparkConf, SparkContext, SparkEnv, TaskContext, TaskContextImpl} import org.apache.spark.executor.{ShuffleWriteMetrics, TaskMetrics} import org.apache.spark.memory.{MemoryConsumer, MemoryMode, SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager, UnifiedMemoryManager} import org.apache.spark.shuffle.api.{ShuffleExecutorComponents, ShuffleMapOutputWriter, ShufflePartitionWriter} @@ -50,7 +50,10 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { private val schema = StructType(Seq(StructField("a", BinaryType))) private val pageSize: Long = 256 * 1024 - private def newTaskContext(tmm: TaskMemoryManager, taskAttemptId: Long): TaskContextImpl = { + private def newTaskContext( + tmm: TaskMemoryManager, + taskAttemptId: Long, + localProperties: Properties = new Properties): TaskContextImpl = { new TaskContextImpl( stageId = 0, stageAttemptNumber = 0, @@ -59,7 +62,7 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { taskAttemptId = taskAttemptId, attemptNumber = 0, taskMemoryManager = tmm, - localProperties = new Properties, + localProperties = localProperties, metricsSystem = null, taskMetrics = TaskMetrics.empty, cpus = 1, @@ -87,18 +90,21 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { val writersA = new JLinkedList[CometDiskBlockWriter]() val writersB = new JLinkedList[CometDiskBlockWriter]() val serializer = new UnsafeRowSerializer(1).newInstance() - val fileA = new File(tempDir, "taskA-partition0") val fileB = new File(tempDir, "taskB-partition0") - val writerA = new CometDiskBlockWriter( - fileA, - allocatorA, - taskContextA, - serializer, - schema, - new ShuffleWriteMetrics, - conf, - false, - writersA) + def newTaskAWriter(partition: Int): CometDiskBlockWriter = + new CometDiskBlockWriter( + new File(tempDir, s"taskA-partition$partition"), + allocatorA, + taskContextA, + serializer, + schema, + new ShuffleWriteMetrics, + conf, + false, + writersA) + val writerA0 = newTaskAWriter(0) + val writerA1 = newTaskAWriter(1) + val writerA2 = newTaskAWriter(2) val writerB = new CometDiskBlockWriter( fileB, allocatorB, @@ -128,21 +134,35 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { threadB.join() assert(allocatorB.getUsed == 12 * pageSize) - // Task A buffers 10 pages, then keeps inserting: its 11th page allocation fails, which - // must spill task A's own writers and leave task B's writer untouched. - var rowsA = 0L + // Task A fills nine pages in one partition and one page in another. When the smaller + // writer needs a second page, spilling its larger sibling alone would satisfy the request, + // but the requesting writer must also flush before initialCurrentPage() replaces its page. + var rowsA0 = 0L + var rowsA1 = 0L + var rowsA2 = 0L + while (allocatorA.getUsed < 9 * pageSize) { + insertOne(writerA1) + rowsA1 += 1 + } while (allocatorA.getUsed < 10 * pageSize) { - insertOne(writerA) - rowsA += 1 + insertOne(writerA0) + rowsA0 += 1 } - (0 until 400).foreach { _ => - insertOne(writerA) - rowsA += 1 + while (writerA0.getOutputRecords == 0) { + insertOne(writerA0) + rowsA0 += 1 } + assert(writerA1.getOutputRecords == 0) + + // A third, empty writer has nothing of its own to release, so it must spill a sibling from + // the shared task list to acquire its first page. + insertOne(writerA2) + rowsA2 += 1 + assert(writerA1.getOutputRecords > 0) // Task A resolved its memory pressure by spilling its own data... assert(taskContextA.taskMetrics.diskBytesSpilled > 0) - assert(writerA.getOutputRecords > 0) + assert(writerA0.getOutputRecords > 0) // ... and task B's buffered rows were not spilled, not written out, and not charged. assert(allocatorB.getUsed == 12 * pageSize) assert(writerB.getActiveMemoryUsage == 12 * pageSize) @@ -150,11 +170,17 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { assert(taskContextB.taskMetrics.diskBytesSpilled == 0) assert(fileB.length() == 0) - val segmentA = writerA.close() + val segmentA0 = writerA0.close() + val segmentA1 = writerA1.close() + val segmentA2 = writerA2.close() val segmentB = writerB.close() - assert(writerA.getOutputRecords == rowsA) + assert(writerA0.getOutputRecords == rowsA0) + assert(writerA1.getOutputRecords == rowsA1) + assert(writerA2.getOutputRecords == rowsA2) assert(writerB.getOutputRecords == rowsB) - assert(segmentA.length > 0) + assert(segmentA0.length > 0) + assert(segmentA1.length > 0) + assert(segmentA2.length > 0) assert(segmentB.length > 0) assert(writersA.isEmpty && writersB.isEmpty) } finally { @@ -173,11 +199,30 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { val memoryManager = new TestMemoryManager(conf) val tmmA = new TaskMemoryManager(memoryManager, 0L) val tmmB = new TaskMemoryManager(memoryManager, 1L) - val taskContextA = newTaskContext(tmmA, 0L) - val taskContextB = newTaskContext(tmmB, 1L) - // Both tasks get the same singleton allocator in on-heap mode. - val allocatorA = CometShuffleMemoryAllocator.getInstance(conf, tmmA, pageSize) - val allocatorB = CometShuffleMemoryAllocator.getInstance(conf, tmmB, pageSize) + val waitTimeoutKey = "spark.comet.shuffle.jvm.memoryWaitTimeout" + val propertiesA = new Properties + propertiesA.setProperty(waitTimeoutKey, "5s") + val propertiesB = new Properties + propertiesB.setProperty(waitTimeoutKey, "100ms") + val taskContextA = newTaskContext(tmmA, 0L, propertiesA) + val taskContextB = newTaskContext(tmmB, 1L, propertiesB) + + // Initialize the shared allocator under B's short timeout, then reuse it under A's longer + // timeout. The wait below must use the requesting task's setting, not the first task's. + TaskContext.setTaskContext(taskContextB) + val allocatorB = + try { + CometShuffleMemoryAllocator.getInstance(conf, tmmB, pageSize) + } finally { + TaskContext.unset() + } + TaskContext.setTaskContext(taskContextA) + val allocatorA = + try { + CometShuffleMemoryAllocator.getInstance(conf, tmmA, pageSize) + } finally { + TaskContext.unset() + } assert(allocatorA eq allocatorB) val tempDir = Utils.createTempDir() @@ -217,13 +262,18 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { var segmentBLength = 0L val poolFilled = new CountDownLatch(1) val threadB = new Thread(() => { - while (writerB.getActiveMemoryUsage < 4 * pageSize) { - insertOne(writerB) - rowsB += 1 + TaskContext.setTaskContext(taskContextB) + try { + while (writerB.getActiveMemoryUsage < 4 * pageSize) { + insertOne(writerB) + rowsB += 1 + } + poolFilled.countDown() + Thread.sleep(500) + segmentBLength = writerB.close().length + } finally { + TaskContext.unset() } - poolFilled.countDown() - Thread.sleep(500) - segmentBLength = writerB.close().length }) threadB.start() poolFilled.await() @@ -231,9 +281,14 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { // Task A's first insert finds the pool exhausted and has nothing of its own to spill; it // must wait for task B to finish instead of throwing SparkOutOfMemoryError. var rowsA = 0L - (0 until 10).foreach { _ => - insertOne(writerA) - rowsA += 1 + TaskContext.setTaskContext(taskContextA) + try { + (0 until 10).foreach { _ => + insertOne(writerA) + rowsA += 1 + } + } finally { + TaskContext.unset() } threadB.join() @@ -471,15 +526,20 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { taskContext.taskMetrics.shuffleWriteMetrics, newShuffleExecutorComponents(), null) + val diskBlockManager = SparkEnv.get.blockManager.diskBlockManager + val filesBeforeWrite = diskBlockManager.getAllFiles().toSet + var spillFiles = Set.empty[File] - // Buffer three pages worth of rows, then hit a fatal error mid-write. + // Force at least one real spill file, then hit a fatal error mid-write. val toUnsafe = UnsafeProjection.create(schema) val rows: Iterator[Product2[Int, UnsafeRow]] = - (0 until 700).iterator.map { i => + (0 until 2000).iterator.map { i => (i % 3, toUnsafe(InternalRow(new Array[Byte](1024)))) } ++ new Iterator[Product2[Int, UnsafeRow]] { override def hasNext: Boolean = true override def next(): Product2[Int, UnsafeRow] = { + spillFiles = diskBlockManager.getAllFiles().toSet -- filesBeforeWrite + assert(spillFiles.nonEmpty) throw new SparkOutOfMemoryError( "UNABLE_TO_ACQUIRE_MEMORY", java.util.Map.of("requestedBytes", "1", "receivedBytes", "0")) @@ -488,6 +548,7 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { intercept[SparkOutOfMemoryError] { writer.write(rows) } + assert(spillFiles.forall(!_.exists())) // All pages buffered by the failed task were reclaimed, so a full-pool allocation // succeeds; without the reclaim the orphaned pages would make it fail forever. @@ -671,17 +732,38 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { serializer = new UnsafeRowSerializer(1), schema = Some(schema), decodeTime = new SQLMetric("nsTiming")) + def newUnsafeWriter(): CometUnsafeShuffleWriter[Int, UnsafeRow] = + new CometUnsafeShuffleWriter[Int, UnsafeRow]( + SparkEnv.get.blockManager, + tmm, + new CometSerializedShuffleHandle[Int, UnsafeRow](0, dep), + 0L, + taskContext, + conf, + taskContext.taskMetrics.shuffleWriteMetrics, + newShuffleExecutorComponents(), + null) + + // If the input fails before open() assigns the allocator, tracing cleanup must preserve the + // original failure instead of masking it with an allocator null dereference. + SQLConf.get.setConfString("spark.comet.tracing.enabled", "true") + val tracingWriter = + try { + newUnsafeWriter() + } finally { + SQLConf.get.unsetConf("spark.comet.tracing.enabled") + } + val inputFailure = new RuntimeException("input failed before the first record") + val thrown = intercept[RuntimeException] { + tracingWriter.write(new Iterator[Product2[Int, UnsafeRow]] { + override def hasNext: Boolean = throw inputFailure + override def next(): Product2[Int, UnsafeRow] = throw new AssertionError("unreachable") + }) + } + assert(thrown eq inputFailure) + // Construct the writer exactly as Spark does before evaluating the input iterator. - val writer = new CometUnsafeShuffleWriter[Int, UnsafeRow]( - SparkEnv.get.blockManager, - tmm, - new CometSerializedShuffleHandle[Int, UnsafeRow](0, dep), - 0L, - taskContext, - conf, - taskContext.taskMetrics.shuffleWriteMetrics, - newShuffleExecutorComponents(), - null) + val writer = newUnsafeWriter() val bounded = CometShuffleMemoryAllocator .getInstance(conf, tmm, pageSize) .asInstanceOf[CometBoundedShuffleMemoryAllocator] From 59b7cf2b27765a7803e0db6526699189cff522c9 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 2 Sep 2026 11:12:48 +0800 Subject: [PATCH 15/15] Address follow-up shuffle review feedback --- .../CometBoundedShuffleMemoryAllocator.java | 32 ++++-- .../CometShuffleMemoryAllocatorTrait.java | 8 +- .../CometBypassMergeSortShuffleWriter.java | 76 +++++++------- .../shuffle/CometDiskBlockWriter.java | 13 +-- .../shuffle/CometDiskBlockWriterSuite.scala | 99 +++++++++++++------ 5 files changed, 135 insertions(+), 93 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java index eeeb857c11f..9692c1efa24 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory; import org.apache.spark.SparkConf; +import org.apache.spark.TaskContext; import org.apache.spark.memory.MemoryConsumer; import org.apache.spark.memory.MemoryMode; import org.apache.spark.memory.SparkOutOfMemoryError; @@ -68,6 +69,9 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory /** How often a thread blocked in {@link #allocateBlocking(long)} logs that it is waiting. */ private static final long WAIT_LOG_INTERVAL_MS = 30_000L; + /** How often a blocked thread checks for cooperative task cancellation. */ + private static final long TASK_KILL_POLL_INTERVAL_MS = 1_000L; + /** The number of bits used to address the page table. */ private static final int PAGE_NUMBER_BITS = 13; @@ -146,14 +150,15 @@ public synchronized MemoryBlock allocate(long required) { /** * Like {@link #allocate(long)}, but waits for other tasks of this shared pool to free memory, * mirroring how Spark's unified memory manager blocks a task until memory becomes available. - * Callers must only use this after spilling their own buffered data. The wait fails fast when it - * can never succeed: when the request does not fit next to the memory this thread itself still - * retains (e.g. the sorter's pointer array), or when all allocated memory is retained by threads - * that are themselves blocked here and none of their requests fits in the free pool. Because the - * holders it depends on may in turn be blocked on resources outside this pool that only a task - * waiting here can release, the wait is also bounded by - * `spark.comet.shuffle.jvm.memoryWaitTimeout`, after which the managed allocation error is thrown - * and Spark's task retry can recover. Interrupting the task (e.g. task kill) aborts the wait. + * Callers must first spill buffered data they can cheaply release; memory this thread still + * retains (e.g. the sorter's pointer array or sibling writers' pages) is included in the liveness + * checks below. The wait fails fast when it can never succeed: when the request does not fit next + * to the requester's retained memory, or when all allocated memory is retained by blocked threads + * and none of their requests fits in the free pool. Because the holders it depends on may in turn + * be blocked on resources outside this pool that only a task waiting here can release, the wait + * is also bounded by `spark.comet.shuffle.jvm.memoryWaitTimeout`, after which the managed + * allocation error is thrown and Spark's task retry can recover. Task cancellation or Java + * interruption aborts the wait. */ @Override public synchronized MemoryBlock allocateBlocking(long required) { @@ -161,10 +166,14 @@ public synchronized MemoryBlock allocateBlocking(long required) { (long) CometConf$.MODULE$.COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT().get(); long size = Math.max(pageSize, required); Thread self = Thread.currentThread(); + TaskContext taskContext = TaskContext.get(); long waitStart = 0; long lastLog = 0; try { while (true) { + if (taskContext != null) { + taskContext.killTaskIfInterrupted(); + } try { return allocateMemoryBlock(size); } catch (SparkOutOfMemoryError e) { @@ -217,7 +226,8 @@ public synchronized MemoryBlock allocateBlocking(long required) { try { wait( Math.max( - 1L, Math.min(WAIT_LOG_INTERVAL_MS, memoryWaitTimeoutMs - (now - waitStart)))); + 1L, + Math.min(TASK_KILL_POLL_INTERVAL_MS, memoryWaitTimeoutMs - (now - waitStart)))); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); // Not an allocation failure: stay non-fatal so that an intentional task kill is @@ -229,7 +239,9 @@ public synchronized MemoryBlock allocateBlocking(long required) { } } } finally { - waitingThreads.remove(self); + if (waitingThreads.remove(self) != null) { + notifyAll(); + } } } diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java index 8b2cf3bd0f2..2c9af24a7d9 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java +++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java @@ -35,10 +35,10 @@ protected CometShuffleMemoryAllocatorTrait( /** * Like {@link #allocate(long)}, but may wait for memory freed by other tasks instead of failing - * immediately. Callers must only use this after spilling their own buffered data, so that a - * waiting task holds no memory that other tasks could be waiting for. The default implementation - * does not wait: `CometUnifiedShuffleMemoryAllocator` delegates to Spark's memory manager, which - * already arbitrates memory between tasks. + * immediately. Callers must first spill buffered data they can cheaply release; an implementation + * that waits must account for any memory the caller retains while blocked. The default + * implementation does not wait: `CometUnifiedShuffleMemoryAllocator` delegates to Spark's memory + * manager, which already arbitrates memory between tasks. */ public MemoryBlock allocateBlocking(long required) { return allocate(required); diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java index 5e788b424fd..fdb4b289d5b 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java @@ -265,32 +265,7 @@ public void write(Iterator> records) throws IOException { // task-memory cleanup (in on-heap mode they live in an executor-shared pool). Free them // and delete their temp files here so they cannot starve other tasks' allocations or leak // disk space. - if (partitionWriters != null) { - try { - for (CometDiskBlockWriter writer : partitionWriters) { - if (writer == null) { - continue; - } - try { - writer.freeMemory(); - } catch (Exception e2) { - logger.error("Failed to free memory of partition writer", e2); - e.addSuppressed(e2); - } - try { - File file = writer.getFile(); - if (file.exists() && !file.delete()) { - logger.error("Error while deleting file {}", file.getAbsolutePath()); - } - } catch (Exception e2) { - logger.error("Failed to delete file of partition writer", e2); - e.addSuppressed(e2); - } - } - } finally { - partitionWriters = null; - } - } + cleanupPartitionWriters(e); try { mapOutputWriter.abort(e); } catch (Exception e2) { @@ -301,6 +276,40 @@ public void write(Iterator> records) throws IOException { } } + private void cleanupPartitionWriters(@Nullable Throwable failure) { + if (partitionWriters == null) { + return; + } + try { + for (CometDiskBlockWriter writer : partitionWriters) { + if (writer == null) { + continue; + } + try { + writer.freeMemory(); + } catch (Exception e) { + logger.error("Failed to free memory of partition writer", e); + if (failure != null) { + failure.addSuppressed(e); + } + } + try { + File file = writer.getFile(); + if (file.exists() && !file.delete()) { + logger.error("Error while deleting file {}", file.getAbsolutePath()); + } + } catch (Exception e) { + logger.error("Failed to delete file of partition writer", e); + if (failure != null) { + failure.addSuppressed(e); + } + } + } + } finally { + partitionWriters = null; + } + } + @Override public long[] getPartitionLengths() { return partitionLengths; @@ -403,20 +412,7 @@ public Option stop(boolean success) { return Option.apply(mapStatus); } else { // The map task failed, so delete our output data. - if (partitionWriters != null) { - try { - for (CometDiskBlockWriter writer : partitionWriters) { - writer.freeMemory(); - - File file = writer.getFile(); - if (!file.delete()) { - logger.error("Error while deleting file {}", file.getAbsolutePath()); - } - } - } finally { - partitionWriters = null; - } - } + cleanupPartitionWriters(null); return None$.empty(); } } diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java index b378ce33f37..6cda37779a5 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java @@ -64,7 +64,8 @@ public final class CometDiskBlockWriter { /** * All writers of the same shuffle task. Spilling under memory pressure only ever touches this - * task's writers, never those of other tasks. + * task's writers, never those of other tasks. Access is normally confined to the task thread; + * synchronization below only guards against overlapping lifecycle calls. */ private final LinkedList currentWriters; @@ -181,8 +182,6 @@ private void doSpill() throws IOException { // Set this into spilling state first, so it cannot recursively trigger another spill on itself. spilling = true; - // This spill could be triggered by other thread (i.e., other `CometDiskBlockWriter`), - // so we need to synchronize it. synchronized (CometDiskBlockWriter.this) { totalWritten += activeWriter.doSpilling(false); activeWriter.freeMemory(); @@ -214,8 +213,6 @@ public void insertRow(UnsafeRow row, int partitionId) throws IOException { final int serializedRecordSize = serBuffer.size(); assert (serializedRecordSize > 0); - // While proceeding with possible spilling and inserting the record, we need to synchronize - // it, because other threads may be spilling this writer at the same time. synchronized (CometDiskBlockWriter.this) { if (activeWriter.numRecords() >= numElementsForSpillThreshold || activeWriter.numRecords() >= columnarBatchSize) { @@ -303,7 +300,6 @@ class ArrowIPCWriter extends SpillWriter { /** Inserts a record into current allocated page. */ void insertRecord(Object recordBase, long recordOffset, int length) { - // This `ArrowIPCWriter` could be spilled by other threads, so we need to synchronize it. final Object base = currentPage.getBaseObject(); // Add row addresses @@ -338,7 +334,6 @@ long doSpilling(boolean isLast) throws IOException { final long written; - // All threads are writing to the same file, so we need to synchronize it. synchronized (file) { outputRecords += rowPartition.getNumRows(); written = @@ -354,7 +349,6 @@ long doSpilling(boolean isLast) throws IOException { } // Update metrics - // Other threads may be updating the metrics at the same time, so we need to synchronize it. synchronized (writeMetrics) { if (!isLast) { writeMetrics.incRecordsWritten( @@ -380,6 +374,9 @@ protected void spill(int required) throws IOException { // spilling a larger sibling would free enough memory for the allocation on its own. long totalFreed = getActiveMemoryUsage(); CometDiskBlockWriter.this.doSpill(); + if (totalFreed >= required) { + return; + } // Spill from the largest writer first to maximize the amount of memory we can // acquire diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 439dd069f9c..3b92acdfd69 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -27,7 +27,7 @@ import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.time.{Seconds, Span} -import org.apache.spark.{Partitioner, SparkConf, SparkContext, SparkEnv, TaskContext, TaskContextImpl} +import org.apache.spark.{Partitioner, SparkConf, SparkContext, SparkEnv, TaskContext, TaskContextImpl, TaskKilledException} import org.apache.spark.executor.{ShuffleWriteMetrics, TaskMetrics} import org.apache.spark.memory.{MemoryConsumer, MemoryMode, SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager, UnifiedMemoryManager} import org.apache.spark.shuffle.api.{ShuffleExecutorComponents, ShuffleMapOutputWriter, ShufflePartitionWriter} @@ -74,9 +74,8 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "1g") val memoryManager = new TestMemoryManager(conf) - // Enough memory for task B to buffer 12 pages and task A to buffer 10 pages; task A's - // 11th page allocation must fail so that it goes through the spilling path. - memoryManager.limit(22 * pageSize) + // Task B holds two pages while task A's three writers fill the other four. + memoryManager.limit(6 * pageSize) val tmmA = new TaskMemoryManager(memoryManager, 0L) val tmmB = new TaskMemoryManager(memoryManager, 1L) @@ -117,55 +116,66 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { writersB) val toUnsafe = UnsafeProjection.create(schema) + def insert(writer: CometDiskBlockWriter, size: Int): Unit = { + writer.insertRow(toUnsafe(InternalRow(new Array[Byte](size))), 0) + } def insertOne(writer: CometDiskBlockWriter): Unit = { - writer.insertRow(toUnsafe(InternalRow(new Array[Byte](1024))), 0) + insert(writer, 1024) } - // Task B buffers 12 pages worth of rows on its own thread and then idles, holding the + // Task B buffers two pages worth of rows on its own thread and then idles, holding the // buffered rows in memory (as a concurrently running task would). var rowsB = 0L val threadB = new Thread(() => { - while (allocatorB.getUsed < 12 * pageSize) { + while (allocatorB.getUsed < 2 * pageSize) { insertOne(writerB) rowsB += 1 } }) threadB.start() threadB.join() - assert(allocatorB.getUsed == 12 * pageSize) + assert(allocatorB.getUsed == 2 * pageSize) - // Task A fills nine pages in one partition and one page in another. When the smaller - // writer needs a second page, spilling its larger sibling alone would satisfy the request, - // but the requesting writer must also flush before initialCurrentPage() replaces its page. + // Task A fills two pages in one writer and one page in each sibling. When A0 needs another + // page, spilling larger A1 alone would satisfy the request, but A0 must flush itself before + // initialCurrentPage() replaces its page. var rowsA0 = 0L var rowsA1 = 0L var rowsA2 = 0L - while (allocatorA.getUsed < 9 * pageSize) { + while (allocatorA.getUsed < 2 * pageSize) { insertOne(writerA1) rowsA1 += 1 } - while (allocatorA.getUsed < 10 * pageSize) { + while (allocatorA.getUsed < 3 * pageSize) { insertOne(writerA0) rowsA0 += 1 } + while (allocatorA.getUsed < 4 * pageSize) { + insertOne(writerA2) + rowsA2 += 1 + } while (writerA0.getOutputRecords == 0) { insertOne(writerA0) rowsA0 += 1 } assert(writerA1.getOutputRecords == 0) - - // A third, empty writer has nothing of its own to release, so it must spill a sibling from - // the shared task list to acquire its first page. - insertOne(writerA2) - rowsA2 += 1 + assert(writerA2.getOutputRecords == 0) + + // A 900 KiB row needs more than A0 plus either sibling can free, so the spill loop must + // continue through both A1 and A2. + val outputA0BeforeLargeRow = writerA0.getOutputRecords + insert(writerA0, 900 * 1024) + rowsA0 += 1 + assert(writerA0.getOutputRecords > outputA0BeforeLargeRow) assert(writerA1.getOutputRecords > 0) + assert(writerA2.getOutputRecords > 0) // Task A resolved its memory pressure by spilling its own data... assert(taskContextA.taskMetrics.diskBytesSpilled > 0) assert(writerA0.getOutputRecords > 0) // ... and task B's buffered rows were not spilled, not written out, and not charged. - assert(allocatorB.getUsed == 12 * pageSize) - assert(writerB.getActiveMemoryUsage == 12 * pageSize) + assert(allocatorB.getUsed == 2 * pageSize) + assert(writerB.getActiveMemoryUsage == 2 * pageSize) assert(writerB.getOutputRecords == 0) assert(taskContextB.taskMetrics.diskBytesSpilled == 0) assert(fileB.length() == 0) @@ -344,22 +354,28 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { } } - test("on-heap shared pool: interrupting a waiting task does not throw a fatal error") { + test("on-heap shared pool: interruption or cancellation aborts a waiting task") { val conf = new SparkConf().set("spark.comet.memoryOverhead", "1") // 1 MiB shared pool resetOnHeapAllocatorSingleton() val memoryManager = new TestMemoryManager(conf) val tmmHolder = new TaskMemoryManager(memoryManager, 0L) val tmmWaiter = new TaskMemoryManager(memoryManager, 1L) + val tmmCancelled = new TaskMemoryManager(memoryManager, 2L) val taskContextHolder = newTaskContext(tmmHolder, 0L) val taskContextWaiter = newTaskContext(tmmWaiter, 1L) + val taskContextCancelled = newTaskContext(tmmCancelled, 2L) val allocatorHolder = CometShuffleMemoryAllocator.getInstance(conf, tmmHolder, pageSize) val allocatorWaiter = CometShuffleMemoryAllocator.getInstance(conf, tmmWaiter, pageSize) + val allocatorCancelled = + CometShuffleMemoryAllocator.getInstance(conf, tmmCancelled, pageSize) val tempDir = Utils.createTempDir() try { val writerHolder = newWriter(new File(tempDir, "holder"), allocatorHolder, taskContextHolder, conf) val writerWaiter = newWriter(new File(tempDir, "waiter"), allocatorWaiter, taskContextWaiter, conf) + val writerCancelled = + newWriter(new File(tempDir, "cancelled"), allocatorCancelled, taskContextCancelled, conf) val toUnsafe = UnsafeProjection.create(schema) def insertOne(writer: CometDiskBlockWriter): Unit = { writer.insertRow(toUnsafe(InternalRow(new Array[Byte](1024))), 0) @@ -380,11 +396,11 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { holderThread.start() poolFilled.await() - // The waiter blocks on its first insert, and is then killed via interrupt. - @volatile var thrown: Throwable = null + // A Java interrupt aborts the wait without surfacing a fatal allocation error. + @volatile var interruptedFailure: Throwable = null val waiterThread = new Thread(() => { try insertOne(writerWaiter) - catch { case t: Throwable => thrown = t } + catch { case t: Throwable => interruptedFailure = t } }) waiterThread.start() while (!isBlockedInWait(waiterThread)) { @@ -393,22 +409,43 @@ class CometDiskBlockWriterSuite extends AnyFunSuite with TimeLimits { waiterThread.interrupt() waiterThread.join() - // The interruption must not surface as a fatal error such as SparkOutOfMemoryError, - // otherwise an intentional task kill is reported as ExceptionFailure instead of - // TaskKilled. - assert(thrown != null) - assert(!thrown.isInstanceOf[OutOfMemoryError]) - assert(thrown.isInstanceOf[RuntimeException]) - assert(thrown.getCause.isInstanceOf[InterruptedException]) + assert(interruptedFailure != null) + assert(!interruptedFailure.isInstanceOf[OutOfMemoryError]) + assert(interruptedFailure.isInstanceOf[RuntimeException]) + assert(interruptedFailure.getCause.isInstanceOf[InterruptedException]) + + // With interruptOnCancel=false Spark only marks TaskContext; it does not interrupt the + // Java thread. The allocator must poll that flag and throw TaskKilledException promptly. + @volatile var cancelledFailure: Throwable = null + val cancelledThread = new Thread(() => { + TaskContext.setTaskContext(taskContextCancelled) + try insertOne(writerCancelled) + catch { case t: Throwable => cancelledFailure = t } + finally TaskContext.unset() + }) + cancelledThread.start() + while (!isBlockedInWait(cancelledThread)) { + Thread.sleep(10) + } + taskContextCancelled.markInterrupted("test cancellation") + cancelledThread.join(5000) + val exitedOnCancellation = !cancelledThread.isAlive release.countDown() holderThread.join() + cancelledThread.join() writerWaiter.freeMemory() + writerCancelled.freeMemory() + + assert(exitedOnCancellation) + assert(!cancelledThread.isInterrupted) + assert(cancelledFailure.isInstanceOf[TaskKilledException]) } } finally { Utils.deleteRecursively(tempDir) tmmHolder.cleanUpAllAllocatedMemory() tmmWaiter.cleanUpAllAllocatedMemory() + tmmCancelled.cleanUpAllAllocatedMemory() resetOnHeapAllocatorSingleton() } }