From acb6d9ffaa72c2a5e8e622aca334412056ad9aa2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 10 May 2025 09:21:54 -0600 Subject: [PATCH 01/19] refactor and add comet and spark task memory --- .../apache/spark/CometTaskMemoryManager.java | 2 +- .../org/apache/comet/CometExecIterator.scala | 53 ++++++++++--------- .../main/scala/org/apache/comet/Tracing.scala | 39 ++++++++++++++ 3 files changed, 68 insertions(+), 26 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/Tracing.scala diff --git a/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java b/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java index e4a08cbc4bf..225317d2a2c 100644 --- a/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java +++ b/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java @@ -33,7 +33,7 @@ public class CometTaskMemoryManager { /** The id uniquely identifies the native plan this memory manager is associated to */ private final long id; - private final TaskMemoryManager internal; + public final TaskMemoryManager internal; private final NativeMemoryConsumer nativeMemoryConsumer; private long used; diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala index 7dc65687ab1..163b2357cc2 100644 --- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala +++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.comet.CometMetricNode import org.apache.spark.sql.vectorized._ import org.apache.comet.CometConf.{COMET_BATCH_SIZE, COMET_DEBUG_ENABLED, COMET_EXEC_MEMORY_POOL_TYPE, COMET_EXPLAIN_NATIVE_ENABLED, COMET_METRICS_UPDATE_INTERVAL} +import org.apache.comet.Tracing.withTrace import org.apache.comet.vector.NativeUtil /** @@ -134,33 +135,35 @@ class CometExecIterator( def getNextBatch(): Option[ColumnarBatch] = { assert(partitionIndex >= 0 && partitionIndex < numParts) - // TODO introduce withTracing method to avoid the try/finish pattern here - try { - if (tracingEnabled) { - nativeLib.traceBegin("CometExecIterator_getNextBatch") - val memoryMXBean = ManagementFactory.getMemoryMXBean - val heap = memoryMXBean.getHeapMemoryUsage - nativeLib.logCounter("jvm_heapUsed", heap.getUsed) - } + if (tracingEnabled) { + val memoryMXBean = ManagementFactory.getMemoryMXBean + val heap = memoryMXBean.getHeapMemoryUsage + nativeLib.logCounter("jvm_heapUsed", heap.getUsed) - nativeUtil.getNextBatch( - numOutputCols, - (arrayAddrs, schemaAddrs) => { - val ctx = TaskContext.get() - nativeLib.executePlan( - ctx.stageId(), - partitionIndex, - plan, - arrayAddrs, - schemaAddrs, - tracingEnabled) - }) - } finally { - if (tracingEnabled) { - nativeLib.traceEnd("CometExecIterator_getNextBatch") - nativeLib.logCounter("CometTaskMemoryManager", cometTaskMemoryManager.getUsed) - } + val totalTaskMemory = cometTaskMemoryManager.internal.getMemoryConsumptionForThisTask + val cometTaskMemory = cometTaskMemoryManager.getUsed + val sparkTaskMemory = totalTaskMemory - cometTaskMemory + + nativeLib.logCounter("task_memory_comet", cometTaskMemory) + nativeLib.logCounter("task_memory_spark", sparkTaskMemory) } + + withTrace( + "getNextBatch[JVM]", + tracingEnabled, { + nativeUtil.getNextBatch( + numOutputCols, + (arrayAddrs, schemaAddrs) => { + val ctx = TaskContext.get() + nativeLib.executePlan( + ctx.stageId(), + partitionIndex, + plan, + arrayAddrs, + schemaAddrs, + tracingEnabled) + }) + }) } override def hasNext: Boolean = { diff --git a/spark/src/main/scala/org/apache/comet/Tracing.scala b/spark/src/main/scala/org/apache/comet/Tracing.scala new file mode 100644 index 00000000000..13f44ce2703 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/Tracing.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +object Tracing { + + private val nativeLib = new Native + + def withTrace[T](label: String, tracingEnabled: Boolean, fun: => T): T = { + try { + if (tracingEnabled) { + nativeLib.traceBegin(label) + } + fun + } finally { + if (tracingEnabled) { + nativeLib.traceEnd(label) + } + } + } + +} From bac6e3524909711c141c2691b08aed117ca8b40f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 10 May 2025 09:29:03 -0600 Subject: [PATCH 02/19] save --- docs/source/contributor-guide/tracing.md | 26 ++++++++++++++----- .../org/apache/comet/CometExecIterator.scala | 3 --- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/docs/source/contributor-guide/tracing.md b/docs/source/contributor-guide/tracing.md index 47c0cc2f2cc..fa3a0bd7392 100644 --- a/docs/source/contributor-guide/tracing.md +++ b/docs/source/contributor-guide/tracing.md @@ -19,18 +19,20 @@ under the License. # Tracing -Tracing can be enabled by setting `spark.comet.tracing.enabled=true`. Additionally, enabling the `jemalloc` -feature will enable tracing of native memory allocations. +Tracing can be enabled by setting `spark.comet.tracing.enabled=true`. + +With this feature enabled, each Spark executor will write a JSON event log file in +Chrome's [Trace Event Format]. The file will be written to the executor's current working +directory with the filename `comet-event-trace.json`. + +[Trace Event Format]: https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview?tab=t.0#heading=h.yr4qxyxotyw + +Additionally, enabling the `jemalloc` feature will enable tracing of native memory allocations. ```shell make release COMET_FEATURES="jemalloc" ``` -With this feature enabled, each Spark executor will write a JSON event log file in -Chrome's [Trace Event Format]. - -[Trace Event Format]: https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview?tab=t.0#heading=h.yr4qxyxotyw - Example output: ```json @@ -50,3 +52,13 @@ Traces can be viewed with [Trace Viewer]. Example trace visualization: ![tracing](../_static/images/tracing.png) + +## Definition of Labels + +| Label | Meaning | +|---------------------|---------------------------------------------------------| +| jvm_heapUsed | JVM heap memory usage of live objects | +| task_memory_comet | Off-heap memory allocated by Comet for query execution | +| task_memory_spark | On-heap & Off-heap memory allocated by Spark | +| offheap_shuffle_NNN | Off-heap memory allocated by Comet for columnar shuffle | + diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala index 163b2357cc2..36826920bc8 100644 --- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala +++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala @@ -216,9 +216,6 @@ class CometExecIterator( } nativeUtil.close() nativeLib.releasePlan(plan) - if (tracingEnabled) { - nativeLib.logCounter("CometTaskMemoryManager", cometTaskMemoryManager.getUsed) - } // The allocator thoughts the exported ArrowArray and ArrowSchema structs are not released, // so it will report: From 81cf02214cda0d25448e25f3ba21dedcb4b7f525 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 10 May 2025 09:31:51 -0600 Subject: [PATCH 03/19] save --- .../apache/spark/CometTaskMemoryManager.java | 19 +++++++++---- .../org/apache/comet/CometExecIterator.scala | 27 ++++++++++++------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java b/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java index 225317d2a2c..cd3097c53c3 100644 --- a/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java +++ b/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java @@ -20,6 +20,10 @@ package org.apache.spark; import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.spark.memory.MemoryConsumer; import org.apache.spark.memory.MemoryMode; @@ -30,36 +34,41 @@ * memory manager. This assumes Spark's off-heap memory mode is enabled. */ public class CometTaskMemoryManager { + + private static final Logger logger = LoggerFactory.getLogger(CometTaskMemoryManager.class); + /** The id uniquely identifies the native plan this memory manager is associated to */ private final long id; public final TaskMemoryManager internal; private final NativeMemoryConsumer nativeMemoryConsumer; - private long used; + private final AtomicLong used = new AtomicLong(); public CometTaskMemoryManager(long id) { this.id = id; this.internal = TaskContext$.MODULE$.get().taskMemoryManager(); this.nativeMemoryConsumer = new NativeMemoryConsumer(); - this.used = 0; } // Called by Comet native through JNI. // Returns the actual amount of memory (in bytes) granted. public long acquireMemory(long size) { long acquired = internal.acquireExecutionMemory(size, nativeMemoryConsumer); - used += acquired; + used.addAndGet(acquired); return acquired; } // Called by Comet native through JNI public void releaseMemory(long size) { - used -= size; + long newUsed = used.addAndGet(-size); + if (newUsed < 0) { + logger.warn("Used memory is negative: " + newUsed); + } internal.releaseExecutionMemory(size, nativeMemoryConsumer); } public long getUsed() { - return used; + return used.get(); } /** diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala index 36826920bc8..b21c83cbe90 100644 --- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala +++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala @@ -61,6 +61,7 @@ class CometExecIterator( with Logging { private val tracingEnabled = CometConf.COMET_TRACING_ENABLED.get() + private val memoryMXBean = ManagementFactory.getMemoryMXBean private val nativeLib = new Native() private val nativeUtil = new NativeUtil() private val cometTaskMemoryManager = new CometTaskMemoryManager(id) @@ -136,16 +137,7 @@ class CometExecIterator( assert(partitionIndex >= 0 && partitionIndex < numParts) if (tracingEnabled) { - val memoryMXBean = ManagementFactory.getMemoryMXBean - val heap = memoryMXBean.getHeapMemoryUsage - nativeLib.logCounter("jvm_heapUsed", heap.getUsed) - - val totalTaskMemory = cometTaskMemoryManager.internal.getMemoryConsumptionForThisTask - val cometTaskMemory = cometTaskMemoryManager.getUsed - val sparkTaskMemory = totalTaskMemory - cometTaskMemory - - nativeLib.logCounter("task_memory_comet", cometTaskMemory) - nativeLib.logCounter("task_memory_spark", sparkTaskMemory) + traceMemoryUsage() } withTrace( @@ -217,6 +209,10 @@ class CometExecIterator( nativeUtil.close() nativeLib.releasePlan(plan) + if (tracingEnabled) { + traceMemoryUsage() + } + // The allocator thoughts the exported ArrowArray and ArrowSchema structs are not released, // so it will report: // Caused by: java.lang.IllegalStateException: Memory was leaked by query. @@ -239,4 +235,15 @@ class CometExecIterator( closed = true } } + + private def traceMemoryUsage(): Unit = { + nativeLib.logCounter("jvm_heapUsed", memoryMXBean.getHeapMemoryUsage.getUsed) + + val totalTaskMemory = cometTaskMemoryManager.internal.getMemoryConsumptionForThisTask + val cometTaskMemory = cometTaskMemoryManager.getUsed + val sparkTaskMemory = totalTaskMemory - cometTaskMemory + + nativeLib.logCounter("task_memory_comet", cometTaskMemory) + nativeLib.logCounter("task_memory_spark", sparkTaskMemory) + } } From eb47aedf54c1cfe12b4e2da4c7446edae9561429 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 10 May 2025 09:41:00 -0600 Subject: [PATCH 04/19] save --- docs/source/contributor-guide/tracing.md | 20 +++++++++---------- .../org/apache/comet/CometExecIterator.scala | 7 +++---- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/source/contributor-guide/tracing.md b/docs/source/contributor-guide/tracing.md index fa3a0bd7392..fa1dd14aa75 100644 --- a/docs/source/contributor-guide/tracing.md +++ b/docs/source/contributor-guide/tracing.md @@ -19,10 +19,10 @@ under the License. # Tracing -Tracing can be enabled by setting `spark.comet.tracing.enabled=true`. +Tracing can be enabled by setting `spark.comet.tracing.enabled=true`. With this feature enabled, each Spark executor will write a JSON event log file in -Chrome's [Trace Event Format]. The file will be written to the executor's current working +Chrome's [Trace Event Format]. The file will be written to the executor's current working directory with the filename `comet-event-trace.json`. [Trace Event Format]: https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview?tab=t.0#heading=h.yr4qxyxotyw @@ -45,7 +45,7 @@ Example output: { "name": "CometExecIterator_getNextBatch", "cat": "PERF", "ph": "B", "pid": 1, "tid": 5, "ts": 10109479156 }, ``` -Traces can be viewed with [Trace Viewer]. +Traces can be viewed with [Trace Viewer]. [Trace Viewer]: https://github.com/catapult-project/catapult/blob/main/tracing/README.md @@ -55,10 +55,10 @@ Example trace visualization: ## Definition of Labels -| Label | Meaning | -|---------------------|---------------------------------------------------------| -| jvm_heapUsed | JVM heap memory usage of live objects | -| task_memory_comet | Off-heap memory allocated by Comet for query execution | -| task_memory_spark | On-heap & Off-heap memory allocated by Spark | -| offheap_shuffle_NNN | Off-heap memory allocated by Comet for columnar shuffle | - +| Label | Meaning | +| --------------------- | -------------------------------------------------------------- | +| jvm_heapUsed | JVM heap memory usage of live objects for the executor process | +| jemalloc_allocated | Native memory usage for the executor process | +| task_memory_comet_NNN | Off-heap memory allocated by Comet for query execution | +| task_memory_spark_NNN | On-heap & Off-heap memory allocated by Spark | +| offheap_shuffle_NNN | Off-heap memory allocated by Comet for columnar shuffle | diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala index b21c83cbe90..1a4c5898cd9 100644 --- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala +++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala @@ -238,12 +238,11 @@ class CometExecIterator( private def traceMemoryUsage(): Unit = { nativeLib.logCounter("jvm_heapUsed", memoryMXBean.getHeapMemoryUsage.getUsed) - val totalTaskMemory = cometTaskMemoryManager.internal.getMemoryConsumptionForThisTask val cometTaskMemory = cometTaskMemoryManager.getUsed val sparkTaskMemory = totalTaskMemory - cometTaskMemory - - nativeLib.logCounter("task_memory_comet", cometTaskMemory) - nativeLib.logCounter("task_memory_spark", sparkTaskMemory) + val threadId = Thread.currentThread().getId + nativeLib.logCounter(s"task_memory_comet_$threadId", cometTaskMemory) + nativeLib.logCounter(s"task_memory_spark_$threadId", sparkTaskMemory) } } From f422d7b5c34f72648d7687c03e9f573f1407c29e Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Mon, 12 May 2025 14:27:17 +0800 Subject: [PATCH 05/19] fix: Check acquired memory when CometMemoryPool grows --- native/core/src/execution/memory_pools/unified_pool.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/native/core/src/execution/memory_pools/unified_pool.rs b/native/core/src/execution/memory_pools/unified_pool.rs index 210ae22f466..016346ceafa 100644 --- a/native/core/src/execution/memory_pools/unified_pool.rs +++ b/native/core/src/execution/memory_pools/unified_pool.rs @@ -89,10 +89,8 @@ unsafe impl Send for CometMemoryPool {} unsafe impl Sync for CometMemoryPool {} impl MemoryPool for CometMemoryPool { - fn grow(&self, _: &MemoryReservation, additional: usize) { - self.acquire(additional) - .unwrap_or_else(|_| panic!("Failed to acquire {} bytes", additional)); - self.used.fetch_add(additional, Relaxed); + fn grow(&self, reservation: &MemoryReservation, additional: usize) { + self.try_grow(reservation, additional).unwrap(); } fn shrink(&self, _: &MemoryReservation, size: usize) { From 64d7563d7ad22233c46d026c392a7297763cfc3c Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Mon, 12 May 2025 14:52:56 +0800 Subject: [PATCH 06/19] add log --- .../main/java/org/apache/spark/CometTaskMemoryManager.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java b/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java index e4a08cbc4bf..7646fcdfc95 100644 --- a/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java +++ b/spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java @@ -48,6 +48,10 @@ public CometTaskMemoryManager(long id) { // Returns the actual amount of memory (in bytes) granted. public long acquireMemory(long size) { long acquired = internal.acquireExecutionMemory(size, nativeMemoryConsumer); + if (acquired < size) { + // If memory manager is not able to acquire the requested size, log memory usage + internal.showMemoryUsage(); + } used += acquired; return acquired; } From 9121eeed4927ee1d320bc549df05d70512f18048 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 11:34:19 -0600 Subject: [PATCH 07/19] more improvements --- native/core/src/execution/jni_api.rs | 5 ++--- .../shuffle/CometBypassMergeSortShuffleWriter.java | 4 ++-- .../execution/shuffle/CometUnsafeShuffleWriter.java | 2 +- .../scala/org/apache/comet/CometExecIterator.scala | 12 +++--------- spark/src/main/scala/org/apache/comet/Native.scala | 3 +-- 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index b7c62d4d923..68c7b84a4b9 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -366,14 +366,13 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan( exec_context: jlong, array_addrs: jlongArray, schema_addrs: jlongArray, - tracing_enabled: jboolean, ) -> jlong { try_unwrap_or_throw(&e, |mut env| { - let _ = TraceGuard::new("executePlan", tracing_enabled != JNI_FALSE); - // Retrieve the query let exec_context = get_execution_context(exec_context); + let _ = TraceGuard::new("executePlan", exec_context.tracing_enabled); + if exec_context.tracing_enabled { #[cfg(feature = "jemalloc")] { diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java index c939e7428e6..bdad1493fb5 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java @@ -234,7 +234,7 @@ public void write(Iterator> records) throws IOException { Native _native = new Native(); if (tracingEnabled) { - _native.logCounter("BypassMergeShortShuffle", allocator.getUsed()); + _native.logCounter("comet_shuffle_", allocator.getUsed()); } long spillRecords = 0; @@ -247,7 +247,7 @@ public void write(Iterator> records) throws IOException { } if (tracingEnabled) { - _native.logCounter("BypassMergeShortShuffle", allocator.getUsed()); + _native.logCounter("comet_shuffle_", allocator.getUsed()); } if (outputRows != spillRecords) { diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java index 686633e7f72..1b3318ffc40 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java @@ -213,7 +213,7 @@ public void write(scala.collection.Iterator> records) throws IOEx if (tracingEnabled) { nativeLib.traceBegin("CometUnsafeShuffleWriter"); } - String offheapMemKey = "offheap_shuffle_" + Thread.currentThread().getId(); + String offheapMemKey = "comet_shuffle_" + Thread.currentThread().getId(); try { while (records.hasNext()) { insertRecordIntoSorter(records.next()); diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala index 1a4c5898cd9..978f1093e31 100644 --- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala +++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala @@ -140,20 +140,14 @@ class CometExecIterator( traceMemoryUsage() } + val ctx = TaskContext.get() withTrace( - "getNextBatch[JVM]", + s"getNextBatch[JVM] stage=${ctx.stageId()}", tracingEnabled, { nativeUtil.getNextBatch( numOutputCols, (arrayAddrs, schemaAddrs) => { - val ctx = TaskContext.get() - nativeLib.executePlan( - ctx.stageId(), - partitionIndex, - plan, - arrayAddrs, - schemaAddrs, - tracingEnabled) + nativeLib.executePlan(ctx.stageId(), partitionIndex, plan, arrayAddrs, schemaAddrs) }) }) } diff --git a/spark/src/main/scala/org/apache/comet/Native.scala b/spark/src/main/scala/org/apache/comet/Native.scala index 0a08e29eeb2..662684caf84 100644 --- a/spark/src/main/scala/org/apache/comet/Native.scala +++ b/spark/src/main/scala/org/apache/comet/Native.scala @@ -91,8 +91,7 @@ class Native extends NativeBase { partition: Int, plan: Long, arrayAddrs: Array[Long], - schemaAddrs: Array[Long], - tracingEnabled: Boolean): Long + schemaAddrs: Array[Long]): Long /** * Release and drop the native query plan object and context object. From 892cf1b3408a007135027701273d3b11b66b4853 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 13:00:15 -0600 Subject: [PATCH 08/19] use micros not nanos --- native/core/src/execution/jni_api.rs | 8 +++++++- native/core/src/execution/tracing.rs | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 68c7b84a4b9..72a8f134508 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -74,6 +74,7 @@ use log::info; use once_cell::sync::Lazy; #[cfg(feature = "jemalloc")] use tikv_jemalloc_ctl::{epoch, stats}; +use datafusion_comet_proto::spark_operator::operator::OpStruct; static TOKIO_RUNTIME: Lazy = Lazy::new(|| { let mut builder = tokio::runtime::Builder::new_multi_thread(); @@ -371,7 +372,12 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan( // Retrieve the query let exec_context = get_execution_context(exec_context); - let _ = TraceGuard::new("executePlan", exec_context.tracing_enabled); + let tracing_event_name = match &exec_context.spark_plan.op_struct { + Some(OpStruct::ShuffleWriter(_)) => "executePlan(ShuffleWriter)", + _ => "executePlan" + }; + + let _ = TraceGuard::new(tracing_event_name, exec_context.tracing_enabled); if exec_context.tracing_enabled { #[cfg(feature = "jemalloc")] diff --git a/native/core/src/execution/tracing.rs b/native/core/src/execution/tracing.rs index 8925f429ddd..d6f4f935285 100644 --- a/native/core/src/execution/tracing.rs +++ b/native/core/src/execution/tracing.rs @@ -60,7 +60,7 @@ impl Recorder { let json = format!( "{{ \"name\": \"{name}\", \"cat\": \"PERF\", \"ph\": \"C\", \"pid\": 1, \"tid\": {}, \"ts\": {}, \"args\": {{ \"{name}\": {value} }} }},\n", Self::get_thread_id(), - self.now.elapsed().as_nanos() + self.now.elapsed().as_micros() ); let mut writer = self.writer.lock().unwrap(); writer.write_all(json.as_bytes()).unwrap(); @@ -71,7 +71,7 @@ impl Recorder { "{{ \"name\": \"{}\", \"cat\": \"PERF\", \"ph\": \"{ph}\", \"pid\": 1, \"tid\": {}, \"ts\": {} }},\n", name, Self::get_thread_id(), - self.now.elapsed().as_nanos() + self.now.elapsed().as_micros() ); let mut writer = self.writer.lock().unwrap(); writer.write_all(json.as_bytes()).unwrap(); From a2244898e425ee9a5598594cfbd86f619b407c58 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 13:39:30 -0600 Subject: [PATCH 09/19] record memory usage in MB --- native/core/src/execution/jni_api.rs | 14 +++++------ native/core/src/execution/tracing.rs | 25 ++++++++++++------- .../CometBypassMergeSortShuffleWriter.java | 4 +-- .../shuffle/CometUnsafeShuffleWriter.java | 4 +-- .../org/apache/comet/CometExecIterator.scala | 6 ++--- .../main/scala/org/apache/comet/Native.scala | 20 ++++++++++++++- 6 files changed, 49 insertions(+), 24 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 72a8f134508..eee4d6ecf0b 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -68,13 +68,13 @@ use crate::execution::shuffle::{read_ipc_compressed, CompressionCodec}; use crate::execution::spark_plan::SparkPlan; use crate::execution::tracing::TraceGuard; -use crate::execution::tracing::{log_counter, trace_begin, trace_end}; +use crate::execution::tracing::{log_memory_usage, trace_begin, trace_end}; +use datafusion_comet_proto::spark_operator::operator::OpStruct; use log::info; use once_cell::sync::Lazy; #[cfg(feature = "jemalloc")] use tikv_jemalloc_ctl::{epoch, stats}; -use datafusion_comet_proto::spark_operator::operator::OpStruct; static TOKIO_RUNTIME: Lazy = Lazy::new(|| { let mut builder = tokio::runtime::Builder::new_multi_thread(); @@ -374,7 +374,7 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan( let tracing_event_name = match &exec_context.spark_plan.op_struct { Some(OpStruct::ShuffleWriter(_)) => "executePlan(ShuffleWriter)", - _ => "executePlan" + _ => "executePlan", }; let _ = TraceGuard::new(tracing_event_name, exec_context.tracing_enabled); @@ -385,8 +385,8 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan( let e = epoch::mib().unwrap(); let allocated = stats::allocated::mib().unwrap(); e.advance().unwrap(); - use crate::execution::tracing::log_counter; - log_counter("jemalloc_allocated", allocated.read().unwrap() as u64); + use crate::execution::tracing::log_memory_usage; + log_memory_usage("jemalloc_allocated", allocated.read().unwrap() as u64); } } @@ -712,7 +712,7 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_traceEnd( #[no_mangle] /// # Safety /// This function is inherently unsafe since it deals with raw pointers passed from JNI. -pub unsafe extern "system" fn Java_org_apache_comet_Native_logCounter( +pub unsafe extern "system" fn Java_org_apache_comet_Native_logMemoryUsage( e: JNIEnv, _class: JClass, name: jstring, @@ -720,7 +720,7 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_logCounter( ) { try_unwrap_or_throw(&e, |mut env| { let name: String = env.get_string(&JString::from_raw(name)).unwrap().into(); - log_counter(&name, value as u64); + log_memory_usage(&name, value as u64); Ok(()) }) } diff --git a/native/core/src/execution/tracing.rs b/native/core/src/execution/tracing.rs index d6f4f935285..c676827223d 100644 --- a/native/core/src/execution/tracing.rs +++ b/native/core/src/execution/tracing.rs @@ -36,13 +36,15 @@ impl Recorder { .create(true) .append(true) .open("comet-event-trace.json") - .unwrap(); + .expect("Error writing tracing"); let mut writer = BufWriter::new(file); // Write start of JSON array. Note that there is no requirement to write // the closing ']'. - writer.write_all("[ ".as_bytes()).unwrap(); + writer + .write_all("[ ".as_bytes()) + .expect("Error writing tracing"); Self { now: Instant::now(), writer: Arc::new(Mutex::new(writer)), @@ -56,14 +58,17 @@ impl Recorder { self.log_event(name, "E") } - pub fn log_counter(&self, name: &str, value: u64) { + pub fn log_memory_usage(&self, name: &str, usage_bytes: u64) { + let usage_mb = (usage_bytes as f64 / 1024.0 / 1024.0) as usize; let json = format!( - "{{ \"name\": \"{name}\", \"cat\": \"PERF\", \"ph\": \"C\", \"pid\": 1, \"tid\": {}, \"ts\": {}, \"args\": {{ \"{name}\": {value} }} }},\n", + "{{ \"name\": \"{name}\", \"cat\": \"PERF\", \"ph\": \"C\", \"pid\": 1, \"tid\": {}, \"ts\": {}, \"args\": {{ \"{name}\": {usage_mb} }} }},\n", Self::get_thread_id(), self.now.elapsed().as_micros() ); let mut writer = self.writer.lock().unwrap(); - writer.write_all(json.as_bytes()).unwrap(); + writer + .write_all(json.as_bytes()) + .expect("Error writing tracing"); } fn log_event(&self, name: &str, ph: &str) { @@ -74,7 +79,9 @@ impl Recorder { self.now.elapsed().as_micros() ); let mut writer = self.writer.lock().unwrap(); - writer.write_all(json.as_bytes()).unwrap(); + writer + .write_all(json.as_bytes()) + .expect("Error writing tracing"); } fn get_thread_id() -> u64 { @@ -83,7 +90,7 @@ impl Recorder { .trim_start_matches("ThreadId(") .trim_end_matches(")") .parse() - .unwrap() + .expect("Error parsing thread id") } } @@ -95,8 +102,8 @@ pub(crate) fn trace_end(name: &str) { RECORDER.end_task(name); } -pub(crate) fn log_counter(name: &str, value: u64) { - RECORDER.log_counter(name, value); +pub(crate) fn log_memory_usage(name: &str, value: u64) { + RECORDER.log_memory_usage(name, value); } pub(crate) struct TraceGuard<'a> { label: &'a str, diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java index bdad1493fb5..a58ec7851bb 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java @@ -234,7 +234,7 @@ public void write(Iterator> records) throws IOException { Native _native = new Native(); if (tracingEnabled) { - _native.logCounter("comet_shuffle_", allocator.getUsed()); + _native.logMemoryUsage("comet_shuffle_", allocator.getUsed()); } long spillRecords = 0; @@ -247,7 +247,7 @@ public void write(Iterator> records) throws IOException { } if (tracingEnabled) { - _native.logCounter("comet_shuffle_", allocator.getUsed()); + _native.logMemoryUsage("comet_shuffle_", allocator.getUsed()); } if (outputRows != spillRecords) { diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java index 1b3318ffc40..552a189ab45 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java @@ -219,7 +219,7 @@ public void write(scala.collection.Iterator> records) throws IOEx insertRecordIntoSorter(records.next()); } if (tracingEnabled) { - nativeLib.logCounter(offheapMemKey, this.allocator.getUsed()); + nativeLib.logMemoryUsage(offheapMemKey, this.allocator.getUsed()); } closeAndWriteOutput(); success = true; @@ -242,7 +242,7 @@ public void write(scala.collection.Iterator> records) throws IOEx } } if (tracingEnabled) { - nativeLib.logCounter(offheapMemKey, this.allocator.getUsed()); + nativeLib.logMemoryUsage(offheapMemKey, this.allocator.getUsed()); } } } diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala index 978f1093e31..349c9cd581b 100644 --- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala +++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala @@ -231,12 +231,12 @@ class CometExecIterator( } private def traceMemoryUsage(): Unit = { - nativeLib.logCounter("jvm_heapUsed", memoryMXBean.getHeapMemoryUsage.getUsed) + nativeLib.logMemoryUsage("jvm_heapUsed", memoryMXBean.getHeapMemoryUsage.getUsed) val totalTaskMemory = cometTaskMemoryManager.internal.getMemoryConsumptionForThisTask val cometTaskMemory = cometTaskMemoryManager.getUsed val sparkTaskMemory = totalTaskMemory - cometTaskMemory val threadId = Thread.currentThread().getId - nativeLib.logCounter(s"task_memory_comet_$threadId", cometTaskMemory) - nativeLib.logCounter(s"task_memory_spark_$threadId", sparkTaskMemory) + nativeLib.logMemoryUsage(s"task_memory_comet_$threadId", cometTaskMemory) + nativeLib.logMemoryUsage(s"task_memory_spark_$threadId", sparkTaskMemory) } } diff --git a/spark/src/main/scala/org/apache/comet/Native.scala b/spark/src/main/scala/org/apache/comet/Native.scala index 662684caf84..afaa3d17db1 100644 --- a/spark/src/main/scala/org/apache/comet/Native.scala +++ b/spark/src/main/scala/org/apache/comet/Native.scala @@ -177,10 +177,28 @@ class Native extends NativeBase { schemaAddrs: Array[Long], tracingEnabled: Boolean): Long + /** + * Log the beginning of an event. + * @param name + * The name of the event. + */ @native def traceBegin(name: String): Unit + /** + * Log the end of an event. + * @param name + * The name of the event. + */ @native def traceEnd(name: String): Unit - @native def logCounter(name: String, value: Long): Unit + /** + * Log the amount of memory currently in use. + * + * @param name + * Type of memory e.g. jvm, native, off-heap pool + * @param memoryUsageBytes + * Number of bytes in use + */ + @native def logMemoryUsage(name: String, memoryUsageBytes: Long): Unit } From e3fba1c1aeab5acad9d5908474d99786db02df65 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 16:13:23 -0600 Subject: [PATCH 10/19] remove TraceGuard --- native/core/src/execution/jni_api.rs | 47 +++++++++++++++---- .../src/execution/shuffle/shuffle_writer.rs | 39 ++++++++++++--- native/core/src/execution/tracing.rs | 24 ---------- 3 files changed, 71 insertions(+), 39 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index eee4d6ecf0b..3610da685b8 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -67,7 +67,6 @@ use crate::execution::operators::ScanExec; use crate::execution::shuffle::{read_ipc_compressed, CompressionCodec}; use crate::execution::spark_plan::SparkPlan; -use crate::execution::tracing::TraceGuard; use crate::execution::tracing::{log_memory_usage, trace_begin, trace_end}; use datafusion_comet_proto::spark_operator::operator::OpStruct; @@ -163,7 +162,9 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_createPlan( tracing_enabled: jboolean, ) -> jlong { try_unwrap_or_throw(&e, |mut env| { - let _ = TraceGuard::new("createPlan", tracing_enabled != JNI_FALSE); + if tracing_enabled != JNI_FALSE { + trace_begin("createPlan"); + } // Init JVM classes JVMClasses::init(&mut env); @@ -246,6 +247,10 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_createPlan( tracing_enabled: tracing_enabled != JNI_FALSE, }); + if tracing_enabled != JNI_FALSE { + trace_end("createPlan"); + } + Ok(Box::into_raw(exec_context) as i64) }) } @@ -377,9 +382,9 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan( _ => "executePlan", }; - let _ = TraceGuard::new(tracing_event_name, exec_context.tracing_enabled); - if exec_context.tracing_enabled { + trace_begin(tracing_event_name); + #[cfg(feature = "jemalloc")] { let e = epoch::mib().unwrap(); @@ -446,13 +451,17 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan( match poll_output { Poll::Ready(Some(output)) => { // prepare output for FFI transfer - return prepare_output( + let addr = prepare_output( &mut env, array_addrs, schema_addrs, output?, exec_context.debug_native, ); + if exec_context.tracing_enabled { + trace_end(tracing_event_name); + } + return addr; } Poll::Ready(None) => { // Reaches EOF of output. @@ -469,6 +478,9 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan( ); } } + if exec_context.tracing_enabled { + trace_end(tracing_event_name); + } return Ok(-1); } // A poll pending means there are more than one blocking operators, @@ -570,7 +582,9 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_writeSortedFileNative tracing_enabled: jboolean, ) -> jlongArray { try_unwrap_or_throw(&e, |mut env| unsafe { - let _ = TraceGuard::new("writeSortedFileNative", tracing_enabled != JNI_FALSE); + if tracing_enabled != JNI_FALSE { + trace_begin("writeSortedFileNative"); + } let data_types = convert_datatype_arrays(&mut env, serialized_datatypes)?; @@ -634,6 +648,10 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_writeSortedFileNative let long_array = env.new_long_array(2)?; env.set_long_array_region(&long_array, 0, &[written_bytes, checksum])?; + if tracing_enabled != JNI_FALSE { + trace_end("writeSortedFileNative"); + } + Ok(long_array.into_raw()) }) } @@ -648,10 +666,15 @@ pub extern "system" fn Java_org_apache_comet_Native_sortRowPartitionsNative( tracing_enabled: jboolean, ) { try_unwrap_or_throw(&e, |_| { - let _ = TraceGuard::new("sortRowPartitionsNative", tracing_enabled != JNI_FALSE); + if tracing_enabled != JNI_FALSE { + trace_begin("sortRowPartitionsNative"); + } // SAFETY: JVM unsafe memory allocation is aligned with long. let array = unsafe { std::slice::from_raw_parts_mut(address as *mut i64, size as usize) }; array.rdxsort(); + if tracing_enabled != JNI_FALSE { + trace_end("sortRowPartitionsNative"); + } Ok(()) }) } @@ -670,12 +693,18 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_decodeShuffleBlock( tracing_enabled: jboolean, ) -> jlong { try_unwrap_or_throw(&e, |mut env| { - let _ = TraceGuard::new("decodeShuffleBlock", tracing_enabled != JNI_FALSE); + if tracing_enabled != JNI_FALSE { + trace_begin("decodeShuffleBlock"); + } let raw_pointer = env.get_direct_buffer_address(&byte_buffer)?; let length = length as usize; let slice: &[u8] = unsafe { std::slice::from_raw_parts(raw_pointer, length) }; let batch = read_ipc_compressed(slice)?; - prepare_output(&mut env, array_addrs, schema_addrs, batch, false) + let addr = prepare_output(&mut env, array_addrs, schema_addrs, batch, false); + if tracing_enabled != JNI_FALSE { + trace_end("decodeShuffleBlock"); + } + addr }) } diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index 24f4d48104f..701f57e6b4d 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -18,7 +18,7 @@ //! Defines the External shuffle repartition plan. use crate::execution::shuffle::{CompressionCodec, ShuffleBlockWriter}; -use crate::execution::tracing::TraceGuard; +use crate::execution::tracing::{trace_begin, trace_end}; use arrow::compute::interleave_record_batch; use async_trait::async_trait; use datafusion::common::utils::proxy::VecAllocExt; @@ -215,7 +215,9 @@ async fn external_shuffle( codec: CompressionCodec, tracing_enabled: bool, ) -> Result { - let _ = TraceGuard::new("external_shuffle", tracing_enabled); + if tracing_enabled { + trace_begin("external_shuffle"); + } let schema = input.schema(); @@ -252,6 +254,10 @@ async fn external_shuffle( repartitioner.shuffle_write().await?; + if tracing_enabled { + trace_end("external_shuffle"); + } + // shuffle writer always has empty output Ok(Box::pin(EmptyRecordBatchStream::new(Arc::clone(&schema)))) } @@ -612,8 +618,6 @@ impl MultiPartitionShuffleRepartitioner { } async fn spill(&mut self) -> Result<()> { - let _ = TraceGuard::new("spill", self.tracing_enabled); - log::debug!( "ShuffleRepartitioner spilling shuffle data of {} to disk while inserting ({} time(s) so far)", self.used(), @@ -625,6 +629,10 @@ impl MultiPartitionShuffleRepartitioner { return Ok(()); } + if self.tracing_enabled { + trace_begin("shuffle_spill"); + } + let num_output_partitions = self.partition_writers.len(); let mut partitioned_batches = self.partitioned_batches(); let mut spilled_bytes = 0; @@ -640,6 +648,11 @@ impl MultiPartitionShuffleRepartitioner { timer.stop(); self.metrics.spill_count.add(1); self.metrics.spilled_bytes.add(spilled_bytes); + + if self.tracing_enabled { + trace_end("shuffle_spill"); + } + Ok(()) } } @@ -650,7 +663,10 @@ impl ShufflePartitioner for MultiPartitionShuffleRepartitioner { /// This function will slice input batch according to configured batch size and then /// shuffle rows into corresponding partition buffer. async fn insert_batch(&mut self, batch: RecordBatch) -> Result<()> { - let _ = TraceGuard::new("insert_batch", self.tracing_enabled); + if self.tracing_enabled { + trace_begin("shuffle_insert_batch"); + } + let start_time = Instant::now(); let mut start = 0; while start < batch.num_rows() { @@ -664,12 +680,19 @@ impl ShufflePartitioner for MultiPartitionShuffleRepartitioner { .baseline .elapsed_compute() .add_duration(start_time.elapsed()); + + if self.tracing_enabled { + trace_end("shuffle_insert_batch"); + } + Ok(()) } /// Writes buffered shuffled record batches into Arrow IPC bytes. async fn shuffle_write(&mut self) -> Result<()> { - let _ = TraceGuard::new("shuffle_write", self.tracing_enabled); + if self.tracing_enabled { + trace_begin("shuffle_write"); + } let start_time = Instant::now(); @@ -739,6 +762,10 @@ impl ShufflePartitioner for MultiPartitionShuffleRepartitioner { .elapsed_compute() .add_duration(start_time.elapsed()); + if self.tracing_enabled { + trace_end("shuffle_write"); + } + Ok(()) } } diff --git a/native/core/src/execution/tracing.rs b/native/core/src/execution/tracing.rs index c676827223d..61e3b3b699d 100644 --- a/native/core/src/execution/tracing.rs +++ b/native/core/src/execution/tracing.rs @@ -105,27 +105,3 @@ pub(crate) fn trace_end(name: &str) { pub(crate) fn log_memory_usage(name: &str, value: u64) { RECORDER.log_memory_usage(name, value); } -pub(crate) struct TraceGuard<'a> { - label: &'a str, - tracing_enabled: bool, -} - -impl<'a> TraceGuard<'a> { - pub fn new(label: &'a str, tracing_enabled: bool) -> Self { - if tracing_enabled { - trace_begin(label); - } - Self { - label, - tracing_enabled, - } - } -} - -impl Drop for TraceGuard<'_> { - fn drop(&mut self) { - if self.tracing_enabled { - trace_end(self.label); - } - } -} From 870a7cb5a6fd823190b4afec4ccf09ebc121aae0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 16:21:55 -0600 Subject: [PATCH 11/19] with_trace --- native/core/src/execution/jni_api.rs | 160 +++++++++++++-------------- native/core/src/execution/tracing.rs | 17 +++ 2 files changed, 94 insertions(+), 83 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 3610da685b8..7603da28ae8 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -67,7 +67,7 @@ use crate::execution::operators::ScanExec; use crate::execution::shuffle::{read_ipc_compressed, CompressionCodec}; use crate::execution::spark_plan::SparkPlan; -use crate::execution::tracing::{log_memory_usage, trace_begin, trace_end}; +use crate::execution::tracing::{log_memory_usage, trace_begin, trace_end, with_trace}; use datafusion_comet_proto::spark_operator::operator::OpStruct; use log::info; @@ -162,96 +162,90 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_createPlan( tracing_enabled: jboolean, ) -> jlong { try_unwrap_or_throw(&e, |mut env| { - if tracing_enabled != JNI_FALSE { - trace_begin("createPlan"); - } - - // Init JVM classes - JVMClasses::init(&mut env); - - let start = Instant::now(); - - let array = unsafe { JPrimitiveArray::from_raw(serialized_query) }; - let bytes = env.convert_byte_array(array)?; + with_trace("createPlan", tracing_enabled != JNI_FALSE, || { + // Init JVM classes + JVMClasses::init(&mut env); - // Deserialize query plan - let spark_plan = serde::deserialize_op(bytes.as_slice())?; - - let metrics = Arc::new(jni_new_global_ref!(env, metrics_node)?); - - // Get the global references of input sources - let mut input_sources = vec![]; - let iter_array = JObjectArray::from_raw(iterators); - let num_inputs = env.get_array_length(&iter_array)?; - for i in 0..num_inputs { - let input_source = env.get_object_array_element(&iter_array, i)?; - let input_source = Arc::new(jni_new_global_ref!(env, input_source)?); - input_sources.push(input_source); - } - - // Create DataFusion memory pool - let task_memory_manager = - Arc::new(jni_new_global_ref!(env, comet_task_memory_manager_obj)?); + let start = Instant::now(); - let memory_pool_type = env.get_string(&JString::from_raw(memory_pool_type))?.into(); - let memory_pool_config = parse_memory_pool_config( - off_heap_mode != JNI_FALSE, - memory_pool_type, - memory_limit, - memory_limit_per_task, - )?; - let memory_pool = - create_memory_pool(&memory_pool_config, task_memory_manager, task_attempt_id); - - // Get local directories for storing spill files - let local_dirs_array = JObjectArray::from_raw(local_dirs); - let num_local_dirs = env.get_array_length(&local_dirs_array)?; - let mut local_dirs = vec![]; - for i in 0..num_local_dirs { - let local_dir: JString = env.get_object_array_element(&local_dirs_array, i)?.into(); - let local_dir = env.get_string(&local_dir)?; - local_dirs.push(local_dir.into()); - } + let array = unsafe { JPrimitiveArray::from_raw(serialized_query) }; + let bytes = env.convert_byte_array(array)?; - // We need to keep the session context alive. Some session state like temporary - // dictionaries are stored in session context. If it is dropped, the temporary - // dictionaries will be dropped as well. - let session = - prepare_datafusion_session_context(batch_size as usize, memory_pool, local_dirs)?; + // Deserialize query plan + let spark_plan = serde::deserialize_op(bytes.as_slice())?; - let plan_creation_time = start.elapsed(); + let metrics = Arc::new(jni_new_global_ref!(env, metrics_node)?); - let metrics_update_interval = if metrics_update_interval > 0 { - Some(Duration::from_millis(metrics_update_interval as u64)) - } else { - None - }; + // Get the global references of input sources + let mut input_sources = vec![]; + let iter_array = JObjectArray::from_raw(iterators); + let num_inputs = env.get_array_length(&iter_array)?; + for i in 0..num_inputs { + let input_source = env.get_object_array_element(&iter_array, i)?; + let input_source = Arc::new(jni_new_global_ref!(env, input_source)?); + input_sources.push(input_source); + } - let exec_context = Box::new(ExecutionContext { - id, - task_attempt_id, - spark_plan, - partition_count: partition_count as usize, - root_op: None, - scans: vec![], - input_sources, - stream: None, - metrics, - metrics_update_interval, - metrics_last_update_time: Instant::now(), - plan_creation_time, - session_ctx: Arc::new(session), - debug_native: debug_native == 1, - explain_native: explain_native == 1, - memory_pool_config, - tracing_enabled: tracing_enabled != JNI_FALSE, - }); + // Create DataFusion memory pool + let task_memory_manager = + Arc::new(jni_new_global_ref!(env, comet_task_memory_manager_obj)?); - if tracing_enabled != JNI_FALSE { - trace_end("createPlan"); - } + let memory_pool_type = env.get_string(&JString::from_raw(memory_pool_type))?.into(); + let memory_pool_config = parse_memory_pool_config( + off_heap_mode != JNI_FALSE, + memory_pool_type, + memory_limit, + memory_limit_per_task, + )?; + let memory_pool = + create_memory_pool(&memory_pool_config, task_memory_manager, task_attempt_id); + + // Get local directories for storing spill files + let local_dirs_array = JObjectArray::from_raw(local_dirs); + let num_local_dirs = env.get_array_length(&local_dirs_array)?; + let mut local_dirs = vec![]; + for i in 0..num_local_dirs { + let local_dir: JString = env.get_object_array_element(&local_dirs_array, i)?.into(); + let local_dir = env.get_string(&local_dir)?; + local_dirs.push(local_dir.into()); + } - Ok(Box::into_raw(exec_context) as i64) + // We need to keep the session context alive. Some session state like temporary + // dictionaries are stored in session context. If it is dropped, the temporary + // dictionaries will be dropped as well. + let session = + prepare_datafusion_session_context(batch_size as usize, memory_pool, local_dirs)?; + + let plan_creation_time = start.elapsed(); + + let metrics_update_interval = if metrics_update_interval > 0 { + Some(Duration::from_millis(metrics_update_interval as u64)) + } else { + None + }; + + let exec_context = Box::new(ExecutionContext { + id, + task_attempt_id, + spark_plan, + partition_count: partition_count as usize, + root_op: None, + scans: vec![], + input_sources, + stream: None, + metrics, + metrics_update_interval, + metrics_last_update_time: Instant::now(), + plan_creation_time, + session_ctx: Arc::new(session), + debug_native: debug_native == 1, + explain_native: explain_native == 1, + memory_pool_config, + tracing_enabled: tracing_enabled != JNI_FALSE, + }); + + Ok(Box::into_raw(exec_context) as i64) + }) }) } diff --git a/native/core/src/execution/tracing.rs b/native/core/src/execution/tracing.rs index 61e3b3b699d..f6953dfb3e5 100644 --- a/native/core/src/execution/tracing.rs +++ b/native/core/src/execution/tracing.rs @@ -105,3 +105,20 @@ pub(crate) fn trace_end(name: &str) { pub(crate) fn log_memory_usage(name: &str, value: u64) { RECORDER.log_memory_usage(name, value); } + +pub(crate) fn with_trace(label: &str, tracing_enabled: bool, f: F) -> T +where + F: FnOnce() -> T, +{ + if tracing_enabled { + trace_begin(label); + } + + let result = f(); + + if tracing_enabled { + trace_end(label); + } + + result +} From 34d967bd366c0819e46c51c02f0c8078e576c506 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 16:23:42 -0600 Subject: [PATCH 12/19] with_trace --- native/core/src/execution/jni_api.rs | 174 +++++++++++++-------------- 1 file changed, 84 insertions(+), 90 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 7603da28ae8..c5616a97b2f 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -377,8 +377,6 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan( }; if exec_context.tracing_enabled { - trace_begin(tracing_event_name); - #[cfg(feature = "jemalloc")] { let e = epoch::mib().unwrap(); @@ -389,105 +387,101 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_executePlan( } } - let exec_context_id = exec_context.id; - - // Initialize the execution stream. - // Because we don't know if input arrays are dictionary-encoded when we create - // query plan, we need to defer stream initialization to first time execution. - if exec_context.root_op.is_none() { - let start = Instant::now(); - let planner = PhysicalPlanner::new(Arc::clone(&exec_context.session_ctx)) - .with_exec_id(exec_context_id); - let (scans, root_op) = planner.create_plan( - &exec_context.spark_plan, - &mut exec_context.input_sources.clone(), - exec_context.partition_count, - )?; - let physical_plan_time = start.elapsed(); - - exec_context.plan_creation_time += physical_plan_time; - exec_context.root_op = Some(Arc::clone(&root_op)); - exec_context.scans = scans; + with_trace(tracing_event_name, exec_context.tracing_enabled, || { + let exec_context_id = exec_context.id; + + // Initialize the execution stream. + // Because we don't know if input arrays are dictionary-encoded when we create + // query plan, we need to defer stream initialization to first time execution. + if exec_context.root_op.is_none() { + let start = Instant::now(); + let planner = PhysicalPlanner::new(Arc::clone(&exec_context.session_ctx)) + .with_exec_id(exec_context_id); + let (scans, root_op) = planner.create_plan( + &exec_context.spark_plan, + &mut exec_context.input_sources.clone(), + exec_context.partition_count, + )?; + let physical_plan_time = start.elapsed(); + + exec_context.plan_creation_time += physical_plan_time; + exec_context.root_op = Some(Arc::clone(&root_op)); + exec_context.scans = scans; + + if exec_context.explain_native { + let formatted_plan_str = + DisplayableExecutionPlan::new(root_op.native_plan.as_ref()).indent(true); + info!("Comet native query plan:\n{formatted_plan_str:}"); + } - if exec_context.explain_native { - let formatted_plan_str = - DisplayableExecutionPlan::new(root_op.native_plan.as_ref()).indent(true); - info!("Comet native query plan:\n{formatted_plan_str:}"); + let task_ctx = exec_context.session_ctx.task_ctx(); + let stream = exec_context + .root_op + .as_ref() + .unwrap() + .native_plan + .execute(partition as usize, task_ctx)?; + exec_context.stream = Some(stream); + } else { + // Pull input batches + pull_input_batches(exec_context)?; } - let task_ctx = exec_context.session_ctx.task_ctx(); - let stream = exec_context - .root_op - .as_ref() - .unwrap() - .native_plan - .execute(partition as usize, task_ctx)?; - exec_context.stream = Some(stream); - } else { - // Pull input batches - pull_input_batches(exec_context)?; - } - - loop { - // Polling the stream. - let next_item = exec_context.stream.as_mut().unwrap().next(); - let poll_output = get_runtime().block_on(async { poll!(next_item) }); - - // update metrics at interval - if let Some(interval) = exec_context.metrics_update_interval { - let now = Instant::now(); - if now - exec_context.metrics_last_update_time >= interval { - update_metrics(&mut env, exec_context)?; - exec_context.metrics_last_update_time = now; + loop { + // Polling the stream. + let next_item = exec_context.stream.as_mut().unwrap().next(); + let poll_output = get_runtime().block_on(async { poll!(next_item) }); + + // update metrics at interval + if let Some(interval) = exec_context.metrics_update_interval { + let now = Instant::now(); + if now - exec_context.metrics_last_update_time >= interval { + update_metrics(&mut env, exec_context)?; + exec_context.metrics_last_update_time = now; + } } - } - match poll_output { - Poll::Ready(Some(output)) => { - // prepare output for FFI transfer - let addr = prepare_output( - &mut env, - array_addrs, - schema_addrs, - output?, - exec_context.debug_native, - ); - if exec_context.tracing_enabled { - trace_end(tracing_event_name); + match poll_output { + Poll::Ready(Some(output)) => { + // prepare output for FFI transfer + return prepare_output( + &mut env, + array_addrs, + schema_addrs, + output?, + exec_context.debug_native, + ); } - return addr; - } - Poll::Ready(None) => { - // Reaches EOF of output. - if exec_context.explain_native { - if let Some(plan) = &exec_context.root_op { - let formatted_plan_str = - DisplayableExecutionPlan::with_metrics(plan.native_plan.as_ref()) - .indent(true); - info!( - "Comet native query plan with metrics (Plan #{} Stage {} Partition {}):\ - \n plan creation (including CometScans fetching first batches) took {:?}:\ - \n{formatted_plan_str:}", - plan.plan_id, stage_id, partition, exec_context.plan_creation_time - ); + Poll::Ready(None) => { + // Reaches EOF of output. + if exec_context.explain_native { + if let Some(plan) = &exec_context.root_op { + let formatted_plan_str = DisplayableExecutionPlan::with_metrics( + plan.native_plan.as_ref(), + ) + .indent(true); + info!( + "Comet native query plan with metrics (Plan #{} Stage {} Partition {}):\ + \n plan creation (including CometScans fetching first batches) took {:?}:\ + \n{formatted_plan_str:}", + plan.plan_id, stage_id, partition, exec_context.plan_creation_time + ); + } } + return Ok(-1); } - if exec_context.tracing_enabled { - trace_end(tracing_event_name); + // A poll pending means there are more than one blocking operators, + // we don't need go back-forth between JVM/Native. Just keeping polling. + Poll::Pending => { + // Pull input batches + pull_input_batches(exec_context)?; + + // Output not ready yet + continue; } - return Ok(-1); - } - // A poll pending means there are more than one blocking operators, - // we don't need go back-forth between JVM/Native. Just keeping polling. - Poll::Pending => { - // Pull input batches - pull_input_batches(exec_context)?; - - // Output not ready yet - continue; } } - } + }) }) } From 784d04b825ed7c9e3299d2666aded494dbd0b0e0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 16:25:51 -0600 Subject: [PATCH 13/19] with_trace --- native/core/src/execution/jni_api.rs | 167 +++++++++++++-------------- 1 file changed, 81 insertions(+), 86 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index c5616a97b2f..abff7043a67 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -570,77 +570,76 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_writeSortedFileNative tracing_enabled: jboolean, ) -> jlongArray { try_unwrap_or_throw(&e, |mut env| unsafe { - if tracing_enabled != JNI_FALSE { - trace_begin("writeSortedFileNative"); - } - - let data_types = convert_datatype_arrays(&mut env, serialized_datatypes)?; - - let row_address_array = JLongArray::from_raw(row_addresses); - let row_num = env.get_array_length(&row_address_array)? as usize; - let row_addresses = env.get_array_elements(&row_address_array, ReleaseMode::NoCopyBack)?; - - let row_size_array = JIntArray::from_raw(row_sizes); - let row_sizes = env.get_array_elements(&row_size_array, ReleaseMode::NoCopyBack)?; - - let row_addresses_ptr = row_addresses.as_ptr(); - let row_sizes_ptr = row_sizes.as_ptr(); + with_trace( + "writeSortedFileNative", + tracing_enabled != JNI_FALSE, + || { + let data_types = convert_datatype_arrays(&mut env, serialized_datatypes)?; - let output_path: String = env - .get_string(&JString::from_raw(file_path)) - .unwrap() - .into(); + let row_address_array = JLongArray::from_raw(row_addresses); + let row_num = env.get_array_length(&row_address_array)? as usize; + let row_addresses = + env.get_array_elements(&row_address_array, ReleaseMode::NoCopyBack)?; - let checksum_enabled = checksum_enabled == 1; - let current_checksum = if current_checksum == i64::MIN { - // Initial checksum is not available. - None - } else { - Some(current_checksum as u32) - }; - - let compression_codec: String = env - .get_string(&JString::from_raw(compression_codec)) - .unwrap() - .into(); + let row_size_array = JIntArray::from_raw(row_sizes); + let row_sizes = env.get_array_elements(&row_size_array, ReleaseMode::NoCopyBack)?; - let compression_codec = match compression_codec.as_str() { - "zstd" => CompressionCodec::Zstd(compression_level), - "lz4" => CompressionCodec::Lz4Frame, - "snappy" => CompressionCodec::Snappy, - _ => CompressionCodec::Lz4Frame, - }; + let row_addresses_ptr = row_addresses.as_ptr(); + let row_sizes_ptr = row_sizes.as_ptr(); - let (written_bytes, checksum) = process_sorted_row_partition( - row_num, - batch_size as usize, - row_addresses_ptr, - row_sizes_ptr, - &data_types, - output_path, - prefer_dictionary_ratio, - checksum_enabled, - checksum_algo, - current_checksum, - &compression_codec, - )?; - - let checksum = if let Some(checksum) = checksum { - checksum as i64 - } else { - // Spark checksums (CRC32 or Adler32) are both u32, so we use i64::MIN to indicate - // checksum is not available. - i64::MIN - }; + let output_path: String = env + .get_string(&JString::from_raw(file_path)) + .unwrap() + .into(); + + let checksum_enabled = checksum_enabled == 1; + let current_checksum = if current_checksum == i64::MIN { + // Initial checksum is not available. + None + } else { + Some(current_checksum as u32) + }; + + let compression_codec: String = env + .get_string(&JString::from_raw(compression_codec)) + .unwrap() + .into(); + + let compression_codec = match compression_codec.as_str() { + "zstd" => CompressionCodec::Zstd(compression_level), + "lz4" => CompressionCodec::Lz4Frame, + "snappy" => CompressionCodec::Snappy, + _ => CompressionCodec::Lz4Frame, + }; + + let (written_bytes, checksum) = process_sorted_row_partition( + row_num, + batch_size as usize, + row_addresses_ptr, + row_sizes_ptr, + &data_types, + output_path, + prefer_dictionary_ratio, + checksum_enabled, + checksum_algo, + current_checksum, + &compression_codec, + )?; - let long_array = env.new_long_array(2)?; - env.set_long_array_region(&long_array, 0, &[written_bytes, checksum])?; + let checksum = if let Some(checksum) = checksum { + checksum as i64 + } else { + // Spark checksums (CRC32 or Adler32) are both u32, so we use i64::MIN to indicate + // checksum is not available. + i64::MIN + }; - if tracing_enabled != JNI_FALSE { - trace_end("writeSortedFileNative"); - } + let long_array = env.new_long_array(2)?; + env.set_long_array_region(&long_array, 0, &[written_bytes, checksum])?; - Ok(long_array.into_raw()) + Ok(long_array.into_raw()) + }, + ) }) } @@ -654,16 +653,17 @@ pub extern "system" fn Java_org_apache_comet_Native_sortRowPartitionsNative( tracing_enabled: jboolean, ) { try_unwrap_or_throw(&e, |_| { - if tracing_enabled != JNI_FALSE { - trace_begin("sortRowPartitionsNative"); - } - // SAFETY: JVM unsafe memory allocation is aligned with long. - let array = unsafe { std::slice::from_raw_parts_mut(address as *mut i64, size as usize) }; - array.rdxsort(); - if tracing_enabled != JNI_FALSE { - trace_end("sortRowPartitionsNative"); - } - Ok(()) + with_trace( + "sortRowPartitionsNative", + tracing_enabled != JNI_FALSE, + || { + // SAFETY: JVM unsafe memory allocation is aligned with long. + let array = + unsafe { std::slice::from_raw_parts_mut(address as *mut i64, size as usize) }; + array.rdxsort(); + Ok(()) + }, + ) }) } @@ -681,18 +681,13 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_decodeShuffleBlock( tracing_enabled: jboolean, ) -> jlong { try_unwrap_or_throw(&e, |mut env| { - if tracing_enabled != JNI_FALSE { - trace_begin("decodeShuffleBlock"); - } - let raw_pointer = env.get_direct_buffer_address(&byte_buffer)?; - let length = length as usize; - let slice: &[u8] = unsafe { std::slice::from_raw_parts(raw_pointer, length) }; - let batch = read_ipc_compressed(slice)?; - let addr = prepare_output(&mut env, array_addrs, schema_addrs, batch, false); - if tracing_enabled != JNI_FALSE { - trace_end("decodeShuffleBlock"); - } - addr + with_trace("decodeShuffleBlock", tracing_enabled != JNI_FALSE, || { + let raw_pointer = env.get_direct_buffer_address(&byte_buffer)?; + let length = length as usize; + let slice: &[u8] = unsafe { std::slice::from_raw_parts(raw_pointer, length) }; + let batch = read_ipc_compressed(slice)?; + prepare_output(&mut env, array_addrs, schema_addrs, batch, false) + }) }) } From fb1ca4f6fa50803be0782c0fa7170a42eaaaf0ac Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 16:29:10 -0600 Subject: [PATCH 14/19] with_trace --- .../src/execution/shuffle/shuffle_writer.rs | 125 +++++++++--------- 1 file changed, 59 insertions(+), 66 deletions(-) diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index 701f57e6b4d..0bd8aaf6429 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -18,7 +18,7 @@ //! Defines the External shuffle repartition plan. use crate::execution::shuffle::{CompressionCodec, ShuffleBlockWriter}; -use crate::execution::tracing::{trace_begin, trace_end}; +use crate::execution::tracing::{trace_begin, trace_end, with_trace}; use arrow::compute::interleave_record_batch; use async_trait::async_trait; use datafusion::common::utils::proxy::VecAllocExt; @@ -690,83 +690,76 @@ impl ShufflePartitioner for MultiPartitionShuffleRepartitioner { /// Writes buffered shuffled record batches into Arrow IPC bytes. async fn shuffle_write(&mut self) -> Result<()> { - if self.tracing_enabled { - trace_begin("shuffle_write"); - } - - let start_time = Instant::now(); - - let mut partitioned_batches = self.partitioned_batches(); - let num_output_partitions = self.partition_indices.len(); - let mut offsets = vec![0; num_output_partitions + 1]; - - let data_file = self.output_data_file.clone(); - let index_file = self.output_index_file.clone(); + with_trace("shuffle_write", self.tracing_enabled, || { + let start_time = Instant::now(); - let output_data = OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(data_file) - .map_err(|e| DataFusionError::Execution(format!("shuffle write error: {:?}", e)))?; + let mut partitioned_batches = self.partitioned_batches(); + let num_output_partitions = self.partition_indices.len(); + let mut offsets = vec![0; num_output_partitions + 1]; - let mut output_data = BufWriter::new(output_data); + let data_file = self.output_data_file.clone(); + let index_file = self.output_index_file.clone(); - #[allow(clippy::needless_range_loop)] - for i in 0..num_output_partitions { - offsets[i] = output_data.stream_position()?; + let output_data = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(data_file) + .map_err(|e| DataFusionError::Execution(format!("shuffle write error: {:?}", e)))?; + + let mut output_data = BufWriter::new(output_data); + + #[allow(clippy::needless_range_loop)] + for i in 0..num_output_partitions { + offsets[i] = output_data.stream_position()?; + + // if we wrote a spill file for this partition then copy the + // contents into the shuffle file + if let Some(spill_data) = self.partition_writers[i].spill_file.as_ref() { + let mut spill_file = + BufReader::new(File::open(spill_data.temp_file.path()).map_err(to_df_err)?); + let mut write_timer = self.metrics.write_time.timer(); + std::io::copy(&mut spill_file, &mut output_data).map_err(to_df_err)?; + write_timer.stop(); + } - // if we wrote a spill file for this partition then copy the - // contents into the shuffle file - if let Some(spill_data) = self.partition_writers[i].spill_file.as_ref() { - let mut spill_file = - BufReader::new(File::open(spill_data.temp_file.path()).map_err(to_df_err)?); - let mut write_timer = self.metrics.write_time.timer(); - std::io::copy(&mut spill_file, &mut output_data).map_err(to_df_err)?; - write_timer.stop(); + // Write in memory batches to output data file + let mut partition_iter = partitioned_batches.produce(i); + Self::shuffle_write_partition( + &mut partition_iter, + &mut self.shuffle_block_writer, + &mut output_data, + &self.metrics.encode_time, + &self.metrics.write_time, + )?; } - // Write in memory batches to output data file - let mut partition_iter = partitioned_batches.produce(i); - Self::shuffle_write_partition( - &mut partition_iter, - &mut self.shuffle_block_writer, - &mut output_data, - &self.metrics.encode_time, - &self.metrics.write_time, - )?; - } - - let mut write_timer = self.metrics.write_time.timer(); - output_data.flush()?; - write_timer.stop(); + let mut write_timer = self.metrics.write_time.timer(); + output_data.flush()?; + write_timer.stop(); - // add one extra offset at last to ease partition length computation - offsets[num_output_partitions] = output_data.stream_position().map_err(to_df_err)?; + // add one extra offset at last to ease partition length computation + offsets[num_output_partitions] = output_data.stream_position().map_err(to_df_err)?; - let mut write_timer = self.metrics.write_time.timer(); - let mut output_index = - BufWriter::new(File::create(index_file).map_err(|e| { + let mut write_timer = self.metrics.write_time.timer(); + let mut output_index = BufWriter::new(File::create(index_file).map_err(|e| { DataFusionError::Execution(format!("shuffle write error: {:?}", e)) })?); - for offset in offsets { - output_index - .write_all(&(offset as i64).to_le_bytes()[..]) - .map_err(to_df_err)?; - } - output_index.flush()?; - write_timer.stop(); - - self.metrics - .baseline - .elapsed_compute() - .add_duration(start_time.elapsed()); + for offset in offsets { + output_index + .write_all(&(offset as i64).to_le_bytes()[..]) + .map_err(to_df_err)?; + } + output_index.flush()?; + write_timer.stop(); - if self.tracing_enabled { - trace_end("shuffle_write"); - } + self.metrics + .baseline + .elapsed_compute() + .add_duration(start_time.elapsed()); - Ok(()) + Ok(()) + }) } } From c3c40e85b5c68fd0c0eaeeee12a1016f888aa5a2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 16:37:33 -0600 Subject: [PATCH 15/19] with_trace --- .../src/execution/shuffle/shuffle_writer.rs | 85 ++++++++++--------- native/core/src/execution/tracing.rs | 18 ++++ 2 files changed, 61 insertions(+), 42 deletions(-) diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index 0bd8aaf6429..2a841c5f981 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -18,7 +18,7 @@ //! Defines the External shuffle repartition plan. use crate::execution::shuffle::{CompressionCodec, ShuffleBlockWriter}; -use crate::execution::tracing::{trace_begin, trace_end, with_trace}; +use crate::execution::tracing::{trace_begin, trace_end, with_trace, with_trace_async}; use arrow::compute::interleave_record_batch; use async_trait::async_trait; use datafusion::common::utils::proxy::VecAllocExt; @@ -215,51 +215,52 @@ async fn external_shuffle( codec: CompressionCodec, tracing_enabled: bool, ) -> Result { - if tracing_enabled { - trace_begin("external_shuffle"); - } - - let schema = input.schema(); + with_trace_async("external_shuffle", tracing_enabled, || async { + let schema = input.schema(); + + let mut repartitioner: Box = match &partitioning { + any if any.partition_count() == 1 => { + Box::new(SinglePartitionShufflePartitioner::try_new( + output_data_file, + output_index_file, + Arc::clone(&schema), + metrics, + context.session_config().batch_size(), + codec, + )?) + } + _ => Box::new(MultiPartitionShuffleRepartitioner::try_new( + partition, + output_data_file, + output_index_file, + Arc::clone(&schema), + partitioning, + metrics, + context.runtime_env(), + context.session_config().batch_size(), + codec, + tracing_enabled, + )?), + }; - let mut repartitioner: Box = match &partitioning { - any if any.partition_count() == 1 => Box::new(SinglePartitionShufflePartitioner::try_new( - output_data_file, - output_index_file, - Arc::clone(&schema), - metrics, - context.session_config().batch_size(), - codec, - )?), - _ => Box::new(MultiPartitionShuffleRepartitioner::try_new( - partition, - output_data_file, - output_index_file, - Arc::clone(&schema), - partitioning, - metrics, - context.runtime_env(), - context.session_config().batch_size(), - codec, - tracing_enabled, - )?), - }; - - while let Some(batch) = input.next().await { - // Block on the repartitioner to insert the batch and shuffle the rows - // into the corresponding partition buffer. - // Otherwise, pull the next batch from the input stream might overwrite the - // current batch in the repartitioner. - block_on(repartitioner.insert_batch(batch?))?; - } + while let Some(batch) = input.next().await { + // Block on the repartitioner to insert the batch and shuffle the rows + // into the corresponding partition buffer. + // Otherwise, pull the next batch from the input stream might overwrite the + // current batch in the repartitioner. + block_on(repartitioner.insert_batch(batch?))?; + } - repartitioner.shuffle_write().await?; + repartitioner.shuffle_write().await?; - if tracing_enabled { - trace_end("external_shuffle"); - } + if tracing_enabled { + trace_end("external_shuffle"); + } - // shuffle writer always has empty output - Ok(Box::pin(EmptyRecordBatchStream::new(Arc::clone(&schema)))) + // shuffle writer always has empty output + Ok(Box::pin(EmptyRecordBatchStream::new(Arc::clone(&schema))) as SendableRecordBatchStream) + }) + .await } struct ShuffleRepartitionerMetrics { diff --git a/native/core/src/execution/tracing.rs b/native/core/src/execution/tracing.rs index f6953dfb3e5..0555e6b1473 100644 --- a/native/core/src/execution/tracing.rs +++ b/native/core/src/execution/tracing.rs @@ -122,3 +122,21 @@ where result } + +pub(crate) async fn with_trace_async(label: &str, tracing_enabled: bool, f: F) -> T +where + F: FnOnce() -> Fut, + Fut: std::future::Future, +{ + if tracing_enabled { + trace_begin(label); + } + + let result = f().await; + + if tracing_enabled { + trace_end(label); + } + + result +} From 0ed98561fc73e2a04b17acd0afe206c4c550a24d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 12 May 2025 16:39:26 -0600 Subject: [PATCH 16/19] with_trace --- .../src/execution/shuffle/shuffle_writer.rs | 85 ++++++++----------- 1 file changed, 34 insertions(+), 51 deletions(-) diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index 2a841c5f981..e15db6762ba 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -18,7 +18,7 @@ //! Defines the External shuffle repartition plan. use crate::execution::shuffle::{CompressionCodec, ShuffleBlockWriter}; -use crate::execution::tracing::{trace_begin, trace_end, with_trace, with_trace_async}; +use crate::execution::tracing::{with_trace, with_trace_async}; use arrow::compute::interleave_record_batch; use async_trait::async_trait; use datafusion::common::utils::proxy::VecAllocExt; @@ -253,10 +253,6 @@ async fn external_shuffle( repartitioner.shuffle_write().await?; - if tracing_enabled { - trace_end("external_shuffle"); - } - // shuffle writer always has empty output Ok(Box::pin(EmptyRecordBatchStream::new(Arc::clone(&schema))) as SendableRecordBatchStream) }) @@ -630,31 +626,24 @@ impl MultiPartitionShuffleRepartitioner { return Ok(()); } - if self.tracing_enabled { - trace_begin("shuffle_spill"); - } - - let num_output_partitions = self.partition_writers.len(); - let mut partitioned_batches = self.partitioned_batches(); - let mut spilled_bytes = 0; - - for partition_id in 0..num_output_partitions { - let partition_writer = &mut self.partition_writers[partition_id]; - let mut iter = partitioned_batches.produce(partition_id); - spilled_bytes += partition_writer.spill(&mut iter, &self.runtime, &self.metrics)?; - } - - let mut timer = self.metrics.mempool_time.timer(); - self.reservation.free(); - timer.stop(); - self.metrics.spill_count.add(1); - self.metrics.spilled_bytes.add(spilled_bytes); + with_trace("shuffle_spill", self.tracing_enabled, || { + let num_output_partitions = self.partition_writers.len(); + let mut partitioned_batches = self.partitioned_batches(); + let mut spilled_bytes = 0; - if self.tracing_enabled { - trace_end("shuffle_spill"); - } + for partition_id in 0..num_output_partitions { + let partition_writer = &mut self.partition_writers[partition_id]; + let mut iter = partitioned_batches.produce(partition_id); + spilled_bytes += partition_writer.spill(&mut iter, &self.runtime, &self.metrics)?; + } - Ok(()) + let mut timer = self.metrics.mempool_time.timer(); + self.reservation.free(); + timer.stop(); + self.metrics.spill_count.add(1); + self.metrics.spilled_bytes.add(spilled_bytes); + Ok(()) + }) } } @@ -664,29 +653,23 @@ impl ShufflePartitioner for MultiPartitionShuffleRepartitioner { /// This function will slice input batch according to configured batch size and then /// shuffle rows into corresponding partition buffer. async fn insert_batch(&mut self, batch: RecordBatch) -> Result<()> { - if self.tracing_enabled { - trace_begin("shuffle_insert_batch"); - } - - let start_time = Instant::now(); - let mut start = 0; - while start < batch.num_rows() { - let end = (start + self.batch_size).min(batch.num_rows()); - let batch = batch.slice(start, end - start); - self.partitioning_batch(batch).await?; - start = end; - } - self.metrics.input_batches.add(1); - self.metrics - .baseline - .elapsed_compute() - .add_duration(start_time.elapsed()); - - if self.tracing_enabled { - trace_end("shuffle_insert_batch"); - } - - Ok(()) + with_trace_async("shuffle_insert_batch", self.tracing_enabled, || async { + let start_time = Instant::now(); + let mut start = 0; + while start < batch.num_rows() { + let end = (start + self.batch_size).min(batch.num_rows()); + let batch = batch.slice(start, end - start); + self.partitioning_batch(batch).await?; + start = end; + } + self.metrics.input_batches.add(1); + self.metrics + .baseline + .elapsed_compute() + .add_duration(start_time.elapsed()); + Ok(()) + }) + .await } /// Writes buffered shuffled record batches into Arrow IPC bytes. From f3ab2b6364ab180fbe4d284b6572814c25294396 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 13 May 2025 06:28:52 -0600 Subject: [PATCH 17/19] remove async marker from spill method --- native/core/src/execution/shuffle/shuffle_writer.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index e15db6762ba..ecc754ab4e0 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -562,7 +562,7 @@ impl MultiPartitionShuffleRepartitioner { result }; if grow_result.is_err() { - self.spill().await?; + self.spill()?; } Ok(()) @@ -614,7 +614,7 @@ impl MultiPartitionShuffleRepartitioner { PartitionedBatchesProducer::new(buffered_batches, indices, self.batch_size) } - async fn spill(&mut self) -> Result<()> { + fn spill(&mut self) -> Result<()> { log::debug!( "ShuffleRepartitioner spilling shuffle data of {} to disk while inserting ({} time(s) so far)", self.used(), @@ -1269,7 +1269,7 @@ mod test { assert!(repartitioner.partition_writers[0].spill_file.is_none()); assert!(repartitioner.partition_writers[1].spill_file.is_none()); - repartitioner.spill().await.unwrap(); + repartitioner.spill().unwrap(); // after spill, there should be spill files assert!(repartitioner.partition_writers[0].spill_file.is_some()); From 0c7f448340b53136d5777b75b210a80d41044356 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 13 May 2025 06:31:07 -0600 Subject: [PATCH 18/19] remove async marker from shuffle_write method --- native/core/src/execution/shuffle/shuffle_writer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index ecc754ab4e0..e2619eed70d 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -251,7 +251,7 @@ async fn external_shuffle( block_on(repartitioner.insert_batch(batch?))?; } - repartitioner.shuffle_write().await?; + repartitioner.shuffle_write()?; // shuffle writer always has empty output Ok(Box::pin(EmptyRecordBatchStream::new(Arc::clone(&schema))) as SendableRecordBatchStream) @@ -309,7 +309,7 @@ trait ShufflePartitioner: Send + Sync { /// Insert a batch into the partitioner async fn insert_batch(&mut self, batch: RecordBatch) -> Result<()>; /// Write shuffle data and shuffle index file to disk - async fn shuffle_write(&mut self) -> Result<()>; + fn shuffle_write(&mut self) -> Result<()>; } /// A partitioner that uses a hash function to partition data into multiple partitions @@ -673,7 +673,7 @@ impl ShufflePartitioner for MultiPartitionShuffleRepartitioner { } /// Writes buffered shuffled record batches into Arrow IPC bytes. - async fn shuffle_write(&mut self) -> Result<()> { + fn shuffle_write(&mut self) -> Result<()> { with_trace("shuffle_write", self.tracing_enabled, || { let start_time = Instant::now(); @@ -887,7 +887,7 @@ impl ShufflePartitioner for SinglePartitionShufflePartitioner { Ok(()) } - async fn shuffle_write(&mut self) -> Result<()> { + fn shuffle_write(&mut self) -> Result<()> { let start_time = Instant::now(); let concatenated_batch = self.concat_buffered_batches()?; From d1c77c45072226dc0ddb92f14665ade4a71baaec Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 13 May 2025 06:40:41 -0600 Subject: [PATCH 19/19] update docs --- docs/source/contributor-guide/tracing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contributor-guide/tracing.md b/docs/source/contributor-guide/tracing.md index fa1dd14aa75..b9b4fe0dcc4 100644 --- a/docs/source/contributor-guide/tracing.md +++ b/docs/source/contributor-guide/tracing.md @@ -61,4 +61,4 @@ Example trace visualization: | jemalloc_allocated | Native memory usage for the executor process | | task_memory_comet_NNN | Off-heap memory allocated by Comet for query execution | | task_memory_spark_NNN | On-heap & Off-heap memory allocated by Spark | -| offheap_shuffle_NNN | Off-heap memory allocated by Comet for columnar shuffle | +| comet_shuffle_NNN | Off-heap memory allocated by Comet for columnar shuffle |