From 664b6e8a291d0100c73c61100b318f4793e43860 Mon Sep 17 00:00:00 2001 From: Lukas Moravec Date: Fri, 28 Feb 2025 16:24:45 +0100 Subject: [PATCH 1/7] fix: spark executor memory overhead overwritten into absurd amounts --- .../scala/org/apache/comet/CometSparkSessionExtensions.scala | 5 +++++ spark/src/main/scala/org/apache/spark/Plugins.scala | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala index 9f295bc3f18..c82d10da290 100644 --- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala +++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala @@ -1409,6 +1409,11 @@ object CometSparkSessionExtensions extends Logging { } } + /** Calculates Comet shuffle memory size in MB */ + def getCometShuffleMemorySizeInMiB(sparkConf: SparkConf, conf: SQLConf = SQLConf.get): Long = { + ByteUnit.BYTE.toMiB(getCometShuffleMemorySize(sparkConf, conf)) + } + def cometUnifiedMemoryManagerEnabled(sparkConf: SparkConf): Boolean = { sparkConf.getBoolean("spark.memory.offHeap.enabled", false) } diff --git a/spark/src/main/scala/org/apache/spark/Plugins.scala b/spark/src/main/scala/org/apache/spark/Plugins.scala index 0e5c5d6b03b..5863e29cedb 100644 --- a/spark/src/main/scala/org/apache/spark/Plugins.scala +++ b/spark/src/main/scala/org/apache/spark/Plugins.scala @@ -63,11 +63,11 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl } val cometMemOverhead = - if (!CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(sc.getConf)) { + if (CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(sc.getConf)) { CometSparkSessionExtensions.getCometMemoryOverheadInMiB(sc.getConf) } else { // comet shuffle unified memory manager is disabled, so we need to add overhead memory - CometSparkSessionExtensions.getCometShuffleMemorySize(sc.getConf) + CometSparkSessionExtensions.getCometShuffleMemorySizeInMiB(sc.getConf) } sc.conf.set(EXECUTOR_MEMORY_OVERHEAD.key, s"${execMemOverhead + cometMemOverhead}M") val newExecMemOverhead = sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) From 80bee244590820ca3bdd592787824d6ce0e91363 Mon Sep 17 00:00:00 2001 From: Lukas Moravec Date: Sun, 2 Mar 2025 10:26:53 +0100 Subject: [PATCH 2/7] fix: prevent executor memory overhead from being overridden in unified memory mode --- .../main/scala/org/apache/spark/Plugins.scala | 20 ++++++++----- .../org/apache/spark/CometPluginsSuite.scala | 30 +++++++++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/Plugins.scala b/spark/src/main/scala/org/apache/spark/Plugins.scala index 5863e29cedb..e0a26c35bdb 100644 --- a/spark/src/main/scala/org/apache/spark/Plugins.scala +++ b/spark/src/main/scala/org/apache/spark/Plugins.scala @@ -100,15 +100,19 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl * unified memory manager. */ private def shouldOverrideMemoryConf(conf: SparkConf): Boolean = { - conf.getBoolean(CometConf.COMET_ENABLED.key, true) && ( - conf.getBoolean( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, - CometConf.COMET_EXEC_SHUFFLE_ENABLED.defaultValue.get) || + // short-circuit if InTestEnabled otherwise would always return true in production use + if (CometSparkSessionExtensions.cometShuffleUnifiedMemoryManagerInTestEnabled(conf)) { + false + } else { + conf.getBoolean(CometConf.COMET_ENABLED.key, true) && ( conf.getBoolean( - CometConf.COMET_EXEC_ENABLED.key, - CometConf.COMET_EXEC_ENABLED.defaultValue.get) - ) && (!CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(conf) || - !CometSparkSessionExtensions.cometShuffleUnifiedMemoryManagerInTestEnabled(conf)) + CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, + CometConf.COMET_EXEC_SHUFFLE_ENABLED.defaultValue.get) || + conf.getBoolean( + CometConf.COMET_EXEC_ENABLED.key, + CometConf.COMET_EXEC_ENABLED.defaultValue.get) + ) && (!CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(conf)) + } } } diff --git a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala index 8b388f2d3af..2089c55a205 100644 --- a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala @@ -143,3 +143,33 @@ class CometPluginsNonOverrideSuite extends CometTestBase { assert(execMemOverhead4 == "2G") } } + +class CometPluginsNonOverrideUnifiedModeSuite extends CometTestBase { + override protected def sparkConf: SparkConf = { + val conf = new SparkConf() + conf.set("spark.driver.memory", "1G") + conf.set("spark.executor.memory", "1G") + conf.set("spark.executor.memoryOverhead", "1G") + conf.set("spark.plugins", "org.apache.spark.CometPlugin") + conf.set("spark.comet.enabled", "true") + conf.set("spark.memory.offHeap.enabled", "true") + conf.set("spark.memory.offHeap.size", "2G") + conf.set("spark.comet.exec.shuffle.enabled", "true") + conf.set("spark.comet.exec.enabled", "true") + conf.set("spark.comet.memory.overhead.factor", "0.5") + conf + } + + /** Since using unified memory, executor memory should not be overridden */ + test("executor memory overhead is not overridden") { + val execMemOverhead1 = spark.conf.get("spark.executor.memoryOverhead") + val execMemOverhead2 = spark.sessionState.conf.getConfString("spark.executor.memoryOverhead") + val execMemOverhead3 = spark.sparkContext.getConf.get("spark.executor.memoryOverhead") + val execMemOverhead4 = spark.sparkContext.conf.get("spark.executor.memoryOverhead") + + assert(execMemOverhead1 == "1G") + assert(execMemOverhead2 == "1G") + assert(execMemOverhead3 == "1G") + assert(execMemOverhead4 == "1G") + } +} From a578101ba40326b001ed1c17a24b8825d87fb932 Mon Sep 17 00:00:00 2001 From: Lukas Moravec Date: Sun, 2 Mar 2025 10:41:49 +0100 Subject: [PATCH 3/7] fix: ensure correct calculation of executor memory overhead in unified memory mode --- spark/src/main/scala/org/apache/spark/Plugins.scala | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/Plugins.scala b/spark/src/main/scala/org/apache/spark/Plugins.scala index e0a26c35bdb..288a3c39a61 100644 --- a/spark/src/main/scala/org/apache/spark/Plugins.scala +++ b/spark/src/main/scala/org/apache/spark/Plugins.scala @@ -62,13 +62,8 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl Math.max((executorMemory * memoryOverheadFactor).toLong, memoryOverheadMinMib) } - val cometMemOverhead = - if (CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(sc.getConf)) { - CometSparkSessionExtensions.getCometMemoryOverheadInMiB(sc.getConf) - } else { - // comet shuffle unified memory manager is disabled, so we need to add overhead memory - CometSparkSessionExtensions.getCometShuffleMemorySizeInMiB(sc.getConf) - } + // shouldOverrideMemoryConf guarantees that Comet is unified mode is disabled + val cometMemOverhead = CometSparkSessionExtensions.getCometShuffleMemorySizeInMiB(sc.getConf) sc.conf.set(EXECUTOR_MEMORY_OVERHEAD.key, s"${execMemOverhead + cometMemOverhead}M") val newExecMemOverhead = sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) From bd6231523e4e9f396aacd29e3020b6150242496c Mon Sep 17 00:00:00 2001 From: Lukas Moravec Date: Sun, 2 Mar 2025 18:04:20 +0100 Subject: [PATCH 4/7] fix: formatting --- spark/src/main/scala/org/apache/spark/Plugins.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/spark/Plugins.scala b/spark/src/main/scala/org/apache/spark/Plugins.scala index 288a3c39a61..951cb4fc4e3 100644 --- a/spark/src/main/scala/org/apache/spark/Plugins.scala +++ b/spark/src/main/scala/org/apache/spark/Plugins.scala @@ -63,7 +63,8 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl } // shouldOverrideMemoryConf guarantees that Comet is unified mode is disabled - val cometMemOverhead = CometSparkSessionExtensions.getCometShuffleMemorySizeInMiB(sc.getConf) + val cometMemOverhead = + CometSparkSessionExtensions.getCometShuffleMemorySizeInMiB(sc.getConf) sc.conf.set(EXECUTOR_MEMORY_OVERHEAD.key, s"${execMemOverhead + cometMemOverhead}M") val newExecMemOverhead = sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) From ecabb8ed4dcb9c5408e8634f38868bafb4429ae1 Mon Sep 17 00:00:00 2001 From: Lukas Moravec Date: Mon, 3 Mar 2025 16:51:17 +0100 Subject: [PATCH 5/7] fix: correctly override executor memory overhead in unified mode --- .../main/scala/org/apache/spark/Plugins.scala | 55 ++++++++++--------- .../org/apache/spark/CometPluginsSuite.scala | 15 +++-- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/Plugins.scala b/spark/src/main/scala/org/apache/spark/Plugins.scala index 951cb4fc4e3..da716e65612 100644 --- a/spark/src/main/scala/org/apache/spark/Plugins.scala +++ b/spark/src/main/scala/org/apache/spark/Plugins.scala @@ -51,22 +51,29 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl CometDriverPlugin.registerCometSessionExtension(sc.conf) if (shouldOverrideMemoryConf(sc.getConf)) { - val execMemOverhead = if (sc.getConf.contains(EXECUTOR_MEMORY_OVERHEAD.key)) { - sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) - } else { - // By default, executorMemory * spark.executor.memoryOverheadFactor, with minimum of 384MB - val executorMemory = sc.getConf.getSizeAsMb(EXECUTOR_MEMORY.key, EXECUTOR_MEMORY_DEFAULT) - val memoryOverheadFactor = getMemoryOverheadFactor(sc.getConf) - val memoryOverheadMinMib = getMemoryOverheadMinMib(sc.getConf) - - Math.max((executorMemory * memoryOverheadFactor).toLong, memoryOverheadMinMib) - } + val execMemOverhead = + if (sc.getConf.contains(EXECUTOR_MEMORY_OVERHEAD.key)) { + sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) + } else { + // By default, executorMemory * spark.executor.memoryOverheadFactor, with minimum of 384MB + val executorMemory = + sc.getConf.getSizeAsMb(EXECUTOR_MEMORY.key, EXECUTOR_MEMORY_DEFAULT) + val memoryOverheadFactor = getMemoryOverheadFactor(sc.getConf) + val memoryOverheadMinMib = getMemoryOverheadMinMib(sc.getConf) + + Math.max((executorMemory * memoryOverheadFactor).toLong, memoryOverheadMinMib) + } - // shouldOverrideMemoryConf guarantees that Comet is unified mode is disabled val cometMemOverhead = - CometSparkSessionExtensions.getCometShuffleMemorySizeInMiB(sc.getConf) + if (!CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(sc.getConf)) { + CometSparkSessionExtensions.getCometMemoryOverheadInMiB(sc.getConf) + } else { + // comet shuffle unified memory manager is disabled, so we need to add overhead memory + CometSparkSessionExtensions.getCometShuffleMemorySizeInMiB(sc.getConf) + } sc.conf.set(EXECUTOR_MEMORY_OVERHEAD.key, s"${execMemOverhead + cometMemOverhead}M") - val newExecMemOverhead = sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) + val newExecMemOverhead = + sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) logInfo(s""" Overriding Spark memory configuration for Comet: @@ -96,19 +103,17 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl * unified memory manager. */ private def shouldOverrideMemoryConf(conf: SparkConf): Boolean = { - // short-circuit if InTestEnabled otherwise would always return true in production use - if (CometSparkSessionExtensions.cometShuffleUnifiedMemoryManagerInTestEnabled(conf)) { - false - } else { - conf.getBoolean(CometConf.COMET_ENABLED.key, true) && ( + conf.getBoolean(CometConf.COMET_ENABLED.key, true) && ( + conf.getBoolean( + CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, + CometConf.COMET_EXEC_SHUFFLE_ENABLED.defaultValue.get) || conf.getBoolean( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, - CometConf.COMET_EXEC_SHUFFLE_ENABLED.defaultValue.get) || - conf.getBoolean( - CometConf.COMET_EXEC_ENABLED.key, - CometConf.COMET_EXEC_ENABLED.defaultValue.get) - ) && (!CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(conf)) - } + CometConf.COMET_EXEC_ENABLED.key, + CometConf.COMET_EXEC_ENABLED.defaultValue.get) + ) && ( + !CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(conf) || + !CometSparkSessionExtensions.cometShuffleUnifiedMemoryManagerInTestEnabled(conf) + ) } } diff --git a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala index 2089c55a205..88c6752b95d 100644 --- a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala @@ -144,7 +144,7 @@ class CometPluginsNonOverrideSuite extends CometTestBase { } } -class CometPluginsNonOverrideUnifiedModeSuite extends CometTestBase { +class CometPluginsUnifiedModeOverrideSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() conf.set("spark.driver.memory", "1G") @@ -161,15 +161,18 @@ class CometPluginsNonOverrideUnifiedModeSuite extends CometTestBase { } /** Since using unified memory, executor memory should not be overridden */ - test("executor memory overhead is not overridden") { + test("executor memory overhead is correctly overridden") { val execMemOverhead1 = spark.conf.get("spark.executor.memoryOverhead") val execMemOverhead2 = spark.sessionState.conf.getConfString("spark.executor.memoryOverhead") val execMemOverhead3 = spark.sparkContext.getConf.get("spark.executor.memoryOverhead") val execMemOverhead4 = spark.sparkContext.conf.get("spark.executor.memoryOverhead") - assert(execMemOverhead1 == "1G") - assert(execMemOverhead2 == "1G") - assert(execMemOverhead3 == "1G") - assert(execMemOverhead4 == "1G") + // in unified memory mode, comet memory overhead is spark.memory.offHeap.size (2G) * spark.comet.memory.overhead.factor (0.5) = 1G + // so the total executor memory overhead is executor memory overhead (1G) + comet memory overhead (1G) = 2G + // and the overhead is overridden in MiB + assert(execMemOverhead1 == "2048M") + assert(execMemOverhead2 == "2048M") + assert(execMemOverhead3 == "2048M") + assert(execMemOverhead4 == "2048M") } } From de101cdb9de8b70238903815976ba22eaaca1383 Mon Sep 17 00:00:00 2001 From: Lukas Moravec Date: Mon, 3 Mar 2025 16:57:16 +0100 Subject: [PATCH 6/7] formatting wrt original --- .../main/scala/org/apache/spark/Plugins.scala | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/Plugins.scala b/spark/src/main/scala/org/apache/spark/Plugins.scala index da716e65612..644ec81b725 100644 --- a/spark/src/main/scala/org/apache/spark/Plugins.scala +++ b/spark/src/main/scala/org/apache/spark/Plugins.scala @@ -51,18 +51,17 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl CometDriverPlugin.registerCometSessionExtension(sc.conf) if (shouldOverrideMemoryConf(sc.getConf)) { - val execMemOverhead = - if (sc.getConf.contains(EXECUTOR_MEMORY_OVERHEAD.key)) { - sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) - } else { - // By default, executorMemory * spark.executor.memoryOverheadFactor, with minimum of 384MB - val executorMemory = - sc.getConf.getSizeAsMb(EXECUTOR_MEMORY.key, EXECUTOR_MEMORY_DEFAULT) - val memoryOverheadFactor = getMemoryOverheadFactor(sc.getConf) - val memoryOverheadMinMib = getMemoryOverheadMinMib(sc.getConf) - - Math.max((executorMemory * memoryOverheadFactor).toLong, memoryOverheadMinMib) - } + val execMemOverhead = if (sc.getConf.contains(EXECUTOR_MEMORY_OVERHEAD.key)) { + sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) + } else { + // By default, executorMemory * spark.executor.memoryOverheadFactor, with minimum of 384MB + val executorMemory = + sc.getConf.getSizeAsMb(EXECUTOR_MEMORY.key, EXECUTOR_MEMORY_DEFAULT) + val memoryOverheadFactor = getMemoryOverheadFactor(sc.getConf) + val memoryOverheadMinMib = getMemoryOverheadMinMib(sc.getConf) + + Math.max((executorMemory * memoryOverheadFactor).toLong, memoryOverheadMinMib) + } val cometMemOverhead = if (!CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(sc.getConf)) { @@ -72,8 +71,7 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl CometSparkSessionExtensions.getCometShuffleMemorySizeInMiB(sc.getConf) } sc.conf.set(EXECUTOR_MEMORY_OVERHEAD.key, s"${execMemOverhead + cometMemOverhead}M") - val newExecMemOverhead = - sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) + val newExecMemOverhead = sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key) logInfo(s""" Overriding Spark memory configuration for Comet: @@ -110,10 +108,8 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl conf.getBoolean( CometConf.COMET_EXEC_ENABLED.key, CometConf.COMET_EXEC_ENABLED.defaultValue.get) - ) && ( - !CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(conf) || - !CometSparkSessionExtensions.cometShuffleUnifiedMemoryManagerInTestEnabled(conf) - ) + ) && (!CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(conf) || + !CometSparkSessionExtensions.cometShuffleUnifiedMemoryManagerInTestEnabled(conf)) } } From ba78cd9a0245f104130b4373fce7f3a248f7ce29 Mon Sep 17 00:00:00 2001 From: "Moravec4, Lukas" Date: Tue, 4 Mar 2025 16:05:44 +0100 Subject: [PATCH 7/7] clarify test comment --- .../src/test/scala/org/apache/spark/CometPluginsSuite.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala index 88c6752b95d..67a06b28bce 100644 --- a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala @@ -160,7 +160,10 @@ class CometPluginsUnifiedModeOverrideSuite extends CometTestBase { conf } - /** Since using unified memory, executor memory should not be overridden */ + /* + * Since using unified memory, but not shuffle unified memory + * executor memory should be overridden by adding comet shuffle memory size + */ test("executor memory overhead is correctly overridden") { val execMemOverhead1 = spark.conf.get("spark.executor.memoryOverhead") val execMemOverhead2 = spark.sessionState.conf.getConfString("spark.executor.memoryOverhead")