diff --git a/benchmarks/README.md b/benchmarks/README.md index 7e2dfc9f2bc..a35b0e6c993 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -85,8 +85,8 @@ $SPARK_HOME/bin/spark-submit \ --conf spark.comet.exec.enabled=true \ --conf spark.comet.exec.all.enabled=true \ --conf spark.comet.cast.allowIncompatible=true \ - --conf spark.comet.exec.shuffle.enabled=true \ - --conf spark.comet.exec.shuffle.mode=auto \ + --conf spark.comet.shuffle.enabled=true \ + --conf spark.comet.shuffle.mode=auto \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.kubernetes.namespace=default \ --conf spark.kubernetes.driver.pod.name=tpcbench \ diff --git a/benchmarks/pyspark/README.md b/benchmarks/pyspark/README.md index 3fc55123f0d..487bc129370 100644 --- a/benchmarks/pyspark/README.md +++ b/benchmarks/pyspark/README.md @@ -88,7 +88,7 @@ spark-submit --master spark://master:7077 \ spark-submit --master spark://master:7077 \ --jars /path/to/comet.jar \ --conf spark.comet.enabled=true \ - --conf spark.comet.exec.shuffle.enabled=true \ + --conf spark.comet.shuffle.enabled=true \ --conf spark.comet.shuffle.mode=jvm \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ run_benchmark.py --data /tmp/shuffle-benchmark-data --mode jvm --benchmark shuffle-hash @@ -97,8 +97,8 @@ spark-submit --master spark://master:7077 \ spark-submit --master spark://master:7077 \ --jars /path/to/comet.jar \ --conf spark.comet.enabled=true \ - --conf spark.comet.exec.shuffle.enabled=true \ - --conf spark.comet.exec.shuffle.mode=native \ + --conf spark.comet.shuffle.enabled=true \ + --conf spark.comet.shuffle.mode=native \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ run_benchmark.py --data /tmp/shuffle-benchmark-data --mode native --benchmark shuffle-roundrobin ``` diff --git a/benchmarks/pyspark/benchmarks/base.py b/benchmarks/pyspark/benchmarks/base.py index 7e8e8db5a96..504ca44ec11 100644 --- a/benchmarks/pyspark/benchmarks/base.py +++ b/benchmarks/pyspark/benchmarks/base.py @@ -107,7 +107,7 @@ def _print_spark_config(self): conf = self.spark.sparkContext.getConf() print(f"Shuffle manager: {conf.get('spark.shuffle.manager', 'default')}") print(f"Comet enabled: {conf.get('spark.comet.enabled', 'false')}") - print(f"Comet shuffle enabled: {conf.get('spark.comet.exec.shuffle.enabled', 'false')}") + print(f"Comet shuffle enabled: {conf.get('spark.comet.shuffle.enabled', 'false')}") print(f"Comet shuffle mode: {conf.get('spark.comet.shuffle.mode', 'not set')}") print(f"Spark UI: {self.spark.sparkContext.uiWebUrl}") diff --git a/benchmarks/pyspark/run_all_benchmarks.sh b/benchmarks/pyspark/run_all_benchmarks.sh index 7461ef5d5a1..33706a01387 100755 --- a/benchmarks/pyspark/run_all_benchmarks.sh +++ b/benchmarks/pyspark/run_all_benchmarks.sh @@ -52,7 +52,7 @@ $SPARK_HOME/bin/spark-submit \ --conf spark.eventLog.enabled=true \ --conf spark.eventLog.dir="$EVENT_LOG_DIR" \ --conf spark.comet.enabled=false \ - --conf spark.comet.exec.shuffle.enabled=false \ + --conf spark.comet.shuffle.enabled=false \ "$SCRIPT_DIR/run_benchmark.py" \ --data "$DATA_PATH" \ --mode spark @@ -76,7 +76,7 @@ $SPARK_HOME/bin/spark-submit \ --conf spark.comet.explain.fallback.log.enabled=true \ --conf spark.comet.explain.fallback.enabled=true \ --conf spark.comet.shuffle.mode=jvm \ - --conf spark.comet.exec.shuffle.mode=jvm \ + --conf spark.comet.shuffle.mode=jvm \ --conf spark.comet.exec.replaceSortMergeJoin=true \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \ @@ -103,7 +103,7 @@ $SPARK_HOME/bin/spark-submit \ --conf spark.comet.parquet.write.enabled=true \ --conf spark.comet.explain.fallback.log.enabled=true \ --conf spark.comet.explain.fallback.enabled=true \ - --conf spark.comet.exec.shuffle.mode=native \ + --conf spark.comet.shuffle.mode=native \ --conf spark.comet.exec.replaceSortMergeJoin=true \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \ diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index 133daf681a0..5db4b36c05a 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -2536,14 +2536,13 @@ index d083cac48ff..3c11bcde807 100644 import testImplicits._ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala -index 266bb343526..f8ad838e2b2 100644 +index 266bb343526..50408a27a0a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala -@@ -19,15 +19,18 @@ package org.apache.spark.sql.sources +@@ -19,15 +19,17 @@ package org.apache.spark.sql.sources import scala.util.Random -+import org.apache.comet.CometConf + import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.BucketSpec @@ -2560,7 +2559,7 @@ index 266bb343526..f8ad838e2b2 100644 import org.apache.spark.sql.execution.joins.SortMergeJoinExec import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf -@@ -101,12 +104,22 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -101,12 +103,22 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti } } @@ -2585,7 +2584,7 @@ index 266bb343526..f8ad838e2b2 100644 // To verify if the bucket pruning works, this function checks two conditions: // 1) Check if the pruned buckets (before filtering) are empty. // 2) Verify the final result is the same as the expected one -@@ -155,7 +168,8 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -155,7 +167,8 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti val planWithoutBucketedScan = bucketedDataFrame.filter(filterCondition) .queryExecution.executedPlan val fileScan = getFileScan(planWithoutBucketedScan) @@ -2595,7 +2594,7 @@ index 266bb343526..f8ad838e2b2 100644 val bucketColumnType = bucketedDataFrame.schema.apply(bucketColumnIndex).dataType val rowsWithInvalidBuckets = fileScan.execute().filter(row => { -@@ -451,28 +465,54 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -451,28 +464,54 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti val joinOperator = if (joined.sqlContext.conf.adaptiveExecutionEnabled) { val executedPlan = joined.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan @@ -2658,7 +2657,7 @@ index 266bb343526..f8ad838e2b2 100644 s"expected sort in the right child to be $sortRight but found\n${joinOperator.right}") // check the output partitioning -@@ -835,11 +875,11 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -835,11 +874,11 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti df1.write.format("parquet").bucketBy(8, "i").saveAsTable("bucketed_table") val scanDF = spark.table("bucketed_table").select("j") @@ -2672,40 +2671,40 @@ index 266bb343526..f8ad838e2b2 100644 checkAnswer(aggDF, df1.groupBy("j").agg(max("k"))) } } -@@ -894,7 +934,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -894,7 +933,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti } test("SPARK-29655 Read bucketed tables obeys spark.sql.shuffle.partitions") { + // Range partitioning uses random samples, so per-partition comparisons do not always yield + // the same results. Disable Comet native range partitioning. withSQLConf( -+ CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "false", ++ "spark.comet.shuffle.native.partitioning.range.enabled" -> "false", SQLConf.SHUFFLE_PARTITIONS.key -> "5", SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "7") { val bucketSpec = Some(BucketSpec(6, Seq("i", "j"), Nil)) -@@ -913,7 +956,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -913,7 +955,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti } test("SPARK-32767 Bucket join should work if SHUFFLE_PARTITIONS larger than bucket number") { + // Range partitioning uses random samples, so per-partition comparisons do not always yield + // the same results. Disable Comet native range partitioning. withSQLConf( -+ CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "false", ++ "spark.comet.shuffle.native.partitioning.range.enabled" -> "false", SQLConf.SHUFFLE_PARTITIONS.key -> "9", SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10") { -@@ -943,7 +989,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -943,7 +988,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti } test("bucket coalescing eliminates shuffle") { + // Range partitioning uses random samples, so per-partition comparisons do not always yield + // the same results. Disable Comet native range partitioning. withSQLConf( -+ CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "false", ++ "spark.comet.shuffle.native.partitioning.range.enabled" -> "false", SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true", SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { // The side with bucketedTableTestSpec1 will be coalesced to have 4 output partitions. -@@ -1026,15 +1075,26 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -1026,15 +1074,26 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti expectedNumShuffles: Int, expectedCoalescedNumBuckets: Option[Int]): Unit = { val plan = sql(query).queryExecution.executedPlan @@ -3000,7 +2999,7 @@ index dd55fcfe42c..d9a3f2df535 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index ed2e309fa07..b94db4ff1ab 100644 +index ed2e309fa07..040013dc8ab 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala @@ -74,6 +74,20 @@ trait SharedSparkSessionBase @@ -3018,7 +3017,7 @@ index ed2e309fa07..b94db4ff1ab 100644 + .set("spark.comet.exec.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") ++ .set("spark.comet.shuffle.enabled", "true") + .set("spark.comet.memoryOverhead", "4g") + } conf.set( @@ -3085,18 +3084,10 @@ index de3b1ffccf0..2a76d127093 100644 override def beforeEach(): Unit = { super.beforeEach() diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala -index a902cb3a69e..800a3acbe99 100644 +index a902cb3a69e..e652edd9f81 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala -@@ -24,6 +24,7 @@ import java.sql.{Date, Timestamp} - import java.util.{Locale, Set} - - import com.google.common.io.Files -+import org.apache.comet.CometConf - import org.apache.hadoop.fs.{FileSystem, Path} - - import org.apache.spark.{SparkException, TestUtils} -@@ -838,8 +839,13 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi +@@ -838,8 +838,13 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi } test("SPARK-2554 SumDistinct partial aggregation") { @@ -3104,7 +3095,7 @@ index a902cb3a69e..800a3acbe99 100644 - sql("SELECT distinct key FROM src order by key").collect().toSeq) + // Range partitioning uses random samples, so per-partition comparisons do not always yield + // the same results. Disable Comet native range partitioning. -+ withSQLConf(CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "false") ++ withSQLConf("spark.comet.shuffle.native.partitioning.range.enabled" -> "false") + { + checkAnswer(sql("SELECT sum( distinct key) FROM src group by key order by key"), + sql("SELECT distinct key FROM src order by key").collect().toSeq) @@ -3113,7 +3104,7 @@ index a902cb3a69e..800a3acbe99 100644 test("SPARK-4963 DataFrame sample on mutable row return wrong result") { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index 07361cfdce9..4fdbcd18656 100644 +index 07361cfdce9..af6dcfc2302 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -55,25 +55,41 @@ object TestHive @@ -3167,7 +3158,7 @@ index 07361cfdce9..4fdbcd18656 100644 + .set("spark.comet.exec.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") ++ .set("spark.comet.shuffle.enabled", "true") + } + conf diff --git a/dev/diffs/3.5.8.diff b/dev/diffs/3.5.8.diff index 428e964bad8..3d00bc13b6b 100644 --- a/dev/diffs/3.5.8.diff +++ b/dev/diffs/3.5.8.diff @@ -2500,14 +2500,13 @@ index d083cac48ff..3c11bcde807 100644 import testImplicits._ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala -index 746f289c393..e5dc13b87d5 100644 +index 746f289c393..f2e0d1c93e5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala -@@ -19,16 +19,19 @@ package org.apache.spark.sql.sources +@@ -19,16 +19,18 @@ package org.apache.spark.sql.sources import scala.util.Random -+import org.apache.comet.CometConf + import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.BucketSpec @@ -2525,7 +2524,7 @@ index 746f289c393..e5dc13b87d5 100644 import org.apache.spark.sql.execution.joins.SortMergeJoinExec import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf -@@ -102,12 +105,22 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -102,12 +104,22 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti } } @@ -2550,7 +2549,7 @@ index 746f289c393..e5dc13b87d5 100644 // To verify if the bucket pruning works, this function checks two conditions: // 1) Check if the pruned buckets (before filtering) are empty. // 2) Verify the final result is the same as the expected one -@@ -156,7 +169,8 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -156,7 +168,8 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti val planWithoutBucketedScan = bucketedDataFrame.filter(filterCondition) .queryExecution.executedPlan val fileScan = getFileScan(planWithoutBucketedScan) @@ -2560,7 +2559,7 @@ index 746f289c393..e5dc13b87d5 100644 val bucketColumnType = bucketedDataFrame.schema.apply(bucketColumnIndex).dataType val rowsWithInvalidBuckets = fileScan.execute().filter(row => { -@@ -452,28 +466,54 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -452,28 +465,54 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti val joinOperator = if (joined.sqlContext.conf.adaptiveExecutionEnabled) { val executedPlan = joined.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan @@ -2623,7 +2622,7 @@ index 746f289c393..e5dc13b87d5 100644 s"expected sort in the right child to be $sortRight but found\n${joinOperator.right}") // check the output partitioning -@@ -836,11 +876,11 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -836,11 +875,11 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti df1.write.format("parquet").bucketBy(8, "i").saveAsTable("bucketed_table") val scanDF = spark.table("bucketed_table").select("j") @@ -2637,40 +2636,40 @@ index 746f289c393..e5dc13b87d5 100644 checkAnswer(aggDF, df1.groupBy("j").agg(max("k"))) } } -@@ -895,7 +935,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -895,7 +934,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti } test("SPARK-29655 Read bucketed tables obeys spark.sql.shuffle.partitions") { + // Range partitioning uses random samples, so per-partition comparisons do not always yield + // the same results. Disable Comet native range partitioning. withSQLConf( -+ CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "false", ++ "spark.comet.shuffle.native.partitioning.range.enabled" -> "false", SQLConf.SHUFFLE_PARTITIONS.key -> "5", SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "7") { val bucketSpec = Some(BucketSpec(6, Seq("i", "j"), Nil)) -@@ -914,7 +957,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -914,7 +956,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti } test("SPARK-32767 Bucket join should work if SHUFFLE_PARTITIONS larger than bucket number") { + // Range partitioning uses random samples, so per-partition comparisons do not always yield + // the same results. Disable Comet native range partitioning. withSQLConf( -+ CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "false", ++ "spark.comet.shuffle.native.partitioning.range.enabled" -> "false", SQLConf.SHUFFLE_PARTITIONS.key -> "9", SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10") { -@@ -944,7 +990,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -944,7 +989,10 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti } test("bucket coalescing eliminates shuffle") { + // Range partitioning uses random samples, so per-partition comparisons do not always yield + // the same results. Disable Comet native range partitioning. withSQLConf( -+ CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "false", ++ "spark.comet.shuffle.native.partitioning.range.enabled" -> "false", SQLConf.COALESCE_BUCKETS_IN_JOIN_ENABLED.key -> "true", SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { // The side with bucketedTableTestSpec1 will be coalesced to have 4 output partitions. -@@ -1029,15 +1078,24 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti +@@ -1029,15 +1077,24 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti Seq(true, false).foreach { aqeEnabled => withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> aqeEnabled.toString) { val plan = sql(query).queryExecution.executedPlan @@ -2949,7 +2948,7 @@ index e937173a590..263934fbe7b 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index ed2e309fa07..1c30220c207 100644 +index ed2e309fa07..2dc45ad821f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala @@ -74,6 +74,20 @@ trait SharedSparkSessionBase @@ -2967,7 +2966,7 @@ index ed2e309fa07..1c30220c207 100644 + .set("spark.comet.exec.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") ++ .set("spark.comet.shuffle.enabled", "true") + .set("spark.comet.memoryOverhead", "2g") + } conf.set( @@ -3034,18 +3033,10 @@ index de3b1ffccf0..2a76d127093 100644 override def beforeEach(): Unit = { super.beforeEach() diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala -index 6160c3e5f6c..0956d7d9edc 100644 +index 6160c3e5f6c..bfc0c618a9b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala -@@ -24,6 +24,7 @@ import java.sql.{Date, Timestamp} - import java.util.{Locale, Set} - - import com.google.common.io.Files -+import org.apache.comet.CometConf - import org.apache.hadoop.fs.{FileSystem, Path} - - import org.apache.spark.{SparkException, TestUtils} -@@ -838,8 +839,13 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi +@@ -838,8 +838,13 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi } test("SPARK-2554 SumDistinct partial aggregation") { @@ -3053,7 +3044,7 @@ index 6160c3e5f6c..0956d7d9edc 100644 - sql("SELECT distinct key FROM src order by key").collect().toSeq) + // Range partitioning uses random samples, so per-partition comparisons do not always yield + // the same results. Disable Comet native range partitioning. -+ withSQLConf(CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "false") ++ withSQLConf("spark.comet.shuffle.native.partitioning.range.enabled" -> "false") + { + checkAnswer(sql("SELECT sum( distinct key) FROM src group by key order by key"), + sql("SELECT distinct key FROM src order by key").collect().toSeq) @@ -3062,7 +3053,7 @@ index 6160c3e5f6c..0956d7d9edc 100644 test("SPARK-4963 DataFrame sample on mutable row return wrong result") { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index 1d646f40b3e..df108c17c42 100644 +index 1d646f40b3e..c8192f52f98 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -53,25 +53,41 @@ object TestHive @@ -3116,7 +3107,7 @@ index 1d646f40b3e..df108c17c42 100644 + .set("spark.comet.exec.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") ++ .set("spark.comet.shuffle.enabled", "true") + } + conf diff --git a/dev/diffs/4.0.2.diff b/dev/diffs/4.0.2.diff index 8ee55fea1d8..a2f6eac74ab 100644 --- a/dev/diffs/4.0.2.diff +++ b/dev/diffs/4.0.2.diff @@ -3649,7 +3649,7 @@ index f0f3f94b811..b7d18771314 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index 245219c1756..fe5384382f3 100644 +index 245219c1756..24318f6bd43 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala @@ -75,6 +75,21 @@ trait SharedSparkSessionBase @@ -3667,7 +3667,7 @@ index 245219c1756..fe5384382f3 100644 + .set("spark.comet.exec.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") ++ .set("spark.comet.shuffle.enabled", "true") + .set("spark.comet.memoryOverhead", "2g") + + } @@ -3792,7 +3792,7 @@ index b67370f6eb9..746b3974b29 100644 override def beforeEach(): Unit = { super.beforeEach() diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index a394d0b7393..3e1f0404a37 100644 +index a394d0b7393..7056c74759c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -53,24 +53,34 @@ object TestHive @@ -3839,7 +3839,7 @@ index a394d0b7393..3e1f0404a37 100644 + .set("spark.comet.exec.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") ++ .set("spark.comet.shuffle.enabled", "true") + } + + conf diff --git a/dev/diffs/4.1.2.diff b/dev/diffs/4.1.2.diff index 599e2cd077e..30f6c2f23fb 100644 --- a/dev/diffs/4.1.2.diff +++ b/dev/diffs/4.1.2.diff @@ -3919,7 +3919,7 @@ index f0f3f94b811..b7d18771314 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index 720b13b812e..813779bd3b2 100644 +index 720b13b812e..93221ef4cf5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala @@ -98,6 +98,21 @@ trait SharedSparkSessionBase @@ -3937,7 +3937,7 @@ index 720b13b812e..813779bd3b2 100644 + .set("spark.comet.exec.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") ++ .set("spark.comet.shuffle.enabled", "true") + .set("spark.comet.memoryOverhead", "2g") + + } @@ -4062,7 +4062,7 @@ index b67370f6eb9..746b3974b29 100644 override def beforeEach(): Unit = { super.beforeEach() diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index a394d0b7393..3e1f0404a37 100644 +index a394d0b7393..7056c74759c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -53,24 +53,34 @@ object TestHive @@ -4109,7 +4109,7 @@ index a394d0b7393..3e1f0404a37 100644 + .set("spark.comet.exec.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") ++ .set("spark.comet.shuffle.enabled", "true") + } + + conf diff --git a/docs/source/contributor-guide/benchmarking_spark_sql_perf.md b/docs/source/contributor-guide/benchmarking_spark_sql_perf.md index d8933194437..4503b414ae1 100644 --- a/docs/source/contributor-guide/benchmarking_spark_sql_perf.md +++ b/docs/source/contributor-guide/benchmarking_spark_sql_perf.md @@ -242,7 +242,7 @@ $SPARK_HOME/bin/spark-shell \ --conf spark.comet.exec.enabled=true \ --conf spark.comet.exec.all.enabled=true \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ - --conf spark.comet.exec.shuffle.enabled=true \ + --conf spark.comet.shuffle.enabled=true \ --conf spark.comet.columnar.shuffle.enabled=true ``` diff --git a/docs/source/contributor-guide/jvm_shuffle.md b/docs/source/contributor-guide/jvm_shuffle.md index e67f79a5242..420278ec492 100644 --- a/docs/source/contributor-guide/jvm_shuffle.md +++ b/docs/source/contributor-guide/jvm_shuffle.md @@ -40,7 +40,7 @@ The JVM shuffle is selected via `CometShuffleDependency.shuffleType`. JVM shuffle (`CometColumnarExchange`) is used instead of native shuffle (`CometExchange`) in the following cases: -1. **Shuffle mode is explicitly set to "jvm"**: When `spark.comet.exec.shuffle.mode` is set to `jvm`. +1. **Shuffle mode is explicitly set to "jvm"**: When `spark.comet.shuffle.mode` is set to `jvm`. 2. **Child plan is not a Comet native operator**: When the child plan is a Spark row-based operator (not a `CometPlan`), JVM shuffle is the only option since native shuffle requires columnar input @@ -187,8 +187,8 @@ Selection logic in `CometShuffleManager.shouldBypassMergeSort()`: ## Configuration -| Config | Description | -| ---------------------------------------------- | ----------------------------------- | -| `spark.comet.columnar.shuffle.batch.size` | Rows per Arrow batch | -| `spark.comet.columnar.shuffle.spill.threshold` | Row count threshold for spill | -| `spark.comet.exec.shuffle.compression.codec` | Compression codec (zstd, lz4, etc.) | +| Config | Description | +| ---------------------------------------- | ----------------------------------- | +| `spark.comet.shuffle.jvm.batchSize` | Rows per Arrow batch | +| `spark.comet.shuffle.jvm.spillThreshold` | Row count threshold for spill | +| `spark.comet.shuffle.compression.codec` | Compression codec (zstd, lz4, etc.) | diff --git a/docs/source/contributor-guide/native_shuffle.md b/docs/source/contributor-guide/native_shuffle.md index 4d6421de5e8..9cd9ebc6c5f 100644 --- a/docs/source/contributor-guide/native_shuffle.md +++ b/docs/source/contributor-guide/native_shuffle.md @@ -43,12 +43,13 @@ Comet Native (columnar) → ColumnarToRowExec → rows → JVM Shuffle → Arrow Native shuffle (`CometExchange`) is selected when all of the following conditions are met: -1. **Shuffle mode allows native**: `spark.comet.exec.shuffle.mode` is `native` or `auto`. +1. **Shuffle mode allows native**: `spark.comet.shuffle.mode` is `native` or `auto`. 2. **Child plan is a Comet native operator**: The child must be a `CometPlan` that produces columnar output. Row-based Spark operators require JVM shuffle. 3. **Supported partitioning type**: Native shuffle supports: + - `HashPartitioning` - `RangePartitioning` - `SinglePartition` @@ -128,6 +129,7 @@ Native shuffle (`CometExchange`) is selected when all of the following condition 1. **Plan construction**: `CometNativeShuffleWriter` builds a protobuf operator tree with a `ShuffleWriter` operator at the root and `childNativeOp` as its child. `childNativeOp` takes one of two shapes: + - The child plan's `nativeOp` directly, when `CometShuffleExchangeExec`'s child is a `CometNativeExec` subtree. The upstream operators run inside the same `CometExecIterator` as the writer, with no JVM-to-native batch boundary between them. @@ -140,6 +142,7 @@ Native shuffle (`CometExchange`) is selected when all of the following condition 2. **Native execution**: A single `CometExecIterator` per partition runs the unified plan. 3. **Partitioning**: `ShuffleWriterExec` receives batches and routes to the appropriate partitioner: + - `MultiPartitionShuffleRepartitioner`: For hash/range/round-robin partitioning - `SinglePartitionShufflePartitioner`: For single partition (simpler path) @@ -147,11 +150,13 @@ Native shuffle (`CometExchange`) is selected when all of the following condition exceeds the threshold, partitions spill to temporary files. 5. **Encoding**: `ShuffleBlockWriter` encodes each partition's data as compressed Arrow IPC: + - Writes compression type header - Writes field count header - Writes compressed IPC stream 6. **Output files**: Two files are produced: + - **Data file**: Concatenated partition data - **Index file**: Array of 8-byte little-endian offsets marking partition boundaries @@ -163,6 +168,7 @@ Native shuffle (`CometExchange`) is selected when all of the following condition 1. `CometBlockStoreShuffleReader` fetches shuffle blocks via `ShuffleBlockFetcherIterator`. 2. For each block, `NativeBatchDecoderIterator`: + - Reads the 8-byte compressed length header - Reads the 8-byte field count header - Reads the compressed IPC data @@ -215,7 +221,7 @@ Native shuffle uses DataFusion's memory management with spilling support: - **Memory pool**: Tracks memory usage across the shuffle operation. - **Spill triggers**: Partitions spill to disk when the memory pool denies an allocation, or - when the buffered bytes reach `spark.comet.shuffle.maxBufferBytes`. That config defaults to + when the buffered bytes reach `spark.comet.shuffle.native.maxBufferBytes`. That config defaults to 0, which disables the fixed limit and leaves memory pressure as the only trigger. - **Per-partition spilling**: Each partition has its own spill file. Multiple spills for a partition are concatenated when writing the final output. @@ -230,7 +236,7 @@ The `MultiPartitionShuffleRepartitioner` manages: ## Compression Native shuffle supports multiple compression codecs configured via -`spark.comet.exec.shuffle.compression.codec`: +`spark.comet.shuffle.compression.codec`: | Codec | Description | | -------- | ------------------------------------------------------ | @@ -244,14 +250,14 @@ independently compressed, allowing parallel decompression during reads. ## Configuration -| Config | Default | Description | -| ------------------------------------------------- | ------- | ---------------------------------------- | -| `spark.comet.exec.shuffle.enabled` | `true` | Enable Comet shuffle | -| `spark.comet.exec.shuffle.mode` | `auto` | Shuffle mode: `native`, `jvm`, or `auto` | -| `spark.comet.exec.shuffle.compression.codec` | `zstd` | Compression codec | -| `spark.comet.exec.shuffle.compression.zstd.level` | `1` | Zstd compression level | -| `spark.comet.shuffle.write.buffer.size` | `1MB` | Write buffer size | -| `spark.comet.columnar.shuffle.batch.size` | `8192` | Target rows per batch | +| Config | Default | Description | +| -------------------------------------------- | ------- | ---------------------------------------- | +| `spark.comet.shuffle.enabled` | `true` | Enable Comet shuffle | +| `spark.comet.shuffle.mode` | `auto` | Shuffle mode: `native`, `jvm`, or `auto` | +| `spark.comet.shuffle.compression.codec` | `zstd` | Compression codec | +| `spark.comet.shuffle.compression.zstd.level` | `1` | Zstd compression level | +| `spark.comet.shuffle.write.buffer.size` | `1MB` | Write buffer size | +| `spark.comet.shuffle.jvm.batchSize` | `8192` | Target rows per batch | ## Comparison with JVM Shuffle diff --git a/docs/source/user-guide/latest/compatibility/operators.md b/docs/source/user-guide/latest/compatibility/operators.md index eec21d283b5..47373d3901c 100644 --- a/docs/source/user-guide/latest/compatibility/operators.md +++ b/docs/source/user-guide/latest/compatibility/operators.md @@ -62,7 +62,7 @@ incorrect result. When any single window expression in a `WindowExec` falls back Comet's native shuffle implementation of round-robin partitioning (`df.repartition(n)`) is not compatible with Spark's implementation and is disabled by default. It can be enabled by setting -`spark.comet.native.shuffle.partitioning.roundrobin.enabled=true`. +`spark.comet.shuffle.native.partitioning.roundrobin.enabled=true`. **Why the incompatibility exists:** diff --git a/docs/source/user-guide/latest/kubernetes.md b/docs/source/user-guide/latest/kubernetes.md index 498eefb614b..5dfb723e655 100644 --- a/docs/source/user-guide/latest/kubernetes.md +++ b/docs/source/user-guide/latest/kubernetes.md @@ -79,8 +79,8 @@ spec: "spark.plugins": "org.apache.spark.CometPlugin" "spark.comet.enabled": "true" "spark.comet.exec.enabled": "true" - "spark.comet.exec.shuffle.enabled": "true" - "spark.comet.exec.shuffle.mode": "auto" + "spark.comet.shuffle.enabled": "true" + "spark.comet.shuffle.mode": "auto" "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager" sparkVersion: 4.1.2 driver: diff --git a/docs/source/user-guide/latest/pyarrow-udfs.md b/docs/source/user-guide/latest/pyarrow-udfs.md index ddfa9cf8bd3..8a5da107edf 100644 --- a/docs/source/user-guide/latest/pyarrow-udfs.md +++ b/docs/source/user-guide/latest/pyarrow-udfs.md @@ -185,7 +185,7 @@ on the unoptimized path. Comet operator and the Python UDF, you need Comet's native shuffle for the optimization to apply. Set `spark.shuffle.manager` to `org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager` and enable - `spark.comet.exec.shuffle.enabled=true` at session startup. With a vanilla Spark `Exchange` + `spark.comet.shuffle.enabled=true` at session startup. With a vanilla Spark `Exchange` in the plan the data leaves the shuffle as rows and the optimization cannot fire. - Spark 4.0 or newer is required. On Spark 3.4 and 3.5 the optimization is a no-op even when enabled; vanilla `PythonMapInArrowExec` / `MapInPandasExec` handle the operation. The Spark 3.5 diff --git a/docs/source/user-guide/latest/tuning.md b/docs/source/user-guide/latest/tuning.md index 6f0e9a684b9..fc1c9e47b82 100644 --- a/docs/source/user-guide/latest/tuning.md +++ b/docs/source/user-guide/latest/tuning.md @@ -129,12 +129,12 @@ To enable Comet shuffle, set the following configuration in your Spark configura ``` spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager -spark.comet.exec.shuffle.enabled=true +spark.comet.shuffle.enabled=true ``` `spark.shuffle.manager` is a Spark static configuration which cannot be changed at runtime. It must be set before the Spark context is created. You can enable or disable Comet shuffle -at runtime by setting `spark.comet.exec.shuffle.enabled` to `true` or `false`. +at runtime by setting `spark.comet.shuffle.enabled` to `true` or `false`. Once it is disabled, Comet will fall back to the default Spark shuffle manager. ### Shuffle Implementations @@ -158,7 +158,7 @@ By default, Comet will convert a Spark `ShuffleExchangeExec` to columnar shuffle non-Comet (Spark) plan. The benefit is that the next query stage can start as native Comet execution, since the shuffle output is already in Arrow format. The cost is a row to columnar conversion at the shuffle boundary on the write side. To restrict columnar shuffle to cases where the child is already a Comet plan, set -`spark.comet.exec.shuffle.convertFromSparkPlan.enabled=false`. Shuffles whose child is a Spark plan will then be left +`spark.comet.shuffle.convertFromSparkPlan.enabled=false`. Shuffles whose child is a Spark plan will then be left as native Spark shuffles, which avoids the row to columnar conversion but means the downstream stage will also start on Spark. @@ -177,7 +177,7 @@ Each revert is logged at `INFO` level on the driver as `Reverting Comet columnar and `, which lets you correlate any unexpected behavior with this optimization. This optimization is enabled by default and can be disabled by setting -`spark.comet.exec.shuffle.revertRedundantColumnar.enabled=false`, in which case Comet will keep the columnar shuffle +`spark.comet.shuffle.revertRedundantColumnar.enabled=false`, in which case Comet will keep the columnar shuffle even when both its parent and child are non-Comet operators. ### Shuffle Compression diff --git a/spark/src/main/java/org/apache/comet/CometShuffleBlockIterator.java b/spark/src/main/java/org/apache/comet/CometShuffleBlockIterator.java index 9f72b20f517..30314278aa5 100644 --- a/spark/src/main/java/org/apache/comet/CometShuffleBlockIterator.java +++ b/spark/src/main/java/org/apache/comet/CometShuffleBlockIterator.java @@ -95,7 +95,7 @@ public int hasNext() throws IOException { + bytesToRead + " exceeds maximum of " + Integer.MAX_VALUE - + ". Try reducing spark.comet.columnar.shuffle.batch.size."); + + ". Try reducing spark.comet.shuffle.jvm.batchSize."); } currentBlockLength = (int) bytesToRead; diff --git a/spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java b/spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java index e8bb611d892..13ea96dfcb2 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java +++ b/spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java @@ -128,21 +128,20 @@ public CometShuffleExternalSorter( this.numPartitions = numPartitions; this.schema = schema; this.numElementsForSpillThreshold = - (int) CometConf$.MODULE$.COMET_COLUMNAR_SHUFFLE_SPILL_THRESHOLD().get(); + (int) CometConf$.MODULE$.COMET_SHUFFLE_JVM_SPILL_THRESHOLD().get(); this.writeMetrics = writeMetrics; this.peakMemoryUsedBytes = getMemoryUsage(); this.partitionChecksums = createPartitionChecksums(numPartitions, conf); this.checksumAlgorithm = getChecksumAlgorithm(conf); - this.compressionCodec = CometConf$.MODULE$.COMET_EXEC_SHUFFLE_COMPRESSION_CODEC().get(); - this.compressionLevel = - (int) CometConf$.MODULE$.COMET_EXEC_SHUFFLE_COMPRESSION_ZSTD_LEVEL().get(); + this.compressionCodec = CometConf$.MODULE$.COMET_SHUFFLE_COMPRESSION_CODEC().get(); + this.compressionLevel = (int) CometConf$.MODULE$.COMET_SHUFFLE_COMPRESSION_ZSTD_LEVEL().get(); this.initialSize = initialSize; this.tracingEnabled = (boolean) CometConf$.MODULE$.COMET_TRACING_ENABLED().get(); this.preferDictionaryRatio = - (double) CometConf$.MODULE$.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO().get(); + (double) CometConf$.MODULE$.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO().get(); this.activeSpillSorter = createSpillSorter(); } diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java index c47953b0519..504de9ac4a3 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java @@ -141,16 +141,15 @@ public long getEncodeNanos() { this.file = file; this.tracingEnabled = tracingEnabled; - this.columnarBatchSize = (int) CometConf$.MODULE$.COMET_COLUMNAR_SHUFFLE_BATCH_SIZE().get(); - this.compressionCodec = CometConf$.MODULE$.COMET_EXEC_SHUFFLE_COMPRESSION_CODEC().get(); - this.compressionLevel = - (int) CometConf$.MODULE$.COMET_EXEC_SHUFFLE_COMPRESSION_ZSTD_LEVEL().get(); + this.columnarBatchSize = (int) CometConf$.MODULE$.COMET_SHUFFLE_JVM_BATCH_SIZE().get(); + this.compressionCodec = CometConf$.MODULE$.COMET_SHUFFLE_COMPRESSION_CODEC().get(); + this.compressionLevel = (int) CometConf$.MODULE$.COMET_SHUFFLE_COMPRESSION_ZSTD_LEVEL().get(); this.numElementsForSpillThreshold = - (int) CometConf$.MODULE$.COMET_COLUMNAR_SHUFFLE_SPILL_THRESHOLD().get(); + (int) CometConf$.MODULE$.COMET_SHUFFLE_JVM_SPILL_THRESHOLD().get(); this.preferDictionaryRatio = - (double) CometConf$.MODULE$.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO().get(); + (double) CometConf$.MODULE$.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO().get(); this.activeWriter = new ArrowIPCWriter(); diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java index 9cb8fac0bca..47e85383bb4 100644 --- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java +++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java @@ -196,7 +196,7 @@ protected long doSpilling( long currentChecksum = checksumEnabled ? checksum : 0L; long start = System.nanoTime(); - int batchSize = (int) CometConf.COMET_COLUMNAR_SHUFFLE_BATCH_SIZE().get(); + int batchSize = (int) CometConf.COMET_SHUFFLE_JVM_BATCH_SIZE().get(); long[] results = nativeLib.writeSortedFileNative( addresses, diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 821897a9dc8..78b0d8fb36f 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -279,8 +279,9 @@ object CometConf extends ShimCometConf { .bytesConf(ByteUnit.MiB) .createWithDefault(1024) - val COMET_EXEC_SHUFFLE_ENABLED: ConfigEntry[Boolean] = - conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.enabled") + val COMET_SHUFFLE_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.shuffle.enabled") + .withAlternative(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.enabled") .category(CATEGORY_SHUFFLE) .doc( "Whether to enable Comet native shuffle. " + @@ -292,17 +293,19 @@ object CometConf extends ShimCometConf { .createWithDefault(true) val COMET_SHUFFLE_DIRECT_READ_ENABLED: ConfigEntry[Boolean] = - conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.directRead.enabled") + conf("spark.comet.shuffle.directRead.enabled") + .withAlternative(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.directRead.enabled") .category(CATEGORY_SHUFFLE) .doc( "When enabled, native operators that consume shuffle output will read " + "compressed shuffle blocks directly in native code, bypassing Arrow FFI. " + "Applies to both native shuffle and JVM columnar shuffle. " + - "Requires spark.comet.exec.shuffle.enabled to be true.") + "Requires spark.comet.shuffle.enabled to be true.") .booleanConf .createWithDefault(true) - val COMET_SHUFFLE_MODE: ConfigEntry[String] = conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.mode") + val COMET_SHUFFLE_MODE: ConfigEntry[String] = conf("spark.comet.shuffle.mode") + .withAlternative(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.mode") .category(CATEGORY_SHUFFLE) .doc( "This is test config to allow tests to force a particular shuffle implementation to be " + @@ -348,22 +351,25 @@ object CometConf extends ShimCometConf { .booleanConf .createWithDefault(true) - val COMET_EXEC_SHUFFLE_WITH_HASH_PARTITIONING_ENABLED: ConfigEntry[Boolean] = - conf("spark.comet.native.shuffle.partitioning.hash.enabled") + val COMET_SHUFFLE_NATIVE_HASH_PARTITIONING_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.shuffle.native.partitioning.hash.enabled") + .withAlternative("spark.comet.native.shuffle.partitioning.hash.enabled") .category(CATEGORY_SHUFFLE) .doc("Whether to enable hash partitioning for Comet native shuffle.") .booleanConf .createWithDefault(true) - val COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED: ConfigEntry[Boolean] = - conf("spark.comet.native.shuffle.partitioning.range.enabled") + val COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.shuffle.native.partitioning.range.enabled") + .withAlternative("spark.comet.native.shuffle.partitioning.range.enabled") .category(CATEGORY_SHUFFLE) .doc("Whether to enable range partitioning for Comet native shuffle.") .booleanConf .createWithDefault(true) - val COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_ENABLED: ConfigEntry[Boolean] = - conf("spark.comet.native.shuffle.partitioning.roundrobin.enabled") + val COMET_SHUFFLE_NATIVE_ROUND_ROBIN_PARTITIONING_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.shuffle.native.partitioning.roundrobin.enabled") + .withAlternative("spark.comet.native.shuffle.partitioning.roundrobin.enabled") .category(CATEGORY_SHUFFLE) .doc( "Whether to enable round robin partitioning for Comet native shuffle. " + @@ -377,8 +383,9 @@ object CometConf extends ShimCometConf { .booleanConf .createWithDefault(false) - val COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_MAX_HASH_COLUMNS: ConfigEntry[Int] = - conf("spark.comet.native.shuffle.partitioning.roundrobin.maxHashColumns") + val COMET_SHUFFLE_NATIVE_ROUND_ROBIN_PARTITIONING_MAX_HASH_COLUMNS: ConfigEntry[Int] = + conf("spark.comet.shuffle.native.partitioning.roundrobin.maxHashColumns") + .withAlternative("spark.comet.native.shuffle.partitioning.roundrobin.maxHashColumns") .category(CATEGORY_SHUFFLE) .doc( "The maximum number of columns to hash for round robin partitioning. " + @@ -392,8 +399,9 @@ object CometConf extends ShimCometConf { "The maximum number of columns to hash for round robin partitioning must be non-negative.") .createWithDefault(0) - val COMET_EXEC_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED: ConfigEntry[Boolean] = - conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.convertFromSparkPlan.enabled") + val COMET_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.shuffle.convertFromSparkPlan.enabled") + .withAlternative(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.convertFromSparkPlan.enabled") .category(CATEGORY_SHUFFLE) .doc( "When enabled, Comet will convert a Spark `ShuffleExchangeExec` to a Comet columnar " + @@ -403,8 +411,9 @@ object CometConf extends ShimCometConf { .booleanConf .createWithDefault(true) - val COMET_EXEC_SHUFFLE_REVERT_REDUNDANT_COLUMNAR_ENABLED: ConfigEntry[Boolean] = - conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.revertRedundantColumnar.enabled") + val COMET_SHUFFLE_REVERT_REDUNDANT_COLUMNAR_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.shuffle.revertRedundantColumnar.enabled") + .withAlternative(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.revertRedundantColumnar.enabled") .category(CATEGORY_SHUFFLE) .doc( "When enabled, Comet reverts a `CometShuffleExchangeExec` with `CometColumnarShuffle` " + @@ -443,8 +452,9 @@ object CometConf extends ShimCometConf { .checkValue(_ >= 0, "Must be >= 0.") .createWithDefault(2) - val COMET_EXEC_SHUFFLE_COMPRESSION_CODEC: ConfigEntry[String] = - conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.compression.codec") + val COMET_SHUFFLE_COMPRESSION_CODEC: ConfigEntry[String] = + conf("spark.comet.shuffle.compression.codec") + .withAlternative(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.compression.codec") .category(CATEGORY_SHUFFLE) .doc( "The codec of Comet native shuffle used to compress shuffle data. lz4, zstd, and " + @@ -454,16 +464,19 @@ object CometConf extends ShimCometConf { .checkValues(Set("zstd", "lz4", "snappy")) .createWithDefault("lz4") - val COMET_EXEC_SHUFFLE_COMPRESSION_ZSTD_LEVEL: ConfigEntry[Int] = - conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.compression.zstd.level") + val COMET_SHUFFLE_COMPRESSION_ZSTD_LEVEL: ConfigEntry[Int] = + conf("spark.comet.shuffle.compression.zstd.level") + .withAlternative(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.compression.zstd.level") .category(CATEGORY_SHUFFLE) .doc("The compression level to use when compressing shuffle files with zstd.") .intConf .createWithDefault(1) - val COMET_COLUMNAR_SHUFFLE_MAX_WRITERS_PER_EXECUTOR: ConfigEntry[Int] = { - conf("spark.comet.columnar.shuffle.max.writers.per.executor") - .withAlternative("spark.comet.columnar.shuffle.async.max.thread.num") + val COMET_SHUFFLE_JVM_MAX_WRITERS_PER_EXECUTOR: ConfigEntry[Int] = { + conf("spark.comet.shuffle.jvm.maxWritersPerExecutor") + .withAlternative( + "spark.comet.columnar.shuffle.max.writers.per.executor", + "spark.comet.columnar.shuffle.async.max.thread.num") .category(CATEGORY_SHUFFLE) .doc( "Maximum number of concurrent hash-based shuffle writers per executor. Comet's " + @@ -474,11 +487,12 @@ object CometConf extends ShimCometConf { .createWithDefault(100) } - val COMET_COLUMNAR_SHUFFLE_SPILL_THRESHOLD: ConfigEntry[Int] = - conf("spark.comet.columnar.shuffle.spill.threshold") + val COMET_SHUFFLE_JVM_SPILL_THRESHOLD: ConfigEntry[Int] = + conf("spark.comet.shuffle.jvm.spillThreshold") + .withAlternative("spark.comet.columnar.shuffle.spill.threshold") .category(CATEGORY_SHUFFLE) .doc( - "Number of rows to be spilled used for Comet columnar shuffle. " + + "Number of rows to be spilled used for Comet JVM (columnar) shuffle. " + "For every configured number of rows, a new spill file will be created. " + "Higher value means more memory requirement to buffer shuffle data before " + "flushing to disk. As Comet uses columnar shuffle which is columnar format, " + @@ -488,11 +502,12 @@ object CometConf extends ShimCometConf { .intConf .createWithDefault(Int.MaxValue) - val COMET_ONHEAP_SHUFFLE_MEMORY_FACTOR: ConfigEntry[Double] = - conf("spark.comet.columnar.shuffle.memory.factor") + val COMET_SHUFFLE_JVM_MEMORY_FACTOR: ConfigEntry[Double] = + conf("spark.comet.shuffle.jvm.memoryFactor") + .withAlternative("spark.comet.columnar.shuffle.memory.factor") .category(CATEGORY_TESTING) - .doc("Fraction of Comet memory to be allocated per executor process for columnar shuffle " + - s"when running in on-heap mode. $TUNING_GUIDE.") + .doc("Fraction of Comet memory to be allocated per executor process for JVM (columnar) " + + s"shuffle when running in on-heap mode. $TUNING_GUIDE.") .doubleConf .checkValue( factor => factor > 0, @@ -506,8 +521,9 @@ object CometConf extends ShimCometConf { .checkValue(v => v > 0, "Batch size must be positive") .createWithDefault(8192) - val COMET_COLUMNAR_SHUFFLE_BATCH_SIZE: ConfigEntry[Int] = - conf("spark.comet.columnar.shuffle.batch.size") + val COMET_SHUFFLE_JVM_BATCH_SIZE: ConfigEntry[Int] = + conf("spark.comet.shuffle.jvm.batchSize") + .withAlternative("spark.comet.columnar.shuffle.batch.size") .category(CATEGORY_SHUFFLE) .doc("Batch size when writing out sorted spill files on the native side. Note that " + "this should not be larger than batch size (i.e., `spark.comet.batchSize`). Otherwise " + @@ -518,8 +534,9 @@ object CometConf extends ShimCometConf { "Should not be larger than batch size `spark.comet.batchSize`") .createWithDefault(8192) - val COMET_SHUFFLE_WRITE_BUFFER_SIZE: ConfigEntry[Long] = - conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.writeBufferSize") + val COMET_SHUFFLE_NATIVE_WRITE_BUFFER_SIZE: ConfigEntry[Long] = + conf("spark.comet.shuffle.native.writeBufferSize") + .withAlternative(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.writeBufferSize") .category(CATEGORY_SHUFFLE) .doc("Size of the write buffer in bytes used by the native shuffle writer when writing " + "shuffle data to disk. Larger values may improve write performance by reducing " + @@ -529,15 +546,16 @@ object CometConf extends ShimCometConf { .checkValue(v => v > 0, "Write buffer size must be positive") .createWithDefault(1) - val COMET_SHUFFLE_PREFER_DICTIONARY_RATIO: ConfigEntry[Double] = conf( - "spark.comet.shuffle.preferDictionary.ratio") + val COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO: ConfigEntry[Double] = conf( + "spark.comet.shuffle.jvm.preferDictionary.ratio") + .withAlternative("spark.comet.shuffle.preferDictionary.ratio") .category(CATEGORY_SHUFFLE) .doc( "The ratio of total values to distinct values in a string column to decide whether to " + "prefer dictionary encoding when shuffling the column. If the ratio is higher than " + "this config, dictionary encoding will be used on shuffling string column. This config " + "is effective if it is higher than 1.0. Note that this " + - "config is only used when `spark.comet.exec.shuffle.mode` is `jvm`.") + "config is only used when `spark.comet.shuffle.mode` is `jvm`.") .doubleConf .createWithDefault(10.0) @@ -552,8 +570,8 @@ object CometConf extends ShimCometConf { .doubleConf .createWithDefault(1.0) - val COMET_SHUFFLE_MAX_BUFFER_BYTES: ConfigEntry[Long] = - conf("spark.comet.shuffle.maxBufferBytes") + val COMET_SHUFFLE_NATIVE_MAX_BUFFER_BYTES: ConfigEntry[Long] = + conf("spark.comet.shuffle.native.maxBufferBytes") .category(CATEGORY_SHUFFLE) .doc( "Maximum number of bytes that the native shuffle writer will buffer in memory " + diff --git a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala index c87e927b483..3511c7e494f 100644 --- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala +++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala @@ -132,12 +132,12 @@ object CometSparkSessionExtensions extends Logging { return false } - if (COMET_EXEC_SHUFFLE_ENABLED.get(conf) && !isCometShuffleManagerEnabled(conf)) { + if (COMET_SHUFFLE_ENABLED.get(conf) && !isCometShuffleManagerEnabled(conf)) { logWarning( "Comet extension is disabled because spark.shuffle.manager is not set to " + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager. " + "Comet provides limited benefit without its shuffle manager. " + - s"Set ${COMET_EXEC_SHUFFLE_ENABLED.key}=false to keep Comet enabled with " + + s"Set ${COMET_SHUFFLE_ENABLED.key}=false to keep Comet enabled with " + "Spark's default shuffle manager.") return false } @@ -172,11 +172,11 @@ object CometSparkSessionExtensions extends Logging { } // Check whether Comet shuffle is enabled: - // 1. `COMET_EXEC_SHUFFLE_ENABLED` is true + // 1. `COMET_SHUFFLE_ENABLED` is true // 2. `spark.shuffle.manager` is set to `CometShuffleManager` // 3. Off-heap memory is enabled || Spark/Comet unit testing def isCometShuffleEnabled(conf: SQLConf): Boolean = - COMET_EXEC_SHUFFLE_ENABLED.get(conf) && isCometShuffleManagerEnabled(conf) + COMET_SHUFFLE_ENABLED.get(conf) && isCometShuffleManagerEnabled(conf) def isCometShuffleManagerEnabled(conf: SQLConf): Boolean = { conf.contains("spark.shuffle.manager") && conf.getConfString("spark.shuffle.manager") == @@ -210,7 +210,7 @@ object CometSparkSessionExtensions extends Logging { */ def shouldOverrideMemoryConf(conf: SparkConf): Boolean = { val cometEnabled = getBooleanConf(conf, CometConf.COMET_ENABLED) - val cometShuffleEnabled = getBooleanConf(conf, CometConf.COMET_EXEC_SHUFFLE_ENABLED) + val cometShuffleEnabled = getBooleanConf(conf, CometConf.COMET_SHUFFLE_ENABLED) val cometExecEnabled = getBooleanConf(conf, CometConf.COMET_EXEC_ENABLED) val offHeapMode = CometSparkSessionExtensions.isOffHeapEnabled(conf) cometEnabled && (cometShuffleEnabled || cometExecEnabled) && !offHeapMode @@ -253,7 +253,7 @@ object CometSparkSessionExtensions extends Logging { val cometMemoryOverhead = getCometMemoryOverheadInMiB(sparkConf) - val overheadFactor = COMET_ONHEAP_SHUFFLE_MEMORY_FACTOR.get(conf) + val overheadFactor = COMET_SHUFFLE_JVM_MEMORY_FACTOR.get(conf) val shuffleMemorySize = (overheadFactor * cometMemoryOverhead).toLong if (shuffleMemorySize > cometMemoryOverhead) { diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index fb88482f15f..f8721fea7d5 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -611,7 +611,7 @@ case class CometExecRule(session: SparkSession) // Revert CometColumnarShuffle to Spark's ShuffleExchangeExec when both its parent and child // are non-Comet HashAggregate/ObjectHashAggregate operators that remained JVM after the main // transform pass. See https://github.com/apache/datafusion-comet/issues/4004. - if (CometConf.COMET_EXEC_SHUFFLE_REVERT_REDUNDANT_COLUMNAR_ENABLED.get()) { + if (CometConf.COMET_SHUFFLE_REVERT_REDUNDANT_COLUMNAR_ENABLED.get()) { newPlan = revertRedundantColumnarShuffle(newPlan) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala index 2f1b2411744..15f620339d1 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala @@ -221,7 +221,7 @@ class CometNativeShuffleWriter[K, V]( shuffleWriterBuilder.setOutputIndexFile(indexFile) if (SparkEnv.get.conf.getBoolean("spark.shuffle.compress", true)) { - val codec = CometConf.COMET_EXEC_SHUFFLE_COMPRESSION_CODEC.get() match { + val codec = CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.get() match { case "zstd" => CompressionCodec.Zstd case "lz4" => CompressionCodec.Lz4 case "snappy" => CompressionCodec.Snappy @@ -231,11 +231,10 @@ class CometNativeShuffleWriter[K, V]( } else { shuffleWriterBuilder.setCodec(CompressionCodec.None) } - shuffleWriterBuilder.setCompressionLevel( - CometConf.COMET_EXEC_SHUFFLE_COMPRESSION_ZSTD_LEVEL.get) + shuffleWriterBuilder.setCompressionLevel(CometConf.COMET_SHUFFLE_COMPRESSION_ZSTD_LEVEL.get) shuffleWriterBuilder.setWriteBufferSize( - CometConf.COMET_SHUFFLE_WRITE_BUFFER_SIZE.get().min(Int.MaxValue).toInt) - shuffleWriterBuilder.setMaxBufferBytes(CometConf.COMET_SHUFFLE_MAX_BUFFER_BYTES.get()) + CometConf.COMET_SHUFFLE_NATIVE_WRITE_BUFFER_SIZE.get().min(Int.MaxValue).toInt) + shuffleWriterBuilder.setMaxBufferBytes(CometConf.COMET_SHUFFLE_NATIVE_MAX_BUFFER_BYTES.get()) outputPartitioning match { case p if isSinglePartitioning(p) => @@ -330,7 +329,7 @@ class CometNativeShuffleWriter[K, V]( val partitioning = PartitioningOuterClass.RoundRobinPartition.newBuilder() partitioning.setNumPartitions(outputPartitioning.numPartitions) partitioning.setMaxHashColumns( - CometConf.COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_MAX_HASH_COLUMNS.get()) + CometConf.COMET_SHUFFLE_NATIVE_ROUND_ROBIN_PARTITIONING_MAX_HASH_COLUMNS.get()) val partitioningBuilder = PartitioningOuterClass.Partitioning.newBuilder() shuffleWriterBuilder.setPartitioning( diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala index 2a50bbd89bf..9505443fb31 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala @@ -50,7 +50,7 @@ import org.apache.spark.util.random.XORShiftRandom import com.google.common.base.Objects import org.apache.comet.{CometConf, CometExplainInfo} -import org.apache.comet.CometConf.{COMET_EXEC_SHUFFLE_ENABLED, COMET_SHUFFLE_MODE} +import org.apache.comet.CometConf.{COMET_SHUFFLE_ENABLED, COMET_SHUFFLE_MODE} import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, isCometShuffleManagerEnabled, withFallbackReasons} import org.apache.comet.serde.{Compatible, OperatorOuterClass, QueryPlanSerde, SupportLevel, Unsupported} import org.apache.comet.serde.operator.CometSink @@ -350,11 +350,11 @@ object CometShuffleExchangeExec } if (!isCometPlan(s.child) && - !CometConf.COMET_EXEC_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED.get(s.conf)) { + !CometConf.COMET_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED.get(s.conf)) { withFallbackReasons( s, Set( - s"${CometConf.COMET_EXEC_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED.key} is disabled " + + s"${CometConf.COMET_SHUFFLE_CONVERT_FROM_SPARK_PLAN_ENABLED.key} is disabled " + "and child is not a Comet plan")) return None } @@ -446,9 +446,9 @@ object CometShuffleExchangeExec val partitioning = s.outputPartitioning partitioning match { case HashPartitioning(expressions, _) => - if (!CometConf.COMET_EXEC_SHUFFLE_WITH_HASH_PARTITIONING_ENABLED.get(conf)) { + if (!CometConf.COMET_SHUFFLE_NATIVE_HASH_PARTITIONING_ENABLED.get(conf)) { reasons += - s"${CometConf.COMET_EXEC_SHUFFLE_WITH_HASH_PARTITIONING_ENABLED.key} is disabled" + s"${CometConf.COMET_SHUFFLE_NATIVE_HASH_PARTITIONING_ENABLED.key} is disabled" } for (expr <- expressions) { if (QueryPlanSerde.exprToProto(expr, inputs).isEmpty) { @@ -485,9 +485,9 @@ object CometShuffleExchangeExec false } - if (!CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.get(conf)) { + if (!CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.get(conf)) { reasons += - s"${CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key} is disabled" + s"${CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.key} is disabled" return reasons.toSeq } for (o <- orderings) { @@ -512,7 +512,7 @@ object CometShuffleExchangeExec } } case RoundRobinPartitioning(_) => - val config = CometConf.COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_ENABLED + val config = CometConf.COMET_SHUFFLE_NATIVE_ROUND_ROBIN_PARTITIONING_ENABLED if (!config.get(conf)) { reasons += s"${config.key} is disabled" } @@ -667,8 +667,8 @@ object CometShuffleExchangeExec * tag the node. */ private def isCometShuffleEnabledReason(op: SparkPlan): Option[String] = { - if (!COMET_EXEC_SHUFFLE_ENABLED.get(op.conf)) { - Some(s"Comet shuffle is not enabled: ${COMET_EXEC_SHUFFLE_ENABLED.key} is not enabled") + if (!COMET_SHUFFLE_ENABLED.get(op.conf)) { + Some(s"Comet shuffle is not enabled: ${COMET_SHUFFLE_ENABLED.key} is not enabled") } else if (!isCometShuffleManagerEnabled(op.conf)) { Some(s"spark.shuffle.manager is not set to ${classOf[CometShuffleManager].getName}") } else { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala index 06f2ff02d3c..398ed66b6a5 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala @@ -303,10 +303,10 @@ object CometShuffleManager extends Logging { val partitionCond = SortShuffleWriter.shouldBypassMergeSort(conf, dep) // Bypass merge sort if we have partition * cores fewer than - // `spark.comet.columnar.shuffle.max.writers.per.executor` + // `spark.comet.shuffle.jvm.maxWritersPerExecutor` val executorCores = conf.get(config.EXECUTOR_CORES) val maxThreads = - CometConf.COMET_COLUMNAR_SHUFFLE_MAX_WRITERS_PER_EXECUTOR.get(SQLConf.get) + CometConf.COMET_SHUFFLE_JVM_MAX_WRITERS_PER_EXECUTOR.get(SQLConf.get) val threadCond = dep.partitioner.numPartitions * executorCores <= maxThreads // Comet columnar shuffle buffers rows in memory. If too many cores are used with diff --git a/spark/src/test/resources/pyspark/test_pyarrow_udf.py b/spark/src/test/resources/pyspark/test_pyarrow_udf.py index 67cdb9f134b..4571932266e 100644 --- a/spark/src/test/resources/pyspark/test_pyarrow_udf.py +++ b/spark/src/test/resources/pyspark/test_pyarrow_udf.py @@ -63,12 +63,12 @@ def spark(): .config("spark.plugins", "org.apache.spark.CometPlugin") .config("spark.comet.enabled", "true") .config("spark.comet.exec.enabled", "true") - # spark.comet.exec.shuffle.enabled defaults to true, and + # spark.comet.shuffle.enabled defaults to true, and # CometSparkSessionExtensions.isCometLoaded refuses to register Comet's rules # at all when shuffle is on but spark.shuffle.manager is not the Comet manager. # These tests do not need Comet shuffle, so disable it explicitly to keep # Comet's scan and exec rules active without configuring shuffle. - .config("spark.comet.exec.shuffle.enabled", "false") + .config("spark.comet.shuffle.enabled", "false") .config("spark.memory.offHeap.enabled", "true") .config("spark.memory.offHeap.size", "2g") .getOrCreate() diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/calendar_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/calendar_interval.sql index b544bc6276a..796e6354acd 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/calendar_interval.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/calendar_interval.sql @@ -16,7 +16,7 @@ -- under the License. -- Config: spark.comet.exec.localTableScan.enabled=true --- Config: spark.comet.exec.shuffle.mode=native +-- Config: spark.comet.shuffle.mode=native query SELECT * FROM VALUES diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_dt_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_dt_interval.sql index 371f01be43f..476930481e6 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_dt_interval.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_dt_interval.sql @@ -16,7 +16,7 @@ -- under the License. -- Config: spark.comet.exec.localTableScan.enabled=true --- Config: spark.comet.exec.shuffle.mode=native +-- Config: spark.comet.shuffle.mode=native -- Routes make_dt_interval through the codegen dispatcher; produces DayTimeIntervalType. diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle.sql index 024865b9041..f3031baa350 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle.sql @@ -17,8 +17,8 @@ -- MinSparkVersion: 4.1 -- Config: spark.sql.timeType.enabled=true --- Config: spark.comet.native.shuffle.partitioning.roundrobin.enabled=true --- ConfigMatrix: spark.comet.exec.shuffle.mode=native,jvm +-- Config: spark.comet.shuffle.native.partitioning.roundrobin.enabled=true +-- ConfigMatrix: spark.comet.shuffle.mode=native,jvm statement CREATE TABLE test_make_time_shuffle(hours int, minutes int, secs decimal(16,6)) USING parquet diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_jvm.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_jvm.sql index 36316adb875..bd0e1972fa7 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_jvm.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_jvm.sql @@ -17,7 +17,7 @@ -- MinSparkVersion: 4.1 -- Config: spark.sql.timeType.enabled=true --- Config: spark.comet.exec.shuffle.mode=jvm +-- Config: spark.comet.shuffle.mode=jvm statement CREATE TABLE test_make_time_range_shuffle_jvm(hours int, minutes int, secs decimal(16,6)) USING parquet diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_native.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_native.sql index 3627c87fe06..6f54adeb34c 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_native.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_native.sql @@ -17,7 +17,7 @@ -- MinSparkVersion: 4.1 -- Config: spark.sql.timeType.enabled=true --- Config: spark.comet.exec.shuffle.mode=native +-- Config: spark.comet.shuffle.mode=native statement CREATE TABLE test_make_time_range_shuffle_native(hours int, minutes int, secs decimal(16,6)) USING parquet diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_ym_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_ym_interval.sql index e2763cce7ab..150b7799ffb 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/make_ym_interval.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_ym_interval.sql @@ -16,7 +16,7 @@ -- under the License. -- Config: spark.comet.exec.localTableScan.enabled=true --- Config: spark.comet.exec.shuffle.mode=native +-- Config: spark.comet.shuffle.mode=native -- Routes make_ym_interval through the codegen dispatcher; produces YearMonthIntervalType. diff --git a/spark/src/test/scala/org/apache/comet/CometConfSuite.scala b/spark/src/test/scala/org/apache/comet/CometConfSuite.scala index 4ddd882bd5e..a5b1dc5d5f0 100644 --- a/spark/src/test/scala/org/apache/comet/CometConfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometConfSuite.scala @@ -122,6 +122,27 @@ class CometConfSuite extends AnyFunSuite { assert(CometConf.COMET_FORCE_SHJ.get(conf)) } + test("COMET_SHUFFLE_ENABLED reads the deprecated exec.shuffle.enabled key as an alias") { + val conf = new SQLConf + conf.setConfString(s"${CometConf.COMET_EXEC_CONFIG_PREFIX}.shuffle.enabled", "false") + + assert(!CometConf.COMET_SHUFFLE_ENABLED.get(conf)) + } + + test("COMET_SHUFFLE_JVM_SPILL_THRESHOLD reads the deprecated dots-in-segment key") { + val conf = new SQLConf + conf.setConfString("spark.comet.columnar.shuffle.spill.threshold", "12345") + + assert(CometConf.COMET_SHUFFLE_JVM_SPILL_THRESHOLD.get(conf) == 12345) + } + + test("COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO reads the deprecated top-level key") { + val conf = new SQLConf + conf.setConfString("spark.comet.shuffle.preferDictionary.ratio", "3.5") + + assert(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.get(conf) == 3.5) + } + test("COMET_EXPLAIN_CODEGEN_ENABLED reads deprecated explainCodegen.enabled as an alias") { val conf = new SQLConf conf.setConfString("spark.comet.explainCodegen.enabled", "true") diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index d6955fb78b7..b8a70760bf4 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -145,8 +145,8 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { StructField("v", IntegerType, nullable = false))) val rows = (0 until 1000).map(i => Row(if (i % 2 == 0) Row(i.toLong) else null, i)) withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_WITH_HASH_PARTITIONING_ENABLED.key -> "true") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_NATIVE_HASH_PARTITIONING_ENABLED.key -> "true") { val df = spark .createDataFrame(spark.sparkContext.parallelize(rows), schema) .repartition(4, col("v")) // materialize the typed struct through a Comet shuffle @@ -2026,7 +2026,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false", + CometConf.COMET_SHUFFLE_ENABLED.key -> "false", EXTENDED_EXPLAIN_PROVIDERS_KEY -> "org.apache.comet.ExtendedExplainInfo") { val table = "test" withTable(table) { @@ -2048,7 +2048,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { "extractintervalmonths is not supported")), ( s"SELECT sum(c0), sum(c2) from $table group by c1", - Set("Comet shuffle is not enabled: spark.comet.exec.shuffle.enabled is not enabled")), + Set("Comet shuffle is not enabled: spark.comet.shuffle.enabled is not enabled")), ( "SELECT A.c1, A.sum_c0, A.sum_c2, B.casted from " + s"(SELECT c1, sum(c0) as sum_c0, sum(c2) as sum_c2 from $table group by c1) as A, " @@ -2056,7 +2056,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { + "where A.c1 = B.c1 ", Set( "Cast from CalendarIntervalType to StringType is not supported", - "Comet shuffle is not enabled: spark.comet.exec.shuffle.enabled is not enabled")), + "Comet shuffle is not enabled: spark.comet.shuffle.enabled is not enabled")), (s"select * from $table LIMIT 10 OFFSET 3", Set("Comet shuffle is not enabled"))) .foreach(test => { val qry = test._1 diff --git a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala index 268fdf94ebc..846fd633a3e 100644 --- a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala @@ -31,7 +31,7 @@ class CometSparkSessionExtensionsSuite extends CometTestBase { val conf = new SQLConf // Disable Comet shuffle so this test can focus on other checks without needing // spark.shuffle.manager to be set. - conf.setConfString(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "false") + conf.setConfString(CometConf.COMET_SHUFFLE_ENABLED.key, "false") conf.setConfString(CometConf.COMET_ENABLED.key, "false") assert(!isCometLoaded(conf)) @@ -61,11 +61,11 @@ class CometSparkSessionExtensionsSuite extends CometTestBase { assert(!isCometLoaded(conf)) // Opt out: shuffle.enabled=false. Comet should load (assumes native lib is available). - conf.setConfString(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "false") + conf.setConfString(CometConf.COMET_SHUFFLE_ENABLED.key, "false") assert(isCometLoaded(conf)) // shuffle.enabled=true with the Comet shuffle manager registered: Comet should load. - conf.setConfString(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true") + conf.setConfString(CometConf.COMET_SHUFFLE_ENABLED.key, "true") conf.setConfString( "spark.shuffle.manager", "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") @@ -123,7 +123,7 @@ class CometSparkSessionExtensionsSuite extends CometTestBase { val conf = new SparkConf() val sqlConf = new SQLConf - sqlConf.setConfString(CometConf.COMET_ONHEAP_SHUFFLE_MEMORY_FACTOR.key, "0.2") + sqlConf.setConfString(CometConf.COMET_SHUFFLE_JVM_MEMORY_FACTOR.key, "0.2") // Minimum Comet memory overhead is 384MB assert( diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index 3248101a2b4..b3a7015385a 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -180,7 +180,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { disableFinal <- Seq(false, true); groupBy <- Seq("", " GROUP BY _1")) { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native", SQLConf.USE_OBJECT_HASH_AGG.key -> "true", CometConf.COMET_ENABLE_PARTIAL_HASH_AGGREGATE.key -> (!disablePartial).toString, @@ -193,7 +193,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("stddev_pop should return NaN for some cases") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { Seq(true, false).foreach { nullOnDivideByZero => withSQLConf("spark.sql.legacy.statisticalAggregate" -> nullOnDivideByZero.toString) { @@ -210,7 +210,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("count with aggregation filter") { withSQLConf( CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { val df1 = sql("SELECT count(DISTINCT 2), count(DISTINCT 2,3)") checkSparkAnswer(df1) @@ -223,7 +223,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("multiple column distinct count") { withSQLConf( CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { val df1 = Seq( ("a", "b", "c"), @@ -244,7 +244,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { lowerCaseData.createOrReplaceTempView("lowerCaseData") withSQLConf( CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { val df = sql("SELECT LAST(n) FROM lowerCaseData") checkSparkAnswer(df) @@ -259,7 +259,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { allNulls.createOrReplaceTempView("allNulls") withSQLConf( CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { val df = sql("select sum(a), avg(a) from allNulls") checkSparkAnswer(df) @@ -272,7 +272,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { withParquetTable(data, "tbl") { withSQLConf( CometConf.COMET_ENABLE_FINAL_HASH_AGGREGATE.key -> "false", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { checkSparkAnswer( "SELECT _4, SUM(_1), SUM(_2), SUM(_3), AVG(_1), AVG(_2), AVG(_3) FROM tbl GROUP BY _4") @@ -285,7 +285,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { withParquetTable(data, "tbl") { withSQLConf( CometConf.COMET_ENABLE_PARTIAL_HASH_AGGREGATE.key -> "false", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { checkSparkAnswer( "SELECT _4, SUM(_1), SUM(_2), SUM(_3), AVG(_1), AVG(_2), AVG(_3) FROM tbl GROUP BY _4") @@ -296,7 +296,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("Aggregation without aggregate expressions should use correct result expressions") { withSQLConf( CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { withTempDir { dir => val path = new Path(dir.toURI.toString, "test") @@ -312,7 +312,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("Final aggregation should not bind to the input of partial aggregation") { withSQLConf( CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -331,7 +331,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { withTable("lineitem", "part") { withSQLConf( CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { sql( @@ -368,7 +368,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { withSQLConf( SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> EliminateSorts.ruleName, CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -387,7 +387,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { withSQLConf( SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> EliminateSorts.ruleName, CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -407,7 +407,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { withTable(table) { withSQLConf( CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false", + CometConf.COMET_SHUFFLE_ENABLED.key -> "false", CometConf.COMET_SHUFFLE_MODE.key -> "native") { withTable(table) { sql(s"CREATE TABLE $table(col DECIMAL(5, 2)) USING PARQUET") @@ -420,7 +420,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("fix: Decimal Average should not enable native final aggregation") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test") @@ -507,7 +507,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { Seq(true, false).foreach { nativeShuffleEnabled => Seq(true, false).foreach { dictionaryEnabled => withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> nativeShuffleEnabled.toString, + CometConf.COMET_SHUFFLE_ENABLED.key -> nativeShuffleEnabled.toString, CometConf.COMET_SHUFFLE_MODE.key -> "native") { withParquetTable( (0 until 100).map(i => (i, (i % 10).toString)), @@ -529,7 +529,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { assume(isSpark41Plus, "Spark 4.1+ supports grouping on map-containing types") withSQLConf( CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { val query = """SELECT col1.data['key'] |FROM VALUES @@ -648,7 +648,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { (-0.0.asInstanceOf[Float], 2), (0.0.asInstanceOf[Float], 3), (Float.NaN, 4)) - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { withParquetTable(data, "tbl", dictionaryEnabled) { checkSparkAnswer("SELECT SUM(_2), MIN(_2), MAX(_2), _1 FROM tbl GROUP BY _1") checkSparkAnswer("SELECT MIN(_1), MAX(_1), MIN(_2), MAX(_2) FROM tbl") @@ -717,7 +717,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { Seq(true, false).foreach { dictionaryEnabled => Seq(true, false).foreach { nativeShuffleEnabled => withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> nativeShuffleEnabled.toString, + CometConf.COMET_SHUFFLE_ENABLED.key -> nativeShuffleEnabled.toString, CometConf.COMET_SHUFFLE_MODE.key -> "native") { withTempDir { dir => val path = new Path(dir.toURI.toString, "test") @@ -854,7 +854,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", "spark.comet.exec.shuffle.fallbackToColumnar" -> "false", "spark.comet.enabled" -> "true") { spark.read.parquet(dir.getAbsolutePath).createOrReplaceTempView("t2") @@ -946,7 +946,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { withTable("t") { sql("CREATE TABLE t(v VARCHAR(3), i INT) USING PARQUET") sql("INSERT INTO t VALUES ('c', 1)") - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { checkSparkAnswerAndNumOfAggregates("SELECT v, sum(i) FROM t GROUP BY v ORDER BY v", 1) } } @@ -1026,7 +1026,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { Seq(128, numValues + 100).foreach { batchSize => withSQLConf( CometConf.COMET_BATCH_SIZE.key -> batchSize.toString, - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { // Test all combinations of different aggregation & group-by types (1 to 14).foreach { gCol => @@ -1075,7 +1075,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { Seq(128, numValues + 100).foreach { batchSize => withSQLConf( CometConf.COMET_BATCH_SIZE.key -> batchSize.toString, - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { // Test all combinations of different aggregation & group-by types (1 to 14).foreach { gCol => @@ -1096,7 +1096,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("test final count") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { Seq(false, true).foreach { dictionaryEnabled => withParquetTable((0 until 5).map(i => (i, i % 2)), "tbl", dictionaryEnabled) { @@ -1113,7 +1113,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("test final min/max") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable((0 until 5).map(i => (i, i % 2)), "tbl", dictionaryEnabled) { @@ -1134,7 +1134,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("test final min/max/count with result expressions") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable((0 until 5).map(i => (i, i % 2)), "tbl", dictionaryEnabled) { @@ -1169,7 +1169,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("test final sum") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { Seq(false, true).foreach { dictionaryEnabled => withParquetTable((0L until 5L).map(i => (i, i % 2)), "tbl", dictionaryEnabled) { @@ -1205,7 +1205,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("test final avg") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable( @@ -1227,7 +1227,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("final decimal avg") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { Seq(true, false).foreach { dictionaryEnabled => withSQLConf("parquet.enable.dictionary" -> dictionaryEnabled.toString) { @@ -1265,7 +1265,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { (0 until 5).map(i => (i.toDouble, i.toDouble % 2)), "tbl", dictionaryEnabled) { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { checkSparkAnswerAndNumOfAggregates("SELECT _2 , AVG(_1) FROM tbl GROUP BY _2", 1) } } @@ -1274,7 +1274,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("avg null handling") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { val table = "avg_null_handling" withTable(table) { @@ -1296,7 +1296,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { Seq(true, false).foreach { dictionaryEnabled => Seq(true, false).foreach { nativeShuffleEnabled => withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> nativeShuffleEnabled.toString, + CometConf.COMET_SHUFFLE_ENABLED.key -> nativeShuffleEnabled.toString, CometConf.COMET_SHUFFLE_MODE.key -> "native", CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true") { withTempDir { dir => @@ -1336,7 +1336,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("distinct") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { Seq("native", "jvm").foreach { cometShuffleMode => withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> cometShuffleMode) { Seq(true, false).foreach { dictionary => @@ -1380,7 +1380,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("first/last") { withSQLConf( SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { Seq(true, false).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { @@ -1416,7 +1416,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("test bool_and/bool_or") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { Seq("native", "jvm").foreach { cometShuffleMode => withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> cometShuffleMode) { Seq(true, false).foreach { dictionary => @@ -1442,7 +1442,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("bitwise aggregate") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { Seq(true, false).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { @@ -1518,7 +1518,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("covariance & correlation") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { Seq("jvm", "native").foreach { cometShuffleMode => withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> cometShuffleMode) { Seq(true, false).foreach { dictionary => @@ -1615,7 +1615,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("var_pop and var_samp") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { Seq("native", "jvm").foreach { cometShuffleMode => withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> cometShuffleMode) { Seq(true, false).foreach { dictionary => @@ -1654,7 +1654,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("stddev_pop and stddev_samp") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { Seq("native", "jvm").foreach { cometShuffleMode => withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> cometShuffleMode) { Seq(true, false).foreach { dictionary => diff --git a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala index 1cbb414c04a..bf354356386 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala @@ -54,10 +54,10 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar pos: Position): Unit = { super.test(testName, testTags: _*) { withSQLConf( - CometConf.COMET_COLUMNAR_SHUFFLE_SPILL_THRESHOLD.key -> numElementsForceSpillThreshold.toString, + CometConf.COMET_SHUFFLE_JVM_SPILL_THRESHOLD.key -> numElementsForceSpillThreshold.toString, CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { testFun } } @@ -105,7 +105,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar test("columnar shuffle on nested struct including nulls") { Seq(10, 201).foreach { numPartitions => Seq("1.0", "10.0").foreach { ratio => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> ratio) { withParquetTable( (0 until 50).map(i => (i, Seq((i + 1, i.toString), null, (i + 3, (i + 3).toString)), i + 1)), @@ -125,7 +125,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar test("columnar shuffle on struct including nulls") { Seq(10, 201).foreach { numPartitions => Seq("1.0", "10.0").foreach { ratio => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> ratio) { val data: Seq[(Int, (Int, String))] = Seq((1, (0, "1")), (2, (3, "3")), (3, null)) withParquetTable(data, "tbl") { @@ -151,7 +151,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar Seq("1.0", "10.0").foreach { ratio => withSQLConf( CometConf.COMET_EXEC_ENABLED.key -> execEnabled, - CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> ratio) { withParquetTable((0 until 50).map(i => (Map(Seq(i, i + 1) -> i), i + 1)), "tbl") { val df = sql("SELECT * FROM tbl") .filter($"_2" > 10) @@ -199,7 +199,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar Seq("1.0", "10.0").foreach { ratio => withSQLConf( CometConf.COMET_EXEC_ENABLED.key -> execEnabled, - CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> ratio) { withParquetTable( (0 until 50).map(i => ((Seq(Map(1 -> i)), Map(2 -> i), Map(3 -> i)), i + 1)), "tbl") { @@ -302,7 +302,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar def columnarShuffleOnMapTest[K: TypeTag](num: Int, keys: Seq[K]): Unit = { Seq(10, 201).foreach { numPartitions => Seq("1.0", "10.0").foreach { ratio => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> ratio) { withParquetTable(genTuples(num, keys), "tbl") { repartitionAndSort(numPartitions) } @@ -367,7 +367,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar Seq(10, 201).foreach { numPartitions => Seq("1.0", "10.0").foreach { ratio => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> ratio) { withParquetTable( (0 until 50).map(i => ( @@ -399,7 +399,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar Seq("false", "true").foreach { _ => Seq(10, 201).foreach { numPartitions => Seq("1.0", "10.0").foreach { ratio => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> ratio) { withParquetTable( (0 until 50).map(i => (Seq(Seq(i + 1), Seq(i + 2), Seq(i + 3)), i + 1)), "tbl") { @@ -419,7 +419,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar test("columnar shuffle on nested struct") { Seq(10, 201).foreach { numPartitions => Seq("1.0", "10.0").foreach { ratio => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> ratio) { withParquetTable( (0 until 50).map(i => ((i, 2.toString, (i + 1).toLong, (3.toString, i + 1, (i + 2).toLong)), i + 1)), @@ -441,8 +441,8 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar withSQLConf( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", CometConf.COMET_BATCH_SIZE.key -> "10", - CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> "1.1", - CometConf.COMET_COLUMNAR_SHUFFLE_SPILL_THRESHOLD.key -> "1000000000") { + CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> "1.1", + CometConf.COMET_SHUFFLE_JVM_SPILL_THRESHOLD.key -> "1000000000") { val table1 = (0 until 1000) .map(i => (111111.toString, 2222222.toString, 3333333.toString, i.toLong)) .toDF("a", "b", "c", "d") @@ -463,7 +463,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar test("fix: Dictionary field should have distinct dict_id") { Seq(10, 201).foreach { numPartitions => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> "2.0") { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> "2.0") { withParquetTable( (0 until 10000).map(i => (1.toString, 2.toString, (i + 1).toLong)), "tbl") { @@ -480,7 +480,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar test("dictionary shuffle") { Seq(10, 201).foreach { numPartitions => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> "2.0") { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> "2.0") { withParquetTable((0 until 10000).map(i => (1.toString, (i + 1).toLong)), "tbl") { assert( sql("SELECT * FROM tbl").repartition(numPartitions, $"_1").count() == sql( @@ -495,7 +495,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar test("dictionary shuffle: fallback to string") { Seq(10, 201).foreach { numPartitions => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> "1000000000.0") { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> "1000000000.0") { withParquetTable((0 until 10000).map(i => (1.toString, (i + 1).toLong)), "tbl") { assert( sql("SELECT * FROM tbl").repartition(numPartitions, $"_1").count() == sql( @@ -808,7 +808,7 @@ class CometShuffleSuite extends CometColumnarShuffleSuite { SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { withParquetTable((0 until 10).map(i => (i, i % 5)), "tbl_a") { val df = sql("SELECT * FROM tbl_a") @@ -828,7 +828,7 @@ class CometShuffleSuite extends CometColumnarShuffleSuite { SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { withParquetTable((0 until 10).map(i => (i, i % 5)), "tbl_a") { withParquetTable((0 until 10).map(i => (i % 10, i + 2)), "tbl_b") { @@ -859,7 +859,7 @@ class DisableAQECometShuffleSuite extends CometColumnarShuffleSuite { SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { withParquetTable((0 until 10).map(i => (i, i % 5)), "tbl_a") { withParquetTable((0 until 10).map(i => (i % 10, i + 2)), "tbl_b") { @@ -898,7 +898,7 @@ class CometShuffleEncryptionSuite extends CometTestBase { (1 until 10).map(i => $"_$i").foreach { col => withSQLConf( CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { readParquetFile(path.toString) { df => val shuffled = df @@ -917,7 +917,7 @@ class CometShuffleEncryptionSuite extends CometTestBase { class CometShuffleManagerSuite extends CometTestBase { test("should not bypass merge sort if executor cores are too high") { - withSQLConf(CometConf.COMET_COLUMNAR_SHUFFLE_MAX_WRITERS_PER_EXECUTOR.key -> "100") { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_MAX_WRITERS_PER_EXECUTOR.key -> "100") { val conf = new SparkConf() conf.set("spark.executor.cores", "1") diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala index 3a406f9a723..d3ecc789309 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala @@ -77,7 +77,7 @@ class CometExec3_4PlusSuite extends CometTestBase { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { super.test(testName, testTags: _*) { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { testFun } } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index ce6071bf1a6..d630648e1ad 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -60,7 +60,7 @@ class CometExecSuite extends CometTestBase { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { super.test(testName, testTags: _*) { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { testFun } } @@ -1837,7 +1837,7 @@ class CometExecSuite extends CometTestBase { SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { val data1 = Seq(Tuple1(null), Tuple1((1, "a")), Tuple1((2, null)), Tuple1((3, "b")), Tuple1(null)) @@ -1871,7 +1871,7 @@ class CometExecSuite extends CometTestBase { SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { sql(""" @@ -1897,7 +1897,7 @@ class CometExecSuite extends CometTestBase { SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { sql(""" @@ -1925,7 +1925,7 @@ class CometExecSuite extends CometTestBase { SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { sql(""" @@ -1952,7 +1952,7 @@ class CometExecSuite extends CometTestBase { SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_EXEC_SORT_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { @@ -2167,7 +2167,7 @@ class CometExecSuite extends CometTestBase { "DECIMAL(38, 10)") dataTypes.map { subqueryType => withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl") { var column1 = s"CAST(max(_1) AS $subqueryType)" @@ -2228,7 +2228,7 @@ class CometExecSuite extends CometTestBase { CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "false", CometConf.COMET_CONVERT_FROM_PARQUET_ENABLED.key -> "true", CometConf.COMET_SPARK_TO_ARROW_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { withParquetTable((0 until 10).map(i => (i, i)), "t") { val df = sql("SELECT * FROM t DISTRIBUTE BY (_1 + (SELECT max(_2) FROM t))") @@ -2242,7 +2242,7 @@ class CometExecSuite extends CometTestBase { // This confirms the "Subquery N not found" crash is specific to the native shuffle path. test("scalar subquery in repartition (columnar shuffle)") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { withParquetTable((0 until 10).map(i => (i, i)), "t") { val df = sql("SELECT * FROM t DISTRIBUTE BY (_1 + (SELECT max(_2) FROM t))") @@ -2483,7 +2483,7 @@ class CometExecSuite extends CometTestBase { SQLConf.EXCHANGE_REUSE_ENABLED.key -> "true", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { withTable(tableName, dim) { @@ -2539,7 +2539,7 @@ class CometExecSuite extends CometTestBase { withSQLConf( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "10485760", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { withParquetTable((0 until 100).map(i => (i, i + 1)), "tbl_a") { withParquetTable((0 until 100).map(i => (i, i + 2)), "tbl_b") { @@ -2574,7 +2574,7 @@ class CometExecSuite extends CometTestBase { withSQLConf( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "10485760", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "auto") { withParquetTable((0 until 100).map(i => (i, i + 1)), "tbl_a") { withParquetTable((0 until 100).map(i => (i, i + 2)), "tbl_b") { @@ -2848,7 +2848,7 @@ class CometExecSuite extends CometTestBase { } test("final aggregation") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { withParquetTable( (0 until 100) .map(_ => (Random.nextInt(), Random.nextInt() % 5)), @@ -2897,7 +2897,7 @@ class CometExecSuite extends CometTestBase { test("global sort (columnar shuffle only)") { withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl") { val df = sql("SELECT * FROM tbl").sort($"_1".desc) @@ -2948,7 +2948,7 @@ class CometExecSuite extends CometTestBase { test("limit") { Seq("native", "jvm").foreach { columnarShuffleMode => withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> columnarShuffleMode) { withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl_a") { val df = sql("SELECT * FROM tbl_a") @@ -3445,7 +3445,7 @@ class CometExecSuite extends CometTestBase { } test("coalesce") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { withTable("t1") { (0 until 5) .map(i => (i, (i + 1).toLong)) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index f01d5e2109c..2da0a4c4d84 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -38,7 +38,7 @@ class CometJoinSuite extends CometTestBase { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { super.test(testName, testTags: _*) { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { testFun } } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index 75c86e35689..ced6531cd8d 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -41,7 +41,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper withSQLConf( CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { testFun } } @@ -84,7 +84,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper Seq("false", "true").foreach { _ => Seq(10, 201).foreach { numPartitions => Seq("1.0", "10.0").foreach { ratio => - withSQLConf(CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> ratio) { + withSQLConf(CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> ratio) { withParquetTable( (0 until 50).map(i => (i, Seq(Seq(i + 1), Seq(i + 2), Seq(i + 3)), i + 1)), "tbl") { @@ -146,8 +146,8 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper test("native operator after native shuffle") { Seq("true", "false").zip(Seq("true", "false")).foreach { partitioning => withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_WITH_HASH_PARTITIONING_ENABLED.key -> partitioning._1, - CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> partitioning._2) { + CometConf.COMET_SHUFFLE_NATIVE_HASH_PARTITIONING_ENABLED.key -> partitioning._1, + CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.key -> partitioning._2) { withParquetTable((0 until 5).map(i => (i, (i + 1).toLong)), "tbl") { val df = sql("SELECT * FROM tbl") @@ -309,7 +309,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper test("range partitioning on floating-point falls back when strictFloatingPoint=true") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.key -> "true", CometConf.COMET_EXEC_STRICT_FLOATING_POINT.key -> "true", // Bypass the CometSortOrder-level Incompatible check so that only // supportedRangePartitioningDataType is exercised as the guard. @@ -331,7 +331,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper test( "range partitioning on floating-point uses native shuffle when strictFloatingPoint=false") { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.key -> "true", CometConf.COMET_EXEC_STRICT_FLOATING_POINT.key -> "false") { withParquetTable(floatingPointRangePartitionData, "tbl") { Seq("FLOAT", "DOUBLE").foreach { sqlType => @@ -386,7 +386,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper // This adapts the PySpark example in https://github.com/apache/datafusion-comet/issues/1906 to // test for incorrect partition values after native RangePartitioning test("fix: range partitioning #1906") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.key -> "true") { withParquetTable((0 until 100000).map(i => (i, i + 1)), "tbl") { val df = sql("SELECT * from tbl") @@ -401,7 +401,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper // This adapts the PySpark example in https://github.com/apache/datafusion-comet/issues/1906 to // test for incorrect partition values after native RangePartitioning test("fix: range partitioning #1906, two columns") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.key -> "true") { withParquetTable((0 until 100000).map(i => (i, i + 1)), "tbl") { val df = sql("SELECT * from tbl") @@ -416,7 +416,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper // This adapts the PySpark example in https://github.com/apache/datafusion-comet/issues/1906 to // test for incorrect partition values after native RangePartitioning test("fix: range partitioning #1906, random sort column with duplicates") { - withSQLConf(CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_NATIVE_RANGE_PARTITIONING_ENABLED.key -> "true") { val random = new Random(42) withParquetTable((0 until 100000).map(i => (random.nextInt(10000), i)), "tbl") { val df = sql("SELECT * from tbl") @@ -450,7 +450,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper withParquetTable((0 until 20000).map(i => (i, (i + 1).toLong, s"str$i")), "tbl") { def spillCountWithMaxBufferBytes(maxBufferBytes: String): Long = { var spillCount = 0L - withSQLConf(CometConf.COMET_SHUFFLE_MAX_BUFFER_BYTES.key -> maxBufferBytes) { + withSQLConf(CometConf.COMET_SHUFFLE_NATIVE_MAX_BUFFER_BYTES.key -> maxBufferBytes) { val shuffled = sql("SELECT * FROM tbl").repartition(10, $"_1") checkShuffleAnswer(shuffled, 1) @@ -481,8 +481,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper } test("native shuffle: round robin partitioning") { - withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_NATIVE_ROUND_ROBIN_PARTITIONING_ENABLED.key -> "true") { withParquetTable((0 until 100).map(i => (i, (i + 1).toLong, s"str$i")), "tbl") { val df = sql("SELECT * FROM tbl") @@ -498,8 +497,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper test("native shuffle: round robin deterministic behavior") { // Test that round robin produces consistent results across multiple executions - withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_NATIVE_ROUND_ROBIN_PARTITIONING_ENABLED.key -> "true") { withParquetTable((0 until 1000).map(i => (i, (i + 1).toLong, s"str$i")), "tbl") { val df = sql("SELECT * FROM tbl") @@ -514,8 +512,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper } test("native shuffle: round robin with filter") { - withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_NATIVE_ROUND_ROBIN_PARTITIONING_ENABLED.key -> "true") { withParquetTable((0 until 100).map(i => (i, (i + 1).toLong)), "tbl") { val df = sql("SELECT * FROM tbl") val shuffled = df @@ -577,8 +574,7 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper .write .parquet(dir.toString) } - withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_ENABLED.key -> "true") { + withSQLConf(CometConf.COMET_SHUFFLE_NATIVE_ROUND_ROBIN_PARTITIONING_ENABLED.key -> "true") { val testDF = spark.read.parquet(dir.toString).repartition(10) // Verify CometShuffleExchangeExec is in the plan assert( diff --git a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala index 2441cc0fa77..7cb6816298d 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala @@ -46,7 +46,7 @@ class CometWindowExecSuite extends CometTestBase { pos: Position): Unit = { super.test(testName, testTags: _*) { withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_EXEC_WINDOW_ENABLED.key -> "true", "spark.comet.operator.WindowExec.allowIncompatible" -> "true", "spark.comet.explain.fallback.enabled" -> "true", @@ -120,7 +120,7 @@ class CometWindowExecSuite extends CometTestBase { test("lead/lag should return the default value if the offset row does not exist") { withSQLConf( CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { checkSparkAnswerAndOperator(sql(""" |SELECT @@ -342,7 +342,7 @@ class CometWindowExecSuite extends CometTestBase { test("Windows support") { Seq("true", "false").foreach(aqeEnabled => withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> aqeEnabled) { withParquetTable((0 until 10).map(i => (i, 10 - i)), "t1") { // TODO: test nulls val aggregateFunctions = diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index 93af47a6059..8825b8af097 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -660,7 +660,7 @@ class CometExecRuleSuite extends CometTestBase { // Both aggregates fall back to JVM as in the prior test, but the revert optimization is // disabled, so the shuffle should still be wrapped in CometColumnarShuffle. withSQLConf( - CometConf.COMET_EXEC_SHUFFLE_REVERT_REDUNDANT_COLUMNAR_ENABLED.key -> "false", + CometConf.COMET_SHUFFLE_REVERT_REDUNDANT_COLUMNAR_ENABLED.key -> "false", CometConf.COMET_ENABLE_PARTIAL_HASH_AGGREGATE.key -> "false", CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") { val transformedPlan = applyCometExecRule(sparkPlan) diff --git a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala index fa5f368e338..a926b018d3d 100644 --- a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala @@ -142,7 +142,7 @@ class CometPluginsDefaultSuite extends CometTestBase { conf.set("spark.executor.memoryOverheadFactor", "0.5") conf.set("spark.plugins", "org.apache.spark.CometPlugin") conf.set("spark.comet.enabled", "true") - conf.set("spark.comet.exec.shuffle.enabled", "true") + conf.set("spark.comet.shuffle.enabled", "true") conf.set("spark.comet.exec.onHeap.enabled", "true") conf } @@ -171,7 +171,7 @@ class CometPluginsNonOverrideSuite extends CometTestBase { conf.set("spark.executor.memoryOverheadFactor", "0.5") conf.set("spark.plugins", "org.apache.spark.CometPlugin") conf.set("spark.comet.enabled", "true") - conf.set("spark.comet.exec.shuffle.enabled", "false") + conf.set("spark.comet.shuffle.enabled", "false") conf.set("spark.comet.exec.enabled", "false") conf.set("spark.comet.exec.onHeap.enabled", "true") conf @@ -200,7 +200,7 @@ class CometPluginsUnifiedModeOverrideSuite extends CometTestBase { conf.set("spark.comet.enabled", "true") conf.set("spark.memory.offHeap.enabled", "true") conf.set("spark.memory.offHeap.size", "2G") - conf.set("spark.comet.exec.shuffle.enabled", "true") + conf.set("spark.comet.shuffle.enabled", "true") conf.set("spark.comet.exec.enabled", "true") conf.set("spark.comet.memory.overhead.factor", "0.5") conf diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala index b8e2d2c9825..a82d6f99083 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala @@ -186,7 +186,7 @@ class CometTPCDSQuerySuite conf.set(CometConf.COMET_ENABLED.key, "true") conf.set(CometConf.COMET_EXEC_ENABLED.key, "true") conf.set(CometConf.COMET_NATIVE_SCAN_ENABLED.key, "true") - conf.set(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true") + conf.set(CometConf.COMET_SHUFFLE_ENABLED.key, "true") conf.set(CometConf.COMET_ONHEAP_MEMORY_OVERHEAD.key, "15g") conf.set(CometConf.COMET_EXPLAIN_TRANSFORMATIONS.key, "true") conf.set("spark.sql.adaptive.coalescePartitions.enabled", "true") diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala index 99184b97f2c..0e2fd64683e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala @@ -172,7 +172,7 @@ class CometTPCDSQueryTestSuite extends QueryTest with TPCDSBase with CometSQLQue CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") + CometConf.COMET_SHUFFLE_ENABLED.key -> "true") val sortMergeJoinConf: Map[String, String] = Map( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala index 608ecfe2c4a..0f942927091 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala @@ -89,7 +89,7 @@ class CometTPCHQuerySuite extends QueryTest with TPCBase with ShimCometTPCHQuery conf.set(CometConf.COMET_ENABLED.key, "true") conf.set(CometConf.COMET_EXEC_ENABLED.key, "true") conf.set(CometConf.COMET_NATIVE_SCAN_ENABLED.key, "true") - conf.set(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true") + conf.set(CometConf.COMET_SHUFFLE_ENABLED.key, "true") conf.set(MEMORY_OFFHEAP_ENABLED.key, "true") conf.set(MEMORY_OFFHEAP_SIZE.key, "2g") } diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryListBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryListBase.scala index 94915d0b829..044ed3190cc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryListBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryListBase.scala @@ -86,7 +86,7 @@ trait CometTPCQueryListBase withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", // Lower bloom filter thresholds to allows us to simulate the plan produced at larger scale. "spark.sql.optimizer.runtime.bloomFilter.creationSideThreshold" -> "1MB", "spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold" -> "1MB") { diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala index ac34151d0e6..07231142e21 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala @@ -80,7 +80,7 @@ abstract class CometTestBase conf.set(CometConf.COMET_ENABLED.key, "true") conf.set(CometConf.COMET_ONHEAP_ENABLED.key, "true") conf.set(CometConf.COMET_EXEC_ENABLED.key, "true") - conf.set(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true") + conf.set(CometConf.COMET_SHUFFLE_ENABLED.key, "true") conf.set(CometConf.COMET_SPARK_TO_ARROW_ENABLED.key, "true") conf.set(CometConf.COMET_NATIVE_SCAN_ENABLED.key, "true") conf.set(CometConf.COMET_PARQUET_UNSIGNED_SMALL_INT_CHECK.key, "false") @@ -1171,7 +1171,7 @@ abstract class CometTestBase df: DataFrame, cometExchangeNum: Int, native: Boolean): Seq[CometShuffleExchangeExec] = { - if (CometConf.COMET_EXEC_SHUFFLE_ENABLED.get()) { + if (CometConf.COMET_SHUFFLE_ENABLED.get()) { val sparkPlan = stripAQEPlan(df.queryExecution.executedPlan) val cometShuffleExecs = sparkPlan.collect { case b: CometShuffleExchangeExec => b } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala index f458d84a3f2..860e6abb4f6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala @@ -46,7 +46,7 @@ object CometExecBenchmark extends CometBenchmarkBase { .set( "spark.shuffle.manager", "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") - .set("spark.comet.columnar.shuffle.spill.threshold", "30000") + .set("spark.comet.shuffle.jvm.spillThreshold", "30000") val sparkSession = SparkSession.builder .config(conf) @@ -125,7 +125,7 @@ object CometExecBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { spark.sql( "SELECT (SELECT max(col1) AS parquetV1Table FROM parquetV1Table) AS a, " + diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala index a8961457ed9..e13cfca353b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala @@ -53,7 +53,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { .set( "spark.shuffle.manager", "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") - .set("spark.comet.columnar.shuffle.spill.threshold", "30000") + .set("spark.comet.shuffle.jvm.spillThreshold", "30000") val sparkSession = SparkSession.builder .config(conf) @@ -93,7 +93,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { spark .sql( s"SELECT ARRAY_REPEAT(CAST(1 AS ${dataType.sql}), 10) AS c1 FROM parquetV1Table") @@ -106,8 +106,8 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", - CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> "1.0", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> "1.0", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { spark .sql( @@ -147,7 +147,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { spark .sql( s"SELECT STRUCT(CAST(c1 AS ${dataType.sql})," + @@ -162,8 +162,8 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", - CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> "1.0", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> "1.0", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { spark .sql( @@ -204,7 +204,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { spark .sql("select c1 from parquetV1Table") .repartition(partitionNum, Column("c1")) @@ -216,8 +216,8 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", - CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> "1.0", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> "1.0", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { spark .sql("select c1 from parquetV1Table") @@ -230,8 +230,8 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", - CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> "2.0", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> "2.0", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { spark .sql("select c1 from parquetV1Table") @@ -244,8 +244,8 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", - CometConf.COMET_SHUFFLE_PREFER_DICTIONARY_RATIO.key -> "1000000000.0", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_JVM_PREFER_DICTIONARY_RATIO.key -> "1000000000.0", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { spark .sql("select c1 from parquetV1Table") @@ -297,7 +297,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { spark .sql("select c1 from parquetV1Table") .repartition(partitionNum, Column("c1")) @@ -309,7 +309,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { spark .sql("select c1 from parquetV1Table") @@ -322,7 +322,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { spark .sql("select c1 from parquetV1Table") .repartition(partitionNum, Column("c1")) @@ -366,7 +366,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { spark .sql(s"select $columns from parquetV1Table") .repartition(partitionNum, Column("c1")) @@ -378,7 +378,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { spark .sql(s"select $columns from parquetV1Table") @@ -391,7 +391,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { spark .sql(s"select $columns from parquetV1Table") @@ -436,7 +436,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { spark .sql(s"select $columns from parquetV1Table") .repartitionByRange(partitionNum, Column("c1")) @@ -448,7 +448,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { spark .sql(s"select $columns from parquetV1Table") @@ -461,7 +461,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "native") { spark .sql(s"select $columns from parquetV1Table") @@ -498,7 +498,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "false") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "false") { spark .sql(sql) .repartition(partitionNum) @@ -511,7 +511,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> shuffle) { spark .sql(sql) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index 117ce67b2e4..f94a425fbf8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -280,7 +280,7 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED.key -> "true", // as well as for v1.4/q9, v1.4/q44, v2.7.0/q6, v2.7.0/q64 CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true", @@ -317,7 +317,7 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa conf.set(CometConf.COMET_ENABLED.key, "true") conf.set(CometConf.COMET_EXEC_ENABLED.key, "true") conf.set(CometConf.COMET_ONHEAP_MEMORY_OVERHEAD.key, "1g") - conf.set(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true") + conf.set(CometConf.COMET_SHUFFLE_ENABLED.key, "true") conf.set(CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key, "true") new TestSparkSession(new SparkContext("local[1]", this.getClass.getCanonicalName, conf)) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala b/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala index 05a9a1eca25..1423981b1c4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala @@ -354,7 +354,7 @@ class ParquetEncryptionITCase extends CometTestBase with SQLTestUtils { "org.apache.parquet.crypto.keytools.mocks.InMemoryKMS", InMemoryKMS.KEY_LIST_PROPERTY_NAME -> s"footerKey: ${footerKey}, key1: ${key1}, key2: ${key2}", - CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { + CometConf.COMET_SHUFFLE_ENABLED.key -> "true") { // Write first file with key1 val inputDF1 = spark