From a4774131cac215bfb8eb0e5444e91432458cdd7c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 14:05:30 -0700 Subject: [PATCH 01/10] Save --- .../apache/comet/CometExpressionSuite.scala | 262 ++++++++++-------- 1 file changed, 140 insertions(+), 122 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 99149d375a1..82682e372fe 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -59,7 +59,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { val DIVIDE_BY_ZERO_EXCEPTION_MSG = """Division by zero. Use `try_divide` to tolerate divisor being 0 and return NULL instead""" - test("sort floating point with negative zero") { + ignore("sort floating point with negative zero") { val schema = StructType( Seq( StructField("c0", DataTypes.FloatType, true), @@ -83,7 +83,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("sort array of floating point with negative zero") { + ignore("sort array of floating point with negative zero") { val schema = StructType( Seq( StructField("c0", DataTypes.createArrayType(DataTypes.FloatType), true), @@ -105,7 +105,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("sort struct containing floating point with negative zero") { + ignore("sort struct containing floating point with negative zero") { val schema = StructType( Seq( StructField( @@ -131,7 +131,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("compare true/false to negative zero") { + ignore("compare true/false to negative zero") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -147,7 +147,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("parquet default values") { + ignore("parquet default values") { withTable("t1") { sql("create table t1(col1 boolean) using parquet") sql("insert into t1 values(true)") @@ -156,7 +156,25 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("decimals divide by zero") { + + // spotless:off + test("Decimal random number tests repro") { + withTable("test") { + sql(s"create table test(a decimal(33, 29), b decimal(28, 17)) using parquet") + sql(s"insert into test values (-6788.53035340376888409034576923353, 70948216565.90127985418365471)") + withSQLConf( + "spark.comet.enabled" -> "true", + "spark.comet.explain.native.enabled" -> "true", + "spark.sql.decimalOperations.allowPrecisionLoss" -> "true") { + val df = sql(s"select a, b, a % b from test") + println(df.queryExecution.executedPlan) + df.collect() + } + } + } + // spotless:on + + ignore("decimals divide by zero") { Seq(true, false).foreach { dictionary => withSQLConf( SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key -> "false", @@ -176,7 +194,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("Integral Division Overflow Handling Matches Spark Behavior") { + ignore("Integral Division Overflow Handling Matches Spark Behavior") { withTable("t1") { val value = Long.MinValue sql("create table t1(c1 long, c2 short) using parquet") @@ -186,7 +204,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("basic data type support") { + ignore("basic data type support") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -200,7 +218,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("uint data type support") { + ignore("uint data type support") { Seq(true, false).foreach { dictionaryEnabled => // TODO: Once the question of what to get back from uint_8, uint_16 types is resolved, // we can also update this test to check for COMET_SCAN_ALLOW_INCOMPATIBLE=true @@ -239,7 +257,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("null literals") { + ignore("null literals") { val batchSize = 1000 Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -267,7 +285,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date and timestamp type literals") { + ignore("date and timestamp type literals") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -281,7 +299,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_add with int scalars") { + ignore("date_add with int scalars") { Seq(true, false).foreach { dictionaryEnabled => Seq("TINYINT", "SHORT", "INT").foreach { intType => withTempDir { dir => @@ -315,7 +333,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_add with int arrays") { + ignore("date_add with int arrays") { Seq(true, false).foreach { dictionaryEnabled => Seq("_2", "_3", "_4").foreach { intColumn => // tinyint, short, int columns withTempDir { dir => @@ -329,7 +347,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_sub with int scalars") { + ignore("date_sub with int scalars") { Seq(true, false).foreach { dictionaryEnabled => Seq("TINYINT", "SHORT", "INT").foreach { intType => withTempDir { dir => @@ -343,7 +361,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_sub with scalar overflow") { + ignore("date_sub with scalar overflow") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -363,7 +381,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_sub with int arrays") { + ignore("date_sub with int arrays") { Seq(true, false).foreach { dictionaryEnabled => Seq("_2", "_3", "_4").foreach { intColumn => // tinyint, short, int columns withTempDir { dir => @@ -377,7 +395,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("try_add") { + ignore("try_add") { val data = Seq((1, 1)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator(spark.sql(""" @@ -393,7 +411,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("try_subtract") { + ignore("try_subtract") { val data = Seq((1, 1)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator(spark.sql(""" @@ -409,7 +427,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("try_multiply") { + ignore("try_multiply") { val data = Seq((1, 1)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator(spark.sql(""" @@ -425,7 +443,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("try_divide") { + ignore("try_divide") { val data = Seq((15121991, 0)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator("SELECT try_divide(_1, _2) FROM tbl") @@ -442,7 +460,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("try_integral_divide overflow cases") { + ignore("try_integral_divide overflow cases") { val data = Seq((15121991, 0)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator("SELECT try_divide(_1, _2) FROM tbl") @@ -457,7 +475,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("dictionary arithmetic") { + ignore("dictionary arithmetic") { // TODO: test ANSI mode withSQLConf(SQLConf.ANSI_ENABLED.key -> "false", "parquet.enable.dictionary" -> "true") { withParquetTable((0 until 10).map(i => (i % 5, i % 3)), "tbl") { @@ -466,7 +484,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("dictionary arithmetic with scalar") { + ignore("dictionary arithmetic with scalar") { withSQLConf("parquet.enable.dictionary" -> "true") { withParquetTable((0 until 10).map(i => (i % 5, i % 3)), "tbl") { checkSparkAnswerAndOperator("SELECT _1 + 1, _1 - 1, _1 * 2, _1 / 2, _1 % 2 FROM tbl") @@ -474,7 +492,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("string type and substring") { + ignore("string type and substring") { withParquetTable((0 until 5).map(i => (i.toString, (i + 100).toString)), "tbl") { checkSparkAnswerAndOperator("SELECT _1, substring(_2, 2, 2) FROM tbl") checkSparkAnswerAndOperator("SELECT _1, substring(_2, 2, -2) FROM tbl") @@ -486,7 +504,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("substring with start < 1") { + ignore("substring with start < 1") { withTempPath { _ => withTable("t") { sql("create table t (col string) using parquet") @@ -497,7 +515,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("substring with dictionary") { + ignore("substring with dictionary") { val data = (0 until 1000) .map(_ % 5) // reduce value space to trigger dictionary encoding .map(i => (i.toString, (i + 100).toString)) @@ -506,7 +524,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("hour, minute, second") { + ignore("hour, minute, second") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "part-r-0.parquet") @@ -532,7 +550,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("time expressions folded on jvm") { + ignore("time expressions folded on jvm") { val ts = "1969-12-31 16:23:45" val functions = Map("hour" -> 16, "minute" -> 23, "second" -> 45) @@ -555,7 +573,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("hour on int96 timestamp column") { + ignore("hour on int96 timestamp column") { import testImplicits._ val N = 100 @@ -584,7 +602,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("cast timestamp and timestamp_ntz") { + ignore("cast timestamp and timestamp_ntz") { withSQLConf( SESSION_LOCAL_TIMEZONE.key -> "Asia/Kathmandu", CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true") { @@ -606,7 +624,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("cast timestamp and timestamp_ntz to string") { + ignore("cast timestamp and timestamp_ntz to string") { withSQLConf( SESSION_LOCAL_TIMEZONE.key -> "Asia/Kathmandu", CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true") { @@ -628,7 +646,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("cast timestamp and timestamp_ntz to long, date") { + ignore("cast timestamp and timestamp_ntz to long, date") { withSQLConf( SESSION_LOCAL_TIMEZONE.key -> "Asia/Kathmandu", CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true") { @@ -652,7 +670,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("trunc") { + ignore("trunc") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "date_trunc.parquet") @@ -666,7 +684,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("trunc with format array") { + ignore("trunc with format array") { val numRows = 1000 Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -685,7 +703,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_trunc") { + ignore("date_trunc") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "timestamp_trunc.parquet") @@ -720,7 +738,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_trunc with timestamp_ntz") { + ignore("date_trunc with timestamp_ntz") { withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -755,7 +773,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_trunc with format array") { + ignore("date_trunc with format array") { val numRows = 1000 Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -781,7 +799,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_trunc on int96 timestamp column") { + ignore("date_trunc on int96 timestamp column") { import testImplicits._ val N = 100 @@ -836,7 +854,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("charvarchar") { + ignore("charvarchar") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "char_tbl4" @@ -873,7 +891,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("char varchar over length values") { + ignore("char varchar over length values") { Seq("char", "varchar").foreach { typ => withTempPath { dir => withTable("t") { @@ -888,7 +906,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("like (LikeSimplification enabled)") { + ignore("like (LikeSimplification enabled)") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -916,7 +934,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("like with custom escape") { + ignore("like with custom escape") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -934,7 +952,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("rlike simple case") { + ignore("rlike simple case") { val table = "rlike_names" Seq(false, true).foreach { withDictionary => val data = Seq("James Smith", "Michael Rose", "Rames Rose", "Rames rose") ++ @@ -950,7 +968,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("withInfo") { + ignore("withInfo") { val table = "with_info" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -981,7 +999,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("rlike fallback for non scalar pattern") { + ignore("rlike fallback for non scalar pattern") { val table = "rlike_fallback" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -995,7 +1013,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("rlike whitespace") { + ignore("rlike whitespace") { val table = "rlike_whitespace" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -1029,7 +1047,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("rlike") { + ignore("rlike") { val table = "rlike_fuzz" val gen = new DataGenerator(new Random(42)) withTable(table) { @@ -1095,7 +1113,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("contains") { + ignore("contains") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -1111,7 +1129,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("startswith") { + ignore("startswith") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -1127,7 +1145,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("endswith") { + ignore("endswith") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -1143,7 +1161,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("add overflow (ANSI disable)") { + ignore("add overflow (ANSI disable)") { // Enabling ANSI will cause native engine failure, but as we cannot catch // native error now, we cannot test it here. withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { @@ -1153,7 +1171,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("divide by zero (ANSI disable)") { + ignore("divide by zero (ANSI disable)") { // Enabling ANSI will cause native engine failure, but as we cannot catch // native error now, we cannot test it here. withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { @@ -1166,7 +1184,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("decimals arithmetic and comparison") { + ignore("decimals arithmetic and comparison") { def makeDecimalRDD(num: Int, decimal: DecimalType, useDictionary: Boolean): DataFrame = { val div = if (useDictionary) 5 else num // narrow the space to make it dictionary encoded spark @@ -1237,7 +1255,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("scalar decimal arithmetic operations") { + ignore("scalar decimal arithmetic operations") { withTable("tbl") { withSQLConf(CometConf.COMET_ENABLED.key -> "true") { sql("CREATE TABLE tbl (a INT) USING PARQUET") @@ -1261,7 +1279,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("cast decimals to int") { + ignore("cast decimals to int") { Seq(16, 1024).foreach { batchSize => withSQLConf( CometConf.COMET_BATCH_SIZE.key -> batchSize.toString, @@ -1305,7 +1323,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { Double.PositiveInfinity, Double.NegativeInfinity) - test("various math scalar functions") { + ignore("various math scalar functions") { val data = doubleValues.map(n => (n, n)) withParquetTable(data, "tbl") { // expressions with single arg @@ -1357,7 +1375,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("disable expression using dynamic config") { + ignore("disable expression using dynamic config") { def countSparkProjectExec(plan: SparkPlan) = { plan.collect { case _: ProjectExec => true @@ -1376,7 +1394,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("enable incompat expression using dynamic config") { + ignore("enable incompat expression using dynamic config") { def countSparkProjectExec(plan: SparkPlan) = { plan.collect { case _: ProjectExec => true @@ -1393,15 +1411,15 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("signum") { + ignore("signum") { testDoubleScalarExpr("signum") } - test("expm1") { + ignore("expm1") { testDoubleScalarExpr("expm1") } - test("ceil and floor") { + ignore("ceil and floor") { Seq("true", "false").foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary) { withParquetTable( @@ -1449,7 +1467,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("round") { + ignore("round") { // https://github.com/apache/datafusion-comet/issues/1441 assume(!usingDataSourceExec) Seq(true, false).foreach { dictionaryEnabled => @@ -1499,7 +1517,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("md5") { + ignore("md5") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1513,7 +1531,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("hex") { + ignore("hex") { // https://github.com/apache/datafusion-comet/issues/1441 assume(!usingDataSourceExec) Seq(true, false).foreach { dictionaryEnabled => @@ -1531,7 +1549,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("unhex") { + ignore("unhex") { val table = "unhex_table" withTable(table) { sql(s"create table $table(col string) using parquet") @@ -1551,7 +1569,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("EqualNullSafe should preserve comet filter") { + ignore("EqualNullSafe should preserve comet filter") { Seq("true", "false").foreach(b => withParquetTable( data = (0 until 8).map(i => (i, if (i > 5) None else Some(i % 2 == 0))), @@ -1575,7 +1593,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { }) } - test("test in(set)/not in(set)") { + ignore("test in(set)/not in(set)") { Seq("100", "0").foreach { inSetThreshold => Seq(false, true).foreach { dictionary => withSQLConf( @@ -1601,7 +1619,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("not") { + ignore("not") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1614,7 +1632,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("negative") { + ignore("negative") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1627,7 +1645,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("basic arithmetic") { + ignore("basic arithmetic") { withSQLConf("parquet.enable.dictionary" -> "false") { withParquetTable((1 until 10).map(i => (i, i + 1)), "tbl", false) { checkSparkAnswerAndOperator("SELECT _1 + _2, _1 - _2, _1 * _2, _1 / _2, _1 % _2 FROM tbl") @@ -1647,7 +1665,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date partition column does not forget date type") { + ignore("date partition column does not forget date type") { withTable("t1") { sql("CREATE TABLE t1(flag LONG, cal_dt DATE) USING PARQUET PARTITIONED BY (cal_dt)") sql(""" @@ -1661,7 +1679,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("DatePart functions: Year/Month/DayOfMonth/DayOfWeek/DayOfYear/WeekOfYear/Quarter") { + ignore("DatePart functions: Year/Month/DayOfMonth/DayOfWeek/DayOfYear/WeekOfYear/Quarter") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1676,7 +1694,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("from_unixtime") { + ignore("from_unixtime") { Seq(false, true).foreach { dictionary => withSQLConf( "parquet.enable.dictionary" -> dictionary.toString, @@ -1715,7 +1733,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("Decimal binary ops multiply is aligned to Spark") { + ignore("Decimal binary ops multiply is aligned to Spark") { Seq(true, false).foreach { allowPrecisionLoss => withSQLConf( "spark.sql.decimalOperations.allowPrecisionLoss" -> allowPrecisionLoss.toString) { @@ -1772,7 +1790,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("Decimal random number tests") { + ignore("Decimal random number tests") { val rand = new scala.util.Random(42) def makeNum(p: Int, s: Int): String = { val int1 = rand.nextLong() @@ -1813,7 +1831,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("test cast utf8 to boolean as compatible with Spark") { + ignore("test cast utf8 to boolean as compatible with Spark") { def testCastedColumn(inputValues: Seq[String]): Unit = { val table = "test_table" withTable(table) { @@ -1836,7 +1854,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { testCastedColumn(inputValues = Seq("car", "Truck")) } - test("explain comet") { + ignore("explain comet") { withSQLConf( SQLConf.ANSI_ENABLED.key -> "false", SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", @@ -1885,7 +1903,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("explain: CollectLimit disabled") { + ignore("explain: CollectLimit disabled") { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", @@ -1911,7 +1929,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("hash functions") { + ignore("hash functions") { Seq(true, false).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1936,7 +1954,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("remainder function") { + ignore("remainder function") { def withAnsiMode(enabled: Boolean)(f: => Unit): Unit = { withSQLConf( SQLConf.ANSI_ENABLED.key -> enabled.toString, @@ -2009,7 +2027,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("hash functions with random input") { + ignore("hash functions with random input") { val dataGen = DataGenerator.DEFAULT // sufficient number of rows to create dictionary encoded ArrowArray. val randomNumRows = 1000 @@ -2046,7 +2064,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("hash function with decimal input") { + ignore("hash function with decimal input") { val testPrecisionScales: Seq[(Int, Int)] = Seq( (1, 0), (17, 2), @@ -2067,7 +2085,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("xxhash64 function with decimal input") { + ignore("xxhash64 function with decimal input") { val testPrecisionScales: Seq[(Int, Int)] = Seq( (1, 0), (17, 2), @@ -2088,7 +2106,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("unary negative integer overflow test") { + ignore("unary negative integer overflow test") { def withAnsiMode(enabled: Boolean)(f: => Unit): Unit = { withSQLConf( SQLConf.ANSI_ENABLED.key -> enabled.toString, @@ -2176,7 +2194,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("readSidePadding") { + ignore("readSidePadding") { // https://stackoverflow.com/a/46290728 val table = "test" withTable(table) { @@ -2190,7 +2208,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("isnan") { + ignore("isnan") { Seq("true", "false").foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary) { withParquetTable( @@ -2206,7 +2224,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("named_struct") { + ignore("named_struct") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -2221,7 +2239,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("named_struct with duplicate field names") { + ignore("named_struct with duplicate field names") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -2241,7 +2259,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("to_json") { + ignore("to_json") { // TODO fix for Spark 4.0.0 assume(!isSpark40Plus) Seq(true, false).foreach { dictionaryEnabled => @@ -2266,7 +2284,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("to_json escaping of field names and string values") { + ignore("to_json escaping of field names and string values") { // TODO fix for Spark 4.0.0 assume(!isSpark40Plus) val gen = new DataGenerator(new Random(42)) @@ -2295,7 +2313,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("to_json unicode") { + ignore("to_json unicode") { // TODO fix for Spark 4.0.0 assume(!isSpark40Plus) Seq(true, false).foreach { dictionaryEnabled => @@ -2320,7 +2338,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("struct and named_struct with dictionary") { + ignore("struct and named_struct with dictionary") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable( (0 until 100).map(i => @@ -2336,7 +2354,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("get_struct_field") { + ignore("get_struct_field") { Seq("", "parquet").foreach { v1List => withSQLConf( SQLConf.USE_V1_SOURCE_LIST.key -> v1List, @@ -2365,7 +2383,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("get_struct_field - select primitive fields") { + ignore("get_struct_field - select primitive fields") { val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() assume(!(scanImpl == CometConf.SCAN_AUTO && CometSparkSessionExtensions.isSpark40Plus)) withTempPath { dir => @@ -2390,7 +2408,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("get_struct_field - select subset of struct") { + ignore("get_struct_field - select subset of struct") { val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() assume(!(scanImpl == CometConf.SCAN_AUTO && CometSparkSessionExtensions.isSpark40Plus)) withTempPath { dir => @@ -2428,7 +2446,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("get_struct_field - read entire struct") { + ignore("get_struct_field - read entire struct") { val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() assume(!(scanImpl == CometConf.SCAN_AUTO && CometSparkSessionExtensions.isSpark40Plus)) withTempPath { dir => @@ -2532,7 +2550,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("get_struct_field with DataFusion ParquetExec - read entire struct") { + ignore("get_struct_field with DataFusion ParquetExec - read entire struct") { assume(usingDataSourceExec(conf)) withTempPath { dir => // create input file with Comet disabled @@ -2569,7 +2587,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("read array[int] from parquet") { + ignore("read array[int] from parquet") { assume(usingDataSourceExec(conf)) withTempPath { dir => @@ -2598,7 +2616,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("CreateArray") { + ignore("CreateArray") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -2620,7 +2638,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("ListExtract") { + ignore("ListExtract") { def assertBothThrow(df: DataFrame): Unit = { checkSparkAnswerMaybeThrows(df) match { case (Some(_), Some(_)) => () @@ -2689,7 +2707,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("GetArrayStructFields") { + ignore("GetArrayStructFields") { Seq(true, false).foreach { dictionaryEnabled => withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> SimplifyExtractValueOps.ruleName) { withTempDir { dir => @@ -2711,7 +2729,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("test integral divide") { + ignore("test integral divide") { // this test requires native_comet scan due to unsigned u8/u16 issue withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_COMET) { Seq(true, false).foreach { dictionaryEnabled => @@ -2759,7 +2777,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("ANSI support for add") { + ignore("ANSI support for add") { val data = Seq((Integer.MAX_VALUE, 1), (Integer.MIN_VALUE, -1)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2779,7 +2797,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("ANSI support for subtract") { + ignore("ANSI support for subtract") { val data = Seq((Integer.MIN_VALUE, 1)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2798,7 +2816,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("ANSI support for multiply") { + ignore("ANSI support for multiply") { val data = Seq((Integer.MAX_VALUE, 10)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2818,7 +2836,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("ANSI support for divide (division by zero)") { + ignore("ANSI support for divide (division by zero)") { val data = Seq((Integer.MIN_VALUE, 0)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2838,7 +2856,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("ANSI support for divide (division by zero) float division") { + ignore("ANSI support for divide (division by zero) float division") { val data = Seq((Float.MinPositiveValue, 0.0)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2858,7 +2876,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("ANSI support for integral divide (division by zero)") { + ignore("ANSI support for integral divide (division by zero)") { val data = Seq((Integer.MAX_VALUE, 0)) Seq("true", "false").foreach { p => withSQLConf(SQLConf.ANSI_ENABLED.key -> p) { @@ -2887,7 +2905,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("ANSI support for round function") { + ignore("ANSI support for round function") { Seq( ( Int.MaxValue, @@ -2922,7 +2940,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("test integral divide overflow for decimal") { + ignore("test integral divide overflow for decimal") { if (isSpark40Plus) { Seq(true, false) } else @@ -2956,7 +2974,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("rand expression with random parameters") { + ignore("rand expression with random parameters") { testOnShuffledRangeWithRandomParameters { df => val seed = Random.nextLong() val dfWithRandParameters = df.withColumn("rnd", rand(seed)) @@ -2968,7 +2986,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("randn expression with random parameters") { + ignore("randn expression with random parameters") { testOnShuffledRangeWithRandomParameters { df => val seed = Random.nextLong() val dfWithRandParameters = df.withColumn("randn", randn(seed)) @@ -2980,7 +2998,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("spark_partition_id expression on random dataset") { + ignore("spark_partition_id expression on random dataset") { testOnShuffledRangeWithRandomParameters { df => val dfWithRandParameters = df.withColumn("pid1", spark_partition_id()) @@ -2990,7 +3008,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("monotonically_increasing_id expression on random dataset") { + ignore("monotonically_increasing_id expression on random dataset") { testOnShuffledRangeWithRandomParameters { df => val dfWithRandParameters = df.withColumn("mid1", monotonically_increasing_id()) @@ -3000,7 +3018,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("multiple nondetermenistic expressions with shuffle") { + ignore("multiple nondetermenistic expressions with shuffle") { testOnShuffledRangeWithRandomParameters { df => val seed1 = Random.nextLong() val seed2 = Random.nextLong() @@ -3015,7 +3033,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("vectorized reader: missing all struct fields") { + ignore("vectorized reader: missing all struct fields") { Seq(true, false).foreach { offheapEnabled => withSQLConf( SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", @@ -3043,7 +3061,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("test length function") { + ignore("test length function") { withTable("t1") { sql( "create table t1 using parquet as select cast(id as string) as c1, cast(id as binary) as c2 from range(10)") @@ -3053,7 +3071,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("test concat function - strings") { + ignore("test concat function - strings") { withTable("t1") { sql( "create table t1 using parquet as select uuid() c1, uuid() c2, uuid() c3, uuid() c4, cast(null as string) c5 from range(10)") @@ -3067,7 +3085,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } // https://github.com/apache/datafusion-comet/issues/2647 - test("test concat function - arrays") { + ignore("test concat function - arrays") { withTable("t1") { sql( "create table t1 using parquet as select array(id, id+1) c1, array(id+2, id+3) c2, CAST(array() AS array) c3, CAST(array(null) as array) c4, cast(null as array) c5 from range(10)") @@ -3090,7 +3108,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } // https://github.com/apache/datafusion-comet/issues/2647 - test("test concat function - binary") { + ignore("test concat function - binary") { withTable("t1") { sql( "create table t1 using parquet as select cast(uuid() as binary) c1, cast(uuid() as binary) c2, cast(uuid() as binary) c3, cast(uuid() as binary) c4, cast(null as binary) c5 from range(10)") @@ -3113,7 +3131,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } // https://github.com/apache/datafusion-comet/issues/2813 - test("make decimal using DataFrame API - integer") { + ignore("make decimal using DataFrame API - integer") { withTable("t1") { sql("create table t1 using parquet as select 123456 as c1 from range(1)") @@ -3132,7 +3150,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("make decimal using DataFrame API - long") { + ignore("make decimal using DataFrame API - long") { withTable("t1") { sql("create table t1 using parquet as select cast(123456 as long) as c1 from range(1)") From 45e7fdbcb815aa9e0459d8f7104b680aee3e13a0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 14:15:31 -0700 Subject: [PATCH 02/10] cargo update --- native/Cargo.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index acdd2797601..56162f57691 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -388,9 +388,9 @@ checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" [[package]] name = "assertables" -version = "9.8.2" +version = "9.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59051ec02907378a67b0ba1b8631121f5388c8dbbb3cec8c749d8f93c2c3c211" +checksum = "cbada39b42413d4db3d9460f6e791702490c40f72924378a1b6fc1a4181188fd" [[package]] name = "async-channel" @@ -598,9 +598,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b5ce75405893cd713f9ab8e297d8e438f624dde7d706108285f7e17a25a180f" +checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" dependencies = [ "aws-lc-sys", "zeroize", @@ -608,9 +608,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "179c3777a8b5e70e90ea426114ffc565b2c1a9f82f6c4a0c5a34aa6ef5e781b6" +checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" dependencies = [ "cc", "cmake", @@ -1361,9 +1361,9 @@ checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "cmake" -version = "0.1.54" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +checksum = "b042e5d8a74ae91bb0961acd039822472ec99f8ab0948cbf6d1369588f8be586" dependencies = [ "cc", ] @@ -4909,9 +4909,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.25" +version = "0.12.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6eff9328d40131d43bd911d42d79eb6a47312002a4daefc9e37f17e74a7701a" +checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f" dependencies = [ "base64", "bytes", @@ -5004,9 +5004,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f08d6a905edb32d74a5d5737a0c9d7e950c312f3c46cb0ca0a2ca09ea11878a0" +checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" dependencies = [ "bytemuck", "byteorder", From 09821df75321babe616f49ce14593e054b146bf4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 14:17:28 -0700 Subject: [PATCH 03/10] scalastyle --- .../test/scala/org/apache/comet/CometExpressionSuite.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 82682e372fe..ba41f627518 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -157,6 +157,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } + // scalastyle:off // spotless:off test("Decimal random number tests repro") { withTable("test") { @@ -173,7 +174,8 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } // spotless:on - + // scalastyle:on + ignore("decimals divide by zero") { Seq(true, false).foreach { dictionary => withSQLConf( From 4939a852a0ecfa925365029cac90babd5ea50f44 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 14:20:33 -0700 Subject: [PATCH 04/10] format --- .../org/apache/comet/CometExpressionSuite.scala | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index ba41f627518..a2ef95093d7 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -156,25 +156,22 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - - // scalastyle:off - // spotless:off test("Decimal random number tests repro") { withTable("test") { - sql(s"create table test(a decimal(33, 29), b decimal(28, 17)) using parquet") - sql(s"insert into test values (-6788.53035340376888409034576923353, 70948216565.90127985418365471)") + sql("create table test(a decimal(33, 29), b decimal(28, 17)) using parquet") + sql( + "insert into test values (-6788.53035340376888409034576923353, " + + "70948216565.90127985418365471)") withSQLConf( "spark.comet.enabled" -> "true", "spark.comet.explain.native.enabled" -> "true", "spark.sql.decimalOperations.allowPrecisionLoss" -> "true") { - val df = sql(s"select a, b, a % b from test") + val df = sql("select a, b, a % b from test") println(df.queryExecution.executedPlan) df.collect() } } } - // spotless:on - // scalastyle:on ignore("decimals divide by zero") { Seq(true, false).foreach { dictionary => From 30d87fddae7bd4f8978fecc275153b4cf636d0f4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 16:33:59 -0700 Subject: [PATCH 05/10] fix --- native/Cargo.lock | 24 +++++++++---------- .../spark-expr/src/math_funcs/modulo_expr.rs | 8 ++++++- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 56162f57691..acdd2797601 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -388,9 +388,9 @@ checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" [[package]] name = "assertables" -version = "9.8.3" +version = "9.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbada39b42413d4db3d9460f6e791702490c40f72924378a1b6fc1a4181188fd" +checksum = "59051ec02907378a67b0ba1b8631121f5388c8dbbb3cec8c749d8f93c2c3c211" [[package]] name = "async-channel" @@ -598,9 +598,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.15.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" +checksum = "6b5ce75405893cd713f9ab8e297d8e438f624dde7d706108285f7e17a25a180f" dependencies = [ "aws-lc-sys", "zeroize", @@ -608,9 +608,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.35.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" +checksum = "179c3777a8b5e70e90ea426114ffc565b2c1a9f82f6c4a0c5a34aa6ef5e781b6" dependencies = [ "cc", "cmake", @@ -1361,9 +1361,9 @@ checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "cmake" -version = "0.1.56" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b042e5d8a74ae91bb0961acd039822472ec99f8ab0948cbf6d1369588f8be586" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] @@ -4909,9 +4909,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.26" +version = "0.12.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f" +checksum = "b6eff9328d40131d43bd911d42d79eb6a47312002a4daefc9e37f17e74a7701a" dependencies = [ "base64", "bytes", @@ -5004,9 +5004,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.3" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +checksum = "f08d6a905edb32d74a5d5737a0c9d7e950c312f3c46cb0ca0a2ca09ea11878a0" dependencies = [ "bytemuck", "byteorder", diff --git a/native/spark-expr/src/math_funcs/modulo_expr.rs b/native/spark-expr/src/math_funcs/modulo_expr.rs index 57deed72e0e..733d653d2c4 100644 --- a/native/spark-expr/src/math_funcs/modulo_expr.rs +++ b/native/spark-expr/src/math_funcs/modulo_expr.rs @@ -100,10 +100,16 @@ pub fn create_modulo_expr( SparkCastOptions::new_without_timezone(EvalMode::Legacy, false), )); + // The UDF's return type must match what Arrow's rem function will actually return. + // Since we're operating on Decimal256 inputs, rem will return Decimal256. + let decimal256_return_type = match &data_type { + DataType::Decimal128(p, s) => DataType::Decimal256(*p, *s), + other => other.clone(), + }; let modulo_scalar_func = create_modulo_scalar_function( left_256, right_256, - &data_type, + &decimal256_return_type, registry, fail_on_error, )?; From 3ff68bbe97bd0db208ac9dd895c009afc9593a2f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 16:35:29 -0700 Subject: [PATCH 06/10] reinstate tests --- .../apache/comet/CometExpressionSuite.scala | 282 +++++++++--------- 1 file changed, 141 insertions(+), 141 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index a2ef95093d7..0577ba6f4b9 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -59,7 +59,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { val DIVIDE_BY_ZERO_EXCEPTION_MSG = """Division by zero. Use `try_divide` to tolerate divisor being 0 and return NULL instead""" - ignore("sort floating point with negative zero") { + test("sort floating point with negative zero") { val schema = StructType( Seq( StructField("c0", DataTypes.FloatType, true), @@ -83,7 +83,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("sort array of floating point with negative zero") { + test("sort array of floating point with negative zero") { val schema = StructType( Seq( StructField("c0", DataTypes.createArrayType(DataTypes.FloatType), true), @@ -105,7 +105,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("sort struct containing floating point with negative zero") { + test("sort struct containing floating point with negative zero") { val schema = StructType( Seq( StructField( @@ -131,7 +131,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("compare true/false to negative zero") { + test("compare true/false to negative zero") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -147,7 +147,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("parquet default values") { + test("parquet default values") { withTable("t1") { sql("create table t1(col1 boolean) using parquet") sql("insert into t1 values(true)") @@ -156,24 +156,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("Decimal random number tests repro") { - withTable("test") { - sql("create table test(a decimal(33, 29), b decimal(28, 17)) using parquet") - sql( - "insert into test values (-6788.53035340376888409034576923353, " + - "70948216565.90127985418365471)") - withSQLConf( - "spark.comet.enabled" -> "true", - "spark.comet.explain.native.enabled" -> "true", - "spark.sql.decimalOperations.allowPrecisionLoss" -> "true") { - val df = sql("select a, b, a % b from test") - println(df.queryExecution.executedPlan) - df.collect() - } - } - } - - ignore("decimals divide by zero") { + test("decimals divide by zero") { Seq(true, false).foreach { dictionary => withSQLConf( SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key -> "false", @@ -193,7 +176,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("Integral Division Overflow Handling Matches Spark Behavior") { + test("Integral Division Overflow Handling Matches Spark Behavior") { withTable("t1") { val value = Long.MinValue sql("create table t1(c1 long, c2 short) using parquet") @@ -203,7 +186,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("basic data type support") { + test("basic data type support") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -217,7 +200,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("uint data type support") { + test("uint data type support") { Seq(true, false).foreach { dictionaryEnabled => // TODO: Once the question of what to get back from uint_8, uint_16 types is resolved, // we can also update this test to check for COMET_SCAN_ALLOW_INCOMPATIBLE=true @@ -256,7 +239,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("null literals") { + test("null literals") { val batchSize = 1000 Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -284,7 +267,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date and timestamp type literals") { + test("date and timestamp type literals") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -298,7 +281,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date_add with int scalars") { + test("date_add with int scalars") { Seq(true, false).foreach { dictionaryEnabled => Seq("TINYINT", "SHORT", "INT").foreach { intType => withTempDir { dir => @@ -313,7 +296,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } // TODO: https://github.com/apache/datafusion-comet/issues/2539 - ignore("date_add with scalar overflow") { + test("date_add with scalar overflow") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -332,7 +315,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date_add with int arrays") { + test("date_add with int arrays") { Seq(true, false).foreach { dictionaryEnabled => Seq("_2", "_3", "_4").foreach { intColumn => // tinyint, short, int columns withTempDir { dir => @@ -346,7 +329,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date_sub with int scalars") { + test("date_sub with int scalars") { Seq(true, false).foreach { dictionaryEnabled => Seq("TINYINT", "SHORT", "INT").foreach { intType => withTempDir { dir => @@ -360,7 +343,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date_sub with scalar overflow") { + test("date_sub with scalar overflow") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -380,7 +363,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date_sub with int arrays") { + test("date_sub with int arrays") { Seq(true, false).foreach { dictionaryEnabled => Seq("_2", "_3", "_4").foreach { intColumn => // tinyint, short, int columns withTempDir { dir => @@ -394,7 +377,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("try_add") { + test("try_add") { val data = Seq((1, 1)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator(spark.sql(""" @@ -410,7 +393,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("try_subtract") { + test("try_subtract") { val data = Seq((1, 1)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator(spark.sql(""" @@ -426,7 +409,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("try_multiply") { + test("try_multiply") { val data = Seq((1, 1)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator(spark.sql(""" @@ -442,7 +425,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("try_divide") { + test("try_divide") { val data = Seq((15121991, 0)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator("SELECT try_divide(_1, _2) FROM tbl") @@ -459,7 +442,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("try_integral_divide overflow cases") { + test("try_integral_divide overflow cases") { val data = Seq((15121991, 0)) withParquetTable(data, "tbl") { checkSparkAnswerAndOperator("SELECT try_divide(_1, _2) FROM tbl") @@ -474,7 +457,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("dictionary arithmetic") { + test("dictionary arithmetic") { // TODO: test ANSI mode withSQLConf(SQLConf.ANSI_ENABLED.key -> "false", "parquet.enable.dictionary" -> "true") { withParquetTable((0 until 10).map(i => (i % 5, i % 3)), "tbl") { @@ -483,7 +466,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("dictionary arithmetic with scalar") { + test("dictionary arithmetic with scalar") { withSQLConf("parquet.enable.dictionary" -> "true") { withParquetTable((0 until 10).map(i => (i % 5, i % 3)), "tbl") { checkSparkAnswerAndOperator("SELECT _1 + 1, _1 - 1, _1 * 2, _1 / 2, _1 % 2 FROM tbl") @@ -491,7 +474,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("string type and substring") { + test("string type and substring") { withParquetTable((0 until 5).map(i => (i.toString, (i + 100).toString)), "tbl") { checkSparkAnswerAndOperator("SELECT _1, substring(_2, 2, 2) FROM tbl") checkSparkAnswerAndOperator("SELECT _1, substring(_2, 2, -2) FROM tbl") @@ -503,7 +486,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("substring with start < 1") { + test("substring with start < 1") { withTempPath { _ => withTable("t") { sql("create table t (col string) using parquet") @@ -514,7 +497,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("substring with dictionary") { + test("substring with dictionary") { val data = (0 until 1000) .map(_ % 5) // reduce value space to trigger dictionary encoding .map(i => (i.toString, (i + 100).toString)) @@ -523,7 +506,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("hour, minute, second") { + test("hour, minute, second") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "part-r-0.parquet") @@ -549,7 +532,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("time expressions folded on jvm") { + test("time expressions folded on jvm") { val ts = "1969-12-31 16:23:45" val functions = Map("hour" -> 16, "minute" -> 23, "second" -> 45) @@ -572,7 +555,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("hour on int96 timestamp column") { + test("hour on int96 timestamp column") { import testImplicits._ val N = 100 @@ -601,7 +584,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("cast timestamp and timestamp_ntz") { + test("cast timestamp and timestamp_ntz") { withSQLConf( SESSION_LOCAL_TIMEZONE.key -> "Asia/Kathmandu", CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true") { @@ -623,7 +606,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("cast timestamp and timestamp_ntz to string") { + test("cast timestamp and timestamp_ntz to string") { withSQLConf( SESSION_LOCAL_TIMEZONE.key -> "Asia/Kathmandu", CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true") { @@ -645,7 +628,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("cast timestamp and timestamp_ntz to long, date") { + test("cast timestamp and timestamp_ntz to long, date") { withSQLConf( SESSION_LOCAL_TIMEZONE.key -> "Asia/Kathmandu", CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true") { @@ -669,7 +652,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("trunc") { + test("trunc") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "date_trunc.parquet") @@ -683,7 +666,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("trunc with format array") { + test("trunc with format array") { val numRows = 1000 Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -702,7 +685,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date_trunc") { + test("date_trunc") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "timestamp_trunc.parquet") @@ -737,7 +720,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date_trunc with timestamp_ntz") { + test("date_trunc with timestamp_ntz") { withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[Cast]) -> "true") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -772,7 +755,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date_trunc with format array") { + test("date_trunc with format array") { val numRows = 1000 Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => @@ -798,7 +781,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date_trunc on int96 timestamp column") { + test("date_trunc on int96 timestamp column") { import testImplicits._ val N = 100 @@ -853,7 +836,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("charvarchar") { + test("charvarchar") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "char_tbl4" @@ -890,7 +873,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("char varchar over length values") { + test("char varchar over length values") { Seq("char", "varchar").foreach { typ => withTempPath { dir => withTable("t") { @@ -905,7 +888,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("like (LikeSimplification enabled)") { + test("like (LikeSimplification enabled)") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -933,7 +916,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("like with custom escape") { + test("like with custom escape") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -951,7 +934,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("rlike simple case") { + test("rlike simple case") { val table = "rlike_names" Seq(false, true).foreach { withDictionary => val data = Seq("James Smith", "Michael Rose", "Rames Rose", "Rames rose") ++ @@ -967,7 +950,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("withInfo") { + test("withInfo") { val table = "with_info" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -998,7 +981,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("rlike fallback for non scalar pattern") { + test("rlike fallback for non scalar pattern") { val table = "rlike_fallback" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -1012,7 +995,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("rlike whitespace") { + test("rlike whitespace") { val table = "rlike_whitespace" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -1046,7 +1029,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("rlike") { + test("rlike") { val table = "rlike_fuzz" val gen = new DataGenerator(new Random(42)) withTable(table) { @@ -1112,7 +1095,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("contains") { + test("contains") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -1128,7 +1111,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("startswith") { + test("startswith") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -1144,7 +1127,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("endswith") { + test("endswith") { val table = "names" withTable(table) { sql(s"create table $table(id int, name varchar(20)) using parquet") @@ -1160,7 +1143,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("add overflow (ANSI disable)") { + test("add overflow (ANSI disable)") { // Enabling ANSI will cause native engine failure, but as we cannot catch // native error now, we cannot test it here. withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { @@ -1170,7 +1153,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("divide by zero (ANSI disable)") { + test("divide by zero (ANSI disable)") { // Enabling ANSI will cause native engine failure, but as we cannot catch // native error now, we cannot test it here. withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { @@ -1183,7 +1166,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("decimals arithmetic and comparison") { + test("decimals arithmetic and comparison") { def makeDecimalRDD(num: Int, decimal: DecimalType, useDictionary: Boolean): DataFrame = { val div = if (useDictionary) 5 else num // narrow the space to make it dictionary encoded spark @@ -1254,7 +1237,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("scalar decimal arithmetic operations") { + test("scalar decimal arithmetic operations") { withTable("tbl") { withSQLConf(CometConf.COMET_ENABLED.key -> "true") { sql("CREATE TABLE tbl (a INT) USING PARQUET") @@ -1278,7 +1261,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("cast decimals to int") { + test("cast decimals to int") { Seq(16, 1024).foreach { batchSize => withSQLConf( CometConf.COMET_BATCH_SIZE.key -> batchSize.toString, @@ -1322,7 +1305,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { Double.PositiveInfinity, Double.NegativeInfinity) - ignore("various math scalar functions") { + test("various math scalar functions") { val data = doubleValues.map(n => (n, n)) withParquetTable(data, "tbl") { // expressions with single arg @@ -1374,7 +1357,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("disable expression using dynamic config") { + test("disable expression using dynamic config") { def countSparkProjectExec(plan: SparkPlan) = { plan.collect { case _: ProjectExec => true @@ -1393,7 +1376,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("enable incompat expression using dynamic config") { + test("enable incompat expression using dynamic config") { def countSparkProjectExec(plan: SparkPlan) = { plan.collect { case _: ProjectExec => true @@ -1410,15 +1393,15 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("signum") { + test("signum") { testDoubleScalarExpr("signum") } - ignore("expm1") { + test("expm1") { testDoubleScalarExpr("expm1") } - ignore("ceil and floor") { + test("ceil and floor") { Seq("true", "false").foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary) { withParquetTable( @@ -1466,7 +1449,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("round") { + test("round") { // https://github.com/apache/datafusion-comet/issues/1441 assume(!usingDataSourceExec) Seq(true, false).foreach { dictionaryEnabled => @@ -1516,7 +1499,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("md5") { + test("md5") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1530,7 +1513,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("hex") { + test("hex") { // https://github.com/apache/datafusion-comet/issues/1441 assume(!usingDataSourceExec) Seq(true, false).foreach { dictionaryEnabled => @@ -1548,7 +1531,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("unhex") { + test("unhex") { val table = "unhex_table" withTable(table) { sql(s"create table $table(col string) using parquet") @@ -1568,7 +1551,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("EqualNullSafe should preserve comet filter") { + test("EqualNullSafe should preserve comet filter") { Seq("true", "false").foreach(b => withParquetTable( data = (0 until 8).map(i => (i, if (i > 5) None else Some(i % 2 == 0))), @@ -1592,7 +1575,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { }) } - ignore("test in(set)/not in(set)") { + test("test in(set)/not in(set)") { Seq("100", "0").foreach { inSetThreshold => Seq(false, true).foreach { dictionary => withSQLConf( @@ -1618,7 +1601,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("not") { + test("not") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1631,7 +1614,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("negative") { + test("negative") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1644,7 +1627,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("basic arithmetic") { + test("basic arithmetic") { withSQLConf("parquet.enable.dictionary" -> "false") { withParquetTable((1 until 10).map(i => (i, i + 1)), "tbl", false) { checkSparkAnswerAndOperator("SELECT _1 + _2, _1 - _2, _1 * _2, _1 / _2, _1 % _2 FROM tbl") @@ -1664,7 +1647,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("date partition column does not forget date type") { + test("date partition column does not forget date type") { withTable("t1") { sql("CREATE TABLE t1(flag LONG, cal_dt DATE) USING PARQUET PARTITIONED BY (cal_dt)") sql(""" @@ -1678,7 +1661,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("DatePart functions: Year/Month/DayOfMonth/DayOfWeek/DayOfYear/WeekOfYear/Quarter") { + test("DatePart functions: Year/Month/DayOfMonth/DayOfWeek/DayOfYear/WeekOfYear/Quarter") { Seq(false, true).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1693,7 +1676,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("from_unixtime") { + test("from_unixtime") { Seq(false, true).foreach { dictionary => withSQLConf( "parquet.enable.dictionary" -> dictionary.toString, @@ -1732,7 +1715,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("Decimal binary ops multiply is aligned to Spark") { + test("Decimal binary ops multiply is aligned to Spark") { Seq(true, false).foreach { allowPrecisionLoss => withSQLConf( "spark.sql.decimalOperations.allowPrecisionLoss" -> allowPrecisionLoss.toString) { @@ -1789,7 +1772,24 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("Decimal random number tests") { + test("Decimal module with Decimal256 intermediate type") { + withTable("test") { + sql("create table test(a decimal(33, 29), b decimal(28, 17)) using parquet") + sql( + "insert into test values (-6788.53035340376888409034576923353, " + + "70948216565.90127985418365471)") + withSQLConf( + "spark.comet.enabled" -> "true", + "spark.comet.explain.native.enabled" -> "true", + "spark.sql.decimalOperations.allowPrecisionLoss" -> "true") { + val df = sql("select a, b, a % b from test") + println(df.queryExecution.executedPlan) + df.collect() + } + } + } + + test("Decimal random number tests") { val rand = new scala.util.Random(42) def makeNum(p: Int, s: Int): String = { val int1 = rand.nextLong() @@ -1830,7 +1830,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("test cast utf8 to boolean as compatible with Spark") { + test("test cast utf8 to boolean as compatible with Spark") { def testCastedColumn(inputValues: Seq[String]): Unit = { val table = "test_table" withTable(table) { @@ -1853,7 +1853,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { testCastedColumn(inputValues = Seq("car", "Truck")) } - ignore("explain comet") { + test("explain comet") { withSQLConf( SQLConf.ANSI_ENABLED.key -> "false", SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", @@ -1902,7 +1902,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("explain: CollectLimit disabled") { + test("explain: CollectLimit disabled") { withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", @@ -1928,7 +1928,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("hash functions") { + test("hash functions") { Seq(true, false).foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { val table = "test" @@ -1953,7 +1953,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("remainder function") { + test("remainder function") { def withAnsiMode(enabled: Boolean)(f: => Unit): Unit = { withSQLConf( SQLConf.ANSI_ENABLED.key -> enabled.toString, @@ -2026,7 +2026,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("hash functions with random input") { + test("hash functions with random input") { val dataGen = DataGenerator.DEFAULT // sufficient number of rows to create dictionary encoded ArrowArray. val randomNumRows = 1000 @@ -2063,7 +2063,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("hash function with decimal input") { + test("hash function with decimal input") { val testPrecisionScales: Seq[(Int, Int)] = Seq( (1, 0), (17, 2), @@ -2084,7 +2084,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("xxhash64 function with decimal input") { + test("xxhash64 function with decimal input") { val testPrecisionScales: Seq[(Int, Int)] = Seq( (1, 0), (17, 2), @@ -2105,7 +2105,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("unary negative integer overflow test") { + test("unary negative integer overflow test") { def withAnsiMode(enabled: Boolean)(f: => Unit): Unit = { withSQLConf( SQLConf.ANSI_ENABLED.key -> enabled.toString, @@ -2193,7 +2193,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("readSidePadding") { + test("readSidePadding") { // https://stackoverflow.com/a/46290728 val table = "test" withTable(table) { @@ -2207,7 +2207,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("isnan") { + test("isnan") { Seq("true", "false").foreach { dictionary => withSQLConf("parquet.enable.dictionary" -> dictionary) { withParquetTable( @@ -2223,7 +2223,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("named_struct") { + test("named_struct") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -2238,7 +2238,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("named_struct with duplicate field names") { + test("named_struct with duplicate field names") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -2258,7 +2258,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("to_json") { + test("to_json") { // TODO fix for Spark 4.0.0 assume(!isSpark40Plus) Seq(true, false).foreach { dictionaryEnabled => @@ -2283,7 +2283,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("to_json escaping of field names and string values") { + test("to_json escaping of field names and string values") { // TODO fix for Spark 4.0.0 assume(!isSpark40Plus) val gen = new DataGenerator(new Random(42)) @@ -2312,7 +2312,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("to_json unicode") { + test("to_json unicode") { // TODO fix for Spark 4.0.0 assume(!isSpark40Plus) Seq(true, false).foreach { dictionaryEnabled => @@ -2337,7 +2337,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("struct and named_struct with dictionary") { + test("struct and named_struct with dictionary") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable( (0 until 100).map(i => @@ -2353,7 +2353,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("get_struct_field") { + test("get_struct_field") { Seq("", "parquet").foreach { v1List => withSQLConf( SQLConf.USE_V1_SOURCE_LIST.key -> v1List, @@ -2382,7 +2382,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("get_struct_field - select primitive fields") { + test("get_struct_field - select primitive fields") { val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() assume(!(scanImpl == CometConf.SCAN_AUTO && CometSparkSessionExtensions.isSpark40Plus)) withTempPath { dir => @@ -2407,7 +2407,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("get_struct_field - select subset of struct") { + test("get_struct_field - select subset of struct") { val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() assume(!(scanImpl == CometConf.SCAN_AUTO && CometSparkSessionExtensions.isSpark40Plus)) withTempPath { dir => @@ -2445,7 +2445,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("get_struct_field - read entire struct") { + test("get_struct_field - read entire struct") { val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() assume(!(scanImpl == CometConf.SCAN_AUTO && CometSparkSessionExtensions.isSpark40Plus)) withTempPath { dir => @@ -2487,7 +2487,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { // we want to make sure we enable the operator check and marking the test as ignore will make it // more obvious // - ignore(s"$testName - V2") { + test(s"$testName - V2") { withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { f } } } @@ -2549,7 +2549,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("get_struct_field with DataFusion ParquetExec - read entire struct") { + test("get_struct_field with DataFusion ParquetExec - read entire struct") { assume(usingDataSourceExec(conf)) withTempPath { dir => // create input file with Comet disabled @@ -2586,7 +2586,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("read array[int] from parquet") { + test("read array[int] from parquet") { assume(usingDataSourceExec(conf)) withTempPath { dir => @@ -2615,7 +2615,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("CreateArray") { + test("CreateArray") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -2637,7 +2637,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("ListExtract") { + test("ListExtract") { def assertBothThrow(df: DataFrame): Unit = { checkSparkAnswerMaybeThrows(df) match { case (Some(_), Some(_)) => () @@ -2706,7 +2706,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("GetArrayStructFields") { + test("GetArrayStructFields") { Seq(true, false).foreach { dictionaryEnabled => withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> SimplifyExtractValueOps.ruleName) { withTempDir { dir => @@ -2728,7 +2728,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("test integral divide") { + test("test integral divide") { // this test requires native_comet scan due to unsigned u8/u16 issue withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_COMET) { Seq(true, false).foreach { dictionaryEnabled => @@ -2776,7 +2776,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("ANSI support for add") { + test("ANSI support for add") { val data = Seq((Integer.MAX_VALUE, 1), (Integer.MIN_VALUE, -1)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2796,7 +2796,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("ANSI support for subtract") { + test("ANSI support for subtract") { val data = Seq((Integer.MIN_VALUE, 1)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2815,7 +2815,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("ANSI support for multiply") { + test("ANSI support for multiply") { val data = Seq((Integer.MAX_VALUE, 10)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2835,7 +2835,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("ANSI support for divide (division by zero)") { + test("ANSI support for divide (division by zero)") { val data = Seq((Integer.MIN_VALUE, 0)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2855,7 +2855,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("ANSI support for divide (division by zero) float division") { + test("ANSI support for divide (division by zero) float division") { val data = Seq((Float.MinPositiveValue, 0.0)) withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withParquetTable(data, "tbl") { @@ -2875,7 +2875,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("ANSI support for integral divide (division by zero)") { + test("ANSI support for integral divide (division by zero)") { val data = Seq((Integer.MAX_VALUE, 0)) Seq("true", "false").foreach { p => withSQLConf(SQLConf.ANSI_ENABLED.key -> p) { @@ -2904,7 +2904,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("ANSI support for round function") { + test("ANSI support for round function") { Seq( ( Int.MaxValue, @@ -2939,7 +2939,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("test integral divide overflow for decimal") { + test("test integral divide overflow for decimal") { if (isSpark40Plus) { Seq(true, false) } else @@ -2973,7 +2973,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("rand expression with random parameters") { + test("rand expression with random parameters") { testOnShuffledRangeWithRandomParameters { df => val seed = Random.nextLong() val dfWithRandParameters = df.withColumn("rnd", rand(seed)) @@ -2985,7 +2985,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("randn expression with random parameters") { + test("randn expression with random parameters") { testOnShuffledRangeWithRandomParameters { df => val seed = Random.nextLong() val dfWithRandParameters = df.withColumn("randn", randn(seed)) @@ -2997,7 +2997,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("spark_partition_id expression on random dataset") { + test("spark_partition_id expression on random dataset") { testOnShuffledRangeWithRandomParameters { df => val dfWithRandParameters = df.withColumn("pid1", spark_partition_id()) @@ -3007,7 +3007,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("monotonically_increasing_id expression on random dataset") { + test("monotonically_increasing_id expression on random dataset") { testOnShuffledRangeWithRandomParameters { df => val dfWithRandParameters = df.withColumn("mid1", monotonically_increasing_id()) @@ -3017,7 +3017,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("multiple nondetermenistic expressions with shuffle") { + test("multiple nondetermenistic expressions with shuffle") { testOnShuffledRangeWithRandomParameters { df => val seed1 = Random.nextLong() val seed2 = Random.nextLong() @@ -3032,7 +3032,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("vectorized reader: missing all struct fields") { + test("vectorized reader: missing all struct fields") { Seq(true, false).foreach { offheapEnabled => withSQLConf( SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", @@ -3060,7 +3060,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("test length function") { + test("test length function") { withTable("t1") { sql( "create table t1 using parquet as select cast(id as string) as c1, cast(id as binary) as c2 from range(10)") @@ -3070,7 +3070,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("test concat function - strings") { + test("test concat function - strings") { withTable("t1") { sql( "create table t1 using parquet as select uuid() c1, uuid() c2, uuid() c3, uuid() c4, cast(null as string) c5 from range(10)") @@ -3084,7 +3084,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } // https://github.com/apache/datafusion-comet/issues/2647 - ignore("test concat function - arrays") { + test("test concat function - arrays") { withTable("t1") { sql( "create table t1 using parquet as select array(id, id+1) c1, array(id+2, id+3) c2, CAST(array() AS array) c3, CAST(array(null) as array) c4, cast(null as array) c5 from range(10)") @@ -3107,7 +3107,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } // https://github.com/apache/datafusion-comet/issues/2647 - ignore("test concat function - binary") { + test("test concat function - binary") { withTable("t1") { sql( "create table t1 using parquet as select cast(uuid() as binary) c1, cast(uuid() as binary) c2, cast(uuid() as binary) c3, cast(uuid() as binary) c4, cast(null as binary) c5 from range(10)") @@ -3130,7 +3130,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } // https://github.com/apache/datafusion-comet/issues/2813 - ignore("make decimal using DataFrame API - integer") { + test("make decimal using DataFrame API - integer") { withTable("t1") { sql("create table t1 using parquet as select 123456 as c1 from range(1)") @@ -3149,7 +3149,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - ignore("make decimal using DataFrame API - long") { + test("make decimal using DataFrame API - long") { withTable("t1") { sql("create table t1 using parquet as select cast(123456 as long) as c1 from range(1)") From e6fd586a56b8d69e8a016ec89ffe72595679afa3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 16:37:27 -0700 Subject: [PATCH 07/10] revert --- .../test/scala/org/apache/comet/CometExpressionSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 0577ba6f4b9..978ca97ad77 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -296,7 +296,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } // TODO: https://github.com/apache/datafusion-comet/issues/2539 - test("date_add with scalar overflow") { + ignore("date_add with scalar overflow") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") @@ -2487,7 +2487,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { // we want to make sure we enable the operator check and marking the test as ignore will make it // more obvious // - test(s"$testName - V2") { + ignore(s"$testName - V2") { withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { f } } } From d694ecb3bc120887df36ec71ce6101e056ab2a2f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 16:38:01 -0700 Subject: [PATCH 08/10] typo --- .../src/test/scala/org/apache/comet/CometExpressionSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 978ca97ad77..4e6717f38e3 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1772,7 +1772,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("Decimal module with Decimal256 intermediate type") { + test("Decimal modulus with Decimal256 intermediate type") { withTable("test") { sql("create table test(a decimal(33, 29), b decimal(28, 17)) using parquet") sql( From 984b05b98dcc216b657b73e66ddb4efba34e54e3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 16:38:31 -0700 Subject: [PATCH 09/10] link to issue --- .../src/test/scala/org/apache/comet/CometExpressionSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 4e6717f38e3..7a64ecebd23 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1773,6 +1773,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("Decimal modulus with Decimal256 intermediate type") { + // regression test for https://github.com/apache/datafusion-comet/issues/2911 withTable("test") { sql("create table test(a decimal(33, 29), b decimal(28, 17)) using parquet") sql( @@ -1783,7 +1784,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { "spark.comet.explain.native.enabled" -> "true", "spark.sql.decimalOperations.allowPrecisionLoss" -> "true") { val df = sql("select a, b, a % b from test") - println(df.queryExecution.executedPlan) df.collect() } } From 1bcd039587b87f1c5253ed0669d10233b1035b83 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 16 Dec 2025 16:38:42 -0700 Subject: [PATCH 10/10] link to issue --- spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 7a64ecebd23..0352da7850c 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1781,7 +1781,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { "70948216565.90127985418365471)") withSQLConf( "spark.comet.enabled" -> "true", - "spark.comet.explain.native.enabled" -> "true", "spark.sql.decimalOperations.allowPrecisionLoss" -> "true") { val df = sql("select a, b, a % b from test") df.collect()