diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index af5ad6d43bdad..f918447a98070 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -4961,8 +4961,6 @@ mod tests { }; let left_list = literals(&[0.0, 1.0, 2.0, 3.0])?; let right_list = literals(&[-0.0, 4.0, 5.0, 6.0])?; - let retains_short_lists = - matches!(data_type, DataType::Float32 | DataType::Float64); let schema = Arc::new(Schema::new(vec![Field::new("x", data_type, false)])); let batch = RecordBatch::try_new(schema, vec![values])?; let schema = batch.schema().to_dfschema_ref()?; @@ -4987,7 +4985,7 @@ mod tests { (left(true).or(right(true)), vec![false, false]), ] { let simplified = simplifier.simplify(expr.clone())?; - if list_len > THRESHOLD_INLINE_INLIST || retains_short_lists { + if list_len > THRESHOLD_INLINE_INLIST { // Floating-point literals must bypass structural set rewrites. assert_eq!(simplified, expr); } else { diff --git a/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs index 32aba76d1605c..5c7a5352d32d4 100644 --- a/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs @@ -8,7 +8,7 @@ use super::THRESHOLD_INLINE_INLIST; use arrow::datatypes::DataType; use datafusion_common::tree_node::{Transformed, TreeNodeRewriter}; -use datafusion_common::{DFSchema, Result}; +use datafusion_common::{DFSchema, Result, ScalarValue}; use datafusion_expr::expr::InList; use datafusion_expr::{Expr, ExprSchemable}; @@ -32,15 +32,25 @@ impl<'a> ShortenInListSimplifier<'a> { let Ok(data_type) = expr.get_type(self.schema) else { return false; }; - matches!( - data_type, - DataType::Int32 - | DataType::Int64 - | DataType::UInt64 - | DataType::Float32 - | DataType::Float64 - | DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16) - ) && list.iter().all(|expr| expr.as_literal().is_some()) + let is_float = matches!(data_type, DataType::Float32 | DataType::Float64); + (is_float + || matches!( + data_type, + DataType::Int32 + | DataType::Int64 + | DataType::UInt64 + | DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16) + )) + && list.iter().all(|expr| match expr.as_literal() { + // The membership filters compare floats by bit pattern, so + // -0.0 and +0.0 would not match each other, while `=` treats + // them as equal. Keep the comparison chain when a zero is in + // the list so both forms agree. + Some(ScalarValue::Float32(Some(v))) if is_float => *v != 0.0, + Some(ScalarValue::Float64(Some(v))) if is_float => *v != 0.0, + Some(_) => true, + None => false, + }) } } diff --git a/datafusion/sqllogictest/test_files/in_list.slt b/datafusion/sqllogictest/test_files/in_list.slt index 8b9e9f58b255e..75965b65972df 100644 --- a/datafusion/sqllogictest/test_files/in_list.slt +++ b/datafusion/sqllogictest/test_files/in_list.slt @@ -1147,3 +1147,30 @@ true statement ok DROP TABLE in_list_nulls; + +# Short float IN lists must treat -0.0 and +0.0 as equal, matching `=` +# https://github.com/apache/datafusion/issues/27121 +statement ok +CREATE TABLE signed_zeros(id BIGINT, x DOUBLE, y FLOAT) AS VALUES + (1, -0.0, -0.0), + (2, 0.0, 0.0), + (3, 5.0, 5.0); + +query IBBBBBB +SELECT + id, + x = 0.0, + x IN (0.0, 5.0), + x NOT IN (0.0, 5.0), + x IN (-0.0, 7.0), + y IN (CAST(0 AS FLOAT), CAST(9 AS FLOAT)), + y NOT IN (CAST(0 AS FLOAT), CAST(9 AS FLOAT)) +FROM signed_zeros +ORDER BY id; +---- +1 true true false true true false +2 true true false true true false +3 false true false false false true + +statement ok +DROP TABLE signed_zeros;