Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4961,8 +4961,6 @@ mod tests {
};
let left_list = literals(&[0.0, 1.0, 2.0, 3.0])?;
let right_list = literals(&[-0.0, 4.0, 5.0, 6.0])?;
let retains_short_lists =
matches!(data_type, DataType::Float32 | DataType::Float64);
let schema = Arc::new(Schema::new(vec![Field::new("x", data_type, false)]));
let batch = RecordBatch::try_new(schema, vec![values])?;
let schema = batch.schema().to_dfschema_ref()?;
Expand All @@ -4987,7 +4985,7 @@ mod tests {
(left(true).or(right(true)), vec![false, false]),
] {
let simplified = simplifier.simplify(expr.clone())?;
if list_len > THRESHOLD_INLINE_INLIST || retains_short_lists {
if list_len > THRESHOLD_INLINE_INLIST {
// Floating-point literals must bypass structural set rewrites.
assert_eq!(simplified, expr);
} else {
Expand Down
30 changes: 20 additions & 10 deletions datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use super::THRESHOLD_INLINE_INLIST;

use arrow::datatypes::DataType;
use datafusion_common::tree_node::{Transformed, TreeNodeRewriter};
use datafusion_common::{DFSchema, Result};
use datafusion_common::{DFSchema, Result, ScalarValue};
use datafusion_expr::expr::InList;
use datafusion_expr::{Expr, ExprSchemable};

Expand All @@ -32,15 +32,25 @@ impl<'a> ShortenInListSimplifier<'a> {
let Ok(data_type) = expr.get_type(self.schema) else {
return false;
};
matches!(
data_type,
DataType::Int32
| DataType::Int64
| DataType::UInt64
| DataType::Float32
| DataType::Float64
| DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16)
) && list.iter().all(|expr| expr.as_literal().is_some())
let is_float = matches!(data_type, DataType::Float32 | DataType::Float64);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is treating the symptom rather than the problem --- specifically that the short InList implementation uses bit equality for floats.

(is_float
|| matches!(
data_type,
DataType::Int32
| DataType::Int64
| DataType::UInt64
| DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16)
))
&& list.iter().all(|expr| match expr.as_literal() {
// The membership filters compare floats by bit pattern, so
// -0.0 and +0.0 would not match each other, while `=` treats
// them as equal. Keep the comparison chain when a zero is in
// the list so both forms agree.
Some(ScalarValue::Float32(Some(v))) if is_float => *v != 0.0,
Some(ScalarValue::Float64(Some(v))) if is_float => *v != 0.0,
Some(_) => true,
None => false,
})
}
}

Expand Down
27 changes: 27 additions & 0 deletions datafusion/sqllogictest/test_files/in_list.slt
Original file line number Diff line number Diff line change
Expand Up @@ -1147,3 +1147,30 @@ true

statement ok
DROP TABLE in_list_nulls;

# Short float IN lists must treat -0.0 and +0.0 as equal, matching `=`
# https://github.com/apache/datafusion/issues/27121
statement ok
CREATE TABLE signed_zeros(id BIGINT, x DOUBLE, y FLOAT) AS VALUES
(1, -0.0, -0.0),
(2, 0.0, 0.0),
(3, 5.0, 5.0);

query IBBBBBB
SELECT
id,
x = 0.0,
x IN (0.0, 5.0),
x NOT IN (0.0, 5.0),
x IN (-0.0, 7.0),
y IN (CAST(0 AS FLOAT), CAST(9 AS FLOAT)),
y NOT IN (CAST(0 AS FLOAT), CAST(9 AS FLOAT))
FROM signed_zeros
ORDER BY id;
----
1 true true false true true false
2 true true false true true false
3 false true false false false true

statement ok
DROP TABLE signed_zeros;