From 7c8ec5db90ad69f23313d3903b65b915d36e2d77 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 18 Sep 2026 08:46:54 -0500 Subject: [PATCH 1/5] feat: add a `BetweenExpr` physical expression `BetweenExpr` evaluates its value one time and compares that single result against both bounds. The planner keeps its current rewrite into `value >= low AND value <= high`, so nothing uses `BetweenExpr` yet. The expression also gets protobuf support, so a plan that holds it still serializes. Co-Authored-By: Claude Fable 5.1 --- .../physical-expr/src/expressions/between.rs | 364 ++++++++++++++++++ .../physical-expr/src/expressions/mod.rs | 2 + .../proto-models/proto/datafusion.proto | 8 + .../proto-models/src/generated/pbjson.rs | 155 ++++++++ .../proto-models/src/generated/prost.rs | 15 +- .../proto/src/physical_plan/from_proto.rs | 7 +- datafusion/proto/tests/cases/plans/filters.rs | 22 +- 7 files changed, 568 insertions(+), 5 deletions(-) create mode 100644 datafusion/physical-expr/src/expressions/between.rs diff --git a/datafusion/physical-expr/src/expressions/between.rs b/datafusion/physical-expr/src/expressions/between.rs new file mode 100644 index 0000000000000..0cb10e6c9258c --- /dev/null +++ b/datafusion/physical-expr/src/expressions/between.rs @@ -0,0 +1,364 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! BETWEEN expression + +use std::fmt; +use std::hash::Hash; +use std::sync::Arc; + +use crate::PhysicalExpr; + +use arrow::array::BooleanArray; +use arrow::compute::kernels::boolean; +use arrow::datatypes::{DataType, Schema}; +use arrow::record_batch::RecordBatch; +use datafusion_common::{Result, ScalarValue, cast::as_boolean_array, internal_err}; +use datafusion_expr::{ColumnarValue, Operator}; +use datafusion_physical_expr_common::datum::apply_cmp; + +/// `BETWEEN` expression: `expr BETWEEN low AND high`. +/// +/// `expr BETWEEN low AND high` gives the same result as +/// `expr >= low AND expr <= high`. The planner uses the second form by default, +/// because the optimizer, the interval analysis and the pruning predicates all +/// understand plain binary comparisons. +/// +/// The second form names `expr` two times, so it also evaluates `expr` two +/// times. That is wrong when `expr` is volatile, because the two evaluations +/// give two different values. This expression evaluates `expr` one time and +/// compares that single result against both bounds. See +/// . +#[derive(Debug, Eq)] +pub struct BetweenExpr { + /// The value to test + expr: Arc, + /// `NOT BETWEEN` when true + negated: bool, + /// The lower bound, inclusive + low: Arc, + /// The upper bound, inclusive + high: Arc, +} + +// Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808 +impl PartialEq for BetweenExpr { + fn eq(&self, other: &Self) -> bool { + self.expr.eq(&other.expr) + && self.negated == other.negated + && self.low.eq(&other.low) + && self.high.eq(&other.high) + } +} + +impl Hash for BetweenExpr { + fn hash(&self, state: &mut H) { + self.expr.hash(state); + self.negated.hash(state); + self.low.hash(state); + self.high.hash(state); + } +} + +impl BetweenExpr { + /// Create a new `BETWEEN` expression + pub fn new( + expr: Arc, + negated: bool, + low: Arc, + high: Arc, + ) -> Self { + Self { + expr, + negated, + low, + high, + } + } + + /// The value to test + pub fn expr(&self) -> &Arc { + &self.expr + } + + /// True for `NOT BETWEEN` + pub fn negated(&self) -> bool { + self.negated + } + + /// The lower bound, inclusive + pub fn low(&self) -> &Arc { + &self.low + } + + /// The upper bound, inclusive + pub fn high(&self) -> &Arc { + &self.high + } + + /// Apply the `NOT` of `NOT BETWEEN` to an array result + fn negate_array(&self, result: BooleanArray) -> Result { + if self.negated { + Ok(boolean::not(&result)?) + } else { + Ok(result) + } + } + + /// Apply the `NOT` of `NOT BETWEEN` to a scalar result + fn negate_scalar(&self, result: Option) -> Option { + if self.negated { + result.map(|v| !v) + } else { + result + } + } +} + +impl fmt::Display for BetweenExpr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let negated = if self.negated { "NOT " } else { "" }; + write!( + f, + "{} {negated}BETWEEN {} AND {}", + self.expr, self.low, self.high + ) + } +} + +impl PhysicalExpr for BetweenExpr { + fn data_type(&self, _input_schema: &Schema) -> Result { + Ok(DataType::Boolean) + } + + fn nullable(&self, input_schema: &Schema) -> Result { + Ok(self.expr.nullable(input_schema)? + || self.low.nullable(input_schema)? + || self.high.nullable(input_schema)?) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + // Evaluate the value one time and compare that single result against + // both bounds. This is what makes a volatile value correct. + let value = self.expr.evaluate(batch)?; + let low = self.low.evaluate(batch)?; + let high = self.high.evaluate(batch)?; + + let ge_low = apply_cmp(Operator::GtEq, &value, &low)?; + let le_high = apply_cmp(Operator::LtEq, &value, &high)?; + + match (ge_low, le_high) { + (ColumnarValue::Scalar(ge_low), ColumnarValue::Scalar(le_high)) => { + let result = + kleene_and(as_boolean_scalar(&ge_low)?, as_boolean_scalar(&le_high)?); + Ok(ColumnarValue::Scalar(ScalarValue::Boolean( + self.negate_scalar(result), + ))) + } + (ge_low, le_high) => { + let ge_low = ge_low.into_array(batch.num_rows())?; + let le_high = le_high.into_array(batch.num_rows())?; + let result = boolean::and_kleene( + as_boolean_array(&ge_low)?, + as_boolean_array(&le_high)?, + )?; + Ok(ColumnarValue::Array(Arc::new(self.negate_array(result)?))) + } + } + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.expr, &self.low, &self.high] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(BetweenExpr::new( + Arc::clone(&children[0]), + self.negated, + Arc::clone(&children[1]), + Arc::clone(&children[2]), + ))) + } + + fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.expr.fmt_sql(f)?; + if self.negated { + write!(f, " NOT")?; + } + write!(f, " BETWEEN ")?; + self.low.fmt_sql(f)?; + write!(f, " AND ")?; + self.high.fmt_sql(f) + } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + Ok(Some(protobuf::PhysicalExprNode { + expr_id: None, + expr_type: Some(protobuf::physical_expr_node::ExprType::Between(Box::new( + protobuf::PhysicalBetweenNode { + expr: Some(Box::new(ctx.encode_child(&self.expr)?)), + negated: self.negated, + low: Some(Box::new(ctx.encode_child(&self.low)?)), + high: Some(Box::new(ctx.encode_child(&self.high)?)), + }, + ))), + })) + } +} + +#[cfg(feature = "proto")] +impl BetweenExpr { + /// Reconstruct a [`BetweenExpr`] from its protobuf representation. + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalExprNode, + ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>, + ) -> Result> { + use datafusion_physical_expr_common::expect_expr_variant; + use datafusion_proto_models::protobuf; + + let between = expect_expr_variant!( + node, + protobuf::physical_expr_node::ExprType::Between, + "BetweenExpr", + ); + + let expr = ctx.decode_required_expression( + between.expr.as_deref(), + "BetweenExpr", + "expr", + )?; + let low = + ctx.decode_required_expression(between.low.as_deref(), "BetweenExpr", "low")?; + let high = ctx.decode_required_expression( + between.high.as_deref(), + "BetweenExpr", + "high", + )?; + + Ok(Arc::new(BetweenExpr::new(expr, between.negated, low, high))) + } +} + +/// Kleene `AND` over two three valued booleans +fn kleene_and(lhs: Option, rhs: Option) -> Option { + match (lhs, rhs) { + (Some(false), _) | (_, Some(false)) => Some(false), + (Some(true), Some(true)) => Some(true), + _ => None, + } +} + +fn as_boolean_scalar(value: &ScalarValue) -> Result> { + match value { + ScalarValue::Boolean(v) => Ok(*v), + other => { + internal_err!("BETWEEN comparison returned a non boolean scalar: {other:?}") + } + } +} + +/// Create a `BETWEEN` expression +pub fn between( + expr: Arc, + negated: bool, + low: Arc, + high: Arc, +) -> Arc { + Arc::new(BetweenExpr::new(expr, negated, low, high)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expressions::{col, lit}; + + use arrow::array::Int32Array; + use arrow::datatypes::Field; + use datafusion_physical_expr_common::physical_expr::fmt_sql; + + fn test_batch() -> Result { + let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); + let a = Int32Array::from(vec![Some(1), Some(3), Some(5), Some(7), None]); + Ok(RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)])?) + } + + fn evaluate_between(negated: bool) -> Result>> { + let batch = test_batch()?; + let schema = batch.schema(); + let expr = between(col("a", &schema)?, negated, lit(3i32), lit(5i32)); + let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?; + Ok(as_boolean_array(&result)?.iter().collect()) + } + + #[test] + fn between_bounds_are_inclusive() -> Result<()> { + assert_eq!( + evaluate_between(false)?, + vec![Some(false), Some(true), Some(true), Some(false), None] + ); + Ok(()) + } + + #[test] + fn not_between_is_the_negation() -> Result<()> { + assert_eq!( + evaluate_between(true)?, + vec![Some(true), Some(false), Some(false), Some(true), None] + ); + Ok(()) + } + + #[test] + fn scalar_input_gives_a_scalar_result() -> Result<()> { + let batch = test_batch()?; + + let expr = between(lit(4i32), false, lit(3i32), lit(5i32)); + let ColumnarValue::Scalar(value) = expr.evaluate(&batch)? else { + panic!("expected a scalar result"); + }; + assert_eq!(value, ScalarValue::Boolean(Some(true))); + + let expr = between(lit(ScalarValue::Int32(None)), false, lit(3i32), lit(5i32)); + let ColumnarValue::Scalar(value) = expr.evaluate(&batch)? else { + panic!("expected a scalar result"); + }; + assert_eq!(value, ScalarValue::Boolean(None)); + Ok(()) + } + + #[test] + fn display_and_fmt_sql() -> Result<()> { + let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); + let expr = between(col("a", &schema)?, false, lit(3i32), lit(5i32)); + assert_eq!(expr.to_string(), "a@0 BETWEEN 3 AND 5"); + assert_eq!(fmt_sql(expr.as_ref()).to_string(), "a BETWEEN 3 AND 5"); + + let expr = between(col("a", &schema)?, true, lit(3i32), lit(5i32)); + assert_eq!(expr.to_string(), "a@0 NOT BETWEEN 3 AND 5"); + assert_eq!(fmt_sql(expr.as_ref()).to_string(), "a NOT BETWEEN 3 AND 5"); + Ok(()) + } +} diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs index 767c0b4ce40f3..73cc1ba12fe36 100644 --- a/datafusion/physical-expr/src/expressions/mod.rs +++ b/datafusion/physical-expr/src/expressions/mod.rs @@ -6,6 +6,7 @@ #[macro_use] mod binary; +mod between; mod case; mod cast; mod column; @@ -28,6 +29,7 @@ pub use crate::PhysicalSortExpr; /// Module with some convenient methods used in expression building pub use crate::aggregate::stats::StatsType; +pub use between::{BetweenExpr, between}; pub use binary::{BinaryExpr, binary, similar_to}; pub use case::{CaseExpr, case}; pub use cast::{CastExpr, cast}; diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index 6c4ef8f0a4b2f..31c8b7e3d866f 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -1117,6 +1117,7 @@ message PhysicalExprNode { PhysicalRangeExprNode range_expr = 27; PhysicalSqlSimilarToPatternNode sql_similar_to_pattern = 28; PhysicalLiteralNode literal_with_metadata = 29; + PhysicalBetweenNode between = 31; } } @@ -1125,6 +1126,13 @@ message PhysicalLiteralNode { map metadata = 2; } +message PhysicalBetweenNode { + PhysicalExprNode expr = 1; + bool negated = 2; + PhysicalExprNode low = 3; + PhysicalExprNode high = 4; +} + message PhysicalDynamicFilterNode { repeated PhysicalExprNode children = 1; repeated PhysicalExprNode remapped_children = 2; diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index 725cceeb75970..053cf969f5837 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -18793,6 +18793,148 @@ impl<'de> serde::Deserialize<'de> for PhysicalAliasNode { deserializer.deserialize_struct("datafusion.PhysicalAliasNode", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for PhysicalBetweenNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.expr.is_some() { + len += 1; + } + if self.negated { + len += 1; + } + if self.low.is_some() { + len += 1; + } + if self.high.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalBetweenNode", len)?; + if let Some(v) = self.expr.as_ref() { + struct_ser.serialize_field("expr", v)?; + } + if self.negated { + struct_ser.serialize_field("negated", &self.negated)?; + } + if let Some(v) = self.low.as_ref() { + struct_ser.serialize_field("low", v)?; + } + if let Some(v) = self.high.as_ref() { + struct_ser.serialize_field("high", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for PhysicalBetweenNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "expr", + "negated", + "low", + "high", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Expr, + Negated, + Low, + High, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "expr" => Ok(GeneratedField::Expr), + "negated" => Ok(GeneratedField::Negated), + "low" => Ok(GeneratedField::Low), + "high" => Ok(GeneratedField::High), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = PhysicalBetweenNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.PhysicalBetweenNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut expr__ = None; + let mut negated__ = None; + let mut low__ = None; + let mut high__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Expr => { + if expr__.is_some() { + return Err(serde::de::Error::duplicate_field("expr")); + } + expr__ = map_.next_value()?; + } + GeneratedField::Negated => { + if negated__.is_some() { + return Err(serde::de::Error::duplicate_field("negated")); + } + negated__ = Some(map_.next_value()?); + } + GeneratedField::Low => { + if low__.is_some() { + return Err(serde::de::Error::duplicate_field("low")); + } + low__ = map_.next_value()?; + } + GeneratedField::High => { + if high__.is_some() { + return Err(serde::de::Error::duplicate_field("high")); + } + high__ = map_.next_value()?; + } + } + } + Ok(PhysicalBetweenNode { + expr: expr__, + negated: negated__.unwrap_or_default(), + low: low__, + high: high__, + }) + } + } + deserializer.deserialize_struct("datafusion.PhysicalBetweenNode", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for PhysicalBinaryExprNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -19909,6 +20051,9 @@ impl serde::Serialize for PhysicalExprNode { physical_expr_node::ExprType::LiteralWithMetadata(v) => { struct_ser.serialize_field("literalWithMetadata", v)?; } + physical_expr_node::ExprType::Between(v) => { + struct_ser.serialize_field("between", v)?; + } } } struct_ser.end() @@ -19970,6 +20115,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode { "sqlSimilarToPattern", "literal_with_metadata", "literalWithMetadata", + "between", ]; #[allow(clippy::enum_variant_names)] @@ -20002,6 +20148,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode { RangeExpr, SqlSimilarToPattern, LiteralWithMetadata, + Between, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -20051,6 +20198,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode { "rangeExpr" | "range_expr" => Ok(GeneratedField::RangeExpr), "sqlSimilarToPattern" | "sql_similar_to_pattern" => Ok(GeneratedField::SqlSimilarToPattern), "literalWithMetadata" | "literal_with_metadata" => Ok(GeneratedField::LiteralWithMetadata), + "between" => Ok(GeneratedField::Between), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -20269,6 +20417,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode { return Err(serde::de::Error::duplicate_field("literalWithMetadata")); } expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::LiteralWithMetadata) +; + } + GeneratedField::Between => { + if expr_type__.is_some() { + return Err(serde::de::Error::duplicate_field("between")); + } + expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::Between) ; } } diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index 1034d0443dbce..1fe6ec080a596 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -1645,7 +1645,7 @@ pub struct PhysicalExprNode { pub expr_id: ::core::option::Option, #[prost( oneof = "physical_expr_node::ExprType", - tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29" + tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31" )] pub expr_type: ::core::option::Option, } @@ -1716,6 +1716,8 @@ pub mod physical_expr_node { ), #[prost(message, tag = "29")] LiteralWithMetadata(super::PhysicalLiteralNode), + #[prost(message, tag = "31")] + Between(::prost::alloc::boxed::Box), } } #[derive(Clone, PartialEq, ::prost::Message)] @@ -1729,6 +1731,17 @@ pub struct PhysicalLiteralNode { >, } #[derive(Clone, PartialEq, ::prost::Message)] +pub struct PhysicalBetweenNode { + #[prost(message, optional, boxed, tag = "1")] + pub expr: ::core::option::Option<::prost::alloc::boxed::Box>, + #[prost(bool, tag = "2")] + pub negated: bool, + #[prost(message, optional, boxed, tag = "3")] + pub low: ::core::option::Option<::prost::alloc::boxed::Box>, + #[prost(message, optional, boxed, tag = "4")] + pub high: ::core::option::Option<::prost::alloc::boxed::Box>, +} +#[derive(Clone, PartialEq, ::prost::Message)] pub struct PhysicalDynamicFilterNode { #[prost(message, repeated, tag = "1")] pub children: ::prost::alloc::vec::Vec, diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs index 734c1d71ff239..cc359738af471 100644 --- a/datafusion/proto/src/physical_plan/from_proto.rs +++ b/datafusion/proto/src/physical_plan/from_proto.rs @@ -22,9 +22,9 @@ use datafusion_physical_expr::{ HigherOrderFunctionExpr, PhysicalSortExpr, ScalarFunctionExpr, }; use datafusion_physical_plan::expressions::{ - BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, - LikeExpr, Literal, NegativeExpr, NotExpr, SqlSimilarToPattern, TryCastExpr, - UnKnownColumn, + BetweenExpr, BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, + IsNullExpr, LikeExpr, Literal, NegativeExpr, NotExpr, SqlSimilarToPattern, + TryCastExpr, UnKnownColumn, }; use datafusion_physical_plan::joins::HashExpr; use datafusion_physical_plan::proto::ExecutionPlanDecodeCtx; @@ -279,6 +279,7 @@ pub fn parse_physical_expr_with_converter( ExprType::IsNullExpr(_) => IsNullExpr::try_from_proto(proto, &decode_ctx)?, ExprType::IsNotNullExpr(_) => IsNotNullExpr::try_from_proto(proto, &decode_ctx)?, ExprType::NotExpr(_) => NotExpr::try_from_proto(proto, &decode_ctx)?, + ExprType::Between(_) => BetweenExpr::try_from_proto(proto, &decode_ctx)?, ExprType::Negative(_) => NegativeExpr::try_from_proto(proto, &decode_ctx)?, ExprType::InList(_) => InListExpr::try_from_proto(proto, &decode_ctx)?, ExprType::Case(_) => CaseExpr::try_from_proto(proto, &decode_ctx)?, diff --git a/datafusion/proto/tests/cases/plans/filters.rs b/datafusion/proto/tests/cases/plans/filters.rs index ab9530252be92..4a07e363c7315 100644 --- a/datafusion/proto/tests/cases/plans/filters.rs +++ b/datafusion/proto/tests/cases/plans/filters.rs @@ -9,7 +9,9 @@ use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::logical_expr::Operator; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::empty::EmptyExec; -use datafusion::physical_plan::expressions::{NotExpr, binary, col, in_list, lit}; +use datafusion::physical_plan::expressions::{ + NotExpr, between, binary, col, in_list, lit, +}; use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder}; use datafusion::prelude::SessionContext; use datafusion::scalar::ScalarValue; @@ -43,6 +45,24 @@ fn roundtrip_filter_with_not_and_in_list() -> Result<()> { )?)) } +#[test] +fn roundtrip_filter_with_between() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)])); + for negated in [false, true] { + let predicate = between( + col("a", &schema)?, + negated, + lit(ScalarValue::Int64(Some(1))), + lit(ScalarValue::Int64(Some(5))), + ); + roundtrip_test(Arc::new(FilterExec::try_new( + predicate, + Arc::new(EmptyExec::new(schema.clone())), + )?))?; + } + Ok(()) +} + #[test] fn roundtrip_filter_with_fetch() -> Result<()> { let field_a = Field::new("a", DataType::Boolean, false); From b0a40f2ac4b3a4d1f346dfaec63a49961105d962 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 18 Sep 2026 08:47:03 -0500 Subject: [PATCH 2/5] fix: evaluate a volatile `BETWEEN` value one time `x BETWEEN low AND high` became `x >= low AND x <= high` in two places: `SimplifyExpressions` and the physical planner. Both places name `x` two times, so both also evaluate `x` two times. A volatile `x` gives a different value each time, so the query returned the wrong rows. `SELECT count(*) FROM t WHERE random() BETWEEN 0.4 AND 0.6` over 100000 rows returned about 36000 rows, which is 0.6 * 0.6, and not the expected 20000. `SimplifyExpressions` now keeps `BETWEEN` together when the value is volatile, and the physical planner lowers that shape to `BetweenExpr`, which evaluates the value one time. A value that is not volatile keeps the rewrite into two comparisons. The optimizer, the interval analysis and the pruning predicates all understand plain binary comparisons, so the plans for those queries do not change. Co-Authored-By: Claude Fable 5.1 --- .../src/simplify_expressions/expr_simplifier.rs | 9 +++++++++ datafusion/physical-expr/src/planner.rs | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 248fc2fa1b38c..02a09424ad800 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -1649,6 +1649,15 @@ impl TreeNodeRewriter for Simplifier<'_> { // a between 3 and 5 --> a >= 3 AND a <=5 // a not between 3 and 5 --> a < 3 OR a > 5 + // + // The rewrite names `a` two times, so it also evaluates `a` two + // times. A volatile `a` gives a different value each time, so keep + // `BETWEEN` together and let the physical planner evaluate the + // value one time. See + // https://github.com/apache/datafusion/issues/25457 + Expr::Between(between) if between.expr.is_volatile() => { + Transformed::no(Expr::Between(between)) + } Expr::Between(between) => Transformed::yes(if between.negated { let l = *between.expr.clone(); let r = *between.expr; diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index de760ee68e156..d5eaf20718e93 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -451,6 +451,21 @@ pub fn create_physical_expr( planning_ctx, )?; + // A volatile value must be evaluated one time. The rewrite below + // names the value two times, so it also evaluates it two times and + // returns the wrong rows for `random() BETWEEN 0.4 AND 0.6`. + // `BetweenExpr` evaluates the value one time. See + // https://github.com/apache/datafusion/issues/25457 + // + // Every other value keeps the rewrite, because the optimizer, the + // interval analysis and the pruning predicates all understand plain + // binary comparisons but not `BetweenExpr`. + if expr.is_volatile() { + return Ok(expressions::between( + value_expr, *negated, low_expr, high_expr, + )); + } + // rewrite the between into the two binary operators let binary_expr = binary( binary( From efb95212306c189d6d13cb059f8bc268df81b1f3 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 18 Sep 2026 08:47:03 -0500 Subject: [PATCH 3/5] test: cover one evaluation per row for a volatile `BETWEEN` A volatile UDF counts the rows it produces values for. `BETWEEN` and `NOT BETWEEN` over 100 rows must give 100 evaluations. Before the fix they gave 200. The sqllogictest checks the row count of `random() BETWEEN 0.4 AND 0.6` over 100000 rows, and the plans of both a volatile and a plain `BETWEEN`. Co-Authored-By: Claude Fable 5.1 --- .../user_defined_scalar_functions.rs | 97 ++++++++++++++++++- datafusion/sqllogictest/test_files/expr.slt | 50 ++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs index b188f95417c22..ba69d0386636c 100644 --- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs @@ -5,10 +5,11 @@ use std::collections::HashMap; use std::hash::{Hash, Hasher}; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use arrow::array::{ - Array, ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, RecordBatch, - StringArray, builder::BooleanBuilder, cast::AsArray, + Array, ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, + RecordBatch, StringArray, builder::BooleanBuilder, cast::AsArray, }; use arrow::array::{Int8Array, UInt64Array, as_string_array, create_array, record_batch}; use arrow::compute::kernels::numeric::add; @@ -2341,3 +2342,95 @@ async fn test_extension_metadata_preserve_in_uncorrelated_scalar_subquery() -> R assert!(!batches.is_empty()); Ok(()) } + +/// Volatile UDF that counts how many values it produces. +/// +/// Each row gets the next value of the shared counter, so the counter also +/// gives the number of rows that the function was evaluated for. +#[derive(Debug)] +struct CountingVolatileUdf { + signature: Signature, + rows_evaluated: Arc, +} + +impl PartialEq for CountingVolatileUdf { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.rows_evaluated, &other.rows_evaluated) + } +} + +impl Eq for CountingVolatileUdf {} + +impl Hash for CountingVolatileUdf { + fn hash(&self, state: &mut H) { + self.signature.hash(state); + } +} + +impl ScalarUDFImpl for CountingVolatileUdf { + fn name(&self) -> &str { + "counting_volatile" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Int64) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let first = self + .rows_evaluated + .fetch_add(args.number_rows, Ordering::SeqCst) as i64; + let values = (0..args.number_rows as i64).map(|offset| first + offset); + Ok(ColumnarValue::Array(Arc::new( + Int64Array::from_iter_values(values), + ))) + } +} + +/// `BETWEEN` must evaluate a volatile value one time per row. +/// +/// See . +#[tokio::test] +async fn between_evaluates_a_volatile_value_one_time_per_row() -> Result<()> { + let row_count = 100; + + for predicate in [ + "counting_volatile() BETWEEN 0 AND 1000000", + "counting_volatile() NOT BETWEEN 0 AND 1000000", + ] { + let rows_evaluated = Arc::new(AtomicUsize::new(0)); + let udf = ScalarUDF::from(CountingVolatileUdf { + signature: Signature::exact(vec![], Volatility::Volatile), + rows_evaluated: Arc::clone(&rows_evaluated), + }); + + let ctx = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(1), + ); + ctx.register_udf(udf); + ctx.register_batch( + "t", + RecordBatch::try_from_iter(vec![( + "a", + Arc::new(Int64Array::from_iter_values(0..row_count as i64)) as ArrayRef, + )])?, + )?; + + ctx.sql(&format!("SELECT a FROM t WHERE {predicate}")) + .await? + .collect() + .await?; + + let evaluated = rows_evaluated.load(Ordering::SeqCst); + assert_eq!( + evaluated, row_count, + "{predicate} evaluated the volatile value {evaluated} times for {row_count} rows" + ); + } + + Ok(()) +} diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index 890603e18b7ef..1ea61b0754bae 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -2617,6 +2617,56 @@ query T select 'A' where null between 2 and null; ---- +# A volatile value must be evaluated one time per row. +# https://github.com/apache/datafusion/issues/25457 +statement ok +create table between_volatile as select value as a from generate_series(1, 100000); + +# One draw of random() keeps about 20000 of the 100000 rows. Two draws keep +# about 36000, which is 0.6 * 0.6. +query B +select count(*) between 18000 and 22000 from between_volatile where random() between 0.4 and 0.6; +---- +true + +# One draw keeps about 80000 rows. Two draws keep about 64000. +query B +select count(*) between 78000 and 82000 from between_volatile where random() not between 0.4 and 0.6; +---- +true + +statement ok +drop table between_volatile; + +statement ok +create table between_plan(a int) as values (1), (3), (5); + +# The plan keeps BETWEEN together, so random() is named one time. +query TT +explain select a from between_plan where random() between 0.4 and 0.6; +---- +logical_plan +01)Filter: random() BETWEEN Float64(0.4) AND Float64(0.6) +02)--TableScan: between_plan projection=[a] +physical_plan +01)FilterExec: random() BETWEEN 0.4 AND 0.6 +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +# A value that is not volatile keeps the expansion into two comparisons, which +# the optimizer and the pruning predicates understand. +query TT +explain select a from between_plan where a between 3 and 5; +---- +logical_plan +01)Filter: between_plan.a >= Int32(3) AND between_plan.a <= Int32(5) +02)--TableScan: between_plan projection=[a] +physical_plan +01)FilterExec: a@0 >= 3 AND a@0 <= 5 +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +statement ok +drop table between_plan; + ### Demonstrate use of E literals for escaping # should not have literal tab query T From 1c10c5f21ce7a54d65cf0f3fd445d49c08c1da8f Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:35:50 -0500 Subject: [PATCH 4/5] fix: skip the upper bound when no row passes the lower bound in BetweenExpr Also return an error instead of panicking on a wrong child count in with_new_children, and add a simplifier test for the volatile guard. Co-Authored-By: Claude Opus 5.5 --- .../simplify_expressions/expr_simplifier.rs | 13 +++ .../physical-expr/src/expressions/between.rs | 101 ++++++++++++------ .../physical-expr/src/expressions/binary.rs | 2 +- 3 files changed, 81 insertions(+), 35 deletions(-) diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 02a09424ad800..473b1323214a9 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -5302,6 +5302,19 @@ mod tests { ); } + #[test] + fn simplify_expr_between_keeps_a_volatile_value_together() { + // Expanding would evaluate the volatile value two times + let fun = Arc::new(ScalarUDF::new_from_impl(VolatileUdf::new())); + let volatile = Expr::ScalarFunction(ScalarFunction::new_udf(fun, vec![])); + for expr in [ + volatile.clone().between(lit(1i16), lit(2i16)), + volatile.not_between(lit(1i16), lit(2i16)), + ] { + assert_eq!(simplify(expr.clone()), expr); + } + } + #[test] fn test_like_and_ilike() { let null = lit(ScalarValue::Utf8(None)); diff --git a/datafusion/physical-expr/src/expressions/between.rs b/datafusion/physical-expr/src/expressions/between.rs index 0cb10e6c9258c..21784fa7b1efd 100644 --- a/datafusion/physical-expr/src/expressions/between.rs +++ b/datafusion/physical-expr/src/expressions/between.rs @@ -22,12 +22,14 @@ use std::hash::Hash; use std::sync::Arc; use crate::PhysicalExpr; +use crate::expressions::binary::is_all_false; -use arrow::array::BooleanArray; use arrow::compute::kernels::boolean; use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; -use datafusion_common::{Result, ScalarValue, cast::as_boolean_array, internal_err}; +use datafusion_common::{ + Result, ScalarValue, cast::as_boolean_array, internal_datafusion_err, internal_err, +}; use datafusion_expr::{ColumnarValue, Operator}; use datafusion_physical_expr_common::datum::apply_cmp; @@ -110,21 +112,18 @@ impl BetweenExpr { &self.high } - /// Apply the `NOT` of `NOT BETWEEN` to an array result - fn negate_array(&self, result: BooleanArray) -> Result { - if self.negated { - Ok(boolean::not(&result)?) - } else { - Ok(result) + /// Apply the `NOT` of `NOT BETWEEN` to a result + fn negate(&self, result: ColumnarValue) -> Result { + if !self.negated { + return Ok(result); } - } - - /// Apply the `NOT` of `NOT BETWEEN` to a scalar result - fn negate_scalar(&self, result: Option) -> Option { - if self.negated { - result.map(|v| !v) - } else { - result + match result { + ColumnarValue::Array(array) => Ok(ColumnarValue::Array(Arc::new( + boolean::not(as_boolean_array(&array)?)?, + ))), + ColumnarValue::Scalar(scalar) => Ok(ColumnarValue::Scalar( + ScalarValue::Boolean(as_boolean_scalar(&scalar)?.map(|v| !v)), + )), } } } @@ -156,29 +155,35 @@ impl PhysicalExpr for BetweenExpr { // both bounds. This is what makes a volatile value correct. let value = self.expr.evaluate(batch)?; let low = self.low.evaluate(batch)?; - let high = self.high.evaluate(batch)?; - let ge_low = apply_cmp(Operator::GtEq, &value, &low)?; + + // Like `expr >= low AND expr <= high`, skip the upper bound when no + // row passes the lower bound. This also keeps a fallible upper bound + // (`1 / b` with `b = 0`) from raising an error the expansion would not. + if is_all_false(&ge_low) { + return self.negate(ge_low); + } + + let high = self.high.evaluate(batch)?; let le_high = apply_cmp(Operator::LtEq, &value, &high)?; - match (ge_low, le_high) { + let result = match (ge_low, le_high) { (ColumnarValue::Scalar(ge_low), ColumnarValue::Scalar(le_high)) => { - let result = - kleene_and(as_boolean_scalar(&ge_low)?, as_boolean_scalar(&le_high)?); - Ok(ColumnarValue::Scalar(ScalarValue::Boolean( - self.negate_scalar(result), + ColumnarValue::Scalar(ScalarValue::Boolean(kleene_and( + as_boolean_scalar(&ge_low)?, + as_boolean_scalar(&le_high)?, ))) } (ge_low, le_high) => { let ge_low = ge_low.into_array(batch.num_rows())?; let le_high = le_high.into_array(batch.num_rows())?; - let result = boolean::and_kleene( + ColumnarValue::Array(Arc::new(boolean::and_kleene( as_boolean_array(&ge_low)?, as_boolean_array(&le_high)?, - )?; - Ok(ColumnarValue::Array(Arc::new(self.negate_array(result)?))) + )?)) } - } + }; + self.negate(result) } fn children(&self) -> Vec<&Arc> { @@ -189,12 +194,14 @@ impl PhysicalExpr for BetweenExpr { self: Arc, children: Vec>, ) -> Result> { - Ok(Arc::new(BetweenExpr::new( - Arc::clone(&children[0]), - self.negated, - Arc::clone(&children[1]), - Arc::clone(&children[2]), - ))) + let [expr, low, high]: [Arc; 3] = + children.try_into().map_err(|children: Vec<_>| { + internal_datafusion_err!( + "BetweenExpr expects 3 children, got {}", + children.len() + ) + })?; + Ok(Arc::new(BetweenExpr::new(expr, self.negated, low, high))) } fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -293,7 +300,7 @@ pub fn between( #[cfg(test)] mod tests { use super::*; - use crate::expressions::{col, lit}; + use crate::expressions::{binary, col, lit}; use arrow::array::Int32Array; use arrow::datatypes::Field; @@ -331,6 +338,32 @@ mod tests { Ok(()) } + #[test] + fn upper_bound_is_skipped_when_no_row_passes_the_lower_bound() -> Result<()> { + let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); + let a = Int32Array::from(vec![1, 3, 5]); + let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?; + + // `1 / 0` fails if it is evaluated + let failing_high = binary(lit(1i32), Operator::Divide, lit(0i32), &schema)?; + assert!(failing_high.evaluate(&batch).is_err()); + + for (negated, expected) in [(false, false), (true, true)] { + let expr = between( + col("a", &schema)?, + negated, + lit(100i32), + Arc::clone(&failing_high), + ); + let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?; + assert_eq!( + as_boolean_array(&result)?.iter().collect::>(), + vec![Some(expected); 3] + ); + } + Ok(()) + } + #[test] fn scalar_input_gives_a_scalar_result() -> Result<()> { let batch = test_batch()?; diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index d3cb36e3f6ab9..7091ce66220e7 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -1500,7 +1500,7 @@ fn for_each_conjunct( } /// Whether every row is already `false`. -fn is_all_false(value: &ColumnarValue) -> bool { +pub(crate) fn is_all_false(value: &ColumnarValue) -> bool { match value { ColumnarValue::Array(array) => { let array = array.as_boolean(); From 63bb268e6a67a2436d52d1c81cfbaec2f2490962 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:10:34 -0500 Subject: [PATCH 5/5] chore: use the SPDX license header in between.rs Co-Authored-By: Claude Opus 5.5 --- .../physical-expr/src/expressions/between.rs | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/between.rs b/datafusion/physical-expr/src/expressions/between.rs index 21784fa7b1efd..c8fdcf86fa31e 100644 --- a/datafusion/physical-expr/src/expressions/between.rs +++ b/datafusion/physical-expr/src/expressions/between.rs @@ -1,19 +1,6 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements //! BETWEEN expression