Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions datafusion/common/src/cse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use crate::tree_node::{
TreeNodeVisitor,
};
use indexmap::IndexMap;
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::hash::{BuildHasher, Hash, Hasher, RandomState};
use std::marker::PhantomData;
use std::sync::Arc;
Expand Down Expand Up @@ -176,6 +176,9 @@ pub trait CSEController {

/// Splits the children to normal and conditionally evaluated ones or returns `None`
/// if all are always evaluated.
///
/// The returned references must point to the children of `node` themselves. The two
/// lists can be in any order.
fn conditional_children(node: &Self::Node) -> Option<ChildrenList<&Self::Node>>;

// Returns true if a node is valid. If a node is invalid then it can't be eliminated.
Expand Down Expand Up @@ -363,14 +366,21 @@ where
// If we are already in a node that can short-circuit then start new
// traversals on its normal conditional children.
match C::conditional_children(node) {
Some((normal, conditional)) => {
normal
.into_iter()
.try_for_each(|n| n.visit(self).map(|_| ()))?;
self.conditional = true;
conditional
Some((_normal, conditional)) => {
// Visit the children in tree order, whatever order the
// controller lists them in. `CSERewriter` reads `id_array` in
// pre-order, so the indexes that this visitor assigns must
// follow the same order.
let conditional = conditional
.into_iter()
.try_for_each(|n| n.visit(self).map(|_| ()))?;
.map(std::ptr::from_ref)
.collect::<HashSet<_>>();
node.apply_children(|child| {
self.conditional =
conditional.contains(&std::ptr::from_ref(child));
child.visit(self)?;
Ok(TreeNodeRecursion::Continue)
})?;
self.conditional = false;

TreeNodeRecursion::Jump
Expand Down
231 changes: 225 additions & 6 deletions datafusion/optimizer/src/common_subexpr_eliminate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,10 @@ use datafusion_common::alias::AliasGenerator;
use datafusion_common::cse::{CSE, CSEController, FoundCommonNodes};
use datafusion_common::tree_node::{Transformed, TreeNode};
use datafusion_common::{Column, DFSchema, DFSchemaRef, Result, qualified_name};
use datafusion_expr::expr::{Alias, HigherOrderFunction, ScalarFunction};
use datafusion_expr::expr::{
AggregateFunction, AggregateFunctionParams, Alias, HigherOrderFunction,
ScalarFunction, WindowFunction, WindowFunctionParams,
};
use datafusion_expr::logical_plan::{
Aggregate, Filter, LogicalPlan, Projection, Sort, Window,
};
Expand Down Expand Up @@ -695,6 +698,62 @@ impl CSEController for ExprCSEController<'_> {
.chain(else_expr.iter().map(|e| e.as_ref()))
.collect(),
)),

// An aggregate function with a `FILTER` clause evaluates the filter for
// every input row, but it evaluates its arguments only for the rows that
// pass the filter. Extracting an argument subexpression into a projection
// below the aggregate would evaluate it for the rejected rows too, which
// can raise an error (for example a failed cast or a division by zero)
// that the filter must prevent. The `ORDER BY` expressions of the
// aggregate are conservatively handled like the arguments.
Expr::AggregateFunction(AggregateFunction {
params:
AggregateFunctionParams {
args,
filter: Some(filter),
order_by,
..
},
..
}) => Some((
vec![filter.as_ref()],
args.iter()
.chain(order_by.iter().map(|sort| &sort.expr))
.collect(),
)),

// The arguments of a window function with a `FILTER` clause are
// conservatively handled like the arguments of a filtered aggregate.
// The window executor does not guarantee the same today:
// `AggregateWindowExpr::get_result_column` evaluates the arguments
// before it applies the filter. This rule must not rely on that order,
// so it does not extract a subexpression that occurs only in such
// arguments. The `PARTITION BY` and `ORDER BY` expressions of the
// window are evaluated for every input row.
Expr::WindowFunction(window_function) => {
let WindowFunction {
params:
WindowFunctionParams {
args,
partition_by,
order_by,
filter: Some(filter),
..
},
..
} = window_function.as_ref()
else {
return None;
};
Some((
partition_by
.iter()
.chain(order_by.iter().map(|sort| &sort.expr))
.chain([filter.as_ref()])
.collect(),
args.iter().collect(),
))
}
_ => None,
}
}
Expand Down Expand Up @@ -846,9 +905,9 @@ mod test {
use arrow::datatypes::{DataType, Field, Schema};
use datafusion_expr::logical_plan::{JoinType, table_scan};
use datafusion_expr::{
AccumulatorFactoryFunction, AggregateUDF, ColumnarValue, ScalarFunctionArgs,
ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF, Volatility,
grouping_set, is_null, not,
AccumulatorFactoryFunction, AggregateUDF, ColumnarValue, ExprFunctionExt,
ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF,
Volatility, WindowFunctionDefinition, grouping_set, is_null, not,
};
use datafusion_expr::{lit, logical_plan::builder::LogicalPlanBuilder};

Expand All @@ -857,7 +916,7 @@ mod test {
use crate::optimizer::OptimizerContext;
use crate::test::udfs::leaf_udf_expr;
use crate::test::*;
use datafusion_expr::test::function_stub::{avg, sum};
use datafusion_expr::test::function_stub::{avg, count, sum, sum_udaf};
use datafusion_functions_window::row_number::row_number_udwf;

macro_rules! assert_optimized_plan_equal {
Expand Down Expand Up @@ -950,7 +1009,7 @@ mod test {
let return_type = DataType::UInt32;
let accumulator: AccumulatorFactoryFunction = Arc::new(|_| unimplemented!());
let udf_agg = |inner: Expr| {
Expr::AggregateFunction(datafusion_expr::expr::AggregateFunction::new_udf(
Expr::AggregateFunction(AggregateFunction::new_udf(
Arc::new(AggregateUDF::from(SimpleAggregateUDF::new_with_signature(
"my_agg",
Signature::exact(vec![DataType::UInt32], Volatility::Stable),
Expand Down Expand Up @@ -1952,4 +2011,164 @@ mod test {
"
)
}

#[test]
fn test_aggregate_filter_arguments() -> Result<()> {
let table_scan = test_table_scan()?;

// An aggregate evaluates its arguments only for the rows that pass its
// `FILTER`. Thus `a + b` must stay in the aggregates. The filter `c > 0` is
// evaluated for every row and is extracted.
let filter = col("c").gt(lit(0));
let plan = LogicalPlanBuilder::from(table_scan.clone())
.aggregate(
iter::empty::<Expr>(),
vec![
sum(col("a") + col("b")).filter(filter.clone()).build()?,
count(col("a") + col("b")).filter(filter.clone()).build()?,
],
)?
.build()?;

assert_optimized_plan_equal!(
plan,
@ "
Aggregate: groupBy=[[]], aggr=[[sum(test.a + test.b) FILTER (WHERE __common_expr_1 AS test.c > Int32(0)) AS sum(test.a + test.b) FILTER (WHERE test.c > Int32(0)), COUNT(test.a + test.b) FILTER (WHERE __common_expr_1 AS test.c > Int32(0)) AS COUNT(test.a + test.b) FILTER (WHERE test.c > Int32(0))]]
Projection: test.c > Int32(0) AS __common_expr_1, test.a, test.b, test.c
TableScan: test
"
)?;

// `a + b` is surely evaluated by the aggregate without a `FILTER`, so it is
// extracted.
let plan = LogicalPlanBuilder::from(table_scan.clone())
.aggregate(
iter::empty::<Expr>(),
vec![
sum(col("a") + col("b")).filter(filter.clone()).build()?,
count(col("a") + col("b")),
],
)?
.build()?;

assert_optimized_plan_equal!(
plan,
@ "
Aggregate: groupBy=[[]], aggr=[[sum(__common_expr_1 AS test.a + test.b) FILTER (WHERE test.c > Int32(0)), COUNT(__common_expr_1 AS test.a + test.b)]]
Projection: test.a + test.b AS __common_expr_1, test.a, test.b, test.c
TableScan: test
"
)?;

// `a + b` is surely evaluated by the filter, so it is extracted.
let plan = LogicalPlanBuilder::from(table_scan.clone())
.aggregate(
iter::empty::<Expr>(),
vec![
sum(col("a") + col("b"))
.filter((col("a") + col("b")).gt(lit(0)))
.build()?,
],
)?
.build()?;

assert_optimized_plan_equal!(
plan,
@ "
Aggregate: groupBy=[[]], aggr=[[sum(__common_expr_1 AS test.a + test.b) FILTER (WHERE __common_expr_1 AS test.a + test.b > Int32(0)) AS sum(test.a + test.b) FILTER (WHERE test.a + test.b > Int32(0))]]
Projection: test.a + test.b AS __common_expr_1, test.a, test.b, test.c
TableScan: test
"
)?;

// Identical aggregates with a `FILTER` are still deduplicated.
let plan = LogicalPlanBuilder::from(table_scan)
.aggregate(
iter::empty::<Expr>(),
vec![
sum(col("a") + col("b"))
.filter(filter.clone())
.build()?
.alias("s1"),
sum(col("a") + col("b")).filter(filter).build()?.alias("s2"),
],
)?
.build()?;

assert_optimized_plan_equal!(
plan,
@ "
Projection: __common_expr_2 AS s1, __common_expr_2 AS s2
Aggregate: groupBy=[[]], aggr=[[sum(test.a + test.b) FILTER (WHERE __common_expr_1) AS __common_expr_2]]
Projection: test.c > Int32(0) AS __common_expr_1, test.a, test.b, test.c
TableScan: test
"
)
}

#[test]
fn test_aggregate_filter_order_by() -> Result<()> {
let table_scan = test_table_scan()?;

// The `ORDER BY` expressions of an aggregate with a `FILTER` are handled
// like its arguments, so `a + b` is not extracted.
let filter = col("c").gt(lit(0));
let order_by = vec![(col("a") + col("b")).sort(true, false)];
let plan = LogicalPlanBuilder::from(table_scan)
.aggregate(
iter::empty::<Expr>(),
vec![
sum(col("a"))
.order_by(order_by.clone())
.filter(filter.clone())
.build()?,
sum(col("b")).order_by(order_by).filter(filter).build()?,
],
)?
.build()?;

assert_optimized_plan_equal!(
plan,
@ "
Aggregate: groupBy=[[]], aggr=[[sum(test.a) FILTER (WHERE __common_expr_1 AS test.c > Int32(0)) ORDER BY [test.a + test.b ASC NULLS LAST] AS sum(test.a) FILTER (WHERE test.c > Int32(0)) ORDER BY [test.a + test.b ASC NULLS LAST], sum(test.b) FILTER (WHERE __common_expr_1 AS test.c > Int32(0)) ORDER BY [test.a + test.b ASC NULLS LAST] AS sum(test.b) FILTER (WHERE test.c > Int32(0)) ORDER BY [test.a + test.b ASC NULLS LAST]]]
Projection: test.c > Int32(0) AS __common_expr_1, test.a, test.b, test.c
TableScan: test
"
)
}

#[test]
fn test_window_filter_arguments() -> Result<()> {
let table_scan = test_table_scan()?;

// CSE conservatively treats the arguments of a window function with a
// `FILTER` as conditionally evaluated, although the window executor
// evaluates them before it applies the filter today. The `PARTITION BY`
// expressions and the filter are evaluated for every input row.
let window_sum = |arg: Expr| {
Expr::from(WindowFunction::new(
WindowFunctionDefinition::AggregateUDF(sum_udaf()),
vec![arg],
))
.partition_by(vec![col("a") * col("b")])
.filter(col("c").gt(lit(0)))
.build()
};
let plan = LogicalPlanBuilder::from(table_scan)
.window(vec![
window_sum(col("a") + col("b"))?,
window_sum((col("a") + col("b")) * lit(2))?,
])?
.build()?;

assert_optimized_plan_equal!(
plan,
@ "
Projection: test.a, test.b, test.c, sum(test.a + test.b) FILTER (WHERE test.c > Int32(0)) PARTITION BY [test.a * test.b] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING, sum(test.a + test.b * Int32(2)) FILTER (WHERE test.c > Int32(0)) PARTITION BY [test.a * test.b] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
WindowAggr: windowExpr=[[sum(test.a + test.b) FILTER (WHERE __common_expr_2 AS test.c > Int32(0)) PARTITION BY [__common_expr_1 AS test.a * test.b] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS sum(test.a + test.b) FILTER (WHERE test.c > Int32(0)) PARTITION BY [test.a * test.b] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING, sum((test.a + test.b) * Int32(2)) FILTER (WHERE __common_expr_2 AS test.c > Int32(0)) PARTITION BY [__common_expr_1 AS test.a * test.b] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS sum(test.a + test.b * Int32(2)) FILTER (WHERE test.c > Int32(0)) PARTITION BY [test.a * test.b] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
Projection: test.a * test.b AS __common_expr_1, test.c > Int32(0) AS __common_expr_2, test.a, test.b, test.c
TableScan: test
"
)
}
}
8 changes: 3 additions & 5 deletions datafusion/sqllogictest/test_files/aggregates_simplify.slt
Original file line number Diff line number Diff line change
Expand Up @@ -226,13 +226,11 @@ query TT
EXPLAIN SELECT SUM(column1 + 1) FILTER (WHERE column1 > 1), SUM(column1 + 1) FILTER (WHERE column1 > 0) FROM sum_simplify_t;
----
logical_plan
01)Aggregate: groupBy=[[]], aggr=[[sum(__common_expr_1 AS sum_simplify_t.column1 + Int64(1)) FILTER (WHERE sum_simplify_t.column1 > Int64(1)), sum(__common_expr_1 AS sum_simplify_t.column1 + Int64(1)) FILTER (WHERE sum_simplify_t.column1 > Int64(0))]]
02)--Projection: sum_simplify_t.column1 + Int64(1) AS __common_expr_1, sum_simplify_t.column1
03)----TableScan: sum_simplify_t projection=[column1]
01)Aggregate: groupBy=[[]], aggr=[[sum(sum_simplify_t.column1 + Int64(1)) FILTER (WHERE sum_simplify_t.column1 > Int64(1)), sum(sum_simplify_t.column1 + Int64(1)) FILTER (WHERE sum_simplify_t.column1 > Int64(0))]]
02)--TableScan: sum_simplify_t projection=[column1]
physical_plan
01)AggregateExec: mode=Single, gby=[], aggr=[sum(sum_simplify_t.column1 + Int64(1)) FILTER (WHERE sum_simplify_t.column1 > Int64(1)), sum(sum_simplify_t.column1 + Int64(1)) FILTER (WHERE sum_simplify_t.column1 > Int64(0))]
02)--ProjectionExec: expr=[column1@0 + 1 as __common_expr_1, column1@0 as column1]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
02)--DataSourceExec: partitions=1, partition_sizes=[1]

# volatile aggregate arguments
query B
Expand Down
Loading
Loading