Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions datafusion/substrait/src/logical_plan/consumer/expr/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ mod nested;
mod scalar_function;
mod singular_or_list;
mod subquery;
mod switch;
mod window_function;

pub use aggregate_function::*;
Expand All @@ -39,6 +40,7 @@ pub use nested::*;
pub use scalar_function::*;
pub use singular_or_list::*;
pub use subquery::*;
pub use switch::*;
pub use window_function::*;

use crate::extensions::Extensions;
Expand Down
66 changes: 66 additions & 0 deletions datafusion/substrait/src/logical_plan/consumer/expr/switch.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

use crate::logical_plan::consumer::{SubstraitConsumer, from_literal};
use datafusion::common::{DFSchema, substrait_err};
use datafusion::logical_expr::{Case, Expr};
use substrait::proto::expression::SwitchExpression;

/// Reads Substrait's [`SwitchExpression`] as a `CASE` over a base expression.
///
/// Substrait defines a [switch] as "an optimized form of a generic if
/// expression where all conditions are equality to the same value", which is
/// what `CASE <base> WHEN <value>` is, down to `CaseExpr` evaluating the base
/// once rather than once per arm.
///
/// [switch]: https://substrait.io/expressions/specialized_record_expressions/#switch-expression
pub async fn from_switch(
consumer: &impl SubstraitConsumer,
switch: &SwitchExpression,
input_schema: &DFSchema,
) -> datafusion::common::Result<Expr> {
let Some(base) = switch.r#match.as_ref() else {
return substrait_err!("Switch expression without a match expression");
};
let expr = Box::new(consumer.consume_expression(base, input_schema).await?);

let mut when_then_expr = Vec::with_capacity(switch.ifs.len());
for arm in &switch.ifs {
let (Some(value), Some(then)) = (arm.r#if.as_ref(), arm.then.as_ref()) else {
return substrait_err!(
"Switch expression arm needs both a value to match and a result"
);
};
when_then_expr.push((
Box::new(from_literal(consumer, value).await?),
Box::new(consumer.consume_expression(then, input_schema).await?),
));
}

let else_expr = match switch.r#else.as_ref() {
Some(e) => Some(Box::new(
consumer.consume_expression(e, input_schema).await?,
)),
None => None,
};

Ok(Expr::Case(Case {
expr: Some(expr),
when_then_expr,
else_expr,
}))
}
57 changes: 40 additions & 17 deletions datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,14 @@

use crate::logical_plan::consumer::{NameTracker, SubstraitConsumer};
use crate::logical_plan::consumer::{from_substrait_agg_func, from_substrait_sorts};
use crate::logical_plan::grouping_set::{
grouping_id_column, grouping_set_columns, grouping_set_ids, grouping_sets_of,
index_from_grouping_id, unique_grouping_set_index_name,
};
use datafusion::common::{Column, DFSchemaRef, internal_err, not_impl_err};
use datafusion::logical_expr::builder::project;
use datafusion::logical_expr::{
Aggregate, Expr, GroupingSet, LogicalPlan, LogicalPlanBuilder,
Aggregate, Expr, ExprSchemable, GroupingSet, LogicalPlan, LogicalPlanBuilder,
};
use substrait::proto::AggregateRel;
use substrait::proto::aggregate_function::AggregationInvocation;
Expand Down Expand Up @@ -125,33 +129,44 @@ pub async fn from_aggregate_rel(
.map(|e| name_tracker.get_uniquely_named_expr(e))
.collect::<Result<Vec<Expr>, _>>()?;

let set_ids = (agg.groupings.len() > 1)
.then(|| {
grouping_set_ids(
&grouping_set_columns(&group_exprs)?,
grouping_sets_of(&group_exprs)?,
)
})
.transpose()?;
let plan = input.aggregate(group_exprs, aggr_exprs)?.build()?;
if agg.groupings.len() > 1 {
reorder_grouping_set_output(plan, agg.measures.len())
} else {
Ok(plan)
match set_ids {
Some(set_ids) => grouping_set_output(plan, agg.measures.len(), &set_ids),
None => Ok(plan),
}
} else {
not_impl_err!("Aggregate without an input is not valid")
}
}

/// Reorders DataFusion's `[groups, grouping_id, measures]` aggregate schema to
/// Substrait's direct output order of `[groups, measures, grouping_id]`.
fn reorder_grouping_set_output(
/// Shapes DataFusion's `[groups, grouping_id, measures]` aggregate schema into
/// the direct output Substrait gives a multi-set aggregate:
/// `[groups, measures, grouping set index]`.
///
/// The trailing column is not DataFusion's `__grouping_id`. Substrait defines it
/// as "the zero-based index of the grouping set that yielded the record", while
/// `__grouping_id` packs a bitmask of the columns the set leaves out together
/// with an ordinal that separates repeated sets. Both identify the set that
/// produced a row, so the column is replaced here by an expression mapping one
/// to the other.
///
/// [Aggregate Operation]: https://substrait.io/relations/logical_relations/#aggregate-operation
fn grouping_set_output(
plan: LogicalPlan,
measure_count: usize,
set_ids: &[u64],
) -> datafusion::common::Result<LogicalPlan> {
let exprs: Vec<Expr> = {
let schema = plan.schema();
let Some(grouping_id_index) =
schema.index_of_column_by_name(None, Aggregate::INTERNAL_GROUPING_ID)
else {
return internal_err!(
"Grouping set aggregate schema is missing {}",
Aggregate::INTERNAL_GROUPING_ID
);
};
let (grouping_id_index, grouping_id) = grouping_id_column(schema)?;
if grouping_id_index + measure_count + 1 != schema.fields().len() {
return internal_err!(
"Grouping set aggregate schema has {} fields after {}, expected {} measures",
Expand All @@ -160,11 +175,19 @@ fn reorder_grouping_set_output(
measure_count
);
}
let grouping_id = Expr::Column(grouping_id);
let grouping_id_type = grouping_id.get_type(schema)?;
// A real column already named like the preferred alias - most likely
// a user column literally called `grouping_set_index` - would
// otherwise collide with it as a duplicate unqualified field.
let index_name = unique_grouping_set_index_name(schema);
let set_index = index_from_grouping_id(&grouping_id, &grouping_id_type, set_ids)?
Comment thread
namanjain24-sudo marked this conversation as resolved.
Resolved
Comment thread
namanjain24-sudo marked this conversation as resolved.
Resolved
.alias(index_name);

(0..grouping_id_index)
.chain(grouping_id_index + 1..schema.fields().len())
.chain(std::iter::once(grouping_id_index))
.map(|index| Expr::Column(Column::from(schema.qualified_field(index))))
.chain(std::iter::once(set_index))
.collect()
};
project(plan, exprs)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use super::{
from_field_reference, from_filter_rel, from_if_then, from_join_rel, from_literal,
from_nested, from_project_rel, from_read_rel, from_scalar_function, from_set_rel,
from_singular_or_list, from_sort_rel, from_subquery, from_substrait_rel,
from_substrait_rex, from_window_function,
from_substrait_rex, from_switch, from_window_function,
};
use crate::extensions::Extensions;
use crate::logical_plan::consumer::{
Expand Down Expand Up @@ -342,10 +342,10 @@ pub trait SubstraitConsumer: Send + Sync + Sized {

async fn consume_switch(
&self,
_expr: &SwitchExpression,
_input_schema: &DFSchema,
expr: &SwitchExpression,
input_schema: &DFSchema,
) -> datafusion::common::Result<Expr> {
not_impl_err!("Switch expression not supported")
from_switch(self, expr, input_schema).await
}

async fn consume_singular_or_list(
Expand Down
Loading
Loading