diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rs index fc54710c97777..93957fc770262 100644 --- a/datafusion/optimizer/src/extract_leaf_expressions.rs +++ b/datafusion/optimizer/src/extract_leaf_expressions.rs @@ -55,17 +55,18 @@ //! [`OptimizeProjections`]: crate::optimize_projections::OptimizeProjections use indexmap::{IndexMap, IndexSet}; -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::sync::Arc; use datafusion_common::alias::AliasGenerator; -use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion_common::{Column, DFSchema, Result, qualified_name}; +use datafusion_common::tree_node::{ + Transformed, TransformedResult, TreeNode, TreeNodeRecursion, +}; +use datafusion_common::{Column, DFSchema, Result}; use datafusion_expr::logical_plan::LogicalPlan; use datafusion_expr::{Expr, ExpressionPlacement, Projection}; use crate::optimizer::ApplyOrder; -use crate::push_down_filter::replace_cols_by_name; use crate::utils::{ColumnReference, has_all_column_refs, schema_columns}; use crate::{OptimizerConfig, OptimizerRule}; @@ -91,22 +92,81 @@ fn has_extractable_expr(exprs: &[Expr]) -> bool { }) } -/// Returns the flat names of `plan`'s output columns whose defining expression +/// Resolves `col` to the index of the one field of `schema` that it names. +/// +/// This is the single place where the leaf rules turn a [`Column`] into a +/// field identity. Every later decision compares indices, so a bare `c` and a +/// qualified `t.c` that name the same field are the same column, and two +/// fields that share a name are never confused. Returns `None` when `col` is +/// not in `schema` or when the name is ambiguous there. +fn column_index(schema: &DFSchema, col: &Column) -> Option { + let mut matches = schema + .iter() + .enumerate() + .filter(|(_, (qualifier, field))| { + field.name() == &col.name + && match (&col.relation, qualifier) { + (Some(r), Some(q)) => r.resolved_eq(q), + (Some(_), None) => false, + (None, _) => true, + } + }) + .map(|(idx, _)| idx); + let idx = matches.next()?; + matches.next().is_none().then_some(idx) +} + +/// Replaces every column of `expr` that resolves in `schema` (see +/// [`column_index`]) with `replacement(idx)`. Columns that do not resolve are +/// left as they are. +fn rewrite_cols_by_index( + expr: Expr, + schema: &DFSchema, + replacement: impl Fn(usize) -> Result, +) -> Result { + expr.transform_up(|e| { + if let Expr::Column(col) = &e + && let Some(idx) = column_index(schema, col) + { + replacement(idx).map(Transformed::yes) + } else { + Ok(Transformed::no(e)) + } + }) + .data() +} + +/// Rewrites `expr` from `from` to `to`, two schemas with the same fields at +/// the same positions: every column that resolves in `from` becomes the column +/// at the same index of `to`, spelled the way `to` spells it. With `from` and +/// `to` the same schema, this moves `expr` into the canonical spelling of that +/// schema. +fn respell_cols(expr: Expr, from: &DFSchema, to: &DFSchema) -> Result { + rewrite_cols_by_index(expr, from, |idx| Ok(Expr::from(to.qualified_field(idx)))) +} + +/// Resolves `col` to the column of `schema` it names, spelled the way `schema` +/// spells it (see [`column_index`]). +fn resolve_against(schema: &DFSchema, col: &Column) -> Option { + column_index(schema, col).map(|idx| Column::from(schema.qualified_field(idx))) +} + +/// Returns the indices of `plan`'s output columns whose defining expression /// is volatile (e.g. `random()`). /// /// Only a [`LogicalPlan::Projection`] can define such a column: anywhere else /// the value has already been materialized by the projection that produced it, /// so referencing the column again does not re-evaluate anything. -fn volatile_output_columns(plan: &LogicalPlan) -> BTreeSet { +fn volatile_output_columns(plan: &LogicalPlan) -> BTreeSet { let LogicalPlan::Projection(projection) = plan else { return BTreeSet::new(); }; projection - .schema + .expr .iter() - .zip(projection.expr.iter()) + .enumerate() .filter(|(_, expr)| expr.is_volatile()) - .map(|((qualifier, field), _)| Column::from((qualifier, field)).flat_name()) + .map(|(idx, _)| idx) .collect() } @@ -116,7 +176,7 @@ fn volatile_output_columns(plan: &LogicalPlan) -> BTreeSet { /// When `input` is already a projection, [`build_extraction_projection_impl`] /// *merges* into it: every column reference in an extracted expression is /// replaced by that column's defining expression (see -/// [`build_projection_replace_map`]). Inlining a volatile definition makes the +/// [`projection_definition`]). Inlining a volatile definition makes the /// merged projection evaluate it a second, independent time, so the extracted /// value no longer matches the column it was derived from: /// @@ -136,10 +196,11 @@ fn would_duplicate_volatile<'a>( if volatile.is_empty() { return false; } + let schema = input.schema(); exprs.into_iter().any(|expr| { - expr.column_refs() - .iter() - .any(|col| volatile.contains(&col.flat_name())) + expr.column_refs().iter().any(|col| { + column_index(schema, col).is_some_and(|idx| volatile.contains(&idx)) + }) }) } @@ -312,11 +373,10 @@ fn extract_from_plan( .collect(); // Build per-input column sets for routing expressions to the correct input - let input_column_sets: Vec> = - input_schemas - .iter() - .map(|schema| schema_columns(schema.as_ref())) - .collect(); + let input_column_sets: Vec> = input_schemas + .iter() + .map(|schema| schema_columns(schema.as_ref())) + .collect(); // Transform expressions via map_expressions with routing let transformed = plan.map_expressions(|expr| { @@ -370,7 +430,7 @@ fn extract_from_plan( /// in both sides of a join). fn find_owning_input( expr: &Expr, - input_column_sets: &[std::collections::HashSet], + input_column_sets: &[HashSet], ) -> Option { let mut found = None; for (idx, cols) in input_column_sets.iter().enumerate() { @@ -390,7 +450,7 @@ fn find_owning_input( fn routing_extract( expr: Expr, extractors: &mut [LeafExpressionExtractor], - input_column_sets: &[std::collections::HashSet], + input_column_sets: &[HashSet], ) -> Result> { expr.transform_down(|e| { // Skip expressions already aliased with extracted expression pattern @@ -438,12 +498,11 @@ fn routing_extract( }) } -/// Rewrites extraction pairs and column references from one qualifier -/// space to another. +/// Rewrites extraction pairs and column references from one schema to +/// another that has the same fields at the same positions. /// -/// Builds a replacement map by zipping `from_schema` (whose qualifiers -/// currently appear in `pairs` / `columns`) with `to_schema` (the -/// qualifiers we want), then applies `replace_cols_by_name`. +/// Each column is resolved to its index in `from_schema` and replaced by the +/// column at the same index of `to_schema`. /// /// Used for SubqueryAlias (alias-space -> input-space) and Union /// (union output-space -> per-branch input-space). @@ -453,32 +512,20 @@ fn remap_pairs_and_columns( from_schema: &DFSchema, to_schema: &DFSchema, ) -> Result { - let mut replace_map = HashMap::with_capacity(from_schema.fields().len()); - for ((from_q, from_f), (to_q, to_f)) in from_schema.iter().zip(to_schema.iter()) { - replace_map.insert( - qualified_name(from_q, from_f.name()), - Expr::Column(Column::new(to_q.cloned(), to_f.name())), - ); - } let remapped_pairs: Vec<(Expr, String)> = pairs .iter() .map(|(expr, alias)| { Ok(( - replace_cols_by_name(expr.clone(), &replace_map)?, + respell_cols(expr.clone(), from_schema, to_schema)?, alias.clone(), )) }) .collect::>()?; let remapped_columns: IndexSet = columns .iter() - .filter_map(|col| { - let rewritten = - replace_cols_by_name(Expr::Column(col.clone()), &replace_map).ok()?; - if let Expr::Column(c) = rewritten { - Some(c) - } else { - Some(col.clone()) - } + .map(|col| match column_index(from_schema, col) { + Some(idx) => Column::from(to_schema.qualified_field(idx)), + None => col.clone(), }) .collect(); Ok(ExtractionTarget { @@ -500,20 +547,19 @@ struct ExtractionTarget { columns: IndexSet, } -/// Build a replacement map from a projection: output_column_name -> underlying_expr. +/// The expression that defines output field `idx` of `projection`, with its +/// columns spelled the way the projection's input schema spells them. /// /// This is used to resolve column references through a renaming projection. -/// For example, if a projection has `user AS x`, this maps `x` -> `col("user")`. -fn build_projection_replace_map(projection: &Projection) -> HashMap { - projection - .schema - .iter() - .zip(projection.expr.iter()) - .map(|((qualifier, field), expr)| { - let key = Column::from((qualifier, field)).flat_name(); - (key, expr.clone().unalias()) - }) - .collect() +/// For example, if a projection has `user AS x`, the field `x` maps to +/// `col("t.user")`. +fn projection_definition(projection: &Projection, idx: usize) -> Result { + let input_schema = projection.input.schema(); + respell_cols( + projection.expr[idx].clone().unalias(), + input_schema, + input_schema, + ) } /// Build a recovery projection to restore the original output schema. @@ -671,20 +717,6 @@ impl<'a> LeafExpressionExtractor<'a> { } } -/// Resolves `col` against `schema` and returns it with the qualifier that -/// `schema` gives the field. Returns `None` if `schema` does not hold `col`, -/// or if the name is ambiguous in `schema`. -/// -/// Two spellings of the same input column (a bare `c` and a qualified `t.c`) -/// resolve to the same `Column`, so they compare equal. A column pushed into -/// a projection this way uses the name that the input gives it. -fn resolve_against(schema: &DFSchema, col: &Column) -> Option { - schema - .qualified_field_from_column(col) - .ok() - .map(Column::from) -} - /// Build an extraction projection above the target node (shared by both passes). /// /// If the target is an existing projection, merges into it. This requires @@ -693,7 +725,11 @@ fn resolve_against(schema: &DFSchema, col: &Column) -> Option { /// `u['name']`, we must rewrite it to `user['name']` since the merged /// projection reads from the same input as the original. /// -/// Deduplicates by resolved expression equality and adds pass-through +/// Column references are resolved by their index in the projection's output +/// schema (see [`column_index`]), never by name. +/// +/// Deduplicates by resolved expression equality, comparing expressions in the +/// canonical spelling of the projection's input schema, and adds pass-through /// columns as needed. Otherwise builds a fresh projection with extracted /// expressions + ALL input schema columns. fn build_extraction_projection_impl( @@ -705,96 +741,65 @@ fn build_extraction_projection_impl( if let LogicalPlan::Projection(existing) = target.as_ref() { // Merge into existing projection let mut proj_exprs = existing.expr.clone(); + let input_schema = existing.input.schema(); - // Build a map of existing expressions (by Expr equality) to their aliases - let existing_extractions: IndexMap = existing - .expr - .iter() - .filter_map(|e| { - if let Expr::Alias(alias) = e - && alias.name.starts_with(EXTRACTED_EXPR_PREFIX) - { - return Some((*alias.expr.clone(), alias.name.clone())); - } - None - }) - .collect(); - - // Resolve column references through the projection's rename mapping - let replace_map = build_projection_replace_map(existing); + // The (expression, alias) extractions the merged projection computes, + // with each expression in the canonical spelling of the input schema. + // The same expression can appear under several aliases, so the alias + // is part of the key. + #[allow(clippy::allow_attributes, clippy::mutable_key_type)] + // Expr contains Arc with interior mutability but is intentionally used as hash key + let mut extractions: HashSet<(Expr, String)> = HashSet::new(); + for e in &existing.expr { + if let Expr::Alias(alias) = e + && alias.name.starts_with(EXTRACTED_EXPR_PREFIX) + { + let expr = respell_cols( + alias.expr.as_ref().clone(), + input_schema, + input_schema, + )?; + extractions.insert((expr, alias.name.clone())); + } + } - // Add new extracted expressions, resolving column refs through the projection + // Add new extracted expressions, resolving column refs through the + // projection by output index. An extraction already computed under + // the same alias is skipped. The same expression under a different + // alias is added again so both names are available in the output: we + // can't reference the existing alias as a column within the same + // projection, so we duplicate the computation. for (expr, alias) in extracted_exprs { - let resolved = replace_cols_by_name(expr.clone().alias(alias), &replace_map)?; - let resolved_inner = if let Expr::Alias(a) = &resolved { - a.expr.as_ref() - } else { - &resolved - }; - if let Some(existing_alias) = existing_extractions.get(resolved_inner) { - // Same expression already extracted under a different alias — - // add the expression with the new alias so both names are - // available in the output. We can't reference the existing alias - // as a column within the same projection, so we duplicate the - // computation. - if existing_alias != alias { - proj_exprs.push(resolved); - } - } else { - proj_exprs.push(resolved); + let resolved = + rewrite_cols_by_index(expr.clone(), &existing.schema, |idx| { + projection_definition(existing, idx) + })?; + if extractions.insert((resolved.clone(), alias.clone())) { + proj_exprs.push(resolved.alias(alias)); } } - // Add any new pass-through columns that aren't already in the projection. - // We check against existing.input.schema() (the projection's source) rather - // than target_schema (the projection's output) because columns produced - // by alias expressions (e.g., CSE's __common_expr_N) exist in the output but - // not the input, and cannot be added as pass-through Column references. - // - // Resolve the pass-through columns of `existing` (including same-name - // aliases such as `t.c AS c`) and the columns in `columns_needed` - // against the input schema before comparing them. Otherwise a bare `c` - // and a qualified `t.c` do not match, and the merged projection holds - // both, which `Projection::try_new` rejects as ambiguous. - let input_schema = existing.input.schema(); - let existing_cols: IndexSet = existing - .expr - .iter() - .filter_map(|e| resolve_against(input_schema, passthrough_column(e)?)) - .collect(); - - // The names the merged projection already produces. The merge keeps - // every expression of `existing`, so the parent can still read each of - // these names, and a pass-through column that carries one of them makes - // the output schema ambiguous. - let output_names: std::collections::HashSet<&str> = existing - .schema - .fields() - .iter() - .map(|f| f.name().as_str()) - .collect(); - + // Add any new pass-through columns that the merged projection does not + // already produce. A column that resolves in the output schema of + // `existing` is produced already: the merge keeps every expression of + // `existing`. This also covers renames such as `t.a AS b` (issue + // #25446): the parent reads `b` from the output, not `t.a` from the + // input. for col in columns_needed { - // The projection produces this name, so the parent reads it from the - // projection's own output. Do not resolve the name through the rename - // map: a rename such as `t.a AS b` is a computed output, and its - // input column `t.a` beside the output field `a` of a second rename - // gives an ambiguous schema (issue #25446). - if output_names.contains(col.name.as_str()) { + if column_index(&existing.schema, col).is_some() { continue; } - let col_expr = Expr::Column(col.clone()); - let resolved = replace_cols_by_name(col_expr, &replace_map)?; - if let Expr::Column(resolved_col) = &resolved - && let Some(input_col) = resolve_against(input_schema, resolved_col) - && !existing_cols.contains(&input_col) - && !output_names.contains(input_col.name.as_str()) + // Columns that are not in the output: add them from the input if + // they resolve there and their name does not collide with an + // output field. Every pass-through of `existing` carries the name + // of an output field, so this also skips those. + if let Some(input_col) = resolve_against(input_schema, col) + && !existing + .schema + .has_column_with_unqualified_name(&input_col.name) { proj_exprs.push(Expr::Column(input_col)); } - // Skip a column that resolves to a non-column expr: the existing - // projection already computes it. Also skip a column that is not in - // the input or is ambiguous there: no single spelling is correct. } Projection::try_new(proj_exprs, Arc::clone(&existing.input)) @@ -923,36 +928,35 @@ fn merge_would_duplicate_kept_expr( columns_needed: &IndexSet, child: &Projection, ) -> bool { - // Columns whose `KeepInPlace` definition must stay put; inlining one would - // duplicate a compute-once expression. Plain columns, pushable exprs, and - // literals are all cheap to duplicate, so only `KeepInPlace` counts. - let kept_columns: std::collections::HashSet = child - .schema + // Output indices whose `KeepInPlace` definition must stay put; inlining + // one would duplicate a compute-once expression. Plain columns, pushable + // exprs, and literals are all cheap to duplicate, so only `KeepInPlace` + // counts. + let kept: BTreeSet = child + .expr .iter() - .zip(child.expr.iter()) - .filter_map(|((qualifier, field), expr)| { - if matches!(expr.placement(), ExpressionPlacement::KeepInPlace) { - Some(Column::from((qualifier, field)).flat_name()) - } else { - None - } - }) + .enumerate() + .filter(|(_, expr)| matches!(expr.placement(), ExpressionPlacement::KeepInPlace)) + .map(|(idx, _)| idx) .collect(); - if kept_columns.is_empty() { + if kept.is_empty() { return false; } + let kept_index = + |col: &Column| column_index(&child.schema, col).filter(|idx| kept.contains(idx)); // Count every site that references a kept column. A pair that references the // same kept column twice still inlines one copy, so count each pair at most // once per column. - let mut usage: HashMap = HashMap::new(); - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut usage: HashMap = HashMap::new(); + let mut seen: BTreeSet = BTreeSet::new(); for (expr, _alias) in pairs { seen.clear(); for col in expr.column_refs() { - let name = col.flat_name(); - if kept_columns.contains(&name) && seen.insert(name.clone()) { - *usage.entry(name).or_insert(0) += 1; + if let Some(idx) = kept_index(col) + && seen.insert(idx) + { + *usage.entry(idx).or_insert(0) += 1; } } } @@ -960,9 +964,8 @@ fn merge_would_duplicate_kept_expr( // a bare `f(c)` alongside `f(c)['a']` also pins the compute-once column, so // counting only `pairs` under-counted and re-duplicated it (#23655). for col in columns_needed { - let name = col.flat_name(); - if kept_columns.contains(&name) { - *usage.entry(name).or_insert(0) += 1; + if let Some(idx) = kept_index(col) { + *usage.entry(idx).or_insert(0) += 1; } } usage.values().any(|&count| count > 1) @@ -1328,7 +1331,7 @@ fn route_to_inputs( pairs: &[(Expr, String)], columns: &IndexSet, node: &LogicalPlan, - input_column_sets: &[std::collections::HashSet], + input_column_sets: &[HashSet], input_schemas: &[Arc], ) -> Result>> { let num_inputs = input_schemas.len(); @@ -1435,7 +1438,7 @@ fn try_push_into_inputs( // Build per-input schemas and column sets for routing let input_schemas: Vec> = inputs.iter().map(|i| Arc::clone(i.schema())).collect(); - let input_column_sets: Vec> = + let input_column_sets: Vec> = input_schemas.iter().map(|s| schema_columns(s)).collect(); // Route pairs and columns to the appropriate inputs @@ -2506,7 +2509,7 @@ mod tests { ## After Extraction Projection: test.a, test.b, d Filter: __datafusion_extracted_1 = Int32(1) - Projection: test.a, test.b, test.c + Int32(1) AS d, leaf_udf(a, Utf8("x")) AS __datafusion_extracted_1 + Projection: test.a, test.b, test.c + Int32(1) AS d, leaf_udf(test.a, Utf8("x")) AS __datafusion_extracted_1 TableScan: test projection=[a, b, c] ## After Pushdown @@ -2546,6 +2549,63 @@ mod tests { Ok(()) } + /// The existing projection spells an extraction with a bare `a`, and the + /// caller asks for the same extraction spelled `test.a`. Both resolve to + /// the same input field, so the merge must find the existing extraction + /// and not add it a second time (issue #25447). + #[test] + fn test_merge_finds_extraction_with_other_column_spelling() -> Result<()> { + let alias = format!("{EXTRACTED_EXPR_PREFIX}_1"); + let existing = Projection::try_new( + vec![leaf_udf(col("a"), "x").alias(&alias), col("test.a")], + Arc::new(test_table_scan()?), + )?; + let existing_exprs = existing.expr.clone(); + let projection = LogicalPlan::Projection(existing); + let target_schema = Arc::clone(projection.schema()); + + let merged = build_extraction_projection_impl( + &[(leaf_udf(col("test.a"), "x"), alias)], + &IndexSet::new(), + &Arc::new(projection), + &target_schema, + )?; + + assert_eq!(merged.expr, existing_exprs); + Ok(()) + } + + /// The existing projection computes the same extraction under two aliases. + /// Asking for it under either alias must find it, whichever comes first, + /// and not add a second field with the same name. + #[test] + fn test_merge_finds_extraction_computed_under_two_aliases() -> Result<()> { + let alias_1 = format!("{EXTRACTED_EXPR_PREFIX}_1"); + let alias_2 = format!("{EXTRACTED_EXPR_PREFIX}_2"); + let existing = Projection::try_new( + vec![ + leaf_udf(col("test.a"), "x").alias(&alias_1), + leaf_udf(col("test.a"), "x").alias(&alias_2), + col("test.a"), + ], + Arc::new(test_table_scan()?), + )?; + let existing_exprs = existing.expr.clone(); + let projection = Arc::new(LogicalPlan::Projection(existing)); + let target_schema = Arc::clone(projection.schema()); + + for alias in [alias_1, alias_2] { + let merged = build_extraction_projection_impl( + &[(leaf_udf(col("test.a"), "x"), alias)], + &IndexSet::new(), + &projection, + &target_schema, + )?; + assert_eq!(merged.expr, existing_exprs); + } + Ok(()) + } + /// A projection can spell a pass-through column as a same-name alias /// (`test.a AS a`). Merging an extraction into it must treat that alias as /// the pass-through it is, and not add `test.a` beside the `a` it outputs,