From 5efbfb6753dd741b0482b243b9972f6418a3d978 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 14 Jan 2025 14:13:48 +0800 Subject: [PATCH 01/13] nlj proj pushdown Signed-off-by: Jay Zhan --- .../src/physical_optimizer/join_selection.rs | 2 + .../physical_optimizer/projection_pushdown.rs | 193 ++++++++++++------ datafusion/core/src/physical_planner.rs | 1 + datafusion/core/tests/fuzz_cases/join_fuzz.rs | 2 +- .../src/joins/nested_loop_join.rs | 177 +++++++++++++--- datafusion/proto/src/physical_plan/mod.rs | 1 + .../tests/cases/roundtrip_physical_plan.rs | 1 + .../sqllogictest/test_files/group_by.slt | 9 +- 8 files changed, 292 insertions(+), 94 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/join_selection.rs b/datafusion/core/src/physical_optimizer/join_selection.rs index 736c3fbd01845..6153092883b12 100644 --- a/datafusion/core/src/physical_optimizer/join_selection.rs +++ b/datafusion/core/src/physical_optimizer/join_selection.rs @@ -1063,6 +1063,7 @@ mod tests_statistical { Arc::clone(&small), nl_join_filter(), &join_type, + None, ) .unwrap(), ); @@ -1131,6 +1132,7 @@ mod tests_statistical { Arc::clone(&small), nl_join_filter(), &join_type, + None, ) .unwrap(), ); diff --git a/datafusion/core/src/physical_optimizer/projection_pushdown.rs b/datafusion/core/src/physical_optimizer/projection_pushdown.rs index d2d35c3877c19..416f51cc24b50 100644 --- a/datafusion/core/src/physical_optimizer/projection_pushdown.rs +++ b/datafusion/core/src/physical_optimizer/projection_pushdown.rs @@ -144,7 +144,10 @@ pub fn remove_unnecessary_projections( } else if let Some(cross_join) = input.downcast_ref::() { try_swapping_with_cross_join(projection, cross_join)? } else if let Some(nl_join) = input.downcast_ref::() { - try_swapping_with_nested_loop_join(projection, nl_join)? + try_pushdown_through_nested_loop_join(projection, nl_join)?.map_or_else( + || try_embed_projection(projection, nl_join), + |e| Ok(Some(e)), + )? } else if let Some(sm_join) = input.downcast_ref::() { try_swapping_with_sort_merge_join(projection, sm_join)? } else if let Some(sym_join) = input.downcast_ref::() { @@ -549,6 +552,12 @@ impl EmbeddedProjection for HashJoinExec { } } +impl EmbeddedProjection for NestedLoopJoinExec { + fn with_projection(&self, projection: Option>) -> Result { + self.with_projection(projection) + } +} + impl EmbeddedProjection for FilterExec { fn with_projection(&self, projection: Option>) -> Result { self.with_projection(projection) @@ -626,6 +635,63 @@ fn collect_column_indices(exprs: &[(Arc, String)]) -> Vec Result>> { + if nl_join.contain_projection() { + return Ok(None); + } + + let Some(projection_as_columns) = physical_to_column_exprs(projection.expr()) else { + return Ok(None); + }; + + let (far_right_left_col_ind, far_left_right_col_ind) = join_table_borders( + nl_join.left().schema().fields().len(), + &projection_as_columns, + ); + + if !join_allows_pushdown( + &projection_as_columns, + &nl_join.schema(), + far_right_left_col_ind, + far_left_right_col_ind, + ) { + return Ok(None); + } + + let new_filter = if let Some(filter) = nl_join.filter() { + match update_join_filter( + &projection_as_columns[0..=far_right_left_col_ind as _], + &projection_as_columns[far_left_right_col_ind as _..], + filter, + nl_join.left().schema().fields().len(), + ) { + Some(updated_filter) => Some(updated_filter), + None => return Ok(None), + } + } else { + None + }; + + let (new_left, new_right) = new_join_children( + &projection_as_columns, + far_right_left_col_ind, + far_left_right_col_ind, + nl_join.left(), + nl_join.right(), + )?; + + Ok(Some(Arc::new(NestedLoopJoinExec::try_new( + Arc::new(new_left), + Arc::new(new_right), + new_filter, + nl_join.join_type(), + None, + )?))) +} + /// Tries to push `projection` down through `hash_join`. If possible, performs the /// pushdown and returns a new [`HashJoinExec`] as the top plan which has projections /// as its children. Otherwise, returns `None`. @@ -741,62 +807,6 @@ fn try_swapping_with_cross_join( )))) } -/// Tries to swap the projection with its input [`NestedLoopJoinExec`]. If it can be done, -/// it returns the new swapped version having the [`NestedLoopJoinExec`] as the top plan. -/// Otherwise, it returns None. -fn try_swapping_with_nested_loop_join( - projection: &ProjectionExec, - nl_join: &NestedLoopJoinExec, -) -> Result>> { - // Convert projected PhysicalExpr's to columns. If not possible, we cannot proceed. - let Some(projection_as_columns) = physical_to_column_exprs(projection.expr()) else { - return Ok(None); - }; - - let (far_right_left_col_ind, far_left_right_col_ind) = join_table_borders( - nl_join.left().schema().fields().len(), - &projection_as_columns, - ); - - if !join_allows_pushdown( - &projection_as_columns, - &nl_join.schema(), - far_right_left_col_ind, - far_left_right_col_ind, - ) { - return Ok(None); - } - - let new_filter = if let Some(filter) = nl_join.filter() { - match update_join_filter( - &projection_as_columns[0..=far_right_left_col_ind as _], - &projection_as_columns[far_left_right_col_ind as _..], - filter, - nl_join.left().schema().fields().len(), - ) { - Some(updated_filter) => Some(updated_filter), - None => return Ok(None), - } - } else { - None - }; - - let (new_left, new_right) = new_join_children( - &projection_as_columns, - far_right_left_col_ind, - far_left_right_col_ind, - nl_join.left(), - nl_join.right(), - )?; - - Ok(Some(Arc::new(NestedLoopJoinExec::try_new( - Arc::new(new_left), - Arc::new(new_right), - new_filter, - nl_join.join_type(), - )?))) -} - /// Tries to swap the projection with its input [`SortMergeJoinExec`]. If it can be done, /// it returns the new swapped version having the [`SortMergeJoinExec`] as the top plan. /// Otherwise, it returns None. @@ -1345,7 +1355,7 @@ mod tests { ColumnarValue, Operator, ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; use datafusion_physical_expr::expressions::{ - BinaryExpr, CaseExpr, CastExpr, NegativeExpr, + binary, col, BinaryExpr, CaseExpr, CastExpr, NegativeExpr }; use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_plan::joins::PartitionMode; @@ -2402,6 +2412,73 @@ mod tests { Ok(()) } + #[test] + fn test_nested_loop_join_after_projection() -> Result<()> { + let left_csv = create_simple_csv_exec(); + let right_csv = create_simple_csv_exec(); + + let col_left_a = col("a", &left_csv.schema())?; + let col_right_b = col("b", &right_csv.schema())?; + let col_left_c = col("c", &left_csv.schema())?; + // left_a < right_b + let filter_expr = + binary(col_left_a, Operator::Lt, col_right_b, &Schema::empty())?; + let filter_column_indices = vec![ + ColumnIndex { + index: 0, + side: JoinSide::Left, + }, + ColumnIndex { + index: 1, + side: JoinSide::Right, + }, + ColumnIndex { + index: 2, + side: JoinSide::Right, + }, + ]; + let filter_schema = Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int32, true), + Field::new("c", DataType::Int32, true), + ]); + + let join: Arc = Arc::new(NestedLoopJoinExec::try_new( + left_csv, + right_csv, + Some(JoinFilter::new( + filter_expr, + filter_column_indices, + filter_schema, + )), + &JoinType::Inner, + None, + )?); + + let projection: Arc = Arc::new(ProjectionExec::try_new( + vec![(col_left_c, "c".to_string())], + Arc::clone(&join), + )?); + let initial = get_plan_string(&projection); + let expected_initial = [ + "ProjectionExec: expr=[c@2 as c]", + " NestedLoopJoinExec: join_type=Inner, filter=a@0 < b@1", + " CsvExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], has_header=false", + " CsvExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], has_header=false", + ]; + assert_eq!(initial, expected_initial); + + let after_optimize = + ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?; + let expected = [ + "NestedLoopJoinExec: join_type=Inner, filter=a@0 < b@1, projection=[c@2]", + " CsvExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], has_header=false", + " CsvExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], has_header=false", + ]; + assert_eq!(get_plan_string(&after_optimize), expected); + Ok(()) + } + #[test] fn test_hash_join_after_projection() -> Result<()> { // sql like diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 6a4f379e4d6d3..f6b16d8bc2445 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -1109,6 +1109,7 @@ impl DefaultPhysicalPlanner { physical_right, join_filter, join_type, + None, )?) } } else if session_state.config().target_partitions() > 1 diff --git a/datafusion/core/tests/fuzz_cases/join_fuzz.rs b/datafusion/core/tests/fuzz_cases/join_fuzz.rs index b331388f4f3f9..04eb48e41ccf2 100644 --- a/datafusion/core/tests/fuzz_cases/join_fuzz.rs +++ b/datafusion/core/tests/fuzz_cases/join_fuzz.rs @@ -486,7 +486,7 @@ impl JoinFuzzTestCase { let filter = JoinFilter::new(expression, column_indices, intermediate_schema); Arc::new( - NestedLoopJoinExec::try_new(left, right, Some(filter), &self.join_type) + NestedLoopJoinExec::try_new(left, right, Some(filter), &self.join_type, None) .unwrap(), ) } diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index c69fa2888806c..ebd8e953274ce 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -29,6 +29,7 @@ use super::utils::{ StatefulStreamResult, }; use crate::coalesce_partitions::CoalescePartitionsExec; +use crate::common::can_project; use crate::execution_plan::{boundedness_from_children, EmissionType}; use crate::joins::utils::{ adjust_indices_by_join_type, apply_join_filter_to_indices, build_batch_from_indices, @@ -48,12 +49,14 @@ use arrow::compute::concat_batches; use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use datafusion_common::{ - exec_datafusion_err, internal_err, JoinSide, Result, Statistics, + exec_datafusion_err, internal_err, project_schema, JoinSide, Result, Statistics, }; use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation}; use datafusion_execution::TaskContext; use datafusion_expr::JoinType; -use datafusion_physical_expr::equivalence::join_equivalence_properties; +use datafusion_physical_expr::equivalence::{ + join_equivalence_properties, ProjectionMapping, +}; use futures::{ready, Stream, StreamExt, TryStreamExt}; use parking_lot::Mutex; @@ -155,7 +158,7 @@ pub struct NestedLoopJoinExec { /// How the join is performed pub(crate) join_type: JoinType, /// The schema once the join is applied - schema: SchemaRef, + join_schema: SchemaRef, /// Future that consumes left input and buffers it in memory /// /// This structure is *shared* across all output streams. @@ -165,6 +168,9 @@ pub struct NestedLoopJoinExec { inner_table: OnceAsync, /// Information of index and left / right placement of columns column_indices: Vec, + /// Projection to apply to the output of the join + projection: Option>, + /// Execution metrics metrics: ExecutionPlanMetricsSet, /// Cache holding plan properties like equivalences, output partitioning etc. @@ -178,24 +184,31 @@ impl NestedLoopJoinExec { right: Arc, filter: Option, join_type: &JoinType, + projection: Option>, ) -> Result { let left_schema = left.schema(); let right_schema = right.schema(); check_join_is_valid(&left_schema, &right_schema, &[])?; - let (schema, column_indices) = + let (join_schema, column_indices) = build_join_schema(&left_schema, &right_schema, join_type); - let schema = Arc::new(schema); - let cache = - Self::compute_properties(&left, &right, Arc::clone(&schema), *join_type); + let join_schema = Arc::new(join_schema); + let cache = Self::compute_properties( + &left, + &right, + Arc::clone(&join_schema), + *join_type, + projection.as_ref(), + )?; Ok(NestedLoopJoinExec { left, right, filter, join_type: *join_type, - schema, + join_schema, inner_table: Default::default(), column_indices, + projection, metrics: Default::default(), cache, }) @@ -227,20 +240,21 @@ impl NestedLoopJoinExec { right: &Arc, schema: SchemaRef, join_type: JoinType, - ) -> PlanProperties { + projection: Option<&Vec>, + ) -> Result { // Calculate equivalence properties: - let eq_properties = join_equivalence_properties( + let mut eq_properties = join_equivalence_properties( left.equivalence_properties().clone(), right.equivalence_properties().clone(), &join_type, - schema, + Arc::clone(&schema), &Self::maintains_input_order(join_type), None, // No on columns in nested loop join &[], ); - let output_partitioning = + let mut output_partitioning = asymmetric_join_output_partitioning(left, right, &join_type); let emission_type = if left.boundedness().is_unbounded() { @@ -265,12 +279,22 @@ impl NestedLoopJoinExec { right.pipeline_behavior() }; - PlanProperties::new( + if let Some(projection) = projection { + // construct a map from the input expressions to the output expression of the Projection + let projection_mapping = + ProjectionMapping::from_indices(projection, &schema)?; + let out_schema = project_schema(&schema, Some(projection))?; + output_partitioning = + output_partitioning.project(&projection_mapping, &eq_properties); + eq_properties = eq_properties.project(&projection_mapping, out_schema); + } + + Ok(PlanProperties::new( eq_properties, output_partitioning, emission_type, boundedness_from_children([left, right]), - ) + )) } /// Returns a vector indicating whether the left and right inputs maintain their order. @@ -298,17 +322,45 @@ impl NestedLoopJoinExec { ] } + pub fn contain_projection(&self) -> bool { + self.projection.is_some() + } + + pub fn with_projection(&self, projection: Option>) -> Result { + // check if the projection is valid + can_project(&self.schema(), projection.as_ref())?; + let projection = match projection { + Some(projection) => match &self.projection { + Some(p) => Some(projection.iter().map(|i| p[*i]).collect()), + None => Some(projection), + }, + None => None, + }; + Self::try_new( + Arc::clone(&self.left), + Arc::clone(&self.right), + self.filter.clone(), + &self.join_type, + projection, + ) + } + /// Returns a new `ExecutionPlan` that runs NestedLoopsJoins with the left /// and right inputs swapped. pub fn swap_inputs(&self) -> Result> { - let new_filter = self.filter().map(JoinFilter::swap); - let new_join_type = &self.join_type().swap(); - + let left = self.left(); + let right = self.right(); let new_join = NestedLoopJoinExec::try_new( - Arc::clone(self.right()), - Arc::clone(self.left()), - new_filter, - new_join_type, + Arc::clone(right), + Arc::clone(left), + self.filter().map(JoinFilter::swap), + &self.join_type().swap(), + swap_join_projection( + left.schema().fields().len(), + right.schema().fields().len(), + self.projection.as_ref(), + self.join_type(), + ), )?; // For Semi/Anti joins, swap result will produce same output schema, @@ -319,7 +371,8 @@ impl NestedLoopJoinExec { | JoinType::RightSemi | JoinType::LeftAnti | JoinType::RightAnti - ) { + ) || self.projection.is_some() + { Arc::new(new_join) } else { reorder_output_after_swap( @@ -333,6 +386,41 @@ impl NestedLoopJoinExec { } } +// TODO: Merge this one with the hash join one +/// The same logic as the hash join one, but for nested loop join +/// This function swaps the given join's projection. +fn swap_join_projection( + left_schema_len: usize, + right_schema_len: usize, + projection: Option<&Vec>, + join_type: &JoinType, +) -> Option> { + match join_type { + // For Anti/Semi join types, projection should remain unmodified, + // since these joins output schema remains the same after swap + JoinType::LeftAnti + | JoinType::LeftSemi + | JoinType::RightAnti + | JoinType::RightSemi => projection.cloned(), + + _ => projection.map(|p| { + p.iter() + .map(|i| { + // If the index is less than the left schema length, it is from + // the left schema, so we add the right schema length to it. + // Otherwise, it is from the right schema, so we subtract the left + // schema length from it. + if *i < left_schema_len { + *i + right_schema_len + } else { + *i - left_schema_len + } + }) + .collect() + }), + } +} + impl DisplayAs for NestedLoopJoinExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { match t { @@ -341,10 +429,28 @@ impl DisplayAs for NestedLoopJoinExec { || "".to_string(), |f| format!(", filter={}", f.expression()), ); + let display_projections = if self.contain_projection() { + format!( + ", projection=[{}]", + self.projection + .as_ref() + .unwrap() + .iter() + .map(|index| format!( + "{}@{}", + self.join_schema.fields().get(*index).unwrap().name(), + index + )) + .collect::>() + .join(", ") + ) + } else { + "".to_string() + }; write!( f, - "NestedLoopJoinExec: join_type={:?}{}", - self.join_type, display_filter + "NestedLoopJoinExec: join_type={:?}{}{}", + self.join_type, display_filter, display_projections ) } } @@ -388,6 +494,7 @@ impl ExecutionPlan for NestedLoopJoinExec { Arc::clone(&children[1]), self.filter.clone(), &self.join_type, + self.projection.clone(), )?)) } @@ -426,14 +533,23 @@ impl ExecutionPlan for NestedLoopJoinExec { let right_side_ordered = self.maintains_input_order()[1] && self.right.output_ordering().is_some(); + // update column indices to reflect the projection + let column_indices_after_projection = match &self.projection { + Some(projection) => projection + .iter() + .map(|i| self.column_indices[*i].clone()) + .collect(), + None => self.column_indices.clone(), + }; + if enforce_batch_size_in_joins { Ok(Box::pin(NestedLoopJoinStream { - schema: Arc::clone(&self.schema), + schema: self.schema(), filter: self.filter.clone(), join_type: self.join_type, outer_table, inner_table, - column_indices: self.column_indices.clone(), + column_indices: column_indices_after_projection, join_metrics, indices_cache, right_side_ordered, @@ -443,12 +559,12 @@ impl ExecutionPlan for NestedLoopJoinExec { })) } else { Ok(Box::pin(NestedLoopJoinStream { - schema: Arc::clone(&self.schema), + schema: self.schema(), filter: self.filter.clone(), join_type: self.join_type, outer_table, inner_table, - column_indices: self.column_indices.clone(), + column_indices: column_indices_after_projection, join_metrics, indices_cache, right_side_ordered, @@ -469,7 +585,7 @@ impl ExecutionPlan for NestedLoopJoinExec { Arc::clone(&self.right), vec![], &self.join_type, - &self.schema, + &self.join_schema, ) } } @@ -1049,7 +1165,7 @@ pub(crate) mod tests { // Use the required distribution for nested loop join to test partition data let nested_loop_join = - NestedLoopJoinExec::try_new(left, right, join_filter, join_type)?; + NestedLoopJoinExec::try_new(left, right, join_filter, join_type, None)?; let columns = columns(&nested_loop_join.schema()); let mut batches = vec![]; for i in 0..partition_count { @@ -1485,6 +1601,7 @@ pub(crate) mod tests { Arc::clone(&right), Some(filter), &join_type, + None, )?) as Arc; assert_eq!(nested_loop_join.maintains_input_order(), vec![false, true]); diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 292ce13d0eded..aed6a35cd8eb0 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -1001,6 +1001,7 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode { right, filter, &join_type.into(), + None, )?)) } PhysicalPlanType::Analyze(analyze) => { diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 0a6ea6c7ff853..2b59f3a904cfb 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -270,6 +270,7 @@ fn roundtrip_nested_loop_join() -> Result<()> { Arc::new(EmptyExec::new(schema_right.clone())), None, join_type, + None, )?))?; } Ok(()) diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt index df7e21c2da447..d5f5f0b965979 100644 --- a/datafusion/sqllogictest/test_files/group_by.slt +++ b/datafusion/sqllogictest/test_files/group_by.slt @@ -3429,11 +3429,10 @@ physical_plan 05)--------CoalesceBatchesExec: target_batch_size=4 06)----------RepartitionExec: partitioning=Hash([sn@0, amount@1], 8), input_partitions=8 07)------------AggregateExec: mode=Partial, gby=[sn@1 as sn, amount@2 as amount], aggr=[sum(l.amount)] -08)--------------ProjectionExec: expr=[amount@1 as amount, sn@2 as sn, amount@3 as amount] -09)----------------NestedLoopJoinExec: join_type=Inner, filter=sn@0 >= sn@1 -10)------------------MemoryExec: partitions=1, partition_sizes=[1] -11)------------------RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1 -12)--------------------MemoryExec: partitions=1, partition_sizes=[1] +08)--------------NestedLoopJoinExec: join_type=Inner, filter=sn@0 >= sn@1, projection=[amount@1, sn@2, amount@3] +09)----------------MemoryExec: partitions=1, partition_sizes=[1] +10)----------------RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1 +11)------------------MemoryExec: partitions=1, partition_sizes=[1] query IRR SELECT r.sn, SUM(l.amount), r.amount From c182586a35d444d250b3f8339629d449b0de0619 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 14 Jan 2025 14:17:39 +0800 Subject: [PATCH 02/13] fmt Signed-off-by: Jay Zhan --- datafusion/core/src/physical_optimizer/projection_pushdown.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_optimizer/projection_pushdown.rs b/datafusion/core/src/physical_optimizer/projection_pushdown.rs index 416f51cc24b50..118c25a0f8e5d 100644 --- a/datafusion/core/src/physical_optimizer/projection_pushdown.rs +++ b/datafusion/core/src/physical_optimizer/projection_pushdown.rs @@ -1355,7 +1355,7 @@ mod tests { ColumnarValue, Operator, ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; use datafusion_physical_expr::expressions::{ - binary, col, BinaryExpr, CaseExpr, CastExpr, NegativeExpr + binary, col, BinaryExpr, CaseExpr, CastExpr, NegativeExpr, }; use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_plan::joins::PartitionMode; From 9b29caf87eb963cb87696544660e2e97efc86e99 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 14 Jan 2025 14:21:59 +0800 Subject: [PATCH 03/13] move swap proj to util Signed-off-by: Jay Zhan --- .../physical-plan/src/joins/hash_join.rs | 34 +--------------- .../src/joins/nested_loop_join.rs | 39 +------------------ datafusion/physical-plan/src/joins/utils.rs | 33 ++++++++++++++++ 3 files changed, 35 insertions(+), 71 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index 4e0b0bf820f2f..536222d4e289a 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -24,7 +24,7 @@ use std::sync::Arc; use std::task::Poll; use std::{any::Any, vec}; -use super::utils::{asymmetric_join_output_partitioning, reorder_output_after_swap}; +use super::utils::{asymmetric_join_output_partitioning, reorder_output_after_swap, swap_join_projection}; use super::{ utils::{OnceAsync, OnceFut}, PartitionMode, @@ -616,38 +616,6 @@ impl HashJoinExec { } } -/// This function swaps the given join's projection. -fn swap_join_projection( - left_schema_len: usize, - right_schema_len: usize, - projection: Option<&Vec>, - join_type: &JoinType, -) -> Option> { - match join_type { - // For Anti/Semi join types, projection should remain unmodified, - // since these joins output schema remains the same after swap - JoinType::LeftAnti - | JoinType::LeftSemi - | JoinType::RightAnti - | JoinType::RightSemi => projection.cloned(), - - _ => projection.map(|p| { - p.iter() - .map(|i| { - // If the index is less than the left schema length, it is from - // the left schema, so we add the right schema length to it. - // Otherwise, it is from the right schema, so we subtract the left - // schema length from it. - if *i < left_schema_len { - *i + right_schema_len - } else { - *i - left_schema_len - } - }) - .collect() - }), - } -} impl DisplayAs for HashJoinExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { match t { diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index ebd8e953274ce..66305cbb0743b 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -24,9 +24,7 @@ use std::sync::Arc; use std::task::Poll; use super::utils::{ - asymmetric_join_output_partitioning, need_produce_result_in_final, - reorder_output_after_swap, BatchSplitter, BatchTransformer, NoopBatchTransformer, - StatefulStreamResult, + asymmetric_join_output_partitioning, need_produce_result_in_final, reorder_output_after_swap, swap_join_projection, BatchSplitter, BatchTransformer, NoopBatchTransformer, StatefulStreamResult }; use crate::coalesce_partitions::CoalescePartitionsExec; use crate::common::can_project; @@ -386,41 +384,6 @@ impl NestedLoopJoinExec { } } -// TODO: Merge this one with the hash join one -/// The same logic as the hash join one, but for nested loop join -/// This function swaps the given join's projection. -fn swap_join_projection( - left_schema_len: usize, - right_schema_len: usize, - projection: Option<&Vec>, - join_type: &JoinType, -) -> Option> { - match join_type { - // For Anti/Semi join types, projection should remain unmodified, - // since these joins output schema remains the same after swap - JoinType::LeftAnti - | JoinType::LeftSemi - | JoinType::RightAnti - | JoinType::RightSemi => projection.cloned(), - - _ => projection.map(|p| { - p.iter() - .map(|i| { - // If the index is less than the left schema length, it is from - // the left schema, so we add the right schema length to it. - // Otherwise, it is from the right schema, so we subtract the left - // schema length from it. - if *i < left_schema_len { - *i + right_schema_len - } else { - *i - left_schema_len - } - }) - .collect() - }), - } -} - impl DisplayAs for NestedLoopJoinExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { match t { diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs index 371949a32598c..a7304d218e2ed 100644 --- a/datafusion/physical-plan/src/joins/utils.rs +++ b/datafusion/physical-plan/src/joins/utils.rs @@ -1775,6 +1775,39 @@ fn swap_reverting_projection( left_cols.chain(right_cols).collect() } +/// This function swaps the given join's projection. +pub(super) fn swap_join_projection( + left_schema_len: usize, + right_schema_len: usize, + projection: Option<&Vec>, + join_type: &JoinType, +) -> Option> { + match join_type { + // For Anti/Semi join types, projection should remain unmodified, + // since these joins output schema remains the same after swap + JoinType::LeftAnti + | JoinType::LeftSemi + | JoinType::RightAnti + | JoinType::RightSemi => projection.cloned(), + + _ => projection.map(|p| { + p.iter() + .map(|i| { + // If the index is less than the left schema length, it is from + // the left schema, so we add the right schema length to it. + // Otherwise, it is from the right schema, so we subtract the left + // schema length from it. + if *i < left_schema_len { + *i + right_schema_len + } else { + *i - left_schema_len + } + }) + .collect() + }), + } +} + #[cfg(test)] mod tests { use std::pin::Pin; From 746552df182ea5757d8a52557ed25694e79bf10c Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 14 Jan 2025 14:26:31 +0800 Subject: [PATCH 04/13] fmt Signed-off-by: Jay Zhan --- datafusion/physical-plan/src/joins/hash_join.rs | 4 +++- datafusion/physical-plan/src/joins/nested_loop_join.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index 536222d4e289a..15cca655ddad4 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -24,7 +24,9 @@ use std::sync::Arc; use std::task::Poll; use std::{any::Any, vec}; -use super::utils::{asymmetric_join_output_partitioning, reorder_output_after_swap, swap_join_projection}; +use super::utils::{ + asymmetric_join_output_partitioning, reorder_output_after_swap, swap_join_projection, +}; use super::{ utils::{OnceAsync, OnceFut}, PartitionMode, diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index 66305cbb0743b..d95984e0658bb 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -24,7 +24,9 @@ use std::sync::Arc; use std::task::Poll; use super::utils::{ - asymmetric_join_output_partitioning, need_produce_result_in_final, reorder_output_after_swap, swap_join_projection, BatchSplitter, BatchTransformer, NoopBatchTransformer, StatefulStreamResult + asymmetric_join_output_partitioning, need_produce_result_in_final, + reorder_output_after_swap, swap_join_projection, BatchSplitter, BatchTransformer, + NoopBatchTransformer, StatefulStreamResult, }; use crate::coalesce_partitions::CoalescePartitionsExec; use crate::common::can_project; From cf53ca57d9c2d1b3aff56aeefb2d25e0f7084e5d Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 14 Jan 2025 15:34:35 +0800 Subject: [PATCH 05/13] fix proto Signed-off-by: Jay Zhan --- .../src/joins/nested_loop_join.rs | 4 ++++ datafusion/proto/proto/datafusion.proto | 1 + datafusion/proto/src/generated/pbjson.rs | 20 +++++++++++++++++++ datafusion/proto/src/generated/prost.rs | 2 ++ datafusion/proto/src/physical_plan/mod.rs | 9 ++++++++- .../tests/cases/roundtrip_physical_plan.rs | 2 +- 6 files changed, 36 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index d95984e0658bb..7c96e1975655b 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -234,6 +234,10 @@ impl NestedLoopJoinExec { &self.join_type } + pub fn projection(&self) -> Option<&Vec> { + self.projection.as_ref() + } + /// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc. fn compute_properties( left: &Arc, diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index 416227f70de9d..5ca4cb3727107 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -1178,6 +1178,7 @@ message NestedLoopJoinExecNode { PhysicalPlanNode right = 2; datafusion_common.JoinType join_type = 3; JoinFilter filter = 4; + repeated uint32 projection = 5; } message CoalesceBatchesExecNode { diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index cffb63018676c..5530ab98af19f 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -11418,6 +11418,9 @@ impl serde::Serialize for NestedLoopJoinExecNode { if self.filter.is_some() { len += 1; } + if !self.projection.is_empty() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion.NestedLoopJoinExecNode", len)?; if let Some(v) = self.left.as_ref() { struct_ser.serialize_field("left", v)?; @@ -11433,6 +11436,9 @@ impl serde::Serialize for NestedLoopJoinExecNode { if let Some(v) = self.filter.as_ref() { struct_ser.serialize_field("filter", v)?; } + if !self.projection.is_empty() { + struct_ser.serialize_field("projection", &self.projection)?; + } struct_ser.end() } } @@ -11448,6 +11454,7 @@ impl<'de> serde::Deserialize<'de> for NestedLoopJoinExecNode { "join_type", "joinType", "filter", + "projection", ]; #[allow(clippy::enum_variant_names)] @@ -11456,6 +11463,7 @@ impl<'de> serde::Deserialize<'de> for NestedLoopJoinExecNode { Right, JoinType, Filter, + Projection, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -11481,6 +11489,7 @@ impl<'de> serde::Deserialize<'de> for NestedLoopJoinExecNode { "right" => Ok(GeneratedField::Right), "joinType" | "join_type" => Ok(GeneratedField::JoinType), "filter" => Ok(GeneratedField::Filter), + "projection" => Ok(GeneratedField::Projection), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -11504,6 +11513,7 @@ impl<'de> serde::Deserialize<'de> for NestedLoopJoinExecNode { let mut right__ = None; let mut join_type__ = None; let mut filter__ = None; + let mut projection__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Left => { @@ -11530,6 +11540,15 @@ impl<'de> serde::Deserialize<'de> for NestedLoopJoinExecNode { } filter__ = map_.next_value()?; } + GeneratedField::Projection => { + if projection__.is_some() { + return Err(serde::de::Error::duplicate_field("projection")); + } + projection__ = + Some(map_.next_value::>>()? + .into_iter().map(|x| x.0).collect()) + ; + } } } Ok(NestedLoopJoinExecNode { @@ -11537,6 +11556,7 @@ impl<'de> serde::Deserialize<'de> for NestedLoopJoinExecNode { right: right__, join_type: join_type__.unwrap_or_default(), filter: filter__, + projection: projection__.unwrap_or_default(), }) } } diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index d2fda5dc8892f..f169426ed309f 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -1722,6 +1722,8 @@ pub struct NestedLoopJoinExecNode { pub join_type: i32, #[prost(message, optional, tag = "4")] pub filter: ::core::option::Option, + #[prost(uint32, repeated, tag = "5")] + pub projection: ::prost::alloc::vec::Vec, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct CoalesceBatchesExecNode { diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index aed6a35cd8eb0..1d435820e7a2d 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -996,12 +996,13 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode { }) .map_or(Ok(None), |v: Result| v.map(Some))?; + let projection = join.projection.iter().map(|i| *i as usize).collect(); Ok(Arc::new(NestedLoopJoinExec::try_new( left, right, filter, &join_type.into(), - None, + Some(projection), )?)) } PhysicalPlanType::Analyze(analyze) => { @@ -1834,6 +1835,11 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode { }) .map_or(Ok(None), |v: Result| v.map(Some))?; + let projection = match exec.projection() { + Some(projection) => projection.iter().map(|&i| i as u32).collect(), + None => Vec::new(), + }; + return Ok(protobuf::PhysicalPlanNode { physical_plan_type: Some(PhysicalPlanType::NestedLoopJoin(Box::new( protobuf::NestedLoopJoinExecNode { @@ -1841,6 +1847,7 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode { right: Some(Box::new(right)), join_type: join_type.into(), filter, + projection, }, ))), }); diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 2b59f3a904cfb..20d1db93c6519 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -270,7 +270,7 @@ fn roundtrip_nested_loop_join() -> Result<()> { Arc::new(EmptyExec::new(schema_right.clone())), None, join_type, - None, + Some(vec![0]) )?))?; } Ok(()) From c4aa176e3b70629030d21554577bddcb89cda5d0 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 14 Jan 2025 15:35:49 +0800 Subject: [PATCH 06/13] fmt Signed-off-by: Jay Zhan --- datafusion/proto/tests/cases/roundtrip_physical_plan.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 20d1db93c6519..50bcc6a680555 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -270,7 +270,7 @@ fn roundtrip_nested_loop_join() -> Result<()> { Arc::new(EmptyExec::new(schema_right.clone())), None, join_type, - Some(vec![0]) + Some(vec![0]), )?))?; } Ok(()) From c02c7d55ff0f13b443b7d1d41a554b82b74d2747 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 14 Jan 2025 15:48:07 +0800 Subject: [PATCH 07/13] use none Signed-off-by: Jay Zhan --- datafusion/core/src/physical_optimizer/projection_pushdown.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_optimizer/projection_pushdown.rs b/datafusion/core/src/physical_optimizer/projection_pushdown.rs index 118c25a0f8e5d..c1dc4c06cb57b 100644 --- a/datafusion/core/src/physical_optimizer/projection_pushdown.rs +++ b/datafusion/core/src/physical_optimizer/projection_pushdown.rs @@ -761,7 +761,8 @@ fn try_pushdown_through_hash_join( new_on, new_filter, hash_join.join_type(), - hash_join.projection.clone(), + // Early return if projection is not None + None, *hash_join.partition_mode(), hash_join.null_equals_null, )?))) From eb8001663f93f494ec119e94a325a0bfca0cc5e5 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 14 Jan 2025 16:26:02 +0800 Subject: [PATCH 08/13] proto fix Signed-off-by: Jay Zhan --- datafusion/proto/src/physical_plan/mod.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 1d435820e7a2d..c7cf25d0654c9 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -996,13 +996,23 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode { }) .map_or(Ok(None), |v: Result| v.map(Some))?; - let projection = join.projection.iter().map(|i| *i as usize).collect(); + let projection = if !join.projection.is_empty() { + Some( + join.projection + .iter() + .map(|i| *i as usize) + .collect::>(), + ) + } else { + None + }; + Ok(Arc::new(NestedLoopJoinExec::try_new( left, right, filter, &join_type.into(), - Some(projection), + projection, )?)) } PhysicalPlanType::Analyze(analyze) => { @@ -1835,11 +1845,6 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode { }) .map_or(Ok(None), |v: Result| v.map(Some))?; - let projection = match exec.projection() { - Some(projection) => projection.iter().map(|&i| i as u32).collect(), - None => Vec::new(), - }; - return Ok(protobuf::PhysicalPlanNode { physical_plan_type: Some(PhysicalPlanType::NestedLoopJoin(Box::new( protobuf::NestedLoopJoinExecNode { @@ -1847,7 +1852,9 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode { right: Some(Box::new(right)), join_type: join_type.into(), filter, - projection, + projection: exec.projection().map_or_else(Vec::new, |v| { + v.iter().map(|x| *x as u32).collect::>() + }), }, ))), }); From 1732644c5eb2140070926674ceee096be66a2844 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 14 Jan 2025 17:30:12 +0800 Subject: [PATCH 09/13] fix slt Signed-off-by: Jay Zhan --- datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part index 80f9919cb46bc..6645ede2a73bf 100644 --- a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part +++ b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part @@ -75,7 +75,7 @@ logical_plan physical_plan 01)SortExec: TopK(fetch=10), expr=[value@1 DESC], preserve_partitioning=[false] 02)--ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value] -03)----NestedLoopJoinExec: join_type=Inner, filter=CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Decimal128(38, 15)) > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@1 +03)----NestedLoopJoinExec: join_type=Inner, filter=CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Decimal128(38, 15)) > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@1, projection=[ps_partkey@0, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1] 04)------CoalescePartitionsExec 05)--------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] 06)----------CoalesceBatchesExec: target_batch_size=8192 From f9cc3b5413e6aaca8773fa9de6b4a44582ea6eca Mon Sep 17 00:00:00 2001 From: berkaysynnada Date: Wed, 15 Jan 2025 11:00:22 +0300 Subject: [PATCH 10/13] Update projection_pushdown.rs --- datafusion/core/src/physical_optimizer/projection_pushdown.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_optimizer/projection_pushdown.rs b/datafusion/core/src/physical_optimizer/projection_pushdown.rs index c1dc4c06cb57b..7c9f11beade03 100644 --- a/datafusion/core/src/physical_optimizer/projection_pushdown.rs +++ b/datafusion/core/src/physical_optimizer/projection_pushdown.rs @@ -639,6 +639,7 @@ fn try_pushdown_through_nested_loop_join( projection: &ProjectionExec, nl_join: &NestedLoopJoinExec, ) -> Result>> { + // TODO: currently if there is projection in NestedLoopJoinExec, we can't push down projection to left or right input. Maybe we can pushdown the mixed projection later. if nl_join.contain_projection() { return Ok(None); } @@ -761,7 +762,7 @@ fn try_pushdown_through_hash_join( new_on, new_filter, hash_join.join_type(), - // Early return if projection is not None + // Returned early if projection is not None None, *hash_join.partition_mode(), hash_join.null_equals_null, From a1bba91e52d8dccca75aa01fa443934df17b57d7 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Wed, 15 Jan 2025 17:03:38 +0800 Subject: [PATCH 11/13] refactor: streamline projection pushdown logic for join operations --- .../physical_optimizer/projection_pushdown.rs | 169 ++++++++---------- 1 file changed, 78 insertions(+), 91 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/projection_pushdown.rs b/datafusion/core/src/physical_optimizer/projection_pushdown.rs index 7c9f11beade03..75bf7b4456e31 100644 --- a/datafusion/core/src/physical_optimizer/projection_pushdown.rs +++ b/datafusion/core/src/physical_optimizer/projection_pushdown.rs @@ -51,6 +51,7 @@ use datafusion_physical_expr::{ utils::collect_columns, Partitioning, PhysicalExpr, PhysicalExprRef, PhysicalSortExpr, PhysicalSortRequirement, }; +use datafusion_physical_plan::joins::utils::{JoinOn, JoinOnRef}; use datafusion_physical_plan::streaming::StreamingTableExec; use datafusion_physical_plan::union::UnionExec; @@ -137,17 +138,11 @@ pub fn remove_unnecessary_projections( } else if let Some(union) = input.downcast_ref::() { try_pushdown_through_union(projection, union)? } else if let Some(hash_join) = input.downcast_ref::() { - try_pushdown_through_hash_join(projection, hash_join)?.map_or_else( - || try_embed_projection(projection, hash_join), - |e| Ok(Some(e)), - )? + try_pushdown_through_hash_join(projection, hash_join)? } else if let Some(cross_join) = input.downcast_ref::() { try_swapping_with_cross_join(projection, cross_join)? } else if let Some(nl_join) = input.downcast_ref::() { - try_pushdown_through_nested_loop_join(projection, nl_join)?.map_or_else( - || try_embed_projection(projection, nl_join), - |e| Ok(Some(e)), - )? + try_pushdown_through_nested_loop_join(projection, nl_join)? } else if let Some(sm_join) = input.downcast_ref::() { try_swapping_with_sort_merge_join(projection, sm_join)? } else if let Some(sym_join) = input.downcast_ref::() { @@ -635,39 +630,40 @@ fn collect_column_indices(exprs: &[(Arc, String)]) -> Vec Result>> { - // TODO: currently if there is projection in NestedLoopJoinExec, we can't push down projection to left or right input. Maybe we can pushdown the mixed projection later. - if nl_join.contain_projection() { - return Ok(None); - } +struct JoinData(ProjectionExec, ProjectionExec, Option, JoinOn); +fn try_pushdown_through_join( + projection: &ProjectionExec, + join_left: &Arc, + join_right: &Arc, + join_on: JoinOnRef, + schema: SchemaRef, + filter: Option<&JoinFilter>, +) -> Result> { + // Convert projected expressions to columns. We can not proceed if this is + // not possible. let Some(projection_as_columns) = physical_to_column_exprs(projection.expr()) else { return Ok(None); }; - let (far_right_left_col_ind, far_left_right_col_ind) = join_table_borders( - nl_join.left().schema().fields().len(), - &projection_as_columns, - ); + let (far_right_left_col_ind, far_left_right_col_ind) = + join_table_borders(join_left.schema().fields().len(), &projection_as_columns); if !join_allows_pushdown( &projection_as_columns, - &nl_join.schema(), + &schema, far_right_left_col_ind, far_left_right_col_ind, ) { return Ok(None); } - let new_filter = if let Some(filter) = nl_join.filter() { + let new_filter = if let Some(filter) = filter { match update_join_filter( &projection_as_columns[0..=far_right_left_col_ind as _], &projection_as_columns[far_left_right_col_ind as _..], filter, - nl_join.left().schema().fields().len(), + join_left.schema().fields().len(), ) { Some(updated_filter) => Some(updated_filter), None => return Ok(None), @@ -676,21 +672,52 @@ fn try_pushdown_through_nested_loop_join( None }; + let Some(new_on) = update_join_on( + &projection_as_columns[0..=far_right_left_col_ind as _], + &projection_as_columns[far_left_right_col_ind as _..], + join_on, + join_left.schema().fields().len(), + ) else { + return Ok(None); + }; + let (new_left, new_right) = new_join_children( &projection_as_columns, far_right_left_col_ind, far_left_right_col_ind, - nl_join.left(), - nl_join.right(), + join_left, + join_right, )?; - Ok(Some(Arc::new(NestedLoopJoinExec::try_new( - Arc::new(new_left), - Arc::new(new_right), - new_filter, - nl_join.join_type(), - None, - )?))) + Ok(Some(JoinData(new_left, new_right, new_filter, new_on))) +} + +fn try_pushdown_through_nested_loop_join( + projection: &ProjectionExec, + nl_join: &NestedLoopJoinExec, +) -> Result>> { + if nl_join.contain_projection() { + return Ok(None); + } + + if let Some(JoinData(left, right, filter, _)) = try_pushdown_through_join( + projection, + nl_join.left(), + nl_join.right(), + &[], + nl_join.schema(), + nl_join.filter(), + )? { + Ok(Some(Arc::new(NestedLoopJoinExec::try_new( + Arc::new(left), + Arc::new(right), + filter, + nl_join.join_type(), + None, + )?))) + } else { + try_embed_projection(projection, nl_join) + } } /// Tries to push `projection` down through `hash_join`. If possible, performs the @@ -705,68 +732,28 @@ fn try_pushdown_through_hash_join( return Ok(None); } - // Convert projected expressions to columns. We can not proceed if this is - // not possible. - let Some(projection_as_columns) = physical_to_column_exprs(projection.expr()) else { - return Ok(None); - }; - - let (far_right_left_col_ind, far_left_right_col_ind) = join_table_borders( - hash_join.left().schema().fields().len(), - &projection_as_columns, - ); - - if !join_allows_pushdown( - &projection_as_columns, - &hash_join.schema(), - far_right_left_col_ind, - far_left_right_col_ind, - ) { - return Ok(None); - } - - let Some(new_on) = update_join_on( - &projection_as_columns[0..=far_right_left_col_ind as _], - &projection_as_columns[far_left_right_col_ind as _..], + if let Some(JoinData(left, right, filter, join_on)) = try_pushdown_through_join( + projection, + hash_join.left(), + hash_join.right(), hash_join.on(), - hash_join.left().schema().fields().len(), - ) else { - return Ok(None); - }; - - let new_filter = if let Some(filter) = hash_join.filter() { - match update_join_filter( - &projection_as_columns[0..=far_right_left_col_ind as _], - &projection_as_columns[far_left_right_col_ind as _..], + hash_join.schema(), + hash_join.filter(), + )? { + Ok(Some(Arc::new(HashJoinExec::try_new( + Arc::new(left), + Arc::new(right), + join_on, filter, - hash_join.left().schema().fields().len(), - ) { - Some(updated_filter) => Some(updated_filter), - None => return Ok(None), - } + hash_join.join_type(), + // Early return if projection is not None + None, + *hash_join.partition_mode(), + hash_join.null_equals_null, + )?))) } else { - None - }; - - let (new_left, new_right) = new_join_children( - &projection_as_columns, - far_right_left_col_ind, - far_left_right_col_ind, - hash_join.left(), - hash_join.right(), - )?; - - Ok(Some(Arc::new(HashJoinExec::try_new( - Arc::new(new_left), - Arc::new(new_right), - new_on, - new_filter, - hash_join.join_type(), - // Returned early if projection is not None - None, - *hash_join.partition_mode(), - hash_join.null_equals_null, - )?))) + try_embed_projection(projection, hash_join) + } } /// Tries to swap the projection with its input [`CrossJoinExec`]. If it can be done, From 47fbc05907855caa7026bf5cc7057bfae7bcc4fa Mon Sep 17 00:00:00 2001 From: berkaysynnada Date: Thu, 16 Jan 2025 10:20:35 +0300 Subject: [PATCH 12/13] minor --- .../physical_optimizer/projection_pushdown.rs | 54 +++++++++++++------ .../physical-plan/src/joins/hash_join.rs | 4 +- .../src/joins/nested_loop_join.rs | 4 +- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/projection_pushdown.rs b/datafusion/core/src/physical_optimizer/projection_pushdown.rs index 75bf7b4456e31..2f1cf3a8e659f 100644 --- a/datafusion/core/src/physical_optimizer/projection_pushdown.rs +++ b/datafusion/core/src/physical_optimizer/projection_pushdown.rs @@ -630,7 +630,12 @@ fn collect_column_indices(exprs: &[(Arc, String)]) -> Vec, JoinOn); +struct JoinData { + projected_left_child: ProjectionExec, + projected_right_child: ProjectionExec, + join_filter: Option, + join_on: JoinOn, +} fn try_pushdown_through_join( projection: &ProjectionExec, @@ -640,8 +645,7 @@ fn try_pushdown_through_join( schema: SchemaRef, filter: Option<&JoinFilter>, ) -> Result> { - // Convert projected expressions to columns. We can not proceed if this is - // not possible. + // Convert projected expressions to columns. We can not proceed if this is not possible. let Some(projection_as_columns) = physical_to_column_exprs(projection.expr()) else { return Ok(None); }; @@ -689,18 +693,32 @@ fn try_pushdown_through_join( join_right, )?; - Ok(Some(JoinData(new_left, new_right, new_filter, new_on))) + Ok(Some(JoinData { + projected_left_child: new_left, + projected_right_child: new_right, + join_filter: new_filter, + join_on: new_on, + })) } +/// Tries to push `projection` down through `nested_loop_join`. If possible, performs the +/// pushdown and returns a new [`NestedLoopJoinExec`] as the top plan which has projections +/// as its children. Otherwise, returns `None`. fn try_pushdown_through_nested_loop_join( projection: &ProjectionExec, nl_join: &NestedLoopJoinExec, ) -> Result>> { - if nl_join.contain_projection() { + // TODO: currently if there is projection in NestedLoopJoinExec, we can't push down projection to left or right input. Maybe we can pushdown the mixed projection later. + if nl_join.contains_projection() { return Ok(None); } - if let Some(JoinData(left, right, filter, _)) = try_pushdown_through_join( + if let Some(JoinData { + projected_left_child, + projected_right_child, + join_filter, + .. + }) = try_pushdown_through_join( projection, nl_join.left(), nl_join.right(), @@ -709,10 +727,11 @@ fn try_pushdown_through_nested_loop_join( nl_join.filter(), )? { Ok(Some(Arc::new(NestedLoopJoinExec::try_new( - Arc::new(left), - Arc::new(right), - filter, + Arc::new(projected_left_child), + Arc::new(projected_right_child), + join_filter, nl_join.join_type(), + // Returned early if projection is not None None, )?))) } else { @@ -728,11 +747,16 @@ fn try_pushdown_through_hash_join( hash_join: &HashJoinExec, ) -> Result>> { // TODO: currently if there is projection in HashJoinExec, we can't push down projection to left or right input. Maybe we can pushdown the mixed projection later. - if hash_join.contain_projection() { + if hash_join.contains_projection() { return Ok(None); } - if let Some(JoinData(left, right, filter, join_on)) = try_pushdown_through_join( + if let Some(JoinData { + projected_left_child, + projected_right_child, + join_filter, + join_on, + }) = try_pushdown_through_join( projection, hash_join.left(), hash_join.right(), @@ -741,12 +765,12 @@ fn try_pushdown_through_hash_join( hash_join.filter(), )? { Ok(Some(Arc::new(HashJoinExec::try_new( - Arc::new(left), - Arc::new(right), + Arc::new(projected_left_child), + Arc::new(projected_right_child), join_on, - filter, + join_filter, hash_join.join_type(), - // Early return if projection is not None + // Returned early if projection is not None None, *hash_join.partition_mode(), hash_join.null_equals_null, diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index 15cca655ddad4..4540a3b90a6ee 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -469,7 +469,7 @@ impl HashJoinExec { } /// Return whether the join contains a projection - pub fn contain_projection(&self) -> bool { + pub fn contains_projection(&self) -> bool { self.projection.is_some() } @@ -626,7 +626,7 @@ impl DisplayAs for HashJoinExec { || "".to_string(), |f| format!(", filter={}", f.expression()), ); - let display_projections = if self.contain_projection() { + let display_projections = if self.contains_projection() { format!( ", projection=[{}]", self.projection diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index 7c96e1975655b..5485e0e0c5662 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -326,7 +326,7 @@ impl NestedLoopJoinExec { ] } - pub fn contain_projection(&self) -> bool { + pub fn contains_projection(&self) -> bool { self.projection.is_some() } @@ -398,7 +398,7 @@ impl DisplayAs for NestedLoopJoinExec { || "".to_string(), |f| format!(", filter={}", f.expression()), ); - let display_projections = if self.contain_projection() { + let display_projections = if self.contains_projection() { format!( ", projection=[{}]", self.projection From 9dc4670bf6323361fd827b62a53963de8c46f35d Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Thu, 16 Jan 2025 23:09:48 +0800 Subject: [PATCH 13/13] fmt Signed-off-by: Jay Zhan --- datafusion/physical-plan/src/joins/hash_join.rs | 4 ++-- datafusion/physical-plan/src/joins/nested_loop_join.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index a61ebda2a9236..e4f3cdce2f6d3 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -25,8 +25,8 @@ use std::task::Poll; use std::{any::Any, vec}; use super::utils::{ - asymmetric_join_output_partitioning, reorder_output_after_swap, swap_join_projection, - get_final_indices_from_shared_bitmap, + asymmetric_join_output_partitioning, get_final_indices_from_shared_bitmap, + reorder_output_after_swap, swap_join_projection, }; use super::{ utils::{OnceAsync, OnceFut}, diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index d0ffb02c5741f..f416694e38308 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -24,9 +24,9 @@ use std::sync::Arc; use std::task::Poll; use super::utils::{ - asymmetric_join_output_partitioning, need_produce_result_in_final, get_final_indices_from_shared_bitmap, - reorder_output_after_swap, swap_join_projection, BatchSplitter, BatchTransformer, - NoopBatchTransformer, StatefulStreamResult, + asymmetric_join_output_partitioning, get_final_indices_from_shared_bitmap, + need_produce_result_in_final, reorder_output_after_swap, swap_join_projection, + BatchSplitter, BatchTransformer, NoopBatchTransformer, StatefulStreamResult, }; use crate::coalesce_partitions::CoalescePartitionsExec; use crate::common::can_project;