Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 65 additions & 19 deletions datafusion/core/src/physical_planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ use arrow_schema::Field;
use datafusion_catalog::ScanArgs;
use datafusion_common::Column;
use datafusion_common::HashMap as DFHashMap;
use datafusion_common::config::ConfigOptions;
use datafusion_common::display::ToStringifiedPlan;
use datafusion_common::format::ExplainAnalyzeCategories;
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
Expand Down Expand Up @@ -103,9 +104,11 @@ use datafusion_physical_expr::{
use datafusion_physical_plan::empty::EmptyExec;
use datafusion_physical_plan::execution_plan::InvariantLevel;
use datafusion_physical_plan::joins::PiecewiseMergeJoinExec;
use datafusion_physical_plan::operator_statistics::StatisticsRegistry;
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::recursive_query::RecursiveQueryExec;
use datafusion_physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink};
use datafusion_physical_plan::statistics::StatisticsContext;
use datafusion_physical_plan::unnest::ListUnnest;
use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Session};

Expand All @@ -123,18 +126,34 @@ pub use datafusion_session::{ExtensionPlanner, PhysicalPlanner};

struct SessionOptimizerContext<'a> {
session: &'a dyn Session,
statistics_context: StatisticsContext,
}

impl<'a> SessionOptimizerContext<'a> {
fn new(session: &'a dyn Session) -> Self {
let statistics_context = match session.statistics_registry() {
Some(registry) => StatisticsContext::new_with_registry(registry.clone()),
None => StatisticsContext::new(),
};
Self {
session,
statistics_context,
}
}
}

impl PhysicalOptimizerContext for SessionOptimizerContext<'_> {
fn config_options(&self) -> &datafusion_common::config::ConfigOptions {
fn config_options(&self) -> &ConfigOptions {
self.session.config_options()
}

fn statistics_registry(
&self,
) -> Option<&datafusion_physical_plan::operator_statistics::StatisticsRegistry> {
fn statistics_registry(&self) -> Option<&StatisticsRegistry> {
self.session.statistics_registry()
}

fn statistics_context(&self) -> Option<&StatisticsContext> {
Some(&self.statistics_context)
}
}

/// Default single node physical query planner that converts a
Expand Down Expand Up @@ -3123,9 +3142,7 @@ impl DefaultPhysicalPlanner {
InvariantChecker(InvariantLevel::Always).check(&plan)?;

let mut new_plan = Arc::clone(&plan);
let optimizer_context = SessionOptimizerContext {
session: session_state,
};
let optimizer_context = SessionOptimizerContext::new(session_state);
Comment thread
asolimando marked this conversation as resolved.
Resolved
for optimizer in optimizers {
let before_schema = new_plan.schema();
new_plan = optimizer
Expand Down Expand Up @@ -3536,6 +3553,7 @@ mod tests {
use arrow::datatypes::{DataType, Field, Int32Type};
use arrow_schema::{FieldRef, SchemaRef};
use datafusion_catalog::CatalogProviderList;
use datafusion_common::Statistics;
use datafusion_common::config::{ConfigOptions, TableOptions};
use datafusion_common::{
DFSchemaRef, ScalarValue, SplitPoint, TableReference, ToDFSchema as _,
Expand All @@ -3558,12 +3576,16 @@ mod tests {
use datafusion_functions_aggregate::expr_fn::sum;
use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
use datafusion_physical_plan::operator_statistics::StatisticsRegistry;
use datafusion_physical_plan::statistics::StatisticsArgs;
use datafusion_physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions};
use datafusion_session::QueryPlanner;
use parking_lot::Mutex as SyncMutex;

/// Records the root statistics computed with the shared statistics context
#[derive(Debug)]
struct ContextCheckingRule {
invoked: Arc<AtomicBool>,
recorded: Arc<SyncMutex<Vec<Arc<Statistics>>>>,
}

impl PhysicalOptimizerRule for ContextCheckingRule {
Expand All @@ -3581,7 +3603,12 @@ mod tests {
context: &dyn PhysicalOptimizerContext,
) -> Result<Arc<dyn ExecutionPlan>> {
assert!(context.statistics_registry().is_some());
self.invoked.store(true, AtomicOrdering::Relaxed);
let statistics_context = context
.statistics_context()
.expect("the planner shares a statistics context");
let statistics =
statistics_context.compute_arc(&plan, &StatisticsArgs::new())?;
self.recorded.lock().push(statistics);
Ok(plan)
}

Expand Down Expand Up @@ -3644,10 +3671,7 @@ mod tests {
self.inner.physical_optimizers()
}

fn statistics_registry(
&self,
) -> Option<&datafusion_physical_plan::operator_statistics::StatisticsRegistry>
{
fn statistics_registry(&self) -> Option<&StatisticsRegistry> {
self.inner.statistics_registry()
}

Expand Down Expand Up @@ -3816,15 +3840,13 @@ mod tests {

#[tokio::test]
async fn plans_with_non_session_state_implementation() -> Result<()> {
let invoked = Arc::new(AtomicBool::new(false));
let recorded = Arc::new(SyncMutex::new(vec![]));
let inner = SessionStateBuilder::new()
.with_default_features()
.with_physical_optimizer_rules(vec![Arc::new(ContextCheckingRule {
invoked: Arc::clone(&invoked),
recorded: Arc::clone(&recorded),
})])
.with_statistics_registry(
datafusion_physical_plan::operator_statistics::StatisticsRegistry::new(),
)
.with_statistics_registry(StatisticsRegistry::new())
.build();
let query_planner_invoked = Arc::new(AtomicBool::new(false));
let session = TestSession {
Expand All @@ -3839,7 +3861,31 @@ mod tests {
let physical_plan = session.create_physical_plan(&logical_plan).await?;
assert!(physical_plan.is::<EmptyExec>());
assert!(query_planner_invoked.load(AtomicOrdering::Relaxed));
assert!(invoked.load(AtomicOrdering::Relaxed));
assert_eq!(recorded.lock().len(), 1);
Ok(())
}

#[tokio::test]
async fn optimizer_rules_share_statistics_context() -> Result<()> {
let recorded = Arc::new(SyncMutex::new(vec![]));
let rule = || -> Arc<dyn PhysicalOptimizerRule + Send + Sync> {
Arc::new(ContextCheckingRule {
recorded: Arc::clone(&recorded),
})
};
let session_state = SessionStateBuilder::new()
.with_default_features()
.with_physical_optimizer_rules(vec![rule(), rule()])
.with_statistics_registry(StatisticsRegistry::new())
.build();

let logical_plan = LogicalPlanBuilder::empty(false).build()?;
session_state.create_physical_plan(&logical_plan).await?;

// The second rule reads the statistics the first rule cached
let recorded = recorded.lock();
assert_eq!(recorded.len(), 2);
assert!(Arc::ptr_eq(&recorded[0], &recorded[1]));
Ok(())
}

Expand Down
49 changes: 49 additions & 0 deletions datafusion/core/tests/physical_optimizer/aggregate_statistics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ use datafusion::datasource::memory::MemTable;
use datafusion::datasource::memory::MemorySourceConfig;
use datafusion::datasource::physical_plan::ParquetSource;
use datafusion::datasource::source::DataSourceExec;
use datafusion::execution::session_state::SessionStateBuilder;
use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_common::cast::as_int64_array;
use datafusion_common::config::ConfigOptions;
Expand All @@ -52,7 +53,11 @@ use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion_physical_plan::common;
use datafusion_physical_plan::displayable;
use datafusion_physical_plan::filter::FilterExec;
use datafusion_physical_plan::operator_statistics::{
ClosureStatisticsProvider, StatisticsRegistry, StatisticsResult,
};
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::statistics::StatisticsArgs;

/// Mock data using a MemorySourceConfig which has an exact count statistic
fn mock_data() -> Result<Arc<DataSourceExec>> {
Expand Down Expand Up @@ -856,3 +861,47 @@ async fn test_sum_from_statistics() -> Result<()> {

Ok(())
}

/// `AggregateStatistics` trusts an `Exact` row count from a registered
/// provider: it answers `COUNT(*)` without scanning
#[tokio::test]
async fn aggregate_statistics_consults_statistics_providers() -> Result<()> {
let provider = ClosureStatisticsProvider::with_matches(
|plan| plan.is::<DataSourceExec>(),
|plan, child_stats| {
let child_stats = child_stats
.iter()
.map(|c| Arc::clone(c.base_arc()))
.collect::<Vec<_>>();
let mut stats = Arc::unwrap_or_clone(
plan.statistics_from_inputs(&child_stats, &StatisticsArgs::new())?,
);
stats.num_rows = Precision::Exact(42);
Ok(StatisticsResult::Computed(stats.into()))
},
);
let state = SessionStateBuilder::new()
.with_default_features()
.with_statistics_registry(StatisticsRegistry::with_providers(vec![Arc::new(
provider,
)]))
.build();
let ctx = SessionContext::new_with_state(state);
ctx.sql("CREATE TABLE t AS VALUES (1), (2), (3)")
.await?
.collect()
.await?;

let batches = ctx.sql("SELECT COUNT(*) FROM t").await?.collect().await?;
assert_batches_eq!(
&[
"+----------+",
"| count(*) |",
"+----------+",
"| 42 |",
"+----------+"
],
&batches
);
Ok(())
}
48 changes: 48 additions & 0 deletions datafusion/core/tests/physical_optimizer/limit_pushdown.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,28 @@ use crate::physical_optimizer::test_utils::{

use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::execution::session_state::SessionStateBuilder;
use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_common::config::ConfigOptions;
use datafusion_common::error::Result;
use datafusion_common::stats::Precision;
use datafusion_expr::{JoinType, Operator};
use datafusion_physical_expr::Partitioning;
use datafusion_physical_expr::expressions::{BinaryExpr, col, lit};
use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;
use datafusion_physical_plan::aggregates::AggregateExec;
use datafusion_physical_plan::empty::EmptyExec;
use datafusion_physical_plan::filter::FilterExec;
use datafusion_physical_plan::joins::NestedLoopJoinExec;
use datafusion_physical_plan::operator_statistics::{
ClosureStatisticsProvider, StatisticsRegistry, StatisticsResult,
};
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::statistics::StatisticsArgs;
use datafusion_physical_plan::{ExecutionPlan, get_plan_string};

fn create_schema() -> SchemaRef {
Expand Down Expand Up @@ -834,3 +842,43 @@ fn outer_offset_with_same_sort_key_still_pushes_limit() -> Result<()> {

Ok(())
}

/// `LimitPushdown` trusts an `Exact(0)` row count from a registered
/// provider: it removes a limit over that input
#[tokio::test]
async fn limit_pushdown_consults_statistics_providers() -> Result<()> {
let provider = ClosureStatisticsProvider::with_matches(
|plan| plan.is::<AggregateExec>(),
|plan, child_stats| {
let child_stats = child_stats
.iter()
.map(|c| Arc::clone(c.base_arc()))
.collect::<Vec<_>>();
let mut stats = Arc::unwrap_or_clone(
plan.statistics_from_inputs(&child_stats, &StatisticsArgs::new())?,
);
stats.num_rows = Precision::Exact(0);
Ok(StatisticsResult::Computed(stats.into()))
},
);
let state = SessionStateBuilder::new()
.with_default_features()
.with_config(SessionConfig::new().with_target_partitions(1))
.with_statistics_registry(StatisticsRegistry::with_providers(vec![Arc::new(
provider,
)]))
.build();
let ctx = SessionContext::new_with_state(state);
ctx.sql("CREATE TABLE t AS VALUES (1), (2), (3)")
.await?
.collect()
.await?;

let batches = ctx
.sql("SELECT column1, count(*) FROM t GROUP BY column1 LIMIT 1")
.await?
.collect()
.await?;
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 3);
Ok(())
}
29 changes: 20 additions & 9 deletions datafusion/physical-optimizer/src/aggregate_statistics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,15 @@ use datafusion_physical_plan::aggregates::{
};
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::statistics::StatisticsArgs;
use datafusion_physical_plan::udaf::{
AggregateFunctionExpr, StatisticsArgs as PlanStatisticsArgs,
};
use datafusion_physical_plan::{ExecutionPlan, expressions};
use std::sync::Arc;

use crate::PhysicalOptimizerRule;
use crate::optimizer::{ConfigOnlyContext, PhysicalOptimizerContext};

/// Optimizer that uses available statistics for aggregate functions
#[derive(Default, Debug)]
Expand All @@ -46,20 +47,26 @@ impl AggregateStatistics {
}

impl PhysicalOptimizerRule for AggregateStatistics {
#[cfg_attr(feature = "recursive_protection", recursive::recursive)]
#[expect(clippy::allow_attributes)] // See https://github.com/apache/datafusion/issues/18881#issuecomment-3621545670
#[allow(clippy::only_used_in_recursion)] // See https://github.com/rust-lang/rust-clippy/issues/14566
fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
self.optimize_with_context(plan, &ConfigOnlyContext::new(config))
}

#[cfg_attr(feature = "recursive_protection", recursive::recursive)]
fn optimize_with_context(
&self,
plan: Arc<dyn ExecutionPlan>,
context: &dyn PhysicalOptimizerContext,
) -> Result<Arc<dyn ExecutionPlan>> {
if let Some(partial_agg_exec) = take_optimizable(&plan) {
let partial_agg_exec = partial_agg_exec
.downcast_ref::<AggregateExec>()
.expect("take_optimizable() ensures that this is a AggregateExec");
let stats = StatisticsContext::new()
.compute(partial_agg_exec.input().as_ref(), &StatisticsArgs::new())?;
let stats = context
Comment thread
asolimando marked this conversation as resolved.
Resolved
.compute_statistics(partial_agg_exec.input(), &StatisticsArgs::new())?;
let mut projections = vec![];
for expr in partial_agg_exec.aggr_expr() {
let field = expr.field();
Expand Down Expand Up @@ -92,13 +99,17 @@ impl PhysicalOptimizerRule for AggregateStatistics {
)?))
} else {
plan.map_children(|child| {
self.optimize(child, config).map(Transformed::yes)
self.optimize_with_context(child, context)
.map(Transformed::yes)
})
.data()
}
} else {
plan.map_children(|child| self.optimize(child, config).map(Transformed::yes))
.data()
plan.map_children(|child| {
self.optimize_with_context(child, context)
.map(Transformed::yes)
})
.data()
}
}

Expand Down
Loading
Loading