diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 63a7797e396ef..8c5c84d779476 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1940,16 +1940,14 @@ config_namespace! { /// projections or aliases. pub enable_unions_to_filter: bool, default = false - /// Prefer native grouped DISTINCT aggregation when target_partitions is 1 - /// and every aggregate explicitly supports a specialized GroupsAccumulator - /// for its DISTINCT argument types. Defaults to false, preserving the - /// two-phase GROUP BY rewrite. + /// When `true`, queries with a single DISTINCT aggregate will take less + /// memory but may also run more slowly, by skipping the two-phase `GROUP BY` + /// rewrite and executing with native accumulators. /// - /// The native plan can avoid materializing distinct (group, value) pairs, - /// but can be slower for many groups with duplicate values. It must fit - /// each group's distinct values in memory, so queries that succeed by - /// spilling with the rewrite may fail with this option enabled. One target - /// partition can still have multiple input partitions and merge native states. + /// Only applies when `target_partitions` is 1 and all DISTINCT arguments are + /// fixed-width, non-float types. See + /// [`SingleDistinctToGroupBy`](https://docs.rs/datafusion-optimizer/latest/datafusion_optimizer/single_distinct_to_groupby/struct.SingleDistinctToGroupBy.html) + /// for details. pub single_distinct_prefer_native: bool, default = false } } diff --git a/datafusion/optimizer/src/single_distinct_to_groupby.rs b/datafusion/optimizer/src/single_distinct_to_groupby.rs index 73efe0de34475..e7f8ee652320c 100644 --- a/datafusion/optimizer/src/single_distinct_to_groupby.rs +++ b/datafusion/optimizer/src/single_distinct_to_groupby.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use crate::optimizer::ApplyOrder; use crate::{OptimizerConfig, OptimizerRule}; +use arrow::datatypes::DataType; use datafusion_common::{ DFSchema, DataFusionError, HashSet, Result, assert_or_internal_err, @@ -17,7 +18,7 @@ use datafusion_expr::builder::project; use datafusion_expr::expr::AggregateFunctionParams; use datafusion_expr::expr_schema::ExprSchemable; use datafusion_expr::{ - AggregateUDF, Expr, cast, col, + AggregateUDF, DistinctHandling, Expr, cast, col, expr::AggregateFunction, lit, logical_plan::{Aggregate, LogicalPlan}, @@ -68,9 +69,20 @@ use datafusion_expr::{ /// `sum` of no rows is NULL. /// /// See `rewrite_pays_for_count` for when the `count` is allowed. +/// +/// # Memory vs Performance Tradeoffs (`single_distinct_prefer_native`) +/// /// With `single_distinct_prefer_native` enabled and one target partition, grouped -/// all-DISTINCT aggregates stay native if every function explicitly supports a -/// specialized `GroupsAccumulator` for its argument types. +/// all-DISTINCT aggregates stay native when every DISTINCT argument is a +/// fixed-width, non-float type (such as integers, decimals, dates). +/// +/// Skipping the two-phase rewrite avoids materializing intermediate +/// `(group, distinct value)` pairs, lowering peak memory when there are few +/// groups with many distinct values each. With many small groups it can use +/// more memory and run slower, because functions without a specialized DISTINCT +/// `GroupsAccumulator` maintain one accumulator per group. Queries without +/// `GROUP BY`, multi-partition queries, or queries with float/variable-length +/// arguments continue to use the two-phase rewrite. #[derive(Default, Debug)] pub struct SingleDistinctToGroupBy {} @@ -110,7 +122,49 @@ impl CountRollup { } } -/// Check whether all aggregate exprs are distinct on a single field. +/// Returns `true` for fixed-width primitive types where single-phase distinct +/// accumulation avoids heap allocation churn **and** where bit-identity equals +/// value-equality, so that skipping the GROUP BY rewrite cannot silently change +/// query results. +/// +/// Float types are deliberately excluded: IEEE 754 defines `-0.0 == 0.0`, so a +/// `GROUP BY` on floats merges those two bit-patterns into one group. The +/// native distinct accumulators hash raw bits and therefore count them +/// separately. Keeping floats in this list would let `single_distinct_prefer_native` +/// skip the rewrite for a float column, producing a different answer than the +/// non-rewritten path — a silent correctness regression. +/// +/// `Duration` is also excluded: `avg` has no native DISTINCT accumulator for duration +/// types, so skipping the rewrite would cause a runtime error. +fn is_fixed_width_type(dt: &DataType) -> bool { + matches!( + dt, + DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Date32 + | DataType::Date64 + | DataType::Time32(_) + | DataType::Time64(_) + | DataType::Timestamp(_, _) + | DataType::Interval(_) + | DataType::Decimal128(_, _) + | DataType::Decimal256(_, _) + ) +} + +/// Check whether all aggregate exprs are eligible for the single-distinct-to-group-by rewrite. +/// +/// When `use_native_grouped_distinct` is set, all-DISTINCT queries on fixed-width, +/// non-float types skip the rewrite and run natively to reduce intermediate memory. +/// All other queries (un-grouped, multi-partition, mixed distinct, unsupported UDAFs, +/// floats, or variable-width types) proceed with the rewrite. fn is_single_distinct_agg( aggr_expr: &[Expr], input_schema: &DFSchema, @@ -158,8 +212,17 @@ fn is_single_distinct_agg( if aggregate_count != aggr_expr.len() || fields_set.len() != 1 { return Ok(false); } - // With one target partition, a native GroupsAccumulator avoids building an - // extra aggregate with a row per (group, distinct value) pair. With multiple + // Aggregates with DistinctHandling::Unsupported (e.g. approx_median) cannot + // run natively with DISTINCT; they must always be rewritten. + if distinct_aggs + .iter() + .any(|(func, _)| func.distinct_handling() == DistinctHandling::Unsupported) + { + return Ok(true); + } + + // With one target partition, native accumulation avoids building an extra + // aggregate with a row per (group, distinct value) pair. With multiple // partitions, the rewrite also distributes deduplication by both keys; // keeping DISTINCT instead repartitions lists of values by group and merges // them in the final accumulator. That can concentrate work on a few groups. @@ -167,15 +230,23 @@ fn is_single_distinct_agg( // ALL or an existing partitioned table), requiring Partial/Final state merging. // This is opt-in: the rule cannot see memory limits or cost the input, and // native DISTINCT must fit each group's distinct values in memory. - if use_native_grouped_distinct - && distinct_aggs.len() == aggregate_count - // An unknown answer must preserve the existing rewrite. - && distinct_groups_support(&distinct_aggs, input_schema)? + // + // The rewrite is skipped only when every aggregate is a DISTINCT aggregate + // over fixed-width primitive types (integers, decimals, dates, etc.) where + // bit-identity equals value-equality. Floats are excluded to avoid -0.0 vs + // 0.0 IEEE 754 discrepancy, and variable-length types (strings, binary) are + // excluded to prevent heap allocation churn. + if use_native_grouped_distinct && distinct_aggs.len() == aggregate_count { + let arg_types = distinct_aggs .iter() - .all(|support| *support == Some(true)) - { - return Ok(false); + .flat_map(|(_, args)| args.iter()) + .map(|arg| arg.get_type(input_schema)) + .collect::>>()?; + if arg_types.iter().all(is_fixed_width_type) { + return Ok(false); + } } + if has_count_rollup && !rewrite_pays_for_count(&distinct_aggs, input_schema)? { return Ok(false); } diff --git a/datafusion/sqllogictest/test_files/grouped_count_distinct.slt b/datafusion/sqllogictest/test_files/grouped_count_distinct.slt index 1490442ab383a..38c4dd42b11b1 100644 --- a/datafusion/sqllogictest/test_files/grouped_count_distinct.slt +++ b/datafusion/sqllogictest/test_files/grouped_count_distinct.slt @@ -2,8 +2,8 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -# With the opt-in enabled and one target partition, grouped integer DISTINCT -# counts keep their specialized accumulator. Multiple target partitions retain +# With the opt-in enabled and one target partition, grouped all-DISTINCT aggregates +# with fixed-width, non-float arguments stay native. Multiple target partitions retain # distributed deduplication. # String and floating-point DISTINCT arguments still use the two-phase rewrite. @@ -80,15 +80,20 @@ logical_plan 03)----Aggregate: groupBy=[[numeric_counts.g, numeric_counts.x AS alias1]], aggr=[[sum(CAST(numeric_counts.x AS Int64)) AS alias2]] 04)------TableScan: numeric_counts projection=[g, x] -# `sum` does not report specialized DISTINCT support, so it keeps the rewrite. +# `sum` with integer DISTINCT argument stays native. query TT EXPLAIN SELECT g, sum(DISTINCT x) FROM numeric_counts GROUP BY g; ---- logical_plan -01)Projection: numeric_counts.g, sum(alias1) AS sum(DISTINCT numeric_counts.x) -02)--Aggregate: groupBy=[[numeric_counts.g]], aggr=[[sum(alias1)]] -03)----Aggregate: groupBy=[[numeric_counts.g, CAST(numeric_counts.x AS Int64) AS alias1]], aggr=[[]] -04)------TableScan: numeric_counts projection=[g, x] +01)Aggregate: groupBy=[[numeric_counts.g]], aggr=[[sum(DISTINCT CAST(numeric_counts.x AS Int64))]] +02)--TableScan: numeric_counts projection=[g, x] + +query II rowsort +SELECT g, sum(DISTINCT x) FROM numeric_counts GROUP BY g; +---- +1 3 +2 NULL +3 3 # The global aggregate does not use a GroupsAccumulator. query TT diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index c1aabc2946ea6..c2f135f29709f 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -505,7 +505,7 @@ datafusion.optimizer.repartition_file_scans true When set to `true`, datasource datafusion.optimizer.repartition_joins true Should DataFusion repartition data using the join keys to execute joins in parallel using the provided `target_partitions` level datafusion.optimizer.repartition_sorts true Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below ```text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` would turn into the plan below which performs better in multithreaded environments ```text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ``` datafusion.optimizer.repartition_windows true Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level -datafusion.optimizer.single_distinct_prefer_native false Prefer native grouped DISTINCT aggregation when target_partitions is 1 and every aggregate explicitly supports a specialized GroupsAccumulator for its DISTINCT argument types. Defaults to false, preserving the two-phase GROUP BY rewrite. The native plan can avoid materializing distinct (group, value) pairs, but can be slower for many groups with duplicate values. It must fit each group's distinct values in memory, so queries that succeed by spilling with the rewrite may fail with this option enabled. One target partition can still have multiple input partitions and merge native states. +datafusion.optimizer.single_distinct_prefer_native false When `true`, queries with a single DISTINCT aggregate will take less memory but may also run more slowly, by skipping the two-phase `GROUP BY` rewrite and executing with native accumulators. Only applies when `target_partitions` is 1 and all DISTINCT arguments are fixed-width, non-float types. See [`SingleDistinctToGroupBy`](https://docs.rs/datafusion-optimizer/latest/datafusion_optimizer/single_distinct_to_groupby/struct.SingleDistinctToGroupBy.html) for details. datafusion.optimizer.skip_failed_rules false When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail datafusion.optimizer.subset_repartition_threshold 4 Partition count threshold for subset satisfaction optimization. When the current partition count is >= this threshold, DataFusion will skip repartitioning if the required partitioning expression is a subset of the current partition expression such as Hash(a) satisfies Hash(a, b). When the current partition count is < this threshold, DataFusion will repartition to increase parallelism even when subset satisfaction applies. Set to 0 to always repartition (disable subset satisfaction optimization). Set to a high value to always use subset satisfaction. Example (subset_repartition_threshold = 4): ```text Hash([a]) satisfies Hash([a, b]) because (Hash([a, b]) is subset of Hash([a]) If current partitions (3) < threshold (4), repartition: AggregateExec: mode=FinalPartitioned, gby=[a, b], aggr=[SUM(x)] RepartitionExec: partitioning=Hash([a, b], 8), input_partitions=3 AggregateExec: mode=Partial, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 3) If current partitions (8) >= threshold (4), use subset satisfaction: AggregateExec: mode=SinglePartitioned, gby=[a, b], aggr=[SUM(x)] DataSourceExec: file_groups={...}, output_partitioning=Hash([a], 8) ``` datafusion.optimizer.top_down_join_key_reordering true When set to true, the physical plan optimizer will run a top down process to reorder the join keys diff --git a/datafusion/sqllogictest/test_files/single_distinct_to_groupby.slt b/datafusion/sqllogictest/test_files/single_distinct_to_groupby.slt index 78308f7897c6e..208deaf678af9 100644 --- a/datafusion/sqllogictest/test_files/single_distinct_to_groupby.slt +++ b/datafusion/sqllogictest/test_files/single_distinct_to_groupby.slt @@ -470,3 +470,173 @@ physical_plan statement ok DROP TABLE t_decimal_distinct; + + +######## +# single_distinct_prefer_native config option +######## + +statement ok +CREATE TABLE t_prefer_mem AS SELECT * FROM (VALUES (1, 10), (1, 20), (2, 10)) AS v(g, x); + +# By default (single_distinct_prefer_native = false), lone-distinct queries rewrite to two-phase GROUP BY +query TT +EXPLAIN SELECT g, count(DISTINCT x) FROM t_prefer_mem GROUP BY g; +---- +logical_plan +01)Projection: t_prefer_mem.g, count(alias1) AS count(DISTINCT t_prefer_mem.x) +02)--Aggregate: groupBy=[[t_prefer_mem.g]], aggr=[[count(alias1)]] +03)----Aggregate: groupBy=[[t_prefer_mem.g, t_prefer_mem.x AS alias1]], aggr=[[]] +04)------TableScan: t_prefer_mem projection=[g, x] +physical_plan +01)ProjectionExec: expr=[g@0 as g, count(alias1)@1 as count(DISTINCT t_prefer_mem.x)] +02)--AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(alias1)] +03)----RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=4 +04)------AggregateExec: mode=Partial, gby=[g@0 as g], aggr=[count(alias1)] +05)--------AggregateExec: mode=FinalPartitioned, gby=[g@0 as g, alias1@1 as alias1], aggr=[] +06)----------RepartitionExec: partitioning=Hash([g@0, alias1@1], 4), input_partitions=1 +07)------------AggregateExec: mode=Partial, gby=[g@0 as g, x@1 as alias1], aggr=[] +08)--------------DataSourceExec: partitions=1, partition_sizes=[1] + +# stddev cannot run DISTINCT itself, so it is rewritten even next to count(*). +query IIR rowsort +SELECT g, count(*), stddev(DISTINCT x) FROM t_prefer_mem GROUP BY g; +---- +1 2 7.071067811865 +2 1 NULL + +# When single_distinct_prefer_native = true and target_partitions = 1, the rewrite is skipped to reduce peak memory +statement ok +set datafusion.execution.target_partitions = 1; + +statement ok +set datafusion.optimizer.single_distinct_prefer_native = true; + +query TT +EXPLAIN SELECT g, count(DISTINCT x) FROM t_prefer_mem GROUP BY g; +---- +logical_plan +01)Aggregate: groupBy=[[t_prefer_mem.g]], aggr=[[count(DISTINCT t_prefer_mem.x)]] +02)--TableScan: t_prefer_mem projection=[g, x] +physical_plan +01)AggregateExec: mode=Single, gby=[g@0 as g], aggr=[count(DISTINCT t_prefer_mem.x)] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +# Query results are identical under both settings +query II +SELECT g, count(DISTINCT x) FROM t_prefer_mem GROUP BY g ORDER BY g; +---- +1 2 +2 1 + +# Queries without GROUP BY retain the rewrite even with single_distinct_prefer_native = true +query TT +EXPLAIN SELECT count(DISTINCT x) FROM t_prefer_mem; +---- +logical_plan +01)Projection: count(alias1) AS count(DISTINCT t_prefer_mem.x) +02)--Aggregate: groupBy=[[]], aggr=[[count(alias1)]] +03)----Aggregate: groupBy=[[t_prefer_mem.x AS alias1]], aggr=[[]] +04)------TableScan: t_prefer_mem projection=[x] +physical_plan +01)ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT t_prefer_mem.x)] +02)--AggregateExec: mode=Single, gby=[], aggr=[count(alias1)] +03)----AggregateExec: mode=Single, gby=[x@0 as alias1], aggr=[] +04)------DataSourceExec: partitions=1, partition_sizes=[1] + +# Variable-length string types retain the rewrite +statement ok +CREATE TABLE t_prefer_mem_str AS SELECT * FROM (VALUES (1, '10'), (1, '20'), (2, '10')) AS v(g, s); + +query TT +EXPLAIN SELECT g, count(DISTINCT s) FROM t_prefer_mem_str GROUP BY g; +---- +logical_plan +01)Projection: t_prefer_mem_str.g, count(alias1) AS count(DISTINCT t_prefer_mem_str.s) +02)--Aggregate: groupBy=[[t_prefer_mem_str.g]], aggr=[[count(alias1)]] +03)----Aggregate: groupBy=[[t_prefer_mem_str.g, t_prefer_mem_str.s AS alias1]], aggr=[[]] +04)------TableScan: t_prefer_mem_str projection=[g, s] +physical_plan +01)ProjectionExec: expr=[g@0 as g, count(alias1)@1 as count(DISTINCT t_prefer_mem_str.s)] +02)--AggregateExec: mode=Single, gby=[g@0 as g], aggr=[count(alias1)] +03)----AggregateExec: mode=Single, gby=[g@0 as g, s@1 as alias1], aggr=[] +04)------DataSourceExec: partitions=1, partition_sizes=[1] + +statement ok +DROP TABLE t_prefer_mem_str; + +# Unsupported DISTINCT aggregates (like stddev) retain the rewrite +query TT +EXPLAIN SELECT g, stddev(DISTINCT x) FROM t_prefer_mem GROUP BY g; +---- +logical_plan +01)Projection: t_prefer_mem.g, stddev(alias1) AS stddev(DISTINCT t_prefer_mem.x) +02)--Aggregate: groupBy=[[t_prefer_mem.g]], aggr=[[stddev(alias1)]] +03)----Aggregate: groupBy=[[t_prefer_mem.g, CAST(t_prefer_mem.x AS Float64) AS alias1]], aggr=[[]] +04)------TableScan: t_prefer_mem projection=[g, x] +physical_plan +01)ProjectionExec: expr=[g@0 as g, stddev(alias1)@1 as stddev(DISTINCT t_prefer_mem.x)] +02)--AggregateExec: mode=Single, gby=[g@0 as g], aggr=[stddev(alias1)] +03)----AggregateExec: mode=Single, gby=[g@0 as g, CAST(x@1 AS Float64) as alias1], aggr=[] +04)------DataSourceExec: partitions=1, partition_sizes=[1] + +# Float DISTINCT keeps the rewrite: the native accumulator counts -0.0 and 0.0 separately. +statement ok +CREATE TABLE t_prefer_mem_float AS SELECT * FROM (VALUES (1, CAST('0.0' AS DOUBLE)), (1, CAST('-0.0' AS DOUBLE)), (1, CAST('1.5' AS DOUBLE))) AS v(g, f); + +query TT +EXPLAIN SELECT g, count(DISTINCT f) FROM t_prefer_mem_float GROUP BY g; +---- +logical_plan +01)Projection: t_prefer_mem_float.g, count(alias1) AS count(DISTINCT t_prefer_mem_float.f) +02)--Aggregate: groupBy=[[t_prefer_mem_float.g]], aggr=[[count(alias1)]] +03)----Aggregate: groupBy=[[t_prefer_mem_float.g, t_prefer_mem_float.f AS alias1]], aggr=[[]] +04)------TableScan: t_prefer_mem_float projection=[g, f] +physical_plan +01)ProjectionExec: expr=[g@0 as g, count(alias1)@1 as count(DISTINCT t_prefer_mem_float.f)] +02)--AggregateExec: mode=Single, gby=[g@0 as g], aggr=[count(alias1)] +03)----AggregateExec: mode=Single, gby=[g@0 as g, f@1 as alias1], aggr=[] +04)------DataSourceExec: partitions=1, partition_sizes=[1] + +query IIR +SELECT g, count(DISTINCT f), avg(DISTINCT f) FROM t_prefer_mem_float GROUP BY g; +---- +1 2 0.75 + +statement ok +DROP TABLE t_prefer_mem_float; + +# `avg` has no DISTINCT accumulator for durations, so durations keep the rewrite. +query I? +SELECT g, avg(DISTINCT d) FROM (SELECT g, arrow_cast(x, 'Duration(Second)') AS d FROM t_prefer_mem) GROUP BY g ORDER BY g; +---- +1 0 days 0 hours 0 mins 15 secs +2 0 days 0 hours 0 mins 10 secs + +# Multiple target partitions retain the rewrite even with single_distinct_prefer_native = true +statement ok +set datafusion.execution.target_partitions = 4; + +query TT +EXPLAIN SELECT g, count(DISTINCT x) FROM t_prefer_mem GROUP BY g; +---- +logical_plan +01)Projection: t_prefer_mem.g, count(alias1) AS count(DISTINCT t_prefer_mem.x) +02)--Aggregate: groupBy=[[t_prefer_mem.g]], aggr=[[count(alias1)]] +03)----Aggregate: groupBy=[[t_prefer_mem.g, t_prefer_mem.x AS alias1]], aggr=[[]] +04)------TableScan: t_prefer_mem projection=[g, x] +physical_plan +01)ProjectionExec: expr=[g@0 as g, count(alias1)@1 as count(DISTINCT t_prefer_mem.x)] +02)--AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(alias1)] +03)----RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=4 +04)------AggregateExec: mode=Partial, gby=[g@0 as g], aggr=[count(alias1)] +05)--------AggregateExec: mode=FinalPartitioned, gby=[g@0 as g, alias1@1 as alias1], aggr=[] +06)----------RepartitionExec: partitioning=Hash([g@0, alias1@1], 4), input_partitions=1 +07)------------AggregateExec: mode=Partial, gby=[g@0 as g, x@1 as alias1], aggr=[] +08)--------------DataSourceExec: partitions=1, partition_sizes=[1] + +statement ok +set datafusion.optimizer.single_distinct_prefer_native = false; + +statement ok +DROP TABLE t_prefer_mem; diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 2c3ca887d4a38..c5d35bb3c0342 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -178,7 +178,7 @@ The following configuration settings are available: | datafusion.optimizer.enable_sort_pushdown | true | Enable sort pushdown optimization. When enabled, attempts to push sort requirements down to data sources that can natively handle them (e.g., by reversing file/row group read order). Returns **inexact ordering**: Sort operator is kept for correctness, but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N), providing significant speedup. Memory: No additional overhead (only changes read order). Future: Will add option to detect perfectly sorted data and eliminate Sort completely. Default: true | | datafusion.optimizer.enable_leaf_expression_pushdown | true | When set to true, the optimizer will extract leaf expressions (such as `get_field`) from filter/sort/join nodes into projections closer to the leaf table scans, and push those projections down towards the leaf nodes. | | datafusion.optimizer.enable_unions_to_filter | false | When set to true, the logical optimizer will rewrite `UNION DISTINCT` branches that read from the same source and differ only by filter predicates into a single branch with a combined filter. This optimization is conservative and only applies when the branches share the same source and compatible wrapper nodes such as identical projections or aliases. | -| datafusion.optimizer.single_distinct_prefer_native | false | Prefer native grouped DISTINCT aggregation when target_partitions is 1 and every aggregate explicitly supports a specialized GroupsAccumulator for its DISTINCT argument types. Defaults to false, preserving the two-phase GROUP BY rewrite. The native plan can avoid materializing distinct (group, value) pairs, but can be slower for many groups with duplicate values. It must fit each group's distinct values in memory, so queries that succeed by spilling with the rewrite may fail with this option enabled. One target partition can still have multiple input partitions and merge native states. | +| datafusion.optimizer.single_distinct_prefer_native | false | When `true`, queries with a single DISTINCT aggregate will take less memory but may also run more slowly, by skipping the two-phase `GROUP BY` rewrite and executing with native accumulators. Only applies when `target_partitions` is 1 and all DISTINCT arguments are fixed-width, non-float types. See [`SingleDistinctToGroupBy`](https://docs.rs/datafusion-optimizer/latest/datafusion_optimizer/single_distinct_to_groupby/struct.SingleDistinctToGroupBy.html) for details. | | datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans | | datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans | | datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans |