From 503dc53b880cabf534abe451269f32535a509cc0 Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Tue, 29 Sep 2026 11:06:28 +0530 Subject: [PATCH] fix: keep singleton distinct counts inexact for estimated rows --- datafusion/physical-plan/src/filter.rs | 81 +++++++++++++------ .../test_files/parquet_statistics.slt | 12 +-- 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index f71d67d04e39c..dcaae294b67d3 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -1254,9 +1254,9 @@ fn scale_byte_size_at_rows( /// Returns the NDV for a column constrained to one non-null value (e.g. /// `column = literal` or a singleton interval), derived from the filtered row -/// estimate: zero rows means zero distinct values, a known positive row count -/// means exactly one, and an unknown row count means an inexact one (the column -/// could still be empty). +/// estimate: zero rows means zero distinct values, an exact positive row count +/// means exactly one, and an estimated or unknown row count means an inexact +/// one (the column could still be empty). /// /// The caller is responsible for proving the singleton domain. fn distinct_count_for_singleton_domain( @@ -1264,10 +1264,10 @@ fn distinct_count_for_singleton_domain( ) -> Precision { match filtered_num_rows { Precision::Exact(0) | Precision::Inexact(0) => filtered_num_rows, - // The row count is unknown, so the column could still be empty (zero - // distinct values); report an inexact one rather than overstating it. - Precision::Absent => Precision::Inexact(1), - _ => Precision::Exact(1), + Precision::Exact(_) => Precision::Exact(1), + // The row count is not known exactly, so the column could still be + // empty (zero distinct values); report an inexact one. + Precision::Inexact(_) | Precision::Absent => Precision::Inexact(1), } } @@ -2990,7 +2990,7 @@ mod tests { Operator::Eq, Arc::new(Literal::new(ScalarValue::Utf8(Some("hello".to_string())))), )), - vec![Precision::Exact(1)], + vec![Precision::Inexact(1)], ), ( "utf8view equality", @@ -3006,7 +3006,7 @@ mod tests { "hello".to_string(), )))), )), - vec![Precision::Exact(1)], + vec![Precision::Inexact(1)], ), ( "largeutf8 equality", @@ -3022,7 +3022,7 @@ mod tests { "hello".to_string(), )))), )), - vec![Precision::Exact(1)], + vec![Precision::Inexact(1)], ), ( "utf8 reversed (literal = column)", @@ -3036,7 +3036,7 @@ mod tests { Operator::Eq, Arc::new(Column::new("name", 0)), )), - vec![Precision::Exact(1)], + vec![Precision::Inexact(1)], ), ( "OR is not collapsed to NDV=1, but NDV is capped at filtered rows", @@ -3093,7 +3093,7 @@ mod tests { Arc::new(Literal::new(ScalarValue::Int32(Some(42)))), )), )), - vec![Precision::Exact(1), Precision::Exact(1)], + vec![Precision::Inexact(1), Precision::Inexact(1)], ), ( "numeric equality with min/max bounds (interval analysis path)", @@ -3109,7 +3109,7 @@ mod tests { Operator::Eq, Arc::new(Literal::new(ScalarValue::Int32(Some(42)))), )), - vec![Precision::Exact(1)], + vec![Precision::Inexact(1)], ), ( "timestamp equality", @@ -3130,7 +3130,7 @@ mod tests { None, ))), )), - vec![Precision::Exact(1)], + vec![Precision::Inexact(1)], ), ( "contradictory numeric equality (infeasible)", @@ -3166,7 +3166,7 @@ mod tests { Operator::Eq, Arc::new(Literal::new(ScalarValue::Utf8(Some("hello".to_string())))), )), - vec![Precision::Exact(1)], + vec![Precision::Inexact(1)], ), ( "contradictory utf8 equality (infeasible)", @@ -3231,7 +3231,7 @@ mod tests { Arc::new(Literal::new(ScalarValue::Int32(Some(2)))), )), )), - vec![Precision::Exact(1), Precision::Exact(1)], + vec![Precision::Inexact(1), Precision::Inexact(1)], ), ]; @@ -3529,7 +3529,7 @@ mod tests { // Equality predicates collapse NDV and reject nulls for their columns. assert_eq!( statistics.column_statistics[0].distinct_count, - Precision::Exact(1) + Precision::Inexact(1) ); assert_eq!( statistics.column_statistics[0].null_count, @@ -3544,7 +3544,7 @@ mod tests { ); assert_eq!( statistics.column_statistics[2].distinct_count, - Precision::Exact(1) + Precision::Inexact(1) ); assert_eq!( statistics.column_statistics[2].null_count, @@ -3582,7 +3582,7 @@ mod tests { StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?; assert_eq!( statistics.column_statistics[0].distinct_count, - Precision::Exact(1) + Precision::Inexact(1) ); Ok(()) } @@ -3616,7 +3616,7 @@ mod tests { StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?; assert_eq!( statistics.column_statistics[0].distinct_count, - Precision::Exact(1) + Precision::Inexact(1) ); Ok(()) } @@ -3650,7 +3650,7 @@ mod tests { StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?; assert_eq!( statistics.column_statistics[0].distinct_count, - Precision::Exact(1) + Precision::Inexact(1) ); Ok(()) } @@ -3684,7 +3684,7 @@ mod tests { StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?; assert_eq!( statistics.column_statistics[0].distinct_count, - Precision::Exact(1) + Precision::Inexact(1) ); Ok(()) } @@ -3719,7 +3719,7 @@ mod tests { StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?; assert_eq!( statistics.column_statistics[0].distinct_count, - Precision::Exact(1) + Precision::Inexact(1) ); Ok(()) } @@ -3766,7 +3766,7 @@ mod tests { StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?; assert_eq!( statistics.column_statistics[0].distinct_count, - Precision::Exact(1) + Precision::Inexact(1) ); Ok(()) } @@ -4174,6 +4174,39 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_filter_statistics_singleton_precision() -> Result<()> { + use Precision::{Exact, Inexact}; + + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); + // The bounds include 5, but no row contains it. + let values = Int32Array::from_iter_values( + (0..1000).map(|i| if i % 2 == 0 { 1 } else { 100 }), + ); + let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(values)])?; + let input = test::TestMemoryExec::try_new_exec( + &[vec![batch]], + Arc::clone(&schema), + None, + )?; + let mut input_stats = StatisticsContext::new() + .compute(input.as_ref(), &StatisticsArgs::new())? + .as_ref() + .clone(); + // Supply the bounds a file scan could report for these rows. + input_stats.column_statistics[0].min_value = Exact(ScalarValue::Int32(Some(1))); + input_stats.column_statistics[0].max_value = Exact(ScalarValue::Int32(Some(100))); + let predicate = binary(col("a", &schema)?, Operator::Eq, lit(5i32), &schema)?; + let filter = FilterExecBuilder::new(predicate, input).build()?; + let stats = filter + .statistics_from_inputs(&[Arc::new(input_stats)], &StatisticsArgs::new())?; + let batches = + collect(filter.execute(0, Arc::new(TaskContext::default()))?).await?; + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 0); + assert_eq!(stats.column_statistics[0].distinct_count, Inexact(1)); + Ok(()) + } + #[tokio::test] async fn test_filter_statistics_is_not_null_rejects_nulls() -> Result<()> { let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]); diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.slt index 71de3833704d2..2a8d1747518d2 100644 --- a/datafusion/sqllogictest/test_files/parquet_statistics.slt +++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt @@ -59,7 +59,7 @@ query TT EXPLAIN SELECT * FROM test_table WHERE column1 = 1; ---- physical_plan -01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]] +01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]] 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]] 03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]] @@ -84,7 +84,7 @@ query TT EXPLAIN SELECT * FROM test_table WHERE column1 = 1; ---- physical_plan -01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]] +01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]] 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]] 03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]] @@ -152,7 +152,7 @@ query TT EXPLAIN SELECT i8 FROM typed_table WHERE i8 = 2; ---- physical_plan -01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]] +01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]] 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]] 03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i8], file_type=parquet, predicate=i8@0 = 2, pruning_predicate=i8_null_count@2 != row_count@3 AND i8_min@0 <= 2 AND 2 <= i8_max@1, required_guarantees=[i8 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]] @@ -161,7 +161,7 @@ query TT EXPLAIN SELECT i64 FROM typed_table WHERE i64 = 2; ---- physical_plan -01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(8))]] +01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(8))]] 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]] 03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i64], file_type=parquet, predicate=i64@1 = 2, pruning_predicate=i64_null_count@2 != row_count@3 AND i64_min@0 <= 2 AND 2 <= i64_max@1, required_guarantees=[i64 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]] @@ -170,7 +170,7 @@ query TT EXPLAIN SELECT f32 FROM typed_table WHERE f32 = 2.5; ---- physical_plan -01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]] +01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]] 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]] 03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f32], file_type=parquet, predicate=CAST(f32@2 AS Float64) = 2.5, pruning_predicate=f32_null_count@2 != row_count@3 AND CAST(f32_min@0 AS Float64) <= 2.5 AND 2.5 <= CAST(f32_max@1 AS Float64), required_guarantees=[], statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]] @@ -179,7 +179,7 @@ query TT EXPLAIN SELECT f64 FROM typed_table WHERE 2.5 = f64; ---- physical_plan -01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]] +01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]] 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]] 03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f64], file_type=parquet, predicate=f64@3 = 2.5, pruning_predicate=f64_null_count@2 != row_count@3 AND f64_min@0 <= 2.5 AND 2.5 <= f64_max@1, required_guarantees=[f64 in (2.5)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]]