Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 57 additions & 24 deletions datafusion/physical-plan/src/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1254,20 +1254,20 @@ fn scale_byte_size_at_rows(

/// Returns the NDV for a column constrained to one non-null value (e.g.
/// `column = literal` or a singleton interval), derived from the filtered row
/// estimate: zero rows means zero distinct values, a known positive row count
/// means exactly one, and an unknown row count means an inexact one (the column
/// could still be empty).
/// estimate: zero rows means zero distinct values, an exact positive row count
/// means exactly one, and an estimated or unknown row count means an inexact
/// one (the column could still be empty).
///
/// The caller is responsible for proving the singleton domain.
fn distinct_count_for_singleton_domain(
filtered_num_rows: Precision<usize>,
) -> Precision<usize> {
match filtered_num_rows {
Precision::Exact(0) | Precision::Inexact(0) => filtered_num_rows,
// The row count is unknown, so the column could still be empty (zero
// distinct values); report an inexact one rather than overstating it.
Precision::Absent => Precision::Inexact(1),
_ => Precision::Exact(1),
Precision::Exact(_) => Precision::Exact(1),
// The row count is not known exactly, so the column could still be
// empty (zero distinct values); report an inexact one.
Precision::Inexact(_) | Precision::Absent => Precision::Inexact(1),
}
}

Expand Down Expand Up @@ -2990,7 +2990,7 @@ mod tests {
Operator::Eq,
Arc::new(Literal::new(ScalarValue::Utf8(Some("hello".to_string())))),
)),
vec![Precision::Exact(1)],
vec![Precision::Inexact(1)],
),
(
"utf8view equality",
Expand All @@ -3006,7 +3006,7 @@ mod tests {
"hello".to_string(),
)))),
)),
vec![Precision::Exact(1)],
vec![Precision::Inexact(1)],
),
(
"largeutf8 equality",
Expand All @@ -3022,7 +3022,7 @@ mod tests {
"hello".to_string(),
)))),
)),
vec![Precision::Exact(1)],
vec![Precision::Inexact(1)],
),
(
"utf8 reversed (literal = column)",
Expand All @@ -3036,7 +3036,7 @@ mod tests {
Operator::Eq,
Arc::new(Column::new("name", 0)),
)),
vec![Precision::Exact(1)],
vec![Precision::Inexact(1)],
),
(
"OR is not collapsed to NDV=1, but NDV is capped at filtered rows",
Expand Down Expand Up @@ -3093,7 +3093,7 @@ mod tests {
Arc::new(Literal::new(ScalarValue::Int32(Some(42)))),
)),
)),
vec![Precision::Exact(1), Precision::Exact(1)],
vec![Precision::Inexact(1), Precision::Inexact(1)],
),
(
"numeric equality with min/max bounds (interval analysis path)",
Expand All @@ -3109,7 +3109,7 @@ mod tests {
Operator::Eq,
Arc::new(Literal::new(ScalarValue::Int32(Some(42)))),
)),
vec![Precision::Exact(1)],
vec![Precision::Inexact(1)],
),
(
"timestamp equality",
Expand All @@ -3130,7 +3130,7 @@ mod tests {
None,
))),
)),
vec![Precision::Exact(1)],
vec![Precision::Inexact(1)],
),
(
"contradictory numeric equality (infeasible)",
Expand Down Expand Up @@ -3166,7 +3166,7 @@ mod tests {
Operator::Eq,
Arc::new(Literal::new(ScalarValue::Utf8(Some("hello".to_string())))),
)),
vec![Precision::Exact(1)],
vec![Precision::Inexact(1)],
),
(
"contradictory utf8 equality (infeasible)",
Expand Down Expand Up @@ -3231,7 +3231,7 @@ mod tests {
Arc::new(Literal::new(ScalarValue::Int32(Some(2)))),
)),
)),
vec![Precision::Exact(1), Precision::Exact(1)],
vec![Precision::Inexact(1), Precision::Inexact(1)],
),
];

Expand Down Expand Up @@ -3529,7 +3529,7 @@ mod tests {
// Equality predicates collapse NDV and reject nulls for their columns.
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
Precision::Inexact(1)
);
assert_eq!(
statistics.column_statistics[0].null_count,
Expand All @@ -3544,7 +3544,7 @@ mod tests {
);
assert_eq!(
statistics.column_statistics[2].distinct_count,
Precision::Exact(1)
Precision::Inexact(1)
);
assert_eq!(
statistics.column_statistics[2].null_count,
Expand Down Expand Up @@ -3582,7 +3582,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
Precision::Inexact(1)
);
Ok(())
}
Expand Down Expand Up @@ -3616,7 +3616,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
Precision::Inexact(1)
);
Ok(())
}
Expand Down Expand Up @@ -3650,7 +3650,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
Precision::Inexact(1)
);
Ok(())
}
Expand Down Expand Up @@ -3684,7 +3684,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
Precision::Inexact(1)
);
Ok(())
}
Expand Down Expand Up @@ -3719,7 +3719,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
Precision::Inexact(1)
);
Ok(())
}
Expand Down Expand Up @@ -3766,7 +3766,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
Precision::Inexact(1)
);
Ok(())
}
Expand Down Expand Up @@ -4174,6 +4174,39 @@ mod tests {
Ok(())
}

#[tokio::test]
async fn test_filter_statistics_singleton_precision() -> Result<()> {
use Precision::{Exact, Inexact};

let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
// The bounds include 5, but no row contains it.
let values = Int32Array::from_iter_values(
(0..1000).map(|i| if i % 2 == 0 { 1 } else { 100 }),
);
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(values)])?;
let input = test::TestMemoryExec::try_new_exec(
&[vec![batch]],
Arc::clone(&schema),
None,
)?;
let mut input_stats = StatisticsContext::new()
.compute(input.as_ref(), &StatisticsArgs::new())?
.as_ref()
.clone();
// Supply the bounds a file scan could report for these rows.
input_stats.column_statistics[0].min_value = Exact(ScalarValue::Int32(Some(1)));
input_stats.column_statistics[0].max_value = Exact(ScalarValue::Int32(Some(100)));
let predicate = binary(col("a", &schema)?, Operator::Eq, lit(5i32), &schema)?;
let filter = FilterExecBuilder::new(predicate, input).build()?;
let stats = filter
.statistics_from_inputs(&[Arc::new(input_stats)], &StatisticsArgs::new())?;
let batches =
collect(filter.execute(0, Arc::new(TaskContext::default()))?).await?;
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 0);
assert_eq!(stats.column_statistics[0].distinct_count, Inexact(1));
Ok(())
}

#[tokio::test]
async fn test_filter_statistics_is_not_null_rejects_nulls() -> Result<()> {
let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]);
Expand Down
12 changes: 6 additions & 6 deletions datafusion/sqllogictest/test_files/parquet_statistics.slt
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ query TT
EXPLAIN SELECT * FROM test_table WHERE column1 = 1;
----
physical_plan
01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]]
01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]

Expand All @@ -84,7 +84,7 @@ query TT
EXPLAIN SELECT * FROM test_table WHERE column1 = 1;
----
physical_plan
01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]]
01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]

Expand Down Expand Up @@ -152,7 +152,7 @@ query TT
EXPLAIN SELECT i8 FROM typed_table WHERE i8 = 2;
----
physical_plan
01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]
01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i8], file_type=parquet, predicate=i8@0 = 2, pruning_predicate=i8_null_count@2 != row_count@3 AND i8_min@0 <= 2 AND 2 <= i8_max@1, required_guarantees=[i8 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]]

Expand All @@ -161,7 +161,7 @@ query TT
EXPLAIN SELECT i64 FROM typed_table WHERE i64 = 2;
----
physical_plan
01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(8))]]
01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(8))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i64], file_type=parquet, predicate=i64@1 = 2, pruning_predicate=i64_null_count@2 != row_count@3 AND i64_min@0 <= 2 AND 2 <= i64_max@1, required_guarantees=[i64 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]]

Expand All @@ -170,7 +170,7 @@ query TT
EXPLAIN SELECT f32 FROM typed_table WHERE f32 = 2.5;
----
physical_plan
01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]
01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f32], file_type=parquet, predicate=CAST(f32@2 AS Float64) = 2.5, pruning_predicate=f32_null_count@2 != row_count@3 AND CAST(f32_min@0 AS Float64) <= 2.5 AND 2.5 <= CAST(f32_max@1 AS Float64), required_guarantees=[], statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]]

Expand All @@ -179,7 +179,7 @@ query TT
EXPLAIN SELECT f64 FROM typed_table WHERE 2.5 = f64;
----
physical_plan
01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]
01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f64], file_type=parquet, predicate=f64@3 = 2.5, pruning_predicate=f64_null_count@2 != row_count@3 AND f64_min@0 <= 2.5 AND 2.5 <= f64_max@1, required_guarantees=[f64 in (2.5)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]]

Expand Down
Loading