Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 92 additions & 8 deletions datafusion/catalog-listing/src/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,19 @@ use datafusion_execution::cache::cache_manager::{
use datafusion_expr::dml::InsertOp;
use datafusion_expr::execution_props::ExecutionProps;
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
use datafusion_expr::utils::conjunction;
use datafusion_expr::{
Expr, Partitioning as LogicalPartitioning, TableProviderFilterPushDown, TableType,
};
use datafusion_physical_expr::{create_lex_ordering, create_physical_partitioning};
use datafusion_physical_expr::{
Partitioning, create_lex_ordering, create_physical_expr, create_physical_partitioning,
};
use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::empty::EmptyExec;
use datafusion_physical_plan::limit::GlobalLimitExec;
use datafusion_pruning::PruningPredicateBuilder;
use datafusion_pruning::{PruningPredicateBuilder, RangePruningStatistics};
use futures::future::BoxFuture;
use futures::{Stream, StreamExt, TryStreamExt, future, stream};
use object_store::ObjectStore;
Expand Down Expand Up @@ -915,14 +918,25 @@ impl ListingTable {
/// be distributed to different threads / executors.
///
/// If [`ListingOptions::output_partitioning`] is set, returns one file
/// group per declared partition, including empty trailing groups.
/// group per declared partition, including empty trailing groups. Filters
/// known not to match declared ranges leave empty groups at those indices.
pub async fn list_files_for_scan<'a>(
&'a self,
ctx: &'a dyn Session,
filters: &'a [Expr],
limit: Option<usize>,
) -> datafusion_common::Result<ListFilesResult> {
self.list_files_for_scan_inner(ctx, filters, &[], limit)
let partition_column_names = self
.options
.table_partition_cols
.iter()
.map(|(name, _)| name.as_str())
.collect::<Vec<_>>();
let (partition_filters, data_filters): (Vec<_>, Vec<_>) =
filters.iter().cloned().partition(|filter| {
can_be_evaluated_for_partition_pruning(&partition_column_names, filter)
});
self.list_files_for_scan_inner(ctx, &partition_filters, &data_filters, limit)
.await
}

Expand All @@ -939,11 +953,18 @@ impl ListingTable {
ctx,
output_partitioning,
partition_filters,
data_filters,
)
.await
} else {
self.list_files_for_regular_scan(ctx, partition_filters, data_filters, limit)
.await
let file_limit = data_filters.is_empty().then_some(limit).flatten();
self.list_files_for_regular_scan(
ctx,
partition_filters,
data_filters,
file_limit,
)
.await
}
}

Expand Down Expand Up @@ -1121,6 +1142,7 @@ impl ListingTable {
ctx: &'a dyn Session,
output_partitioning: &LogicalPartitioning,
filters: &'a [Expr],
data_filters: &[Expr],
) -> datafusion_common::Result<ListFilesResult> {
let Some(file_group_count) = output_partitioning.partition_count() else {
return datafusion_common::not_impl_err!(
Expand Down Expand Up @@ -1149,12 +1171,74 @@ impl ListingTable {
if !file_groups.is_empty() {
file_groups.resize_with(file_group_count, || FileGroup::new(vec![]));
}
let file_groups =
let mut file_groups =
self.filter_declared_file_groups_by_partition_filters(file_groups, filters)?;

if matches!(output_partitioning, LogicalPartitioning::Range(_)) {
let range_filters = filters
.iter()
.chain(data_filters)
.cloned()
.collect::<Vec<_>>();
self.prune_range_file_groups(
ctx,
output_partitioning,
&mut file_groups,
&range_filters,
)?;
}

self.list_files_result_from_groups(ctx, file_groups, inexact_stats, false, vec![])
}

fn prune_range_file_groups(
&self,
ctx: &dyn Session,
output_partitioning: &LogicalPartitioning,
file_groups: &mut [FileGroup],
filters: &[Expr],
) -> datafusion_common::Result<()> {
// Do not invoke volatile expressions while planning. Unsupported
// expressions and pruning evaluation errors must keep the partitions.
let Some(filter) =
conjunction(filters.iter().filter(|f| !f.is_volatile()).cloned())
else {
return Ok(());
};
let schema = DFSchema::try_from(Arc::clone(&self.table_schema))?;
let planning_ctx = PhysicalPlanningContext::default();
let Partitioning::Range(range) = create_physical_partitioning(
output_partitioning,
&schema,
ctx.execution_props(),
&planning_ctx,
)?
else {
return Ok(());
};
let retained =
create_physical_expr(&filter, &schema, ctx.execution_props(), &planning_ctx)
.and_then(|expr| {
PruningPredicateBuilder::new()
.with_file_schema(Arc::clone(&self.table_schema))
.try_build(expr)
})
.and_then(|predicate| {
predicate.prune(&RangePruningStatistics::new(&range))
});
match retained {
Ok(retained) => {
for (group, keep) in file_groups.iter_mut().zip(retained) {
if !keep {
*group = FileGroup::new(vec![]);
}
}
}
Err(error) => log::debug!("Skipping range partition pruning: {error}"),
}
Ok(())
}

fn filter_declared_file_groups_by_partition_filters(
&self,
file_groups: Vec<FileGroup>,
Expand Down Expand Up @@ -1533,7 +1617,7 @@ mod tests {
fn prune_test_files(expr: Expr, statistics: Vec<Option<Statistics>>) -> Vec<String> {
use arrow::datatypes::DataType;
let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Int64, true)]));
let physical = datafusion_physical_expr::create_physical_expr(
let physical = create_physical_expr(
&expr,
&DFSchema::try_from(schema.as_ref().clone()).unwrap(),
&ExecutionProps::new(),
Expand Down
145 changes: 145 additions & 0 deletions datafusion/core/src/datasource/listing/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1418,6 +1418,151 @@ mod tests {
Ok(())
}

#[cfg(feature = "parquet")]
#[tokio::test]
async fn test_list_files_data_filter_does_not_limit_before_pruning() -> Result<()> {
let mut config = SessionConfig::new()
.with_collect_statistics(true)
.with_target_partitions(1);
config.options_mut().execution.meta_fetch_concurrency =
datafusion_common::config::ConfigNonZeroUsize::try_new(1)?;
let ctx = SessionContext::new_with_config(config);
let directory = TempDir::new()?;
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
for (name, values) in [("0.parquet", vec![0, 0]), ("1.parquet", vec![20])] {
let batch = RecordBatch::try_new(
Arc::clone(&schema),
vec![Arc::new(arrow::array::Int32Array::from(values))],
)?;
let file = std::fs::File::create(directory.path().join(name))?;
let mut writer =
parquet::arrow::ArrowWriter::try_new(file, Arc::clone(&schema), None)?;
writer.write(&batch)?;
writer.close()?;
}
let options = ListingOptions::new(Arc::new(ParquetFormat::default()))
.with_file_extension(".parquet");
let table = ListingTable::try_new(
ListingTableConfig::new_with_multi_paths(
["0.parquet", "1.parquet"]
.into_iter()
.map(|name| {
ListingTableUrl::parse(
directory.path().join(name).to_str().unwrap(),
)
})
.collect::<Result<Vec<_>>>()?,
)
.with_listing_options(options)
.with_schema(schema),
)?;

// Establish that the unfiltered soft limit would stop at the first file.
let limited = table
.list_files_for_scan(&ctx.state(), &[], Some(1))
.await?;
assert_eq!(
limited
.file_groups
.iter()
.map(|group| group.len())
.sum::<usize>(),
1
);
assert_eq!(
limited.file_groups[0]
.iter()
.next()
.unwrap()
.object_meta
.location
.filename(),
Some("0.parquet")
);

let files = table
.list_files_for_scan(&ctx.state(), &[col("a").gt_eq(lit(20_i32))], Some(1))
.await?;
let paths = files
.file_groups
.iter()
.flat_map(|group| group.iter())
.map(|file| file.object_meta.location.filename().unwrap())
.collect::<Vec<_>>();
assert_eq!(paths, ["1.parquet"]);
assert_eq!(files.statistics.num_rows, Precision::Exact(1));
Ok(())
}

#[tokio::test]
async fn test_list_files_prunes_normalized_range_boundaries() -> Result<()> {
let files = [
"bucket/key-prefix/file0",
"bucket/key-prefix/file1",
"bucket/key-prefix/file2",
"bucket/key-prefix/file3",
];
let ctx = SessionContext::new();
// Logical split points need not have the physical range column's type.
let opt = ListingOptions::new(Arc::new(JsonFormat::default()))
.with_file_extension_opt(Some(""))
.with_output_partitioning(Some(LogicalPartitioning::Range(
LogicalRangePartitioning::try_new(
vec![col("a").sort(true, true)],
[10_i64, 20, 30]
.into_iter()
.map(|value| SplitPoint::new(vec![ScalarValue::from(value)]))
.collect(),
)?,
)));
let table = listing_table_with_files(
&ctx,
&files,
"test:///bucket/key-prefix/",
opt,
Schema::new(vec![Field::new("a", DataType::Int32, true)]),
)?;

for (filter, expected) in [
(col("a").lt(lit(10_i32)), vec![1, 0, 0, 0]),
(col("a").eq(lit(10_i32)), vec![0, 1, 0, 0]),
(col("a").gt_eq(lit(30_i32)), vec![0, 0, 0, 1]),
(col("a").is_null(), vec![1, 0, 0, 0]),
] {
let listed = table
.list_files_for_scan(&ctx.state(), std::slice::from_ref(&filter), Some(1))
.await?;
assert!(!listed.grouped_by_partition);
assert!(listed.partition_split_points.is_empty());
assert_eq!(
listed
.file_groups
.iter()
.map(|g| g.len())
.collect::<Vec<_>>(),
expected,
"{filter}"
);
let scan = table.scan(&ctx.state(), None, &[filter], None).await?;
assert_eq!(scan.output_partitioning().partition_count(), 4);
}
// Invalid pruning expressions are conservative; the normal scan/filter
// planning remains responsible for reporting any expression error.
let invalid_filter = col("missing").eq(lit(10_i32));
let listed = table
.list_files_for_scan(&ctx.state(), &[invalid_filter], None)
.await?;
assert_eq!(
listed
.file_groups
.iter()
.map(|g| g.len())
.collect::<Vec<_>>(),
vec![1, 1, 1, 1]
);
Ok(())
}

#[tokio::test]
async fn test_range_output_partitioning_normalizes_split_point_types() -> Result<()> {
let files = ["bucket/key-prefix/file0"];
Expand Down
18 changes: 16 additions & 2 deletions datafusion/core/tests/catalog_listing/file_statistics_pruning.rs
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,7 @@ async fn volatile_predicate_is_not_evaluated_while_listing() -> Result<()> {
}

#[tokio::test]
async fn declared_range_partitioning_preserves_groups() -> Result<()> {
async fn declared_range_pruning_preserves_partition_indexes() -> Result<()> {
use datafusion_common::SplitPoint;
use datafusion_expr::RangePartitioning;
let dir = fixture(vec![vec![Some(1)], vec![Some(25)]], false, true);
Expand All @@ -423,7 +423,21 @@ async fn declared_range_partitioning_preserves_groups() -> Result<()> {
let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;
assert_eq!(
names(scan_config(&plan)),
vec![vec!["f0.parquet"], vec!["f1.parquet"], vec![], vec![]]
vec![vec![], vec!["f1.parquet"], vec![], vec![]]
);
// Empty declared groups retain conservative summary statistics.
assert_eq!(scan_config(&plan).statistics().num_rows, Precision::Absent);
assert_eq!(
scan_config(&plan).file_groups[1]
.file_statistics(None)
.unwrap()
.num_rows,
Precision::Exact(1)
);
ctx.register_table("t", Arc::new(table))?;
assert_eq!(
rows(&ctx, "select * from t where d >= 20 and d < 40").await?,
1
);
Ok(())
}
2 changes: 2 additions & 0 deletions datafusion/pruning/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@ mod in_list;
mod key_range_bitmap_expr;
mod primitive_in_list;
mod pruning_predicate;
mod range;
mod string_in_list;

pub use file_pruner::FilePruner;
pub use pruning_predicate::{
MAX_IN_LIST_SIZE, PredicateRewriter, PruningPredicate, PruningPredicateBuilder,
PruningStatistics, RequiredColumns, UnhandledPredicateHook, build_pruning_predicate,
};
pub use range::RangePruningStatistics;
Loading
Loading