Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 115 additions & 40 deletions datafusion/catalog-listing/src/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::empty::EmptyExec;
use futures::future::BoxFuture;
use futures::{Stream, StreamExt, TryStreamExt, future, stream};
use object_store::ObjectStore;
use std::collections::{HashMap, HashSet};
Expand Down Expand Up @@ -489,21 +490,94 @@ impl TableProvider for ListingTable {
TableType::Base
}

async fn scan(
&self,
state: &dyn Session,
projection: Option<&Vec<usize>>,
filters: &[Expr],
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
projection: Option<&'life2 Vec<usize>>,
filters: &'life3 [Expr],
limit: Option<usize>,
) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
let options = ScanArgs::default()
.with_projection(projection.map(|p| p.as_slice()))
.with_filters(Some(filters))
.with_limit(limit);
Ok(self.scan_with_args(state, options).await?.into_inner())
) -> BoxFuture<'async_trait, datafusion_common::Result<Arc<dyn ExecutionPlan>>>
where
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait,
Self: 'async_trait,
{
self.scan_boxed(state, projection, filters, limit)
}

fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
args: ScanArgs<'a>,
) -> BoxFuture<'async_trait, datafusion_common::Result<ScanResult>>
where
'a: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Self: 'async_trait,
{
self.scan_with_args_boxed(state, args)
}

fn supports_filters_pushdown(
&self,
filters: &[&Expr],
) -> datafusion_common::Result<Vec<TableProviderFilterPushDown>> {
let partition_column_names = self
.options
.table_partition_cols
.iter()
.map(|col| col.0.as_str())
.collect::<Vec<_>>();
filters
.iter()
.map(|filter| {
if can_be_evaluated_for_partition_pruning(&partition_column_names, filter)
{
// if filter can be handled by partition pruning, it is exact
return Ok(TableProviderFilterPushDown::Exact);
}

Ok(TableProviderFilterPushDown::Inexact)
})
.collect()
}

fn get_table_definition(&self) -> Option<&str> {
self.definition.as_deref()
}

fn insert_into<'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
input: Arc<dyn ExecutionPlan>,
insert_op: InsertOp,
) -> BoxFuture<'async_trait, datafusion_common::Result<Arc<dyn ExecutionPlan>>>
where
'life0: 'async_trait,
'life1: 'async_trait,
Self: 'async_trait,
{
self.insert_into_boxed(state, input, insert_op)
}

fn get_column_default(&self, column: &str) -> Option<&Expr> {
self.column_defaults.get(column)
}
}

impl ListingTable {
fn scan_with_args_boxed<'a>(
&'a self,
state: &'a dyn Session,
args: ScanArgs<'a>,
) -> BoxFuture<'a, datafusion_common::Result<ScanResult>> {
Box::pin(self.scan_with_args_inner(state, args))
}

async fn scan_with_args<'a>(
async fn scan_with_args_inner<'a>(
&self,
state: &dyn Session,
args: ScanArgs<'a>,
Expand Down Expand Up @@ -670,35 +744,40 @@ impl TableProvider for ListingTable {
Ok(ScanResult::new(plan))
}

fn supports_filters_pushdown(
&self,
filters: &[&Expr],
) -> datafusion_common::Result<Vec<TableProviderFilterPushDown>> {
let partition_column_names = self
.options
.table_partition_cols
.iter()
.map(|col| col.0.as_str())
.collect::<Vec<_>>();
filters
.iter()
.map(|filter| {
if can_be_evaluated_for_partition_pruning(&partition_column_names, filter)
{
// if filter can be handled by partition pruning, it is exact
return Ok(TableProviderFilterPushDown::Exact);
}
fn scan_boxed<'a>(
&'a self,
state: &'a dyn Session,
projection: Option<&'a Vec<usize>>,
filters: &'a [Expr],
limit: Option<usize>,
) -> BoxFuture<'a, datafusion_common::Result<Arc<dyn ExecutionPlan>>> {
Box::pin(self.scan_inner(state, projection, filters, limit))
}

Ok(TableProviderFilterPushDown::Inexact)
})
.collect()
async fn scan_inner(
&self,
state: &dyn Session,
projection: Option<&Vec<usize>>,
filters: &[Expr],
limit: Option<usize>,
) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
let options = ScanArgs::default()
.with_projection(projection.map(|p| p.as_slice()))
.with_filters(Some(filters))
.with_limit(limit);
Ok(self.scan_with_args(state, options).await?.into_inner())
}

fn get_table_definition(&self) -> Option<&str> {
self.definition.as_deref()
fn insert_into_boxed<'a>(
&'a self,
state: &'a dyn Session,
input: Arc<dyn ExecutionPlan>,
insert_op: InsertOp,
) -> BoxFuture<'a, datafusion_common::Result<Arc<dyn ExecutionPlan>>> {
Box::pin(self.insert_into_inner(state, input, insert_op))
}

async fn insert_into(
async fn insert_into_inner(
&self,
state: &dyn Session,
input: Arc<dyn ExecutionPlan>,
Expand Down Expand Up @@ -766,10 +845,6 @@ impl TableProvider for ListingTable {
.create_writer_physical_plan(input, state, config, order_requirements)
.await
}

fn get_column_default(&self, column: &str) -> Option<&Expr> {
self.column_defaults.get(column)
}
}

impl ListingTable {
Expand Down
Loading