diff --git a/crates/iceberg/src/scan/context.rs b/crates/iceberg/src/scan/context.rs index 211ebf8f0c..f82f75c276 100644 --- a/crates/iceberg/src/scan/context.rs +++ b/crates/iceberg/src/scan/context.rs @@ -25,15 +25,21 @@ use crate::delete_file_index::DeleteFileIndex; use crate::expr::{Bind, BoundPredicate, Predicate}; use crate::io::object_cache::ObjectCache; use crate::scan::{ - BoundPredicates, ExpressionEvaluatorCache, FileScanTask, ManifestEvaluatorCache, - PartitionFilterCache, + ExpressionEvaluatorCache, FileScanTask, ManifestEvaluatorCache, PartitionFilterCache, }; use crate::spec::{ - ManifestContentType, ManifestEntryRef, ManifestFile, ManifestList, NameMapping, - PartitionSpecRef, SchemaRef, SnapshotRef, SortOrderRef, StructType, TableMetadataRef, + ManifestContentType, ManifestEntryRef, ManifestFile, NameMapping, PartitionSpecRef, SchemaRef, + SnapshotRef, SortOrderRef, StructType, TableMetadataRef, }; use crate::{Error, ErrorKind, Result}; +/// The predicates a manifest entry is evaluated against: one bound to the +/// manifest's partition spec, one bound to the scan's schema. +pub(crate) struct BoundPredicates { + pub(crate) partition_bound_predicate: BoundPredicate, + pub(crate) scan_bound_predicate: BoundPredicate, +} + /// Wraps a [`ManifestFile`] alongside the objects that are needed /// to process it in a thread-safe manner pub(crate) struct ManifestFileContext { @@ -44,7 +50,7 @@ pub(crate) struct ManifestFileContext { field_ids: Arc>, bound_predicates: Option>, object_cache: Arc, - snapshot_schema: SchemaRef, + scan_schema: SchemaRef, expression_evaluator_cache: Arc, delete_file_index: DeleteFileIndex, name_mapping: Option>, @@ -63,7 +69,7 @@ pub(crate) struct ManifestEntryContext { pub field_ids: Arc>, pub bound_predicates: Option>, pub partition_spec_id: i32, - pub snapshot_schema: SchemaRef, + pub scan_schema: SchemaRef, pub delete_file_index: DeleteFileIndex, pub name_mapping: Option>, pub case_sensitive: bool, @@ -81,7 +87,7 @@ impl ManifestFileContext { object_cache, manifest_file, bound_predicates, - snapshot_schema, + scan_schema, field_ids, mut sender, expression_evaluator_cache, @@ -114,7 +120,7 @@ impl ManifestFileContext { field_ids: field_ids.clone(), partition_spec_id: manifest_file.partition_spec_id, bound_predicates: bound_predicates.clone(), - snapshot_schema: snapshot_schema.clone(), + scan_schema: scan_schema.clone(), delete_file_index: delete_file_index.clone(), name_mapping: name_mapping.clone(), case_sensitive, @@ -155,11 +161,11 @@ impl ManifestEntryContext { .with_data_sequence_number(self.manifest_entry.sequence_number()) .with_data_file_path(self.manifest_entry.file_path().to_string()) .with_data_file_format(self.manifest_entry.file_format()) - .with_schema(self.snapshot_schema) + .with_schema(self.scan_schema) .with_project_field_ids(self.field_ids.to_vec()) .with_predicate( self.bound_predicates - .map(|x| x.as_ref().snapshot_bound_predicate.clone()), + .map(|x| x.as_ref().scan_bound_predicate.clone()), ) .with_deletes(deletes) .with_partition(Some(self.manifest_entry.data_file.partition.clone())) @@ -191,20 +197,24 @@ impl ManifestEntryContext { } } -/// PlanContext wraps a [`SnapshotRef`] alongside all the other -/// objects that are required to perform a scan file plan. +/// PlanContext holds everything required to perform a scan file plan: the +/// snapshot whose manifests are read, and how to project, filter and evaluate +/// their entries. #[derive(Debug)] pub(crate) struct PlanContext { - pub snapshot: SnapshotRef, + /// The snapshot to plan against, or `None` for a table that has no + /// snapshots yet. Such a table lists no manifests and so plans no files, + /// but is still projected and filtered against its current schema. + pub snapshot: Option, pub table_metadata: TableMetadataRef, /// The schema the scan projects. For a scan of the current state, this is /// the table's current schema, not the snapshot's; for time travel, it is /// the schema the snapshot was written with. - pub snapshot_schema: SchemaRef, + pub scan_schema: SchemaRef, pub case_sensitive: bool, pub predicate: Option>, - pub snapshot_bound_predicate: Option>, + pub scan_bound_predicate: Option>, pub object_cache: Arc, pub field_ids: Arc>, pub name_mapping: Option>, @@ -222,13 +232,6 @@ pub(crate) struct PlanContext { } impl PlanContext { - pub(crate) async fn get_manifest_list(&self) -> Result> { - self.object_cache - .as_ref() - .get_manifest_list(&self.snapshot, &self.table_metadata) - .await - } - /// Returns the partition filter for a manifest. See [`PartitionFilterCache::get`] for the /// always-true fallback when the manifest's spec cannot be resolved against the scan schema. fn get_partition_filter(&self, manifest_file: &ManifestFile) -> Result> { @@ -237,7 +240,7 @@ impl PlanContext { let partition_filter = self.partition_filter_cache.get( partition_spec_id, &self.table_metadata, - &self.snapshot_schema, + &self.scan_schema, self.case_sensitive, self.predicate .as_ref() @@ -246,20 +249,33 @@ impl PlanContext { "Expected a predicate but none present", ))? .as_ref() - .bind(self.snapshot_schema.clone(), self.case_sensitive)?, + .bind(self.scan_schema.clone(), self.case_sensitive)?, )?; Ok(partition_filter) } - pub(crate) fn build_manifest_file_contexts( + pub(crate) async fn build_manifest_file_contexts( &self, - manifest_list: Arc, tx_data: Sender, delete_file_idx: DeleteFileIndex, delete_file_tx: Sender, ) -> Result> + 'static>> { - let mut manifest_files = manifest_list.entries().iter().collect::>(); + // A table with no snapshots lists no manifests, so the plan is empty. + let manifest_list = match self.snapshot.as_ref() { + Some(snapshot) => Some( + self.object_cache + .get_manifest_list(snapshot, &self.table_metadata) + .await?, + ), + None => None, + }; + + let mut manifest_files: Vec<&ManifestFile> = manifest_list + .iter() + .flat_map(|manifest_list| manifest_list.entries()) + .collect(); + // Sort manifest files to process delete manifests first. // This avoids a deadlock where the producer blocks on sending data manifest entries // (because the data channel is full) while the delete manifest consumer is waiting @@ -322,12 +338,12 @@ impl PlanContext { delete_file_index: DeleteFileIndex, ) -> ManifestFileContext { let bound_predicates = - if let (Some(ref partition_bound_predicate), Some(snapshot_bound_predicate)) = - (partition_filter, &self.snapshot_bound_predicate) + if let (Some(ref partition_bound_predicate), Some(scan_bound_predicate)) = + (partition_filter, &self.scan_bound_predicate) { Some(Arc::new(BoundPredicates { partition_bound_predicate: partition_bound_predicate.as_ref().clone(), - snapshot_bound_predicate: snapshot_bound_predicate.as_ref().clone(), + scan_bound_predicate: scan_bound_predicate.as_ref().clone(), })) } else { None @@ -338,7 +354,7 @@ impl PlanContext { bound_predicates, sender, object_cache: self.object_cache.clone(), - snapshot_schema: self.snapshot_schema.clone(), + scan_schema: self.scan_schema.clone(), field_ids: self.field_ids.clone(), expression_evaluator_cache: self.expression_evaluator_cache.clone(), delete_file_index, diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 6d23e94704..a3b28bc1a7 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -21,6 +21,7 @@ mod cache; use cache::*; mod context; use context::*; +mod plan; mod task; use std::collections::HashMap; @@ -28,23 +29,20 @@ use std::future::Future; use std::sync::Arc; use arrow_array::RecordBatch; -use futures::channel::mpsc::{Sender, channel}; use futures::stream::BoxStream; -use futures::{SinkExt, StreamExt, TryStreamExt}; pub use task::*; use crate::arrow::ArrowReaderBuilder; pub use crate::arrow::{ScanMetrics, ScanResult}; -use crate::delete_file_index::DeleteFileIndex; use crate::error::invalid_data; -use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator; use crate::expr::{Bind, BoundPredicate, Predicate}; use crate::io::FileIO; use crate::metadata_columns::{ RESERVED_FIELD_ID_PARTITION, get_metadata_field_id, is_metadata_column_name, }; use crate::runtime::Runtime; -use crate::spec::{DataContentType, Schema, SchemaRef, SnapshotRef, SortOrderRef, StructType}; +use crate::scan::plan::plan_tasks; +use crate::spec::{Schema, SchemaRef, SnapshotRef, SortOrderRef, StructType}; use crate::table::Table; use crate::util::available_parallelism; use crate::{Error, ErrorKind, Result}; @@ -289,41 +287,25 @@ impl<'a> TableScanBuilder<'a> { /// Build the table scan. pub fn build(self) -> Result { let snapshot = match self.snapshot_id { - Some(snapshot_id) => self - .table - .metadata() - .snapshot_by_id(snapshot_id) - .ok_or_else(|| invalid_data!("Snapshot with id {snapshot_id} not found"))? - .clone(), - None => { - let Some(current_snapshot_id) = self.table.metadata().current_snapshot() else { - return Ok(TableScan { - batch_size: self.batch_size, - column_names: self.column_names, - file_io: self.table.file_io().clone(), - plan_context: None, - concurrency_limit_data_files: self.concurrency_limit_data_files, - concurrency_limit_manifest_entries: self.concurrency_limit_manifest_entries, - concurrency_limit_manifest_files: self.concurrency_limit_manifest_files, - row_group_filtering_enabled: self.row_group_filtering_enabled, - row_selection_enabled: self.row_selection_enabled, - bloom_filter_enabled: self.bloom_filter_enabled, - runtime: self.table.runtime().clone(), - }); - }; - current_snapshot_id.clone() - } + Some(snapshot_id) => Some( + self.table + .metadata() + .snapshot_by_id(snapshot_id) + .ok_or_else(|| invalid_data!("Snapshot with id {snapshot_id} not found"))? + .clone(), + ), + None => self.table.metadata().current_snapshot().cloned(), }; - // A scan of the current state uses the table's current schema; time travel uses the - // schema its snapshot was written with. - let schema = match self.snapshot_id { - Some(_) => snapshot.schema(self.table.metadata())?, - None => self.table.metadata().current_schema().clone(), + // A scan of the current state, including of a table with no snapshots, uses the + // table's current schema; time travel uses the schema its snapshot was written with. + let schema = match (self.snapshot_id, snapshot.as_ref()) { + (Some(_), Some(snapshot)) => snapshot.schema(self.table.metadata())?, + _ => self.table.metadata().current_schema().clone(), }; let field_ids = collect_scan_field_ids(&schema, self.column_names.as_deref(), self.case_sensitive)?; - let snapshot_bound_predicate = + let scan_bound_predicate = bind_scan_predicate(&schema, self.filter.as_ref(), self.case_sensitive)?; let name_mapping = self .table @@ -346,10 +328,10 @@ impl<'a> TableScanBuilder<'a> { let plan_context = PlanContext { snapshot, table_metadata: self.table.metadata_ref(), - snapshot_schema: schema, + scan_schema: schema, case_sensitive: self.case_sensitive, predicate: self.filter.map(Arc::new), - snapshot_bound_predicate, + scan_bound_predicate, object_cache: self.table.object_cache(), field_ids: Arc::new(field_ids), name_mapping, @@ -364,7 +346,7 @@ impl<'a> TableScanBuilder<'a> { batch_size: self.batch_size, column_names: self.column_names, file_io: self.table.file_io().clone(), - plan_context: Some(plan_context), + plan_context, concurrency_limit_data_files: self.concurrency_limit_data_files, concurrency_limit_manifest_entries: self.concurrency_limit_manifest_entries, concurrency_limit_manifest_files: self.concurrency_limit_manifest_files, @@ -379,10 +361,7 @@ impl<'a> TableScanBuilder<'a> { /// Table scan. #[derive(Debug)] pub struct TableScan { - /// A [PlanContext], if this table has at least one snapshot, otherwise None. - /// - /// If this is None, then the scan contains no rows. - plan_context: Option, + plan_context: PlanContext, batch_size: Option, file_io: FileIO, column_names: Option>, @@ -427,124 +406,14 @@ impl TableScan { F: Fn(ManifestEntryContext) -> Fut + Copy + Send + Sync + 'static, Fut: Future> + Send + 'static, { - let Some(plan_context) = self.plan_context.as_ref() else { - return Ok(Box::pin(futures::stream::empty())); - }; - - let concurrency_limit_manifest_files = self.concurrency_limit_manifest_files; - let concurrency_limit_manifest_entries = self.concurrency_limit_manifest_entries; - - // used to stream ManifestEntryContexts between stages of the file plan operation - let (manifest_entry_data_ctx_tx, manifest_entry_data_ctx_rx) = - channel(concurrency_limit_manifest_files); - let (manifest_entry_delete_ctx_tx, manifest_entry_delete_ctx_rx) = - channel(concurrency_limit_manifest_files); - - // used to stream the planned data file results back to the caller - let (planned_file_tx, planned_file_rx) = channel(concurrency_limit_manifest_entries); - - let (delete_file_idx, delete_file_tx) = DeleteFileIndex::new(self.runtime.clone()); - - let manifest_list = plan_context.get_manifest_list().await?; - - // get the [`ManifestFile`]s from the [`ManifestList`], filtering out any - // whose partitions cannot match this - // scan's filter - let manifest_file_contexts = plan_context.build_manifest_file_contexts( - manifest_list, - manifest_entry_data_ctx_tx, - delete_file_idx.clone(), - manifest_entry_delete_ctx_tx, - )?; - - let mut channel_for_manifest_error = planned_file_tx.clone(); - let mut channel_for_data_manifest_entry_error = planned_file_tx.clone(); - let mut channel_for_delete_manifest_entry_error = planned_file_tx.clone(); - - let rt = self.runtime.clone(); - - // Concurrently load all [`Manifest`]s and stream their [`ManifestEntry`]s - rt.io().spawn(async move { - let result = futures::stream::iter(manifest_file_contexts) - .try_for_each_concurrent(concurrency_limit_manifest_files, |ctx| async move { - ctx.fetch_manifest_and_stream_manifest_entries().await - }) - .await; - - if let Err(error) = result { - let _ = channel_for_manifest_error.send(Err(error)).await; - } - }); - - // Process the delete file [`ManifestEntry`] stream in parallel - { - let rt = rt.clone(); - let rt_inner = rt.clone(); - rt.cpu().spawn(async move { - let result = manifest_entry_delete_ctx_rx - .map(|me_ctx| Ok((me_ctx, delete_file_tx.clone()))) - .try_for_each_concurrent( - concurrency_limit_manifest_entries, - |(manifest_entry_context, tx)| { - let rt_inner = rt_inner.clone(); - async move { - rt_inner - .cpu() - .spawn(async move { - Self::process_delete_manifest_entry( - manifest_entry_context, - tx, - ) - .await - }) - .await? - } - }, - ) - .await; - - if let Err(error) = result { - let _ = channel_for_delete_manifest_entry_error - .send(Err(error)) - .await; - } - }); - } - - // Process the data file [`ManifestEntry`] stream in parallel - { - let rt_inner = rt.clone(); - rt.cpu().spawn(async move { - let result = manifest_entry_data_ctx_rx - .map(|me_ctx| Ok((me_ctx, planned_file_tx.clone()))) - .try_for_each_concurrent( - concurrency_limit_manifest_entries, - |(manifest_entry_context, tx)| { - let rt_inner = rt_inner.clone(); - async move { - rt_inner - .cpu() - .spawn(async move { - Self::process_data_manifest_entry( - manifest_entry_context, - tx, - build_result, - ) - .await - }) - .await? - } - }, - ) - .await; - - if let Err(error) = result { - let _ = channel_for_data_manifest_entry_error.send(Err(error)).await; - } - }); - } - - Ok(planned_file_rx.boxed()) + plan_tasks( + &self.plan_context, + &self.runtime, + self.concurrency_limit_manifest_files, + self.concurrency_limit_manifest_entries, + build_result, + ) + .await } /// Returns an [`ArrowRecordBatchStream`]. @@ -573,120 +442,10 @@ impl TableScan { /// Returns a reference to the snapshot of the table scan. pub fn snapshot(&self) -> Option<&SnapshotRef> { - self.plan_context.as_ref().map(|x| &x.snapshot) - } - - async fn process_data_manifest_entry( - manifest_entry_context: ManifestEntryContext, - mut planned_file_tx: Sender>, - build_result: F, - ) -> Result<()> - where - F: Fn(ManifestEntryContext) -> Fut + Send + Sync + 'static, - Fut: Future> + Send + 'static, - { - // skip processing this manifest entry if it has been marked as deleted - if !manifest_entry_context.manifest_entry.is_alive() { - return Ok(()); - } - - // abort the plan if we encounter a manifest entry for a delete file - if manifest_entry_context.manifest_entry.content_type() != DataContentType::Data { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "Encountered an entry for a delete file in a data file manifest", - )); - } - - if let Some(ref bound_predicates) = manifest_entry_context.bound_predicates { - let BoundPredicates { - snapshot_bound_predicate, - partition_bound_predicate, - } = bound_predicates.as_ref(); - - let expression_evaluator_cache = - manifest_entry_context.expression_evaluator_cache.as_ref(); - - let expression_evaluator = expression_evaluator_cache.get( - manifest_entry_context.partition_spec_id, - partition_bound_predicate, - )?; - - // skip any data file whose partition data indicates that it can't contain - // any data that matches this scan's filter - if !expression_evaluator.eval(manifest_entry_context.manifest_entry.data_file())? { - return Ok(()); - } - - // skip any data file whose metrics don't match this scan's filter - if !InclusiveMetricsEvaluator::eval( - snapshot_bound_predicate, - manifest_entry_context.manifest_entry.data_file(), - false, - )? { - return Ok(()); - } - } - - // congratulations! the manifest entry has made its way through the - // entire plan without getting filtered out. Build the planned file before - // sending so delete-file lookup preserves the original scan timing. - planned_file_tx - .send(Ok(build_result(manifest_entry_context).await?)) - .await?; - - Ok(()) - } - - async fn process_delete_manifest_entry( - manifest_entry_context: ManifestEntryContext, - mut delete_file_ctx_tx: Sender, - ) -> Result<()> { - // skip processing this manifest entry if it has been marked as deleted - if !manifest_entry_context.manifest_entry.is_alive() { - return Ok(()); - } - - // abort the plan if we encounter a manifest entry that is not for a delete file - if manifest_entry_context.manifest_entry.content_type() == DataContentType::Data { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "Encountered an entry for a data file in a delete manifest", - )); - } - - if let Some(ref bound_predicates) = manifest_entry_context.bound_predicates { - let expression_evaluator_cache = - manifest_entry_context.expression_evaluator_cache.as_ref(); - - let expression_evaluator = expression_evaluator_cache.get( - manifest_entry_context.partition_spec_id, - &bound_predicates.partition_bound_predicate, - )?; - - // skip any data file whose partition data indicates that it can't contain - // any data that matches this scan's filter - if !expression_evaluator.eval(manifest_entry_context.manifest_entry.data_file())? { - return Ok(()); - } - } - - delete_file_ctx_tx - .send(DeleteFileContext { - manifest_entry: manifest_entry_context.manifest_entry.clone(), - partition_spec_id: manifest_entry_context.partition_spec_id, - }) - .await?; - - Ok(()) + self.plan_context.snapshot.as_ref() } } -pub(crate) struct BoundPredicates { - partition_bound_predicate: BoundPredicate, - snapshot_bound_predicate: BoundPredicate, -} - #[cfg(test)] mod tests { //! shared tests for the table scan API @@ -915,7 +674,7 @@ mod tests { let table_scan = table.scan().build().unwrap(); assert_eq!(table_scan.snapshot(), table.metadata().current_snapshot()); assert_eq!( - table_scan.plan_context.as_ref().unwrap().snapshot_schema, + table_scan.plan_context.scan_schema, *table.metadata().current_schema() ); } @@ -934,7 +693,7 @@ mod tests { .unwrap(); assert_eq!(table_scan.snapshot(), Some(snapshot)); assert_eq!( - table_scan.plan_context.as_ref().unwrap().snapshot_schema, + table_scan.plan_context.scan_schema, snapshot.schema(table.metadata()).unwrap() ); } @@ -960,14 +719,7 @@ mod tests { let table = TableTestFixture::new().table; let table_scan = table.scan().build().unwrap(); - assert!( - table_scan - .plan_context - .as_ref() - .unwrap() - .name_mapping - .is_none() - ); + assert!(table_scan.plan_context.name_mapping.is_none()); } #[test] @@ -979,8 +731,6 @@ mod tests { let table_scan = table.scan().build().unwrap(); let mapping = table_scan .plan_context - .as_ref() - .unwrap() .name_mapping .as_ref() .expect("name_mapping should be parsed from the table property"); @@ -1154,6 +904,24 @@ mod tests { assert!(batches.is_empty()); } + #[test] + fn test_scan_without_any_snapshots_still_validates_projection() { + let table = TableTestFixture::new_empty().table; + + table + .scan() + .select(["x"]) + .build() + .expect("a column of the current schema should be projectable"); + + let error = table + .scan() + .select(["nonexistent"]) + .build() + .expect_err("an absent column should be rejected even with no snapshots"); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + #[tokio::test] async fn test_plan_files_no_deletions() { let mut fixture = TableTestFixture::new(); @@ -2462,23 +2230,12 @@ mod tests { .build() .unwrap_or_else(|e| panic!("scan of data column `{column_name}` failed: {e}")); - assert_eq!( - table_scan.plan_context.as_ref().unwrap().field_ids.as_ref(), - &[2] - ); + assert_eq!(table_scan.plan_context.field_ids.as_ref(), &[2]); // The default projection (all columns) must resolve to the real field ids // too, not shadow the data column with a reserved delete-file id. let default_scan = table.scan().build().unwrap(); - assert_eq!( - default_scan - .plan_context - .as_ref() - .unwrap() - .field_ids - .as_ref(), - &[1, 2] - ); + assert_eq!(default_scan.plan_context.field_ids.as_ref(), &[1, 2]); } } diff --git a/crates/iceberg/src/scan/plan.rs b/crates/iceberg/src/scan/plan.rs new file mode 100644 index 0000000000..d697094897 --- /dev/null +++ b/crates/iceberg/src/scan/plan.rs @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Drives a scan plan: fans manifests out to their entries and turns the +//! surviving entries into planned files. + +use std::future::Future; + +use futures::channel::mpsc::{Sender, channel}; +use futures::stream::BoxStream; +use futures::{SinkExt, StreamExt, TryStreamExt}; + +use crate::delete_file_index::DeleteFileIndex; +use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator; +use crate::runtime::Runtime; +use crate::scan::DeleteFileContext; +use crate::scan::context::{BoundPredicates, ManifestEntryContext, PlanContext}; +use crate::spec::DataContentType; +use crate::{Error, ErrorKind, Result}; + +pub(crate) async fn plan_tasks( + plan_context: &PlanContext, + runtime: &Runtime, + concurrency_limit_manifest_files: usize, + concurrency_limit_manifest_entries: usize, + build_result: F, +) -> Result>> +where + T: Send + 'static, + F: Fn(ManifestEntryContext) -> Fut + Copy + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + // used to stream ManifestEntryContexts between stages of the file plan operation + let (manifest_entry_data_ctx_tx, manifest_entry_data_ctx_rx) = + channel(concurrency_limit_manifest_files); + let (manifest_entry_delete_ctx_tx, manifest_entry_delete_ctx_rx) = + channel(concurrency_limit_manifest_files); + + // used to stream the planned data file results back to the caller + let (planned_file_tx, planned_file_rx) = channel(concurrency_limit_manifest_entries); + + let (delete_file_idx, delete_file_tx) = DeleteFileIndex::new(runtime.clone()); + + // get the [`ManifestFile`]s from the [`ManifestList`], filtering out any + // whose partitions cannot match this + // scan's filter + let manifest_file_contexts = plan_context + .build_manifest_file_contexts( + manifest_entry_data_ctx_tx, + delete_file_idx.clone(), + manifest_entry_delete_ctx_tx, + ) + .await?; + + let mut channel_for_manifest_error = planned_file_tx.clone(); + let mut channel_for_data_manifest_entry_error = planned_file_tx.clone(); + let mut channel_for_delete_manifest_entry_error = planned_file_tx.clone(); + + let rt = runtime.clone(); + + // Concurrently load all [`Manifest`]s and stream their [`ManifestEntry`]s + rt.io().spawn(async move { + let result = futures::stream::iter(manifest_file_contexts) + .try_for_each_concurrent(concurrency_limit_manifest_files, |ctx| async move { + ctx.fetch_manifest_and_stream_manifest_entries().await + }) + .await; + + if let Err(error) = result { + let _ = channel_for_manifest_error.send(Err(error)).await; + } + }); + + // Process the delete file [`ManifestEntry`] stream in parallel + { + let rt = rt.clone(); + let rt_inner = rt.clone(); + rt.cpu().spawn(async move { + let result = manifest_entry_delete_ctx_rx + .map(|me_ctx| Ok((me_ctx, delete_file_tx.clone()))) + .try_for_each_concurrent( + concurrency_limit_manifest_entries, + |(manifest_entry_context, tx)| { + let rt_inner = rt_inner.clone(); + async move { + rt_inner + .cpu() + .spawn(async move { + process_delete_manifest_entry(manifest_entry_context, tx).await + }) + .await? + } + }, + ) + .await; + + if let Err(error) = result { + let _ = channel_for_delete_manifest_entry_error + .send(Err(error)) + .await; + } + }); + } + + // Process the data file [`ManifestEntry`] stream in parallel + { + let rt_inner = rt.clone(); + rt.cpu().spawn(async move { + let result = manifest_entry_data_ctx_rx + .map(|me_ctx| Ok((me_ctx, planned_file_tx.clone()))) + .try_for_each_concurrent( + concurrency_limit_manifest_entries, + |(manifest_entry_context, tx)| { + let rt_inner = rt_inner.clone(); + async move { + rt_inner + .cpu() + .spawn(async move { + process_data_manifest_entry( + manifest_entry_context, + tx, + build_result, + ) + .await + }) + .await? + } + }, + ) + .await; + + if let Err(error) = result { + let _ = channel_for_data_manifest_entry_error.send(Err(error)).await; + } + }); + } + + Ok(planned_file_rx.boxed()) +} + +async fn process_delete_manifest_entry( + manifest_entry_context: ManifestEntryContext, + mut delete_file_ctx_tx: Sender, +) -> Result<()> { + // skip processing this manifest entry if it has been marked as deleted + if !manifest_entry_context.manifest_entry.is_alive() { + return Ok(()); + } + + // abort the plan if we encounter a manifest entry that is not for a delete file + if manifest_entry_context.manifest_entry.content_type() == DataContentType::Data { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Encountered an entry for a data file in a delete manifest", + )); + } + + if let Some(ref bound_predicates) = manifest_entry_context.bound_predicates { + let expression_evaluator_cache = manifest_entry_context.expression_evaluator_cache.as_ref(); + + let expression_evaluator = expression_evaluator_cache.get( + manifest_entry_context.partition_spec_id, + &bound_predicates.partition_bound_predicate, + )?; + + // skip any data file whose partition data indicates that it can't contain + // any data that matches this scan's filter + if !expression_evaluator.eval(manifest_entry_context.manifest_entry.data_file())? { + return Ok(()); + } + } + + delete_file_ctx_tx + .send(DeleteFileContext { + manifest_entry: manifest_entry_context.manifest_entry.clone(), + partition_spec_id: manifest_entry_context.partition_spec_id, + }) + .await?; + + Ok(()) +} + +async fn process_data_manifest_entry( + manifest_entry_context: ManifestEntryContext, + mut planned_file_tx: Sender>, + build_result: F, +) -> Result<()> +where + F: Fn(ManifestEntryContext) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + // skip processing this manifest entry if it has been marked as deleted + if !manifest_entry_context.manifest_entry.is_alive() { + return Ok(()); + } + + // abort the plan if we encounter a manifest entry for a delete file + if manifest_entry_context.manifest_entry.content_type() != DataContentType::Data { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Encountered an entry for a delete file in a data file manifest", + )); + } + + if let Some(ref bound_predicates) = manifest_entry_context.bound_predicates { + let BoundPredicates { + scan_bound_predicate, + partition_bound_predicate, + } = bound_predicates.as_ref(); + + let expression_evaluator_cache = manifest_entry_context.expression_evaluator_cache.as_ref(); + + let expression_evaluator = expression_evaluator_cache.get( + manifest_entry_context.partition_spec_id, + partition_bound_predicate, + )?; + + // skip any data file whose partition data indicates that it can't contain + // any data that matches this scan's filter + if !expression_evaluator.eval(manifest_entry_context.manifest_entry.data_file())? { + return Ok(()); + } + + // skip any data file whose metrics don't match this scan's filter + if !InclusiveMetricsEvaluator::eval( + scan_bound_predicate, + manifest_entry_context.manifest_entry.data_file(), + false, + )? { + return Ok(()); + } + } + + // congratulations! the manifest entry has made its way through the + // entire plan without getting filtered out. Build the planned file before + // sending so delete-file lookup preserves the original scan timing. + planned_file_tx + .send(Ok(build_result(manifest_entry_context).await?)) + .await?; + + Ok(()) +}