Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 47 additions & 31 deletions crates/iceberg/src/scan/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,21 @@ use crate::delete_file_index::DeleteFileIndex;
use crate::expr::{Bind, BoundPredicate, Predicate};
use crate::io::object_cache::ObjectCache;
use crate::scan::{
BoundPredicates, ExpressionEvaluatorCache, FileScanTask, ManifestEvaluatorCache,
PartitionFilterCache,
ExpressionEvaluatorCache, FileScanTask, ManifestEvaluatorCache, PartitionFilterCache,
};
use crate::spec::{
ManifestContentType, ManifestEntryRef, ManifestFile, ManifestList, NameMapping,
PartitionSpecRef, SchemaRef, SnapshotRef, SortOrderRef, StructType, TableMetadataRef,
ManifestContentType, ManifestEntryRef, ManifestFile, NameMapping, PartitionSpecRef, SchemaRef,
SnapshotRef, SortOrderRef, StructType, TableMetadataRef,
};
use crate::{Error, ErrorKind, Result};

/// The predicates a manifest entry is evaluated against: one bound to the
/// manifest's partition spec, one bound to the scan's schema.
pub(crate) struct BoundPredicates {
pub(crate) partition_bound_predicate: BoundPredicate,
pub(crate) scan_bound_predicate: BoundPredicate,
}

/// Wraps a [`ManifestFile`] alongside the objects that are needed
/// to process it in a thread-safe manner
pub(crate) struct ManifestFileContext {
Expand All @@ -44,7 +50,7 @@ pub(crate) struct ManifestFileContext {
field_ids: Arc<Vec<i32>>,
bound_predicates: Option<Arc<BoundPredicates>>,
object_cache: Arc<ObjectCache>,
snapshot_schema: SchemaRef,
scan_schema: SchemaRef,
expression_evaluator_cache: Arc<ExpressionEvaluatorCache>,
delete_file_index: DeleteFileIndex,
name_mapping: Option<Arc<NameMapping>>,
Expand All @@ -63,7 +69,7 @@ pub(crate) struct ManifestEntryContext {
pub field_ids: Arc<Vec<i32>>,
pub bound_predicates: Option<Arc<BoundPredicates>>,
pub partition_spec_id: i32,
pub snapshot_schema: SchemaRef,
pub scan_schema: SchemaRef,
pub delete_file_index: DeleteFileIndex,
pub name_mapping: Option<Arc<NameMapping>>,
pub case_sensitive: bool,
Expand All @@ -81,7 +87,7 @@ impl ManifestFileContext {
object_cache,
manifest_file,
bound_predicates,
snapshot_schema,
scan_schema,
field_ids,
mut sender,
expression_evaluator_cache,
Expand Down Expand Up @@ -114,7 +120,7 @@ impl ManifestFileContext {
field_ids: field_ids.clone(),
partition_spec_id: manifest_file.partition_spec_id,
bound_predicates: bound_predicates.clone(),
snapshot_schema: snapshot_schema.clone(),
scan_schema: scan_schema.clone(),
delete_file_index: delete_file_index.clone(),
name_mapping: name_mapping.clone(),
case_sensitive,
Expand Down Expand Up @@ -155,11 +161,11 @@ impl ManifestEntryContext {
.with_data_sequence_number(self.manifest_entry.sequence_number())
.with_data_file_path(self.manifest_entry.file_path().to_string())
.with_data_file_format(self.manifest_entry.file_format())
.with_schema(self.snapshot_schema)
.with_schema(self.scan_schema)
.with_project_field_ids(self.field_ids.to_vec())
.with_predicate(
self.bound_predicates
.map(|x| x.as_ref().snapshot_bound_predicate.clone()),
.map(|x| x.as_ref().scan_bound_predicate.clone()),
)
.with_deletes(deletes)
.with_partition(Some(self.manifest_entry.data_file.partition.clone()))
Expand Down Expand Up @@ -191,20 +197,24 @@ impl ManifestEntryContext {
}
}

/// PlanContext wraps a [`SnapshotRef`] alongside all the other
/// objects that are required to perform a scan file plan.
/// PlanContext holds everything required to perform a scan file plan: the
/// snapshot whose manifests are read, and how to project, filter and evaluate
/// their entries.
#[derive(Debug)]
pub(crate) struct PlanContext {
pub snapshot: SnapshotRef,
/// The snapshot to plan against, or `None` for a table that has no
/// snapshots yet. Such a table lists no manifests and so plans no files,
/// but is still projected and filtered against its current schema.
pub snapshot: Option<SnapshotRef>,

pub table_metadata: TableMetadataRef,
/// The schema the scan projects. For a scan of the current state, this is
/// the table's current schema, not the snapshot's; for time travel, it is
/// the schema the snapshot was written with.
pub snapshot_schema: SchemaRef,
pub scan_schema: SchemaRef,
pub case_sensitive: bool,
pub predicate: Option<Arc<Predicate>>,
pub snapshot_bound_predicate: Option<Arc<BoundPredicate>>,
pub scan_bound_predicate: Option<Arc<BoundPredicate>>,
pub object_cache: Arc<ObjectCache>,
pub field_ids: Arc<Vec<i32>>,
pub name_mapping: Option<Arc<NameMapping>>,
Expand All @@ -222,13 +232,6 @@ pub(crate) struct PlanContext {
}

impl PlanContext {
pub(crate) async fn get_manifest_list(&self) -> Result<Arc<ManifestList>> {
self.object_cache
.as_ref()
.get_manifest_list(&self.snapshot, &self.table_metadata)
.await
}

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


/// Returns the partition filter for a manifest. See [`PartitionFilterCache::get`] for the
/// always-true fallback when the manifest's spec cannot be resolved against the scan schema.
fn get_partition_filter(&self, manifest_file: &ManifestFile) -> Result<Arc<BoundPredicate>> {
Expand All @@ -237,7 +240,7 @@ impl PlanContext {
let partition_filter = self.partition_filter_cache.get(
partition_spec_id,
&self.table_metadata,
&self.snapshot_schema,
&self.scan_schema,
self.case_sensitive,
self.predicate
.as_ref()
Expand All @@ -246,20 +249,33 @@ impl PlanContext {
"Expected a predicate but none present",
))?
.as_ref()
.bind(self.snapshot_schema.clone(), self.case_sensitive)?,
.bind(self.scan_schema.clone(), self.case_sensitive)?,
)?;

Ok(partition_filter)
}

pub(crate) fn build_manifest_file_contexts(
pub(crate) async fn build_manifest_file_contexts(
&self,
manifest_list: Arc<ManifestList>,
tx_data: Sender<ManifestEntryContext>,
delete_file_idx: DeleteFileIndex,
delete_file_tx: Sender<ManifestEntryContext>,
) -> Result<Box<impl Iterator<Item = Result<ManifestFileContext>> + 'static>> {
let mut manifest_files = manifest_list.entries().iter().collect::<Vec<_>>();
// A table with no snapshots lists no manifests, so the plan is empty.
let manifest_list = match self.snapshot.as_ref() {
Some(snapshot) => Some(
self.object_cache
.get_manifest_list(snapshot, &self.table_metadata)
.await?,
),
None => None,
};

let mut manifest_files: Vec<&ManifestFile> = manifest_list
.iter()
.flat_map(|manifest_list| manifest_list.entries())
.collect();

// Sort manifest files to process delete manifests first.
// This avoids a deadlock where the producer blocks on sending data manifest entries
// (because the data channel is full) while the delete manifest consumer is waiting
Expand Down Expand Up @@ -322,12 +338,12 @@ impl PlanContext {
delete_file_index: DeleteFileIndex,
) -> ManifestFileContext {
let bound_predicates =
if let (Some(ref partition_bound_predicate), Some(snapshot_bound_predicate)) =
(partition_filter, &self.snapshot_bound_predicate)
if let (Some(ref partition_bound_predicate), Some(scan_bound_predicate)) =
(partition_filter, &self.scan_bound_predicate)
{
Some(Arc::new(BoundPredicates {
partition_bound_predicate: partition_bound_predicate.as_ref().clone(),
snapshot_bound_predicate: snapshot_bound_predicate.as_ref().clone(),
scan_bound_predicate: scan_bound_predicate.as_ref().clone(),
}))
} else {
None
Expand All @@ -338,7 +354,7 @@ impl PlanContext {
bound_predicates,
sender,
object_cache: self.object_cache.clone(),
snapshot_schema: self.snapshot_schema.clone(),
scan_schema: self.scan_schema.clone(),
field_ids: self.field_ids.clone(),
expression_evaluator_cache: self.expression_evaluator_cache.clone(),
delete_file_index,
Expand Down
Loading
Loading