Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 11 additions & 14 deletions vortex-datafusion/src/persistent/opener.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,6 @@ pub(crate) struct VortexOpener {
pub projection_pushdown: bool,
pub scan_concurrency: Option<usize>,
pub scan_v2: bool,
/// How many partitions the engine runs the scan on, so a V2 scan parallelises inside a
/// partition only as far as the engine has not.
pub partitions: usize,
}

/// Identifies a dynamic filter of a scan's predicate by its allocation, which the predicate keeps
Expand Down Expand Up @@ -230,7 +227,6 @@ impl FileOpener for VortexOpener {
let file_schemas = Arc::clone(&self.file_schemas);
let has_output_ordering = self.has_output_ordering;
let scan_concurrency = self.scan_concurrency;
let partitions = self.partitions.max(1);
let scan_v2 = self.scan_v2;

let expr_converter = Arc::clone(&self.expression_converter);
Expand Down Expand Up @@ -505,6 +501,11 @@ impl FileOpener for VortexOpener {
// one applies it again, so the scan may stop evaluating one that rejects
// almost nothing.
.with_skip_unselective_dynamic(true);
// How many partitions split the file, assuming they cut it evenly.
let readers = byte_range.as_ref().map_or(1, |bytes| {
let len = (bytes.end - bytes.start).max(1);
usize::try_from(file.object_meta.size.div_ceil(len)).unwrap_or(usize::MAX)
});
if let Some(bytes) = byte_range {
// Every partition maps the same byte boundary to the same row boundary.
let rows = partition_row_range(bytes, file.object_meta.size, vxf.row_count());
Expand All @@ -520,14 +521,16 @@ impl FileOpener for VortexOpener {
{
builder = builder.with_selection(selection.clone());
}
// Unless set, chunks run beside each other only as far as the engine's partitions
// leave cores idle: at one partition per core they run in the scan's own run.
let concurrency = scan_concurrency.unwrap_or_else(|| {
// Unless set, chunks run beside each other only as far as the cores outnumber the
// partitions reading this file: a file split a partition per core is read in each
// scan's own run, a file read whole by one partition on every core.
let concurrency = scan_concurrency.filter(|&c| c > 0).unwrap_or_else(|| {
get_available_parallelism()
.unwrap_or(1)
.div_ceil(partitions)
.div_ceil(readers)
.max(1)
});
builder = builder.with_concurrency(concurrency);
let handle = vxf.session().handle();
// Lowering and planning a wide file is real CPU: it runs where compute runs.
let scan = handle
Expand Down Expand Up @@ -1195,7 +1198,6 @@ mod tests {
layout_readers: Default::default(),
segment_sources: Default::default(),
scan_caches: Default::default(),
partitions: 1,
natural_splits: Default::default(),
file_schemas: Default::default(),
dynamic_filters: Default::default(),
Expand Down Expand Up @@ -1606,7 +1608,6 @@ mod tests {
layout_readers: Default::default(),
segment_sources: Default::default(),
scan_caches: Default::default(),
partitions: 1,
natural_splits: Default::default(),
file_schemas: Default::default(),
dynamic_filters: Default::default(),
Expand Down Expand Up @@ -1699,7 +1700,6 @@ mod tests {
layout_readers: Default::default(),
segment_sources: Default::default(),
scan_caches: Default::default(),
partitions: 1,
natural_splits: Default::default(),
file_schemas: Default::default(),
dynamic_filters: Default::default(),
Expand Down Expand Up @@ -1883,7 +1883,6 @@ mod tests {
layout_readers: Default::default(),
segment_sources: Default::default(),
scan_caches: Default::default(),
partitions: 1,
natural_splits: Default::default(),
file_schemas: Default::default(),
dynamic_filters: Default::default(),
Expand Down Expand Up @@ -1949,7 +1948,6 @@ mod tests {
layout_readers: Default::default(),
segment_sources: Default::default(),
scan_caches: Default::default(),
partitions: 1,
natural_splits: Default::default(),
file_schemas: Default::default(),
dynamic_filters: Default::default(),
Expand Down Expand Up @@ -2166,7 +2164,6 @@ mod tests {
layout_readers: Default::default(),
segment_sources: Default::default(),
scan_caches: Default::default(),
partitions: 1,
natural_splits: Default::default(),
file_schemas: Default::default(),
dynamic_filters: Default::default(),
Expand Down
16 changes: 7 additions & 9 deletions vortex-datafusion/src/persistent/scan_v2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,20 @@ use vortex::array::ArrayRef;
use vortex::error::VortexResult;
use vortex::error::vortex_err;
use vortex::io::request::IoService;
use vortex::io::request::IoSource;
use vortex::io::runtime::Handle;
use vortex::layout::scan::v2::PreparedScan;
use vortex::scan::planning::driver::Progress;
use vortex::scan::planning::driver::Run;
use vortex::scan::planning::planner::Planner;
use vortex::scan::planning::planner::WorkScope;
use vortex_utils::parallelism::get_available_parallelism;

/// Maps a scan's batches to Arrow on the task that produced them.
type MapBatch = Arc<dyn Fn(ArrayRef) -> VortexResult<RecordBatch> + Send + Sync>;

/// Runs one partition's V2 scan on DataFusion's runtime.
pub(crate) struct ScanDriver {
handle: Handle,
/// Chunks run beside each other per worker thread; at 1 they run on the scan's own run.
/// The scan's chunks run at once; at 1 they run on the scan's own run.
concurrency: usize,
/// Whether batches are yielded in row order, which holds them until the scan is done.
ordered: bool,
Expand All @@ -65,8 +63,8 @@ impl ScanDriver {
map: impl Fn(ArrayRef) -> VortexResult<RecordBatch> + Send + Sync + 'static,
) -> BoxStream<'static, VortexResult<RecordBatch>> {
let map: MapBatch = Arc::new(map);
let in_flight = self.concurrency * get_available_parallelism().unwrap_or(1);
let inline = in_flight <= 1;
let in_flight = self.concurrency;
let inline = self.concurrency <= 1;
let scope = WorkScope {
file_ordinal: 0,
rows: scan.rows().clone(),
Expand All @@ -83,7 +81,7 @@ impl ScanDriver {
&chunk_handle,
chunk,
scope,
chunk_io.session(),
Arc::clone(&chunk_io),
Arc::clone(&chunk_map),
),
})
Expand Down Expand Up @@ -214,13 +212,13 @@ fn chunk_stream(
handle: &Handle,
root: Box<dyn Planner>,
scope: WorkScope,
io: Arc<dyn IoSource>,
io: Arc<dyn IoService>,
map: MapBatch,
) -> BoxStream<'static, VortexResult<(u64, RecordBatch)>> {
let (mut send, recv) = mpsc::channel(1);
let task = handle.spawn(async move {
let mut run = Run::new();
run.admit(root, scope, Arc::clone(&io));
run.admit(root, scope, io.session());
loop {
match run.advance()? {
Progress::Batch(batch) => {
Expand All @@ -232,7 +230,7 @@ fn chunk_stream(
}
}
Progress::Spawn(scope, child) => {
run.admit(child, scope, Arc::clone(&io));
run.admit(child, scope, io.session());
}
Progress::RootDone(_) if run.live_roots() > 0 => {}
Progress::RootDone(_) | Progress::Idle => return Ok(()),
Expand Down
31 changes: 30 additions & 1 deletion vortex-datafusion/src/persistent/source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,31 @@ use crate::persistent::opener::DynamicFilterKey;
use crate::persistent::reader::DefaultVortexReaderFactory;
use crate::persistent::reader::VortexReaderFactory;

/// The fewest rows a filtered V2 scan partition reads when the engine splits files. Each
/// partition proves zone maps, plans its filter and compiles its plans once, so a partition much
/// smaller than a few output windows spends more on that than on reading rows. An unfiltered scan
/// keeps the engine's partitions: every row it reads flows into the operators above it, which
/// run a task per partition.
const V2_MIN_ROWS_PER_PARTITION: usize = 256 * 1024;

/// Caps `target_partitions` so no V2 partition reads fewer than [`V2_MIN_ROWS_PER_PARTITION`]
/// rows, when the files' row counts are known.
fn v2_target_partitions(target_partitions: usize, config: &FileScanConfig) -> usize {
let rows = config
.file_groups
.iter()
.flat_map(|group| group.iter())
.map(|file| {
file.statistics
.as_ref()
.and_then(|statistics| statistics.num_rows.get_value().copied())
})
.sum::<Option<usize>>();
rows.map_or(target_partitions, |rows| {
target_partitions.min(rows.div_ceil(V2_MIN_ROWS_PER_PARTITION).max(1))
})
}

/// File scan implementation for reading one or more `.vortex` files.
///
/// `VortexSource` is the lower-level read component underneath
Expand Down Expand Up @@ -385,7 +410,6 @@ impl VortexSource {
projection_pushdown: self.options.projection_pushdown,
scan_concurrency: self.options.scan_concurrency,
scan_v2: self.options.scan_v2,
partitions: base_config.file_groups.len(),
};

Ok(opener)
Expand Down Expand Up @@ -475,6 +499,11 @@ impl FileSource for VortexSource {
if config.file_compression_type.is_compressed() {
return Ok(None);
}
let target_partitions = if self.options.scan_v2 && self.vortex_predicate.is_some() {
v2_target_partitions(target_partitions, config)
} else {
target_partitions
};
let Some(file_groups) = FileGroupPartitioner::new()
.with_target_partitions(target_partitions)
.with_repartition_file_min_size(repartition_file_min_size)
Expand Down
43 changes: 38 additions & 5 deletions vortex-datafusion/src/persistent/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -630,9 +630,10 @@ async fn test_repartitioned_scan_matches_non_repartitioned_for_uneven_splits(
let session = VortexSession::default();
let path = object_store::path::Path::parse("/split-aligned-repartition.vortex")?;

let chunk_1_len = 2_000;
let chunk_2_len = 5_000;
let chunk_3_len = 6_000;
// Enough rows that a V2 scan, which keeps a minimum of rows per partition, still splits.
let chunk_1_len = 200_000;
let chunk_2_len = 250_000;
let chunk_3_len = 300_000;
let row_count = chunk_1_len + chunk_2_len + chunk_3_len;

let chunk_1 = StructArray::try_new(
Expand Down Expand Up @@ -726,7 +727,7 @@ async fn test_repartitioned_scan_matches_non_repartitioned_for_uneven_splits(
Ok(())
}

/// With a scan concurrency of 0, a V2 scan runs every chunk on its own run as another root. Each
/// With a scan concurrency of 1, a V2 scan runs every chunk on its own run as another root. Each
/// root needs an IO session of its own: the run clears a root's session when the root retires, so
/// a shared session left the remaining chunks waiting for reads that were never issued.
#[tokio::test]
Expand Down Expand Up @@ -764,7 +765,7 @@ async fn v2_inline_chunks_read_through_their_own_sessions() -> anyhow::Result<()
let factory = Arc::new(
VortexFormatFactory::new().with_options(crate::VortexTableOptions {
scan_v2: true,
scan_concurrency: Some(0),
scan_concurrency: Some(1),
..Default::default()
}),
);
Expand Down Expand Up @@ -794,6 +795,38 @@ async fn v2_inline_chunks_read_through_their_own_sessions() -> anyhow::Result<()
Ok(())
}

/// A filtered V2 scan of a file smaller than the minimum rows per partition reads it in one
/// partition. An unfiltered scan keeps the engine's partitions: every row it reads flows into the
/// operators above it.
#[tokio::test]
async fn v2_scan_keeps_partitions_unless_filtered() -> anyhow::Result<()> {
let store = Arc::new(InMemory::new()) as _;
let session = VortexSession::default();
let path = object_store::path::Path::parse("/small.vortex")?;
let table = StructArray::try_new(
["value"].into(),
vec![Buffer::from_iter(0_i32..100_000).into_array()],
100_000,
Validity::NonNullable,
)?
.into_array();
let mut writer = ObjectStoreWrite::new(Arc::clone(&store), &path).await?;
session
.write_options()
.write(&mut writer, table.to_array_stream())
.await?;
writer.shutdown().await?;

let ctx = make_session(store, true, true);
let unfiltered = count_query_partitions(&ctx, "SELECT value FROM '/small.vortex'").await?;
let filtered =
count_query_partitions(&ctx, "SELECT value FROM '/small.vortex' WHERE value > 10").await?;

assert_eq!(unfiltered, 4);
assert_eq!(filtered, 1);
Ok(())
}

/// Roundtrip an `arrow.uuid` extension column through a Vortex file: write the column directly
/// via the session-aware Arrow→Vortex conversion, then `SELECT *` and assert both the field
/// metadata and the underlying values survive the trip.
Expand Down
Loading
Loading