From 4e4ed9d4b6f5b18ee23c440e0435e303165b9745 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 29 Oct 2026 14:15:54 +0000 Subject: [PATCH 01/28] Improve Scan V2 TopK feedback and sparse payload retrieval Emit small startup chunks per DuckDB worker, refresh dynamic pruning and row filters when bounds change, and adapt predicate order using observed cost and selectivity with occasional exploration. Preserve exact file and row pairs during DuckDB V2 late materialization. Retain the file-index hint for pre-open file pruning and gather sparse dictionary selections before conversion when encoded take kernels apply. Add benchmark controls, diagnostics, and regression coverage for startup, dynamic thresholds, adaptive ordering, paired row lookup, and export. Validation before develop integration: 301 DuckDB tests passed (2 S3 skips), workspace all-target/all-feature Clippy, Rust/C++ formatting, and diff checks. Q23 improved 37-41% across repeated 40-warm-sample local comparisons; payload rows 70 -> 10 and payload arrays 49 -> 7. Signed-off-by: Joe Isaacs --- Cargo.lock | 2 + benchmarks/datafusion-bench/Cargo.toml | 1 + benchmarks/datafusion-bench/src/lib.rs | 9 +- benchmarks/datafusion-bench/src/main.rs | 18 +- benchmarks/duckdb-bench/src/lib.rs | 5 + benchmarks/duckdb-bench/src/main.rs | 6 + vortex-datafusion/src/persistent/format.rs | 9 + vortex-datafusion/src/persistent/opener.rs | 99 +++-- vortex-datafusion/src/persistent/scan_v2.rs | 81 ++++ .../src/persistent/scan_v2/tests.rs | 52 +++ vortex-datafusion/src/persistent/source.rs | 54 +++ vortex-datafusion/src/persistent/stream.rs | 65 +++- vortex-duckdb/Cargo.toml | 1 + vortex-duckdb/cpp/include/table_function.hpp | 1 + vortex-duckdb/cpp/multi_file_reader.cpp | 4 +- vortex-duckdb/cpp/table_function.cpp | 20 +- vortex-duckdb/include/vortex.h | 8 +- vortex-duckdb/src/convert/table_filter.rs | 18 +- .../src/e2e_test/vortex_scan_test.rs | 90 +++++ vortex-duckdb/src/exporter/cache.rs | 11 + vortex-duckdb/src/exporter/dict.rs | 159 +------- vortex-duckdb/src/exporter/dict/tests.rs | 206 ++++++++++ vortex-duckdb/src/ffi.rs | 9 +- vortex-duckdb/src/file_reader.rs | 103 +++-- vortex-duckdb/src/lib.rs | 10 + vortex-duckdb/src/projection.rs | 23 ++ vortex-duckdb/src/row_id.rs | 195 ++++++++++ vortex-duckdb/src/row_id/tests.rs | 73 ++++ vortex-duckdb/src/scan_v2/diagnostics.rs | 148 +++++++ vortex-duckdb/src/scan_v2/startup.rs | 174 +++++++++ vortex-duckdb/src/table_function.rs | 21 +- vortex-layout/src/scan/execution/chunk.rs | 14 +- vortex-layout/src/scan/execution/filter.rs | 362 +++++++++++++++--- .../src/scan/execution/filter/ordering.rs | 198 ++++++++++ .../scan/execution/filter/ordering/tests.rs | 188 +++++++++ vortex-layout/src/scan/execution/mod.rs | 12 +- .../src/scan/execution/projection.rs | 16 +- vortex-layout/src/scan/execution/prune.rs | 28 +- vortex-layout/src/scan/execution/selection.rs | 2 +- vortex-layout/src/scan/v2/cache.rs | 159 +++++--- vortex-layout/src/scan/v2/mod.rs | 54 ++- vortex-layout/src/scan/v2/tests.rs | 295 +++++++++++++- 42 files changed, 2615 insertions(+), 388 deletions(-) create mode 100644 vortex-datafusion/src/persistent/scan_v2/tests.rs create mode 100644 vortex-duckdb/src/exporter/dict/tests.rs create mode 100644 vortex-duckdb/src/row_id.rs create mode 100644 vortex-duckdb/src/row_id/tests.rs create mode 100644 vortex-duckdb/src/scan_v2/diagnostics.rs create mode 100644 vortex-duckdb/src/scan_v2/startup.rs create mode 100644 vortex-layout/src/scan/execution/filter/ordering.rs create mode 100644 vortex-layout/src/scan/execution/filter/ordering/tests.rs diff --git a/Cargo.lock b/Cargo.lock index d70c38aa80c..e3a7cdd02fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2815,6 +2815,7 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "tokio", + "tracing", "url", "vortex", "vortex-arrow", @@ -11900,6 +11901,7 @@ dependencies = [ "vortex", "vortex-array", "vortex-duckdb", + "vortex-onpair", "vortex-sequence", "vortex-spatial", "vortex-utils", diff --git a/benchmarks/datafusion-bench/Cargo.toml b/benchmarks/datafusion-bench/Cargo.toml index 1e6968db76a..0eb71e52223 100644 --- a/benchmarks/datafusion-bench/Cargo.toml +++ b/benchmarks/datafusion-bench/Cargo.toml @@ -42,6 +42,7 @@ opentelemetry-otlp.workspace = true opentelemetry_sdk.workspace = true parking_lot = { workspace = true } tokio = { workspace = true, features = ["full"] } +tracing = { workspace = true } url = { workspace = true } vortex = { workspace = true, features = ["object_store", "files", "tokio"] } vortex-arrow = { workspace = true } diff --git a/benchmarks/datafusion-bench/src/lib.rs b/benchmarks/datafusion-bench/src/lib.rs index 5f0b1a6800e..b25b5de03e6 100644 --- a/benchmarks/datafusion-bench/src/lib.rs +++ b/benchmarks/datafusion-bench/src/lib.rs @@ -98,9 +98,16 @@ pub fn make_object_store( } } -pub fn format_to_df_format(format: Format, scan_v2: bool) -> anyhow::Result> { +pub fn format_to_df_format( + format: Format, + scan_v2: bool, + dynamic_filter_initial_chunks: Option, +) -> anyhow::Result> { let mut options = vortex_table_options(); options.scan_v2 = scan_v2; + if let Some(chunks) = dynamic_filter_initial_chunks { + options.dynamic_filter_initial_chunks = chunks; + } Ok(match format { Format::Csv => Arc::new(CsvFormat::default()) as _, Format::Parquet => Arc::new(ParquetFormat::new()), diff --git a/benchmarks/datafusion-bench/src/main.rs b/benchmarks/datafusion-bench/src/main.rs index e2acd536fdc..ca4e7be9c5c 100644 --- a/benchmarks/datafusion-bench/src/main.rs +++ b/benchmarks/datafusion-bench/src/main.rs @@ -124,6 +124,10 @@ struct Args { #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] scan_v1: bool, + /// Override the initial 8K chunks per worker for Vortex scans with dynamic filters. + #[arg(long)] + dynamic_filter_initial_chunks: Option, + #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))] options: Vec, } @@ -136,6 +140,7 @@ async fn main() -> anyhow::Result<()> { vortex_bench::configure_allocator(); let args = Args::parse(); let opts = Opts::from(args.options); + let dynamic_filter_initial_chunks = args.dynamic_filter_initial_chunks; let scan_v2 = !args.scan_v1; set_join_set_tracer(get_static_tracer())?; @@ -196,7 +201,14 @@ async fn main() -> anyhow::Result<()> { session.sql(&sql).await?.collect().await?; } datafusion_bench::make_object_store(&session, benchmark.data_url())?; - register_benchmark_tables(&session, benchmark, format, scan_v2).await?; + register_benchmark_tables( + &session, + benchmark, + format, + scan_v2, + dynamic_filter_initial_chunks, + ) + .await?; Ok((session, format)) } }, @@ -208,6 +220,7 @@ async fn main() -> anyhow::Result<()> { Box::pin( async move { let timer = Instant::now(); + tracing::debug!(query_idx, "benchmark query started"); let (batches, plan) = execute_query(session, query) .with_labelset(get_labelset_from_global()) .await?; @@ -261,13 +274,14 @@ async fn register_benchmark_tables( benchmark: &B, format: Format, scan_v2: bool, + dynamic_filter_initial_chunks: Option, ) -> anyhow::Result<()> { if !scan_v2 && use_scan_api() && matches!(format, Format::OnDiskVortex | Format::VortexCompact) { register_v2_tables(session, benchmark, format).await } else { let benchmark_base = benchmark.data_url().join(&format!("{}/", format.name()))?; - let file_format = format_to_df_format(format, scan_v2)?; + let file_format = format_to_df_format(format, scan_v2, dynamic_filter_initial_chunks)?; for table in benchmark.table_specs().iter() { let pattern = benchmark.pattern(table.name, format); diff --git a/benchmarks/duckdb-bench/src/lib.rs b/benchmarks/duckdb-bench/src/lib.rs index a5c90ce3c23..4d23ea4e172 100644 --- a/benchmarks/duckdb-bench/src/lib.rs +++ b/benchmarks/duckdb-bench/src/lib.rs @@ -243,6 +243,11 @@ impl DuckClient { let time_instant = Instant::now(); let result = self.connection().query(query)?; let query_time = time_instant.elapsed(); + tracing::debug!( + elapsed_us = query_time.as_micros(), + result_rows = result.row_count(), + "benchmark query completed" + ); Ok((Some(query_time), DuckQueryResult(result))) } } diff --git a/benchmarks/duckdb-bench/src/main.rs b/benchmarks/duckdb-bench/src/main.rs index bac918790b3..0e82645894c 100644 --- a/benchmarks/duckdb-bench/src/main.rs +++ b/benchmarks/duckdb-bench/src/main.rs @@ -106,6 +106,10 @@ struct Args { /// Use the V1 Vortex scan. Pass `--scan-v1 false` to use the planner-driven V2 scan. #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] scan_v1: bool, + + /// Initial 8K-row chunks per worker for V2 scans with a dynamic filter; zero disables. + #[arg(long, default_value_t = 0)] + dynamic_filter_initial_chunks: usize, } #[global_allocator] @@ -117,6 +121,7 @@ fn main() -> anyhow::Result<()> { let opts = Opts::from(args.options); vortex_duckdb::set_scan_v2(!args.scan_v1); + vortex_duckdb::set_dynamic_filter_initial_chunks(args.dynamic_filter_initial_chunks); setup_logging_and_tracing(args.verbose, args.tracing)?; @@ -236,6 +241,7 @@ fn main() -> anyhow::Result<()> { if !args.reuse { ctx.reopen()?; } + tracing::debug!(query_idx, "benchmark query started"); ctx.execute_query_result(query) }, )?; diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index fb50ed26d17..994b42795d8 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -239,6 +239,13 @@ extensions_options! { /// with `SET vortex.scan_concurrency = `, or per table with /// `OPTIONS(scan_concurrency '')`. pub scan_concurrency: Option, default = None + /// Initial 8,192-row chunks per worker for V2 scans carrying a dynamic filter. + /// + /// Pruning, filtering and projection of these chunks run concurrently before bulk + /// work, allowing operators such as TopK to update their bounds sooner. The budget is + /// shared across a partition's files. Zero disables this startup phase. Scans that + /// must preserve input order use the normal scan path. Disabled by default. + pub dynamic_filter_initial_chunks: usize, default = 0 } } @@ -1121,6 +1128,7 @@ mod tests { projection_pushdown: true, predicate_pushdown: false, scan_concurrency: Some(3), + dynamic_filter_initial_chunks: 4, ..Default::default() }; let format = VortexFormat::new_with_options(VortexSession::default(), opts.clone()); @@ -1138,6 +1146,7 @@ mod tests { assert_eq!(source.options().predicate_pushdown, opts.predicate_pushdown); assert_eq!(source.options().scan_concurrency, opts.scan_concurrency); assert_eq!(source.options().scan_v2, opts.scan_v2); + assert_eq!(source.options().dynamic_filter_initial_chunks, 4); Ok(()) } diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 208691f0b0e..83e69bbaf8a 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -5,6 +5,8 @@ use std::future::ready; use std::ops::Range; use std::sync::Arc; use std::sync::Weak; +use std::sync::atomic::AtomicUsize; +use std::time::Instant; use arrow_array::RecordBatchOptions; use arrow_schema::DataType; @@ -56,17 +58,13 @@ use vortex::io::InstrumentedReadAt; use vortex::io::session::RuntimeSessionExt; use vortex::layout::LayoutReader; use vortex::layout::scan::scan_builder::ScanBuilder; -use vortex::layout::scan::v2::ChunkPlanner; -use vortex::layout::scan::v2::PrunePlanner; use vortex::layout::scan::v2::ScanBuilder as V2ScanBuilder; use vortex::layout::scan::v2::ScanCache; use vortex::layout::scan::v2::WeakScanCache; use vortex::layout::scan::v2::partition_row_range; use vortex::layout::segments::SegmentSource; -use vortex::mask::Mask; use vortex::metrics::Label; use vortex::metrics::MetricsRegistry; -use vortex::scan::planning::planner::Planner; use vortex::session::VortexSession; use vortex_arrow::ArrowSessionExt; use vortex_utils::aliases::dash_map::DashMap; @@ -83,6 +81,8 @@ use crate::metrics::PATH_LABEL; use crate::persistent::cache::CachedVortexMetadata; use crate::persistent::reader::VortexReaderFactory; use crate::persistent::scan_v2::ScanDriver; +use crate::persistent::scan_v2::initial_ranges; +use crate::persistent::scan_v2::scan_planner; use crate::persistent::stream::PrunableStream; #[derive(Clone)] @@ -140,6 +140,8 @@ pub(crate) struct VortexOpener { /// How many partitions the engine runs the scan on, so a V2 scan parallelises inside a /// partition only as far as the engine has not. pub partitions: usize, + /// Startup work shared by every file opened on this execution partition. + pub initial_chunks: Arc, } impl FileOpener for VortexOpener { @@ -177,6 +179,7 @@ impl FileOpener for VortexOpener { let natural_splits = Arc::clone(&self.natural_splits); let file_schemas = Arc::clone(&self.file_schemas); let has_output_ordering = self.has_output_ordering; + let initial_chunks = Arc::clone(&self.initial_chunks); let scan_concurrency = self.scan_concurrency; let partitions = self.partitions.max(1); let scan_v2 = self.scan_v2; @@ -211,7 +214,11 @@ impl FileOpener for VortexOpener { .transpose()?; } + let scan_span = tracing::debug_span!("v2_scan", path = %file.path()); Ok(async move { + let started = Instant::now(); + let observed_predicate = file_pruning_predicate.clone(); + let observed_path = file.path().to_string(); // FilePruner now has all the logic to figure out if its worthwhile, and will return `None` if not. let mut file_pruner = file_pruning_predicate.and_then(|predicate| { FilePruner::try_new( @@ -423,6 +430,7 @@ impl FileOpener for VortexOpener { let native_limit = limit .filter(|_| scan_v2 && residual_filter.is_none()) .map(|limit| (limit, Arc::clone(&output_schema))); + let mut first_batch = true; let map_batch = move |chunk| { let mut ctx = session.create_execution_ctx(); let arrow_session = ctx.session().clone(); @@ -475,27 +483,21 @@ impl FileOpener for VortexOpener { .map_err(|e| exec_datafusion_err!("Failed to plan Vortex scan: {e}"))?; let scan = Arc::new(scan); - // The planners, composed: with a filter, its pruning plans first over the whole - // partition, continuing into the chunks of the rows they keep; without a - // filter, the chunks directly. A chunk plans its conjuncts, then its - // projection. - let chunks = { - let scan = Arc::clone(&scan); - move |kept: Option| -> Box { - let rows = scan.rows().clone(); - let selection = scan.selection().clone(); - Box::new(ChunkPlanner::new(scan, rows, selection, kept)) - } - }; - let root: Box = if scan.filter().is_some() { - let rows = scan.rows().clone(); - Box::new(PrunePlanner::new(Arc::clone(&scan), rows, Box::new(chunks))) + let initial = if !has_output_ordering { + initial_ranges(scan.rows().clone(), &initial_chunks) } else { - chunks(None) + Vec::new() }; + let bulk_start = initial.last().map_or_else(|| scan.rows().start, |rows| rows.end); + tracing::debug!( + initial_chunks = initial.len(), + bulk_start, + "v2 topk startup" + ); + let root = scan_planner(Arc::clone(&scan), bulk_start..scan.rows().end); let driver = ScanDriver::new(handle, concurrency, has_output_ordering); - Ok(driver.run(scan, root, map_batch)) + Ok(driver.run(scan, root, initial, map_batch)) } else { // Share V1 layout readers across partitions of the file. let layout_reader = match layout_readers.entry(file.object_meta.location.clone()) { @@ -587,6 +589,11 @@ impl FileOpener for VortexOpener { } } let batch = projector.project_batch(&batch)?; + tracing::debug!(path = %residual_path, rows = batch.num_rows(), "scan batch returned"); + if first_batch { + first_batch = false; + tracing::debug!(path = %residual_path, elapsed_us = started.elapsed().as_micros(), rows = batch.num_rows(), "scan first batch"); + } let (_, columns, row_count) = batch.into_parts(); Ok(Some(RecordBatch::try_new_with_options( @@ -616,12 +623,16 @@ impl FileOpener for VortexOpener { if let Some(file_pruner) = file_pruner && file_pruner.is_watching() { - Ok(PrunableStream::new(file_pruner, stream).boxed()) + let mut stream = PrunableStream::new(file_pruner, stream); + if scan_v2 && let Some(predicate) = observed_predicate { + stream = stream.observe_filter(predicate, observed_path, started)?; + } + Ok(stream.boxed()) } else { Ok(stream) } } - .in_current_span() + .instrument(scan_span) .boxed()) } } @@ -832,6 +843,7 @@ mod tests { use std::fmt; use std::sync::Arc; use std::sync::LazyLock; + use std::sync::atomic::Ordering; use arrow_array::record_batch; use arrow_schema::Field; @@ -854,6 +866,7 @@ mod tests { use datafusion::scalar::ScalarValue; use datafusion_common::Statistics; use datafusion_common::arrow::compute::concat_batches; + use datafusion_common::arrow::compute::sort; use datafusion_common::assert_batches_eq; use datafusion_common::stats::Precision; use datafusion_datasource::file::FileSource; @@ -872,6 +885,7 @@ mod tests { use object_store::memory::InMemory; use rstest::rstest; use vortex::VortexSessionDefault; + use vortex::array::VortexSessionExecute; use vortex::buffer::Buffer; use vortex::file::WriteOptionsSessionExt; use vortex::io::VortexWrite; @@ -880,6 +894,7 @@ mod tests { use vortex::scan::selection::Selection; use vortex::scan::strict_sorted_buffer::StrictSortedBuffer; use vortex::session::VortexSession; + use vortex_array::assert_arrays_eq; use super::*; use crate::VortexAccessPlan; @@ -1127,6 +1142,38 @@ mod tests { Ok(summary.size()) } + #[tokio::test] + async fn startup_and_bulk_keep_filtered_selected_rows_once() -> anyhow::Result<()> { + let object_store = Arc::new(InMemory::new()) as Arc; + let batch = record_batch!(("a", Int32, (0..30_000).collect::>()))?; + let schema = batch.schema(); + let size = + write_arrow_to_vortex(Arc::clone(&object_store), "startup.vortex", batch).await?; + let table_schema = TableSchema::from(Arc::clone(&schema)); + let filter = logical2physical(&col("a").gt(lit(10)), table_schema.table_schema()); + let mut opener = make_opener(object_store, table_schema, Some(filter)); + opener.scan_v2 = true; + opener.initial_chunks = Arc::new(AtomicUsize::new(2)); + let indices = StrictSortedBuffer::try_new(Buffer::from(vec![ + 3, 17, 8188, 8192, 16_390, 20_000, 29_999, + ]))?; + let file = PartitionedFile::new("startup.vortex".to_string(), size).with_extension( + VortexAccessPlan::default().with_selection(Selection::IncludeByIndex(indices)), + ); + let batches = opener.open(file)?.await?.try_collect::>().await?; + assert_eq!(opener.initial_chunks.load(Ordering::Relaxed), 0); + let actual = concat_batches(&schema, &batches)?; + let actual = + RecordBatch::try_new(Arc::clone(&schema), vec![sort(actual.column(0), None)?])?; + let expected = record_batch!(("a", Int32, [17, 8188, 8192, 16_390, 20_000, 29_999]))?; + assert_arrays_eq!( + SESSION.arrow().from_arrow_record_batch(actual, &schema)?, + SESSION.arrow().from_arrow_record_batch(expected, &schema)?, + &mut SESSION.create_execution_ctx() + ); + Ok(()) + } + fn make_opener( object_store: Arc, table_schema: TableSchema, @@ -1148,6 +1195,7 @@ mod tests { segment_sources: Default::default(), scan_caches: Default::default(), partitions: 1, + initial_chunks: Default::default(), natural_splits: Default::default(), file_schemas: Default::default(), has_output_ordering: false, @@ -1558,6 +1606,7 @@ mod tests { segment_sources: Default::default(), scan_caches: Default::default(), partitions: 1, + initial_chunks: Default::default(), natural_splits: Default::default(), file_schemas: Default::default(), has_output_ordering: false, @@ -1650,6 +1699,7 @@ mod tests { segment_sources: Default::default(), scan_caches: Default::default(), partitions: 1, + initial_chunks: Default::default(), natural_splits: Default::default(), file_schemas: Default::default(), has_output_ordering: false, @@ -1833,6 +1883,7 @@ mod tests { segment_sources: Default::default(), scan_caches: Default::default(), partitions: 1, + initial_chunks: Default::default(), natural_splits: Default::default(), file_schemas: Default::default(), has_output_ordering: false, @@ -1898,6 +1949,7 @@ mod tests { segment_sources: Default::default(), scan_caches: Default::default(), partitions: 1, + initial_chunks: Default::default(), natural_splits: Default::default(), file_schemas: Default::default(), has_output_ordering: false, @@ -2114,6 +2166,7 @@ mod tests { segment_sources: Default::default(), scan_caches: Default::default(), partitions: 1, + initial_chunks: Default::default(), natural_splits: Default::default(), file_schemas: Default::default(), has_output_ordering: false, diff --git a/vortex-datafusion/src/persistent/scan_v2.rs b/vortex-datafusion/src/persistent/scan_v2.rs index 10ea7b969ea..cc3973f45f1 100644 --- a/vortex-datafusion/src/persistent/scan_v2.rs +++ b/vortex-datafusion/src/persistent/scan_v2.rs @@ -12,7 +12,10 @@ //! itself. use std::future; +use std::ops::Range; use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use std::task::Poll; use arrow_array::RecordBatch; @@ -27,16 +30,69 @@ use vortex::error::VortexResult; use vortex::error::vortex_err; use vortex::io::request::IoSource; use vortex::io::runtime::Handle; +use vortex::layout::scan::v2::ChunkPlanner; use vortex::layout::scan::v2::PreparedScan; +use vortex::layout::scan::v2::PrunePlanner; +use vortex::mask::Mask; use vortex::scan::planning::driver::Progress; use vortex::scan::planning::driver::Run; use vortex::scan::planning::planner::Planner; use vortex::scan::planning::planner::WorkScope; use vortex_utils::parallelism::get_available_parallelism; +#[cfg(test)] +mod tests; + /// Maps a scan's batches to Arrow on the task that produced them. type MapBatch = Arc VortexResult + Send + Sync>; +const INITIAL_CHUNK_ROWS: u64 = 8192; + +/// Claims a partition's startup budget across its files, without duplicating their rows. +pub(crate) fn initial_ranges(rows: Range, remaining: &AtomicUsize) -> Vec> { + let needed = + usize::try_from((rows.end - rows.start).div_ceil(INITIAL_CHUNK_ROWS)).unwrap_or(usize::MAX); + let mut previous = remaining.load(Ordering::Relaxed); + let claimed = loop { + let claimed = previous.min(needed); + if claimed == 0 { + break 0; + } + match remaining.compare_exchange_weak( + previous, + previous - claimed, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break claimed, + Err(current) => previous = current, + } + }; + (0..claimed) + .map(|index| { + let start = rows.start + index as u64 * INITIAL_CHUNK_ROWS; + start..start.saturating_add(INITIAL_CHUNK_ROWS).min(rows.end) + }) + .collect() +} + +/// Composes pruning, filtering and projection for an independent file-local range. +pub(crate) fn scan_planner(scan: Arc, rows: Range) -> Box { + let chunks = { + let scan = Arc::clone(&scan); + let rows = rows.clone(); + move |kept: Option| -> Box { + let selection = scan.selection().clone(); + Box::new(ChunkPlanner::new(scan, rows, selection, kept)) + } + }; + if scan.filter().is_some() { + Box::new(PrunePlanner::new(scan, rows, Box::new(chunks))) + } else { + chunks(None) + } +} + /// Runs one partition's V2 scan on DataFusion's runtime. pub(crate) struct ScanDriver { handle: Handle, @@ -61,6 +117,7 @@ impl ScanDriver { self, scan: Arc, root: Box, + initial: Vec>, map: impl Fn(ArrayRef) -> VortexResult + Send + Sync + 'static, ) -> BoxStream<'static, VortexResult> { let map: MapBatch = Arc::new(map); @@ -71,6 +128,28 @@ impl ScanDriver { rows: scan.rows().clone(), }; let io = scan.io(); + let initial_count = initial.len().max(1); + let (initial_handle, initial_io, initial_map, initial_scan) = ( + self.handle.clone(), + Arc::clone(&io), + Arc::clone(&map), + Arc::clone(&scan), + ); + let initial = stream::iter(initial) + .map(move |rows| { + let scope = WorkScope { + file_ordinal: 0, + rows: rows.clone(), + }; + chunk_stream( + &initial_handle, + scan_planner(Arc::clone(&initial_scan), rows), + scope, + initial_io.session(), + Arc::clone(&initial_map), + ) + }) + .flatten_unordered(initial_count); let (chunk_handle, chunk_io, chunk_map) = (self.handle.clone(), Arc::clone(&io), Arc::clone(&map)); let batches = run_stream(self.handle, root, scope, io.session(), map, inline) @@ -87,6 +166,8 @@ impl ScanDriver { ), }) .try_flatten_unordered(in_flight); + // Drain startup batches before admitting bulk work, so TopK can tighten its bound. + let batches = initial.chain(batches); // The prepared scan holds the file's shared cache: keep it until the stream ends, so // scans of the file that start meanwhile find it. let batches = batches.chain(stream::poll_fn(move |_| { diff --git a/vortex-datafusion/src/persistent/scan_v2/tests.rs b/vortex-datafusion/src/persistent/scan_v2/tests.rs new file mode 100644 index 00000000000..8af81ec5743 --- /dev/null +++ b/vortex-datafusion/src/persistent/scan_v2/tests.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#![allow(clippy::single_range_in_vec_init)] + +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::thread; + +use super::initial_ranges; + +#[test] +fn initial_ranges_leave_a_disjoint_bulk_suffix() { + let remaining = AtomicUsize::new(2); + let initial = initial_ranges(100..30_000, &remaining); + assert_eq!(initial, [100..8292, 8292..16_484]); + assert_eq!(remaining.load(Ordering::Relaxed), 0); + assert!(initial_ranges(0..30_000, &remaining).is_empty()); +} + +#[test] +fn short_files_leave_budget_for_the_next_file() { + let remaining = AtomicUsize::new(2); + assert!(initial_ranges(0..0, &remaining).is_empty()); + assert_eq!(initial_ranges(0..4000, &remaining), [0..4000]); + assert_eq!(remaining.load(Ordering::Relaxed), 1); + assert_eq!(initial_ranges(0..10_000, &remaining), [0..8192]); + assert_eq!(remaining.load(Ordering::Relaxed), 0); +} + +#[test] +fn initial_ranges_at_the_end_of_the_row_domain_do_not_overflow() { + assert_eq!( + initial_ranges(u64::MAX - 4..u64::MAX, &AtomicUsize::new(2)), + [u64::MAX - 4..u64::MAX] + ); +} + +#[test] +fn concurrent_files_share_one_budget() { + let remaining = AtomicUsize::new(2); + let claimed = AtomicUsize::new(0); + thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + claimed.fetch_add(initial_ranges(0..4000, &remaining).len(), Ordering::Relaxed); + }); + } + }); + assert_eq!(remaining.load(Ordering::Relaxed), 0); + assert_eq!(claimed.load(Ordering::Relaxed), 2); +} diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs index 644fea4e71c..cdb313aac46 100644 --- a/vortex-datafusion/src/persistent/source.rs +++ b/vortex-datafusion/src/persistent/source.rs @@ -4,6 +4,7 @@ use std::fmt::Formatter; use std::sync::Arc; use std::sync::Weak; +use std::sync::atomic::AtomicUsize; use arrow_schema::SchemaRef; use datafusion_common::Result as DFResult; @@ -15,6 +16,7 @@ use datafusion_datasource::file_groups::FileGroupPartitioner; use datafusion_datasource::file_scan_config::FileScanConfig; use datafusion_datasource::file_stream::FileOpener; use datafusion_execution::cache::cache_manager::FileMetadataCache; +use datafusion_physical_expr::DynamicFilterTracking; use datafusion_physical_expr::EquivalenceProperties; use datafusion_physical_expr::LexOrdering; use datafusion_physical_expr::PhysicalExprRef; @@ -40,6 +42,7 @@ use vortex::metrics::DefaultMetricsRegistry; use vortex::metrics::MetricsRegistry; use vortex::session::VortexSession; use vortex_utils::aliases::dash_map::DashMap; +use vortex_utils::parallelism::get_available_parallelism; use super::opener::NaturalSplits; use super::opener::VortexOpener; @@ -379,6 +382,21 @@ impl VortexSource { scan_concurrency: self.options.scan_concurrency, scan_v2: self.options.scan_v2, partitions: base_config.file_groups.len(), + initial_chunks: Arc::new(AtomicUsize::new( + if self.options.scan_v2 + && self.full_predicate.as_ref().is_some_and(|predicate| { + DynamicFilterTracking::classify(predicate).contains_dynamic_filter() + }) + { + self.options.dynamic_filter_initial_chunks.saturating_mul( + get_available_parallelism() + .unwrap_or(1) + .div_ceil(base_config.file_groups.len().max(1)), + ) + } else { + 0 + }, + )), }; Ok(opener) @@ -592,6 +610,8 @@ impl FileSource for VortexSource { #[cfg(test)] mod tests { + use std::sync::atomic::Ordering; + use arrow_schema::DataType; use arrow_schema::Field; use arrow_schema::Schema; @@ -605,7 +625,9 @@ mod tests { use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_expr::expressions as df_expr; use datafusion_physical_expr::expressions::Column; + use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr; use object_store::memory::InMemory; + use rstest::rstest; use vortex::VortexSessionDefault; use super::*; @@ -754,6 +776,38 @@ mod tests { Ok(()) } + #[rstest] + fn startup_is_reserved_for_v2_dynamic_filters( + #[values(false, true)] scan_v2: bool, + #[values(false, true)] dynamic: bool, + #[values(0, 2)] chunks: usize, + ) -> anyhow::Result<()> { + let schema = sort_test_schema(); + let predicate = + Arc::new(df_expr::Literal::new(ScalarValue::Boolean(Some(true)))) as PhysicalExprRef; + let mut source = sort_test_source(Arc::clone(&schema)); + source.options.scan_v2 = scan_v2; + source.options.dynamic_filter_initial_chunks = chunks; + source.full_predicate = Some(if dynamic { + Arc::new(DynamicFilterPhysicalExpr::new(vec![], predicate)) + } else { + predicate + }); + let config = FileScanConfigBuilder::new( + ObjectStoreUrl::local_filesystem(), + Arc::new(source.clone()), + ) + .build(); + let opener = source.create_vortex_opener(Arc::new(InMemory::new()), &config, 0)?; + let expected = if scan_v2 && dynamic { + chunks * get_available_parallelism().unwrap_or(1) + } else { + 0 + }; + assert_eq!(opener.initial_chunks.load(Ordering::Relaxed), expected); + Ok(()) + } + #[test] fn try_pushdown_filters_respects_disabled_predicate_pushdown() -> anyhow::Result<()> { let schema = Arc::new(Schema::new(vec![Field::new("name", DataType::Utf8, false)])); diff --git a/vortex-datafusion/src/persistent/stream.rs b/vortex-datafusion/src/persistent/stream.rs index 9038ba81db8..b81a76f8bdf 100644 --- a/vortex-datafusion/src/persistent/stream.rs +++ b/vortex-datafusion/src/persistent/stream.rs @@ -2,11 +2,17 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::pin::Pin; +use std::sync::Arc; use std::task::Context; use std::task::Poll; +use std::time::Instant; use datafusion_common::Result as DFResult; use datafusion_common::arrow::array::RecordBatch; +use datafusion_common::tree_node::TreeNode; +use datafusion_common::tree_node::TreeNodeRecursion; +use datafusion_physical_expr::PhysicalExprRef; +use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr; use datafusion_pruning::FilePruner; use futures::Stream; use futures::StreamExt; @@ -18,6 +24,7 @@ use futures::stream::BoxStream; pub(crate) struct PrunableStream { file_pruner: FilePruner, stream: Option>>, + observer: Option, } impl PrunableStream { @@ -25,15 +32,71 @@ impl PrunableStream { Self { file_pruner, stream: Some(stream), + observer: None, } } + + pub(crate) fn observe_filter( + mut self, + predicate: PhysicalExprRef, + path: String, + started: Instant, + ) -> DFResult { + if tracing::enabled!(tracing::Level::DEBUG) { + let mut filters = Vec::new(); + predicate.apply(|expr| { + if expr.is::() { + filters.push((Arc::clone(expr), 0)); + } + Ok(TreeNodeRecursion::Continue) + })?; + self.observer = Some(FilterObserver { + filters, + path, + started, + }); + } + Ok(self) + } +} + +struct FilterObserver { + filters: Vec<(PhysicalExprRef, u64)>, + path: String, + started: Instant, +} + +impl FilterObserver { + fn observe(&mut self) -> bool { + let mut changed = false; + for (filter, previous) in &mut self.filters { + let generation = filter.snapshot_generation(); + if generation != *previous { + changed = true; + tracing::debug!(path = %self.path, elapsed_us = self.started.elapsed().as_micros(), filter_id = ?filter.expression_id(), generation, previous_generation = *previous, predicate = %filter, "dynamic filter observed"); + *previous = generation; + } + } + changed + } } impl Stream for PrunableStream { type Item = DFResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if self.as_mut().file_pruner.should_prune()? { + let changed = self.observer.as_mut().is_some_and(FilterObserver::observe); + let started = self.observer.as_ref().map(|_| Instant::now()); + let pruned = self.as_mut().file_pruner.should_prune()?; + if let Some(started) = started { + tracing::debug!( + elapsed_us = started.elapsed().as_micros(), + changed, + pruned, + "dynamic file pruning checked" + ); + } + if pruned { self.stream.take(); Poll::Ready(None) } else { diff --git a/vortex-duckdb/Cargo.toml b/vortex-duckdb/Cargo.toml index 1a25ebe47d9..26a4c8caf9a 100644 --- a/vortex-duckdb/Cargo.toml +++ b/vortex-duckdb/Cargo.toml @@ -63,6 +63,7 @@ rstest = { workspace = true } tempfile = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } vortex-duckdb = { path = ".", features = ["static-extensions"] } +vortex-onpair = { workspace = true } vortex-sequence = { workspace = true } wkb = { workspace = true } diff --git a/vortex-duckdb/cpp/include/table_function.hpp b/vortex-duckdb/cpp/include/table_function.hpp index c8ed62a9885..b3b1188c3ab 100644 --- a/vortex-duckdb/cpp/include/table_function.hpp +++ b/vortex-duckdb/cpp/include/table_function.hpp @@ -19,6 +19,7 @@ static_assert(sizeof(idx_t) == 8); bool is_vortex_scan(const BoundTableFunction &function); constexpr inline idx_t COUNT_STAR_PROJ_IDX = std::numeric_limits::max(); +constexpr inline column_t VORTEX_ROW_ID_COLUMN = UINT64_C(9223372036854775811); struct TableFunctionUngroupedAggregateInput { const LogicalGet &get; diff --git a/vortex-duckdb/cpp/multi_file_reader.cpp b/vortex-duckdb/cpp/multi_file_reader.cpp index 7b1036597e7..24ccbb29a55 100644 --- a/vortex-duckdb/cpp/multi_file_reader.cpp +++ b/vortex-duckdb/cpp/multi_file_reader.cpp @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #include "multi_file_reader.hpp" +#include "table_function.hpp" #include "error.hpp" #include "table_function.h" #include "vortex_duckdb.h" @@ -89,7 +90,7 @@ VortexMultiFileReader::InitializeReader(MultiFileReaderData &reader_data, duckdb_vx_error error = nullptr; const void *const ffi_global = global.ffi_global_state->DataPtr(); void *const ffi_file = reader.ffi_file->DataPtr(); - const bool skip = duckdb_reader_initialize(ffi_global, ffi_file, &error); + const bool skip = duckdb_reader_initialize(ffi_global, ffi_file, reader.file_list_idx.GetIndex(), &error); if (error) { throw InvalidInputException(IntoErrString(error)); } @@ -286,6 +287,7 @@ void VortexReaderInterface::GetVirtualColumns(ClientContext &, // "filename", "file_index" and "empty" come from MultiFileReader result.insert( {MultiFileReader::COLUMN_IDENTIFIER_FILE_ROW_NUMBER, {"file_row_number", LogicalType::UBIGINT}}); + result.insert({VORTEX_ROW_ID_COLUMN, {"__vortex_row_id", LogicalType::HUGEINT}}); } bool VortexReaderInterface::FinalizeScan(ClientContext &, diff --git a/vortex-duckdb/cpp/table_function.cpp b/vortex-duckdb/cpp/table_function.cpp index ffe408e1d4f..a79885ce91e 100644 --- a/vortex-duckdb/cpp/table_function.cpp +++ b/vortex-duckdb/cpp/table_function.cpp @@ -28,6 +28,7 @@ #include "duckdb/logging/logger.hpp" #include "duckdb/main/capi/capi_internal.hpp" #include "duckdb/main/connection.hpp" +#include "duckdb/optimizer/column_lifetime_analyzer.hpp" #include "duckdb/parser/keyword_helper.hpp" #include "duckdb/parser/parsed_data/create_table_function_info.hpp" #include "duckdb/planner/operator/logical_get.hpp" @@ -349,7 +350,20 @@ duckdb_state register_table_function(DatabaseInstance &db, LogicalType parameter fn.pushdown_expression = [](auto &, const auto &, Expression &expression) { return duckdb_table_function_pushdown_expression(reinterpret_cast(&expression)); }; - fn.pushdown_complex_filter = [](auto &, auto &, FunctionData *bind_data, FilterVec &filters) { + fn.pushdown_complex_filter = [](auto &, LogicalGet &get, FunctionData *bind_data, FilterVec &filters) { + for (const auto &filter : filters) { + vector bindings; + ColumnLifetimeAnalyzer::ExtractColumnBindings(*filter, bindings); + for (const auto &binding : bindings) { + if (binding.table_index == get.table_index && + get.GetColumnIndex(binding).GetPrimaryIndex() == VORTEX_ROW_ID_COLUMN) { + // Rust consumes optional join filters on this key. Keep SQL predicates in + // DuckDB so arbitrary HUGEINT expressions still receive exact evaluation. + get.function.filter_pushdown = false; + return; + } + } + } pushdown_complex_filter(*bind_data, filters); }; fn.projection_expression_pushdown = projection_expression_pushdown; @@ -359,6 +373,10 @@ duckdb_state register_table_function(DatabaseInstance &db, LogicalType parameter // Columns that uniquely identify a row for deferred re-fetch in a multi // file scan: (file index, row number in file). fn.get_row_id_columns = [](auto &, auto) -> vector { + if (duckdb_reader_use_paired_row_ids()) { + // Keep the file hint for pre-open pruning; the packed key retains row pairing. + return {COLUMN_IDENTIFIER_FILE_INDEX, VORTEX_ROW_ID_COLUMN}; + } return {COLUMN_IDENTIFIER_FILE_INDEX, COLUMN_IDENTIFIER_FILE_ROW_NUMBER}; }; diff --git a/vortex-duckdb/include/vortex.h b/vortex-duckdb/include/vortex.h index 03fe4f336ad..67a0166586b 100644 --- a/vortex-duckdb/include/vortex.h +++ b/vortex-duckdb/include/vortex.h @@ -96,7 +96,13 @@ bool duckdb_footer_get_statistics(const void *footer, size_t column_name_len, duckdb_column_statistics *stats_out); -extern bool duckdb_reader_initialize(const void *global, void *file, duckdb_vx_error *error); +extern bool duckdb_reader_use_paired_row_ids(void); + +extern +bool duckdb_reader_initialize(const void *global, + void *file, + uint64_t file_index, + duckdb_vx_error *error); extern duckdb_logical_type duckdb_reader_bind_column_type(const void *bind, size_t index); diff --git a/vortex-duckdb/src/convert/table_filter.rs b/vortex-duckdb/src/convert/table_filter.rs index 2dce92d3d04..158e203d50c 100644 --- a/vortex-duckdb/src/convert/table_filter.rs +++ b/vortex-duckdb/src/convert/table_filter.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::ops::Range; +use std::time::Instant; use itertools::Itertools; use tracing::debug; @@ -35,6 +36,7 @@ use crate::duckdb::ExtractedValue; use crate::duckdb::TableFilterClass; use crate::duckdb::TableFilterRef; use crate::duckdb::ValueRef; +use crate::scan_v2::FilterObserver; pub fn try_from_table_filter( value: &TableFilterRef, @@ -127,14 +129,22 @@ pub(super) fn dynamic_filter_expr( } }; let data = dynamic.data; + let observer = (crate::scan_v2() && tracing::enabled!(tracing::Level::DEBUG)) + .then(|| FilterObserver::new(child.to_string())); Ok(vortex::expr::dynamic( op, move || { - let value = data.latest()?; - let scalar = Scalar::try_from(&*value) - .vortex_expect("failed to convert dynamic filter value to scalar"); - scalar.into_value() + let started = observer.as_ref().map(|_| Instant::now()); + let value = data.latest().and_then(|value| { + Scalar::try_from(&*value) + .vortex_expect("failed to convert dynamic filter value to scalar") + .into_value() + }); + if let (Some(started), Some(observer)) = (started, &observer) { + observer.observe(value.as_ref(), started.elapsed().as_nanos()); + } + value }, dtype, true, // If there is no value, we say that all rows pass the dynamic filter. diff --git a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs index d804d168123..55e5b44bab0 100644 --- a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs +++ b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs @@ -1435,3 +1435,93 @@ fn test_vortex_scan_v2_matches_v1(#[case] query: &str) -> Result<()> { assert_eq!(v2?, v1); Ok(()) } + +#[rstest] +#[case::topk("SELECT id FROM ? ORDER BY score DESC, id LIMIT 10", None)] +#[case::both_row_columns( + "SELECT id + file_row_number FROM ? ORDER BY score DESC, id LIMIT 10", + None +)] +#[case::paired_selection( + "SELECT id FROM ? WHERE __vortex_row_id IN (7, 18446744073709551627, 36893488147419103249, 7, NULL) ORDER BY id", + Some(vec![7, 100011, 200017]) +)] +#[case::empty_selection( + "SELECT id FROM ? WHERE __vortex_row_id = 55340232221128654848", + Some(vec![]) +)] +#[case::row_id_range( + "SELECT id FROM ? WHERE __vortex_row_id BETWEEN 18446744073709551627 AND 18446744073709551629 ORDER BY id", + Some(vec![100011, 100012, 100013]) +)] +#[case::row_id_or( + "SELECT id FROM ? WHERE __vortex_row_id = 7 OR __vortex_row_id = 36893488147419103249 ORDER BY id", + Some(vec![7, 200017]) +)] +fn test_vortex_scan_v2_multifile_row_ids( + #[case] query: &str, + #[case] expected: Option>, +) -> Result<()> { + let files = (0..3i64) + .map(|file| { + let rows = 4096 + file * 2048; + RUNTIME.block_on(write_vortex_file( + [ + ( + "id", + (0..rows) + .map(|row| file * 100000 + row) + .collect::() + .into_array(), + ), + ( + "score", + (0..rows) + .map(|row| (row * 37 + file * 11) % 10000) + .collect::() + .into_array(), + ), + ] + .into_iter(), + )) + }) + .collect::>(); + let source = format!( + "read_vortex([{}])", + files + .iter() + .map(|file| format!("'{}'", file.path().display())) + .collect::>() + .join(",") + ); + let query = query.replace('?', &source); + let read = |late_materialization: bool| -> Result> { + let conn = database_connection(); + conn.query(if late_materialization { + "SET late_materialization_max_rows = 100000" + } else { + "SET late_materialization_max_rows = 0" + })?; + let mut values = Vec::new(); + for chunk in conn.query(&query)? { + values.extend_from_slice( + chunk + .get_vector(0) + .as_slice_with_len::(chunk.len().as_()), + ); + } + Ok(values) + }; + crate::set_scan_v2(false); + let v1 = read(false)?; + crate::set_scan_v2(true); + let v2 = read(true); + crate::set_scan_v2(false); + assert_eq!(v2?, v1); + if let Some(expected) = expected { + assert_eq!(v1, expected); + } else { + assert_eq!(v1.len(), 10); + } + Ok(()) +} diff --git a/vortex-duckdb/src/exporter/cache.rs b/vortex-duckdb/src/exporter/cache.rs index 56f9afc1c78..cf830249045 100644 --- a/vortex-duckdb/src/exporter/cache.rs +++ b/vortex-duckdb/src/exporter/cache.rs @@ -17,6 +17,7 @@ use crate::duckdb::Vector; /// to avoid redundant conversions of the same array instances. /// We hold on to the `ArrayRef` to ensure that the key (ptr addr) doesn't get reused. pub struct ConversionCache { + pub(super) sparse_dictionaries: bool, pub(crate) dict_cache: DashMap, pub(crate) values_cache: DashMap>)>, pub(crate) canonical_cache: DashMap, @@ -25,9 +26,19 @@ pub struct ConversionCache { impl Default for ConversionCache { fn default() -> Self { Self { + sparse_dictionaries: false, dict_cache: DashMap::with_hasher_and_shard_amount(Default::default(), 64), values_cache: DashMap::with_hasher_and_shard_amount(Default::default(), 64), canonical_cache: DashMap::with_hasher_and_shard_amount(Default::default(), 64), } } } + +impl ConversionCache { + pub(crate) fn with_sparse_dictionaries() -> Self { + Self { + sparse_dictionaries: true, + ..Self::default() + } + } +} diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs index d744172d670..a7d30aaa5fa 100644 --- a/vortex-duckdb/src/exporter/dict.rs +++ b/vortex-duckdb/src/exporter/dict.rs @@ -69,6 +69,26 @@ pub(crate) fn new_exporter_with_flatten( let values_key = values.addr(); let codes = array.codes().clone().execute::(ctx)?; + // A dictionary vector needs every value converted, even when only a handful of codes + // are selected. Let encoded take kernels gather these rows before canonicalization. + if cache.sparse_dictionaries + && codes_len > 0 + && values.len() >= 1024 + && codes_len.saturating_mul(16) < values.len() + && !cache.dict_cache.contains_key(&values_key) + && !cache.canonical_cache.contains_key(&values_key) + { + let selected = DictArray::new(codes.into_array(), values.clone()) + .into_array() + .execute::(ctx)?; + tracing::debug!( + dictionary_values = values.len(), + selected_rows = codes_len, + "v2 sparse dictionary export" + ); + return new_array_exporter(selected.into_array(), cache, ctx); + } + if flatten { let canonical = cache .canonical_cache @@ -131,141 +151,4 @@ impl> ColumnExporter for DictExporter { } #[cfg(test)] -mod tests { - use vortex::VortexSessionDefault; - use vortex::array::IntoArray; - use vortex::array::VortexSessionExecute; - use vortex::array::arrays::ConstantArray; - use vortex::array::arrays::DictArray; - use vortex::array::arrays::PrimitiveArray; - use vortex::buffer::Buffer; - use vortex::error::VortexResult; - use vortex::session::VortexSession; - - use crate::SESSION; - use crate::cpp; - use crate::duckdb::DataChunk; - use crate::duckdb::LogicalType; - use crate::exporter::ColumnExporter; - use crate::exporter::ConversionCache; - use crate::exporter::dict::new_exporter_with_flatten; - use crate::exporter::new_array_exporter; - - pub(crate) fn new_exporter( - array: &DictArray, - cache: &ConversionCache, - ) -> VortexResult> { - new_exporter_with_flatten(array, cache, &mut SESSION.create_execution_ctx(), false) - } - - #[test] - fn test_constant_dict() -> VortexResult<()> { - let arr = DictArray::new( - PrimitiveArray::from_option_iter([None, Some(0u32)]).into_array(), - ConstantArray::new(10, 1).into_array(), - ); - - let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); - - new_exporter(&arr, &ConversionCache::default())?.export( - 0, - 2, - chunk.get_vector_mut(0), - &mut SESSION.create_execution_ctx(), - )?; - chunk.set_len(2); - - assert_eq!( - String::try_from(&*chunk)?, - r#"Chunk - [1 Columns] -- FLAT INTEGER: 2 = [ NULL, 10] -"# - ); - - Ok(()) - } - - #[test] - fn test_constant_dict_null() -> VortexResult<()> { - let arr = DictArray::new( - PrimitiveArray::from_option_iter([None::, None]).into_array(), - ConstantArray::new(10, 1).into_array(), - ); - - let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); - - let mut ctx = VortexSession::default().create_execution_ctx(); - new_exporter_with_flatten(&arr, &ConversionCache::default(), &mut ctx, false)?.export( - 0, - 2, - chunk.get_vector_mut(0), - &mut ctx, - )?; - chunk.set_len(2); - - assert_eq!( - String::try_from(&*chunk)?, - r#"Chunk - [1 Columns] -- CONSTANT INTEGER: 2 = [ NULL] -"# - ); - - Ok(()) - } - - #[test] - fn test_nullable_dict() -> VortexResult<()> { - let arr = DictArray::new( - PrimitiveArray::from_option_iter([None, Some(0u32), Some(1)]).into_array(), - PrimitiveArray::from_option_iter([Some(10), None]).into_array(), - ); - - let mut flat_chunk = - DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); - let mut ctx = SESSION.create_execution_ctx(); - - new_array_exporter(arr.into_array(), &ConversionCache::default(), &mut ctx)?.export( - 0, - 3, - flat_chunk.get_vector_mut(0), - &mut ctx, - )?; - flat_chunk.set_len(3); - - assert_eq!( - String::try_from(&*flat_chunk)?, - r#"Chunk - [1 Columns] -- FLAT INTEGER: 3 = [ NULL, 10, NULL] -"# - ); - - Ok(()) - } - - #[test] - fn test_export_empty_dict() -> VortexResult<()> { - let arr = DictArray::new( - Buffer::::empty().into_array(), - Buffer::::empty().into_array(), - ); - - let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); - - new_exporter(&arr, &ConversionCache::default())?.export( - 0, - 0, - chunk.get_vector_mut(0), - &mut SESSION.create_execution_ctx(), - )?; - chunk.set_len(0); - - assert_eq!( - String::try_from(&*chunk)?, - r#"Chunk - [1 Columns] -- DICTIONARY INTEGER: 0 = [ ] -"# - ); - - Ok(()) - } -} +mod tests; diff --git a/vortex-duckdb/src/exporter/dict/tests.rs b/vortex-duckdb/src/exporter/dict/tests.rs new file mode 100644 index 00000000000..a1a89c69060 --- /dev/null +++ b/vortex-duckdb/src/exporter/dict/tests.rs @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex::VortexSessionDefault; +use vortex::array::IntoArray; +use vortex::array::VortexSessionExecute; +use vortex::array::arrays::ConstantArray; +use vortex::array::arrays::DictArray; +use vortex::array::arrays::PrimitiveArray; +use vortex::array::arrays::VarBinArray; +use vortex::buffer::Buffer; +use vortex::dtype::DType; +use vortex::error::VortexResult; +use vortex::session::VortexSession; +use vortex_onpair::DEFAULT_CONFIG; +use vortex_onpair::onpair_compress; + +use crate::SESSION; +use crate::cpp; +use crate::duckdb::DataChunk; +use crate::duckdb::LogicalType; +use crate::exporter::ColumnExporter; +use crate::exporter::ConversionCache; +use crate::exporter::dict::new_exporter_with_flatten; +use crate::exporter::new_array_exporter; + +#[rstest::rstest] +fn sparse_strings_preserve_nulls_and_duplicates( + #[values(false, true)] null_code: bool, + #[values(false, true)] null_value: bool, +) -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let values = VarBinArray::from_iter( + (0..2048).map(|i| (!(null_value && i == 17)).then(|| format!("value-{i}"))), + DType::Utf8(null_value.into()), + ) + .into_array(); + let values = onpair_compress(&values, DEFAULT_CONFIG, &mut ctx)?; + let array = DictArray::new( + PrimitiveArray::from_option_iter([ + Some(2047u32), + (!null_code).then_some(4), + Some(17), + Some(2047), + ]) + .into_array(), + values, + ); + let cache = ConversionCache::with_sparse_dictionaries(); + let mut chunk = DataChunk::new([LogicalType::varchar()]); + new_exporter(&array, &cache)?.export(0, 4, chunk.get_vector_mut(0), &mut ctx)?; + chunk.set_len(4); + let second = if null_code { "NULL" } else { "value-4" }; + let third = if null_value { "NULL" } else { "value-17" }; + assert_eq!( + String::try_from(&*chunk)?, + format!( + "Chunk - [1 Columns]\n- FLAT VARCHAR: 4 = [ value-2047, {second}, {third}, value-2047]\n" + ) + ); + assert!(cache.dict_cache.is_empty()); + assert!(cache.canonical_cache.is_empty()); + Ok(()) +} + +#[test] +fn sparse_strings_reuse_an_already_converted_dictionary() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let values = VarBinArray::from_iter( + (0..2048).map(|i| Some(format!("value-{i}"))), + DType::Utf8(false.into()), + ) + .into_array(); + let values = onpair_compress(&values, DEFAULT_CONFIG, &mut ctx)?; + let cache = ConversionCache::with_sparse_dictionaries(); + let dense = DictArray::new(Buffer::from_iter(0..2048u32).into_array(), values.clone()); + let _dense = new_exporter(&dense, &cache)?; + assert_eq!(cache.dict_cache.len(), 1); + let sparse = DictArray::new(Buffer::from_iter([2047u32, 17]).into_array(), values); + let mut chunk = DataChunk::new([LogicalType::varchar()]); + new_exporter(&sparse, &cache)?.export(0, 2, chunk.get_vector_mut(0), &mut ctx)?; + chunk.set_len(2); + assert_eq!( + String::try_from(&*chunk)?, + "Chunk - [1 Columns]\n- DICTIONARY VARCHAR: 2 = [ value-2047, value-17]\n" + ); + assert_eq!(cache.dict_cache.len(), 1); + Ok(()) +} + +pub(crate) fn new_exporter( + array: &DictArray, + cache: &ConversionCache, +) -> VortexResult> { + new_exporter_with_flatten(array, cache, &mut SESSION.create_execution_ctx(), false) +} + +#[test] +fn test_constant_dict() -> VortexResult<()> { + let arr = DictArray::new( + PrimitiveArray::from_option_iter([None, Some(0u32)]).into_array(), + ConstantArray::new(10, 1).into_array(), + ); + + let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); + + new_exporter(&arr, &ConversionCache::default())?.export( + 0, + 2, + chunk.get_vector_mut(0), + &mut SESSION.create_execution_ctx(), + )?; + chunk.set_len(2); + + assert_eq!( + String::try_from(&*chunk)?, + r#"Chunk - [1 Columns] +- FLAT INTEGER: 2 = [ NULL, 10] +"# + ); + + Ok(()) +} + +#[test] +fn test_constant_dict_null() -> VortexResult<()> { + let arr = DictArray::new( + PrimitiveArray::from_option_iter([None::, None]).into_array(), + ConstantArray::new(10, 1).into_array(), + ); + + let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); + + let mut ctx = VortexSession::default().create_execution_ctx(); + new_exporter_with_flatten(&arr, &ConversionCache::default(), &mut ctx, false)?.export( + 0, + 2, + chunk.get_vector_mut(0), + &mut ctx, + )?; + chunk.set_len(2); + + assert_eq!( + String::try_from(&*chunk)?, + r#"Chunk - [1 Columns] +- CONSTANT INTEGER: 2 = [ NULL] +"# + ); + + Ok(()) +} + +#[test] +fn test_nullable_dict() -> VortexResult<()> { + let arr = DictArray::new( + PrimitiveArray::from_option_iter([None, Some(0u32), Some(1)]).into_array(), + PrimitiveArray::from_option_iter([Some(10), None]).into_array(), + ); + + let mut flat_chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); + let mut ctx = SESSION.create_execution_ctx(); + + new_array_exporter(arr.into_array(), &ConversionCache::default(), &mut ctx)?.export( + 0, + 3, + flat_chunk.get_vector_mut(0), + &mut ctx, + )?; + flat_chunk.set_len(3); + + assert_eq!( + String::try_from(&*flat_chunk)?, + r#"Chunk - [1 Columns] +- FLAT INTEGER: 3 = [ NULL, 10, NULL] +"# + ); + + Ok(()) +} + +#[test] +fn test_export_empty_dict() -> VortexResult<()> { + let arr = DictArray::new( + Buffer::::empty().into_array(), + Buffer::::empty().into_array(), + ); + + let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); + + new_exporter(&arr, &ConversionCache::default())?.export( + 0, + 0, + chunk.get_vector_mut(0), + &mut SESSION.create_execution_ctx(), + )?; + chunk.set_len(0); + + assert_eq!( + String::try_from(&*chunk)?, + r#"Chunk - [1 Columns] +- DICTIONARY INTEGER: 0 = [ ] +"# + ); + + Ok(()) +} diff --git a/vortex-duckdb/src/ffi.rs b/vortex-duckdb/src/ffi.rs index 8f25fd944db..b280b60a752 100644 --- a/vortex-duckdb/src/ffi.rs +++ b/vortex-duckdb/src/ffi.rs @@ -324,15 +324,22 @@ pub unsafe extern "C-unwind" fn duckdb_footer_get_statistics( true } +/// Whether new late-materialization plans should use V2's paired file and row key. +#[unsafe(no_mangle)] +pub extern "C" fn duckdb_reader_use_paired_row_ids() -> bool { + crate::scan_v2() +} + #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn duckdb_reader_initialize( global: *const c_void, file: *mut c_void, + file_index: u64, error: *mut cpp::duckdb_vx_error, ) -> bool { let global = unsafe { global.cast::().as_ref() }.vortex_expect("null pointer"); let file = unsafe { file.cast::().as_mut() }.vortex_expect("null pointer"); - try_or(error, || reader_initialize(file, global)) + try_or(error, || reader_initialize(file, global, file_index)) } #[unsafe(no_mangle)] diff --git a/vortex-duckdb/src/file_reader.rs b/vortex-duckdb/src/file_reader.rs index 70d8d30c6cb..b4b6398aea2 100644 --- a/vortex-duckdb/src/file_reader.rs +++ b/vortex-duckdb/src/file_reader.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use std::sync::OnceLock; use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use futures::FutureExt; @@ -11,10 +12,7 @@ use object_store::registry::ObjectStoreRegistry; use url::Url; use vortex::array::ArrayRef; use vortex::array::VortexSessionExecute as _; -use vortex::array::arrays::Chunked; -use vortex::array::arrays::chunked::ChunkedArrayExt as _; use vortex::array::arrays::struct_::StructArrayExt as _; -use vortex::array::scalar_fn::fns::dynamic::DynamicExprUpdates; use vortex::dtype::FieldPath; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -35,6 +33,7 @@ use vortex::layout::scan::scan_builder::ScanBuilder; use vortex::layout::scan::v2::ScanBuilder as V2ScanBuilder; use vortex::layout::scan::v2::ScanFile; use vortex::mask::Mask; +use vortex::scan::selection::Selection; use vortex::session::SessionExt as _; use crate::REGISTRY; @@ -126,6 +125,8 @@ pub struct OpenFileReader { /// The state of an open file that the threads scanning it share. pub struct FileScanState { + path: String, + file_index: AtomicU64, reader: LayoutReaderRef, /// The file as the V2 executor reads it. scan_file: ScanFile, @@ -144,10 +145,16 @@ impl OpenFileReader { scan_file.cache.get_or_insert_default(); Ok(OpenFileReader { scan_state: Some(Arc::new(FileScanState { + path, + file_index: AtomicU64::new(0), reader: file.layout_reader()?, scan_file, v2: OnceLock::new(), - cache: ConversionCache::default(), + cache: if crate::scan_v2() { + ConversionCache::with_sparse_dictionaries() + } else { + ConversionCache::default() + }, })), splits: vec![], total_splits: 0, @@ -211,8 +218,38 @@ pub fn reader_bind(file: &OpenFileReader, result: &mut BindResultRef) -> VortexR /// Called once per file by one thread under file-local lock. Determines /// whether the opened file should be skipped. If this function returns false, /// duckdb closes the file and doesn't call reader_try_initialize_scan on it. -pub fn reader_initialize(file: &mut OpenFileReader, global: &GlobalState) -> VortexResult { - if file.can_skip(&global.filter)? { +pub fn reader_initialize( + file: &mut OpenFileReader, + global: &GlobalState, + file_index: u64, +) -> VortexResult { + let state = file.scan_state(); + state.file_index.store(file_index, Ordering::Relaxed); + let selection = match &global.filter.row_ids { + Some(ids) => ids.selection( + file_index, + state.reader.row_count(), + &global.filter.row_selection, + )?, + None => global.filter.row_selection.clone(), + }; + let started = global + .scan_diagnostics + .as_ref() + .map(|_| std::time::Instant::now()); + let skipped = selection.row_count_in_range(&(0..state.reader.row_count())) == 0 + || file.can_skip(&global.filter)?; + if let Some(started) = started { + tracing::debug!( + path = file.scan_state().path, + elapsed_ns = started.elapsed().as_nanos(), + skipped, + file_index, + paired = global.filter.row_ids.is_some(), + "v2 file pruning checked" + ); + } + if skipped { // DuckDB releases a skipped file under the global lock. Here only the // file-local lock is held, so drop the scan state now. file.scan_state = None; @@ -224,16 +261,12 @@ pub fn reader_initialize(file: &mut OpenFileReader, global: &GlobalState) -> Vor // global lock. if global.scan_v2 { let state = file.scan_state(); - let splits = ScanBuilder::new(SESSION.clone(), Arc::clone(&state.reader)) - .with_projection(global.projection.clone()) - .with_some_filter(global.filter.filter.clone()) - .full_file_splits()?; - let dynamic = global - .filter - .filter - .as_ref() - .is_some_and(|filter| DynamicExprUpdates::new(filter).is_some()); - let scan = FileScan::try_new(v2_builder(state, global), &splits, dynamic)?; + let scan = FileScan::try_new( + v2_builder(state, global, selection), + &global.initial_chunks, + global.scan_diagnostics.clone(), + &state.path, + )?; if state.v2.set(scan).is_err() { vortex_panic!("file initialized twice"); } @@ -245,7 +278,7 @@ pub fn reader_initialize(file: &mut OpenFileReader, global: &GlobalState) -> Vor let builder = ScanBuilder::new(SESSION.clone(), reader) .with_projection(global.projection.clone()) .with_some_filter(filter.filter.clone()) - .with_selection(filter.row_selection.clone()); + .with_selection(selection); let scan = builder.prepare()?; let mut splits: Vec = scan .execute(filter.row_range.clone())? @@ -261,7 +294,7 @@ pub fn reader_initialize(file: &mut OpenFileReader, global: &GlobalState) -> Vor } /// The V2 scan of the file's rows that the scan's row range and selection keep. -fn v2_builder(state: &FileScanState, global: &GlobalState) -> V2ScanBuilder { +fn v2_builder(state: &FileScanState, global: &GlobalState, selection: Selection) -> V2ScanBuilder { let filter = &global.filter; let row_count = state.reader.row_count(); let rows = filter.row_range.as_ref().map_or(0..row_count, |range| { @@ -270,7 +303,7 @@ fn v2_builder(state: &FileScanState, global: &GlobalState) -> V2ScanBuilder { V2ScanBuilder::new(SESSION.clone(), state.scan_file.clone()) .with_projection(global.projection.clone()) .with_some_filter(filter.filter.clone()) - .with_selection(filter.row_selection.clone()) + .with_selection(selection) .with_row_range(rows) } @@ -296,23 +329,8 @@ fn next_split_item(local: &mut LocalState) -> VortexResult { .v2 .get() .vortex_expect("V2 split of a file without a V2 scan"); - let array = match local.pending.pop() { - Some(array) => array, - None => { - let Some(array) = scan.next_array(&mut local.run)? else { - return Ok(SplitItem::Done); - }; - // A V2 batch can join chunks of several layout chunks. Canonicalizing it - // would decode their dictionaries, so export each chunk on its own, as a V1 - // scan yields them. - match array.as_opt::() { - Some(chunked) => { - local.pending.extend(chunked.chunks().into_iter().rev()); - local.pending.pop().unwrap_or(array) - } - None => array, - } - } + let Some(array) = scan.next_array(&mut local.run, &mut local.initial_roots)? else { + return Ok(SplitItem::Done); }; local.split = Some(Split::V2); Ok(if array.is_empty() { @@ -387,13 +405,24 @@ pub fn reader_scan( SplitItem::Array(array) => array, }; let mut ctx = SESSION.create_execution_ctx(); - let array = convert_result(array, &mut ctx)?; let file = local.file.as_ref().vortex_expect("split without a file"); + let mut array = convert_result(array, &mut ctx)?; + if let Some(field) = global.row_id_field { + array = crate::row_id::replace_row_id( + array, + field, + file.file_index.load(Ordering::Relaxed), + &mut ctx, + )?; + } local.exporter = Some(ArrayExporter::try_new(&array, &file.cache, ctx)?); } let exporter = local.exporter.as_mut().vortex_expect("no exporter"); let has_more_data = exporter.export(chunk, global.file_row_number_column_pos)?; + if let Some(diagnostics) = &global.scan_diagnostics { + diagnostics.exported(chunk.len()); + } if !has_more_data { local.exporter = None; } diff --git a/vortex-duckdb/src/lib.rs b/vortex-duckdb/src/lib.rs index 57712698666..c5364f7905d 100644 --- a/vortex-duckdb/src/lib.rs +++ b/vortex-duckdb/src/lib.rs @@ -8,6 +8,7 @@ use std::ffi::c_void; use std::sync::LazyLock; use std::sync::OnceLock; use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use vortex::VortexSessionDefault; @@ -31,6 +32,7 @@ mod exporter; mod ffi; mod file_reader; mod projection; +mod row_id; mod scan_v2; mod table_function; @@ -79,6 +81,14 @@ pub fn scan_v2() -> bool { SCAN_V2.load(Ordering::Relaxed) } +static DYNAMIC_FILTER_INITIAL_CHUNKS: AtomicUsize = AtomicUsize::new(0); + +/// Sets the number of initial 8K-row chunks per worker for V2 scans with a dynamic filter. +/// Zero (the default) disables the startup path. The budget is shared across all files of a scan. +pub fn set_dynamic_filter_initial_chunks(chunks: usize) { + DYNAMIC_FILTER_INITIAL_CHUNKS.store(chunks, Ordering::Relaxed); +} + // Duckdb's logger requires a *Context as first argument which // would be hard to integrate with tracing::. We use logging for // debugging only anyway, so that's good enough. diff --git a/vortex-duckdb/src/projection.rs b/vortex-duckdb/src/projection.rs index 68a7c8cd454..b35e4b6bce5 100644 --- a/vortex-duckdb/src/projection.rs +++ b/vortex-duckdb/src/projection.rs @@ -30,6 +30,9 @@ use crate::duckdb::LogicalType; use crate::duckdb::TableFilterClass; use crate::duckdb::TableFilterRef; use crate::duckdb::TableFilterSetRef; +use crate::row_id::ROW_ID_COLUMN_IDX; +use crate::row_id::ROW_ID_NAME; +use crate::row_id::RowIds; use crate::table_function::ColumnAggregate; // See MultiFileReader for constants @@ -55,6 +58,8 @@ pub struct DuckdbField { pub struct Projection { pub projection: Expression, pub file_row_number_column_pos: Option, + /// Field of the Vortex struct to replace with the packed row key before export. + pub row_id_field: Option, } #[cfg_attr(test, derive(Clone))] @@ -82,11 +87,18 @@ impl Projection { let mut exprs = Vec::with_capacity(input.column_ids.len() + 1); let mut file_row_number_column_pos = None; + let mut row_id_field = None; let mut is_star = true; let mut real_column_count = 0; // DuckDB uses u64 as column indices but Rust uses usize for (column_pos, &column_id) in input.column_ids.iter().enumerate() { + if column_id == ROW_ID_COLUMN_IDX { + is_star = false; + row_id_field = Some(exprs.len()); + exprs.push((ROW_ID_NAME, row_idx())); + continue; + } if column_id == FILE_ROW_NUMBER_COLUMN_IDX { is_star = false; if is_projected(column_pos) { @@ -141,15 +153,18 @@ impl Projection { return Projection { projection: root(), file_row_number_column_pos: None, + row_id_field: None, }; } if file_row_number_column_pos.is_some() { // row_idx will be moved to correct position in scan(), prepend here exprs.insert(0, ("file_row_number", row_idx())); + row_id_field = row_id_field.map(|index| index + 1); } Self { projection: pack(exprs, false.into()), file_row_number_column_pos, + row_id_field, } } @@ -185,6 +200,7 @@ impl Projection { Projection { projection, file_row_number_column_pos: None, + row_id_field: None, } } } @@ -194,6 +210,7 @@ pub struct Filter { pub row_selection: Selection, pub row_range: Option>, pub has_non_optional_filter: bool, + pub(crate) row_ids: Option, } fn push_filter_expr(filter_exprs: &mut Vec, expr: &Expression) { @@ -248,12 +265,17 @@ impl Filter { let mut row_selection = Selection::All; let mut row_range = None; + let mut row_ids = None; if let Some(filter) = table_filter_set { for (idx, expression) in filter.into_iter() { let idx: usize = idx.as_(); if column_ids[idx] == FILE_ROW_NUMBER_COLUMN_IDX { (row_selection, row_range) = try_from_virtual_column_filter(expression)?; } + if column_ids[idx] == ROW_ID_COLUMN_IDX { + row_ids = RowIds::from_filter(expression); + tracing::debug!(selection = ?row_ids, "v2 paired row filter"); + } } }; @@ -266,6 +288,7 @@ impl Filter { row_selection, row_range, has_non_optional_filter, + row_ids, }; Ok(out) } diff --git a/vortex-duckdb/src/row_id.rs b/vortex-duckdb/src/row_id.rs new file mode 100644 index 00000000000..88cd8968b5d --- /dev/null +++ b/vortex-duckdb/src/row_id.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! One late-materialization key preserves the association between a file and its rows. + +use std::collections::BTreeSet; + +use vortex::array::ExecutionCtx; +use vortex::array::IntoArray; +use vortex::array::arrays::DecimalArray; +use vortex::array::arrays::PrimitiveArray; +use vortex::array::arrays::StructArray; +use vortex::array::arrays::struct_::StructArrayExt; +use vortex::array::validity::Validity; +use vortex::buffer::Buffer; +use vortex::dtype::DecimalDType; +use vortex::error::VortexResult; +use vortex::error::vortex_err; +use vortex::scan::selection::Selection; +use vortex::scan::strict_sorted_buffer::StrictSortedBuffer; + +use crate::cpp::DUCKDB_VX_EXPR_TYPE; +use crate::duckdb::ExpressionClass; +use crate::duckdb::ExpressionRef; +use crate::duckdb::ExtractedValue; +use crate::duckdb::TableFilterClass; +use crate::duckdb::TableFilterRef; +use crate::duckdb::ValueRef; + +// Kept in sync with VORTEX_ROW_ID_COLUMN in cpp/include/table_function.hpp. +pub(crate) const ROW_ID_COLUMN_IDX: u64 = 9223372036854775811; +pub(crate) const ROW_ID_NAME: &str = "__vortex_row_id"; + +fn encode(file: u64, row: u64) -> VortexResult { + i128::from(file) + .checked_mul(1i128 << 64) + .and_then(|prefix| prefix.checked_add(i128::from(row))) + .ok_or_else(|| vortex_err!("file index exceeds the packed row identifier range")) +} + +#[derive(Debug)] +pub(crate) struct RowIds(BTreeSet); + +impl RowIds { + pub(crate) fn from_filter(filter: &TableFilterRef) -> Option { + match filter.as_class() { + TableFilterClass::Optional(child) => Self::from_filter(child), + TableFilterClass::InFilter(values) => Self::from_values(values.iter()), + TableFilterClass::ConstantComparison(comparison) + if comparison.operator + == DUCKDB_VX_EXPR_TYPE::DUCKDB_VX_EXPR_TYPE_COMPARE_EQUAL => + { + Self::from_values([comparison.value].into_iter()) + } + TableFilterClass::ExpressionRef(expression) => Self::from_expression(expression), + TableFilterClass::ConjunctionAnd(conjunction) => conjunction + .children() + .filter_map(Self::from_filter) + .reduce(|left, right| Self(left.0.intersection(&right.0).copied().collect())), + TableFilterClass::ConjunctionOr(conjunction) => { + let children = conjunction + .children() + .map(Self::from_filter) + .collect::>>()?; + Some(Self( + children.into_iter().flat_map(|child| child.0).collect(), + )) + } + _ => None, + } + } + + fn from_expression(expression: &ExpressionRef) -> Option { + match expression.as_class()? { + ExpressionClass::BoundFunction(function) => Self::from_expression(function.optional()?), + ExpressionClass::BoundOperator(operator) + if operator.op == DUCKDB_VX_EXPR_TYPE::DUCKDB_VX_EXPR_TYPE_COMPARE_IN => + { + let mut children = operator.children(); + if !matches!( + children.next()?.as_class()?, + ExpressionClass::BoundRef | ExpressionClass::BoundColumnRef(_) + ) { + return None; + } + let values = children + .map(|child| match child.as_class()? { + ExpressionClass::BoundConstant(constant) => Some(constant.value), + _ => None, + }) + .collect::>>()?; + Self::from_values(values.into_iter()) + } + ExpressionClass::BoundComparison(comparison) + if comparison.op == DUCKDB_VX_EXPR_TYPE::DUCKDB_VX_EXPR_TYPE_COMPARE_EQUAL => + { + let ExpressionClass::BoundConstant(constant) = comparison.right.as_class()? else { + return None; + }; + if !matches!( + comparison.left.as_class()?, + ExpressionClass::BoundRef | ExpressionClass::BoundColumnRef(_) + ) { + return None; + } + Self::from_values([constant.value].into_iter()) + } + _ => None, + } + } + + fn from_values<'a>(values: impl Iterator) -> Option { + let mut ids = BTreeSet::new(); + for value in values { + let value = match value.extract() { + ExtractedValue::HugeInt(value) | ExtractedValue::Decimal(_, 0, value) => value, + ExtractedValue::UBigInt(value) => i128::from(value), + ExtractedValue::BigInt(value) => i128::from(value), + ExtractedValue::Null => continue, + _ => return None, + }; + if value >= 0 { + ids.insert(value); + } + } + Some(Self(ids)) + } + + pub(crate) fn selection( + &self, + file: u64, + row_count: u64, + selection: &Selection, + ) -> VortexResult { + let first = encode(file, 0)?; + let last = encode(file, u64::MAX)?; + let mut rows = Vec::new(); + for &id in self.0.range(first..=last) { + let row = u64::try_from(id - first)?; + let keep = match selection { + Selection::All => true, + Selection::IncludeByIndex(indices) => indices.binary_search(&row).is_ok(), + Selection::ExcludeByIndex(indices) => indices.binary_search(&row).is_err(), + Selection::IncludeRoaring(indices) => indices.contains(row), + Selection::ExcludeRoaring(indices) => !indices.contains(row), + }; + if row < row_count && keep { + rows.push(row); + } + } + Ok(Selection::IncludeByIndex(StrictSortedBuffer::try_new( + Buffer::from(rows), + )?)) + } +} + +pub(crate) fn replace_row_id( + array: StructArray, + field: usize, + file: u64, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let rows = array + .unmasked_field(field) + .clone() + .execute::(ctx)?; + let prefix = encode(file, 0)?; + let values = Buffer::from_iter( + rows.as_slice::() + .iter() + .map(|&row| prefix | i128::from(row)), + ); + // Decimal's I128 exporter shares HUGEINT's physical representation. This column is only + // an in-memory carrier; DuckDB's declared virtual-column type is HUGEINT. + let packed = + DecimalArray::new(values, DecimalDType::new(38, 0), Validity::NonNullable).into_array(); + StructArray::try_new( + array.names().clone(), + array + .iter_unmasked_fields() + .enumerate() + .map(|(index, value)| { + if index == field { + packed.clone() + } else { + value.clone() + } + }), + array.len(), + array.struct_validity(), + ) +} + +#[cfg(test)] +mod tests; diff --git a/vortex-duckdb/src/row_id/tests.rs b/vortex-duckdb/src/row_id/tests.rs new file mode 100644 index 00000000000..d46a205257d --- /dev/null +++ b/vortex-duckdb/src/row_id/tests.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::collections::BTreeSet; + +use vortex::array::IntoArray; +use vortex::array::VortexSessionExecute; +use vortex::array::arrays::StructArray; +use vortex::buffer::Buffer; +use vortex::error::VortexResult; +use vortex::error::vortex_bail; +use vortex::scan::selection::Selection; +use vortex::scan::strict_sorted_buffer::StrictSortedBuffer; + +use super::RowIds; +use super::encode; +use super::replace_row_id; +use crate::SESSION; +use crate::cpp; +use crate::duckdb::DataChunk; +use crate::duckdb::LogicalType; +use crate::exporter::ArrayExporter; +use crate::exporter::ConversionCache; + +#[test] +fn row_ids_select_only_the_rows_belonging_to_each_file() -> VortexResult<()> { + let ids = RowIds(BTreeSet::from([ + encode(2, 7)?, + encode(9, 12)?, + encode(2, 99)?, + ])); + for (file, expected) in [(2, vec![7, 99]), (9, vec![12]), (3, vec![])] { + let Selection::IncludeByIndex(rows) = ids.selection(file, 100, &Selection::All)? else { + vortex_bail!("expected an index selection"); + }; + assert_eq!(rows.as_ref(), expected.as_slice()); + } + let selection = + Selection::IncludeByIndex(StrictSortedBuffer::try_new(Buffer::from_iter([99u64]))?); + let Selection::IncludeByIndex(rows) = ids.selection(2, 50, &selection)? else { + vortex_bail!("expected an index selection"); + }; + assert!(rows.is_empty()); + Ok(()) +} + +#[test] +fn packed_ids_preserve_the_entire_row_number() -> VortexResult<()> { + assert_eq!(encode(0, u64::MAX)?, i128::from(u64::MAX)); + assert!(encode(1, 0)? > encode(0, u64::MAX)?); + assert_ne!(encode(1, 7)?, encode(7, 1)?); + assert!(encode(u64::MAX, 0).is_err()); + Ok(()) +} + +#[test] +fn packed_ids_export_as_duckdb_hugeints() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = StructArray::from_fields(&[("id", Buffer::from_iter([7u64, 12]).into_array())])?; + let array = replace_row_id(array, 0, 2, &mut ctx)?; + let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_HUGEINT)]); + let mut exporter = ArrayExporter::try_new(&array, &ConversionCache::default(), ctx)?; + exporter.export(&mut chunk, None)?; + assert_eq!( + String::try_from(&*chunk)?, + format!( + "Chunk - [1 Columns]\n- FLAT HUGEINT: 2 = [ {}, {}]\n", + encode(2, 7)?, + encode(2, 12)? + ) + ); + Ok(()) +} diff --git a/vortex-duckdb/src/scan_v2/diagnostics.rs b/vortex-duckdb/src/scan_v2/diagnostics.rs new file mode 100644 index 00000000000..6a34b545fef --- /dev/null +++ b/vortex-duckdb/src/scan_v2/diagnostics.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::time::Instant; + +/// Debug-only counters shared by all files and workers of one V2 table scan. +pub(crate) struct Diagnostics { + pub(super) id: u64, + started: Instant, + first_array: AtomicBool, + first_export: AtomicBool, + arrays: AtomicU64, + array_rows: AtomicU64, + batches: AtomicU64, + returned_rows: AtomicU64, + compute_ns: AtomicU64, + io_wait_ns: AtomicU64, +} + +impl Diagnostics { + pub(crate) fn new() -> Self { + static NEXT_ID: AtomicU64 = AtomicU64::new(0); + Self { + id: NEXT_ID.fetch_add(1, Ordering::Relaxed), + started: Instant::now(), + first_array: AtomicBool::new(false), + first_export: AtomicBool::new(false), + arrays: AtomicU64::new(0), + array_rows: AtomicU64::new(0), + batches: AtomicU64::new(0), + returned_rows: AtomicU64::new(0), + compute_ns: AtomicU64::new(0), + io_wait_ns: AtomicU64::new(0), + } + } + + pub(super) fn array(&self, rows: usize) { + if !self.first_array.swap(true, Ordering::Relaxed) { + tracing::debug!( + scan_id = self.id, + elapsed_us = self.started.elapsed().as_micros(), + rows, + "v2 first native array" + ); + } + self.arrays.fetch_add(1, Ordering::Relaxed); + self.array_rows.fetch_add(rows as u64, Ordering::Relaxed); + tracing::debug!( + scan_id = self.id, + elapsed_us = self.started.elapsed().as_micros(), + rows, + "v2 native array returned" + ); + } + + pub(crate) fn exported(&self, rows: u64) { + if rows == 0 { + return; + } + if !self.first_export.swap(true, Ordering::Relaxed) { + tracing::debug!( + scan_id = self.id, + elapsed_us = self.started.elapsed().as_micros(), + rows, + "v2 first DuckDB batch" + ); + } + self.batches.fetch_add(1, Ordering::Relaxed); + self.returned_rows.fetch_add(rows, Ordering::Relaxed); + } + + pub(super) fn compute(&self, ns: u128) { + self.compute_ns + .fetch_add(u64::try_from(ns).unwrap_or(u64::MAX), Ordering::Relaxed); + } + pub(super) fn io_wait(&self, ns: u128) { + self.io_wait_ns + .fetch_add(u64::try_from(ns).unwrap_or(u64::MAX), Ordering::Relaxed); + } +} + +impl Drop for Diagnostics { + fn drop(&mut self) { + tracing::debug!( + scan_id = self.id, + elapsed_us = self.started.elapsed().as_micros(), + arrays = self.arrays.load(Ordering::Relaxed), + array_rows = self.array_rows.load(Ordering::Relaxed), + batches = self.batches.load(Ordering::Relaxed), + returned_rows = self.returned_rows.load(Ordering::Relaxed), + compute_ns = self.compute_ns.load(Ordering::Relaxed), + io_wait_ns = self.io_wait_ns.load(Ordering::Relaxed), + "v2 scan summary" + ); + } +} + +/// Records bound changes when Vortex reads DuckDB's shared dynamic filter. +pub(crate) struct FilterObserver { + started: Instant, + previous: parking_lot::Mutex>>, + reads: AtomicU64, + read_ns: AtomicU64, + updates: AtomicU64, + column: String, +} + +impl FilterObserver { + pub(crate) fn new(column: String) -> Self { + Self { + started: Instant::now(), + previous: parking_lot::Mutex::new(None), + reads: AtomicU64::new(0), + read_ns: AtomicU64::new(0), + updates: AtomicU64::new(0), + column, + } + } + + pub(crate) fn observe(&self, value: Option<&vortex::scalar::ScalarValue>, read_ns: u128) { + self.reads.fetch_add(1, Ordering::Relaxed); + self.read_ns.fetch_add( + u64::try_from(read_ns).unwrap_or(u64::MAX), + Ordering::Relaxed, + ); + let mut previous = self.previous.lock(); + if previous.as_ref().map(Option::as_ref) != Some(value) { + self.updates.fetch_add(1, Ordering::Relaxed); + tracing::debug!(elapsed_us = self.started.elapsed().as_micros(), column = self.column, bound = ?value, previous = ?previous.as_ref(), "v2 dynamic filter observed"); + *previous = Some(value.cloned()); + } + } +} + +impl Drop for FilterObserver { + fn drop(&mut self) { + tracing::debug!( + column = self.column, + reads = self.reads.load(Ordering::Relaxed), + read_ns = self.read_ns.load(Ordering::Relaxed), + observations = self.updates.load(Ordering::Relaxed), + "v2 dynamic filter summary" + ); + } +} diff --git a/vortex-duckdb/src/scan_v2/startup.rs b/vortex-duckdb/src/scan_v2/startup.rs new file mode 100644 index 00000000000..39bc6adf3a1 --- /dev/null +++ b/vortex-duckdb/src/scan_v2/startup.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::ops::Range; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use vortex::error::VortexResult; +use vortex::expr::BoundExpression; +use vortex::io::request::IoConsumer; +use vortex::io::request::IoRequestId; +use vortex::io::request::IoResult; +use vortex::scalar_fn::fns::dynamic::DynamicComparison; +use vortex::scan::planning::planner::Planner; +use vortex::scan::planning::planner::PlannerOutput; +use vortex::scan::planning::planner::State; + +const INITIAL_ROWS: u64 = 8192; + +/// Limits the startup path to V2 scans with a pushed dynamic comparison. +pub(crate) fn initial_budget( + scan_v2: bool, + filter: Option<&BoundExpression>, + chunks: usize, + workers: usize, +) -> VortexResult { + if !scan_v2 || chunks == 0 { + return Ok(0); + } + if filter + .map(|filter| filter.contains::()) + .transpose()? + .unwrap_or(false) + { + Ok(chunks.saturating_mul(workers)) + } else { + Ok(0) + } +} + +/// Reserves disjoint startup ranges, carrying unused budget across short files. +pub(super) fn initial_ranges(rows: Range, remaining: &AtomicUsize) -> Vec> { + let needed = + usize::try_from((rows.end - rows.start).div_ceil(INITIAL_ROWS)).unwrap_or(usize::MAX); + let mut previous = remaining.load(Ordering::Relaxed); + let claimed = loop { + let claimed = previous.min(needed); + if claimed == 0 { + break 0; + } + match remaining.compare_exchange_weak( + previous, + previous - claimed, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break claimed, + Err(current) => previous = current, + } + }; + (0..claimed) + .map(|index| { + let start = rows.start + index as u64 * INITIAL_ROWS; + start..start.saturating_add(INITIAL_ROWS).min(rows.end) + }) + .collect() +} + +/// Keeps an initial range in one root, including children the chunk planner normally spawns. +pub(super) struct InitialPlanner(pub Box); + +impl IoConsumer for InitialPlanner { + fn set_io_result(&mut self, request: IoRequestId, result: IoResult) { + self.0.set_io_result(request, result); + } +} + +impl Planner for InitialPlanner { + fn state(&self) -> State { + self.0.state() + } + + fn compute(&mut self) -> VortexResult { + Ok(match self.0.compute()? { + PlannerOutput::Spawn(scope, child) | PlannerOutput::Planner(scope, child) => { + PlannerOutput::Planner(scope, Box::new(Self(child))) + } + output => output, + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::single_range_in_vec_init)] + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + use rstest::rstest; + use vortex::dtype::DType; + use vortex::dtype::PType; + use vortex::error::VortexResult; + use vortex::expr::dynamic; + use vortex::expr::lit; + use vortex::expr::root; + use vortex::scalar_fn::fns::operators::CompareOperator; + + use super::initial_budget; + use super::initial_ranges; + + #[rstest] + #[case::v1_dynamic(false, true, 2, 0)] + #[case::v2_static(true, false, 2, 0)] + #[case::disabled(true, true, 0, 0)] + #[case::v2_dynamic(true, true, 2, 16)] + fn only_dynamic_v2_scans_use_startup( + #[case] scan_v2: bool, + #[case] is_dynamic: bool, + #[case] chunks: usize, + #[case] expected: usize, + ) -> VortexResult<()> { + let dtype = DType::Primitive(PType::I64, false.into()); + let filter = if is_dynamic { + dynamic(CompareOperator::Lt, || None, dtype.clone(), true, root()) + } else { + lit(true) + } + .bind(&dtype)?; + assert_eq!(initial_budget(scan_v2, Some(&filter), chunks, 8)?, expected); + assert_eq!(initial_budget(scan_v2, None, chunks, 8)?, 0); + Ok(()) + } + + #[test] + fn startup_budget_tiles_prefix_and_carries_across_files() { + let budget = AtomicUsize::new(4); + assert_eq!(initial_ranges(5..10, &budget), vec![5..10]); + assert_eq!( + initial_ranges(7..40_000, &budget), + vec![7..8199, 8199..16391, 16391..24583] + ); + assert!(initial_ranges(0..100, &budget).is_empty()); + assert_eq!(budget.load(Ordering::Relaxed), 0); + } + + #[test] + fn startup_budget_handles_empty_and_overflow_boundary() { + let budget = AtomicUsize::new(2); + assert!(initial_ranges(5..5, &budget).is_empty()); + assert_eq!( + initial_ranges(u64::MAX - 10..u64::MAX, &budget), + vec![u64::MAX - 10..u64::MAX] + ); + assert_eq!(budget.load(Ordering::Relaxed), 1); + } + + #[test] + fn concurrent_files_share_one_budget() { + let budget = AtomicUsize::new(16); + let claimed = AtomicUsize::new(0); + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + claimed.fetch_add( + initial_ranges(0..1_000_000, &budget).len(), + Ordering::Relaxed, + ); + }); + } + }); + assert_eq!(claimed.load(Ordering::Relaxed), 16); + assert_eq!(budget.load(Ordering::Relaxed), 0); + } +} diff --git a/vortex-duckdb/src/table_function.rs b/vortex-duckdb/src/table_function.rs index e96dd83d12c..9f13c73f2bf 100644 --- a/vortex-duckdb/src/table_function.rs +++ b/vortex-duckdb/src/table_function.rs @@ -7,6 +7,7 @@ use std::fmt::{self}; use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; #[cfg(unix)] @@ -42,6 +43,7 @@ use vortex::expr::proto::ExprSerializeProtoExt; use vortex::metrics::tracing::get_global_labels; use vortex::proto::expr as expr_pb; use vortex::scalar::Scalar; +use vortex::scan::planning::driver::RootId; use vortex::scan::planning::driver::Run; use vortex_utils::aliases::hash_map::HashMap; @@ -185,8 +187,11 @@ pub struct GlobalState { pub projection: BoundExpression, pub filter: Filter, pub file_row_number_column_pos: Option, + pub(crate) row_id_field: Option, /// Whether the scan runs on the planner-driven V2 executor. pub scan_v2: bool, + pub(crate) initial_chunks: AtomicUsize, + pub(crate) scan_diagnostics: Option>, // Following fields are used only in aggregate scans. pub aggregate_state: Mutex, @@ -225,8 +230,7 @@ pub struct LocalState { pub(crate) retired_files: Vec>, /// The planning run on which this thread runs the chunks of V2 scans. pub(crate) run: Run, - /// Chunks of a V2 batch that this thread has yet to export, in reverse order. - pub(crate) pending: Vec, + pub(crate) initial_roots: Vec, /// Empty for non-aggregate scan pub partials: Partials, @@ -318,6 +322,7 @@ pub fn init_global(init_input: &TableInitInput) -> VortexResult { let Projection { projection, file_row_number_column_pos, + row_id_field, } = if bind_data.aggregates.is_empty() { let input = ProjectionInput { column_ids, @@ -353,7 +358,16 @@ pub fn init_global(init_input: &TableInitInput) -> VortexResult { ); let projection = optimize_and_bind(projection, &bind_data.dtype)?; + let initial_chunks = crate::scan_v2::initial_budget( + crate::scan_v2(), + filter.filter.as_ref(), + crate::DYNAMIC_FILTER_INITIAL_CHUNKS.load(Ordering::Relaxed), + vortex_utils::parallelism::get_available_parallelism().unwrap_or(1), + )?; Ok(GlobalState { + initial_chunks: AtomicUsize::new(initial_chunks), + scan_diagnostics: (crate::scan_v2() && tracing::enabled!(tracing::Level::DEBUG)) + .then(|| Arc::new(crate::scan_v2::Diagnostics::new())), projection, filter, aggregates: bind_data.aggregates.clone(), @@ -361,6 +375,7 @@ pub fn init_global(init_input: &TableInitInput) -> VortexResult { aggregate_state: Mutex::new(AggregateState::default()), row_count: AtomicU64::new(0), file_row_number_column_pos, + row_id_field, scan_v2: crate::scan_v2(), }) } @@ -431,7 +446,7 @@ pub fn init_local(bind_data: &BindState, global: &GlobalState) -> LocalState { file: None, retired_files: Vec::new(), run: Run::new(), - pending: Vec::new(), + initial_roots: Vec::new(), finished: false, } } diff --git a/vortex-layout/src/scan/execution/chunk.rs b/vortex-layout/src/scan/execution/chunk.rs index 448be7bc407..690ab404817 100644 --- a/vortex-layout/src/scan/execution/chunk.rs +++ b/vortex-layout/src/scan/execution/chunk.rs @@ -25,15 +25,16 @@ use super::selection::plan_selection; /// The row grid every chunk boundary lies on. pub(crate) const ROW_BLOCK_SIZE: usize = 8192; -/// The rows of a projection chunk. -pub(crate) const MAX_ROWS_PER_CHUNK: usize = 8 * ROW_BLOCK_SIZE; -/// The rows of a filtered chunk, balancing conjunct setup against parallelism. +/// Projection chunks stream stored batches, amortizing planning without concatenating them. +pub(crate) const MAX_ROWS_PER_CHUNK: usize = 128 * ROW_BLOCK_SIZE; +/// Bound a filtered chunk's projection work so selective scans do not leave one busy worker, +/// while keeping enough rows together to amortize conjunct setup and reads. pub(crate) const FILTERED_ROWS_PER_CHUNK: usize = 16 * ROW_BLOCK_SIZE; const TARGET_BYTES_PER_CHUNK: u64 = 8 * 1024 * 1024; /// Limit estimated bytes per chunk without changing the writer-aligned row granularity. pub(crate) fn rows_per_chunk(row_count: u64, locations: &[SegmentLocation]) -> usize { - if row_count <= MAX_ROWS_PER_CHUNK as u64 { + if row_count <= ROW_BLOCK_SIZE as u64 { return MAX_ROWS_PER_CHUNK; } let bytes: u64 = locations @@ -42,7 +43,10 @@ pub(crate) fn rows_per_chunk(row_count: u64, locations: &[SegmentLocation]) -> u .sum(); let bytes_per_row = bytes.div_ceil(row_count).max(1); let blocks = TARGET_BYTES_PER_CHUNK / bytes_per_row / ROW_BLOCK_SIZE as u64; - blocks.clamp(1, 8) as usize * ROW_BLOCK_SIZE + usize::try_from(blocks) + .unwrap_or(usize::MAX) + .clamp(1, MAX_ROWS_PER_CHUNK / ROW_BLOCK_SIZE) + * ROW_BLOCK_SIZE } /// Fixed boundaries in the file's row domain, clipped to the requested range. diff --git a/vortex-layout/src/scan/execution/filter.rs b/vortex-layout/src/scan/execution/filter.rs index d3d7b640892..5080f1e49c8 100644 --- a/vortex-layout/src/scan/execution/filter.rs +++ b/vortex-layout/src/scan/execution/filter.rs @@ -1,8 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod ordering; + use std::ops::Range; use std::sync::Arc; +use std::time::Instant; use bit_vec::BitVec; use rustc_hash::FxHashSet; @@ -26,6 +29,9 @@ use vortex_scan::planning::planner::PlannerOutput; use vortex_scan::planning::planner::State; use vortex_session::VortexSession; +use self::ordering::AdaptiveOrder; +use self::ordering::ChunkOrder; +use self::ordering::Observation; use super::PreparedScan; use super::SegmentLocation; use super::SelectedRows; @@ -50,6 +56,8 @@ pub struct FilterPlans { predicates: Vec, /// A metadata-only pruning plan per conjunct, where the conjunct has one. pruning: Vec>, + dynamic_indices: Vec, + ordering: AdaptiveOrder, } impl FilterPlans { @@ -88,10 +96,15 @@ impl FilterPlans { } }) .collect::>()?; + let dynamic_indices = (0..expression.conjuncts().len()) + .filter(|&index| expression.dynamic_updates(index).is_some()) + .collect(); Ok(Self { + ordering: AdaptiveOrder::new(expression.conjuncts().len()), expression, predicates, pruning, + dynamic_indices, }) } @@ -122,21 +135,93 @@ fn metadata_only(plan: &PlanRef) -> VortexResult { /// Evaluates a chunk's conjuncts over its selected rows, then hands the mask to projection. /// -/// The first compute starts the reads of every conjunct's segments for the selected rows and -/// announces the projection's, so the conjuncts run back to back and the projection's reads -/// coalesce with them. Conjuncts run in the order the shared [`FilterExpr`] picks. +/// Dynamic pruning is refreshed before data prefetch. Each chunk takes an order from the shared +/// cost estimates, occasionally exploring another order. A dynamic conjunct is revisited if its +/// version changes before projection. Static conjuncts retain their results. pub(super) struct FilterPlanner { scan: Arc, selected: Option, filter: Arc, remaining: BitVec, - active: Option<(usize, ScanAdaptor)>, + order: ChunkOrder, + active: Option, + dynamic: Vec, pieces: Vec, + rechecks: RecheckStats, announced: bool, /// Ids for hints, counting down from `u32::MAX` so they never meet an adaptor's fetch ids. next_hint: u32, } +/// Versions are sampled before evaluation so an update during I/O or compute is checked again. +struct DynamicConjunct { + index: usize, + pruned: Option, + filtered: Option, +} + +#[derive(Clone, Copy)] +enum Evaluation { + Prune, + Filter { recheck: bool }, +} + +struct ActiveConjunct { + index: usize, + version: Option, + evaluation: Evaluation, + adaptor: ScanAdaptor, + started: Option, + cpu_ns: u128, +} + +impl ActiveConjunct { + fn new( + scan: &PreparedScan, + selected: &SelectedRows, + plan: PlanRef, + index: usize, + version: Option, + evaluation: Evaluation, + ) -> VortexResult { + let learn = matches!(evaluation, Evaluation::Filter { recheck: false }) + && scan + .filter + .as_ref() + .is_some_and(|filter| filter.predicates.len() > 1); + let started = (learn || tracing::enabled!(tracing::Level::DEBUG)).then(Instant::now); + let pipeline = Scan::try_new( + scan.session.clone(), + plan, + vec![Split { + rows: selected.scope.rows.clone(), + mask: selected.mask.clone(), + }], + )? + .with_row_offset(scan.row_offset) + .with_value_cache(scan.values.clone()); + let cpu_ns = started.map_or(0, |started| started.elapsed().as_nanos()); + Ok(Self { + index, + version, + evaluation, + adaptor: ScanAdaptor::new(pipeline, Arc::clone(&scan.locations)), + started, + cpu_ns, + }) + } +} + +#[derive(Default)] +struct RecheckStats { + prunes: usize, + pruned_rows: usize, + prune_cpu_ns: u128, + filters: usize, + filtered_rows: usize, + filter_cpu_ns: u128, +} + /// The segments `plans` read over `rows` under `mask`, each once, in id order. Compiling finds /// them without IO. pub(super) fn segments_read<'a>( @@ -155,19 +240,189 @@ pub(super) fn segments_read<'a>( } impl FilterPlanner { - pub(super) fn new(scan: Arc, selected: SelectedRows) -> Self { + pub(super) fn new( + scan: Arc, + selected: SelectedRows, + already_pruned: bool, + ) -> Self { let filter = Arc::clone(scan.filter.as_ref().vortex_expect("filter plans exist")); + let order = filter.ordering.begin_chunk(); + tracing::debug!( + row_start = selected.scope.rows.start, + row_end = selected.scope.rows.end, + order = ?order.indices, + exploration = order.exploration, + "v2 predicate order selected" + ); Self { + order, remaining: BitVec::from_elem(filter.predicates.len(), true), + dynamic: filter + .dynamic_indices + .iter() + .map(|&index| DynamicConjunct { + index, + // Version zero was captured when the filter plans were built, before any + // scan could prune. A supplied pruning mask is at least this recent. + pruned: already_pruned.then_some(0), + filtered: None, + }) + .collect(), scan, selected: Some(selected), filter, active: None, pieces: Vec::new(), + rechecks: RecheckStats::default(), announced: false, next_hint: u32::MAX, } } + + fn advance_active(&mut self) -> VortexResult> { + let Some(mut active) = self.active.take() else { + return Ok(None); + }; + if active.adaptor.state() != State::Done { + let started = active.started.map(|_| Instant::now()); + let output = match active.adaptor.compute()? { + AdaptorStep::Yield => PlannerOutput::Continue, + AdaptorStep::NeedsIO(batch) => PlannerOutput::NeedsIO(batch), + AdaptorStep::Piece(piece) => { + self.pieces.push(piece_mask(&self.scan.session, piece)?); + PlannerOutput::Continue + } + }; + if let Some(started) = started { + active.cpu_ns += started.elapsed().as_nanos(); + } + self.active = Some(active); + return Ok(Some(output)); + } + self.finish_active(active)?; + Ok(None) + } + + fn finish_active(&mut self, mut active: ActiveConjunct) -> VortexResult<()> { + let selected = self.selected.as_mut().vortex_expect("selected rows exist"); + let finished = active.started.map(|_| Instant::now()); + let input = selected.mask.true_count(); + let result = Mask::concat(self.pieces.iter())?; + vortex_ensure_eq!(result.len(), input); + selected.mask = selected.mask.intersect_by_rank(&match active.evaluation { + Evaluation::Prune => !result, + Evaluation::Filter { .. } => result, + }); + if let Some(finished) = finished { + active.cpu_ns += finished.elapsed().as_nanos(); + } + let output = selected.mask.true_count(); + let dynamic = self + .dynamic + .iter_mut() + .find(|dynamic| dynamic.index == active.index); + match active.evaluation { + Evaluation::Prune => { + if let Some(dynamic) = dynamic { + dynamic.pruned = active.version; + } + self.rechecks.prunes += 1; + self.rechecks.pruned_rows += input - output; + self.rechecks.prune_cpu_ns += active.cpu_ns; + tracing::debug!( + row_start = selected.scope.rows.start, + row_end = selected.scope.rows.end, + conjunct = active.index, + version = active.version, + input_rows = input, + output_rows = output, + cpu_ns = active.cpu_ns, + elapsed_ns = active + .started + .map_or(0, |started| started.elapsed().as_nanos()), + "v2 dynamic pruning evaluated" + ); + } + Evaluation::Filter { recheck } => { + if let Some(dynamic) = dynamic { + dynamic.filtered = active.version; + } + if recheck { + self.rechecks.filters += 1; + self.rechecks.filtered_rows += input - output; + self.rechecks.filter_cpu_ns += active.cpu_ns; + } + tracing::debug!( + row_start = selected.scope.rows.start, + row_end = selected.scope.rows.end, + conjunct = active.index, + version = active.version, + recheck, + input_rows = input, + output_rows = output, + cpu_ns = active.cpu_ns, + elapsed_ns = active + .started + .map_or(0, |started| started.elapsed().as_nanos()), + "v2 filter conjunct evaluated" + ); + self.filter.ordering.report( + active.index, + Observation { + version: active.version.unwrap_or(0), + input_rows: input, + output_rows: output, + compute_ns: active.cpu_ns, + recheck, + }, + ); + } + } + self.pieces.clear(); + if matches!(active.evaluation, Evaluation::Prune) && !selected.mask.all_false() { + // Filter plans evaluate their entire row domain before selecting their result. + // Trim newly excluded edge zones so they never enter that domain or its prefetch. + let first = selected.mask.first().vortex_expect("non-empty mask"); + let end = selected.mask.last().vortex_expect("non-empty mask") + 1; + selected.scope.rows = + selected.scope.rows.start + first as u64..selected.scope.rows.start + end as u64; + selected.mask = selected.mask.slice(first..end); + } + Ok(()) + } + + fn refresh_dynamic(&mut self) -> VortexResult { + let selected = self.selected.as_ref().vortex_expect("selected rows exist"); + // Refresh metadata before prefetching data, and revisit only dynamic conjuncts whose + // version changed while another conjunct or an I/O request was running. + for dynamic in &mut self.dynamic { + let version = self + .filter + .expression + .dynamic_updates(dynamic.index) + .vortex_expect("dynamic conjunct has a tracker") + .version(); + if dynamic.pruned != Some(version) { + self.filter.ordering.version_changed(dynamic.index, version); + if let Some(plan) = &self.filter.pruning[dynamic.index] { + self.active = Some(ActiveConjunct::new( + &self.scan, + selected, + plan.clone(), + dynamic.index, + Some(version), + Evaluation::Prune, + )?); + return Ok(true); + } + dynamic.pruned = Some(version); + } + if dynamic.filtered.is_some_and(|filtered| filtered != version) { + self.remaining.set(dynamic.index, true); + } + } + Ok(false) + } } /// Publishes a hint for each of `segments` not in `seen`, and adds it there. @@ -200,8 +455,8 @@ pub(super) fn piece_mask(session: &VortexSession, piece: Piece) -> VortexResult< impl IoConsumer for FilterPlanner { fn set_io_result(&mut self, request: IoRequestId, result: IoResult) { - if let Some((_, adaptor)) = &mut self.active { - adaptor.set_io_result(request, result); + if let Some(active) = &mut self.active { + active.adaptor.set_io_result(request, result); } } } @@ -212,15 +467,30 @@ impl Planner for FilterPlanner { return State::Done; } match &self.active { - Some((_, adaptor)) if adaptor.state() != State::Done => adaptor.state(), + Some(active) if active.adaptor.state() != State::Done => active.adaptor.state(), _ => State::NeedsCompute, } } fn compute(&mut self) -> VortexResult { - let Some(selected) = &mut self.selected else { + if self.selected.is_none() { return Ok(PlannerOutput::Done); - }; + } + if let Some(output) = self.advance_active()? { + return Ok(output); + } + if self + .selected + .as_ref() + .is_some_and(|selected| selected.mask.all_false()) + { + self.selected = None; + return Ok(PlannerOutput::Done); + } + if self.refresh_dynamic()? { + return Ok(PlannerOutput::Continue); + } + let selected = self.selected.as_ref().vortex_expect("selected rows exist"); if !self.announced { self.announced = true; let session = &self.scan.session; @@ -260,55 +530,19 @@ impl Planner for FilterPlanner { return Ok(PlannerOutput::NeedsIO(batch)); } } - if let Some((index, adaptor)) = &mut self.active { - if adaptor.state() != State::Done { - return Ok(match adaptor.compute()? { - AdaptorStep::Yield => PlannerOutput::Continue, - AdaptorStep::NeedsIO(batch) => PlannerOutput::NeedsIO(batch), - AdaptorStep::Piece(piece) => { - self.pieces.push(piece_mask(&self.scan.session, piece)?); - PlannerOutput::Continue - } - }); - } - let input = selected.mask.true_count(); - let result = Mask::concat(self.pieces.iter())?; - vortex_ensure_eq!(result.len(), input); - selected.mask = selected.mask.intersect_by_rank(&result); - tracing::debug!( - row_start = selected.scope.rows.start, - row_end = selected.scope.rows.end, - input_rows = input, - output_rows = selected.mask.true_count(), - conjunct = *index, - "v2 filter conjunct evaluated" - ); - self.filter - .expression - .report_selectivity(*index, selected.mask.true_count() as f64 / input as f64); - self.active = None; - self.pieces.clear(); - } - if selected.mask.all_false() { - self.selected = None; - return Ok(PlannerOutput::Done); - } - if let Some(index) = self.filter.expression.next_conjunct(&self.remaining) { + if let Some(index) = self.order.next_conjunct(&self.remaining) { self.remaining.set(index, false); - let scan = Scan::try_new( - self.scan.session.clone(), + let dynamic = self.dynamic.iter().find(|dynamic| dynamic.index == index); + let version = dynamic.and_then(|dynamic| dynamic.pruned); + let recheck = dynamic.is_some_and(|dynamic| dynamic.filtered.is_some()); + self.active = Some(ActiveConjunct::new( + &self.scan, + selected, self.filter.predicates[index].clone(), - vec![Split { - rows: selected.scope.rows.clone(), - mask: selected.mask.clone(), - }], - )? - .with_row_offset(self.scan.row_offset) - .with_value_cache(self.scan.values.clone()); - self.active = Some(( index, - ScanAdaptor::new(scan, Arc::clone(&self.scan.locations)), - )); + version, + Evaluation::Filter { recheck }, + )?); return Ok(PlannerOutput::Continue); } let selected = self.selected.take().vortex_expect("selected rows exist"); @@ -318,3 +552,19 @@ impl Planner for FilterPlanner { )) } } + +impl Drop for FilterPlanner { + fn drop(&mut self) { + if !self.dynamic.is_empty() { + tracing::debug!( + dynamic_prunes = self.rechecks.prunes, + dynamic_pruned_rows = self.rechecks.pruned_rows, + dynamic_prune_cpu_ns = self.rechecks.prune_cpu_ns, + filter_rechecks = self.rechecks.filters, + refiltered_rows = self.rechecks.filtered_rows, + refilter_cpu_ns = self.rechecks.filter_cpu_ns, + "v2 dynamic recheck summary" + ); + } + } +} diff --git a/vortex-layout/src/scan/execution/filter/ordering.rs b/vortex-layout/src/scan/execution/filter/ordering.rs new file mode 100644 index 00000000000..60a34fabfe3 --- /dev/null +++ b/vortex-layout/src/scan/execution/filter/ordering.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Recent predicate costs and occasional alternative orders for Scan V2 chunks. + +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; + +use bit_vec::BitVec; +use parking_lot::Mutex; +use smallvec::SmallVec; +use vortex_error::VortexExpect; + +const MIN_OBSERVATIONS: usize = 4; +const RECENT_WEIGHT: f64 = 0.25; +const SWITCH_RATIO: f64 = 0.9; +const EXPLORATION_INTERVAL: u64 = 32; + +type Indices = SmallVec<[usize; 4]>; + +pub(super) struct AdaptiveOrder(Mutex); + +pub(super) struct ChunkOrder { + pub(super) indices: Indices, + pub(super) exploration: bool, +} + +impl ChunkOrder { + pub(super) fn next_conjunct(&self, remaining: &BitVec) -> Option { + self.indices.iter().copied().find(|&index| remaining[index]) + } +} + +pub(super) struct Observation { + pub(super) version: u64, + pub(super) input_rows: usize, + pub(super) output_rows: usize, + pub(super) compute_ns: u128, + pub(super) recheck: bool, +} + +impl AdaptiveOrder { + pub(super) fn new(conjuncts: usize) -> Self { + static NEXT_SEED: AtomicU64 = AtomicU64::new(0); + Self(Mutex::new(OrderState::new( + conjuncts, + NEXT_SEED.fetch_add(1, Ordering::Relaxed), + ))) + } + + pub(super) fn begin_chunk(&self) -> ChunkOrder { + self.0.lock().begin_chunk() + } + + pub(super) fn version_changed(&self, index: usize, version: u64) { + self.0.lock().version_changed(index, version); + } + + pub(super) fn report(&self, index: usize, observation: Observation) { + // Rechecks see rows that already passed this predicate. Their conditional selectivity + // cannot predict the cost or rejection rate of a first evaluation on a new chunk. + if observation.recheck || observation.input_rows == 0 { + return; + } + let mut state = self.0.lock(); + if state.estimates.len() < 2 || observation.version < state.estimates[index].version { + return; + } + state.version_changed(index, observation.version); + state.estimates[index].observe(observation); + state.reorder(); + } +} + +struct OrderState { + preferred: Indices, + estimates: Vec, + random: u64, +} + +impl OrderState { + fn new(conjuncts: usize, seed: u64) -> Self { + Self { + preferred: (0..conjuncts).collect(), + estimates: (0..conjuncts).map(|_| Estimate::default()).collect(), + random: seed, + } + } + + fn begin_chunk(&mut self) -> ChunkOrder { + let mut indices = self.preferred.clone(); + let exploration = + indices.len() > 1 && self.next_random().is_multiple_of(EXPLORATION_INTERVAL); + if exploration { + let left = usize::try_from(self.next_random() % (indices.len() - 1) as u64) + .vortex_expect("exploration index is smaller than the conjunct count"); + indices.swap(left, left + 1); + } + ChunkOrder { + indices, + exploration, + } + } + + fn next_random(&mut self) -> u64 { + // SplitMix64 disperses adjacent per-file seeds and avoids synchronizing exploration + // across files or workers. This generator is only used under the ordering lock. + self.random = self.random.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut value = self.random; + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) + } + + fn version_changed(&mut self, index: usize, version: u64) { + let estimate = &mut self.estimates[index]; + if version > estimate.version { + tracing::debug!( + conjunct = index, + previous_version = estimate.version, + version, + "v2 predicate estimates invalidated" + ); + estimate.version = version; + estimate.observations = 0; + } + } + + fn reorder(&mut self) { + if self + .estimates + .iter() + .any(|estimate| estimate.observations < MIN_OBSERVATIONS) + { + return; + } + let mut candidate = self.preferred.clone(); + candidate.sort_by(|&left, &right| { + self.estimates[left] + .cost_per_rejected_row() + .total_cmp(&self.estimates[right].cost_per_rejected_row()) + }); + if candidate == self.preferred + || self.expected_cost(&candidate) >= self.expected_cost(&self.preferred) * SWITCH_RATIO + { + return; + } + tracing::debug!( + previous = ?self.preferred, + order = ?candidate, + estimated_before_ns = self.expected_cost(&self.preferred), + estimated_after_ns = self.expected_cost(&candidate), + "v2 predicate order changed" + ); + self.preferred = candidate; + } + + fn expected_cost(&self, order: &[usize]) -> f64 { + let mut surviving = 1.0; + let mut cost = 0.0; + for &index in order { + let estimate = &self.estimates[index]; + cost += surviving * estimate.cost_per_row_ns; + surviving *= estimate.pass_fraction; + } + cost + } +} + +#[derive(Default)] +struct Estimate { + version: u64, + observations: usize, + cost_per_row_ns: f64, + pass_fraction: f64, +} + +impl Estimate { + fn observe(&mut self, observation: Observation) { + let cost = observation.compute_ns.max(1) as f64 / observation.input_rows as f64; + let passed = observation.output_rows as f64 / observation.input_rows as f64; + if self.observations == 0 { + self.cost_per_row_ns = cost; + self.pass_fraction = passed; + } else { + self.cost_per_row_ns += RECENT_WEIGHT * (cost - self.cost_per_row_ns); + self.pass_fraction += RECENT_WEIGHT * (passed - self.pass_fraction); + } + self.observations = self.observations.saturating_add(1); + } + + fn cost_per_rejected_row(&self) -> f64 { + self.cost_per_row_ns / (1.0 - self.pass_fraction).max(f64::EPSILON) + } +} + +#[cfg(test)] +mod tests; diff --git a/vortex-layout/src/scan/execution/filter/ordering/tests.rs b/vortex-layout/src/scan/execution/filter/ordering/tests.rs new file mode 100644 index 00000000000..388b881faa2 --- /dev/null +++ b/vortex-layout/src/scan/execution/filter/ordering/tests.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use parking_lot::Mutex; + +use super::AdaptiveOrder; +use super::MIN_OBSERVATIONS; +use super::Observation; +use super::OrderState; + +fn ordering(conjuncts: usize) -> AdaptiveOrder { + AdaptiveOrder(Mutex::new(OrderState::new(conjuncts, 17))) +} + +fn report(order: &AdaptiveOrder, index: usize, cost: u128, output_rows: usize) { + order.report( + index, + Observation { + version: 0, + input_rows: 1000, + output_rows, + compute_ns: cost * 1000, + recheck: false, + }, + ); +} + +#[test] +fn expensive_selective_predicate_does_not_displace_a_cheap_filter() { + let order = ordering(2); + for _ in 0..MIN_OBSERVATIONS { + report(&order, 0, 2, 500); + report(&order, 1, 100, 1); + } + assert_eq!(order.0.lock().preferred.as_slice(), &[0, 1]); +} + +#[test] +fn waits_for_evidence_then_moves_the_cheaper_filter_first() { + let order = ordering(2); + for _ in 0..MIN_OBSERVATIONS { + report(&order, 0, 100, 500); + } + for _ in 0..MIN_OBSERVATIONS - 1 { + report(&order, 1, 2, 500); + } + assert_eq!(order.0.lock().preferred.as_slice(), &[0, 1]); + report(&order, 1, 2, 500); + assert_eq!(order.0.lock().preferred.as_slice(), &[1, 0]); +} + +#[test] +fn recent_selectivity_can_replace_a_long_history() { + let order = ordering(2); + for _ in 0..1000 { + report(&order, 0, 10, 1); + report(&order, 1, 10, 900); + } + assert_eq!(order.0.lock().preferred.as_slice(), &[0, 1]); + for _ in 0..16 { + report(&order, 0, 10, 990); + report(&order, 1, 10, 1); + } + assert_eq!(order.0.lock().preferred.as_slice(), &[1, 0]); +} + +#[test] +fn small_estimated_improvements_do_not_switch_order() { + let order = ordering(2); + for _ in 0..MIN_OBSERVATIONS { + report(&order, 0, 10, 500); + report(&order, 1, 9, 500); + } + assert_eq!(order.0.lock().preferred.as_slice(), &[0, 1]); +} + +#[test] +fn changed_bound_requires_fresh_evidence_and_ignores_older_results() { + let order = ordering(2); + for _ in 0..MIN_OBSERVATIONS { + report(&order, 0, 10, 1); + report(&order, 1, 10, 500); + } + order.version_changed(0, 2); + report(&order, 0, 1, 0); + assert_eq!(order.0.lock().estimates[0].observations, 0); + for sample in 0..MIN_OBSERVATIONS { + order.report( + 0, + Observation { + version: 2, + input_rows: 1000, + output_rows: 1000, + compute_ns: 10_000, + recheck: false, + }, + ); + if sample + 1 < MIN_OBSERVATIONS { + assert_eq!(order.0.lock().preferred.as_slice(), &[0, 1]); + } + } + assert_eq!(order.0.lock().preferred.as_slice(), &[1, 0]); + order.version_changed(0, 1); + assert_eq!(order.0.lock().estimates[0].version, 2); +} + +#[test] +fn rechecks_do_not_train_the_discovery_order() { + let order = ordering(2); + for _ in 0..100 { + order.report( + 0, + Observation { + version: 0, + input_rows: 3, + output_rows: 0, + compute_ns: 1, + recheck: true, + }, + ); + } + assert_eq!(order.0.lock().estimates[0].observations, 0); +} + +#[test] +fn exploration_observes_a_predicate_hidden_by_short_circuiting() { + let order = ordering(2); + let mut explored = 0; + for _ in 0..4096 { + let chunk = order.begin_chunk(); + if chunk.exploration { + explored += 1; + assert_eq!(chunk.indices.as_slice(), &[1, 0]); + report(&order, 1, 100, 1000); + } else { + assert_eq!(chunk.indices.as_slice(), &[0, 1]); + } + report(&order, 0, 1, 0); + } + assert!((64..192).contains(&explored)); + assert_eq!(order.0.lock().estimates[1].observations, explored); + assert_eq!(order.0.lock().preferred.as_slice(), &[0, 1]); +} + +#[test] +fn an_admitted_chunk_keeps_its_order_while_other_chunks_train() { + let order = ordering(2); + let chunk = order.begin_chunk(); + let original = chunk.indices.clone(); + for _ in 0..MIN_OBSERVATIONS { + report(&order, 0, 100, 500); + report(&order, 1, 2, 500); + } + assert_eq!(order.0.lock().preferred.as_slice(), &[1, 0]); + assert_eq!(chunk.indices, original); +} + +#[test] +fn empty_inputs_do_not_create_invalid_estimates() { + let order = ordering(2); + order.report( + 0, + Observation { + version: 0, + input_rows: 0, + output_rows: 0, + compute_ns: 0, + recheck: false, + }, + ); + assert_eq!(order.0.lock().estimates[0].observations, 0); +} + +#[test] +fn exploration_preserves_every_predicate_once() { + for conjuncts in 0..8 { + let order = ordering(conjuncts); + for _ in 0..128 { + let chunk = order.begin_chunk(); + let mut indices = chunk.indices; + indices.sort_unstable(); + assert_eq!(indices.as_slice(), (0..conjuncts).collect::>()); + if conjuncts < 2 { + assert!(!chunk.exploration); + } + } + } +} diff --git a/vortex-layout/src/scan/execution/mod.rs b/vortex-layout/src/scan/execution/mod.rs index 1411b96f01e..474193a760a 100644 --- a/vortex-layout/src/scan/execution/mod.rs +++ b/vortex-layout/src/scan/execution/mod.rs @@ -131,9 +131,17 @@ impl PreparedScan { /// The planner of one chunk's selected rows: its conjuncts when the scan has a filter, else /// its projection. - pub(super) fn split_planner(self: &Arc, selected: SelectedRows) -> Box { + pub(super) fn split_planner( + self: &Arc, + selected: SelectedRows, + already_pruned: bool, + ) -> Box { if self.filter.is_some() { - Box::new(FilterPlanner::new(Arc::clone(self), selected)) + Box::new(FilterPlanner::new( + Arc::clone(self), + selected, + already_pruned, + )) } else { Box::new(ProjectionPlanner::new(Arc::clone(self), selected)) } diff --git a/vortex-layout/src/scan/execution/projection.rs b/vortex-layout/src/scan/execution/projection.rs index 141b7377c26..364c7ae49b0 100644 --- a/vortex-layout/src/scan/execution/projection.rs +++ b/vortex-layout/src/scan/execution/projection.rs @@ -31,8 +31,8 @@ use crate::scan::execution::adaptor::ScanAdaptor; const MAX_OUTPUT_ROWS: usize = 65_536; -/// Keeps sparse selections together and subdivides dense selections to avoid large copies when -/// converting chunked projection results to Arrow. +/// Keeps sparse selections together and subdivides dense filtered selections to bound copies +/// when converting chunked projection results to Arrow. Unfiltered scans stream their pieces. pub(super) struct ProjectionPlanner { scan: Arc, selected: IntoIter, @@ -41,7 +41,7 @@ pub(super) struct ProjectionPlanner { impl ProjectionPlanner { /// Creates a planner for `selected`. pub(super) fn new(scan: Arc, selected: SelectedRows) -> Self { - let selected = if selected.mask.true_count() <= MAX_OUTPUT_ROWS { + let selected = if scan.filter.is_none() || selected.mask.true_count() <= MAX_OUTPUT_ROWS { vec![selected] } else { (0..selected.mask.len()) @@ -97,8 +97,8 @@ impl Planner for ProjectionPlanner { } } -/// Runs the projection plan over a split's selected rows and returns them as one batch, in row -/// order. A selection that projects to no rows finishes without a batch. +/// Projects selected rows in order, streaming unfiltered pieces and gathering filtered pieces +/// into one batch. A selection that projects to no rows finishes without a batch. struct ProjectionMorsel { scan: Arc, dtype: DType, @@ -186,6 +186,12 @@ impl Morsel for ProjectionMorsel { Ok(match adaptor.compute()? { AdaptorStep::Yield => MorselOutput::Continue, AdaptorStep::NeedsIO(batch) => MorselOutput::NeedsIO(batch), + AdaptorStep::Piece(piece) if piece.array.is_empty() => MorselOutput::Continue, + // Preserve stored boundaries instead of making Arrow conversion copy across them. + // The chunk planner bounds the row domain of these pieces. + AdaptorStep::Piece(piece) if self.scan.filter.is_none() => { + MorselOutput::Batch(piece.array) + } AdaptorStep::Piece(piece) => { self.pieces.push(piece); MorselOutput::Continue diff --git a/vortex-layout/src/scan/execution/prune.rs b/vortex-layout/src/scan/execution/prune.rs index 325f5f897d7..6c2ac717026 100644 --- a/vortex-layout/src/scan/execution/prune.rs +++ b/vortex-layout/src/scan/execution/prune.rs @@ -5,6 +5,7 @@ use std::ops::Range; use std::sync::Arc; +use std::time::Instant; use std::vec::IntoIter; use rustc_hash::FxHashSet; @@ -51,6 +52,8 @@ pub struct PrunePlanner { pruning: IntoIter, active: Option, pieces: Vec, + active_started: Option, + active_cpu_ns: u128, announced: bool, /// Ids for hints, counting down from `u32::MAX` so they never meet an adaptor's fetch ids. next_hint: u32, @@ -75,6 +78,8 @@ impl PrunePlanner { pruning: pruning.into_iter(), active: None, pieces: Vec::new(), + active_started: None, + active_cpu_ns: 0, announced: false, next_hint: u32::MAX, done: false, @@ -131,20 +136,34 @@ impl Planner for PrunePlanner { } if let Some(adaptor) = &mut self.active { if adaptor.state() != State::Done { - return Ok(match adaptor.compute()? { + let started = self.active_started.map(|_| Instant::now()); + let output = match adaptor.compute()? { AdaptorStep::Yield => PlannerOutput::Continue, AdaptorStep::NeedsIO(batch) => PlannerOutput::NeedsIO(batch), AdaptorStep::Piece(piece) => { self.pieces.push(piece_mask(&self.scan.session, piece)?); PlannerOutput::Continue } - }); + }; + if let Some(started) = started { + self.active_cpu_ns += started.elapsed().as_nanos(); + } + return Ok(output); } + let finished = self.active_started.map(|_| Instant::now()); let input = self.mask.true_count(); let excluded = Mask::concat(self.pieces.iter())?; vortex_ensure_eq!(excluded.len(), input); self.mask = self.mask.intersect_by_rank(&!excluded); + if let Some(finished) = finished { + self.active_cpu_ns += finished.elapsed().as_nanos(); + } tracing::debug!( + cpu_ns = self.active_cpu_ns, + elapsed_ns = self + .active_started + .map(|started| started.elapsed().as_nanos()) + .unwrap_or(0), row_start = self.rows.start, row_end = self.rows.end, input_rows = input, @@ -160,6 +179,8 @@ impl Planner for PrunePlanner { return Ok(PlannerOutput::Done); } if let Some(plan) = self.pruning.next() { + self.active_started = tracing::enabled!(tracing::Level::DEBUG).then(Instant::now); + self.active_cpu_ns = 0; let scan = Scan::try_new( self.scan.session.clone(), plan, @@ -170,6 +191,9 @@ impl Planner for PrunePlanner { )? .with_row_offset(self.scan.row_offset) .with_value_cache(self.scan.values.clone()); + if let Some(started) = self.active_started { + self.active_cpu_ns += started.elapsed().as_nanos(); + } self.active = Some(ScanAdaptor::new(scan, Arc::clone(&self.scan.locations))); return Ok(PlannerOutput::Continue); } diff --git a/vortex-layout/src/scan/execution/selection.rs b/vortex-layout/src/scan/execution/selection.rs index d4bd8f0f41e..f7d1ea129b0 100644 --- a/vortex-layout/src/scan/execution/selection.rs +++ b/vortex-layout/src/scan/execution/selection.rs @@ -37,7 +37,7 @@ pub(super) fn plan_selection( selected.mask = mask.slice(first..end); } let scope = selected.scope.clone(); - Some((scope, scan.split_planner(selected))) + Some((scope, scan.split_planner(selected, kept.is_some()))) } fn select_rows(mut scope: WorkScope, selection: &Selection) -> Option { diff --git a/vortex-layout/src/scan/v2/cache.rs b/vortex-layout/src/scan/v2/cache.rs index 53b6051686c..1217b374ebd 100644 --- a/vortex-layout/src/scan/v2/cache.rs +++ b/vortex-layout/src/scan/v2/cache.rs @@ -8,9 +8,11 @@ use std::hash::Hasher; use std::sync::Arc; use std::sync::Weak; +use once_cell::sync::OnceCell; use parking_lot::Mutex; use rustc_hash::FxHashMap; use vortex_array::expr::BoundExpression; +use vortex_error::VortexResult; use super::io::SegmentRanges; use super::retain::RetainedSegments; @@ -40,19 +42,30 @@ pub struct ScanCache(Arc); #[derive(Default)] struct Inner { - root: Mutex>, + root: OnceCell, /// Optimized plans of non-identity projections. Values are cached by the plan that produced /// them, so a projection must reuse its plan for its values to be found again. - projections: Mutex>, - ranges: Mutex>, + projections: ExpressionCache, + ranges: OnceCell, /// Plans of filters, keyed by the filter expression. A pruning plan remembers the zones it /// excluded, so sharing the plans prunes a file's zones once for every scan of that filter. - filters: Mutex>>, + filters: ExpressionCache>, values: ValueCache, /// Recently read segments, shared by the chunks of every scan of the file. retained: Arc, } +type ExpressionCache = Mutex>>>; + +fn cached_expression( + cache: &ExpressionCache, + expression: &BoundExpression, + create: impl FnOnce() -> VortexResult, +) -> VortexResult { + let cell = Arc::clone(cache.lock().entry(ExprKey(expression.clone())).or_default()); + cell.get_or_try_init(create).cloned() +} + impl ScanCache { /// A cache whose scans retain up to `bytes` of recently read segments, so chunks that share /// a stored segment read it once. The default retains 64 MiB; zero retains nothing. @@ -63,65 +76,35 @@ impl ScanCache { })) } - /// The lowered root plan an earlier scan kept. - pub(super) fn root(&self) -> Option { - self.0.root.lock().clone() - } - - /// Keeps `root`, returning the plan already kept if another scan got there first. - pub(super) fn insert_root(&self, root: PlanRef) -> PlanRef { - self.0.root.lock().get_or_insert(root).clone() - } - - /// The optimized plan of `projection` an earlier scan kept. - pub(super) fn projection(&self, projection: &BoundExpression) -> Option { - self.0 - .projections - .lock() - .get(&ExprKey(projection.clone())) - .cloned() - } - - /// Keeps the optimized plan of `projection`, returning the plan already kept if another scan - /// got there first. - pub(super) fn insert_projection(&self, projection: BoundExpression, plan: PlanRef) -> PlanRef { - self.0 - .projections - .lock() - .entry(ExprKey(projection)) - .or_insert(plan) - .clone() - } - - /// The segment byte-range index an earlier scan kept. - pub(super) fn ranges(&self) -> Option { - self.0.ranges.lock().clone() + /// Builds the lowered root once, with concurrent scans waiting for the same plan. + pub(super) fn root( + &self, + create: impl FnOnce() -> VortexResult, + ) -> VortexResult { + self.0.root.get_or_try_init(create).cloned() } - /// Keeps `ranges`, returning the index already kept if another scan got there first. - pub(super) fn insert_ranges(&self, ranges: SegmentRanges) -> SegmentRanges { - Arc::clone(self.0.ranges.lock().get_or_insert(ranges)) + /// Builds each optimized projection once, independently of other projections. + pub(super) fn projection( + &self, + projection: &BoundExpression, + create: impl FnOnce() -> VortexResult, + ) -> VortexResult { + cached_expression(&self.0.projections, projection, create) } - /// The plans of `filter` an earlier scan kept. - pub(super) fn filter_plans(&self, filter: &BoundExpression) -> Option> { - self.0.filters.lock().get(&ExprKey(filter.clone())).cloned() + /// Builds the segment byte-range index once. + pub(super) fn ranges(&self, create: impl FnOnce() -> SegmentRanges) -> SegmentRanges { + Arc::clone(self.0.ranges.get_or_init(create)) } - /// Keeps the plans of `filter`, returning the plans already kept if another scan got there - /// first. - pub(super) fn insert_filter_plans( + /// Builds each filter's conjunct and pruning plans once. + pub(super) fn filter_plans( &self, - filter: BoundExpression, - plans: Arc, - ) -> Arc { - Arc::clone( - self.0 - .filters - .lock() - .entry(ExprKey(filter)) - .or_insert(plans), - ) + filter: &BoundExpression, + create: impl FnOnce() -> VortexResult>, + ) -> VortexResult> { + cached_expression(&self.0.filters, filter, create) } /// Serves `segments` through the segments the file's scans retain. @@ -199,6 +182,11 @@ impl WeakScanCache { #[cfg(test)] mod tests { use std::hash::BuildHasher; + use std::sync::Barrier; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + use std::thread; + use std::time::Duration; use rstest::rstest; use vortex_array::dtype::DType; @@ -211,8 +199,12 @@ mod tests { use vortex_array::expr::lt; use vortex_array::expr::root; use vortex_error::VortexResult; + use vortex_error::vortex_err; use super::ExprKey; + use super::ScanCache; + use crate::plan::physical::PlanRef; + use crate::plan::physical::RowIdxPlan; fn ints_gt(value: i32) -> Expression { gt(get_item("ints", root()), lit(value)) @@ -225,6 +217,61 @@ mod tests { ) } + #[rstest] + #[case::root(false)] + #[case::projection(true)] + fn concurrent_scans_build_a_plan_once(#[case] projection: bool) -> VortexResult<()> { + let cache = ScanCache::default(); + let calls = AtomicUsize::new(0); + let barrier = Barrier::new(8); + let expression = root().bind(&DType::Primitive(PType::U64, Nullability::NonNullable))?; + let plans = thread::scope(|scope| { + let workers = (0..8) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + let create = || { + calls.fetch_add(1, Ordering::Relaxed); + thread::sleep(Duration::from_millis(2)); + Ok(RowIdxPlan::new(10).into_plan()) + }; + if projection { + cache.projection(&expression, create) + } else { + cache.root(create) + } + }) + }) + .collect::>(); + workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| vortex_err!("cache worker panicked"))? + }) + .collect::>>() + })?; + assert_eq!(calls.load(Ordering::Relaxed), 1); + assert!(plans.iter().all(|plan| PlanRef::ptr_eq(&plans[0], plan))); + Ok(()) + } + + #[test] + fn failed_plan_build_can_be_retried() -> VortexResult<()> { + let cache = ScanCache::default(); + let expression = root().bind(&DType::Primitive(PType::U64, Nullability::NonNullable))?; + assert!( + cache + .projection(&expression, || Err(vortex_err!("build failed"))) + .is_err() + ); + let plan = cache.projection(&expression, || Ok(RowIdxPlan::new(10).into_plan()))?; + let cached = cache.projection(&expression, || Err(vortex_err!("already initialized")))?; + assert!(PlanRef::ptr_eq(&plan, &cached)); + Ok(()) + } + #[test] fn separately_bound_equal_expressions_share_a_key() -> VortexResult<()> { let dtype = table(PType::I32); diff --git a/vortex-layout/src/scan/v2/mod.rs b/vortex-layout/src/scan/v2/mod.rs index 2c401bd46d6..02f369aa60f 100644 --- a/vortex-layout/src/scan/v2/mod.rs +++ b/vortex-layout/src/scan/v2/mod.rs @@ -12,11 +12,12 @@ //! //! Import this module's [`ScanBuilder`] to prepare a V2 scan. It accepts a [`ScanFile`] directly //! and supports projection, filters, row ranges, and selections. Unsupported layouts return an -//! error when the plan is prepared; limits are not part of this builder's API. Work is cut into -//! chunks of up to 65,536 rows, or 131,072 rows for filtered scans. Filtered scans reading only -//! one primitive or boolean field use 1,048,576 rows to amortize setup, independently of the stored -//! layout's chunk boundaries. Full-file projections use smaller 8,192-row multiples when -//! serialized bytes per row are high. +//! error when the plan is prepared; limits are not part of this builder's API. Work uses chunks of +//! up to 1,048,576 rows, or 131,072 rows for filtered scans. Filtered scans reading only one +//! primitive or boolean field use 1,048,576 rows to amortize setup. Chunks are independent of +//! stored layout boundaries. Unfiltered projections stream stored batches without concatenating +//! them. Full-file projections use smaller 8,192-row multiples when serialized bytes per row +//! are high. //! //! A [`ScanFile`] carrying a [`ScanCache`] shares its lowered plan, filter plans and pruned //! zones, segment index, dictionary values, and recently read segments with every other scan of @@ -167,10 +168,7 @@ impl ScanBuilder { pub fn prepare(self) -> VortexResult { let file = self.file; let cache = file.cache.unwrap_or_default(); - let root = match cache.root() { - Some(root) => root, - None => cache.insert_root(lower(&file.layout)?), - }; + let root = cache.root(|| lower(&file.layout))?; let row_count = root.row_count(); let filtered_chunk_rows = self .filter @@ -179,30 +177,25 @@ impl ScanBuilder { .transpose()?; let filter = self .filter - .map(|filter| -> VortexResult<_> { - match cache.filter_plans(&filter) { - Some(plans) => Ok(plans), - None => { - let plans = - Arc::new(FilterPlans::new(filter.clone(), &root, &self.session)?); - Ok(cache.insert_filter_plans(filter, plans)) - } - } + .map(|filter| { + cache.filter_plans(&filter, || { + Ok(Arc::new(FilterPlans::new( + filter.clone(), + &root, + &self.session, + )?)) + }) }) .transpose()?; // Optimizing drops the zone tables a projection never reads and keeps the chunks of a // chunked column lazy, so an identity projection is optimized like any other. let identity = self.projection.is_root() && self.projection.dtype() == root.dtype(); - let projection = match cache.projection(&self.projection) { - Some(plan) => plan, - None => { - let plan = optimize( - plan_row_idx_expression(self.projection.clone(), root)?, - &self.session, - )?; - cache.insert_projection(self.projection, plan) - } - }; + let projection = cache.projection(&self.projection, || { + optimize( + plan_row_idx_expression(self.projection.clone(), root)?, + &self.session, + ) + })?; let chunk_rows = if let Some(rows) = filtered_chunk_rows { rows } else if identity { @@ -210,10 +203,7 @@ impl ScanBuilder { } else { MAX_ROWS_PER_CHUNK }; - let ranges = match cache.ranges() { - Some(ranges) => ranges, - None => cache.insert_ranges(segment_ranges(&file.locations)), - }; + let ranges = cache.ranges(|| segment_ranges(&file.locations)); let values = cache.values(); Ok(PreparedScan { session: self.session, diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index b80a0479b88..a65043071d0 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -7,6 +7,8 @@ use std::sync::Arc; use std::sync::LazyLock; use std::sync::atomic::AtomicI32; use std::sync::atomic::Ordering; +use std::task::Context; +use std::task::Poll; use arrow_array::cast::AsArray; use arrow_array::types::Int32Type; @@ -43,6 +45,12 @@ use vortex_buffer::Alignment; use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_error::vortex_err; +use vortex_io::request::Completion; +use vortex_io::request::IoBatch; +use vortex_io::request::IoIntent; +use vortex_io::request::IoOwnerId; +use vortex_io::request::IoSource; use vortex_io::session::RuntimeSessionExt; use vortex_mask::Mask; use vortex_scan::planning::driver::Batch; @@ -55,7 +63,6 @@ use vortex_scan::strict_sorted_buffer::StrictSortedBuffer; use vortex_session::VortexSession; use super::ChunkPlanner; -use super::MAX_ROWS_PER_CHUNK; use super::PreparedScan; use super::PrunePlanner; use super::ROW_BLOCK_SIZE; @@ -529,10 +536,11 @@ fn scans_join_short_edge_chunks( } #[rstest] -#[case::empty(0, 0, 65_536)] -#[case::single_chunk(1000, 100_000_000, 65_536)] -#[case::no_bytes(100_000, 0, 65_536)] -#[case::small_rows(1_000_000, 64_000_000, 65_536)] +#[case::empty(0, 0, 1_048_576)] +#[case::single_chunk(1000, 100_000_000, 1_048_576)] +#[case::no_bytes(100_000, 0, 1_048_576)] +#[case::small_rows(1_000_000, 64_000_000, 131_072)] +#[case::wide_small_file(65_536, 128_000_000, 8192)] #[case::large_rows(1_000_000, 384_000_000, 16_384)] #[case::minimum(1_000_000, 2_000_000_000, 8192)] fn chunk_byte_budget(#[case] rows: u64, #[case] bytes: u32, #[case] expected: usize) { @@ -587,11 +595,14 @@ async fn dictionary_values_are_reused_between_splits() -> VortexResult<()> { let mut file = scan_file(&segments, &layout)?; // Without retained segments, so a chunk reads every codes chunk it overlaps. file.cache = Some(ScanCache::with_retained_bytes(0)); - let scan = Arc::new(ScanBuilder::new(session.clone(), file).prepare()?); + let mut scan = ScanBuilder::new(session.clone(), file).prepare()?; + // Exercise reuse across work chunks even when the default work chunk holds the whole file. + scan.chunk_rows = 8 * ROW_BLOCK_SIZE; + let scan = Arc::new(scan); let chunks = row_chunks(scan.rows().clone(), scan.chunk_rows); assert_eq!( chunks.len(), - (STORED_CHUNKS * STORED_CHUNK_ROWS).div_ceil(MAX_ROWS_PER_CHUNK) + (STORED_CHUNKS * STORED_CHUNK_ROWS).div_ceil(scan.chunk_rows) ); for (index, rows) in chunks.into_iter().enumerate() { let before = counted.requests(); @@ -708,10 +719,10 @@ async fn primitive_column() -> VortexResult<()> { Ok(()) } -/// Stored chunk boundaries do not subdivide a fixed work chunk. +/// Stored batches stream within one work chunk without being concatenated for Arrow conversion. #[rstest] #[tokio::test] -async fn work_chunks_ignore_layout_boundaries( +async fn stored_batches_share_one_work_chunk( #[values(Strategy::Flat, Strategy::Table, Strategy::ZonedTable)] strategy: Strategy, ) -> VortexResult<()> { let session = new_session().with_tokio(); @@ -721,9 +732,53 @@ async fn work_chunks_ignore_layout_boundaries( row_chunks(scan.rows().clone(), scan.chunk_rows), vec![0..(CHUNKS * CHUNK_ROWS) as u64] ); - let arrays = run_rows(&scan, scan.rows().clone()).await?; - assert_eq!(arrays.len(), 1); - assert_eq!(arrays[0].len(), CHUNKS * CHUNK_ROWS); + let batches = run_batches(&scan, scan.rows().clone()).await?; + assert_eq!(batches.len(), CHUNKS); + assert!(batches.iter().all(|batch| batch.scope.rows == *scan.rows())); + let arrays = batches + .into_iter() + .map(|batch| batch.array) + .collect::>(); + let expected = (0..CHUNKS) + .map(|chunk| table_chunk(chunk * CHUNK_ROWS)) + .collect::>>()?; + assert_arrays_eq!( + ChunkedArray::try_new(arrays, layout.dtype().clone())?, + ChunkedArray::try_new(expected, layout.dtype().clone())?, + &mut session.create_execution_ctx() + ); + Ok(()) +} + +#[tokio::test] +async fn large_stored_batch_is_not_subdivided_for_projection() -> VortexResult<()> { + let session = new_session().with_tokio(); + let segments = Arc::new(TestSegments::default()); + let (mut sequence, eof) = SequenceId::root().split(); + let expected = PrimitiveArray::from_iter(0..150_000i32).into_array(); + let layout = FlatLayoutStrategy::default() + .write_stream( + ArrayContext::empty().into(), + Arc::::clone(&segments), + SequentialStreamAdapter::new( + expected.dtype().clone(), + stream::iter([Ok((sequence.advance(), expected.clone()))]), + ) + .sendable(), + eof, + &session, + ) + .await?; + let segments = segments as Arc; + let scan = Arc::new(builder(&session, &segments, &layout)?.prepare()?); + let batches = run_batches(&scan, scan.rows().clone()).await?; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].scope.rows, 0..150_000); + assert_arrays_eq!( + batches[0].array, + expected, + &mut session.create_execution_ctx() + ); Ok(()) } @@ -996,7 +1051,11 @@ async fn shared_cache_prunes_zones_once() -> VortexResult<()> { file.cache = Some(cache.clone()); let filter = || gt(get_item("ints", root()), lit(2_999i32)).bind(layout.dtype()); let projection = || get_item("ints", root()).bind(layout.dtype()); - assert!(cache.filter_plans(&filter()?).is_none()); + assert!( + cache + .filter_plans(&filter()?, || vortex_bail!("the cache should be empty")) + .is_err() + ); let mut reads = Vec::new(); for _ in 0..2 { @@ -1009,11 +1068,215 @@ async fn shared_cache_prunes_zones_once() -> VortexResult<()> { assert_eq!(actual.len(), 1_000); reads.push(counted.requests() - before); } - let plans = cache - .filter_plans(&filter()?) - .expect("the first scan kept its filter plans"); + let plans = cache.filter_plans(&filter()?, || { + vortex_bail!("the first scan should have kept its filter plans") + })?; assert_eq!(plans.pruning().count(), 1); // Each scan reads the one chunk pruning keeps; only the first reads the zone map. assert_eq!(reads, vec![2, 1]); Ok(()) } + +/// Fetch only demanded segments so these tests distinguish avoided reads from eager hints. +struct DemandOnlyIo(Arc); + +impl IoSource for DemandOnlyIo { + fn submit(&self, owner: IoOwnerId, batch: IoBatch) -> VortexResult<()> { + self.0.submit( + owner, + batch + .into_iter() + .filter(|request| request.intent == IoIntent::Fetch) + .collect(), + ) + } + fn poll(&self) -> VortexResult> { + self.0.poll() + } + fn poll_completion(&self, cx: &mut Context<'_>) -> Poll> { + self.0.poll_completion(cx) + } + fn wait(&self) -> VortexResult { + self.0.wait() + } + fn release(&self, owner: IoOwnerId) { + self.0.release(owner); + } + fn clear(&self) { + self.0.clear(); + } +} + +#[rstest] +#[case::already_pruned(true, 1)] +#[case::not_yet_pruned(false, 2)] +#[tokio::test] +async fn dynamic_threshold_reprunes_a_queued_chunk( + #[case] initially_pruned: bool, + #[case] expected_reads: usize, +) -> VortexResult<()> { + let session = new_session().with_tokio(); + let (segments, layout) = write(&session, zoned_table().as_ref()).await?; + let counted = Arc::new(CountingSegments { + inner: segments, + reads: AtomicUsize::new(0), + }); + let segments = Arc::clone(&counted) as Arc; + let threshold = Arc::new(AtomicI32::new(i32::MAX)); + let bound = Arc::clone(&threshold); + let filter = dynamic( + CompareOperator::Lt, + move || Some(bound.load(Ordering::Relaxed).into()), + DType::Primitive(vortex_array::dtype::PType::I32, false.into()), + true, + get_item("ints", root()), + ) + .bind(layout.dtype())?; + let scan = Arc::new( + builder(&session, &segments, &layout)? + .with_projection(get_item("ints", root()).bind(layout.dtype())?) + .with_some_filter(Some(filter)) + .prepare()?, + ); + let io = scan.io(); + let mut run = Run::new(); + let root = if initially_pruned { + compose(&scan, scan.rows().clone()) + } else { + Box::new(ChunkPlanner::new( + Arc::clone(&scan), + scan.rows().clone(), + scan.selection().clone(), + None, + )) + }; + run.admit( + root, + WorkScope { + file_ordinal: 0, + rows: scan.rows().clone(), + }, + Arc::new(DemandOnlyIo(io.session())), + ); + let (scope, chunk) = loop { + match run.advance()? { + Progress::Spawn(scope, chunk) => break (scope, chunk), + Progress::Waiting => std::future::poll_fn(|cx| run.poll_completion(cx)).await?, + Progress::RootDone(_) => {} + _ => vortex_bail!("expected pruning to emit a chunk"), + } + }; + let before = counted.reads.load(Ordering::Relaxed); + threshold.store(500, Ordering::Relaxed); + run.admit(chunk, scope, Arc::new(DemandOnlyIo(io.session()))); + let mut arrays = Vec::new(); + loop { + match run.advance()? { + Progress::Batch(batch) => arrays.push(batch.array), + Progress::Spawn(scope, chunk) => { + run.admit(chunk, scope, Arc::new(DemandOnlyIo(io.session()))); + } + Progress::Waiting => std::future::poll_fn(|cx| run.poll_completion(cx)).await?, + Progress::RootDone(_) if run.live_roots() > 0 => {} + Progress::RootDone(_) | Progress::Idle => break, + } + } + assert_arrays_eq!( + ChunkedArray::try_new( + arrays, + DType::Primitive(vortex_array::dtype::PType::I32, false.into()) + )?, + PrimitiveArray::from_iter(0..500i32), + &mut session.create_execution_ctx() + ); + assert_eq!( + counted.reads.load(Ordering::Relaxed) - before, + expected_reads, + "tightened zones must be excluded before reading data" + ); + Ok(()) +} + +struct TighteningSegments { + inner: Arc, + threshold: Arc, + trigger: Vec, +} + +impl SegmentSource for TighteningSegments { + fn request(&self, id: SegmentId) -> SegmentFuture { + if self.trigger.contains(&id) { + self.threshold.store(500, Ordering::Relaxed); + } + self.inner.request(id) + } +} + +#[tokio::test] +async fn dynamic_threshold_refilters_after_another_conjunct() -> VortexResult<()> { + let session = new_session().with_tokio(); + let (segments, layout) = write(&session, zoned_table().as_ref()).await?; + let strings = layout + .slot(2)? + .ok_or_else(|| vortex_err!("strings column missing"))?; + let data = strings + .slot(0)? + .ok_or_else(|| vortex_err!("strings data missing"))?; + let mut trigger = Vec::new(); + for child in data.depth_first_traversal() { + trigger.extend(child?.segment_ids()); + } + let threshold = Arc::new(AtomicI32::new(3500)); + let bound = Arc::clone(&threshold); + let segments = Arc::new(TighteningSegments { + inner: segments, + threshold: Arc::clone(&threshold), + trigger, + }) as Arc; + let dynamic = dynamic( + CompareOperator::Lt, + move || Some(bound.load(Ordering::Relaxed).into()), + DType::Primitive(vortex_array::dtype::PType::I32, false.into()), + true, + get_item("ints", root()), + ); + let filter = and(dynamic, gt(get_item("strings", root()), lit(""))).bind(layout.dtype())?; + let scan = Arc::new( + builder(&session, &segments, &layout)? + .with_projection(get_item("ints", root()).bind(layout.dtype())?) + .with_some_filter(Some(filter)) + .prepare()?, + ); + let io = scan.io(); + let mut run = Run::new(); + run.admit( + compose(&scan, scan.rows().clone()), + WorkScope { + file_ordinal: 0, + rows: scan.rows().clone(), + }, + Arc::new(DemandOnlyIo(io.session())), + ); + let mut arrays = Vec::new(); + loop { + match run.advance()? { + Progress::Batch(batch) => arrays.push(batch.array), + Progress::Spawn(scope, chunk) => { + run.admit(chunk, scope, Arc::new(DemandOnlyIo(io.session()))); + } + Progress::Waiting => std::future::poll_fn(|cx| run.poll_completion(cx)).await?, + Progress::RootDone(_) if run.live_roots() > 0 => {} + Progress::RootDone(_) | Progress::Idle => break, + } + } + assert_eq!(threshold.load(Ordering::Relaxed), 500); + assert_arrays_eq!( + ChunkedArray::try_new( + arrays, + DType::Primitive(vortex_array::dtype::PType::I32, false.into()) + )?, + PrimitiveArray::from_iter((0..500i32).filter(|row| row % 3 != 0)), + &mut session.create_execution_ctx() + ); + Ok(()) +} From d0be576356beb6f04b95a7ae9185384bbf32f531 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 29 Oct 2026 23:11:52 +0000 Subject: [PATCH 02/28] Choose Scan V2 access from input row selections Select sparse row lookup from positive input selections, schedule only selected regions, and preserve encoded take kernels at segment leaves and dictionary values. Keep zero-copy native slices and the dense scan path. Expose an access-mode override and diagnostics for mode selection and gather work. Validation: 1,864 affected library tests passed (2 S3 skips), workspace all-target/all-feature Clippy, pinned formatting, layout doctests, and matched DuckDB/DataFusion benchmarks. Signed-off-by: Joe Isaacs --- .../src/plan/physical/operators/scan.rs | 67 +++++++++++++ .../src/plan/physical/operators/take.rs | 21 ++++- .../src/plan/physical/pipeline/compile.rs | 5 + .../src/plan/physical/pipeline/mod.rs | 15 +++ .../src/plan/physical/pipeline/scan.rs | 8 ++ .../src/plan/physical/pipeline/tests.rs | 32 +++++++ vortex-layout/src/plan/physical/plans/take.rs | 4 +- vortex-layout/src/scan/execution/chunk.rs | 16 +++- vortex-layout/src/scan/execution/filter.rs | 3 +- vortex-layout/src/scan/execution/mod.rs | 10 +- .../src/scan/execution/projection.rs | 3 +- vortex-layout/src/scan/execution/selection.rs | 93 +++++++++++++++++++ vortex-layout/src/scan/v2/mod.rs | 29 +++++- vortex-layout/src/scan/v2/tests.rs | 54 +++++++++++ 14 files changed, 348 insertions(+), 12 deletions(-) diff --git a/vortex-layout/src/plan/physical/operators/scan.rs b/vortex-layout/src/plan/physical/operators/scan.rs index 842bd33ee87..f3c5d845944 100644 --- a/vortex-layout/src/plan/physical/operators/scan.rs +++ b/vortex-layout/src/plan/physical/operators/scan.rs @@ -6,8 +6,14 @@ use std::ops::Range; use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::arrays::Bool; +use vortex_array::arrays::Decimal; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::VarBinView; use vortex_array::buffer::BufferHandle; use vortex_array::serde::SerializedArray; +use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_mask::Mask; @@ -21,6 +27,8 @@ use crate::plan::physical::operator::Source; use crate::plan::physical::operator::SourceBlocked; use crate::plan::physical::operator::SourceReady; use crate::plan::physical::operator::Step; +use crate::plan::physical::pipeline::AccessMode; +use crate::plan::physical::pipeline::sparse_selection; use crate::segments::SegmentId; /// Decodes the whole of a scan's segment. @@ -46,7 +54,35 @@ fn select( array: ArrayRef, slice: Option<&Range>, filter: Option<&Mask>, + access_mode: AccessMode, ) -> VortexResult { + if access_mode == AccessMode::RandomAccess { + let rows = slice.cloned().unwrap_or(0..array.len()); + let selected = filter.map_or_else(|| rows.len(), Mask::true_count); + let native_slice = filter.is_none_or(Mask::all_true) + && (array.is::() + || array.is::() + || array.is::() + || array.is::()); + if !native_slice && sparse_selection(selected, array.len()) { + // Gather from the original encoding. Slicing first can hide the sparse demand from + // kernels that compare the selected count with the encoded array's full length. + let indices: Buffer = match filter { + Some(Mask::Values(values)) => values + .indices() + .iter() + .map(|&index| (rows.start + index) as u64) + .collect(), + _ => (rows.start..rows.end).map(|index| index as u64).collect(), + }; + tracing::debug!( + selected_rows = selected, + total_rows = array.len(), + "v2 sparse segment take" + ); + return array.take(indices.into_array()); + } + } let array = match slice { Some(slice) => array.slice(slice.clone())?, None => array, @@ -62,6 +98,7 @@ pub(crate) struct ScanSource { plan: SegmentScanPlan, slice: Option>, filter: Option, + access_mode: AccessMode, /// The segment to read, until it has been asked for. unread: Option, /// The segment has been emitted. @@ -73,12 +110,14 @@ impl ScanSource { plan: SegmentScanPlan, slice: Option>, filter: Option, + access_mode: AccessMode, ) -> Self { Self { unread: Some(plan.segment_id()), plan, slice, filter, + access_mode, done: false, } } @@ -112,6 +151,34 @@ impl Operator for ScanSource { array, self.slice.as_ref(), self.filter.as_ref(), + self.access_mode, )?)) } } + +#[cfg(test)] +mod tests { + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + + use super::*; + + #[test] + fn contiguous_native_lookup_keeps_zero_copy_slice() -> VortexResult<()> { + let source = PrimitiveArray::from_iter(0..1024i32).into_array(); + let actual = select( + source, + Some(&(7..8)), + Some(&Mask::new_true(1)), + AccessMode::RandomAccess, + )?; + assert!(actual.is::()); + assert_arrays_eq!( + actual, + PrimitiveArray::from_iter([7i32]), + &mut VortexSession::empty().create_execution_ctx() + ); + Ok(()) + } +} diff --git a/vortex-layout/src/plan/physical/operators/take.rs b/vortex-layout/src/plan/physical/operators/take.rs index 859b139746f..21472405180 100644 --- a/vortex-layout/src/plan/physical/operators/take.rs +++ b/vortex-layout/src/plan/physical/operators/take.rs @@ -20,8 +20,10 @@ use crate::plan::physical::operator::Source; use crate::plan::physical::operator::SourceBlocked; use crate::plan::physical::operator::SourceReady; use crate::plan::physical::operator::Step; +use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::pipeline::DEFAULT_CAPACITY; use crate::plan::physical::pipeline::ValueCache; +use crate::plan::physical::pipeline::sparse_selection; /// Wraps each batch of codes as a dictionary over the values, once the values inlet has ended. pub(crate) struct TakeSource { @@ -30,6 +32,7 @@ pub(crate) struct TakeSource { values: Option, /// Where values joined from the inlet are kept for the split's later scans. cache: Option, + access_mode: AccessMode, } const CODES: usize = 0; @@ -38,11 +41,17 @@ const VALUES: usize = 1; impl TakeSource { /// A take over `values`, or over what its values inlet produces when `None`, keeping those /// in `cache`. - pub(crate) fn new(plan: TakePlan, values: Option, cache: Option) -> Self { + pub(crate) fn new( + plan: TakePlan, + values: Option, + cache: Option, + access_mode: AccessMode, + ) -> Self { Self { plan, values, cache, + access_mode, } } @@ -87,11 +96,13 @@ impl Operator for TakeSource { }; Ok(match cx.inlet(CODES).take() { Some(codes) => { - // A sparse string projection can gather encoded values without decoding the - // whole dictionary. Keep sharing decoded values for small or dense takes. - let values = if (values.dtype().is_utf8() || values.dtype().is_binary()) + // Sparse lookups keep encoded take kernels visible, including numeric kernels + // in random-access mode. Small and dense takes reuse whole-value conversions. + let values = if (self.access_mode == AccessMode::RandomAccess + || values.dtype().is_utf8() + || values.dtype().is_binary()) && values.len() >= 1024 - && codes.len().saturating_mul(16) < values.len() + && sparse_selection(codes.len(), values.len()) && let Some(shared) = values.as_opt::() { shared.current_array_ref().clone() diff --git a/vortex-layout/src/plan/physical/pipeline/compile.rs b/vortex-layout/src/plan/physical/pipeline/compile.rs index 0e18ebe4e4c..e9518075f2b 100644 --- a/vortex-layout/src/plan/physical/pipeline/compile.rs +++ b/vortex-layout/src/plan/physical/pipeline/compile.rs @@ -130,6 +130,10 @@ impl Compiler<'_> { &self.core.values } + pub(crate) fn access_mode(&self) -> super::AccessMode { + self.core.access_mode + } + /// The rows `rows` of `scan`'s segment, filtered by `filter`, read by the split compiling. /// A segment several readers read is decoded once and shared. pub(crate) fn scan( @@ -153,6 +157,7 @@ impl Compiler<'_> { scan.clone(), slice, filter, + self.access_mode(), )))) } } diff --git a/vortex-layout/src/plan/physical/pipeline/mod.rs b/vortex-layout/src/plan/physical/pipeline/mod.rs index 869babeaf8d..c8359ca4ce6 100644 --- a/vortex-layout/src/plan/physical/pipeline/mod.rs +++ b/vortex-layout/src/plan/physical/pipeline/mod.rs @@ -61,6 +61,21 @@ pub use self::values::ValueCache; /// otherwise; see [`Compiler::join_with`]. pub const DEFAULT_CAPACITY: usize = 8; +/// How a scan accesses rows within encoded segments. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AccessMode { + /// Stream slices and filter batches, retaining whole-value conversions for reuse. + #[default] + Scan, + /// Prefer encoded gathers for sparse selections before materializing values. + RandomAccess, +} + +/// Leave dense selections to bulk kernels; sparse gathers avoid decoding unused values. +pub(crate) fn sparse_selection(selected: usize, total: usize) -> bool { + selected > 0 && selected.saturating_mul(16) < total +} + #[cfg(test)] mod scheduling_tests; #[cfg(test)] diff --git a/vortex-layout/src/plan/physical/pipeline/scan.rs b/vortex-layout/src/plan/physical/pipeline/scan.rs index 816c3566d7e..3c5cb1c2ec7 100644 --- a/vortex-layout/src/plan/physical/pipeline/scan.rs +++ b/vortex-layout/src/plan/physical/pipeline/scan.rs @@ -269,6 +269,7 @@ pub(crate) struct Core { pub(crate) exec: ExecutionCtx, /// The global row index of plan row zero. pub(crate) row_offset: u64, + pub(crate) access_mode: super::AccessMode, pub(crate) arena: Arena, pipelines: Slab>, free_pipelines: Vec, @@ -296,6 +297,7 @@ impl Core { exec: session.create_execution_ctx(), session, row_offset, + access_mode: super::AccessMode::Scan, arena: Arena::default(), pipelines: Slab::default(), free_pipelines: Vec::new(), @@ -685,6 +687,12 @@ impl Scan { self } + /// Chooses streaming filters or sparse encoded gathers within selected segments. + pub fn with_access_mode(mut self, access_mode: super::AccessMode) -> Self { + self.core.access_mode = access_mode; + self + } + /// Runs pipelines until the owner has something to do. pub fn step(&mut self) -> VortexResult { loop { diff --git a/vortex-layout/src/plan/physical/pipeline/tests.rs b/vortex-layout/src/plan/physical/pipeline/tests.rs index a4da689e016..612c21e66fe 100644 --- a/vortex-layout/src/plan/physical/pipeline/tests.rs +++ b/vortex-layout/src/plan/physical/pipeline/tests.rs @@ -44,6 +44,7 @@ use super::synthetic::row_dtype; use super::*; use crate::LayoutRef; use crate::OwnedLayoutChildren; +use crate::arrays::shared::Shared; use crate::layouts::chunked::ChunkedLayout; use crate::layouts::dict::DictLayout; use crate::layouts::flat::FlatLayout; @@ -741,6 +742,37 @@ fn sparse_string_take_preserves_encoded_values(#[case] dtype: DType) -> VortexRe assert_view(&expected, &(0..4), &mask, result.arrays) } +#[rstest] +fn random_access_keeps_sparse_numeric_dictionary_kernels_visible( + #[values(AccessMode::Scan, AccessMode::RandomAccess)] mode: AccessMode, +) -> VortexResult<()> { + let mut store = Store::default(); + let values = + PrimitiveArray::from_option_iter((0..1536i32).map(|row| (row != 1).then_some(row))) + .into_array(); + let codes = + PrimitiveArray::from_option_iter([Some(1535u32), Some(1), None, Some(0), Some(1535)]) + .into_array(); + let layout = DictLayout::new(store.flat(&values)?, store.flat(&codes)?).into_layout(); + let plan = lower(&layout)?; + let cache = ValueCache::default(); + for _ in 0..2 { + let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..5)])? + .with_value_cache(cache.clone()) + .with_access_mode(mode); + let result = drive(&store, scan, delivery(Delivery::Lifo))?; + assert!(result.arrays.iter().all(|array| { + array.as_::().values().is::() == (mode == AccessMode::Scan) + })); + assert_arrays_eq!( + join(plan.dtype(), result.arrays)?, + PrimitiveArray::from_option_iter([Some(1535i32), None, None, Some(0), Some(1535)]), + &mut SESSION.create_execution_ctx() + ); + } + Ok(()) +} + #[test] fn eval_applies_expression_to_selected_rows() -> VortexResult<()> { let mut store = Store::default(); diff --git a/vortex-layout/src/plan/physical/plans/take.rs b/vortex-layout/src/plan/physical/plans/take.rs index 81a3026d815..c97c621759c 100644 --- a/vortex-layout/src/plan/physical/plans/take.rs +++ b/vortex-layout/src/plan/physical/plans/take.rs @@ -152,7 +152,7 @@ impl PlanVTable for Take { let cache = compiler.values().clone(); let values_plan = plan.values()?; if let Some(values) = cache.get(&values_plan) { - let source = TakeSource::new(plan.clone(), Some(values), None); + let source = TakeSource::new(plan.clone(), Some(values), None, compiler.access_mode()); return Ok(Some(compiler.join(vec![codes], source))); } let values = values_plan; @@ -164,7 +164,7 @@ impl PlanVTable for Take { Canonical::empty(values.dtype()).into_array(), )), }; - let source = TakeSource::new(plan.clone(), None, Some(cache)); + let source = TakeSource::new(plan.clone(), None, Some(cache), compiler.access_mode()); Ok(Some(compiler.join_with( vec![codes, values], TakeSource::capacity, diff --git a/vortex-layout/src/scan/execution/chunk.rs b/vortex-layout/src/scan/execution/chunk.rs index 690ab404817..d42571664fb 100644 --- a/vortex-layout/src/scan/execution/chunk.rs +++ b/vortex-layout/src/scan/execution/chunk.rs @@ -22,6 +22,8 @@ use vortex_scan::selection::Selection; use super::PreparedScan; use super::SegmentLocation; use super::selection::plan_selection; +use super::selection::selected_chunks; +use crate::plan::physical::pipeline::AccessMode; /// The row grid every chunk boundary lies on. pub(crate) const ROW_BLOCK_SIZE: usize = 8192; @@ -102,7 +104,19 @@ impl ChunkPlanner { selection: Selection, kept: Option, ) -> Self { - let chunks = scan_chunks(rows.clone(), scan.chunk_rows).into_iter(); + let chunks = if scan.access_mode() == AccessMode::RandomAccess { + selected_chunks(rows.clone(), &selection) + .unwrap_or_else(|| scan_chunks(rows.clone(), scan.chunk_rows)) + } else { + scan_chunks(rows.clone(), scan.chunk_rows) + }; + tracing::debug!( + access_mode = ?scan.access_mode(), + chunks = chunks.len(), + covered_rows = chunks.iter().map(|chunk| chunk.end - chunk.start).sum::(), + "v2 selection work planned" + ); + let chunks = chunks.into_iter(); Self { scan, rows, diff --git a/vortex-layout/src/scan/execution/filter.rs b/vortex-layout/src/scan/execution/filter.rs index 5080f1e49c8..3a7596ee356 100644 --- a/vortex-layout/src/scan/execution/filter.rs +++ b/vortex-layout/src/scan/execution/filter.rs @@ -199,7 +199,8 @@ impl ActiveConjunct { }], )? .with_row_offset(scan.row_offset) - .with_value_cache(scan.values.clone()); + .with_value_cache(scan.values.clone()) + .with_access_mode(scan.access_mode()); let cpu_ns = started.map_or(0, |started| started.elapsed().as_nanos()); Ok(Self { index, diff --git a/vortex-layout/src/scan/execution/mod.rs b/vortex-layout/src/scan/execution/mod.rs index 474193a760a..844e902ee18 100644 --- a/vortex-layout/src/scan/execution/mod.rs +++ b/vortex-layout/src/scan/execution/mod.rs @@ -17,7 +17,6 @@ mod filter; mod projection; mod prune; mod selection; - use std::ops::Range; use std::sync::Arc; @@ -44,7 +43,9 @@ pub use self::filter::FilterPlans; use self::projection::ProjectionPlanner; pub use self::prune::PruneNext; pub use self::prune::PrunePlanner; +pub(crate) use self::selection::access_mode; use crate::plan::physical::PlanRef; +use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::pipeline::ValueCache; use crate::scan::v2::ScanCache; use crate::scan::v2::io::SegmentRanges; @@ -102,6 +103,8 @@ pub struct PreparedScan { pub(super) values: ValueCache, /// The rows of one chunk, the unit of work a chunk planner emits. pub(super) chunk_rows: usize, + /// Streaming or sparse row lookup, chosen from the scan's input selection. + pub(super) access_mode: AccessMode, } impl PreparedScan { @@ -115,6 +118,11 @@ impl PreparedScan { &self.selection } + /// The access mode chosen from the input selection, or explicitly set on the builder. + pub fn access_mode(&self) -> AccessMode { + self.access_mode + } + /// The filter's plans, when the scan has a filter. pub fn filter(&self) -> Option<&FilterPlans> { self.filter.as_deref() diff --git a/vortex-layout/src/scan/execution/projection.rs b/vortex-layout/src/scan/execution/projection.rs index 364c7ae49b0..6722f4cc775 100644 --- a/vortex-layout/src/scan/execution/projection.rs +++ b/vortex-layout/src/scan/execution/projection.rs @@ -176,7 +176,8 @@ impl Morsel for ProjectionMorsel { }], )? .with_row_offset(self.scan.row_offset) - .with_value_cache(self.scan.values.clone()); + .with_value_cache(self.scan.values.clone()) + .with_access_mode(self.scan.access_mode()); self.adaptor = Some(ScanAdaptor::new(scan, Arc::clone(&self.scan.locations))); return Ok(MorselOutput::Continue); }; diff --git a/vortex-layout/src/scan/execution/selection.rs b/vortex-layout/src/scan/execution/selection.rs index f7d1ea129b0..699b5bc7f32 100644 --- a/vortex-layout/src/scan/execution/selection.rs +++ b/vortex-layout/src/scan/execution/selection.rs @@ -13,6 +13,56 @@ use vortex_scan::selection::Selection; use super::PreparedScan; use super::SelectedRows; +use crate::plan::physical::pipeline::AccessMode; + +const RANDOM_ACCESS_ROWS_PER_CHUNK: u64 = 65_536; + +pub(crate) fn access_mode(selection: &Selection, rows: &std::ops::Range) -> AccessMode { + if matches!( + selection, + Selection::IncludeByIndex(_) | Selection::IncludeRoaring(_) + ) && selection.row_count_in_range(rows).saturating_mul(16) + < rows.end.saturating_sub(rows.start) + { + AccessMode::RandomAccess + } else { + AccessMode::Scan + } +} + +/// Visit only selected blocks, keeping each mask bounded even when row IDs are far apart. +pub(super) fn selected_chunks( + rows: std::ops::Range, + selection: &Selection, +) -> Option>> { + if rows.is_empty() { + return Some(Vec::new()); + } + let indices: Box + '_> = match selection { + Selection::IncludeByIndex(indices) => { + let first = indices.partition_point(|&row| row < rows.start); + let end = indices.partition_point(|&row| row < rows.end); + Box::new(indices[first..end].iter().copied()) + } + Selection::IncludeRoaring(indices) => { + let mut iter = indices.iter(); + iter.advance_to(rows.start); + Box::new(iter.take_while(move |&row| row < rows.end)) + } + _ => return None, + }; + let mut chunks: Vec> = Vec::new(); + for row in indices { + if let Some(chunk) = chunks.last_mut() + && chunk.start / RANDOM_ACCESS_ROWS_PER_CHUNK == row / RANDOM_ACCESS_ROWS_PER_CHUNK + { + chunk.end = row + 1; + } else { + chunks.push(row..row + 1); + } + } + Some(chunks) +} /// Plans only the selected span of a chunk, with one planner and one mask per chunk. /// @@ -124,4 +174,47 @@ mod tests { ); Ok(()) } + + #[rstest] + #[case::point(vec![7], 0..100, AccessMode::RandomAccess)] + #[case::empty(vec![100], 0..16, AccessMode::RandomAccess)] + #[case::threshold(vec![0], 0..16, AccessMode::Scan)] + #[case::dense(vec![0, 1, 2, 3], 0..32, AccessMode::Scan)] + #[case::clipped(vec![1, 7, 31], 7..8, AccessMode::Scan)] + fn access_follows_selection_density( + #[case] indices: Vec, + #[case] rows: Range, + #[case] expected: AccessMode, + #[values(false, true)] roaring: bool, + ) -> VortexResult<()> { + let selection = if roaring { + Selection::IncludeRoaring(indices.into_iter().collect()) + } else { + Selection::IncludeByIndex(StrictSortedBuffer::try_new(Buffer::from(indices))?) + }; + assert_eq!(access_mode(&selection, &rows), expected); + Ok(()) + } + + #[rstest] + #[case::spread(vec![7, 9, 65_540, 2_000_000], 0..3_000_000, + vec![7..10, 65_540..65_541, 2_000_000..2_000_001])] + #[case::clipped(vec![0, 7, 9, 100], 7..10, vec![7..10])] + #[case::empty(vec![1, 2], 3..100, vec![])] + #[case::near_max(vec![u64::MAX - 1, u64::MAX], u64::MAX - 3..u64::MAX, + vec![u64::MAX - 1..u64::MAX])] + fn random_access_skips_empty_blocks( + #[case] indices: Vec, + #[case] rows: Range, + #[case] expected: Vec>, + #[values(false, true)] roaring: bool, + ) -> VortexResult<()> { + let selection = if roaring { + Selection::IncludeRoaring(indices.into_iter().collect()) + } else { + Selection::IncludeByIndex(StrictSortedBuffer::try_new(Buffer::from(indices))?) + }; + assert_eq!(selected_chunks(rows, &selection), Some(expected)); + Ok(()) + } } diff --git a/vortex-layout/src/scan/v2/mod.rs b/vortex-layout/src/scan/v2/mod.rs index 02f369aa60f..0e2ec4f6ba5 100644 --- a/vortex-layout/src/scan/v2/mod.rs +++ b/vortex-layout/src/scan/v2/mod.rs @@ -19,6 +19,11 @@ //! them. Full-file projections use smaller 8,192-row multiples when serialized bytes per row //! are high. //! +//! Sparse positive input selections use [`AccessMode::RandomAccess`]: only selected 65,536-row +//! regions are scheduled, and segment leaves gather encoded values before materialization. +//! Dense inputs retain [`AccessMode::Scan`]. [`ScanBuilder::with_access_mode`] overrides the +//! automatic choice. The access mode does not change the rows selected or their order. +//! //! A [`ScanFile`] carrying a [`ScanCache`] shares its lowered plan, filter plans and pruned //! zones, segment index, dictionary values, and recently read segments with every other scan of //! that file, like a cached layout reader does for V1 scans. @@ -46,6 +51,7 @@ pub use self::cache::WeakScanCache; use crate::LayoutRef; use crate::plan::physical::lower; use crate::plan::physical::optimize; +pub use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::plan_row_idx_expression; pub use crate::scan::execution::ChunkPlanner; use crate::scan::execution::FILTERED_ROWS_PER_CHUNK; @@ -56,6 +62,7 @@ pub use crate::scan::execution::PruneNext; pub use crate::scan::execution::PrunePlanner; use crate::scan::execution::ROW_BLOCK_SIZE; pub use crate::scan::execution::SegmentLocation; +use crate::scan::execution::access_mode; use crate::scan::execution::rows_per_chunk; use crate::scan::v2::io::segment_ranges; use crate::segments::SegmentSource; @@ -101,6 +108,7 @@ pub struct ScanBuilder { row_range: Option>, selection: Selection, row_offset: u64, + access_mode: Option, } impl ScanBuilder { @@ -114,6 +122,7 @@ impl ScanBuilder { row_range: None, selection: Selection::default(), row_offset: 0, + access_mode: None, } } @@ -151,6 +160,13 @@ impl ScanBuilder { self } + /// Overrides automatic access selection. By default, sparse positive row selections use + /// random access; other inputs stream and filter batches. Both modes keep row order. + pub fn with_access_mode(mut self, access_mode: AccessMode) -> Self { + self.access_mode = Some(access_mode); + self + } + /// Selects rows by strictly sorted file-local indices. pub fn with_row_indices(self, row_indices: StrictSortedBuffer) -> Self { self.with_selection(Selection::IncludeByIndex(row_indices)) @@ -205,9 +221,19 @@ impl ScanBuilder { }; let ranges = cache.ranges(|| segment_ranges(&file.locations)); let values = cache.values(); + let rows = self.row_range.unwrap_or(0..row_count); + let access_mode = self + .access_mode + .unwrap_or_else(|| access_mode(&self.selection, &rows)); + tracing::debug!( + ?access_mode, + selected_rows = self.selection.row_count_in_range(&rows), + input_rows = rows.end.saturating_sub(rows.start), + "v2 scan access mode" + ); Ok(PreparedScan { session: self.session, - rows: self.row_range.unwrap_or(0..row_count), + rows, selection: self.selection, segments: cache.retaining(file.segments), ranges, @@ -218,6 +244,7 @@ impl ScanBuilder { row_offset: self.row_offset, values, chunk_rows, + access_mode, }) } } diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index a65043071d0..f0f1804dde8 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -62,6 +62,7 @@ use vortex_scan::planning::planner::WorkScope; use vortex_scan::strict_sorted_buffer::StrictSortedBuffer; use vortex_session::VortexSession; +use super::AccessMode; use super::ChunkPlanner; use super::PreparedScan; use super::PrunePlanner; @@ -311,6 +312,59 @@ async fn collect_scan(dtype: DType, builder: ScanBuilder) -> VortexResult, + #[values(false, true)] roaring: bool, + #[values(false, true)] filtered: bool, +) -> VortexResult<()> { + let session = new_session().with_tokio(); + let (segments, layout) = write(&session, table().as_ref()).await?; + let indices = vec![0, 100, 999, 1001, 2000, 3999]; + let selection = if roaring { + vortex_scan::selection::Selection::IncludeRoaring(indices.into_iter().collect()) + } else { + vortex_scan::selection::Selection::IncludeByIndex(StrictSortedBuffer::try_new( + Buffer::from(indices), + )?) + }; + let mut builder = builder(&session, &segments, &layout)? + .with_row_range(200..2500) + .with_selection(selection); + if let Some(mode) = mode { + builder = builder.with_access_mode(mode); + } + if filtered { + builder = builder.with_some_filter(Some( + gt(get_item("ints", root()), lit(1000i32)).bind(layout.dtype())?, + )); + } + let scan = Arc::new(builder.prepare()?); + assert_eq!(scan.access_mode(), mode.unwrap_or(AccessMode::RandomAccess)); + let actual = ChunkedArray::try_new(run_rows(&scan, 200..2500).await?, layout.dtype().clone())?; + let source = ChunkedArray::try_new( + (0..CHUNKS) + .map(|i| table_chunk(i * CHUNK_ROWS)) + .collect::>>()?, + layout.dtype().clone(), + )? + .into_array(); + let expected = if filtered { + vec![1001u64, 2000] + } else { + vec![999u64, 1001, 2000] + }; + assert_arrays_eq!( + actual, + source.take(Buffer::from(expected).into_array())?, + &mut session.create_execution_ctx() + ); + Ok(()) +} + /// Runs `builder` on both executors and checks the planner-driven one returns the same rows. async fn assert_same_as_v1( session: &VortexSession, From 3f5325a6c3249c559ad0f3a7822c886a0b7c8520 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 00:51:13 +0000 Subject: [PATCH 03/28] Choose sparse dictionary access from predicate and row demand Preserve selective predicate demand at scan leaves while retaining vectorized evaluation for dense scan masks. Memoize referenced string dictionary predicates and reuse packed whole results after bulk evaluation; keep numeric comparisons on their vectorized path. Read dictionary codes before referenced value chunks for sparse random access, preserve duplicate/null code order, and keep partial reads out of whole-value caches. Fetch flat value segments alongside codes. Validation: 1,890 affected library tests passed (2 S3 skips), workspace all-target/all-feature Clippy with warnings denied, pinned formatting, layout doctests, and five complete Q23 row comparisons against V1. Signed-off-by: Joe Isaacs --- .../src/plan/physical/operators/mod.rs | 2 + .../src/plan/physical/operators/take.rs | 28 ++ .../plan/physical/operators/take_sparse.rs | 224 +++++++++++ .../src/plan/physical/pipeline/mod.rs | 2 + .../src/plan/physical/pipeline/predicate.rs | 187 +++++++++ .../src/plan/physical/pipeline/tests.rs | 362 ++++++++++++++++++ .../src/plan/physical/pipeline/values.rs | 32 +- .../src/plan/physical/plans/share.rs | 7 + vortex-layout/src/plan/physical/plans/take.rs | 130 +++++-- vortex-layout/src/scan/execution/filter.rs | 23 +- vortex-layout/src/scan/v2/tests.rs | 47 ++- 11 files changed, 1002 insertions(+), 42 deletions(-) create mode 100644 vortex-layout/src/plan/physical/operators/take_sparse.rs create mode 100644 vortex-layout/src/plan/physical/pipeline/predicate.rs diff --git a/vortex-layout/src/plan/physical/operators/mod.rs b/vortex-layout/src/plan/physical/operators/mod.rs index b0f5937d71d..990eb3215e2 100644 --- a/vortex-layout/src/plan/physical/operators/mod.rs +++ b/vortex-layout/src/plan/physical/operators/mod.rs @@ -13,6 +13,7 @@ mod once; mod pack; mod scan; mod take; +mod take_sparse; mod whole; pub(crate) use concat::*; @@ -23,6 +24,7 @@ pub(crate) use pack::*; pub(crate) use scan::*; use smallvec::SmallVec; pub(crate) use take::*; +pub(crate) use take_sparse::*; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; diff --git a/vortex-layout/src/plan/physical/operators/take.rs b/vortex-layout/src/plan/physical/operators/take.rs index 21472405180..68281fb2efd 100644 --- a/vortex-layout/src/plan/physical/operators/take.rs +++ b/vortex-layout/src/plan/physical/operators/take.rs @@ -3,9 +3,12 @@ //! Looking codes up in dictionary values. +use std::sync::Arc; + use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::arrays::DictArray; +use vortex_array::expr::BoundExpression; use vortex_error::VortexResult; use super::UNBOUNDED; @@ -22,6 +25,7 @@ use crate::plan::physical::operator::SourceReady; use crate::plan::physical::operator::Step; use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::pipeline::DEFAULT_CAPACITY; +use crate::plan::physical::pipeline::PredicateCache; use crate::plan::physical::pipeline::ValueCache; use crate::plan::physical::pipeline::sparse_selection; @@ -33,6 +37,7 @@ pub(crate) struct TakeSource { /// Where values joined from the inlet are kept for the split's later scans. cache: Option, access_mode: AccessMode, + predicate: Option<(BoundExpression, Arc)>, } const CODES: usize = 0; @@ -52,9 +57,20 @@ impl TakeSource { values, cache, access_mode, + predicate: None, } } + /// Evaluate a static predicate only for dictionary entries the codes reference. + pub(crate) fn with_predicate( + mut self, + expression: BoundExpression, + cache: Arc, + ) -> Self { + self.predicate = Some((expression, cache)); + self + } + /// The batches inlet `inlet` holds: the values are taken whole, so they never block. pub(crate) fn capacity(inlet: usize) -> usize { if inlet == VALUES { @@ -96,6 +112,18 @@ impl Operator for TakeSource { }; Ok(match cx.inlet(CODES).take() { Some(codes) => { + if let Some((expression, cache)) = &self.predicate { + let values = values.as_opt::().map_or_else( + || values.clone(), + |shared| shared.current_array_ref().clone(), + ); + return Ok(Step::Final(cache.evaluate( + values, + codes, + expression, + cx.exec(), + )?)); + } // Sparse lookups keep encoded take kernels visible, including numeric kernels // in random-access mode. Small and dense takes reuse whole-value conversions. let values = if (self.access_mode == AccessMode::RandomAccess diff --git a/vortex-layout/src/plan/physical/operators/take_sparse.rs b/vortex-layout/src/plan/physical/operators/take_sparse.rs new file mode 100644 index 00000000000..07476853192 --- /dev/null +++ b/vortex-layout/src/plan/physical/operators/take_sparse.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Read dictionary codes first, then request only the referenced regions of the values plan. + +use std::collections::VecDeque; + +use rustc_hash::FxHashMap; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::IntoArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::DictArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::PType; +use vortex_array::scalar::Scalar; +use vortex_array::validity::Validity; +use vortex_buffer::Buffer; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; +use vortex_error::vortex_ensure_eq; +use vortex_error::vortex_err; +use vortex_mask::AllOr; +use vortex_mask::Mask; + +use super::join; +use crate::plan::physical::TakePlan; +use crate::plan::physical::operator::Cx; +use crate::plan::physical::operator::Input; +use crate::plan::physical::operator::Operator; +use crate::plan::physical::operator::Source; +use crate::plan::physical::operator::SourceBlocked; +use crate::plan::physical::operator::SourceReady; +use crate::plan::physical::operator::Step; +use crate::plan::physical::pipeline::AccessMode; +use crate::plan::physical::pipeline::ReadId; +use crate::plan::physical::pipeline::Scan; +use crate::plan::physical::pipeline::Split; +use crate::plan::physical::pipeline::Turn; +use crate::plan::physical::pipeline::ValueCache; +use crate::segments::SegmentId; + +const VALUE_BLOCK_ROWS: u64 = 65_536; + +pub(crate) struct SparseTakeSource { + plan: TakePlan, + cache: ValueCache, + row_offset: u64, + active: Option, + output: Option, +} + +struct ValueScan { + scan: Scan, + codes: ArrayRef, + pieces: Vec<(usize, ArrayRef)>, + pending: FxHashMap>, + selected: usize, + reads: usize, +} + +impl SparseTakeSource { + pub(crate) fn new(plan: TakePlan, cache: ValueCache, row_offset: u64) -> Self { + Self { + plan, + cache, + row_offset, + active: None, + output: None, + } + } + + fn start(&mut self, codes: ArrayRef, cx: &mut Cx<'_>) -> VortexResult<()> { + let values = self.plan.values()?; + let codes = codes + .cast(DType::Primitive(PType::U64, codes.dtype().nullability()))? + .execute::(cx.exec())?; + let valid = codes.validity()?.execute_mask(codes.len(), cx.exec())?; + let original = codes.as_slice::(); + let mut selected = Vec::new(); + let mut select = |row: usize| -> VortexResult<()> { + vortex_ensure!( + original[row] < values.row_count(), + "Dictionary code out of bounds" + ); + selected.push(original[row]); + Ok(()) + }; + match valid.indices() { + AllOr::All => (0..codes.len()).try_for_each(&mut select)?, + AllOr::Some(rows) => rows.iter().copied().try_for_each(&mut select)?, + AllOr::None => {} + } + selected.sort_unstable(); + selected.dedup(); + if selected.is_empty() { + self.output = Some(if codes.is_empty() { + Canonical::empty(self.plan.dtype()).into_array() + } else { + ConstantArray::new(Scalar::null(self.plan.dtype().clone()), codes.len()) + .into_array() + }); + return Ok(()); + } + let mut remapped = vec![0u64; codes.len()]; + let mut remap = |row: usize| { + remapped[row] = selected + .binary_search(&original[row]) + .ok() + .vortex_expect("selected codes include every valid code") + as u64; + }; + match valid.indices() { + AllOr::All => (0..codes.len()).for_each(&mut remap), + AllOr::Some(rows) => rows.iter().copied().for_each(&mut remap), + AllOr::None => {} + } + let remapped = PrimitiveArray::new( + Buffer::from(remapped), + Validity::from_mask(valid, codes.dtype().nullability()), + ) + .into_array(); + let splits = selected + .chunk_by(|left, right| left / VALUE_BLOCK_ROWS == right / VALUE_BLOCK_ROWS) + .map(|group| { + let first = group[0]; + let end = group[group.len() - 1] + 1; + Ok(Split { + rows: first..end, + mask: Mask::from_indices( + usize::try_from(end - first)?, + group.iter().map(|&row| { + usize::try_from(row - first) + .vortex_expect("value block indices fit in usize") + }), + ), + }) + }) + .collect::>>()?; + self.active = Some(ValueScan { + scan: Scan::try_new(cx.session().clone(), values, splits)? + .with_value_cache(self.cache.clone()) + .with_access_mode(AccessMode::RandomAccess) + .with_row_offset(self.row_offset), + codes: remapped, + pieces: Vec::new(), + pending: FxHashMap::default(), + selected: selected.len(), + reads: 0, + }); + Ok(()) + } +} + +impl Source for SparseTakeSource { + fn ready(&mut self, cx: &mut Cx<'_>) -> VortexResult { + if self.output.is_some() { + return Ok(SourceReady::Ready); + } + if self.active.is_none() { + match cx.inlet(0).take() { + Some(codes) => self.start(codes, cx)?, + None if cx.inlet(0).closed() => return Ok(SourceReady::Ready), + None => return Ok(SourceReady::Blocked(SourceBlocked::Inlet(0))), + } + } + if self.output.is_some() { + return Ok(SourceReady::Ready); + } + let active = self.active.as_mut().vortex_expect("value scan started"); + while let Some((segment, bytes)) = cx.take_segment() { + let read = active + .pending + .get_mut(&segment) + .and_then(VecDeque::pop_front) + .ok_or_else(|| vortex_err!("Unrequested dictionary values segment {segment}"))?; + active.scan.deliver(read, bytes)?; + } + loop { + match active.scan.step()? { + Turn::Read(read) => { + active + .pending + .entry(read.segment_id) + .or_default() + .push_back(read.id); + active.reads += 1; + cx.request_read(read.segment_id); + } + Turn::Output(split, array) => active.pieces.push((split, array)), + Turn::Waiting => return Ok(SourceReady::Blocked(SourceBlocked::Io)), + Turn::Done => { + let mut active = self.active.take().vortex_expect("value scan exists"); + active.pieces.sort_by_key(|(split, _)| *split); + let values = join( + self.plan.values()?.dtype(), + active.pieces.into_iter().map(|(_, array)| array).collect(), + )?; + vortex_ensure_eq!(values.len(), active.selected); + tracing::debug!( + selected_values = active.selected, + dictionary_values = self.plan.values()?.row_count(), + value_reads = active.reads, + "v2 selected dictionary reads" + ); + self.output = Some(DictArray::try_new(active.codes, values)?.into_array()); + return Ok(SourceReady::Ready); + } + } + } + } +} + +impl Operator for SparseTakeSource { + fn compute(&mut self, _input: Input, _cx: &mut Cx<'_>) -> VortexResult { + Ok(match self.output.take() { + Some(array) => Step::Final(array), + None => Step::End, + }) + } +} diff --git a/vortex-layout/src/plan/physical/pipeline/mod.rs b/vortex-layout/src/plan/physical/pipeline/mod.rs index c8359ca4ce6..47af7dc21e0 100644 --- a/vortex-layout/src/plan/physical/pipeline/mod.rs +++ b/vortex-layout/src/plan/physical/pipeline/mod.rs @@ -40,6 +40,7 @@ mod compile; mod port; +mod predicate; mod scan; pub mod synthetic; mod values; @@ -50,6 +51,7 @@ pub(crate) use self::compile::overlapping; pub(super) use self::port::Arena; pub use self::port::Inlet; pub(super) use self::port::PortId; +pub(crate) use self::predicate::PredicateCache; pub use self::scan::ReadId; pub use self::scan::ReadRequest; pub use self::scan::Scan; diff --git a/vortex-layout/src/plan/physical/pipeline/predicate.rs b/vortex-layout/src/plan/physical/pipeline/predicate.rs new file mode 100644 index 00000000000..fa3206d9650 --- /dev/null +++ b/vortex-layout/src/plan/physical/pipeline/predicate.rs @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Dictionary predicates evaluated only for referenced values, shared across scan chunks. + +use std::sync::OnceLock; +use std::sync::atomic::AtomicU8; +#[cfg(test)] +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::DictArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::PType; +use vortex_array::expr::BoundExpression; +use vortex_array::validity::Validity; +use vortex_buffer::BitBuffer; +use vortex_buffer::Buffer; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; +use vortex_mask::AllOr; +use vortex_mask::Mask; + +const UNKNOWN: u8 = 0; +const FALSE: u8 = 1; +const TRUE: u8 = 2; +const NULL: u8 = 3; + +/// Entries only transition from unknown to their immutable predicate result. Racing readers may +/// evaluate the same missing entry, but never block each other while executing a predicate. +pub(crate) struct PredicateCache { + entries: Vec, + whole: OnceLock, + #[cfg(test)] + evaluated: AtomicUsize, +} + +impl PredicateCache { + pub(crate) fn new(len: usize) -> Self { + Self { + entries: (0..len).map(|_| AtomicU8::new(UNKNOWN)).collect(), + whole: OnceLock::new(), + #[cfg(test)] + evaluated: AtomicUsize::new(0), + } + } + + pub(crate) fn evaluate( + &self, + values: ArrayRef, + codes: ArrayRef, + expression: &BoundExpression, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + if let Some(whole) = self.whole.get() { + tracing::debug!( + selected_rows = codes.len(), + dictionary_values = whole.len(), + missing_values = 0, + evaluated_values = 0, + dense = false, + cached = true, + "v2 dictionary predicate work" + ); + return Ok(DictArray::try_new(codes, whole.clone())?.into_array()); + } + let nullable = codes.dtype().is_nullable() || expression.dtype().is_nullable(); + let codes = codes + .cast(DType::Primitive(PType::U64, codes.dtype().nullability()))? + .execute::(ctx)?; + let valid = codes.validity()?.execute_mask(codes.len(), ctx)?; + let indices = codes.as_slice::(); + let mut missing = Vec::new(); + let mut check = |row: usize| -> VortexResult<()> { + let code = usize::try_from(indices[row])?; + vortex_ensure!(code < self.entries.len(), "Dictionary code out of bounds"); + if self.entries[code].load(Ordering::Relaxed) == UNKNOWN { + missing.push(indices[row]); + } + Ok(()) + }; + match valid.indices() { + AllOr::All => (0..codes.len()).try_for_each(&mut check)?, + AllOr::Some(rows) => rows.iter().copied().try_for_each(&mut check)?, + AllOr::None => {} + } + missing.sort_unstable(); + missing.dedup(); + let dense = missing.len().saturating_mul(4) >= values.len(); + let evaluated = if missing.is_empty() { + 0 + } else if dense { + values.len() + } else { + missing.len() + }; + if evaluated > 0 { + let selected = if dense { + values.clone() + } else { + values.take(Buffer::copy_from(&missing).into_array())? + }; + let result = selected + .apply_bound(expression)? + .execute::(ctx)?; + let mut states = vec![FALSE; evaluated]; + result + .to_bit_buffer() + .for_each_set_index(|row| states[row] = TRUE); + let invalid = !BoolArrayExt::validity(&result).execute_mask(result.len(), ctx)?; + match invalid.indices() { + AllOr::All => states.fill(NULL), + AllOr::Some(rows) => rows.iter().for_each(|&row| states[row] = NULL), + AllOr::None => {} + } + for (row, state) in states.into_iter().enumerate() { + let index = if dense { + row + } else { + usize::try_from(missing[row])? + }; + self.entries[index].store(state, Ordering::Relaxed); + } + #[cfg(test)] + self.evaluated.fetch_add(evaluated, Ordering::Relaxed); + if dense { + // Bulk evaluation has already paid for every value. Keep its packed result so + // later batches use the normal dictionary kernels without per-code atomics. + drop(self.whole.set(result.into_array())); + } + } + tracing::debug!( + selected_rows = codes.len(), + dictionary_values = values.len(), + missing_values = missing.len(), + evaluated_values = evaluated, + dense, + "v2 dictionary predicate work" + ); + if let Some(whole) = self.whole.get() { + return Ok(DictArray::try_new(codes.into_array(), whole.clone())?.into_array()); + } + self.gather(&codes, &valid, nullable) + } + + fn gather( + &self, + codes: &PrimitiveArray, + valid: &Mask, + nullable: bool, + ) -> VortexResult { + let indices = codes.as_slice::(); + let mut states = vec![NULL; codes.len()]; + let mut read = |row: usize| -> VortexResult<()> { + // Codes were checked before evaluation. Relaxed ordering suffices because each + // atomic stores the complete result, with no separate data to publish. + states[row] = self.entries[usize::try_from(indices[row])?].load(Ordering::Relaxed); + Ok(()) + }; + match valid.indices() { + AllOr::All => (0..codes.len()).try_for_each(&mut read)?, + AllOr::Some(rows) => rows.iter().copied().try_for_each(&mut read)?, + AllOr::None => {} + } + let bits = BitBuffer::collect_bool(states.len(), |row| states[row] == TRUE); + let validity = if nullable { + Validity::from(BitBuffer::collect_bool(states.len(), |row| { + states[row] != NULL + })) + } else { + Validity::NonNullable + }; + Ok(BoolArray::new(bits, validity).into_array()) + } + + #[cfg(test)] + pub(crate) fn evaluated_rows(&self) -> usize { + self.evaluated.load(Ordering::Relaxed) + } +} diff --git a/vortex-layout/src/plan/physical/pipeline/tests.rs b/vortex-layout/src/plan/physical/pipeline/tests.rs index 612c21e66fe..0de57d57a1e 100644 --- a/vortex-layout/src/plan/physical/pipeline/tests.rs +++ b/vortex-layout/src/plan/physical/pipeline/tests.rs @@ -15,6 +15,7 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::Dict; +use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBin; @@ -25,8 +26,10 @@ use vortex_array::assert_arrays_eq; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::StructFields; +use vortex_array::expr::eq; use vortex_array::expr::get_item; use vortex_array::expr::gt; +use vortex_array::expr::like; use vortex_array::expr::lit; use vortex_array::expr::root; use vortex_array::serde::SerializeOptions; @@ -50,13 +53,16 @@ use crate::layouts::dict::DictLayout; use crate::layouts::flat::FlatLayout; use crate::layouts::struct_::StructLayout; use crate::plan::physical::ConcatPlan; +use crate::plan::physical::Eval; use crate::plan::physical::EvalPlan; use crate::plan::physical::Filter; use crate::plan::physical::PackPlan; use crate::plan::physical::PlanChildren; use crate::plan::physical::PlanRef; use crate::plan::physical::SegmentScan; +use crate::plan::physical::SharePlan; use crate::plan::physical::Take; +use crate::plan::physical::TakePlan; use crate::plan::physical::lower; use crate::plan::physical::operator::Cx; use crate::plan::physical::operator::InletSet; @@ -1001,3 +1007,359 @@ fn bound_selection_keeps_whole_dictionary_values() -> VortexResult<()> { ); Ok(()) } + +#[rstest] +fn sparse_dictionary_predicate_evaluates_after_gather( + #[values(AccessMode::Scan, AccessMode::RandomAccess)] mode: AccessMode, + #[values(false, true)] nullable_codes: bool, +) -> VortexResult<()> { + let mut store = Store::default(); + let values = VarBinArray::from_iter( + (0..1536).map(|row| (row != 1).then(|| format!("value {row}"))), + DType::Utf8(true.into()), + ) + .into_array(); + let codes = if nullable_codes { + PrimitiveArray::from_option_iter([Some(1535u32), Some(1), None, Some(0), Some(1535)]) + .into_array() + } else { + Buffer::from(vec![1535u32, 1, 0, 0, 1535]).into_array() + }; + let layout = DictLayout::new(store.flat(&values)?, store.flat(&codes)?).into_layout(); + let source = lower(&layout)?; + let expression = like(root(), lit("%1535%")).bind(source.dtype())?; + let plan = optimize(EvalPlan::try_new(expression, source)?.into_plan(), &SESSION)?; + let take = plan.as_::(); + let predicate = take.values()?; + assert!(predicate.is::()); + let cache = ValueCache::default(); + for _ in 0..2 { + let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..5)])? + .with_value_cache(cache.clone()) + .with_access_mode(mode); + let result = drive(&store, scan, delivery(Delivery::Lifo))?; + assert_arrays_eq!( + join(plan.dtype(), result.arrays)?, + BoolArray::from_iter([ + Some(true), + None, + if nullable_codes { None } else { Some(false) }, + Some(false), + Some(true) + ]), + &mut SESSION.create_execution_ctx() + ); + assert!( + cache.get(&predicate).is_none(), + "sparse evaluation must not compute the whole predicate" + ); + assert_eq!( + cache + .predicate(&predicate) + .ok_or_else(|| vortex_err!("missing predicate cache"))? + .evaluated_rows(), + 3 + ); + } + Ok(()) +} + +#[test] +fn sparse_dictionary_predicate_reuses_a_dense_result() -> VortexResult<()> { + let mut store = Store::default(); + let values = VarBinArray::from_iter( + (0..128).map(|row| Some(format!("value {row}"))), + DType::Utf8(false.into()), + ) + .into_array(); + let codes = Buffer::from_iter((0..256u32).map(|row| row % 128)).into_array(); + let layout = DictLayout::new(store.flat(&values)?, store.flat(&codes)?).into_layout(); + let source = lower(&layout)?; + let expression = like(root(), lit("%12%")).bind(source.dtype())?; + let plan = optimize(EvalPlan::try_new(expression, source)?.into_plan(), &SESSION)?; + let predicate = plan.as_::().values()?; + let expected = + BoolArray::from_iter((0..256u32).map(|row| (row % 128).to_string().contains("12"))) + .into_array(); + let cache = ValueCache::default(); + for mask in [Mask::new_true(256), Mask::from_indices(256, [12, 255])] { + let scan = Scan::try_new( + SESSION.clone(), + plan.clone(), + vec![Split { + rows: 0..256, + mask: mask.clone(), + }], + )? + .with_value_cache(cache.clone()); + let result = drive(&store, scan, delivery(Delivery::Lifo))?; + assert_view(&expected, &(0..256), &mask, result.arrays)?; + assert!( + cache.get(&predicate).is_some(), + "whole predicate results remain reusable" + ); + } + Ok(()) +} + +#[test] +fn dictionary_predicate_cache_fills_new_entries_and_switches_to_dense() -> VortexResult<()> { + let values = VarBinArray::from_iter( + (0..1024).map(|row| (row != 1).then(|| format!("value {row}"))), + DType::Utf8(true.into()), + ) + .into_array(); + let expression = like(root(), lit("%12%")).bind(values.dtype())?; + let cache = PredicateCache::new(1024); + let mut ctx = SESSION.create_execution_ctx(); + for (codes, expected, evaluated) in [ + ( + vec![Some(12u64), Some(12), Some(1), None], + vec![Some(true), Some(true), None, None], + 2, + ), + ( + vec![Some(12), Some(100), Some(1)], + vec![Some(true), Some(false), None], + 3, + ), + ] { + let result = cache.evaluate( + values.clone(), + PrimitiveArray::from_option_iter(codes).into_array(), + &expression, + &mut ctx, + )?; + assert_arrays_eq!(result, BoolArray::from_iter(expected), &mut ctx); + assert_eq!(cache.evaluated_rows(), evaluated); + } + let dense = cache.evaluate( + values.clone(), + Buffer::from_iter(0..1024u64).into_array(), + &expression, + &mut ctx, + )?; + assert_arrays_eq!(dense, values.clone().apply_bound(&expression)?, &mut ctx); + assert_eq!(cache.evaluated_rows(), 1027); + let result = cache.evaluate( + values, + PrimitiveArray::from_option_iter([Some(12u64), Some(100), Some(1), None]).into_array(), + &expression, + &mut ctx, + )?; + assert!(result.is::(), "bulk results retain dictionary reuse"); + assert_arrays_eq!( + result, + BoolArray::from_iter([Some(true), Some(false), None, None]), + &mut ctx + ); + assert_eq!(cache.evaluated_rows(), 1027); + Ok(()) +} + +#[test] +fn dictionary_predicate_cache_is_shared_across_workers() -> VortexResult<()> { + let values = VarBinArray::from_iter( + (0..1024).map(|row| Some(format!("value {row}"))), + DType::Utf8(false.into()), + ) + .into_array(); + let expression = like(root(), lit("%12%")).bind(values.dtype())?; + let cache = std::sync::Arc::new(PredicateCache::new(1024)); + std::thread::scope(|scope| -> VortexResult<()> { + let mut workers = Vec::new(); + for _ in 0..8 { + let values = values.clone(); + let expression = expression.clone(); + let cache = std::sync::Arc::clone(&cache); + workers.push(scope.spawn(move || -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let actual = cache.evaluate( + values, + Buffer::from(vec![12u64, 12, 0, 1023]).into_array(), + &expression, + &mut ctx, + )?; + assert_arrays_eq!( + actual, + BoolArray::from_iter([true, true, false, false]), + &mut ctx + ); + Ok(()) + })); + } + for worker in workers { + worker + .join() + .map_err(|_| vortex_err!("predicate worker panicked"))??; + } + Ok(()) + }) +} + +#[rstest] +fn sparse_dictionary_reads_only_referenced_value_chunks( + #[values(AccessMode::Scan, AccessMode::RandomAccess)] mode: AccessMode, + #[values(false, true)] shared_values: bool, + #[values(Delivery::Fifo, Delivery::Lifo)] order: Delivery, +) -> VortexResult<()> { + let mut store = Store::default(); + let values = + PrimitiveArray::from_option_iter((0..2048i32).map(|row| (row != 1023).then_some(row))) + .into_array(); + let codes = + PrimitiveArray::from_option_iter([Some(0u32), Some(1023), Some(2047), Some(0), None]) + .into_array(); + let values_plan = lower(&store.chunked(&values, &[256; 8])?)?; + let values_plan = if shared_values { + SharePlan::new(values_plan).into_plan() + } else { + values_plan + }; + let codes_plan = lower(&store.flat(&codes)?)?; + let plan = TakePlan::new(codes_plan, values_plan).into_plan(); + let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..5)])? + .with_access_mode(mode); + let result = drive(&store, scan, delivery(order))?; + assert_eq!( + reads(&result.events), + if mode == AccessMode::RandomAccess { + 4 + } else { + 9 + } + ); + assert_arrays_eq!( + join(plan.dtype(), result.arrays)?, + DictArray::try_new(codes, values)?, + &mut SESSION.create_execution_ctx() + ); + Ok(()) +} + +#[test] +fn sparse_dictionary_null_codes_do_not_read_values() -> VortexResult<()> { + let mut store = Store::default(); + let values = PrimitiveArray::from_iter(0..2048i32).into_array(); + let codes = PrimitiveArray::from_option_iter([None::, None, None]).into_array(); + let plan = TakePlan::new( + lower(&store.flat(&codes)?)?, + lower(&store.chunked(&values, &[256; 8])?)?, + ) + .into_plan(); + let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..3)])? + .with_access_mode(AccessMode::RandomAccess); + let result = drive(&store, scan, delivery(Delivery::Lifo))?; + assert_eq!(reads(&result.events), 1); + assert_arrays_eq!( + join(plan.dtype(), result.arrays)?, + PrimitiveArray::from_option_iter([None::, None, None]), + &mut SESSION.create_execution_ctx() + ); + Ok(()) +} + +#[test] +fn sparse_dictionary_multiple_code_batches_preserve_order_and_whole_cache() -> VortexResult<()> { + let mut store = Store::default(); + let values = PrimitiveArray::from_iter(0..131072i32).into_array(); + let codes = + PrimitiveArray::from_option_iter([Some(0u32), Some(131071), None, Some(0), Some(65536)]) + .into_array(); + let values_plan = lower(&store.chunked(&values, &[32768; 4])?)?; + let plan = TakePlan::new( + lower(&store.chunked(&codes, &[2, 3])?)?, + values_plan.clone(), + ) + .into_plan(); + let cache = ValueCache::default(); + for mode in [AccessMode::RandomAccess, AccessMode::Scan] { + let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..5)])? + .with_access_mode(mode) + .with_value_cache(cache.clone()); + let result = drive(&store, scan, delivery(Delivery::Lifo))?; + assert_arrays_eq!( + join(plan.dtype(), result.arrays)?, + DictArray::try_new(codes.clone(), values.clone())?, + &mut SESSION.create_execution_ctx() + ); + assert_eq!(cache.get(&values_plan).is_some(), mode == AccessMode::Scan); + } + Ok(()) +} + +#[test] +fn sparse_dictionary_values_can_themselves_be_dictionaries() -> VortexResult<()> { + let mut store = Store::default(); + let values = PrimitiveArray::from_iter(0..4096i32).into_array(); + let inner_codes = Buffer::from_iter((2048..4096u32).rev()).into_array(); + let inner = TakePlan::new( + lower(&store.chunked(&inner_codes, &[512; 4])?)?, + lower(&store.chunked(&values, &[512; 8])?)?, + ) + .into_plan(); + let codes = + PrimitiveArray::from_option_iter([Some(0u32), Some(1023), Some(2047), Some(0), None]) + .into_array(); + let plan = TakePlan::new(lower(&store.flat(&codes)?)?, inner).into_plan(); + let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..5)])? + .with_access_mode(AccessMode::RandomAccess); + let result = drive(&store, scan, delivery(Delivery::Lifo))?; + assert_arrays_eq!( + join(plan.dtype(), result.arrays)?, + PrimitiveArray::from_option_iter([Some(4095i32), Some(3072), Some(2048), Some(4095), None]), + &mut SESSION.create_execution_ctx() + ); + Ok(()) +} + +#[rstest] +fn sparse_dictionary_flat_values_read_beside_codes( + #[values(Delivery::Fifo, Delivery::Lifo)] order: Delivery, +) -> VortexResult<()> { + let mut store = Store::default(); + let values = PrimitiveArray::from_iter(0..2048i32).into_array(); + let codes = Buffer::from(vec![2047u32, 0, 2047]).into_array(); + let plan = + TakePlan::new(lower(&store.flat(&codes)?)?, lower(&store.flat(&values)?)?).into_plan(); + let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..3)])? + .with_access_mode(AccessMode::RandomAccess); + let result = drive(&store, scan, delivery(order))?; + assert!( + matches!(result.events.first(), Some(Event::Io(batch)) if batch.len() == 2), + "flat values cannot skip stored chunks, so both reads should start together" + ); + assert_arrays_eq!( + join(plan.dtype(), result.arrays)?, + DictArray::try_new(codes, values)?, + &mut SESSION.create_execution_ctx() + ); + Ok(()) +} + +#[rstest] +fn sparse_numeric_predicate_keeps_vectorized_dictionary_evaluation( + #[values(AccessMode::Scan, AccessMode::RandomAccess)] mode: AccessMode, +) -> VortexResult<()> { + let mut store = Store::default(); + let values = PrimitiveArray::from_iter(0..2048i32).into_array(); + let codes = Buffer::from(vec![2047u32, 0, 2047]).into_array(); + let layout = DictLayout::new(store.flat(&values)?, store.flat(&codes)?).into_layout(); + let source = lower(&layout)?; + let expression = eq(root(), lit(2047i32)).bind(source.dtype())?; + let plan = optimize(EvalPlan::try_new(expression, source)?.into_plan(), &SESSION)?; + let predicate = plan.as_::().values()?; + let cache = ValueCache::default(); + let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..3)])? + .with_value_cache(cache.clone()) + .with_access_mode(mode); + let result = drive(&store, scan, delivery(Delivery::Lifo))?; + assert_arrays_eq!( + join(plan.dtype(), result.arrays)?, + BoolArray::from_iter([true, false, true]), + &mut SESSION.create_execution_ctx() + ); + assert!(cache.predicate(&predicate).is_none()); + assert!(cache.get(&predicate).is_some()); + Ok(()) +} diff --git a/vortex-layout/src/plan/physical/pipeline/values.rs b/vortex-layout/src/plan/physical/pipeline/values.rs index 18c2116b637..ddfd7366185 100644 --- a/vortex-layout/src/plan/physical/pipeline/values.rs +++ b/vortex-layout/src/plan/physical/pipeline/values.rs @@ -9,30 +9,50 @@ use parking_lot::Mutex; use rustc_hash::FxHashMap; use vortex_array::ArrayRef; +use super::PredicateCache; use crate::plan::physical::PlanByPtr; use crate::plan::physical::PlanRef; /// Values a plan produces over its whole domain whatever rows it is compiled for, kept for the /// later scans that share this cache. /// -/// A dictionary's values are read whole by every reader of its codes, so readers sharing this -/// cache can reuse them across filters, projections, and splits. The caller controls its lifetime; -/// only whole values that a reader requested are retained. +/// Readers reuse whole dictionary values across filters, projections, and splits. Sparse reads +/// never populate the whole-value cache; static predicates have a separate per-entry cache. +/// The caller controls the lifetime of both caches. #[derive(Clone, Default)] -pub struct ValueCache(Arc>>); +pub struct ValueCache { + values: Arc>>, + predicates: Arc>>>, +} impl ValueCache { /// The values `plan` produced earlier. pub(crate) fn get(&self, plan: &PlanRef) -> Option { - self.0.lock().get(&PlanByPtr(plan.clone())).cloned() + self.values.lock().get(&PlanByPtr(plan.clone())).cloned() } /// Records the values of `plan`, keeping the first when two race. pub(crate) fn insert(&self, plan: &PlanRef, values: ArrayRef) -> ArrayRef { - self.0 + self.values .lock() .entry(PlanByPtr(plan.clone())) .or_insert(values) .clone() } + + pub(crate) fn predicate(&self, plan: &PlanRef) -> Option> { + self.predicates + .lock() + .get(&PlanByPtr(plan.clone())) + .cloned() + } + + pub(crate) fn predicate_or_insert(&self, plan: &PlanRef, len: usize) -> Arc { + Arc::clone( + self.predicates + .lock() + .entry(PlanByPtr(plan.clone())) + .or_insert_with(|| Arc::new(PredicateCache::new(len))), + ) + } } diff --git a/vortex-layout/src/plan/physical/plans/share.rs b/vortex-layout/src/plan/physical/plans/share.rs index 96166e7c81f..5707eeda7df 100644 --- a/vortex-layout/src/plan/physical/plans/share.rs +++ b/vortex-layout/src/plan/physical/plans/share.rs @@ -24,8 +24,10 @@ use crate::plan::physical::check_child_count; use crate::plan::physical::operators::OnceSource; use crate::plan::physical::operators::UNBOUNDED; use crate::plan::physical::operators::WholeSource; +use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::pipeline::Chain; use crate::plan::physical::pipeline::Compiler; +use crate::plan::physical::pipeline::sparse_selection; /// Produces its child's whole value once per split, as a /// [`SharedArray`](crate::arrays::shared::SharedArray) that the split's later scans reuse. @@ -120,6 +122,11 @@ impl PlanVTable for Share { } let child = plan.child_plan()?; let len = usize::try_from(plan.row_count())?; + if compiler.access_mode() == AccessMode::RandomAccess + && sparse_selection(mask.true_count(), len) + { + return compiler.compile(&child, rows, mask); + } let Some(chain) = compiler.compile(&child, 0..plan.row_count(), &Mask::new_true(len))? else { return Ok(None); diff --git a/vortex-layout/src/plan/physical/plans/take.rs b/vortex-layout/src/plan/physical/plans/take.rs index c97c621759c..42b43a91af2 100644 --- a/vortex-layout/src/plan/physical/plans/take.rs +++ b/vortex-layout/src/plan/physical/plans/take.rs @@ -3,6 +3,7 @@ use std::borrow::Cow; use std::ops::Range; +use std::sync::Arc; use vortex_array::Canonical; use vortex_array::EmptyMetadata; @@ -19,6 +20,7 @@ use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_session::registry::CachedId; +use crate::plan::physical::Concat; use crate::plan::physical::Eval; use crate::plan::physical::EvalPlan; use crate::plan::physical::Plan; @@ -30,10 +32,14 @@ use crate::plan::physical::PlanVTable; use crate::plan::physical::Share; use crate::plan::physical::check_child_count; use crate::plan::physical::operators::OnceSource; +use crate::plan::physical::operators::SparseTakeSource; use crate::plan::physical::operators::TakeSource; use crate::plan::physical::optimizer::PlanParentReduceRule; +use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::pipeline::Chain; use crate::plan::physical::pipeline::Compiler; +use crate::plan::physical::pipeline::PredicateCache; +use crate::plan::physical::pipeline::sparse_selection; const CODES: usize = 0; const VALUES: usize = 1; @@ -138,39 +144,113 @@ impl PlanVTable for Take { } } - /// A take whose values an earlier scan of the split produced looks its codes up in them; - /// otherwise it joins its codes with the whole of its values. + /// Reuses whole-value results or chooses sparse predicate evaluation and value reads from + /// the selected code density. fn compile( plan: &Plan, rows: Range, mask: &Mask, compiler: &mut Compiler<'_>, ) -> VortexResult> { - let Some(codes) = compiler.compile(&plan.codes()?, rows, mask)? else { - return Ok(None); - }; - let cache = compiler.values().clone(); - let values_plan = plan.values()?; - if let Some(values) = cache.get(&values_plan) { - let source = TakeSource::new(plan.clone(), Some(values), None, compiler.access_mode()); - return Ok(Some(compiler.join(vec![codes], source))); + let values = plan.values()?; + if let Some(eval) = values.as_opt::() + && eval.expression().dtype().is_boolean() + && (eval.child_plan()?.dtype().is_utf8() || eval.child_plan()?.dtype().is_binary()) + && (mask.true_count().saturating_mul(4) < usize::try_from(values.row_count())? + || compiler.values().predicate(&values).is_some()) + && compiler.values().get(&values).is_none() + && expression_is_strict_and_infallible(eval.expression()) + && !eval.expression().contains::()? + { + let cache = compiler + .values() + .predicate_or_insert(&values, usize::try_from(values.row_count())?); + let take = TakePlan::new(plan.codes()?, eval.child_plan()?); + return compile_take( + &take, + rows, + mask, + compiler, + Some((eval.expression().clone(), cache)), + ); } - let values = values_plan; - let values_rows = 0..values.row_count(); - let len = usize::try_from(values_rows.end)?; - let values = match compiler.compile(&values, values_rows, &Mask::new_true(len))? { - Some(values) => values, - None => Chain::new(OnceSource::new( - Canonical::empty(values.dtype()).into_array(), - )), - }; - let source = TakeSource::new(plan.clone(), None, Some(cache), compiler.access_mode()); - Ok(Some(compiler.join_with( - vec![codes, values], - TakeSource::capacity, - source, - ))) + compile_take(plan, rows, mask, compiler, None) + } +} + +/// Chooses cached, predicate-aware, or staged dictionary retrieval for the selected codes. +fn compile_take( + plan: &TakePlan, + rows: Range, + mask: &Mask, + compiler: &mut Compiler<'_>, + predicate: Option<(BoundExpression, Arc)>, +) -> VortexResult> { + let Some(codes) = compiler.compile(&plan.codes()?, rows, mask)? else { + return Ok(None); + }; + let cache = compiler.values().clone(); + let values_plan = plan.values()?; + let cached = cache.get(&values_plan); + if predicate.is_none() + && cached.is_none() + && compiler.access_mode() == AccessMode::RandomAccess + && sparse_selection(mask.true_count(), usize::try_from(values_plan.row_count())?) + && has_value_chunks(&values_plan)? + { + let source = SparseTakeSource::new(plan.clone(), cache, compiler.row_offset()); + return Ok(Some(compiler.join(vec![codes], source))); + } + let mut source = TakeSource::new( + plan.clone(), + cached.clone(), + cached.is_none().then_some(cache), + compiler.access_mode(), + ); + if let Some((expression, cache)) = predicate { + source = source.with_predicate(expression, cache); } + if cached.is_some() { + return Ok(Some(compiler.join(vec![codes], source))); + } + let values_rows = 0..values_plan.row_count(); + let len = usize::try_from(values_rows.end)?; + let values = match compiler.compile(&values_plan, values_rows, &Mask::new_true(len))? { + Some(values) => values, + None => Chain::new(OnceSource::new( + Canonical::empty(values_plan.dtype()).into_array(), + )), + }; + Ok(Some(compiler.join_with( + vec![codes, values], + TakeSource::capacity, + source, + ))) +} + +/// A second read stage only pays when demand can eliminate stored value chunks. A flat value +/// segment still has to be fetched whole, and is cheaper to read beside the codes in one batch. +fn has_value_chunks(plan: &PlanRef) -> VortexResult { + if plan.is::() && plan.children().len() > 1 { + return Ok(true); + } + if plan.children().len() == 1 + && let Some(child) = plan.child(0)? + && child.row_count() == plan.row_count() + { + return has_value_chunks(&child); + } + Ok(false) +} + +fn expression_is_strict_and_infallible(expression: &BoundExpression) -> bool { + expression + .as_scalar() + .is_none_or(|scalar| scalar.signature().is_strict() && scalar.signature().is_infallible()) + && expression + .children() + .iter() + .all(expression_is_strict_and_infallible) } /// Pushes an expression, or the part of it that reads the values, onto the dictionary values of diff --git a/vortex-layout/src/scan/execution/filter.rs b/vortex-layout/src/scan/execution/filter.rs index 3a7596ee356..f9a28cf05fc 100644 --- a/vortex-layout/src/scan/execution/filter.rs +++ b/vortex-layout/src/scan/execution/filter.rs @@ -44,6 +44,7 @@ use crate::plan::physical::PlanRef; use crate::plan::physical::SegmentScan; use crate::plan::physical::Zoned; use crate::plan::physical::optimize; +use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::pipeline::Scan; use crate::plan::physical::pipeline::Split; use crate::plan::physical::plan_row_idx_expression; @@ -71,11 +72,12 @@ impl FilterPlans { .conjuncts() .iter() .map(|expr| { - let plan = optimize( + // Leaf filters already apply the demand. Wrapping the predicate in another + // Filter would reset it to all rows before evaluating the expression. + optimize( plan_row_idx_expression(expr.clone(), root.clone())?, session, - )?; - Ok(FilterPlan::new(plan).into_plan()) + ) }) .collect::>()?; let pruning = expression @@ -190,6 +192,17 @@ impl ActiveConjunct { .as_ref() .is_some_and(|filter| filter.predicates.len() > 1); let started = (learn || tracing::enabled!(tracing::Level::DEBUG)).then(Instant::now); + // Dense demand keeps vectorized evaluation cheaper than gathering almost every row. + // Random access and selective predicates retain demand all the way to their leaves. + let plan = if matches!(evaluation, Evaluation::Filter { .. }) + && scan.access_mode() == AccessMode::Scan + && !selected.mask.all_true() + && selected.mask.true_count().saturating_mul(2) >= selected.mask.len() + { + FilterPlan::new(plan).into_plan() + } else { + plan + }; let pipeline = Scan::try_new( scan.session.clone(), plan, @@ -381,8 +394,8 @@ impl FilterPlanner { } self.pieces.clear(); if matches!(active.evaluation, Evaluation::Prune) && !selected.mask.all_false() { - // Filter plans evaluate their entire row domain before selecting their result. - // Trim newly excluded edge zones so they never enter that domain or its prefetch. + // Trim newly excluded edge zones before compiling another predicate or prefetching + // its data, including dictionary values living in a separate row domain. let first = selected.mask.first().vortex_expect("non-empty mask"); let end = selected.mask.last().vortex_expect("non-empty mask") + 1; selected.scope.rows = diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index f0f1804dde8..3c926085bbb 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -614,6 +614,44 @@ fn smaller_chunks_keep_file_alignment() { ); } +#[rstest] +#[case::sparse(vec![1u64, 3001], 2)] +#[case::dense((0..4000u64).filter(|row| row % 10 != 0).collect(), 4)] +#[tokio::test] +async fn predicate_only_reads_selected_chunks( + #[values(AccessMode::Scan, AccessMode::RandomAccess)] mode: AccessMode, + #[case] indices: Vec, + #[case] expected_reads: usize, +) -> VortexResult<()> { + let session = new_session().with_tokio(); + let (segments, layout) = write(&session, table().as_ref()).await?; + let counted = Arc::new(CountingSegmentSource::new(segments)); + let segments = Arc::clone(&counted) as Arc; + let expected = Buffer::from_iter(indices.iter().copied().filter(|row| row % 3 != 0)); + let selection = vortex_scan::selection::Selection::IncludeByIndex(StrictSortedBuffer::try_new( + Buffer::from(indices), + )?); + let scan = builder(&session, &segments, &layout)? + .with_access_mode(mode) + .with_selection(selection) + .with_projection(row_idx().bind(layout.dtype())?) + .with_some_filter(Some( + gt(get_item("strings", root()), lit("")).bind(layout.dtype())?, + )); + let actual = collect_scan( + DType::Primitive(vortex_array::dtype::PType::U64, false.into()), + scan, + ) + .await?; + assert_arrays_eq!( + actual, + expected.into_array(), + &mut session.create_execution_ctx() + ); + assert_eq!(counted.requests(), expected_reads); + Ok(()) +} + #[tokio::test] async fn dictionary_values_are_reused_between_splits() -> VortexResult<()> { let session = new_session().with_tokio(); @@ -1171,10 +1209,7 @@ async fn dynamic_threshold_reprunes_a_queued_chunk( ) -> VortexResult<()> { let session = new_session().with_tokio(); let (segments, layout) = write(&session, zoned_table().as_ref()).await?; - let counted = Arc::new(CountingSegments { - inner: segments, - reads: AtomicUsize::new(0), - }); + let counted = Arc::new(CountingSegmentSource::new(segments)); let segments = Arc::clone(&counted) as Arc; let threshold = Arc::new(AtomicI32::new(i32::MAX)); let bound = Arc::clone(&threshold); @@ -1220,7 +1255,7 @@ async fn dynamic_threshold_reprunes_a_queued_chunk( _ => vortex_bail!("expected pruning to emit a chunk"), } }; - let before = counted.reads.load(Ordering::Relaxed); + let before = counted.requests(); threshold.store(500, Ordering::Relaxed); run.admit(chunk, scope, Arc::new(DemandOnlyIo(io.session()))); let mut arrays = Vec::new(); @@ -1244,7 +1279,7 @@ async fn dynamic_threshold_reprunes_a_queued_chunk( &mut session.create_execution_ctx() ); assert_eq!( - counted.reads.load(Ordering::Relaxed) - before, + counted.requests() - before, expected_reads, "tightened zones must be excluded before reading data" ); From 0679729bd6a93ba05fe3b1d59f8bf8b77bb5c0ba Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 09:46:46 +0000 Subject: [PATCH 04/28] Simplify Scan V2 dictionary retrieval to one execution path Signed-off-by: Joe Isaacs --- vortex-datafusion/src/persistent/opener.rs | 30 ++- .../src/plan/physical/operators/mod.rs | 2 - .../plan/physical/operators/take_sparse.rs | 224 ------------------ .../src/plan/physical/pipeline/tests.rs | 37 +-- .../src/plan/physical/pipeline/values.rs | 5 +- .../src/plan/physical/plans/share.rs | 7 - vortex-layout/src/plan/physical/plans/take.rs | 33 +-- vortex-layout/src/scan/v2/mod.rs | 5 + 8 files changed, 33 insertions(+), 310 deletions(-) delete mode 100644 vortex-layout/src/plan/physical/operators/take_sparse.rs diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 83e69bbaf8a..e5771db681c 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -1142,10 +1142,14 @@ mod tests { Ok(summary.size()) } + #[rstest] #[tokio::test] - async fn startup_and_bulk_keep_filtered_selected_rows_once() -> anyhow::Result<()> { + async fn startup_and_bulk_keep_filtered_selected_rows_once( + #[values(0, 2)] initial_chunks: usize, + #[values(false, true)] roaring: bool, + ) -> anyhow::Result<()> { let object_store = Arc::new(InMemory::new()) as Arc; - let batch = record_batch!(("a", Int32, (0..30_000).collect::>()))?; + let batch = record_batch!(("a", Int32, (0..150_000).collect::>()))?; let schema = batch.schema(); let size = write_arrow_to_vortex(Arc::clone(&object_store), "startup.vortex", batch).await?; @@ -1153,19 +1157,25 @@ mod tests { let filter = logical2physical(&col("a").gt(lit(10)), table_schema.table_schema()); let mut opener = make_opener(object_store, table_schema, Some(filter)); opener.scan_v2 = true; - opener.initial_chunks = Arc::new(AtomicUsize::new(2)); - let indices = StrictSortedBuffer::try_new(Buffer::from(vec![ - 3, 17, 8188, 8192, 16_390, 20_000, 29_999, - ]))?; - let file = PartitionedFile::new("startup.vortex".to_string(), size).with_extension( - VortexAccessPlan::default().with_selection(Selection::IncludeByIndex(indices)), - ); + opener.initial_chunks = Arc::new(AtomicUsize::new(initial_chunks)); + let indices = vec![3, 17, 8188, 8192, 16_390, 65_535, 65_536, 140_000, 149_999]; + let selection = if roaring { + Selection::IncludeRoaring(indices.into_iter().collect()) + } else { + Selection::IncludeByIndex(StrictSortedBuffer::try_new(Buffer::from(indices))?) + }; + let file = PartitionedFile::new("startup.vortex".to_string(), size) + .with_extension(VortexAccessPlan::default().with_selection(selection)); let batches = opener.open(file)?.await?.try_collect::>().await?; assert_eq!(opener.initial_chunks.load(Ordering::Relaxed), 0); let actual = concat_batches(&schema, &batches)?; let actual = RecordBatch::try_new(Arc::clone(&schema), vec![sort(actual.column(0), None)?])?; - let expected = record_batch!(("a", Int32, [17, 8188, 8192, 16_390, 20_000, 29_999]))?; + let expected = record_batch!(( + "a", + Int32, + [17, 8188, 8192, 16_390, 65_535, 65_536, 140_000, 149_999] + ))?; assert_arrays_eq!( SESSION.arrow().from_arrow_record_batch(actual, &schema)?, SESSION.arrow().from_arrow_record_batch(expected, &schema)?, diff --git a/vortex-layout/src/plan/physical/operators/mod.rs b/vortex-layout/src/plan/physical/operators/mod.rs index 990eb3215e2..b0f5937d71d 100644 --- a/vortex-layout/src/plan/physical/operators/mod.rs +++ b/vortex-layout/src/plan/physical/operators/mod.rs @@ -13,7 +13,6 @@ mod once; mod pack; mod scan; mod take; -mod take_sparse; mod whole; pub(crate) use concat::*; @@ -24,7 +23,6 @@ pub(crate) use pack::*; pub(crate) use scan::*; use smallvec::SmallVec; pub(crate) use take::*; -pub(crate) use take_sparse::*; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; diff --git a/vortex-layout/src/plan/physical/operators/take_sparse.rs b/vortex-layout/src/plan/physical/operators/take_sparse.rs deleted file mode 100644 index 07476853192..00000000000 --- a/vortex-layout/src/plan/physical/operators/take_sparse.rs +++ /dev/null @@ -1,224 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Read dictionary codes first, then request only the referenced regions of the values plan. - -use std::collections::VecDeque; - -use rustc_hash::FxHashMap; -use vortex_array::ArrayRef; -use vortex_array::Canonical; -use vortex_array::IntoArray; -use vortex_array::arrays::ConstantArray; -use vortex_array::arrays::DictArray; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::builtins::ArrayBuiltins; -use vortex_array::dtype::DType; -use vortex_array::dtype::PType; -use vortex_array::scalar::Scalar; -use vortex_array::validity::Validity; -use vortex_buffer::Buffer; -use vortex_error::VortexExpect; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_error::vortex_ensure_eq; -use vortex_error::vortex_err; -use vortex_mask::AllOr; -use vortex_mask::Mask; - -use super::join; -use crate::plan::physical::TakePlan; -use crate::plan::physical::operator::Cx; -use crate::plan::physical::operator::Input; -use crate::plan::physical::operator::Operator; -use crate::plan::physical::operator::Source; -use crate::plan::physical::operator::SourceBlocked; -use crate::plan::physical::operator::SourceReady; -use crate::plan::physical::operator::Step; -use crate::plan::physical::pipeline::AccessMode; -use crate::plan::physical::pipeline::ReadId; -use crate::plan::physical::pipeline::Scan; -use crate::plan::physical::pipeline::Split; -use crate::plan::physical::pipeline::Turn; -use crate::plan::physical::pipeline::ValueCache; -use crate::segments::SegmentId; - -const VALUE_BLOCK_ROWS: u64 = 65_536; - -pub(crate) struct SparseTakeSource { - plan: TakePlan, - cache: ValueCache, - row_offset: u64, - active: Option, - output: Option, -} - -struct ValueScan { - scan: Scan, - codes: ArrayRef, - pieces: Vec<(usize, ArrayRef)>, - pending: FxHashMap>, - selected: usize, - reads: usize, -} - -impl SparseTakeSource { - pub(crate) fn new(plan: TakePlan, cache: ValueCache, row_offset: u64) -> Self { - Self { - plan, - cache, - row_offset, - active: None, - output: None, - } - } - - fn start(&mut self, codes: ArrayRef, cx: &mut Cx<'_>) -> VortexResult<()> { - let values = self.plan.values()?; - let codes = codes - .cast(DType::Primitive(PType::U64, codes.dtype().nullability()))? - .execute::(cx.exec())?; - let valid = codes.validity()?.execute_mask(codes.len(), cx.exec())?; - let original = codes.as_slice::(); - let mut selected = Vec::new(); - let mut select = |row: usize| -> VortexResult<()> { - vortex_ensure!( - original[row] < values.row_count(), - "Dictionary code out of bounds" - ); - selected.push(original[row]); - Ok(()) - }; - match valid.indices() { - AllOr::All => (0..codes.len()).try_for_each(&mut select)?, - AllOr::Some(rows) => rows.iter().copied().try_for_each(&mut select)?, - AllOr::None => {} - } - selected.sort_unstable(); - selected.dedup(); - if selected.is_empty() { - self.output = Some(if codes.is_empty() { - Canonical::empty(self.plan.dtype()).into_array() - } else { - ConstantArray::new(Scalar::null(self.plan.dtype().clone()), codes.len()) - .into_array() - }); - return Ok(()); - } - let mut remapped = vec![0u64; codes.len()]; - let mut remap = |row: usize| { - remapped[row] = selected - .binary_search(&original[row]) - .ok() - .vortex_expect("selected codes include every valid code") - as u64; - }; - match valid.indices() { - AllOr::All => (0..codes.len()).for_each(&mut remap), - AllOr::Some(rows) => rows.iter().copied().for_each(&mut remap), - AllOr::None => {} - } - let remapped = PrimitiveArray::new( - Buffer::from(remapped), - Validity::from_mask(valid, codes.dtype().nullability()), - ) - .into_array(); - let splits = selected - .chunk_by(|left, right| left / VALUE_BLOCK_ROWS == right / VALUE_BLOCK_ROWS) - .map(|group| { - let first = group[0]; - let end = group[group.len() - 1] + 1; - Ok(Split { - rows: first..end, - mask: Mask::from_indices( - usize::try_from(end - first)?, - group.iter().map(|&row| { - usize::try_from(row - first) - .vortex_expect("value block indices fit in usize") - }), - ), - }) - }) - .collect::>>()?; - self.active = Some(ValueScan { - scan: Scan::try_new(cx.session().clone(), values, splits)? - .with_value_cache(self.cache.clone()) - .with_access_mode(AccessMode::RandomAccess) - .with_row_offset(self.row_offset), - codes: remapped, - pieces: Vec::new(), - pending: FxHashMap::default(), - selected: selected.len(), - reads: 0, - }); - Ok(()) - } -} - -impl Source for SparseTakeSource { - fn ready(&mut self, cx: &mut Cx<'_>) -> VortexResult { - if self.output.is_some() { - return Ok(SourceReady::Ready); - } - if self.active.is_none() { - match cx.inlet(0).take() { - Some(codes) => self.start(codes, cx)?, - None if cx.inlet(0).closed() => return Ok(SourceReady::Ready), - None => return Ok(SourceReady::Blocked(SourceBlocked::Inlet(0))), - } - } - if self.output.is_some() { - return Ok(SourceReady::Ready); - } - let active = self.active.as_mut().vortex_expect("value scan started"); - while let Some((segment, bytes)) = cx.take_segment() { - let read = active - .pending - .get_mut(&segment) - .and_then(VecDeque::pop_front) - .ok_or_else(|| vortex_err!("Unrequested dictionary values segment {segment}"))?; - active.scan.deliver(read, bytes)?; - } - loop { - match active.scan.step()? { - Turn::Read(read) => { - active - .pending - .entry(read.segment_id) - .or_default() - .push_back(read.id); - active.reads += 1; - cx.request_read(read.segment_id); - } - Turn::Output(split, array) => active.pieces.push((split, array)), - Turn::Waiting => return Ok(SourceReady::Blocked(SourceBlocked::Io)), - Turn::Done => { - let mut active = self.active.take().vortex_expect("value scan exists"); - active.pieces.sort_by_key(|(split, _)| *split); - let values = join( - self.plan.values()?.dtype(), - active.pieces.into_iter().map(|(_, array)| array).collect(), - )?; - vortex_ensure_eq!(values.len(), active.selected); - tracing::debug!( - selected_values = active.selected, - dictionary_values = self.plan.values()?.row_count(), - value_reads = active.reads, - "v2 selected dictionary reads" - ); - self.output = Some(DictArray::try_new(active.codes, values)?.into_array()); - return Ok(SourceReady::Ready); - } - } - } - } -} - -impl Operator for SparseTakeSource { - fn compute(&mut self, _input: Input, _cx: &mut Cx<'_>) -> VortexResult { - Ok(match self.output.take() { - Some(array) => Step::Final(array), - None => Step::End, - }) - } -} diff --git a/vortex-layout/src/plan/physical/pipeline/tests.rs b/vortex-layout/src/plan/physical/pipeline/tests.rs index 0de57d57a1e..8b88d84793a 100644 --- a/vortex-layout/src/plan/physical/pipeline/tests.rs +++ b/vortex-layout/src/plan/physical/pipeline/tests.rs @@ -1198,7 +1198,7 @@ fn dictionary_predicate_cache_is_shared_across_workers() -> VortexResult<()> { } #[rstest] -fn sparse_dictionary_reads_only_referenced_value_chunks( +fn sparse_dictionary_reuses_bulk_values( #[values(AccessMode::Scan, AccessMode::RandomAccess)] mode: AccessMode, #[values(false, true)] shared_values: bool, #[values(Delivery::Fifo, Delivery::Lifo)] order: Delivery, @@ -1221,14 +1221,7 @@ fn sparse_dictionary_reads_only_referenced_value_chunks( let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..5)])? .with_access_mode(mode); let result = drive(&store, scan, delivery(order))?; - assert_eq!( - reads(&result.events), - if mode == AccessMode::RandomAccess { - 4 - } else { - 9 - } - ); + assert_eq!(reads(&result.events), 9); assert_arrays_eq!( join(plan.dtype(), result.arrays)?, DictArray::try_new(codes, values)?, @@ -1238,29 +1231,7 @@ fn sparse_dictionary_reads_only_referenced_value_chunks( } #[test] -fn sparse_dictionary_null_codes_do_not_read_values() -> VortexResult<()> { - let mut store = Store::default(); - let values = PrimitiveArray::from_iter(0..2048i32).into_array(); - let codes = PrimitiveArray::from_option_iter([None::, None, None]).into_array(); - let plan = TakePlan::new( - lower(&store.flat(&codes)?)?, - lower(&store.chunked(&values, &[256; 8])?)?, - ) - .into_plan(); - let scan = Scan::try_new(SESSION.clone(), plan.clone(), vec![Split::all(0..3)])? - .with_access_mode(AccessMode::RandomAccess); - let result = drive(&store, scan, delivery(Delivery::Lifo))?; - assert_eq!(reads(&result.events), 1); - assert_arrays_eq!( - join(plan.dtype(), result.arrays)?, - PrimitiveArray::from_option_iter([None::, None, None]), - &mut SESSION.create_execution_ctx() - ); - Ok(()) -} - -#[test] -fn sparse_dictionary_multiple_code_batches_preserve_order_and_whole_cache() -> VortexResult<()> { +fn dictionary_code_batches_preserve_order_and_share_whole_values() -> VortexResult<()> { let mut store = Store::default(); let values = PrimitiveArray::from_iter(0..131072i32).into_array(); let codes = @@ -1283,7 +1254,7 @@ fn sparse_dictionary_multiple_code_batches_preserve_order_and_whole_cache() -> V DictArray::try_new(codes.clone(), values.clone())?, &mut SESSION.create_execution_ctx() ); - assert_eq!(cache.get(&values_plan).is_some(), mode == AccessMode::Scan); + assert!(cache.get(&values_plan).is_some()); } Ok(()) } diff --git a/vortex-layout/src/plan/physical/pipeline/values.rs b/vortex-layout/src/plan/physical/pipeline/values.rs index ddfd7366185..bb847b0fe0d 100644 --- a/vortex-layout/src/plan/physical/pipeline/values.rs +++ b/vortex-layout/src/plan/physical/pipeline/values.rs @@ -16,9 +16,8 @@ use crate::plan::physical::PlanRef; /// Values a plan produces over its whole domain whatever rows it is compiled for, kept for the /// later scans that share this cache. /// -/// Readers reuse whole dictionary values across filters, projections, and splits. Sparse reads -/// never populate the whole-value cache; static predicates have a separate per-entry cache. -/// The caller controls the lifetime of both caches. +/// Readers reuse whole dictionary values across filters, projections, and splits. Sparse static +/// predicates use a separate per-entry cache. The caller controls the lifetime of both caches. #[derive(Clone, Default)] pub struct ValueCache { values: Arc>>, diff --git a/vortex-layout/src/plan/physical/plans/share.rs b/vortex-layout/src/plan/physical/plans/share.rs index 5707eeda7df..96166e7c81f 100644 --- a/vortex-layout/src/plan/physical/plans/share.rs +++ b/vortex-layout/src/plan/physical/plans/share.rs @@ -24,10 +24,8 @@ use crate::plan::physical::check_child_count; use crate::plan::physical::operators::OnceSource; use crate::plan::physical::operators::UNBOUNDED; use crate::plan::physical::operators::WholeSource; -use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::pipeline::Chain; use crate::plan::physical::pipeline::Compiler; -use crate::plan::physical::pipeline::sparse_selection; /// Produces its child's whole value once per split, as a /// [`SharedArray`](crate::arrays::shared::SharedArray) that the split's later scans reuse. @@ -122,11 +120,6 @@ impl PlanVTable for Share { } let child = plan.child_plan()?; let len = usize::try_from(plan.row_count())?; - if compiler.access_mode() == AccessMode::RandomAccess - && sparse_selection(mask.true_count(), len) - { - return compiler.compile(&child, rows, mask); - } let Some(chain) = compiler.compile(&child, 0..plan.row_count(), &Mask::new_true(len))? else { return Ok(None); diff --git a/vortex-layout/src/plan/physical/plans/take.rs b/vortex-layout/src/plan/physical/plans/take.rs index 42b43a91af2..93c4936680e 100644 --- a/vortex-layout/src/plan/physical/plans/take.rs +++ b/vortex-layout/src/plan/physical/plans/take.rs @@ -20,7 +20,6 @@ use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::plan::physical::Concat; use crate::plan::physical::Eval; use crate::plan::physical::EvalPlan; use crate::plan::physical::Plan; @@ -32,14 +31,11 @@ use crate::plan::physical::PlanVTable; use crate::plan::physical::Share; use crate::plan::physical::check_child_count; use crate::plan::physical::operators::OnceSource; -use crate::plan::physical::operators::SparseTakeSource; use crate::plan::physical::operators::TakeSource; use crate::plan::physical::optimizer::PlanParentReduceRule; -use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::pipeline::Chain; use crate::plan::physical::pipeline::Compiler; use crate::plan::physical::pipeline::PredicateCache; -use crate::plan::physical::pipeline::sparse_selection; const CODES: usize = 0; const VALUES: usize = 1; @@ -144,8 +140,7 @@ impl PlanVTable for Take { } } - /// Reuses whole-value results or chooses sparse predicate evaluation and value reads from - /// the selected code density. + /// Reuses whole-value results or evaluates a static string predicate over selected values. fn compile( plan: &Plan, rows: Range, @@ -178,7 +173,7 @@ impl PlanVTable for Take { } } -/// Chooses cached, predicate-aware, or staged dictionary retrieval for the selected codes. +/// Reads codes and reusable dictionary values together, applying sparse predicates after gather. fn compile_take( plan: &TakePlan, rows: Range, @@ -192,15 +187,6 @@ fn compile_take( let cache = compiler.values().clone(); let values_plan = plan.values()?; let cached = cache.get(&values_plan); - if predicate.is_none() - && cached.is_none() - && compiler.access_mode() == AccessMode::RandomAccess - && sparse_selection(mask.true_count(), usize::try_from(values_plan.row_count())?) - && has_value_chunks(&values_plan)? - { - let source = SparseTakeSource::new(plan.clone(), cache, compiler.row_offset()); - return Ok(Some(compiler.join(vec![codes], source))); - } let mut source = TakeSource::new( plan.clone(), cached.clone(), @@ -228,21 +214,6 @@ fn compile_take( ))) } -/// A second read stage only pays when demand can eliminate stored value chunks. A flat value -/// segment still has to be fetched whole, and is cheaper to read beside the codes in one batch. -fn has_value_chunks(plan: &PlanRef) -> VortexResult { - if plan.is::() && plan.children().len() > 1 { - return Ok(true); - } - if plan.children().len() == 1 - && let Some(child) = plan.child(0)? - && child.row_count() == plan.row_count() - { - return has_value_chunks(&child); - } - Ok(false) -} - fn expression_is_strict_and_infallible(expression: &BoundExpression) -> bool { expression .as_scalar() diff --git a/vortex-layout/src/scan/v2/mod.rs b/vortex-layout/src/scan/v2/mod.rs index 0e2ec4f6ba5..cd240c1809b 100644 --- a/vortex-layout/src/scan/v2/mod.rs +++ b/vortex-layout/src/scan/v2/mod.rs @@ -24,6 +24,11 @@ //! Dense inputs retain [`AccessMode::Scan`]. [`ScanBuilder::with_access_mode`] overrides the //! automatic choice. The access mode does not change the rows selected or their order. //! +//! Engine drivers share these selection and predicate policies: sparse string predicates gather +//! referenced dictionary entries and cache their results, while dense scans and numeric +//! dictionary predicates retain bulk evaluation. Drivers supply scheduling and IO, rather than +//! their own selection heuristics. +//! //! A [`ScanFile`] carrying a [`ScanCache`] shares its lowered plan, filter plans and pruned //! zones, segment index, dictionary values, and recently read segments with every other scan of //! that file, like a cached layout reader does for V1 scans. From f01efc1204a38886e45f4d17bbbe87ba83edf06c Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 14:05:23 +0000 Subject: [PATCH 05/28] Default DuckDB to Scan V2 and size work using file parallelism Signed-off-by: Joe Isaacs --- vortex-duckdb/cpp/include/table_function.h | 2 ++ vortex-duckdb/cpp/multi_file_reader.cpp | 4 ++++ vortex-duckdb/src/e2e_test/vortex_scan_test.rs | 6 ++++-- vortex-duckdb/src/file_reader.rs | 1 + vortex-duckdb/src/lib.rs | 4 ++-- vortex-duckdb/src/table_function.rs | 3 +++ vortex-layout/src/scan/execution/mod.rs | 6 ++++++ 7 files changed, 22 insertions(+), 4 deletions(-) diff --git a/vortex-duckdb/cpp/include/table_function.h b/vortex-duckdb/cpp/include/table_function.h index af4951977ce..c0fcd8ffa5e 100644 --- a/vortex-duckdb/cpp/include/table_function.h +++ b/vortex-duckdb/cpp/include/table_function.h @@ -25,6 +25,8 @@ void duckdb_vx_string_map_insert(duckdb_vx_string_map map, const char *key, cons // Input data passed into the init_global and init_local callbacks. typedef struct { const void *bind_data; + size_t file_count; + size_t max_threads; /** * Projected columns that are requested to be read. These are not diff --git a/vortex-duckdb/cpp/multi_file_reader.cpp b/vortex-duckdb/cpp/multi_file_reader.cpp index 24ccbb29a55..8ef12657d91 100644 --- a/vortex-duckdb/cpp/multi_file_reader.cpp +++ b/vortex-duckdb/cpp/multi_file_reader.cpp @@ -9,6 +9,7 @@ #include "vortex.h" #include "duckdb/execution/operator/scan/physical_table_scan.hpp" +#include "duckdb/parallel/task_scheduler.hpp" unique_ptr VortexBindData::Copy() const { auto result = make_uniq(); @@ -173,8 +174,11 @@ VortexReaderInterface::InitializeGlobalState(ClientContext &context, } void *const ffi_bind = bind.ffi_bind_data->DataPtr(); + const auto threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); duckdb_vx_tfunc_init_input ffi_input = { .bind_data = ffi_bind, + .file_count = bind_data.file_list->GetFileCount(threads).count, + .max_threads = threads, .column_ids = column_ids.data(), .column_ids_count = column_ids.size(), .projection_ids = projection_ids, diff --git a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs index 55e5b44bab0..9e8cd167093 100644 --- a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs +++ b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs @@ -1426,11 +1426,12 @@ fn test_vortex_scan_v2_matches_v1(#[case] query: &str) -> Result<()> { )); let path = file.path().to_string_lossy(); + let previous = crate::scan_v2(); crate::set_scan_v2(false); let v1 = query_i64_column(&path, query)?; crate::set_scan_v2(true); let v2 = query_i64_column(&path, query); - crate::set_scan_v2(false); + crate::set_scan_v2(previous); assert_eq!(v2?, v1); Ok(()) @@ -1512,11 +1513,12 @@ fn test_vortex_scan_v2_multifile_row_ids( } Ok(values) }; + let previous = crate::scan_v2(); crate::set_scan_v2(false); let v1 = read(false)?; crate::set_scan_v2(true); let v2 = read(true); - crate::set_scan_v2(false); + crate::set_scan_v2(previous); assert_eq!(v2?, v1); if let Some(expected) = expected { assert_eq!(v1, expected); diff --git a/vortex-duckdb/src/file_reader.rs b/vortex-duckdb/src/file_reader.rs index b4b6398aea2..ea30a45eb2d 100644 --- a/vortex-duckdb/src/file_reader.rs +++ b/vortex-duckdb/src/file_reader.rs @@ -264,6 +264,7 @@ pub fn reader_initialize( let scan = FileScan::try_new( v2_builder(state, global, selection), &global.initial_chunks, + global.parallel_files, global.scan_diagnostics.clone(), &state.path, )?; diff --git a/vortex-duckdb/src/lib.rs b/vortex-duckdb/src/lib.rs index c5364f7905d..17b5fa25ec3 100644 --- a/vortex-duckdb/src/lib.rs +++ b/vortex-duckdb/src/lib.rs @@ -67,9 +67,9 @@ static SESSION: LazyLock = LazyLock::new(|| { session }); -/// Whether scans run on the planner-driven V2 executor. Set `VORTEX_SCAN_V2=1` to enable it. +/// Whether scans run on the planner-driven V2 executor. Set `VORTEX_SCAN_V2=0` to use V1. static SCAN_V2: LazyLock = - LazyLock::new(|| AtomicBool::new(std::env::var("VORTEX_SCAN_V2").is_ok_and(|v| v == "1"))); + LazyLock::new(|| AtomicBool::new(!std::env::var("VORTEX_SCAN_V2").is_ok_and(|v| v == "0"))); /// Selects the planner-driven V2 scan executor for scans initialized from now on. pub fn set_scan_v2(enabled: bool) { diff --git a/vortex-duckdb/src/table_function.rs b/vortex-duckdb/src/table_function.rs index 9f13c73f2bf..0557491588b 100644 --- a/vortex-duckdb/src/table_function.rs +++ b/vortex-duckdb/src/table_function.rs @@ -190,6 +190,8 @@ pub struct GlobalState { pub(crate) row_id_field: Option, /// Whether the scan runs on the planner-driven V2 executor. pub scan_v2: bool, + /// File parallelism already supplies one scan per worker, allowing larger in-file chunks. + pub(crate) parallel_files: bool, pub(crate) initial_chunks: AtomicUsize, pub(crate) scan_diagnostics: Option>, @@ -377,6 +379,7 @@ pub fn init_global(init_input: &TableInitInput) -> VortexResult { file_row_number_column_pos, row_id_field, scan_v2: crate::scan_v2(), + parallel_files: init_input.input.file_count >= init_input.input.max_threads.max(1), }) } diff --git a/vortex-layout/src/scan/execution/mod.rs b/vortex-layout/src/scan/execution/mod.rs index 844e902ee18..6252257b419 100644 --- a/vortex-layout/src/scan/execution/mod.rs +++ b/vortex-layout/src/scan/execution/mod.rs @@ -123,6 +123,12 @@ impl PreparedScan { self.access_mode } + /// Preferred rows per compute chunk. Drivers should retain this granularity when dividing + /// a file into work, so small scheduling partitions do not repeat chunk setup and decoding. + pub fn chunk_rows(&self) -> usize { + self.chunk_rows + } + /// The filter's plans, when the scan has a filter. pub fn filter(&self) -> Option<&FilterPlans> { self.filter.as_deref() From f94b70a37a73eee75ef7f767f5d171187960874b Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 14:49:32 +0000 Subject: [PATCH 06/28] Bind DuckDB file scans to their stored column order Signed-off-by: Joe Isaacs --- vortex-duckdb/src/file_reader.rs | 64 ++++++++++++++++--- .../slt/clickbench/duckdb/plans/q23.slt.no | 8 +-- .../slt/clickbench/duckdb/plans/q24.slt.no | 8 +-- .../duckdb/agg_partition_stats_multi_file.slt | 11 ++++ 4 files changed, 75 insertions(+), 16 deletions(-) diff --git a/vortex-duckdb/src/file_reader.rs b/vortex-duckdb/src/file_reader.rs index ea30a45eb2d..71d176d0b56 100644 --- a/vortex-duckdb/src/file_reader.rs +++ b/vortex-duckdb/src/file_reader.rs @@ -13,11 +13,18 @@ use url::Url; use vortex::array::ArrayRef; use vortex::array::VortexSessionExecute as _; use vortex::array::arrays::struct_::StructArrayExt as _; +use vortex::dtype::DType; use vortex::dtype::FieldPath; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_err; use vortex::error::vortex_panic; +use vortex::expr::BoundExpression; +use vortex::expr::root; +use vortex::expr::select; +use vortex::expr::traversal::NodeExt; +use vortex::expr::traversal::Transformed; +use vortex::expr::traversal::TraversalOrder; use vortex::file::Footer; use vortex::file::multi::MultiFileSession; use vortex::file::multi::open_cached; @@ -262,7 +269,7 @@ pub fn reader_initialize( if global.scan_v2 { let state = file.scan_state(); let scan = FileScan::try_new( - v2_builder(state, global, selection), + v2_builder(state, global, selection)?, &global.initial_chunks, global.parallel_files, global.scan_diagnostics.clone(), @@ -276,9 +283,16 @@ pub fn reader_initialize( let reader = Arc::clone(file.reader()); let filter = &global.filter; + let dtype = reader.dtype(); + let projection = file_expression(&global.projection, dtype)?; + let bound_filter = filter + .filter + .as_ref() + .map(|filter| file_expression(filter, dtype)) + .transpose()?; let builder = ScanBuilder::new(SESSION.clone(), reader) - .with_projection(global.projection.clone()) - .with_some_filter(filter.filter.clone()) + .with_projection(projection) + .with_some_filter(bound_filter) .with_selection(selection); let scan = builder.prepare()?; let mut splits: Vec = scan @@ -295,17 +309,51 @@ pub fn reader_initialize( } /// The V2 scan of the file's rows that the scan's row range and selection keep. -fn v2_builder(state: &FileScanState, global: &GlobalState, selection: Selection) -> V2ScanBuilder { +fn v2_builder( + state: &FileScanState, + global: &GlobalState, + selection: Selection, +) -> VortexResult { let filter = &global.filter; let row_count = state.reader.row_count(); let rows = filter.row_range.as_ref().map_or(0..row_count, |range| { range.start.min(row_count)..range.end.min(row_count) }); - V2ScanBuilder::new(SESSION.clone(), state.scan_file.clone()) - .with_projection(global.projection.clone()) - .with_some_filter(filter.filter.clone()) + let dtype = state.scan_file.layout.dtype(); + Ok(V2ScanBuilder::new(SESSION.clone(), state.scan_file.clone()) + .with_projection(file_expression(&global.projection, dtype)?) + .with_some_filter( + filter + .filter + .as_ref() + .map(|filter| file_expression(filter, dtype)) + .transpose()?, + ) .with_selection(selection) - .with_row_range(rows) + .with_row_range(rows)) +} + +/// Files can store columns in a different order from the first file that bound the query. +/// Preserve the query's root schema, including its output order, over the file's actual root. +fn file_expression(expression: &BoundExpression, dtype: &DType) -> VortexResult { + if expression.is_root_bound_to(dtype) { + return Ok(expression.clone()); + } + Ok(expression + .clone() + .transform_down(|node| { + if node.is_root() { + Ok(Transformed { + value: select(node.dtype().as_struct_fields().names().clone(), root()) + .bind(dtype)?, + changed: true, + order: TraversalOrder::Skip, + }) + } else { + Ok(Transformed::no(node)) + } + })? + .into_inner()) } /// What a thread's split yields next. diff --git a/vortex-sqllogictest/slt/clickbench/duckdb/plans/q23.slt.no b/vortex-sqllogictest/slt/clickbench/duckdb/plans/q23.slt.no index 70483220f52..4dc8401827c 100644 --- a/vortex-sqllogictest/slt/clickbench/duckdb/plans/q23.slt.no +++ b/vortex-sqllogictest/slt/clickbench/duckdb/plans/q23.slt.no @@ -308,7 +308,7 @@ logical_opt [ "Expressions": [ "EventTime", "file_index", - "file_row_number" + "__vortex_row_id" ], "Estimated Cardinality": "200000" } @@ -318,7 +318,7 @@ logical_opt [ "Expressions": [ "EventTime", "file_index", - "file_row_number" + "__vortex_row_id" ], "Estimated Cardinality": "200000" } @@ -334,7 +334,7 @@ logical_opt [ "Join Type": "SEMI", "Conditions": [ "(file_index = file_index)", - "(file_row_number = file_row_number)" + "(__vortex_row_id = __vortex_row_id)" ], "Estimated Cardinality": "200000" } @@ -605,7 +605,7 @@ physical_plan [ "Join Type": "SEMI", "Conditions": [ "file_index = file_index", - "file_row_number = file_row_number" + "__vortex_row_id = __vortex_row_id" ], "Estimated Cardinality": "200000" } diff --git a/vortex-sqllogictest/slt/clickbench/duckdb/plans/q24.slt.no b/vortex-sqllogictest/slt/clickbench/duckdb/plans/q24.slt.no index b9e90f0a85a..a0529200fe0 100644 --- a/vortex-sqllogictest/slt/clickbench/duckdb/plans/q24.slt.no +++ b/vortex-sqllogictest/slt/clickbench/duckdb/plans/q24.slt.no @@ -216,7 +216,7 @@ logical_opt [ "Expressions": [ "EventTime", "file_index", - "file_row_number" + "__vortex_row_id" ], "Estimated Cardinality": "200000" } @@ -226,7 +226,7 @@ logical_opt [ "Expressions": [ "EventTime", "file_index", - "file_row_number" + "__vortex_row_id" ], "Estimated Cardinality": "200000" } @@ -242,7 +242,7 @@ logical_opt [ "Join Type": "SEMI", "Conditions": [ "(file_index = file_index)", - "(file_row_number = file_row_number)" + "(__vortex_row_id = __vortex_row_id)" ], "Estimated Cardinality": "200000" } @@ -316,7 +316,7 @@ physical_plan [ "Join Type": "SEMI", "Conditions": [ "file_index = file_index", - "file_row_number = file_row_number" + "__vortex_row_id = __vortex_row_id" ], "Estimated Cardinality": "200000" } diff --git a/vortex-sqllogictest/slt/duckdb/agg_partition_stats_multi_file.slt b/vortex-sqllogictest/slt/duckdb/agg_partition_stats_multi_file.slt index 59f883cd474..055d1a835c7 100644 --- a/vortex-sqllogictest/slt/duckdb/agg_partition_stats_multi_file.slt +++ b/vortex-sqllogictest/slt/duckdb/agg_partition_stats_multi_file.slt @@ -45,6 +45,17 @@ SELECT a FROM '${WORK_DIR}/reordered_*.vortex' ORDER BY a; 1 2 +query II +SELECT * FROM '${WORK_DIR}/reordered_*.vortex' ORDER BY a; +---- +1 100 +2 200 + +query II +SELECT b + 1, a FROM '${WORK_DIR}/reordered_*.vortex' WHERE b >= 150 AND a < 3; +---- +201 2 + query TT EXPLAIN SELECT max(a), min(b) FROM '${WORK_DIR}/reordered_*.vortex'; ---- From 61b075d54c2d1ed645729a4af399254313e8b387 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 15:05:53 +0000 Subject: [PATCH 07/28] Size wide filtered Scan V2 work by bytes Signed-off-by: Joe Isaacs --- vortex-layout/src/scan/v2/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vortex-layout/src/scan/v2/mod.rs b/vortex-layout/src/scan/v2/mod.rs index cd240c1809b..1a4a258d194 100644 --- a/vortex-layout/src/scan/v2/mod.rs +++ b/vortex-layout/src/scan/v2/mod.rs @@ -218,7 +218,11 @@ impl ScanBuilder { ) })?; let chunk_rows = if let Some(rows) = filtered_chunk_rows { - rows + if identity { + rows.min(rows_per_chunk(row_count, &file.locations)) + } else { + rows + } } else if identity { rows_per_chunk(row_count, &file.locations) } else { From 2c6d5e630e237436e5562d37e647ff5f0d8b2ed6 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 17:40:16 +0000 Subject: [PATCH 08/28] Integrate sparse scans with layout-aware DuckDB work scheduling Signed-off-by: Joe Isaacs --- vortex-duckdb/cpp/include/table_function.h | 1 - vortex-duckdb/cpp/multi_file_reader.cpp | 1 - vortex-duckdb/src/file_reader.rs | 54 +++++- vortex-duckdb/src/scan_v2.rs | 200 +++++++++++++++------ vortex-duckdb/src/scan_v2/tests.rs | 135 ++++++++++++++ vortex-duckdb/src/table_function.rs | 10 +- vortex-layout/src/scan/v2/tests.rs | 2 + 7 files changed, 334 insertions(+), 69 deletions(-) create mode 100644 vortex-duckdb/src/scan_v2/tests.rs diff --git a/vortex-duckdb/cpp/include/table_function.h b/vortex-duckdb/cpp/include/table_function.h index c0fcd8ffa5e..c76da70bf94 100644 --- a/vortex-duckdb/cpp/include/table_function.h +++ b/vortex-duckdb/cpp/include/table_function.h @@ -25,7 +25,6 @@ void duckdb_vx_string_map_insert(duckdb_vx_string_map map, const char *key, cons // Input data passed into the init_global and init_local callbacks. typedef struct { const void *bind_data; - size_t file_count; size_t max_threads; /** diff --git a/vortex-duckdb/cpp/multi_file_reader.cpp b/vortex-duckdb/cpp/multi_file_reader.cpp index 8ef12657d91..743623e9320 100644 --- a/vortex-duckdb/cpp/multi_file_reader.cpp +++ b/vortex-duckdb/cpp/multi_file_reader.cpp @@ -177,7 +177,6 @@ VortexReaderInterface::InitializeGlobalState(ClientContext &context, const auto threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); duckdb_vx_tfunc_init_input ffi_input = { .bind_data = ffi_bind, - .file_count = bind_data.file_list->GetFileCount(threads).count, .max_threads = threads, .column_ids = column_ids.data(), .column_ids_count = column_ids.size(), diff --git a/vortex-duckdb/src/file_reader.rs b/vortex-duckdb/src/file_reader.rs index 71d176d0b56..d621d14cec1 100644 --- a/vortex-duckdb/src/file_reader.rs +++ b/vortex-duckdb/src/file_reader.rs @@ -12,6 +12,8 @@ use object_store::registry::ObjectStoreRegistry; use url::Url; use vortex::array::ArrayRef; use vortex::array::VortexSessionExecute as _; +use vortex::array::arrays::Chunked; +use vortex::array::arrays::chunked::ChunkedArrayExt; use vortex::array::arrays::struct_::StructArrayExt as _; use vortex::dtype::DType; use vortex::dtype::FieldPath; @@ -40,6 +42,7 @@ use vortex::layout::scan::scan_builder::ScanBuilder; use vortex::layout::scan::v2::ScanBuilder as V2ScanBuilder; use vortex::layout::scan::v2::ScanFile; use vortex::mask::Mask; +use vortex::scalar_fn::fns::dynamic::DynamicExprUpdates; use vortex::scan::selection::Selection; use vortex::session::SessionExt as _; @@ -268,10 +271,28 @@ pub fn reader_initialize( // global lock. if global.scan_v2 { let state = file.scan_state(); + let builder = v2_builder(state, global, selection)?; + let splits = ScanBuilder::new(SESSION.clone(), Arc::clone(&state.reader)) + .with_projection(file_expression(&global.projection, state.reader.dtype())?) + .with_some_filter( + global + .filter + .filter + .as_ref() + .map(|filter| file_expression(filter, state.reader.dtype())) + .transpose()?, + ) + .full_file_splits()?; + let dynamic = global + .filter + .filter + .as_ref() + .is_some_and(|filter| DynamicExprUpdates::new(filter).is_some()); let scan = FileScan::try_new( - v2_builder(state, global, selection)?, + builder, + &splits, + dynamic, &global.initial_chunks, - global.parallel_files, global.scan_diagnostics.clone(), &state.path, )?; @@ -378,8 +399,25 @@ fn next_split_item(local: &mut LocalState) -> VortexResult { .v2 .get() .vortex_expect("V2 split of a file without a V2 scan"); - let Some(array) = scan.next_array(&mut local.run, &mut local.initial_roots)? else { - return Ok(SplitItem::Done); + let array = match local.pending.pop() { + Some(array) => array, + None => { + let Some(array) = scan.next_array( + &mut local.run, + &mut local.initial_roots, + local.v2_work.take(), + )? + else { + return Ok(SplitItem::Done); + }; + match array.as_opt::() { + Some(chunked) => { + local.pending.extend(chunked.chunks().into_iter().rev()); + local.pending.pop().unwrap_or(array) + } + None => array, + } + } }; local.split = Some(Split::V2); Ok(if array.is_empty() { @@ -400,13 +438,15 @@ fn next_split_item(local: &mut LocalState) -> VortexResult { /// the global lock. It only moves references between the file and the local /// state, and reader_scan drops the retired ones. pub fn reader_try_initialize_scan(file: &mut OpenFileReader, local: &mut LocalState) -> bool { - // A V2 scan's threads take chunks from the file's scan until it has spawned - // them all, so the file has work for another thread until then. + // Reserve a real part before releasing the global lock so idle workers move to other files. let v2 = file .scan_state .as_ref() .and_then(|state| state.v2.get()) - .map(|scan| (!scan.is_exhausted()).then_some(Split::V2)); + .map(|scan| { + local.v2_work = scan.take_work(true); + local.v2_work.as_ref().map(|_| Split::V2) + }); let Some(split) = v2.unwrap_or_else(|| file.splits.pop()) else { // DuckDB releases the file after this call, under the global lock. // Give the reference of the file to this thread so it drops outside. diff --git a/vortex-duckdb/src/scan_v2.rs b/vortex-duckdb/src/scan_v2.rs index ee879e25939..16765a435d2 100644 --- a/vortex-duckdb/src/scan_v2.rs +++ b/vortex-duckdb/src/scan_v2.rs @@ -18,7 +18,9 @@ use std::ops::Range; use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; +use std::time::Instant; use parking_lot::Mutex; use vortex::array::ArrayRef; @@ -29,21 +31,30 @@ use vortex::io::request::IoRequestId; use vortex::io::request::IoResult; use vortex::io::request::IoService; use vortex::io::runtime::BlockingRuntime as _; +use vortex::layout::scan::v2::AccessMode; use vortex::layout::scan::v2::ChunkPlanner; use vortex::layout::scan::v2::PreparedScan; use vortex::layout::scan::v2::PrunePlanner; use vortex::layout::scan::v2::ScanBuilder; use vortex::mask::Mask; use vortex::scan::planning::driver::Progress; +use vortex::scan::planning::driver::RootId; use vortex::scan::planning::driver::Run; use vortex::scan::planning::planner::Planner; use vortex::scan::planning::planner::PlannerOutput; use vortex::scan::planning::planner::State; use vortex::scan::planning::planner::WorkScope; -use vortex::scan::selection::Selection; use crate::RUNTIME; +mod diagnostics; +mod startup; +pub(crate) use diagnostics::Diagnostics; +pub(crate) use diagnostics::FilterObserver; +use startup::InitialPlanner; +pub(crate) use startup::initial_budget; +use startup::initial_ranges; + /// The chunks a thread runs beside each other on its run, so one chunk's reads overlap another's /// compute. const CHUNKS_PER_THREAD: usize = 1; @@ -52,6 +63,8 @@ const CHUNKS_PER_THREAD: usize = 1; pub(crate) struct FileScan { scan: Arc, io: Arc, + diagnostics: Option>, + span: tracing::Span, /// The work the root spawned, in row order, that no thread has taken yet. queue: Mutex>, /// Set once a thread has taken the last of `queue`. @@ -61,9 +74,11 @@ pub(crate) struct FileScan { } /// What a thread takes from the file next. -enum Work { +pub(crate) enum Work { /// A chunk to run on the thread's run. Chunk(WorkScope, Box), + /// An 8K startup range, kept together until it returns to DuckDB. + Initial(WorkScope, Box), /// A batch the root produced itself. Batch(ArrayRef), } @@ -83,45 +98,82 @@ impl FileScan { builder: ScanBuilder, splits: &[u64], dynamic: bool, + remaining: &AtomicUsize, + diagnostics: Option>, + path: &str, ) -> VortexResult { let scan = Arc::new(builder.prepare()?); let io = scan.io(); let rows = scan.rows().clone(); + let initial = if scan.access_mode() == AccessMode::RandomAccess { + Vec::new() + } else { + initial_ranges(rows.clone(), remaining) + }; + let bulk_start = initial.last().map_or(rows.start, |range| range.end); + let scan_id = diagnostics.as_ref().map(|diagnostics| diagnostics.id); + let span = tracing::debug_span!("v2_scan", path, scan_id); + let _entered = span.enter(); + let mut queue = VecDeque::new(); + for rows in &initial { + let planner = chunks(&scan, rows.clone(), None); + let planner = PrunePlanner::new(Arc::clone(&scan), rows.clone(), Box::new(planner)); + queue.push_back(Work::Initial( + WorkScope { + file_ordinal: 0, + rows: rows.clone(), + }, + Box::new(InitialPlanner(Box::new(planner))), + )); + } let mut root = Run::new(); root.admit( - compose(&scan, splits, dynamic), + compose( + &scan, + bulk_start..rows.end, + splits, + dynamic && initial.is_empty(), + ), WorkScope { file_ordinal: 0, rows, }, io.session(), ); - let mut queue = VecDeque::new(); loop { - match root.advance()? { + let started = diagnostics.as_ref().map(|_| Instant::now()); + let progress = root.advance()?; + if let (Some(started), Some(diagnostics)) = (started, &diagnostics) { + diagnostics.compute(started.elapsed().as_nanos()); + } + match progress { Progress::Batch(batch) => queue.push_back(Work::Batch(batch.array)), Progress::Spawn(scope, chunk) => queue.push_back(Work::Chunk(scope, chunk)), Progress::RootDone(_) if root.live_roots() > 0 => {} Progress::RootDone(_) | Progress::Idle => break, Progress::Waiting => { + let started = diagnostics.as_ref().map(|_| Instant::now()); RUNTIME.block_on(future::poll_fn(|cx| root.poll_completion(cx)))?; + if let (Some(started), Some(diagnostics)) = (started, &diagnostics) { + diagnostics.io_wait(started.elapsed().as_nanos()); + } } } } + tracing::debug!(path, access_mode = ?scan.access_mode(), initial_chunks = initial.len(), + worker_slots = queue.len(), bulk_start, "v2 scan startup"); + drop(_entered); Ok(Self { scan, io, + diagnostics, + span, exhausted: AtomicBool::new(queue.is_empty()), queue: Mutex::new(queue), rows_taken: AtomicU64::new(0), }) } - /// Whether every chunk has been taken, so a thread joining the scan finds no work. - pub(crate) fn is_exhausted(&self) -> bool { - self.exhausted.load(Ordering::Acquire) - } - /// The percentage of the scanned rows whose chunks threads have taken. pub(crate) fn progress(&self) -> f64 { let rows = self.scan.rows(); @@ -135,39 +187,84 @@ impl FileScan { /// The next array of this file for the thread whose run is `run`, or `None` once the file has /// no more work for the thread. Keeps the thread's run fed with chunks from the file, and /// waits for reads on the calling thread. - pub(crate) fn next_array(&self, run: &mut Run) -> VortexResult> { + pub(crate) fn next_array( + &self, + run: &mut Run, + initial_roots: &mut Vec, + mut reserved: Option, + ) -> VortexResult> { + let _entered = self.span.enter(); loop { - while run.live_roots() < CHUNKS_PER_THREAD { - match self.next_work() { + while run.live_roots() + < if initial_roots.is_empty() { + CHUNKS_PER_THREAD + } else { + 2 + } + { + match reserved + .take() + .or_else(|| self.take_work(initial_roots.is_empty())) + { + Some(Work::Initial(scope, chunk)) => { + tracing::debug!(worker = ?std::thread::current().id(), row_start = scope.rows.start, + row_end = scope.rows.end, "v2 initial chunk admitted"); + initial_roots.push(run.admit(chunk, scope, self.io.session())); + } Some(Work::Chunk(scope, chunk)) => { run.admit(chunk, scope, self.io.session()); } - Some(Work::Batch(array)) => return Ok(Some(array)), + Some(Work::Batch(array)) => { + if let Some(diagnostics) = &self.diagnostics { + diagnostics.array(array.len()); + } + return Ok(Some(array)); + } None => break, } } - match run.advance()? { - Progress::Batch(batch) => return Ok(Some(batch.array)), + let started = self.diagnostics.as_ref().map(|_| Instant::now()); + let progress = run.advance()?; + if let (Some(started), Some(diagnostics)) = (started, &self.diagnostics) { + diagnostics.compute(started.elapsed().as_nanos()); + } + match progress { + Progress::Batch(batch) => { + if let Some(diagnostics) = &self.diagnostics { + diagnostics.array(batch.array.len()); + } + return Ok(Some(batch.array)); + } Progress::Spawn(scope, child) => { run.admit(child, scope, self.io.session()); } - Progress::RootDone(_) => {} + Progress::RootDone(root) => { + initial_roots.retain(|id| *id != root); + } Progress::Idle => return Ok(None), Progress::Waiting => { + let started = self.diagnostics.as_ref().map(|_| Instant::now()); RUNTIME.block_on(future::poll_fn(|cx| run.poll_completion(cx)))?; + if let (Some(started), Some(diagnostics)) = (started, &self.diagnostics) { + diagnostics.io_wait(started.elapsed().as_nanos()); + } } } } } - /// Takes the next chunk or batch of the file, or `None` once every one has been taken. - fn next_work(&self) -> Option { + /// Reserves actual queued work under DuckDB's global lock without executing it there. + /// When startup work is still running, `allow_bulk` prevents delaying its first output. + pub(crate) fn take_work(&self, allow_bulk: bool) -> Option { let mut queue = self.queue.lock(); + if !allow_bulk && !matches!(queue.front(), Some(Work::Initial(..))) { + return None; + } let work = queue.pop_front()?; if queue.is_empty() { self.exhausted.store(true, Ordering::Release); } - if let Work::Chunk(scope, _) = &work { + if let Work::Chunk(scope, _) | Work::Initial(scope, _) = &work { self.rows_taken .fetch_add(scope.rows.end - scope.rows.start, Ordering::Relaxed); } @@ -177,8 +274,15 @@ impl FileScan { /// The planners of a scan, composed: with a dynamic filter, its pruning plans first over the scan's /// rows, continuing into the parts of the rows they keep; otherwise, the parts directly. -fn compose(scan: &Arc, splits: &[u64], dynamic: bool) -> Box { - let rows = scan.rows().clone(); +fn compose( + scan: &Arc, + rows: Range, + splits: &[u64], + dynamic: bool, +) -> Box { + if scan.access_mode() == AccessMode::RandomAccess { + return chunks(scan, rows, None)(None); + } let parts = { let scan = Arc::clone(scan); let parts = part_ranges(rows.clone(), splits); @@ -194,6 +298,22 @@ fn compose(scan: &Arc, splits: &[u64], dynamic: bool) -> Box, + rows: Range, + kept: Option, +) -> impl FnOnce(Option) -> Box + Send + 'static { + let scan = Arc::clone(scan); + move |again| { + let kept = match (kept, again) { + (Some(kept), Some(again)) => Some(kept & &again), + (kept, again) => kept.or(again), + }; + let selection = scan.selection().clone(); + Box::new(ChunkPlanner::new(scan, rows, selection, kept)) + } +} + /// Spawns a planner per part of a scan's rows that zone pruning left something of, in row order. /// Parts are the file's layout splits, so a layout chunk, and the dictionary it carries, is /// exported by one thread, which lets DuckDB evaluate an expression over that dictionary once. @@ -206,7 +326,6 @@ fn compose(scan: &Arc, splits: &[u64], dynamic: bool) -> Box, - selection: Selection, parts: std::vec::IntoIter>, /// The first row of `kept`. first_row: u64, @@ -222,7 +341,6 @@ impl PartPlanner { kept: Option, ) -> Self { Self { - selection: scan.selection().clone(), scan, parts: parts.into_iter(), first_row, @@ -258,18 +376,7 @@ impl Planner for PartPlanner { if kept.as_ref().is_some_and(Mask::all_false) { continue; } - let chunks = { - let scan = Arc::clone(&self.scan); - let selection = self.selection.clone(); - let part = part.clone(); - move |again: Option| -> Box { - let kept = match (kept, again) { - (Some(kept), Some(again)) => Some(kept & &again), - (kept, again) => kept.or(again), - }; - Box::new(ChunkPlanner::new(scan, part, selection, kept)) - } - }; + let chunks = chunks(&self.scan, part.clone(), kept); let planner: Box = if self.scan.filter().is_some() { Box::new(PrunePlanner::new( Arc::clone(&self.scan), @@ -299,23 +406,4 @@ fn part_ranges(rows: Range, splits: &[u64]) -> Vec> { } #[cfg(test)] -mod tests { - use std::ops::Range; - - use rstest::rstest; - - use super::part_ranges; - - #[rstest] - #[case::whole(0..30, &[0, 10, 20, 30], vec![0..10, 10..20, 20..30])] - #[case::clipped(5..25, &[0, 10, 20, 30], vec![5..10, 10..20, 20..25])] - #[case::inside_one_split(12..18, &[0, 10, 20, 30], vec![12..18])] - #[case::empty(5..5, &[0, 10], vec![])] - fn parts_follow_splits( - #[case] rows: Range, - #[case] splits: &[u64], - #[case] expected: Vec>, - ) { - assert_eq!(part_ranges(rows, splits), expected); - } -} +mod tests; diff --git a/vortex-duckdb/src/scan_v2/tests.rs b/vortex-duckdb/src/scan_v2/tests.rs new file mode 100644 index 00000000000..aec3640eff9 --- /dev/null +++ b/vortex-duckdb/src/scan_v2/tests.rs @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::ops::Range; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; + +use rstest::rstest; +use vortex::array::Canonical; +use vortex::array::IntoArray; +use vortex::array::VortexSessionExecute; +use vortex::array::arrays::PrimitiveArray; +use vortex::array::assert_arrays_eq; +use vortex::buffer::ByteBufferMut; +use vortex::error::VortexResult; +use vortex::error::vortex_err; +use vortex::expr::gt; +use vortex::expr::lit; +use vortex::expr::root; +use vortex::file::OpenOptionsSessionExt; +use vortex::file::WriteOptionsSessionExt; +use vortex::io::runtime::BlockingRuntime; +use vortex::layout::scan::v2::ScanBuilder; +use vortex::scan::planning::driver::Run; +use vortex::scan::strict_sorted_buffer::StrictSortedBuffer; + +use super::FileScan; +use super::part_ranges; +use crate::RUNTIME; +use crate::SESSION; + +#[rstest] +#[case::whole(0..30, &[0, 10, 20, 30], vec![0..10, 10..20, 20..30])] +#[case::clipped(5..25, &[0, 10, 20, 30], vec![5..10, 10..20, 20..25])] +#[case::inside_one_split(12..18, &[0, 10, 20, 30], vec![12..18])] +#[case::empty(5..5, &[0, 10], vec![])] +fn parts_follow_splits( + #[case] rows: Range, + #[case] splits: &[u64], + #[case] expected: Vec>, +) { + assert_eq!(part_ranges(rows, splits), expected); +} + +#[rstest] +#[case::disabled(0)] +#[case::prefix_and_bulk(2)] +#[case::entire_file_initial(16)] +fn reserved_work_returns_selected_rows_once( + #[case] initial: usize, + #[values(false, true)] sparse: bool, + #[values(false, true)] dynamic: bool, +) -> VortexResult<()> { + let array = (0..100_000i64).collect::().into_array(); + let file = RUNTIME.block_on(async { + let mut bytes = ByteBufferMut::empty(); + SESSION + .write_options() + .write(&mut bytes, array.to_array_stream()) + .await?; + SESSION.open_options().open_buffer(bytes) + })?; + let filter = gt(root(), lit(5000i64)).bind(file.dtype())?; + let mut builder = ScanBuilder::new(SESSION.clone(), file.scan_file()) + .with_some_filter(Some(filter)) + .with_row_range(100..99_900); + if sparse { + builder = builder.with_row_indices(StrictSortedBuffer::try_new( + (0..100_000u64).step_by(997).collect(), + )?); + } + let scan = Arc::new(FileScan::try_new( + builder, + &[0, 25_000, 50_000, 75_000, 100_000], + dynamic, + &AtomicUsize::new(initial), + None, + "test", + )?); + // Reserve every assignment before workers execute, as DuckDB can under its global lock. + let mut reserved = Vec::new(); + while let Some(work) = scan.take_work(true) { + reserved.push(work); + } + assert!(scan.take_work(true).is_none()); + assert_eq!(scan.progress(), 100.0); + if !sparse && initial == 0 { + assert_eq!(reserved.len(), 4); + } + let mut actual = std::thread::scope(|scope| -> VortexResult> { + let handles: Vec<_> = reserved + .into_iter() + .map(|work| { + let scan = Arc::clone(&scan); + scope.spawn(move || -> VortexResult> { + let mut run = Run::new(); + let mut initial_roots = Vec::new(); + let mut values = Vec::new(); + let mut ctx = SESSION.create_execution_ctx(); + let mut reserved = Some(work); + while let Some(array) = + scan.next_array(&mut run, &mut initial_roots, reserved.take())? + { + values.extend_from_slice( + array + .execute::(&mut ctx)? + .into_primitive() + .as_slice::(), + ); + } + Ok(values) + }) + }) + .collect(); + let mut values = Vec::new(); + for handle in handles { + values.extend( + handle + .join() + .map_err(|_| vortex_err!("scan worker failed"))??, + ); + } + Ok(values) + })?; + actual.sort_unstable(); + let expected = (5001..99_900i64) + .filter(|value| !sparse || value % 997 == 0) + .collect::(); + assert_arrays_eq!( + PrimitiveArray::from_iter(actual), + expected, + &mut SESSION.create_execution_ctx() + ); + Ok(()) +} diff --git a/vortex-duckdb/src/table_function.rs b/vortex-duckdb/src/table_function.rs index 0557491588b..38682992aae 100644 --- a/vortex-duckdb/src/table_function.rs +++ b/vortex-duckdb/src/table_function.rs @@ -190,8 +190,6 @@ pub struct GlobalState { pub(crate) row_id_field: Option, /// Whether the scan runs on the planner-driven V2 executor. pub scan_v2: bool, - /// File parallelism already supplies one scan per worker, allowing larger in-file chunks. - pub(crate) parallel_files: bool, pub(crate) initial_chunks: AtomicUsize, pub(crate) scan_diagnostics: Option>, @@ -233,6 +231,9 @@ pub struct LocalState { /// The planning run on which this thread runs the chunks of V2 scans. pub(crate) run: Run, pub(crate) initial_roots: Vec, + pub(crate) v2_work: Option, + /// Root chunks awaiting conversion, preserving their dictionary encodings. + pub(crate) pending: Vec, /// Empty for non-aggregate scan pub partials: Partials, @@ -364,7 +365,7 @@ pub fn init_global(init_input: &TableInitInput) -> VortexResult { crate::scan_v2(), filter.filter.as_ref(), crate::DYNAMIC_FILTER_INITIAL_CHUNKS.load(Ordering::Relaxed), - vortex_utils::parallelism::get_available_parallelism().unwrap_or(1), + init_input.input.max_threads.max(1), )?; Ok(GlobalState { initial_chunks: AtomicUsize::new(initial_chunks), @@ -379,7 +380,6 @@ pub fn init_global(init_input: &TableInitInput) -> VortexResult { file_row_number_column_pos, row_id_field, scan_v2: crate::scan_v2(), - parallel_files: init_input.input.file_count >= init_input.input.max_threads.max(1), }) } @@ -450,6 +450,8 @@ pub fn init_local(bind_data: &BindState, global: &GlobalState) -> LocalState { retired_files: Vec::new(), run: Run::new(), initial_roots: Vec::new(), + v2_work: None, + pending: Vec::new(), finished: false, } } diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index 3c926085bbb..f625ee9eccf 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -87,6 +87,8 @@ use crate::scan::execution::row_chunks; use crate::scan::execution::scan_chunks; use crate::scan::scan_builder::ScanBuilder as V1ScanBuilder; use crate::segments::CountingSegmentSource; +use crate::segments::SegmentFuture; +use crate::segments::SegmentId; use crate::segments::SegmentSource; use crate::segments::TestSegments; use crate::sequence::SequenceId; From 6c489ce14232e387c54ee92614a340275f759e09 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 10:06:08 +0000 Subject: [PATCH 09/28] Overlap reads across two DuckDB scan work items per thread Keep two work items active so storage waits can overlap other scan work. Extend exact-once coverage to leave work queued for active workers as well as reserving all assignments up front. Signed-off-by: "Joe Isaacs" --- vortex-duckdb/src/scan_v2.rs | 2 +- vortex-duckdb/src/scan_v2/tests.rs | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/vortex-duckdb/src/scan_v2.rs b/vortex-duckdb/src/scan_v2.rs index 16765a435d2..bfd6be43460 100644 --- a/vortex-duckdb/src/scan_v2.rs +++ b/vortex-duckdb/src/scan_v2.rs @@ -57,7 +57,7 @@ use startup::initial_ranges; /// The chunks a thread runs beside each other on its run, so one chunk's reads overlap another's /// compute. -const CHUNKS_PER_THREAD: usize = 1; +const CHUNKS_PER_THREAD: usize = 2; /// One file's V2 scan, shared by the threads scanning it. pub(crate) struct FileScan { diff --git a/vortex-duckdb/src/scan_v2/tests.rs b/vortex-duckdb/src/scan_v2/tests.rs index aec3640eff9..22b2cb1b2df 100644 --- a/vortex-duckdb/src/scan_v2/tests.rs +++ b/vortex-duckdb/src/scan_v2/tests.rs @@ -46,10 +46,11 @@ fn parts_follow_splits( #[case::disabled(0)] #[case::prefix_and_bulk(2)] #[case::entire_file_initial(16)] -fn reserved_work_returns_selected_rows_once( +fn reserved_and_queued_work_return_selected_rows_once( #[case] initial: usize, #[values(false, true)] sparse: bool, #[values(false, true)] dynamic: bool, + #[values(false, true)] reserve_all: bool, ) -> VortexResult<()> { let array = (0..100_000i64).collect::().into_array(); let file = RUNTIME.block_on(async { @@ -77,15 +78,21 @@ fn reserved_work_returns_selected_rows_once( None, "test", )?); - // Reserve every assignment before workers execute, as DuckDB can under its global lock. + // Reserve assignments as DuckDB can under its global lock. Two workers leave the rest + // queued so each run can admit more work while its earlier roots are still active. let mut reserved = Vec::new(); while let Some(work) = scan.take_work(true) { reserved.push(work); + if !reserve_all && reserved.len() == 2 { + break; + } + } + if reserve_all { + assert!(scan.take_work(true).is_none()); + assert_eq!(scan.progress(), 100.0); } - assert!(scan.take_work(true).is_none()); - assert_eq!(scan.progress(), 100.0); if !sparse && initial == 0 { - assert_eq!(reserved.len(), 4); + assert_eq!(reserved.len(), if reserve_all { 4 } else { 2 }); } let mut actual = std::thread::scope(|scope| -> VortexResult> { let handles: Vec<_> = reserved From 963fdfa0c094de94fc10ee59aad602ca3d9bf32f Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 10:06:08 +0000 Subject: [PATCH 10/28] Amortize narrow fixed-width scans over larger row chunks Allow referenced fixed-width columns totaling up to 32 bytes per row to share the existing one-million-row chunk budget. This reduces setup and physical read fragmentation while retaining smaller chunks for wide and variable-width scans. Signed-off-by: "Joe Isaacs" --- vortex-layout/src/scan/v2/mod.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/vortex-layout/src/scan/v2/mod.rs b/vortex-layout/src/scan/v2/mod.rs index 1a4a258d194..def9d76afd4 100644 --- a/vortex-layout/src/scan/v2/mod.rs +++ b/vortex-layout/src/scan/v2/mod.rs @@ -13,11 +13,11 @@ //! Import this module's [`ScanBuilder`] to prepare a V2 scan. It accepts a [`ScanFile`] directly //! and supports projection, filters, row ranges, and selections. Unsupported layouts return an //! error when the plan is prepared; limits are not part of this builder's API. Work uses chunks of -//! up to 1,048,576 rows, or 131,072 rows for filtered scans. Filtered scans reading only one -//! primitive or boolean field use 1,048,576 rows to amortize setup. Chunks are independent of -//! stored layout boundaries. Unfiltered projections stream stored batches without concatenating -//! them. Full-file projections use smaller 8,192-row multiples when serialized bytes per row -//! are high. +//! up to 1,048,576 rows, or 131,072 rows for filtered scans. Filtered scans whose referenced +//! fixed-width fields total at most 32 bytes per row use 1,048,576 rows to amortize setup and IO. +//! Chunks are independent of stored layout boundaries. Unfiltered projections stream stored +//! batches without concatenating them. Full-file projections use smaller 8,192-row multiples +//! when serialized bytes per row are high. //! //! Sparse positive input selections use [`AccessMode::RandomAccess`]: only selected 65,536-row //! regions are scheduled, and segment leaves gather encoded values before materialization. @@ -265,14 +265,12 @@ fn filtered_rows_per_chunk( ) -> VortexResult { let mut fields = referenced_field_paths(projection)?; fields.extend(referenced_field_paths(filter)?); - let mut fields = fields.into_iter(); - let first = fields.next(); - // A scan of one fixed-width field is cheap enough that smaller chunks mainly add setup. + let bytes_per_row = fields.into_iter().try_fold(0usize, |bytes, field| { + bytes.checked_add(field.resolve(dtype.clone())?.element_size()?) + }); + // Up to 32 MiB of fixed-width values amortize setup and coalesce adjacent stored batches. // Wider and variable-width scans need more chunks to avoid long tails on a few workers. - let narrow = fields.next().is_none() - && first - .and_then(|field| field.resolve(dtype.clone())) - .is_some_and(|dtype| matches!(dtype, DType::Primitive(..) | DType::Bool(..))); + let narrow = bytes_per_row.is_some_and(|bytes| bytes <= 32); Ok(if narrow { 128 * ROW_BLOCK_SIZE } else { From a991c95d5e942743acdb56490cc0f83e21bf933d Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 10:57:42 +0000 Subject: [PATCH 11/28] Compare scan versions on TPC-H, TPC-DS and ClickBench in Actions Add a manual scan-v2 SQL preset that builds the pinned base and head, then compares base V2, head V2 and head V1 sequentially on the same requested-scale files. Preserve raw samples, verify binary commits and complete query coverage, and report paired-round uncertainty. Signed-off-by: "Joe Isaacs" --- .github/workflows/pr-bench-sql.yml | 10 + .github/workflows/scan-v2-comparison.yml | 177 +++++++++++++ scripts/compare-scan-versions.py | 316 +++++++++++++++++++++++ scripts/tests/test_scan_versions.py | 166 ++++++++++++ 4 files changed, 669 insertions(+) create mode 100644 .github/workflows/scan-v2-comparison.yml create mode 100644 scripts/compare-scan-versions.py create mode 100644 scripts/tests/test_scan_versions.py diff --git a/.github/workflows/pr-bench-sql.yml b/.github/workflows/pr-bench-sql.yml index 40789d631a8..d37c627ad9d 100644 --- a/.github/workflows/pr-bench-sql.yml +++ b/.github/workflows/pr-bench-sql.yml @@ -29,6 +29,7 @@ on: - "pr-compact" - "pr-all" - "pr-full" + - "scan-v2" permissions: contents: read @@ -37,8 +38,17 @@ permissions: jobs: sql: + if: inputs.matrix_preset != 'scan-v2' uses: ./.github/workflows/sql-bench-matrix.yml secrets: inherit with: mode: "pr" matrix_preset: ${{ inputs.matrix_preset }} + + scan-comparison: + if: inputs.matrix_preset == 'scan-v2' + uses: ./.github/workflows/scan-v2-comparison.yml + secrets: inherit + with: + head_sha: ${{ github.event.pull_request.head.sha || github.sha }} + base_ref: ${{ github.event.pull_request.base.sha || 'origin/develop' }} diff --git a/.github/workflows/scan-v2-comparison.yml b/.github/workflows/scan-v2-comparison.yml new file mode 100644 index 00000000000..250d03cbea0 --- /dev/null +++ b/.github/workflows/scan-v2-comparison.yml @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +name: Paired Scan V2 Comparisons + +on: + workflow_call: + inputs: + head_sha: + required: true + type: string + base_ref: + required: true + type: string + +permissions: + contents: read + id-token: write + +concurrency: + group: scan-v2-comparison-${{ inputs.head_sha }} + cancel-in-progress: false + +jobs: + resolve: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + base_sha: ${{ steps.base.outputs.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.head_sha }} + fetch-depth: 0 + persist-credentials: false + - id: base + name: Pin the comparison base + env: + BASE_REF: ${{ inputs.base_ref }} + run: | + sha=$(git rev-parse --verify "${BASE_REF}^{commit}") + echo "sha=$sha" >> "$GITHUB_OUTPUT" + + build: + needs: resolve + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + revision: [base, head] + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.8xlarge+c8g.8xlarge+m8gd.8xlarge+m8g.8xlarge+r8gd.8xlarge+r8g.8xlarge/image=ubuntu26-full-arm64-pre-v2/tag=scan-{1}/extras=s3-cache', github.run_id, matrix.revision) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ matrix.revision == 'base' && needs.resolve.outputs.base_sha || inputs.head_sha }} + persist-credentials: false + - uses: ./.github/actions/assert-cpu + if: github.repository == 'vortex-data/vortex' + - uses: ./.github/actions/setup-prebuild + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} + - name: Build identical native benchmark targets + env: + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" + run: cargo build --locked --bin data-gen --bin datafusion-bench --bin duckdb-bench --profile release_debug + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: scan-comparison-${{ matrix.revision }}-binaries + path: | + target/release_debug/data-gen + target/release_debug/datafusion-bench + target/release_debug/duckdb-bench + target/release_debug/build/vortex-duckdb-cache/duckdb-lib-*-prebuilt/libduckdb.so + + bench: + needs: [resolve, build] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + include: + - id: tpch-10 + suite: tpch + scale: "10.0" + - id: tpch-50 + suite: tpch + scale: "50.0" + - id: tpcds-10 + suite: tpcds + scale: "10.0" + - id: tpcds-50 + suite: tpcds + scale: "50.0" + - id: clickbench + suite: clickbench + scale: "" + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.metal-24xl/image=ubuntu26-full-arm64-pre-v2/tag=scan-{1}/extras=s3-cache', github.run_id, matrix.id) + || 'ubuntu-latest' }} + env: + FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.head_sha }} + persist-credentials: false + - uses: ./.github/actions/assert-cpu + if: github.repository == 'vortex-data/vortex' + - uses: ./.github/actions/setup-prebuild + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/setup-duckdb + - uses: ./.github/actions/system-info + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: scan-comparison-head-binaries + path: target/scan-comparison/head + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: scan-comparison-base-binaries + path: target/scan-comparison/base + - name: Prepare shared benchmark files + env: + SUITE: ${{ matrix.suite }} + SCALE: ${{ matrix.scale }} + run: | + chmod +x target/scan-comparison/{head,base}/{data-gen,datafusion-bench,duckdb-bench} + options=() + if [[ -n "$SCALE" ]]; then + options+=(--opt "scale-factor=$SCALE") + fi + target/scan-comparison/head/data-gen "$SUITE" --formats vortex "${options[@]}" + - name: Prepare benchmark CPUs + run: sudo bash scripts/setup-benchmark.sh + - name: Compare base V2, head V2 and head V1 sequentially + env: + SUITE: ${{ matrix.suite }} + SCALE: ${{ matrix.scale }} + BASE_SHA: ${{ needs.resolve.outputs.base_sha }} + HEAD_SHA: ${{ inputs.head_sha }} + run: | + options=() + if [[ -n "$SCALE" ]]; then + options+=(--scale-factor "$SCALE") + fi + bash scripts/bench-taskset.sh python3 scripts/compare-scan-versions.py "$SUITE" \ + --base-dir target/scan-comparison/base --head-dir target/scan-comparison/head \ + --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" \ + --iterations 7 --warmup 2 --rounds 2 --output-dir scan-comparison-results "${options[@]}" + - name: Publish comparison summary + if: always() + run: | + if [[ -f scan-comparison-results/summary.md ]]; then + cat scan-comparison-results/summary.md >> "$GITHUB_STEP_SUMMARY" + else + echo "Comparison incomplete; see job logs and partial measurement artifacts." >> "$GITHUB_STEP_SUMMARY" + fi + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: always() + with: + name: scan-comparison-${{ matrix.id }} + path: | + scan-comparison-results/ + !scan-comparison-results/tmp/** diff --git a/scripts/compare-scan-versions.py b/scripts/compare-scan-versions.py new file mode 100644 index 00000000000..d5f53516fe4 --- /dev/null +++ b/scripts/compare-scan-versions.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Compare base V2, head V2, and head V1 on the same prepared SQL benchmark files.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import random +import statistics +import subprocess +from pathlib import Path +from typing import Any + +VARIANTS = ("base-v2", "head-v2", "head-v1") + + +def write_json(path: Path, value: object) -> None: + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def binary_environment(binary: Path, threads: int, temporary: Path) -> dict[str, str]: + env = os.environ.copy() + for key in ("VORTEX_SCAN_V2", "VORTEX_USE_SCAN_API", "BLOCKING_MAX_THREADS"): + env.pop(key, None) + env.update( + RUST_LOG="warn", + TOKIO_WORKER_THREADS=str(threads), + DATAFUSION_EXECUTION_TARGET_PARTITIONS=str(threads), + TMPDIR=str(temporary), + ) + libraries = sorted(binary.parent.glob("build/vortex-duckdb-cache/duckdb-lib-*-prebuilt")) + if libraries: + env["LD_LIBRARY_PATH"] = os.pathsep.join([*(str(path) for path in libraries), env.get("LD_LIBRARY_PATH", "")]) + return env + + +def command( + binary: Path, + suite: str, + options: list[str], + engine: str, + threads: int, + version: int, +) -> list[str]: + args = [str(binary), suite, "--formats", "vortex", "--scan-v1", str(version == 1).lower()] + for option in options: + args.extend(["--opt", option]) + if engine == "duckdb": + args.extend(["--threads", str(threads)]) + return args + + +def measurement(path: Path, *, query: int, engine: str, commit: str, iterations: int, warmup: int) -> dict[str, Any]: + records = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + if len(records) != 1: + raise ValueError(f"Expected one query record in {path}, found {len(records)}") + record = records[0] + name = record["name"].split("/", 1)[0] + if int(name.rsplit("_q", 1)[1]) != query: + raise ValueError(f"Wrong query in {path}: {name}") + if record["commit_id"] != commit: + raise ValueError(f"Wrong binary commit in {path}: {record['commit_id']} != {commit}") + if record["target"] != {"engine": engine, "format": "vortex"} or record["unit"] != "ns": + raise ValueError(f"Unexpected measurement target or unit in {path}") + samples = record["all_runtimes"] + if len(samples) != iterations or any(not math.isfinite(value) or value <= 0 for value in samples): + raise ValueError(f"Invalid runtime samples in {path}") + return {"cold_ns": samples[0], "warm_ns": samples[warmup:], "record": record} + + +def ratio_interval( + numerators: list[list[list[float]]], denominators: list[list[list[float]]], resamples: int = 1000 +) -> tuple[float, float]: + """Resample paired process rounds, then warm samples, retaining the fixed query workload.""" + rng = random.Random(0) + ratios = [] + for _ in range(resamples): + query_ratios = [] + for top, bottom in zip(numerators, denominators, strict=True): + selected = rng.choices(range(len(top)), k=len(top)) + numerator = [value for index in selected for value in rng.choices(top[index], k=len(top[index]))] + denominator = [value for index in selected for value in rng.choices(bottom[index], k=len(bottom[index]))] + query_ratios.append(math.log(statistics.median(numerator) / statistics.median(denominator))) + ratios.append(math.exp(statistics.mean(query_ratios))) + ratios.sort() + return ratios[int(0.025 * resamples)], ratios[min(int(0.975 * resamples), resamples - 1)] + + +def summarize(rows: list[dict[str, Any]], engines: list[str], queries: list[int], rounds: int) -> dict[str, Any]: + expected = { + (engine, query, turn, variant) + for engine in engines + for query in queries + for turn in range(rounds) + for variant in VARIANTS + } + keys = [(row["engine"], row["query"], row["round"], row["variant"]) for row in rows] + if len(set(keys)) != len(keys) or set(keys) != expected: + raise ValueError("Comparison has duplicate or missing engine/query/round/variant measurements") + result: dict[str, Any] = {"complete": True, "engines": {}} + for engine in engines: + per_query: list[dict[str, Any]] = [] + samples_by_variant: dict[str, list[list[list[float]]]] = {variant: [] for variant in VARIANTS} + for query in queries: + samples = { + variant: [ + value + for row in rows + if row["engine"] == engine and row["query"] == query and row["variant"] == variant + for value in row["warm_ns"] + ] + for variant in VARIANTS + } + medians = {variant: statistics.median(values) for variant, values in samples.items()} + per_query.append( + { + "query": query, + "median_ms": {variant: value / 1e6 for variant, value in medians.items()}, + "head_v2_vs_base_v2": medians["head-v2"] / medians["base-v2"], + "head_v2_vs_head_v1": medians["head-v2"] / medians["head-v1"], + } + ) + for variant in VARIANTS: + samples_by_variant[variant].append( + [ + row["warm_ns"] + for row in sorted(rows, key=lambda row: row["round"]) + if row["engine"] == engine and row["query"] == query and row["variant"] == variant + ] + ) + comparisons = {} + for label, baseline in (("head_v2_vs_base_v2", "base-v2"), ("head_v2_vs_head_v1", "head-v1")): + ratio = math.exp(statistics.mean(math.log(row[label]) for row in per_query)) + low, high = ratio_interval(samples_by_variant["head-v2"], samples_by_variant[baseline]) + comparisons[label] = { + "ratio": ratio, + "interval_95": [low, high], + "verdict": "improved" if high < 1 else "regressed" if low > 1 else "unclear", + } + result["engines"][engine] = {"queries": per_query, "comparisons": comparisons} + return result + + +def markdown(summary: dict[str, Any], manifest: dict[str, Any]) -> str: + lines = [ + f"# Scan comparison: {manifest['suite']} SF={manifest['scale_factor'] or 'n/a'}", + "", + f"Head `{manifest['head_sha']}`; base `{manifest['base_sha']}`.", + f"{manifest['rounds']} reversed-order rounds; {manifest['iterations']} iterations per process; " + f"first {manifest['warmup']} excluded. {manifest['threads']} workers, " + f"{len(manifest['affinity'])} CPUs in the affinity mask; identical prepared files.", + "", + "Ratios below 1 mean faster head V2. " + "Intervals resample paired process rounds and warm samples within each query.", + "", + "| Engine | Queries | Head V2 / base V2 (95% interval) | Head V2 / head V1 (95% interval) |", + "| --- | ---: | --- | --- |", + ] + for engine, data in summary["engines"].items(): + cells = [] + for label in ("head_v2_vs_base_v2", "head_v2_vs_head_v1"): + value = data["comparisons"][label] + low, high = value["interval_95"] + cells.append(f"{value['ratio']:.3f} ({low:.3f}–{high:.3f}), {value['verdict']}") + lines.append(f"| {engine} | {len(data['queries'])} | {' | '.join(cells)} |") + for engine, data in summary["engines"].items(): + lines.extend( + [ + "", + f"## {engine}", + "", + "| Query | Base V2 ms | Head V2 ms | Head V1 ms | V2/base | V2/V1 |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for row in data["queries"]: + values = row["median_ms"] + lines.append( + f"| {row['query']} | {values['base-v2']:.3f} | {values['head-v2']:.3f} | " + f"{values['head-v1']:.3f} | {row['head_v2_vs_base_v2']:.3f} | " + f"{row['head_v2_vs_head_v1']:.3f} |" + ) + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("suite", choices=("tpch", "tpcds", "clickbench")) + parser.add_argument("--scale-factor") + parser.add_argument("--base-dir", type=Path, required=True) + parser.add_argument("--head-dir", type=Path, required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--head-sha", required=True) + parser.add_argument("--engines", default="duckdb,datafusion") + parser.add_argument("--queries", help="Comma-separated query indices; omitted runs the complete suite") + parser.add_argument("--iterations", type=int, default=7) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument("--rounds", type=int, default=2) + parser.add_argument("--threads", type=int, default=len(os.sched_getaffinity(0))) + parser.add_argument("--opt", action="append", default=[]) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + if not 0 <= args.warmup < args.iterations or args.rounds < 1 or args.threads < 1: + parser.error("Require 0 <= warmup < iterations and positive rounds/threads") + engines = args.engines.split(",") + if len(set(engines)) != len(engines) or not set(engines) <= {"duckdb", "datafusion"}: + parser.error("Engines must be unique members of duckdb,datafusion") + output = args.output_dir.resolve() + output.mkdir(parents=True, exist_ok=True) + temporary = output / "tmp" + temporary.mkdir(exist_ok=True) + options = list(args.opt) + if args.scale_factor: + options.append(f"scale-factor={args.scale_factor}") + manifest: dict[str, Any] = { + "suite": args.suite, + "scale_factor": args.scale_factor, + "head_sha": args.head_sha, + "base_sha": args.base_sha, + "iterations": args.iterations, + "warmup": args.warmup, + "rounds": args.rounds, + "threads": args.threads, + "options": options, + "affinity": sorted(os.sched_getaffinity(0)), + "binaries": {}, + "queries": {}, + } + rows: list[dict[str, Any]] = [] + for engine in engines: + binaries = { + "base": args.base_dir.resolve() / f"{engine}-bench", + "head": args.head_dir.resolve() / f"{engine}-bench", + } + envs = {label: binary_environment(binary, args.threads, temporary) for label, binary in binaries.items()} + queries_by_revision = {} + for label, binary in binaries.items(): + manifest["binaries"][f"{label}-{engine}"] = { + "path": str(binary), + "bytes": binary.stat().st_size, + "elf_notes": subprocess.check_output(["readelf", "-n", str(binary)], text=True), + } + discover = command(binary, args.suite, options, engine, args.threads, 2) + ["--print-queries"] + queries_by_revision[label] = [ + int(line) for line in subprocess.check_output(discover, env=envs[label], text=True).splitlines() + ] + if queries_by_revision["base"] != queries_by_revision["head"]: + raise ValueError(f"Base and head enumerate different queries for {engine}") + available = queries_by_revision["head"] + queries = [int(query) for query in args.queries.split(",")] if args.queries else available + if not queries or len(set(queries)) != len(queries) or not set(queries) <= set(available): + raise ValueError(f"Invalid requested query set for {engine}: {queries}") + manifest["queries"][engine] = queries + write_json(output / "manifest.json", manifest) + for turn in range(args.rounds): + for index, query in enumerate(queries): + start = index % len(VARIANTS) + order = VARIANTS[start:] + VARIANTS[:start] + if turn % 2: + order = order[::-1] + for variant in order: + revision, version = variant.split("-") + binary = binaries[revision] + stem = output / f"{engine}-q{query}-r{turn}-{variant}" + cmd = command(binary, args.suite, options, engine, args.threads, int(version[1:])) + cmd.extend( + [ + "--queries", + str(query), + "--iterations", + str(args.iterations), + "--hide-progress-bar", + "--display-format", + "gh-json", + "-o", + str(stem.with_suffix(".jsonl")), + ] + ) + print(f"START {engine} q{query} round={turn} {variant}", flush=True) + with stem.with_suffix(".log").open("w", encoding="utf-8") as log: + subprocess.run(cmd, env=envs[revision], stdout=log, stderr=subprocess.STDOUT, check=True) + value = measurement( + stem.with_suffix(".jsonl"), + query=query, + engine=engine, + commit=args.base_sha if revision == "base" else args.head_sha, + iterations=args.iterations, + warmup=args.warmup, + ) + rows.append( + {"engine": engine, "query": query, "round": turn, "variant": variant, "command": cmd, **value} + ) + write_json(output / "measurements.json", rows) + print( + f"DONE {engine} q{query} {variant}: {statistics.median(value['warm_ns']) / 1e6:.3f} ms", + flush=True, + ) + query_sets = list(manifest["queries"].values()) + if any(queries != query_sets[0] for queries in query_sets): + raise ValueError("Engines enumerate different query sets; report them separately") + summary = summarize(rows, engines, query_sets[0], args.rounds) + write_json(output / "summary.json", summary) + report = markdown(summary, manifest) + (output / "summary.md").write_text(report, encoding="utf-8") + print(report, flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_scan_versions.py b/scripts/tests/test_scan_versions.py new file mode 100644 index 00000000000..cf45d2ce4f5 --- /dev/null +++ b/scripts/tests/test_scan_versions.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +import pytest + + +@pytest.fixture +def comparison() -> ModuleType: + path = Path(__file__).resolve().parents[1] / "compare-scan-versions.py" + spec = importlib.util.spec_from_file_location("compare_scan_versions", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_excludes_startup_samples_instead_of_using_runner_value(comparison: ModuleType, tmp_path: Path): + path = tmp_path / "query.jsonl" + record = { + "name": "tpch_q06/datafusion:vortex-file-compressed", + "commit_id": "head-sha", + "target": {"engine": "datafusion", "format": "vortex"}, + "unit": "ns", + "value": 9999, + "all_runtimes": [9000, 8000, 100, 110, 90], + } + path.write_text(json.dumps(record), encoding="utf-8") + result = comparison.measurement(path, query=6, engine="datafusion", commit="head-sha", iterations=5, warmup=2) + assert result["cold_ns"] == 9000 + assert result["warm_ns"] == [100, 110, 90] + + +@pytest.mark.parametrize( + "field,value", + [ + ("commit_id", "wrong-sha"), + ("name", "tpch_q07/datafusion:vortex"), + ("unit", "ms"), + ("all_runtimes", [100, 0, 100]), + ("all_runtimes", [100, float("nan"), 100]), + ("all_runtimes", [100, 100]), + ], +) +def test_rejects_wrong_or_incomplete_measurements(comparison: ModuleType, tmp_path: Path, field: str, value: object): + path = tmp_path / "query.jsonl" + record = { + "name": "tpch_q06/datafusion:vortex-file-compressed", + "commit_id": "head-sha", + "target": {"engine": "datafusion", "format": "vortex"}, + "unit": "ns", + "all_runtimes": [100, 100, 100], + } + record[field] = value + path.write_text(json.dumps(record), encoding="utf-8") + with pytest.raises(ValueError): + comparison.measurement(path, query=6, engine="datafusion", commit="head-sha", iterations=3, warmup=1) + + +def test_missing_and_duplicate_variants_cannot_produce_complete_summary(comparison: ModuleType): + rows = [ + {"engine": "duckdb", "query": 1, "round": 0, "variant": variant, "warm_ns": [100, 100]} + for variant in comparison.VARIANTS + ] + with pytest.raises(ValueError, match="duplicate or missing"): + comparison.summarize(rows[:-1], ["duckdb"], [1], 1) + with pytest.raises(ValueError, match="duplicate or missing"): + comparison.summarize(rows + [rows[0]], ["duckdb"], [1], 1) + + +def test_interval_retains_process_level_variation(comparison: ModuleType): + top = [[[90.0] * 20, [100.0] * 20, [110.0] * 20]] + bottom = [[[100.0] * 20] * 3] + low, high = comparison.ratio_interval(top, bottom) + assert low <= 0.9 + assert high >= 1.1 + + +def test_runs_each_engine_and_scan_version_with_the_expected_binary( + comparison: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + program = f"""#!{sys.executable} +import json +from pathlib import Path +import sys +if '--print-queries' in sys.argv: + print('1\\n2') + raise SystemExit() +def arg(name): + return sys.argv[sys.argv.index(name) + 1] +revision = Path(__file__).parent.name +engine = Path(__file__).name.removesuffix('-bench') +version = 1 if arg('--scan-v1') == 'true' else 2 +duration = 150 if revision == 'base' else 100 if version == 2 else 200 +samples = [9000, 8000] + [duration] * (int(arg('--iterations')) - 2) +record = dict(name=f'tpch_q{{int(arg("--queries")):02}}/{{engine}}:vortex-file-compressed', + commit_id=revision + '-sha', target=dict(engine=engine, format='vortex'), + unit='ns', all_runtimes=samples) +Path(arg('-o')).write_text(json.dumps(record)) +""" + for revision in ("base", "head"): + directory = tmp_path / revision + directory.mkdir() + for engine in ("duckdb", "datafusion"): + binary = directory / f"{engine}-bench" + binary.write_text(program, encoding="utf-8") + binary.chmod(0o755) + check_output = subprocess.check_output + + def with_elf_notes(args: list[str], *, text: bool, env: dict[str, str] | None = None) -> str: + if args[0] == "readelf": + return "Build ID: test-binary\n" + result = check_output(args, text=text, env=env) + assert isinstance(result, str) + return result + + monkeypatch.setattr(subprocess, "check_output", with_elf_notes) + output = tmp_path / "results" + monkeypatch.setattr( + sys, + "argv", + [ + "compare-scan-versions.py", + "tpch", + "--scale-factor", + "10.0", + "--base-dir", + str(tmp_path / "base"), + "--head-dir", + str(tmp_path / "head"), + "--base-sha", + "base-sha", + "--head-sha", + "head-sha", + "--rounds", + "1", + "--iterations", + "4", + "--warmup", + "2", + "--threads", + "2", + "--output-dir", + str(output), + ], + ) + comparison.main() + rows = json.loads((output / "measurements.json").read_text(encoding="utf-8")) + assert len(rows) == 12 + for row in rows: + revision, version = row["variant"].split("-") + assert Path(row["command"][0]).parent.name == revision + flag = row["command"].index("--scan-v1") + assert row["command"][flag + 1] == str(version == "v1").lower() + summary = json.loads((output / "summary.json").read_text(encoding="utf-8")) + assert summary["complete"] is True + for engine in ("duckdb", "datafusion"): + comparisons = summary["engines"][engine]["comparisons"] + assert comparisons["head_v2_vs_base_v2"]["ratio"] == pytest.approx(2 / 3) + assert comparisons["head_v2_vs_head_v1"]["ratio"] == pytest.approx(0.5) From 9c678bb9bea420f7b79a708adf009d4f549ccdf6 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 11:07:39 +0000 Subject: [PATCH 12/28] Identify paired benchmark runs in CI Signed-off-by: "Joe Isaacs" --- scripts/compare-scan-versions.py | 20 +++++++++++++++++--- scripts/tests/test_scan_versions.py | 4 ++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/compare-scan-versions.py b/scripts/compare-scan-versions.py index d5f53516fe4..c49eacaa6e2 100644 --- a/scripts/compare-scan-versions.py +++ b/scripts/compare-scan-versions.py @@ -49,7 +49,16 @@ def command( threads: int, version: int, ) -> list[str]: - args = [str(binary), suite, "--formats", "vortex", "--scan-v1", str(version == 1).lower()] + args = [ + str(binary), + suite, + "--formats", + "vortex", + "--scan-v1", + str(version == 1).lower(), + "--runner", + "scan-v2-comparison", + ] for option in options: args.extend(["--opt", option]) if engine == "duckdb": @@ -284,8 +293,13 @@ def main() -> None: ] ) print(f"START {engine} q{query} round={turn} {variant}", flush=True) - with stem.with_suffix(".log").open("w", encoding="utf-8") as log: - subprocess.run(cmd, env=envs[revision], stdout=log, stderr=subprocess.STDOUT, check=True) + log_path = stem.with_suffix(".log") + try: + with log_path.open("w", encoding="utf-8") as log: + subprocess.run(cmd, env=envs[revision], stdout=log, stderr=subprocess.STDOUT, check=True) + except subprocess.CalledProcessError: + print(log_path.read_text(encoding="utf-8")[-8000:], flush=True) + raise value = measurement( stem.with_suffix(".jsonl"), query=query, diff --git a/scripts/tests/test_scan_versions.py b/scripts/tests/test_scan_versions.py index cf45d2ce4f5..564519ecd12 100644 --- a/scripts/tests/test_scan_versions.py +++ b/scripts/tests/test_scan_versions.py @@ -87,6 +87,7 @@ def test_runs_each_engine_and_scan_version_with_the_expected_binary( ): program = f"""#!{sys.executable} import json +import os from pathlib import Path import sys if '--print-queries' in sys.argv: @@ -94,6 +95,8 @@ def test_runs_each_engine_and_scan_version_with_the_expected_binary( raise SystemExit() def arg(name): return sys.argv[sys.argv.index(name) + 1] +if os.environ.get('CI') == 'true' and ('--runner' not in sys.argv or arg('--runner') == 'unknown'): + raise SystemExit('benchmark runner must not be unknown in CI; pass --runner') revision = Path(__file__).parent.name engine = Path(__file__).name.removesuffix('-bench') version = 1 if arg('--scan-v1') == 'true' else 2 @@ -104,6 +107,7 @@ def arg(name): unit='ns', all_runtimes=samples) Path(arg('-o')).write_text(json.dumps(record)) """ + monkeypatch.setenv("CI", "true") for revision in ("base", "head"): directory = tmp_path / revision directory.mkdir() From f3bf313506d07a91061e916a03842fe7f6ee43a0 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 11:16:24 +0000 Subject: [PATCH 13/28] Verify benchmark artifacts against build manifests Benchmark commit_id records the runtime checkout. Verify the source revision and executable SHA-256 from each build artifact independently, and preserve the runtime checkout in the comparison manifest. Signed-off-by: "Joe Isaacs" --- .github/workflows/scan-v2-comparison.yml | 18 ++++++++++++++++- scripts/compare-scan-versions.py | 18 +++++++++++++++-- scripts/tests/test_scan_versions.py | 25 +++++++++++++++++++++++- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scan-v2-comparison.yml b/.github/workflows/scan-v2-comparison.yml index 250d03cbea0..016e00a50b9 100644 --- a/.github/workflows/scan-v2-comparison.yml +++ b/.github/workflows/scan-v2-comparison.yml @@ -70,7 +70,22 @@ jobs: - name: Build identical native benchmark targets env: RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" - run: cargo build --locked --bin data-gen --bin datafusion-bench --bin duckdb-bench --profile release_debug + run: | + cargo build --locked --bin data-gen --bin datafusion-bench --bin duckdb-bench --profile release_debug + python3 - <<'PY' + import hashlib + import json + import subprocess + from pathlib import Path + directory = Path("target/release_debug") + binaries = {} + for name in ("data-gen", "datafusion-bench", "duckdb-bench"): + with (directory / name).open("rb") as executable: + binaries[name] = hashlib.file_digest(executable, "sha256").hexdigest() + manifest = {"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + "binaries": binaries} + (directory / "build-manifest.json").write_text(json.dumps(manifest, indent=2)) + PY - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: scan-comparison-${{ matrix.revision }}-binaries @@ -78,6 +93,7 @@ jobs: target/release_debug/data-gen target/release_debug/datafusion-bench target/release_debug/duckdb-bench + target/release_debug/build-manifest.json target/release_debug/build/vortex-duckdb-cache/duckdb-lib-*-prebuilt/libduckdb.so bench: diff --git a/scripts/compare-scan-versions.py b/scripts/compare-scan-versions.py index c49eacaa6e2..ca2ec4e9cc4 100644 --- a/scripts/compare-scan-versions.py +++ b/scripts/compare-scan-versions.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse +import hashlib import json import math import os @@ -19,6 +20,17 @@ VARIANTS = ("base-v2", "head-v2", "head-v1") +def binary_provenance(binary: Path, commit: str) -> dict[str, str]: + build = json.loads((binary.parent / "build-manifest.json").read_text(encoding="utf-8")) + if build["commit"] != commit: + raise ValueError(f"Wrong build revision for {binary}: {build['commit']} != {commit}") + with binary.open("rb") as executable: + digest = hashlib.file_digest(executable, "sha256").hexdigest() + if build["binaries"][binary.name] != digest: + raise ValueError(f"Binary hash does not match its build manifest: {binary}") + return {"commit": commit, "sha256": digest} + + def write_json(path: Path, value: object) -> None: temporary = path.with_suffix(".tmp") temporary.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") @@ -75,7 +87,7 @@ def measurement(path: Path, *, query: int, engine: str, commit: str, iterations: if int(name.rsplit("_q", 1)[1]) != query: raise ValueError(f"Wrong query in {path}: {name}") if record["commit_id"] != commit: - raise ValueError(f"Wrong binary commit in {path}: {record['commit_id']} != {commit}") + raise ValueError(f"Wrong measurement checkout in {path}: {record['commit_id']} != {commit}") if record["target"] != {"engine": engine, "format": "vortex"} or record["unit"] != "ns": raise ValueError(f"Unexpected measurement target or unit in {path}") samples = record["all_runtimes"] @@ -241,6 +253,7 @@ def main() -> None: "affinity": sorted(os.sched_getaffinity(0)), "binaries": {}, "queries": {}, + "checkout_commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), } rows: list[dict[str, Any]] = [] for engine in engines: @@ -254,6 +267,7 @@ def main() -> None: manifest["binaries"][f"{label}-{engine}"] = { "path": str(binary), "bytes": binary.stat().st_size, + **binary_provenance(binary, args.base_sha if label == "base" else args.head_sha), "elf_notes": subprocess.check_output(["readelf", "-n", str(binary)], text=True), } discover = command(binary, args.suite, options, engine, args.threads, 2) + ["--print-queries"] @@ -304,7 +318,7 @@ def main() -> None: stem.with_suffix(".jsonl"), query=query, engine=engine, - commit=args.base_sha if revision == "base" else args.head_sha, + commit=manifest["checkout_commit"], iterations=args.iterations, warmup=args.warmup, ) diff --git a/scripts/tests/test_scan_versions.py b/scripts/tests/test_scan_versions.py index 564519ecd12..9810702712e 100644 --- a/scripts/tests/test_scan_versions.py +++ b/scripts/tests/test_scan_versions.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors +import hashlib import importlib.util import json import subprocess @@ -82,12 +83,26 @@ def test_interval_retains_process_level_variation(comparison: ModuleType): assert high >= 1.1 +def test_rejects_mismatched_build_revision_and_executable(comparison: ModuleType, tmp_path: Path): + binary = tmp_path / "duckdb-bench" + binary.write_bytes(b"executable") + build = {"commit": "base-sha", "binaries": {binary.name: hashlib.sha256(binary.read_bytes()).hexdigest()}} + (tmp_path / "build-manifest.json").write_text(json.dumps(build), encoding="utf-8") + assert comparison.binary_provenance(binary, "base-sha")["commit"] == "base-sha" + with pytest.raises(ValueError, match="Wrong build revision"): + comparison.binary_provenance(binary, "head-sha") + binary.write_bytes(b"different executable") + with pytest.raises(ValueError, match="Binary hash"): + comparison.binary_provenance(binary, "base-sha") + + def test_runs_each_engine_and_scan_version_with_the_expected_binary( comparison: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ): program = f"""#!{sys.executable} import json import os +import subprocess from pathlib import Path import sys if '--print-queries' in sys.argv: @@ -103,7 +118,8 @@ def arg(name): duration = 150 if revision == 'base' else 100 if version == 2 else 200 samples = [9000, 8000] + [duration] * (int(arg('--iterations')) - 2) record = dict(name=f'tpch_q{{int(arg("--queries")):02}}/{{engine}}:vortex-file-compressed', - commit_id=revision + '-sha', target=dict(engine=engine, format='vortex'), + commit_id=subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(), + target=dict(engine=engine, format='vortex'), unit='ns', all_runtimes=samples) Path(arg('-o')).write_text(json.dumps(record)) """ @@ -115,6 +131,13 @@ def arg(name): binary = directory / f"{engine}-bench" binary.write_text(program, encoding="utf-8") binary.chmod(0o755) + build = { + "commit": revision + "-sha", + "binaries": { + binary.name: hashlib.sha256(binary.read_bytes()).hexdigest() for binary in directory.iterdir() + }, + } + (directory / "build-manifest.json").write_text(json.dumps(build), encoding="utf-8") check_output = subprocess.check_output def with_elf_notes(args: list[str], *, text: bool, env: dict[str, str] | None = None) -> str: From 8a6097d6d0f610d31dbc95a0e275dbf37f245c7d Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 11:25:56 +0000 Subject: [PATCH 14/28] Respect DataFusion scan partition parallelism Give each admitted planning root its own IO session so retiring one root preserves pending reads of other roots. Bound default per-partition chunk streams by the workers left per engine partition, while retaining explicit per-thread overrides. Exercise delayed payload reads, startup ranges and sparse selections with inline execution. Signed-off-by: "Joe Isaacs" --- vortex-datafusion/src/persistent/opener.rs | 66 +++++++++++++++------ vortex-datafusion/src/persistent/scan_v2.rs | 34 +++++------ 2 files changed, 64 insertions(+), 36 deletions(-) diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index e5771db681c..cac0ff7e4f3 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -469,12 +469,13 @@ impl FileOpener for VortexOpener { } // Unless set, chunks run beside each other only as far as the engine's partitions // leave cores idle: at one partition per core they run in the scan's own run. - let concurrency = scan_concurrency.unwrap_or_else(|| { - get_available_parallelism() - .unwrap_or(1) - .div_ceil(partitions) - .max(1) - }); + let workers = get_available_parallelism().unwrap_or(1); + let in_flight = scan_concurrency + .map_or_else( + || workers.div_ceil(partitions), + |concurrency| concurrency.saturating_mul(workers), + ) + .max(1); let handle = vxf.session().handle(); // Lowering and planning a wide file is real CPU: it runs where compute runs. let scan = handle @@ -496,7 +497,7 @@ impl FileOpener for VortexOpener { ); let root = scan_planner(Arc::clone(&scan), bulk_start..scan.rows().end); - let driver = ScanDriver::new(handle, concurrency, has_output_ordering); + let driver = ScanDriver::new(handle, in_flight, has_output_ordering); Ok(driver.run(scan, root, initial, map_batch)) } else { // Share V1 layout readers across partitions of the file. @@ -844,6 +845,7 @@ mod tests { use std::sync::Arc; use std::sync::LazyLock; use std::sync::atomic::Ordering; + use std::time::Duration; use arrow_array::record_batch; use arrow_schema::Field; @@ -866,7 +868,8 @@ mod tests { use datafusion::scalar::ScalarValue; use datafusion_common::Statistics; use datafusion_common::arrow::compute::concat_batches; - use datafusion_common::arrow::compute::sort; + use datafusion_common::arrow::compute::sort_to_indices; + use datafusion_common::arrow::compute::take_record_batch; use datafusion_common::assert_batches_eq; use datafusion_common::stats::Precision; use datafusion_datasource::file::FileSource; @@ -883,6 +886,8 @@ mod tests { use itertools::Itertools; use object_store::ObjectStore; use object_store::memory::InMemory; + use object_store::throttle::ThrottleConfig; + use object_store::throttle::ThrottledStore; use rstest::rstest; use vortex::VortexSessionDefault; use vortex::array::VortexSessionExecute; @@ -1147,16 +1152,35 @@ mod tests { async fn startup_and_bulk_keep_filtered_selected_rows_once( #[values(0, 2)] initial_chunks: usize, #[values(false, true)] roaring: bool, + #[values(0, 1)] scan_concurrency: usize, ) -> anyhow::Result<()> { - let object_store = Arc::new(InMemory::new()) as Arc; - let batch = record_batch!(("a", Int32, (0..150_000).collect::>()))?; + // Delay payload reads so one root can retire while another still waits for IO. + let object_store = Arc::new(ThrottledStore::new( + InMemory::new(), + ThrottleConfig { + wait_get_per_call: Duration::from_millis(10), + ..Default::default() + }, + )) as Arc; + let payload = |row: u32| { + u64::from(row) + .wrapping_mul(0x9e37_79b9_7f4a_7c15) + .rotate_left(row % 64) + }; + let batch = record_batch!( + ("a", UInt32, (0..150_000).collect::>()), + ("b", UInt64, (0..150_000).map(payload).collect::>()) + )?; let schema = batch.schema(); let size = write_arrow_to_vortex(Arc::clone(&object_store), "startup.vortex", batch).await?; let table_schema = TableSchema::from(Arc::clone(&schema)); - let filter = logical2physical(&col("a").gt(lit(10)), table_schema.table_schema()); + let filter = logical2physical(&col("a").gt(lit(10_u32)), table_schema.table_schema()); + let projection = ProjectionExprs::from_indices(&[0, 1], table_schema.file_schema()); let mut opener = make_opener(object_store, table_schema, Some(filter)); + opener.projection = projection; opener.scan_v2 = true; + opener.scan_concurrency = Some(scan_concurrency); opener.initial_chunks = Arc::new(AtomicUsize::new(initial_chunks)); let indices = vec![3, 17, 8188, 8192, 16_390, 65_535, 65_536, 140_000, 149_999]; let selection = if roaring { @@ -1166,16 +1190,20 @@ mod tests { }; let file = PartitionedFile::new("startup.vortex".to_string(), size) .with_extension(VortexAccessPlan::default().with_selection(selection)); - let batches = opener.open(file)?.await?.try_collect::>().await?; + let batches = tokio::time::timeout( + Duration::from_secs(5), + opener.open(file)?.await?.try_collect::>(), + ) + .await??; assert_eq!(opener.initial_chunks.load(Ordering::Relaxed), 0); let actual = concat_batches(&schema, &batches)?; - let actual = - RecordBatch::try_new(Arc::clone(&schema), vec![sort(actual.column(0), None)?])?; - let expected = record_batch!(( - "a", - Int32, - [17, 8188, 8192, 16_390, 65_535, 65_536, 140_000, 149_999] - ))?; + let order = sort_to_indices(actual.column(0), None, None)?; + let actual = take_record_batch(&actual, &order)?; + let expected_rows = [17, 8188, 8192, 16_390, 65_535, 65_536, 140_000, 149_999]; + let expected = record_batch!( + ("a", UInt32, expected_rows.to_vec()), + ("b", UInt64, expected_rows.map(payload).to_vec()) + )?; assert_arrays_eq!( SESSION.arrow().from_arrow_record_batch(actual, &schema)?, SESSION.arrow().from_arrow_record_batch(expected, &schema)?, diff --git a/vortex-datafusion/src/persistent/scan_v2.rs b/vortex-datafusion/src/persistent/scan_v2.rs index cc3973f45f1..92bb2e281d9 100644 --- a/vortex-datafusion/src/persistent/scan_v2.rs +++ b/vortex-datafusion/src/persistent/scan_v2.rs @@ -28,7 +28,7 @@ use futures::stream::BoxStream; use vortex::array::ArrayRef; use vortex::error::VortexResult; use vortex::error::vortex_err; -use vortex::io::request::IoSource; +use vortex::io::request::IoService; use vortex::io::runtime::Handle; use vortex::layout::scan::v2::ChunkPlanner; use vortex::layout::scan::v2::PreparedScan; @@ -38,7 +38,6 @@ use vortex::scan::planning::driver::Progress; use vortex::scan::planning::driver::Run; use vortex::scan::planning::planner::Planner; use vortex::scan::planning::planner::WorkScope; -use vortex_utils::parallelism::get_available_parallelism; #[cfg(test)] mod tests; @@ -96,17 +95,17 @@ pub(crate) fn scan_planner(scan: Arc, rows: Range) -> Box Self { + pub(crate) fn new(handle: Handle, in_flight: usize, ordered: bool) -> Self { Self { handle, - concurrency, + in_flight, ordered, } } @@ -121,7 +120,7 @@ impl ScanDriver { map: impl Fn(ArrayRef) -> VortexResult + Send + Sync + 'static, ) -> BoxStream<'static, VortexResult> { let map: MapBatch = Arc::new(map); - let in_flight = self.concurrency * get_available_parallelism().unwrap_or(1); + let in_flight = self.in_flight; let inline = in_flight <= 1; let scope = WorkScope { file_ordinal: 0, @@ -145,14 +144,14 @@ impl ScanDriver { &initial_handle, scan_planner(Arc::clone(&initial_scan), rows), scope, - initial_io.session(), + Arc::clone(&initial_io), Arc::clone(&initial_map), ) }) .flatten_unordered(initial_count); let (chunk_handle, chunk_io, chunk_map) = (self.handle.clone(), Arc::clone(&io), Arc::clone(&map)); - let batches = run_stream(self.handle, root, scope, io.session(), map, inline) + let batches = run_stream(self.handle, root, scope, Arc::clone(&io), map, inline) .map_ok(move |yielded| match yielded { Yielded::Batch(start, batch) => { stream::once(future::ready(Ok((start, batch)))).boxed() @@ -161,7 +160,7 @@ impl ScanDriver { &chunk_handle, chunk, scope, - chunk_io.session(), + Arc::clone(&chunk_io), Arc::clone(&chunk_map), ), }) @@ -209,7 +208,8 @@ struct RunState { handle: Handle, /// Taken while a CPU step runs on its task and put back with its result. run: Option, - io: Arc, + /// Each admitted root owns a session, cleared independently when it retires. + io: Arc, map: MapBatch, /// Whether a spawned chunk runs on this run, as another root, instead of being yielded. inline: bool, @@ -224,12 +224,12 @@ fn run_stream( handle: Handle, root: Box, scope: WorkScope, - io: Arc, + io: Arc, map: MapBatch, inline: bool, ) -> BoxStream<'static, VortexResult> { let mut run = Run::new(); - run.admit(root, scope, Arc::clone(&io)); + run.admit(root, scope, io.session()); let state = RunState { handle, run: Some(run), @@ -268,7 +268,7 @@ fn run_stream( return Ok(Some((Yielded::Batch(start, batch), state))); } Step::Spawn(scope, child) if state.inline => { - run.admit(child, scope, Arc::clone(&state.io)); + run.admit(child, scope, state.io.session()); } Step::Spawn(scope, child) => { return Ok(Some((Yielded::Spawn(scope, child), state))); @@ -292,13 +292,13 @@ fn chunk_stream( handle: &Handle, root: Box, scope: WorkScope, - io: Arc, + io: Arc, map: MapBatch, ) -> BoxStream<'static, VortexResult<(u64, RecordBatch)>> { let (mut send, recv) = mpsc::channel(1); let task = handle.spawn(async move { let mut run = Run::new(); - run.admit(root, scope, Arc::clone(&io)); + run.admit(root, scope, io.session()); loop { match run.advance()? { Progress::Batch(batch) => { @@ -310,7 +310,7 @@ fn chunk_stream( } } Progress::Spawn(scope, child) => { - run.admit(child, scope, Arc::clone(&io)); + run.admit(child, scope, io.session()); } Progress::RootDone(_) if run.live_roots() > 0 => {} Progress::RootDone(_) | Progress::Idle => return Ok(()), From 05b8b051cd9169fc3ed3e074e48fd7b58194a9fd Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 11:41:18 +0000 Subject: [PATCH 15/28] Continue scan comparisons after recording query failures Retain failures and incomplete coverage explicitly while comparing the remaining complete query pairs. Bound individual query processes, skip failed queries on later rounds, and use dedicated NVMe paths for input and spill files. Exercise process failures and timeouts through real subprocesses. Signed-off-by: "Joe Isaacs" --- .github/workflows/scan-v2-comparison.yml | 20 ++++++- scripts/compare-scan-versions.py | 67 ++++++++++++++++++++++-- scripts/tests/test_scan_versions.py | 31 +++++++++-- 3 files changed, 106 insertions(+), 12 deletions(-) diff --git a/.github/workflows/scan-v2-comparison.yml b/.github/workflows/scan-v2-comparison.yml index 016e00a50b9..6f61b75a6f9 100644 --- a/.github/workflows/scan-v2-comparison.yml +++ b/.github/workflows/scan-v2-comparison.yml @@ -148,6 +148,19 @@ jobs: with: name: scan-comparison-base-binaries path: target/scan-comparison/base + - name: Place inputs and DuckDB spill files on local NVMe + env: + CASE_ID: ${{ matrix.id }} + run: | + data_root="$RUNNER_TEMP/scan-comparison-data" + if [[ -d /mnt/ephemeral ]]; then + data_root=/mnt/ephemeral/scan-comparison-data + fi + data_dir="$data_root/$GITHUB_RUN_ID/$CASE_ID" + sudo install -d -o "$(id -u)" -g "$(id -g)" "$data_dir" "$data_dir/target" + ln -s "$GITHUB_WORKSPACE/Cargo.lock" "$data_dir/Cargo.lock" + echo "COMPARISON_DATA_DIR=$data_dir" >> "$GITHUB_ENV" + echo "COMPARISON_DATA_URL=file://$data_dir/" >> "$GITHUB_ENV" - name: Prepare shared benchmark files env: SUITE: ${{ matrix.suite }} @@ -158,7 +171,9 @@ jobs: if [[ -n "$SCALE" ]]; then options+=(--opt "scale-factor=$SCALE") fi - target/scan-comparison/head/data-gen "$SUITE" --formats vortex "${options[@]}" + options+=(--opt "remote-data-dir=$COMPARISON_DATA_URL") + cd "$COMPARISON_DATA_DIR" + "$GITHUB_WORKSPACE/target/scan-comparison/head/data-gen" "$SUITE" --formats vortex "${options[@]}" - name: Prepare benchmark CPUs run: sudo bash scripts/setup-benchmark.sh - name: Compare base V2, head V2 and head V1 sequentially @@ -175,7 +190,8 @@ jobs: bash scripts/bench-taskset.sh python3 scripts/compare-scan-versions.py "$SUITE" \ --base-dir target/scan-comparison/base --head-dir target/scan-comparison/head \ --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" \ - --iterations 7 --warmup 2 --rounds 2 --output-dir scan-comparison-results "${options[@]}" + --iterations 7 --warmup 2 --rounds 2 --query-timeout 300 \ + --opt "remote-data-dir=$COMPARISON_DATA_URL" --output-dir scan-comparison-results "${options[@]}" - name: Publish comparison summary if: always() run: | diff --git a/scripts/compare-scan-versions.py b/scripts/compare-scan-versions.py index ca2ec4e9cc4..8ddb5923ebf 100644 --- a/scripts/compare-scan-versions.py +++ b/scripts/compare-scan-versions.py @@ -184,13 +184,20 @@ def markdown(summary: dict[str, Any], manifest: dict[str, Any]) -> str: "| Engine | Queries | Head V2 / base V2 (95% interval) | Head V2 / head V1 (95% interval) |", "| --- | ---: | --- | --- |", ] + if not summary["complete"]: + lines.extend(["**INCOMPLETE: failed queries are excluded; these ratios do not describe the full suite.**", ""]) for engine, data in summary["engines"].items(): cells = [] for label in ("head_v2_vs_base_v2", "head_v2_vs_head_v1"): + if label not in data["comparisons"]: + cells.append("No complete pairs") + continue value = data["comparisons"][label] low, high = value["interval_95"] cells.append(f"{value['ratio']:.3f} ({low:.3f}–{high:.3f}), {value['verdict']}") - lines.append(f"| {engine} | {len(data['queries'])} | {' | '.join(cells)} |") + lines.append(f"| {engine} | {len(data['queries'])}/{len(manifest['queries'][engine])} | {' | '.join(cells)} |") + for failure in summary.get("failures", []): + lines.append(f"\nFailed {failure['engine']} Q{failure['query']} {failure['variant']}: {failure['error']}") for engine, data in summary["engines"].items(): lines.extend( [ @@ -211,6 +218,23 @@ def markdown(summary: dict[str, Any], manifest: dict[str, Any]) -> str: return "\n".join(lines) + "\n" +def completed_comparison( + rows: list[dict[str, Any]], manifest: dict[str, Any], failures: list[dict[str, Any]] +) -> dict[str, Any]: + result: dict[str, Any] = {"complete": not failures, "engines": {}, "failures": failures} + for engine, expected in manifest["queries"].items(): + failed = {failure["query"] for failure in failures if failure["engine"] == engine} + queries = [query for query in expected if query not in failed] + kept = [row for row in rows if row["engine"] == engine and row["query"] in queries] + if queries: + data = summarize(kept, [engine], queries, manifest["rounds"])["engines"][engine] + else: + data = {"queries": [], "comparisons": {}} + data["failed_queries"] = sorted(failed) + result["engines"][engine] = data + return result + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("suite", choices=("tpch", "tpcds", "clickbench")) @@ -224,12 +248,17 @@ def main() -> None: parser.add_argument("--iterations", type=int, default=7) parser.add_argument("--warmup", type=int, default=2) parser.add_argument("--rounds", type=int, default=2) + parser.add_argument( + "--query-timeout", type=float, default=300, help="Seconds per query process before recording a failure" + ) parser.add_argument("--threads", type=int, default=len(os.sched_getaffinity(0))) parser.add_argument("--opt", action="append", default=[]) parser.add_argument("--output-dir", type=Path, required=True) args = parser.parse_args() if not 0 <= args.warmup < args.iterations or args.rounds < 1 or args.threads < 1: parser.error("Require 0 <= warmup < iterations and positive rounds/threads") + if not math.isfinite(args.query_timeout) or args.query_timeout <= 0: + parser.error("Require a positive finite query timeout") engines = args.engines.split(",") if len(set(engines)) != len(engines) or not set(engines) <= {"duckdb", "datafusion"}: parser.error("Engines must be unique members of duckdb,datafusion") @@ -248,6 +277,7 @@ def main() -> None: "iterations": args.iterations, "warmup": args.warmup, "rounds": args.rounds, + "query_timeout_seconds": args.query_timeout, "threads": args.threads, "options": options, "affinity": sorted(os.sched_getaffinity(0)), @@ -256,6 +286,8 @@ def main() -> None: "checkout_commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), } rows: list[dict[str, Any]] = [] + failures: list[dict[str, Any]] = [] + failed_queries: set[tuple[str, int]] = set() for engine in engines: binaries = { "base": args.base_dir.resolve() / f"{engine}-bench", @@ -284,6 +316,8 @@ def main() -> None: write_json(output / "manifest.json", manifest) for turn in range(args.rounds): for index, query in enumerate(queries): + if (engine, query) in failed_queries: + continue start = index % len(VARIANTS) order = VARIANTS[start:] + VARIANTS[:start] if turn % 2: @@ -310,10 +344,31 @@ def main() -> None: log_path = stem.with_suffix(".log") try: with log_path.open("w", encoding="utf-8") as log: - subprocess.run(cmd, env=envs[revision], stdout=log, stderr=subprocess.STDOUT, check=True) - except subprocess.CalledProcessError: + subprocess.run( + cmd, + env=envs[revision], + stdout=log, + stderr=subprocess.STDOUT, + check=True, + timeout=args.query_timeout, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: print(log_path.read_text(encoding="utf-8")[-8000:], flush=True) - raise + failures.append( + { + "engine": engine, + "query": query, + "round": turn, + "variant": variant, + "error": str(error), + "log": str(log_path), + "command": cmd, + } + ) + failed_queries.add((engine, query)) + write_json(output / "failures.json", failures) + print(f"FAILED {engine} q{query}; continuing with other queries", flush=True) + break value = measurement( stem.with_suffix(".jsonl"), query=query, @@ -333,11 +388,13 @@ def main() -> None: query_sets = list(manifest["queries"].values()) if any(queries != query_sets[0] for queries in query_sets): raise ValueError("Engines enumerate different query sets; report them separately") - summary = summarize(rows, engines, query_sets[0], args.rounds) + summary = completed_comparison(rows, manifest, failures) write_json(output / "summary.json", summary) report = markdown(summary, manifest) (output / "summary.md").write_text(report, encoding="utf-8") print(report, flush=True) + if failures: + raise SystemExit(1) if __name__ == "__main__": diff --git a/scripts/tests/test_scan_versions.py b/scripts/tests/test_scan_versions.py index 9810702712e..0bdd661c1f2 100644 --- a/scripts/tests/test_scan_versions.py +++ b/scripts/tests/test_scan_versions.py @@ -96,17 +96,19 @@ def test_rejects_mismatched_build_revision_and_executable(comparison: ModuleType comparison.binary_provenance(binary, "base-sha") +@pytest.mark.parametrize("failure_mode", [None, "exit", "timeout"]) def test_runs_each_engine_and_scan_version_with_the_expected_binary( - comparison: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + comparison: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_mode: str | None ): program = f"""#!{sys.executable} import json import os import subprocess +import time from pathlib import Path import sys if '--print-queries' in sys.argv: - print('1\\n2') + print('1\\n2\\n3') raise SystemExit() def arg(name): return sys.argv[sys.argv.index(name) + 1] @@ -114,6 +116,10 @@ def arg(name): raise SystemExit('benchmark runner must not be unknown in CI; pass --runner') revision = Path(__file__).parent.name engine = Path(__file__).name.removesuffix('-bench') +if os.environ.get('FAIL_QUERY') == arg('--queries') and engine == 'duckdb': + if os.environ.get('FAIL_MODE') == 'timeout': + time.sleep(2) + raise SystemExit('intentional query failure') version = 1 if arg('--scan-v1') == 'true' else 2 duration = 150 if revision == 'base' else 100 if version == 2 else 200 samples = [9000, 8000] + [duration] * (int(arg('--iterations')) - 2) @@ -124,6 +130,9 @@ def arg(name): Path(arg('-o')).write_text(json.dumps(record)) """ monkeypatch.setenv("CI", "true") + if failure_mode: + monkeypatch.setenv("FAIL_QUERY", "2") + monkeypatch.setenv("FAIL_MODE", failure_mode) for revision in ("base", "head"): directory = tmp_path / revision directory.mkdir() @@ -175,18 +184,30 @@ def with_elf_notes(args: list[str], *, text: bool, env: dict[str, str] | None = "2", "--output-dir", str(output), + "--query-timeout", + "0.5" if failure_mode == "timeout" else "300", ], ) - comparison.main() + if failure_mode: + with pytest.raises(SystemExit) as failure: + comparison.main() + assert failure.value.code == 1 + else: + comparison.main() rows = json.loads((output / "measurements.json").read_text(encoding="utf-8")) - assert len(rows) == 12 + assert len(rows) == (15 if failure_mode else 18) for row in rows: revision, version = row["variant"].split("-") assert Path(row["command"][0]).parent.name == revision flag = row["command"].index("--scan-v1") assert row["command"][flag + 1] == str(version == "v1").lower() summary = json.loads((output / "summary.json").read_text(encoding="utf-8")) - assert summary["complete"] is True + assert summary["complete"] is (failure_mode is None) + assert [row["query"] for row in summary["engines"]["duckdb"]["queries"]] == ([1, 3] if failure_mode else [1, 2, 3]) + assert [row["query"] for row in summary["engines"]["datafusion"]["queries"]] == [1, 2, 3] + assert len(summary["failures"]) == (1 if failure_mode else 0) + if failure_mode: + assert "INCOMPLETE" in (output / "summary.md").read_text(encoding="utf-8") for engine in ("duckdb", "datafusion"): comparisons = summary["engines"][engine]["comparisons"] assert comparisons["head_v2_vs_base_v2"]["ratio"] == pytest.approx(2 / 3) From 4efceab7a6ea85c1664b2b7e256e4f5e891d86ef Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 11:41:18 +0000 Subject: [PATCH 16/28] Keep inline DataFusion scans on one task across IO waits Drive the partition through one owned task when engine partitions already fill the workers. Defer that task until startup work has drained, and retain independent IO sessions for admitted roots. Avoid the repeated CPU-task handoffs of the previous inline path. Signed-off-by: "Joe Isaacs" --- vortex-datafusion/src/persistent/scan_v2.rs | 56 +++++++++------------ 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/vortex-datafusion/src/persistent/scan_v2.rs b/vortex-datafusion/src/persistent/scan_v2.rs index 92bb2e281d9..87edd3a140f 100644 --- a/vortex-datafusion/src/persistent/scan_v2.rs +++ b/vortex-datafusion/src/persistent/scan_v2.rs @@ -121,7 +121,6 @@ impl ScanDriver { ) -> BoxStream<'static, VortexResult> { let map: MapBatch = Arc::new(map); let in_flight = self.in_flight; - let inline = in_flight <= 1; let scope = WorkScope { file_ordinal: 0, rows: scan.rows().clone(), @@ -149,22 +148,30 @@ impl ScanDriver { ) }) .flatten_unordered(initial_count); - let (chunk_handle, chunk_io, chunk_map) = - (self.handle.clone(), Arc::clone(&io), Arc::clone(&map)); - let batches = run_stream(self.handle, root, scope, Arc::clone(&io), map, inline) - .map_ok(move |yielded| match yielded { - Yielded::Batch(start, batch) => { - stream::once(future::ready(Ok((start, batch)))).boxed() - } - Yielded::Spawn(scope, chunk) => chunk_stream( - &chunk_handle, - chunk, - scope, - Arc::clone(&chunk_io), - Arc::clone(&chunk_map), - ), - }) - .try_flatten_unordered(in_flight); + let batches = if in_flight <= 1 { + // Start only after startup batches drain, and keep the run on one task across IO waits. + stream::once(async move { chunk_stream(&self.handle, root, scope, io, map) }) + .flatten() + .boxed() + } else { + let (chunk_handle, chunk_io, chunk_map) = + (self.handle.clone(), Arc::clone(&io), Arc::clone(&map)); + run_stream(self.handle, root, scope, io, map) + .map_ok(move |yielded| match yielded { + Yielded::Batch(start, batch) => { + stream::once(future::ready(Ok((start, batch)))).boxed() + } + Yielded::Spawn(scope, chunk) => chunk_stream( + &chunk_handle, + chunk, + scope, + Arc::clone(&chunk_io), + Arc::clone(&chunk_map), + ), + }) + .try_flatten_unordered(in_flight) + .boxed() + }; // Drain startup batches before admitting bulk work, so TopK can tighten its bound. let batches = initial.chain(batches); // The prepared scan holds the file's shared cache: keep it until the stream ends, so @@ -208,34 +215,24 @@ struct RunState { handle: Handle, /// Taken while a CPU step runs on its task and put back with its result. run: Option, - /// Each admitted root owns a session, cleared independently when it retires. - io: Arc, map: MapBatch, - /// Whether a spawned chunk runs on this run, as another root, instead of being yielded. - inline: bool, } /// Runs `root` over `scope` on a planning run of its own, reading through `io`, each CPU step -/// on a `spawn_cpu` task, and yields its mapped batches and, unless `inline`, the chunks it -/// spawns. Inline, a spawned chunk runs on this run beside the others: the run's steps already -/// interleave every chunk's work, and a run per chunk would only add a hand-off per chunk for -/// parallelism the engine provides. +/// on a `spawn_cpu` task, and yields its mapped batches and the chunks it spawns. fn run_stream( handle: Handle, root: Box, scope: WorkScope, io: Arc, map: MapBatch, - inline: bool, ) -> BoxStream<'static, VortexResult> { let mut run = Run::new(); run.admit(root, scope, io.session()); let state = RunState { handle, run: Some(run), - io, map, - inline, }; stream::try_unfold(state, |mut state| async move { loop { @@ -267,9 +264,6 @@ fn run_stream( Step::Batch(start, batch) => { return Ok(Some((Yielded::Batch(start, batch), state))); } - Step::Spawn(scope, child) if state.inline => { - run.admit(child, scope, state.io.session()); - } Step::Spawn(scope, child) => { return Ok(Some((Yielded::Spawn(scope, child), state))); } From a8499d11dbbab5dbafd34c7d0c5da91b719dcaf2 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 11:55:02 +0000 Subject: [PATCH 17/28] Keep benchmark cache paths consistent on NVMe Redirect the normal data and target directories to local NVMe instead of changing the workspace root and supplying a local URL override. ClickBench generation uses its default cache path; both generators and benchmark binaries now share that path while downloads and spill files remain on NVMe. Signed-off-by: "Joe Isaacs" --- .github/workflows/scan-v2-comparison.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/scan-v2-comparison.yml b/.github/workflows/scan-v2-comparison.yml index 6f61b75a6f9..9d2d85937f8 100644 --- a/.github/workflows/scan-v2-comparison.yml +++ b/.github/workflows/scan-v2-comparison.yml @@ -157,10 +157,12 @@ jobs: data_root=/mnt/ephemeral/scan-comparison-data fi data_dir="$data_root/$GITHUB_RUN_ID/$CASE_ID" - sudo install -d -o "$(id -u)" -g "$(id -g)" "$data_dir" "$data_dir/target" - ln -s "$GITHUB_WORKSPACE/Cargo.lock" "$data_dir/Cargo.lock" - echo "COMPARISON_DATA_DIR=$data_dir" >> "$GITHUB_ENV" - echo "COMPARISON_DATA_URL=file://$data_dir/" >> "$GITHUB_ENV" + sudo install -d -o "$(id -u)" -g "$(id -g)" "$data_dir" "$data_dir/data" "$data_dir/tmp" + mv target "$data_dir/target" + ln -s "$data_dir/target" target + ln -s "$data_dir/data" vortex-bench/data + mkdir -p scan-comparison-results + ln -s "$data_dir/tmp" scan-comparison-results/tmp - name: Prepare shared benchmark files env: SUITE: ${{ matrix.suite }} @@ -171,9 +173,7 @@ jobs: if [[ -n "$SCALE" ]]; then options+=(--opt "scale-factor=$SCALE") fi - options+=(--opt "remote-data-dir=$COMPARISON_DATA_URL") - cd "$COMPARISON_DATA_DIR" - "$GITHUB_WORKSPACE/target/scan-comparison/head/data-gen" "$SUITE" --formats vortex "${options[@]}" + target/scan-comparison/head/data-gen "$SUITE" --formats vortex "${options[@]}" - name: Prepare benchmark CPUs run: sudo bash scripts/setup-benchmark.sh - name: Compare base V2, head V2 and head V1 sequentially @@ -191,7 +191,7 @@ jobs: --base-dir target/scan-comparison/base --head-dir target/scan-comparison/head \ --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" \ --iterations 7 --warmup 2 --rounds 2 --query-timeout 300 \ - --opt "remote-data-dir=$COMPARISON_DATA_URL" --output-dir scan-comparison-results "${options[@]}" + --output-dir scan-comparison-results "${options[@]}" - name: Publish comparison summary if: always() run: | From 9d0e2717d6ac4bee9c5496feebece602e0ee39c0 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 12:09:37 +0000 Subject: [PATCH 18/28] Stream stored projection pieces through filtered scans Keep filter and IO work in large chunks while returning stored scan pieces directly for conversion. Avoid gathering filtered scan results into a chunked array and copying across stored boundaries. Keep bounded gathering for filtered random-access work. Verify filtered and unfiltered stored boundaries and array equality across three layout strategies. Signed-off-by: "Joe Isaacs" --- vortex-layout/src/scan/execution/projection.rs | 18 ++++++++++++------ vortex-layout/src/scan/v2/tests.rs | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/vortex-layout/src/scan/execution/projection.rs b/vortex-layout/src/scan/execution/projection.rs index 6722f4cc775..507b2487b76 100644 --- a/vortex-layout/src/scan/execution/projection.rs +++ b/vortex-layout/src/scan/execution/projection.rs @@ -20,6 +20,7 @@ use vortex_scan::planning::planner::PlannerOutput; use vortex_scan::planning::planner::State; use vortex_scan::planning::planner::WorkScope; +use crate::plan::physical::pipeline::AccessMode; use crate::plan::physical::pipeline::Scan; use crate::plan::physical::pipeline::Split; use crate::plan::physical::selection::selected; @@ -31,8 +32,8 @@ use crate::scan::execution::adaptor::ScanAdaptor; const MAX_OUTPUT_ROWS: usize = 65_536; -/// Keeps sparse selections together and subdivides dense filtered selections to bound copies -/// when converting chunked projection results to Arrow. Unfiltered scans stream their pieces. +/// Scan projections stream stored pieces. Random-access projections keep sparse selections +/// together and subdivide dense filtered selections to bound copies during Arrow conversion. pub(super) struct ProjectionPlanner { scan: Arc, selected: IntoIter, @@ -41,7 +42,10 @@ pub(super) struct ProjectionPlanner { impl ProjectionPlanner { /// Creates a planner for `selected`. pub(super) fn new(scan: Arc, selected: SelectedRows) -> Self { - let selected = if scan.filter.is_none() || selected.mask.true_count() <= MAX_OUTPUT_ROWS { + let selected = if scan.access_mode() == AccessMode::Scan + || scan.filter.is_none() + || selected.mask.true_count() <= MAX_OUTPUT_ROWS + { vec![selected] } else { (0..selected.mask.len()) @@ -97,8 +101,8 @@ impl Planner for ProjectionPlanner { } } -/// Projects selected rows in order, streaming unfiltered pieces and gathering filtered pieces -/// into one batch. A selection that projects to no rows finishes without a batch. +/// Projects selected rows in order, streaming scan pieces and gathering filtered random-access +/// pieces into one batch. A selection that projects to no rows finishes without a batch. struct ProjectionMorsel { scan: Arc, dtype: DType, @@ -190,7 +194,9 @@ impl Morsel for ProjectionMorsel { AdaptorStep::Piece(piece) if piece.array.is_empty() => MorselOutput::Continue, // Preserve stored boundaries instead of making Arrow conversion copy across them. // The chunk planner bounds the row domain of these pieces. - AdaptorStep::Piece(piece) if self.scan.filter.is_none() => { + AdaptorStep::Piece(piece) + if self.scan.filter.is_none() || self.scan.access_mode() == AccessMode::Scan => + { MorselOutput::Batch(piece.array) } AdaptorStep::Piece(piece) => { diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index f625ee9eccf..b15bb41aafc 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -818,10 +818,18 @@ async fn primitive_column() -> VortexResult<()> { #[tokio::test] async fn stored_batches_share_one_work_chunk( #[values(Strategy::Flat, Strategy::Table, Strategy::ZonedTable)] strategy: Strategy, + #[values(false, true)] filtered: bool, ) -> VortexResult<()> { let session = new_session().with_tokio(); let (segments, layout) = write(&session, strategy.get().as_ref()).await?; - let scan = Arc::new(builder(&session, &segments, &layout)?.prepare()?); + let filter = filtered + .then(|| gt(get_item("ints", root()), lit(0i32)).bind(layout.dtype())) + .transpose()?; + let scan = Arc::new( + builder(&session, &segments, &layout)? + .with_some_filter(filter) + .prepare()?, + ); assert_eq!( row_chunks(scan.rows().clone(), scan.chunk_rows), vec![0..(CHUNKS * CHUNK_ROWS) as u64] @@ -834,7 +842,9 @@ async fn stored_batches_share_one_work_chunk( .map(|batch| batch.array) .collect::>(); let expected = (0..CHUNKS) - .map(|chunk| table_chunk(chunk * CHUNK_ROWS)) + .map(|chunk| { + table_chunk(chunk * CHUNK_ROWS)?.slice(usize::from(filtered && chunk == 0)..CHUNK_ROWS) + }) .collect::>>()?; assert_arrays_eq!( ChunkedArray::try_new(arrays, layout.dtype().clone())?, From 44e1952f6dfb09930220f9442abe67ce2095c91c Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 12:47:56 +0000 Subject: [PATCH 19/28] Preserve DataFusion read overlap by default Retain the previous per-worker chunk overlap after measurements showed that limiting each partition to one stream regressed Q6, Q15 and ClickBench. Keep independent IO sessions and the single-task path for explicit inline execution. Signed-off-by: "Joe Isaacs" --- vortex-datafusion/src/persistent/opener.rs | 16 ++++++---------- vortex-datafusion/src/persistent/scan_v2.rs | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index cac0ff7e4f3..643458cdb6a 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -137,8 +137,7 @@ pub(crate) struct VortexOpener { pub projection_pushdown: bool, pub scan_concurrency: Option, pub scan_v2: bool, - /// How many partitions the engine runs the scan on, so a V2 scan parallelises inside a - /// partition only as far as the engine has not. + /// How many partitions the engine runs the scan on, used to scale its default read concurrency. pub partitions: usize, /// Startup work shared by every file opened on this execution partition. pub initial_chunks: Arc, @@ -467,15 +466,12 @@ impl FileOpener for VortexOpener { { builder = builder.with_selection(selection.clone()); } - // Unless set, chunks run beside each other only as far as the engine's partitions - // leave cores idle: at one partition per core they run in the scan's own run. + // Reads and downstream consumers can leave a partition waiting, so keep multiple + // chunk streams available per worker even when partitions fill the cores. let workers = get_available_parallelism().unwrap_or(1); - let in_flight = scan_concurrency - .map_or_else( - || workers.div_ceil(partitions), - |concurrency| concurrency.saturating_mul(workers), - ) - .max(1); + let concurrency = + scan_concurrency.unwrap_or_else(|| workers.div_ceil(partitions).max(1)); + let in_flight = concurrency.saturating_mul(workers).max(1); let handle = vxf.session().handle(); // Lowering and planning a wide file is real CPU: it runs where compute runs. let scan = handle diff --git a/vortex-datafusion/src/persistent/scan_v2.rs b/vortex-datafusion/src/persistent/scan_v2.rs index 87edd3a140f..878738ddf9e 100644 --- a/vortex-datafusion/src/persistent/scan_v2.rs +++ b/vortex-datafusion/src/persistent/scan_v2.rs @@ -7,7 +7,7 @@ //! opener composes the planners over a [`PreparedScan`] and hands the driver the first; the //! driver runs it and everything it emits on planning runs it owns: the root run stepped on //! `spawn_cpu` tasks as the stream is polled, each spawned chunk on a task of its own, or on the -//! root run when the engine's partitions already fill the cores. Every read is a `spawn_io` task +//! root run when its chunk concurrency is one. Every read is a `spawn_io` task //! of the file's segment source. The DataFusion thread polling the stream does no scan work //! itself. From bd2653805c0629b3bdf2936410ff6aca50e85604 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 12:47:56 +0000 Subject: [PATCH 20/28] Keep incomplete comparison warnings outside result tables Signed-off-by: "Joe Isaacs" --- scripts/compare-scan-versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/compare-scan-versions.py b/scripts/compare-scan-versions.py index 8ddb5923ebf..ee0132ab641 100644 --- a/scripts/compare-scan-versions.py +++ b/scripts/compare-scan-versions.py @@ -185,7 +185,7 @@ def markdown(summary: dict[str, Any], manifest: dict[str, Any]) -> str: "| --- | ---: | --- | --- |", ] if not summary["complete"]: - lines.extend(["**INCOMPLETE: failed queries are excluded; these ratios do not describe the full suite.**", ""]) + lines[2:2] = ["**INCOMPLETE: failed queries are excluded; these ratios do not describe the full suite.**", ""] for engine, data in summary["engines"].items(): cells = [] for label in ("head_v2_vs_base_v2", "head_v2_vs_head_v1"): From a114a2a44b2bf1fa998ff3e1d70d7e41a57685c0 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 13:35:26 +0000 Subject: [PATCH 21/28] Reuse ordered segment locations for V2 range lookup Avoid constructing a hash map of every file segment before a narrow scan can read. Ordered locations share their existing storage and use binary search; unordered tables retain indexed lookup and duplicate ranges keep their last segment ID. Signed-off-by: "Joe Isaacs" --- vortex-layout/src/scan/v2/io.rs | 78 ++++++++++++++++++++++++++++---- vortex-layout/src/scan/v2/mod.rs | 2 +- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/vortex-layout/src/scan/v2/io.rs b/vortex-layout/src/scan/v2/io.rs index 0e690ebfde7..6998c98a021 100644 --- a/vortex-layout/src/scan/v2/io.rs +++ b/vortex-layout/src/scan/v2/io.rs @@ -34,15 +34,42 @@ use crate::segments::SegmentFuture; use crate::segments::SegmentId; use crate::segments::SegmentSource; -/// Byte ranges of a file's segments, keyed by `(offset, length)`, back to their segment ids. -pub(crate) type SegmentRanges = Arc>; +/// Byte ranges of a file's segments back to their segment ids. +pub(crate) type SegmentRanges = Arc; -pub(super) fn segment_ranges(locations: &[SegmentLocation]) -> SegmentRanges { +pub(crate) enum SegmentRangeIndex { + Ordered(Arc<[SegmentLocation]>), + Indexed(HashMap<(u64, usize), SegmentId>), +} + +impl SegmentRangeIndex { + fn get(&self, key: &(u64, usize)) -> Option { + match self { + Self::Ordered(locations) => { + let end = locations.partition_point(|location| { + (location.offset, location.length as usize) <= *key + }); + let index = end.checked_sub(1)?; + let location = &locations[index]; + ((location.offset, location.length as usize) == *key) + .then(|| SegmentId::from(u32::try_from(index).unwrap_or(u32::MAX))) + } + Self::Indexed(ranges) => ranges.get(key).copied(), + } + } +} + +pub(super) fn segment_ranges(locations: Arc<[SegmentLocation]>) -> SegmentRanges { + // File segment tables are ordered already. Reuse their storage so a narrow scan need not + // allocate and hash the ranges of every column before it can issue its first read. + if locations.is_sorted_by_key(|location| (location.offset, location.length)) { + return Arc::new(SegmentRangeIndex::Ordered(locations)); + } let ranges = locations.iter().enumerate().map(|(id, location)| { let id = SegmentId::from(u32::try_from(id).unwrap_or(u32::MAX)); ((location.offset, location.length as usize), id) }); - Arc::new(ranges.collect()) + Arc::new(SegmentRangeIndex::Indexed(ranges.collect())) } /// Serves splits from any [`SegmentSource`], by segment id. @@ -115,7 +142,7 @@ impl IoSource for SegmentIoSource { request.target ); }; - let segment = *self + let segment = self .ranges .get(&(offset, len)) .ok_or_else(|| vortex_err!("No segment at bytes {offset}+{len}"))?; @@ -240,6 +267,7 @@ mod tests { use futures::channel::oneshot; use futures::future; use parking_lot::Mutex; + use rstest::rstest; use vortex_array::buffer::BufferHandle; use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; @@ -259,6 +287,34 @@ mod tests { use crate::segments::SegmentId; use crate::segments::SegmentSource; + #[rstest] + #[case::empty(vec![])] + #[case::ordered(vec![(8, 0), (8, 4), (8, 4), (16, 4)])] + #[case::unordered(vec![(16, 4), (8, 4), (8, 0), (8, 4)])] + fn ranges_preserve_offsets_lengths_and_duplicate_ids( + #[case] keys: Vec<(u64, u32)>, + ) -> VortexResult<()> { + let locations = keys + .iter() + .map(|&(offset, length)| SegmentLocation { + offset, + length, + alignment: Alignment::none(), + }) + .collect(); + let ranges = segment_ranges(locations); + for key in [(0, 4), (8, 0), (8, 1), (8, 4), (16, 4), (20, 4)] { + let expected = keys + .iter() + .rposition(|&(offset, length)| (offset, length as usize) == key) + .map(u32::try_from) + .transpose()? + .map(SegmentId::from); + assert_eq!(ranges.get(&key), expected, "range {key:?}"); + } + Ok(()) + } + /// Serves four bytes per segment and counts the reads it is asked for. #[derive(Default)] struct CountingSegments(AtomicUsize); @@ -307,7 +363,7 @@ mod tests { length: 4, alignment: Alignment::none(), }); - let io = SegmentIoSource::new(segments, segment_ranges(&locations)); + let io = SegmentIoSource::new(segments, segment_ranges(Arc::from(locations))); let mut pending = request(IoIntent::Fetch, 1); pending.target = IoTarget::range(4, 4); io.submit(IoOwnerId(0), vec![pending, request(IoIntent::Fetch, 0)])?; @@ -340,7 +396,10 @@ mod tests { length: 4, alignment: Alignment::none(), }; - let io = SegmentIoSource::new(Arc::clone(&segments) as _, segment_ranges(&[location])); + let io = SegmentIoSource::new( + Arc::clone(&segments) as _, + segment_ranges(Arc::from([location])), + ); io.submit(IoOwnerId(0), vec![request(IoIntent::Prefetch, 0)])?; assert_eq!(segments.0.load(Ordering::Relaxed), 1); @@ -378,7 +437,10 @@ mod tests { length: 4, alignment: Alignment::none(), }; - let io = SegmentIoSource::new(Arc::clone(&segments) as _, segment_ranges(&[location])); + let io = SegmentIoSource::new( + Arc::clone(&segments) as _, + segment_ranges(Arc::from([location])), + ); io.submit( IoOwnerId(0), vec![ diff --git a/vortex-layout/src/scan/v2/mod.rs b/vortex-layout/src/scan/v2/mod.rs index def9d76afd4..f72afec32b8 100644 --- a/vortex-layout/src/scan/v2/mod.rs +++ b/vortex-layout/src/scan/v2/mod.rs @@ -228,7 +228,7 @@ impl ScanBuilder { } else { MAX_ROWS_PER_CHUNK }; - let ranges = cache.ranges(|| segment_ranges(&file.locations)); + let ranges = cache.ranges(|| segment_ranges(Arc::clone(&file.locations))); let values = cache.values(); let rows = self.row_range.unwrap_or(0..row_count); let access_mode = self From 7245606651a915f48e15ade3ba56aabd7c4f2dd6 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 14:12:22 +0000 Subject: [PATCH 22/28] Bound DataFusion Arrow batches within large V2 work chunks Slice oversized encoded output pieces before Arrow conversion, retaining the scan work and read window while bounding decoded values and downstream casts. Cover filtered and unfiltered inline and overlapping scans with exact output verification. Signed-off-by: "Joe Isaacs" --- vortex-datafusion/src/persistent/opener.rs | 38 +++++++++++++++++++++ vortex-datafusion/src/persistent/scan_v2.rs | 26 +++++++++++--- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 643458cdb6a..38cd0e15b81 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -1143,6 +1143,44 @@ mod tests { Ok(summary.size()) } + #[rstest] + #[tokio::test] + async fn v2_bounds_arrow_batches( + #[values(0, 2)] scan_concurrency: usize, + #[values(false, true)] filtered: bool, + ) -> anyhow::Result<()> { + let object_store = Arc::new(InMemory::new()) as Arc; + let batch = record_batch!(("a", UInt32, (0..150_000).collect::>()))?; + let schema = batch.schema(); + let size = + write_arrow_to_vortex(Arc::clone(&object_store), "batches.vortex", batch.clone()) + .await?; + let filter = filtered.then(|| logical2physical(&col("a").gt(lit(0_u32)), &schema)); + let mut opener = make_opener(object_store, TableSchema::from(Arc::clone(&schema)), filter); + opener.scan_v2 = true; + opener.scan_concurrency = Some(scan_concurrency); + opener.has_output_ordering = true; + let batches = opener + .open(PartitionedFile::new("batches.vortex", size))? + .await? + .try_collect::>() + .await?; + assert!(batches.iter().all(|batch| batch.num_rows() <= 65_536)); + let expected = if filtered { + batch.slice(1, batch.num_rows() - 1) + } else { + batch + }; + assert_arrays_eq!( + SESSION + .arrow() + .from_arrow_record_batch(concat_batches(&schema, &batches)?, &schema)?, + SESSION.arrow().from_arrow_record_batch(expected, &schema)?, + &mut SESSION.create_execution_ctx() + ); + Ok(()) + } + #[rstest] #[tokio::test] async fn startup_and_bulk_keep_filtered_selected_rows_once( diff --git a/vortex-datafusion/src/persistent/scan_v2.rs b/vortex-datafusion/src/persistent/scan_v2.rs index 878738ddf9e..827abddb6ac 100644 --- a/vortex-datafusion/src/persistent/scan_v2.rs +++ b/vortex-datafusion/src/persistent/scan_v2.rs @@ -46,6 +46,7 @@ mod tests; type MapBatch = Arc VortexResult + Send + Sync>; const INITIAL_CHUNK_ROWS: u64 = 8192; +const MAX_ARROW_BATCH_ROWS: usize = 65_536; /// Claims a partition's startup budget across its files, without duplicating their rows. pub(crate) fn initial_ranges(rows: Range, remaining: &AtomicUsize) -> Vec> { @@ -297,10 +298,27 @@ fn chunk_stream( match run.advance()? { Progress::Batch(batch) => { let start = batch.scope.rows.start; - let batch = map(batch.array); - let failed = batch.is_err(); - if send.send(batch.map(|batch| (start, batch))).await.is_err() || failed { - return Ok(()); + let len = batch.array.len(); + if len <= MAX_ARROW_BATCH_ROWS { + let batch = map(batch.array); + let failed = batch.is_err(); + if send.send(batch.map(|batch| (start, batch))).await.is_err() || failed { + return Ok(()); + } + } else { + // Retain large scan work chunks, but bound materialization before Arrow + // conversion so decoded values and downstream casts fit smaller working sets. + for offset in (0..len).step_by(MAX_ARROW_BATCH_ROWS) { + let array = batch.array.slice( + offset..offset.saturating_add(MAX_ARROW_BATCH_ROWS).min(len), + )?; + let batch = map(array); + let failed = batch.is_err(); + if send.send(batch.map(|batch| (start, batch))).await.is_err() || failed + { + return Ok(()); + } + } } } Progress::Spawn(scope, child) => { From 6c3f59295798f6e3ae5e53b5e5f0b903486b025f Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 14:29:48 +0000 Subject: [PATCH 23/28] Allow longer TPC-DS SF50 benchmark sample processes Allow 900 seconds for a seven-iteration SF50 process, retaining the 300-second limit for other cases and the 180-minute job budget. Continue to report failed or timed-out queries as incomplete. Signed-off-by: "Joe Isaacs" --- .github/workflows/scan-v2-comparison.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scan-v2-comparison.yml b/.github/workflows/scan-v2-comparison.yml index 9d2d85937f8..47c7f4354ca 100644 --- a/.github/workflows/scan-v2-comparison.yml +++ b/.github/workflows/scan-v2-comparison.yml @@ -182,6 +182,7 @@ jobs: SCALE: ${{ matrix.scale }} BASE_SHA: ${{ needs.resolve.outputs.base_sha }} HEAD_SHA: ${{ inputs.head_sha }} + QUERY_TIMEOUT_SECONDS: ${{ matrix.id == 'tpcds-50' && '900' || '300' }} run: | options=() if [[ -n "$SCALE" ]]; then @@ -190,7 +191,7 @@ jobs: bash scripts/bench-taskset.sh python3 scripts/compare-scan-versions.py "$SUITE" \ --base-dir target/scan-comparison/base --head-dir target/scan-comparison/head \ --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" \ - --iterations 7 --warmup 2 --rounds 2 --query-timeout 300 \ + --iterations 7 --warmup 2 --rounds 2 --query-timeout "$QUERY_TIMEOUT_SECONDS" \ --output-dir scan-comparison-results "${options[@]}" - name: Publish comparison summary if: always() From 0ac01b8ab611eb520c33008f0a15f608f0ddfe43 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 15:25:25 +0000 Subject: [PATCH 24/28] Capture dedicated Scan V2 diagnostic profiles in Actions Reuse source-verified native artifacts on Graviton4 for serial V1/V2 TPC-H Q13 and ClickBench Q19 metrics and Samply profiles. Resolve names on the runner after checking GNU build IDs, retaining every sample and stack table. Signed-off-by: "Joe Isaacs" --- .github/workflows/pr-bench-sql.yml | 33 +++++- .github/workflows/scan-v2-profile.yml | 164 ++++++++++++++++++++++++++ scripts/symbolicate-samply.py | 89 ++++++++++++++ 3 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/scan-v2-profile.yml create mode 100644 scripts/symbolicate-samply.py diff --git a/.github/workflows/pr-bench-sql.yml b/.github/workflows/pr-bench-sql.yml index d37c627ad9d..e46265a05ce 100644 --- a/.github/workflows/pr-bench-sql.yml +++ b/.github/workflows/pr-bench-sql.yml @@ -17,6 +17,14 @@ on: required: false type: string default: "pr" + profile_run_id: + required: false + type: string + default: "" + profile_source_sha: + required: false + type: string + default: "" workflow_dispatch: inputs: matrix_preset: @@ -30,6 +38,17 @@ on: - "pr-all" - "pr-full" - "scan-v2" + - "scan-v2-profile" + profile_run_id: + description: "Existing scan comparison run supplying profile binaries" + required: false + type: string + default: "" + profile_source_sha: + description: "Exact source SHA in the profile binary build manifest" + required: false + type: string + default: "" permissions: contents: read @@ -38,7 +57,7 @@ permissions: jobs: sql: - if: inputs.matrix_preset != 'scan-v2' + if: inputs.matrix_preset != 'scan-v2' && inputs.matrix_preset != 'scan-v2-profile' uses: ./.github/workflows/sql-bench-matrix.yml secrets: inherit with: @@ -52,3 +71,15 @@ jobs: with: head_sha: ${{ github.event.pull_request.head.sha || github.sha }} base_ref: ${{ github.event.pull_request.base.sha || 'origin/develop' }} + + scan-profile: + if: inputs.matrix_preset == 'scan-v2-profile' + permissions: + actions: read + contents: read + id-token: write + uses: ./.github/workflows/scan-v2-profile.yml + secrets: inherit + with: + source_run_id: ${{ inputs.profile_run_id }} + source_sha: ${{ inputs.profile_source_sha }} diff --git a/.github/workflows/scan-v2-profile.yml b/.github/workflows/scan-v2-profile.yml new file mode 100644 index 00000000000..01cfe79919c --- /dev/null +++ b/.github/workflows/scan-v2-profile.yml @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +name: Scan V2 Profiles + +on: + workflow_call: + inputs: + source_run_id: + required: true + type: string + source_sha: + required: true + type: string + +permissions: + actions: read + contents: read + id-token: write + +jobs: + profile: + if: github.repository == 'vortex-data/vortex' + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - id: tpch-10-q13 + suite: tpch + scale: "10.0" + query: "13" + - id: clickbench-q19 + suite: clickbench + scale: "" + query: "19" + runs-on: >- + ${{ format('runs-on={0}/region=us-east-1/runner=bench-dedicated/family=c8gd.metal-24xl/image=ubuntu26-full-arm64-pre-v2/tag=profile-{1}/extras=s3-cache', github.run_id, matrix.id) }} + env: + FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" + steps: + - name: Validate profile inputs + env: + SOURCE_RUN_ID: ${{ inputs.source_run_id }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + [[ "$SOURCE_RUN_ID" =~ ^[0-9]+$ ]] + [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.source_sha }} + persist-credentials: false + - uses: ./.github/actions/assert-cpu + - uses: ./.github/actions/setup-prebuild + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/setup-duckdb + - uses: ./.github/actions/system-info + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ inputs.source_run_id }} + name: scan-comparison-head-binaries + path: target/scan-profile + - name: Place inputs and profiles on local NVMe + env: + CASE_ID: ${{ matrix.id }} + run: | + data_dir="/mnt/ephemeral/scan-profile-data/$GITHUB_RUN_ID/$CASE_ID" + sudo install -d -o "$(id -u)" -g "$(id -g)" "$data_dir" "$data_dir/data" "$data_dir/tmp" + mv target "$data_dir/target" + ln -s "$data_dir/target" target + ln -s "$data_dir/data" vortex-bench/data + mkdir -p scan-profile-results + ln -s "$data_dir/tmp" scan-profile-results/tmp + - name: Verify measured binaries + env: + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + chmod +x target/scan-profile/{data-gen,datafusion-bench} + python3 - <<'PY' + import hashlib + import json + import os + from pathlib import Path + directory = Path("target/scan-profile") + manifest = json.loads((directory / "build-manifest.json").read_text()) + assert manifest["commit"] == os.environ["SOURCE_SHA"] + for name in ("data-gen", "datafusion-bench"): + with (directory / name).open("rb") as executable: + assert hashlib.file_digest(executable, "sha256").hexdigest() == manifest["binaries"][name] + (Path("scan-profile-results") / "build-manifest.json").write_text(json.dumps(manifest, indent=2)) + PY + readelf -n target/scan-profile/datafusion-bench > scan-profile-results/elf-notes.txt + - name: Install pinned profiler + run: cargo install --locked samply --version 0.13.1 --root "$RUNNER_TEMP/samply" + - name: Prepare shared input files + env: + SUITE: ${{ matrix.suite }} + SCALE: ${{ matrix.scale }} + run: | + options=() + if [[ -n "$SCALE" ]]; then + options+=(--opt "scale-factor=$SCALE") + fi + export LD_LIBRARY_PATH="$(find target/scan-profile -name libduckdb.so -printf '%h:')${LD_LIBRARY_PATH:-}" + target/scan-profile/data-gen "$SUITE" --formats vortex "${options[@]}" + - name: Prepare profile CPUs + run: sudo bash scripts/setup-benchmark.sh + - name: Record V1 and V2 metrics and CPU profiles serially + env: + SUITE: ${{ matrix.suite }} + SCALE: ${{ matrix.scale }} + QUERY: ${{ matrix.query }} + RUST_LOG: warn + TOKIO_WORKER_THREADS: "94" + DATAFUSION_EXECUTION_TARGET_PARTITIONS: "94" + run: | + options=() + if [[ -n "$SCALE" ]]; then + options+=(--opt "scale-factor=$SCALE") + fi + export TMPDIR="$PWD/scan-profile-results/tmp" + export LD_LIBRARY_PATH="$(find target/scan-profile -name libduckdb.so -printf '%h:')${LD_LIBRARY_PATH:-}" + for version in 1 2; do + scan_v1=true + if [[ "$version" == 2 ]]; then + scan_v1=false + fi + command=("$PWD/target/scan-profile/datafusion-bench" "$SUITE" + --formats vortex --scan-v1 "$scan_v1" --runner scan-v2-profile + --queries "$QUERY" --hide-progress-bar --display-format gh-json "${options[@]}") + bash scripts/bench-taskset.sh "${command[@]}" --iterations 1 --show-metrics --emit-plan \ + -o "scan-profile-results/v$version-metrics.jsonl" > "scan-profile-results/v$version-metrics.log" 2>&1 + bash scripts/bench-taskset.sh sudo -n env \ + RUST_LOG="$RUST_LOG" TOKIO_WORKER_THREADS="$TOKIO_WORKER_THREADS" \ + DATAFUSION_EXECUTION_TARGET_PARTITIONS="$DATAFUSION_EXECUTION_TARGET_PARTITIONS" TMPDIR="$TMPDIR" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH" \ + "$RUNNER_TEMP/samply/bin/samply" record --save-only --rate 1000 \ + --output "$PWD/scan-profile-results/v$version.profile.json.gz" -- \ + "${command[@]}" --iterations 300 -o "$PWD/scan-profile-results/v$version-sampled.jsonl" \ + > "scan-profile-results/v$version-profile.log" 2>&1 + sudo chmod a+r "scan-profile-results/v$version.profile.json.gz" + done + - name: Resolve native symbols on the profile host + run: | + curl --fail --location --retry 3 \ + "https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_SHA/scripts/symbolicate-samply.py" \ + --output scan-profile-results/symbolicate-samply.py + for version in 1 2; do + python3 scan-profile-results/symbolicate-samply.py \ + "scan-profile-results/v$version.profile.json.gz" --binary target/scan-profile/datafusion-bench \ + --output "scan-profile-results/v$version-symbolicated.profile.json.gz" + done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: always() + with: + name: scan-profile-${{ matrix.id }} + path: | + scan-profile-results/ + !scan-profile-results/tmp/** diff --git a/scripts/symbolicate-samply.py b/scripts/symbolicate-samply.py new file mode 100644 index 00000000000..9265ae55162 --- /dev/null +++ b/scripts/symbolicate-samply.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Resolve one ELF's Samply function names without changing samples or stack tables.""" + +import argparse +import gzip +import json +import re +import shutil +import subprocess +from pathlib import Path +from typing import Any + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("profile", type=Path) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + notes = subprocess.check_output(["readelf", "-n", str(args.binary)], text=True) + matched = re.search(r"Build ID: ([0-9a-fA-F]+)", notes) + if matched is None: + raise ValueError("The ELF has no GNU build ID") + build_id = matched.group(1).lower() + with gzip.open(args.profile, "rt", encoding="utf-8") as source: + profile = json.load(source) + libraries = { + index for index, library in enumerate(profile["libs"]) if (library.get("codeId") or "").lower() == build_id + } + if not libraries: + raise ValueError(f"The profile does not contain ELF build ID {build_id}") + entries: list[tuple[dict[str, Any], int, int]] = [] + addresses: set[int] = set() + for thread in profile["threads"]: + functions = thread["funcTable"] + resources = thread["resourceTable"] + strings = thread["stringArray"] + for index, resource in enumerate(functions["resource"]): + if resource is None or resource < 0 or resources["lib"][resource] not in libraries: + continue + name = strings[functions["name"][index]] + if re.fullmatch(r"0x[0-9a-fA-F]+", name): + address = int(name, 16) + entries.append((thread, index, address)) + addresses.add(address) + symbols: dict[int, str] = {} + if addresses: + symbolizer = shutil.which("llvm-symbolizer") + if symbolizer is None: + candidates = { + int(path.name.rsplit("-", 1)[1]): path for path in Path("/usr/bin").glob("llvm-symbolizer-[0-9]*") + } + if not candidates: + raise ValueError("llvm-symbolizer is required") + symbolizer = str(candidates[max(candidates)]) + result = subprocess.run( + [symbolizer, f"--obj={args.binary}", "--no-inlines", "--output-style=JSON"], + input="".join(f"0x{address:x}\n" for address in sorted(addresses)), + text=True, + capture_output=True, + check=True, + ) + for line in result.stdout.splitlines(): + value = json.loads(line) + for item in value if isinstance(value, list) else [value]: + names = item.get("Symbol", []) + if names and names[0].get("FunctionName"): + symbols[int(item["Address"], 16)] = names[0]["FunctionName"] + if not symbols: + raise ValueError("No sampled addresses could be resolved") + for thread, index, address in entries: + if address in symbols: + thread["funcTable"]["name"][index] = len(thread["stringArray"]) + thread["stringArray"].append(symbols[address]) + profile["meta"]["nativeSymbolication"] = { + "buildId": build_id, + "resolvedAddresses": len(symbols), + "sampledAddresses": len(addresses), + } + with gzip.open(args.output, "wt", encoding="utf-8") as output: + json.dump(profile, output, separators=(",", ":")) + print(f"Resolved {len(symbols)}/{len(addresses)} native addresses for {build_id}") + + +if __name__ == "__main__": + main() From f62d4e38c9af8e2f9a6b6377a848262a24ee90e6 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 15:36:27 +0000 Subject: [PATCH 25/28] Bound Arrow materialization for narrow fixed-width scans Retain stored boundaries for wide and variable-width conversions while bounding projected rows up to 16 bytes to 65536 rows. This keeps the narrow aggregate working-set benefit without the measured wide TPC-H Q1 regression. Signed-off-by: "Joe Isaacs" --- vortex-datafusion/src/persistent/opener.rs | 2 +- vortex-datafusion/src/persistent/scan_v2.rs | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 38cd0e15b81..de219b358bc 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -1145,7 +1145,7 @@ mod tests { #[rstest] #[tokio::test] - async fn v2_bounds_arrow_batches( + async fn v2_bounds_narrow_arrow_batches( #[values(0, 2)] scan_concurrency: usize, #[values(false, true)] filtered: bool, ) -> anyhow::Result<()> { diff --git a/vortex-datafusion/src/persistent/scan_v2.rs b/vortex-datafusion/src/persistent/scan_v2.rs index 827abddb6ac..05ddda6888d 100644 --- a/vortex-datafusion/src/persistent/scan_v2.rs +++ b/vortex-datafusion/src/persistent/scan_v2.rs @@ -47,6 +47,7 @@ type MapBatch = Arc VortexResult + Send + Sync> const INITIAL_CHUNK_ROWS: u64 = 8192; const MAX_ARROW_BATCH_ROWS: usize = 65_536; +const MAX_ARROW_ROW_BYTES: usize = 16; /// Claims a partition's startup budget across its files, without duplicating their rows. pub(crate) fn initial_ranges(rows: Range, remaining: &AtomicUsize) -> Vec> { @@ -299,7 +300,15 @@ fn chunk_stream( Progress::Batch(batch) => { let start = batch.scope.rows.start; let len = batch.array.len(); - if len <= MAX_ARROW_BATCH_ROWS { + // Narrow values and their casts benefit from a small working set. Wider + // conversions retain stored boundaries to amortize their setup cost. + let split = len > MAX_ARROW_BATCH_ROWS + && batch + .array + .dtype() + .element_size() + .is_some_and(|bytes| (1..=MAX_ARROW_ROW_BYTES).contains(&bytes)); + if !split { let batch = map(batch.array); let failed = batch.is_err(); if send.send(batch.map(|batch| (start, batch))).await.is_err() || failed { From ddf747c1131cdbf94efa107c640e1f04b9ab9a44 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 16:05:33 +0000 Subject: [PATCH 26/28] Stream completed static filter pieces into projection For dense scans with one static predicate, project each completed piece within the existing I/O session instead of waiting for the full filter window. Keep the combined-mask path for multiple predicates, dynamic bounds and sparse demand. A gated-read regression fails before the change and verifies early output plus the complete result after it. Signed-off-by: "Joe Isaacs" --- vortex-layout/src/scan/execution/filter.rs | 43 ++++++- .../src/scan/execution/projection.rs | 4 +- vortex-layout/src/scan/v2/tests.rs | 110 +++++++++++++++++- 3 files changed, 151 insertions(+), 6 deletions(-) diff --git a/vortex-layout/src/scan/execution/filter.rs b/vortex-layout/src/scan/execution/filter.rs index f9a28cf05fc..543196e77db 100644 --- a/vortex-layout/src/scan/execution/filter.rs +++ b/vortex-layout/src/scan/execution/filter.rs @@ -27,6 +27,7 @@ use vortex_mask::Mask; use vortex_scan::planning::planner::Planner; use vortex_scan::planning::planner::PlannerOutput; use vortex_scan::planning::planner::State; +use vortex_scan::planning::planner::WorkScope; use vortex_session::VortexSession; use self::ordering::AdaptiveOrder; @@ -38,6 +39,7 @@ use super::SelectedRows; use super::adaptor::AdaptorStep; use super::adaptor::Piece; use super::adaptor::ScanAdaptor; +use super::projection::ProjectionMorsel; use super::projection::ProjectionPlanner; use crate::plan::physical::FilterPlan; use crate::plan::physical::PlanRef; @@ -135,7 +137,8 @@ fn metadata_only(plan: &PlanRef) -> VortexResult { Ok(true) } -/// Evaluates a chunk's conjuncts over its selected rows, then hands the mask to projection. +/// Evaluates a chunk's conjuncts over its selected rows. A single static predicate streams its +/// completed pieces to projection; other filters hand off their combined mask. /// /// Dynamic pruning is refreshed before data prefetch. Each chunk takes an order from the shared /// cost estimates, occasionally exploring another order. A dynamic conjunct is revisited if its @@ -149,6 +152,8 @@ pub(super) struct FilterPlanner { active: Option, dynamic: Vec, pieces: Vec, + /// Dense input lets a single static predicate hand each completed piece to projection. + streamed_rows: Option, rechecks: RecheckStats, announced: bool, /// Ids for hints, counting down from `u32::MAX` so they never meet an adaptor's fetch ids. @@ -269,6 +274,11 @@ impl FilterPlanner { "v2 predicate order selected" ); Self { + streamed_rows: (scan.access_mode() == AccessMode::Scan + && filter.predicates.len() == 1 + && filter.dynamic_indices.is_empty() + && selected.mask.all_true()) + .then_some(0), order, remaining: BitVec::from_elem(filter.predicates.len(), true), dynamic: filter @@ -303,8 +313,31 @@ impl FilterPlanner { AdaptorStep::Yield => PlannerOutput::Continue, AdaptorStep::NeedsIO(batch) => PlannerOutput::NeedsIO(batch), AdaptorStep::Piece(piece) => { - self.pieces.push(piece_mask(&self.scan.session, piece)?); - PlannerOutput::Continue + let mask = piece_mask(&self.scan.session, piece)?; + let output = if let Some(consumed) = &mut self.streamed_rows { + let selected = self.selected.as_ref().vortex_expect("selected rows exist"); + let start = selected.scope.rows.start + *consumed as u64; + *consumed += mask.len(); + if mask.all_false() { + PlannerOutput::Continue + } else { + let piece = SelectedRows { + scope: WorkScope { + file_ordinal: selected.scope.file_ordinal, + rows: start..start + mask.len() as u64, + }, + mask: mask.clone(), + }; + PlannerOutput::Morsel( + piece.scope.clone(), + Box::new(ProjectionMorsel::new(Arc::clone(&self.scan), piece)), + ) + } + } else { + PlannerOutput::Continue + }; + self.pieces.push(mask); + output } }; if let Some(started) = started { @@ -493,6 +526,10 @@ impl Planner for FilterPlanner { if let Some(output) = self.advance_active()? { return Ok(output); } + if self.streamed_rows.is_some() && self.remaining.none() { + self.selected = None; + return Ok(PlannerOutput::Done); + } if self .selected .as_ref() diff --git a/vortex-layout/src/scan/execution/projection.rs b/vortex-layout/src/scan/execution/projection.rs index 507b2487b76..2fcfa067e1b 100644 --- a/vortex-layout/src/scan/execution/projection.rs +++ b/vortex-layout/src/scan/execution/projection.rs @@ -103,7 +103,7 @@ impl Planner for ProjectionPlanner { /// Projects selected rows in order, streaming scan pieces and gathering filtered random-access /// pieces into one batch. A selection that projects to no rows finishes without a batch. -struct ProjectionMorsel { +pub(super) struct ProjectionMorsel { scan: Arc, dtype: DType, selected: SelectedRows, @@ -114,7 +114,7 @@ struct ProjectionMorsel { impl ProjectionMorsel { /// Creates a morsel projecting `selected`. - fn new(scan: Arc, selected: SelectedRows) -> Self { + pub(super) fn new(scan: Arc, selected: SelectedRows) -> Self { Self { dtype: scan.projection.dtype().clone(), scan, diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index b15bb41aafc..0d7ee332eee 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -9,13 +9,16 @@ use std::sync::atomic::AtomicI32; use std::sync::atomic::Ordering; use std::task::Context; use std::task::Poll; +use std::time::Duration; use arrow_array::cast::AsArray; use arrow_array::types::Int32Type; use arrow_schema::Field; +use futures::FutureExt; use futures::TryStreamExt; use futures::stream; use rstest::rstest; +use tokio::sync::Notify; use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; @@ -836,7 +839,14 @@ async fn stored_batches_share_one_work_chunk( ); let batches = run_batches(&scan, scan.rows().clone()).await?; assert_eq!(batches.len(), CHUNKS); - assert!(batches.iter().all(|batch| batch.scope.rows == *scan.rows())); + for (chunk, batch) in batches.iter().enumerate() { + let expected = if filtered { + (chunk * CHUNK_ROWS) as u64..((chunk + 1) * CHUNK_ROWS) as u64 + } else { + scan.rows().clone() + }; + assert_eq!(batch.scope.rows, expected); + } let arrays = batches .into_iter() .map(|batch| batch.array) @@ -1181,6 +1191,104 @@ async fn shared_cache_prunes_zones_once() -> VortexResult<()> { Ok(()) } +struct GatedSegment { + inner: Arc, + segment: SegmentId, + gate: Arc, +} + +impl SegmentSource for GatedSegment { + fn request(&self, id: SegmentId) -> SegmentFuture { + let read = self.inner.request(id); + let gate = Arc::clone(&self.gate); + let gated = id == self.segment; + async move { + if gated { + gate.notified().await; + } + read.await + } + .boxed() + } +} + +#[tokio::test] +async fn projects_completed_filter_pieces_before_later_reads() -> VortexResult<()> { + let session = new_session().with_tokio(); + let (segments, layout) = write(&session, table().as_ref()).await?; + let ints = layout.slot(1)?.ok_or_else(|| vortex_err!("ints missing"))?; + let later = ints.slot(1)?.ok_or_else(|| vortex_err!("chunk missing"))?; + let segment = later + .segment_ids() + .first() + .copied() + .ok_or_else(|| vortex_err!("segment missing"))?; + let gate = Arc::new(Notify::new()); + let segments = Arc::new(GatedSegment { + inner: segments, + segment, + gate: Arc::clone(&gate), + }) as Arc; + let projection = get_item("ints", root()).bind(layout.dtype())?; + let scan = Arc::new( + builder(&session, &segments, &layout)? + .with_projection(projection.clone()) + .with_some_filter(Some( + gt(get_item("ints", root()), lit(0i32)).bind(layout.dtype())?, + )) + .prepare()?, + ); + let io = scan.io(); + let mut run = Run::new(); + run.admit( + compose(&scan, scan.rows().clone()), + WorkScope { + file_ordinal: 0, + rows: scan.rows().clone(), + }, + Arc::new(DemandOnlyIo(io.session())), + ); + let first = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match run.advance()? { + Progress::Batch(batch) => return Ok(batch.array), + Progress::Spawn(scope, chunk) => { + run.admit(chunk, scope, Arc::new(DemandOnlyIo(io.session()))); + } + Progress::Waiting => std::future::poll_fn(|cx| run.poll_completion(cx)).await?, + Progress::RootDone(_) => {} + Progress::Idle => vortex_bail!("scan ended before producing a batch"), + } + } + }) + .await + .map_err(|_| vortex_err!("first projection waited for a later predicate read"))??; + assert_arrays_eq!( + first.clone(), + PrimitiveArray::from_iter(1..i32::try_from(CHUNK_ROWS)?), + &mut session.create_execution_ctx() + ); + gate.notify_one(); + let mut arrays = vec![first]; + loop { + match run.advance()? { + Progress::Batch(batch) => arrays.push(batch.array), + Progress::Spawn(scope, chunk) => { + run.admit(chunk, scope, Arc::new(DemandOnlyIo(io.session()))); + } + Progress::Waiting => std::future::poll_fn(|cx| run.poll_completion(cx)).await?, + Progress::RootDone(_) if run.live_roots() > 0 => {} + Progress::RootDone(_) | Progress::Idle => break, + } + } + assert_arrays_eq!( + ChunkedArray::try_new(arrays, projection.dtype().clone())?, + PrimitiveArray::from_iter(1..i32::try_from(CHUNKS * CHUNK_ROWS)?), + &mut session.create_execution_ctx() + ); + Ok(()) +} + /// Fetch only demanded segments so these tests distinguish avoided reads from eager hints. struct DemandOnlyIo(Arc); From 9ad658b95cf7fa42914d2db03b831dd679738e7b Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 16:18:18 +0000 Subject: [PATCH 27/28] Drop filter piece streaming after the TPC-H comparison The guarded seven-query comparison was inconclusive overall and Q13 was 1.7% slower than the preserved V2 baseline. Keep the existing combined-mask handoff; early-output correctness alone does not justify this performance experiment. The ClickBench attempt never started because the shared host remained busy. Signed-off-by: "Joe Isaacs" --- vortex-layout/src/scan/execution/filter.rs | 43 +------ .../src/scan/execution/projection.rs | 4 +- vortex-layout/src/scan/v2/tests.rs | 110 +----------------- 3 files changed, 6 insertions(+), 151 deletions(-) diff --git a/vortex-layout/src/scan/execution/filter.rs b/vortex-layout/src/scan/execution/filter.rs index 543196e77db..f9a28cf05fc 100644 --- a/vortex-layout/src/scan/execution/filter.rs +++ b/vortex-layout/src/scan/execution/filter.rs @@ -27,7 +27,6 @@ use vortex_mask::Mask; use vortex_scan::planning::planner::Planner; use vortex_scan::planning::planner::PlannerOutput; use vortex_scan::planning::planner::State; -use vortex_scan::planning::planner::WorkScope; use vortex_session::VortexSession; use self::ordering::AdaptiveOrder; @@ -39,7 +38,6 @@ use super::SelectedRows; use super::adaptor::AdaptorStep; use super::adaptor::Piece; use super::adaptor::ScanAdaptor; -use super::projection::ProjectionMorsel; use super::projection::ProjectionPlanner; use crate::plan::physical::FilterPlan; use crate::plan::physical::PlanRef; @@ -137,8 +135,7 @@ fn metadata_only(plan: &PlanRef) -> VortexResult { Ok(true) } -/// Evaluates a chunk's conjuncts over its selected rows. A single static predicate streams its -/// completed pieces to projection; other filters hand off their combined mask. +/// Evaluates a chunk's conjuncts over its selected rows, then hands the mask to projection. /// /// Dynamic pruning is refreshed before data prefetch. Each chunk takes an order from the shared /// cost estimates, occasionally exploring another order. A dynamic conjunct is revisited if its @@ -152,8 +149,6 @@ pub(super) struct FilterPlanner { active: Option, dynamic: Vec, pieces: Vec, - /// Dense input lets a single static predicate hand each completed piece to projection. - streamed_rows: Option, rechecks: RecheckStats, announced: bool, /// Ids for hints, counting down from `u32::MAX` so they never meet an adaptor's fetch ids. @@ -274,11 +269,6 @@ impl FilterPlanner { "v2 predicate order selected" ); Self { - streamed_rows: (scan.access_mode() == AccessMode::Scan - && filter.predicates.len() == 1 - && filter.dynamic_indices.is_empty() - && selected.mask.all_true()) - .then_some(0), order, remaining: BitVec::from_elem(filter.predicates.len(), true), dynamic: filter @@ -313,31 +303,8 @@ impl FilterPlanner { AdaptorStep::Yield => PlannerOutput::Continue, AdaptorStep::NeedsIO(batch) => PlannerOutput::NeedsIO(batch), AdaptorStep::Piece(piece) => { - let mask = piece_mask(&self.scan.session, piece)?; - let output = if let Some(consumed) = &mut self.streamed_rows { - let selected = self.selected.as_ref().vortex_expect("selected rows exist"); - let start = selected.scope.rows.start + *consumed as u64; - *consumed += mask.len(); - if mask.all_false() { - PlannerOutput::Continue - } else { - let piece = SelectedRows { - scope: WorkScope { - file_ordinal: selected.scope.file_ordinal, - rows: start..start + mask.len() as u64, - }, - mask: mask.clone(), - }; - PlannerOutput::Morsel( - piece.scope.clone(), - Box::new(ProjectionMorsel::new(Arc::clone(&self.scan), piece)), - ) - } - } else { - PlannerOutput::Continue - }; - self.pieces.push(mask); - output + self.pieces.push(piece_mask(&self.scan.session, piece)?); + PlannerOutput::Continue } }; if let Some(started) = started { @@ -526,10 +493,6 @@ impl Planner for FilterPlanner { if let Some(output) = self.advance_active()? { return Ok(output); } - if self.streamed_rows.is_some() && self.remaining.none() { - self.selected = None; - return Ok(PlannerOutput::Done); - } if self .selected .as_ref() diff --git a/vortex-layout/src/scan/execution/projection.rs b/vortex-layout/src/scan/execution/projection.rs index 2fcfa067e1b..507b2487b76 100644 --- a/vortex-layout/src/scan/execution/projection.rs +++ b/vortex-layout/src/scan/execution/projection.rs @@ -103,7 +103,7 @@ impl Planner for ProjectionPlanner { /// Projects selected rows in order, streaming scan pieces and gathering filtered random-access /// pieces into one batch. A selection that projects to no rows finishes without a batch. -pub(super) struct ProjectionMorsel { +struct ProjectionMorsel { scan: Arc, dtype: DType, selected: SelectedRows, @@ -114,7 +114,7 @@ pub(super) struct ProjectionMorsel { impl ProjectionMorsel { /// Creates a morsel projecting `selected`. - pub(super) fn new(scan: Arc, selected: SelectedRows) -> Self { + fn new(scan: Arc, selected: SelectedRows) -> Self { Self { dtype: scan.projection.dtype().clone(), scan, diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index 0d7ee332eee..b15bb41aafc 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -9,16 +9,13 @@ use std::sync::atomic::AtomicI32; use std::sync::atomic::Ordering; use std::task::Context; use std::task::Poll; -use std::time::Duration; use arrow_array::cast::AsArray; use arrow_array::types::Int32Type; use arrow_schema::Field; -use futures::FutureExt; use futures::TryStreamExt; use futures::stream; use rstest::rstest; -use tokio::sync::Notify; use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; @@ -839,14 +836,7 @@ async fn stored_batches_share_one_work_chunk( ); let batches = run_batches(&scan, scan.rows().clone()).await?; assert_eq!(batches.len(), CHUNKS); - for (chunk, batch) in batches.iter().enumerate() { - let expected = if filtered { - (chunk * CHUNK_ROWS) as u64..((chunk + 1) * CHUNK_ROWS) as u64 - } else { - scan.rows().clone() - }; - assert_eq!(batch.scope.rows, expected); - } + assert!(batches.iter().all(|batch| batch.scope.rows == *scan.rows())); let arrays = batches .into_iter() .map(|batch| batch.array) @@ -1191,104 +1181,6 @@ async fn shared_cache_prunes_zones_once() -> VortexResult<()> { Ok(()) } -struct GatedSegment { - inner: Arc, - segment: SegmentId, - gate: Arc, -} - -impl SegmentSource for GatedSegment { - fn request(&self, id: SegmentId) -> SegmentFuture { - let read = self.inner.request(id); - let gate = Arc::clone(&self.gate); - let gated = id == self.segment; - async move { - if gated { - gate.notified().await; - } - read.await - } - .boxed() - } -} - -#[tokio::test] -async fn projects_completed_filter_pieces_before_later_reads() -> VortexResult<()> { - let session = new_session().with_tokio(); - let (segments, layout) = write(&session, table().as_ref()).await?; - let ints = layout.slot(1)?.ok_or_else(|| vortex_err!("ints missing"))?; - let later = ints.slot(1)?.ok_or_else(|| vortex_err!("chunk missing"))?; - let segment = later - .segment_ids() - .first() - .copied() - .ok_or_else(|| vortex_err!("segment missing"))?; - let gate = Arc::new(Notify::new()); - let segments = Arc::new(GatedSegment { - inner: segments, - segment, - gate: Arc::clone(&gate), - }) as Arc; - let projection = get_item("ints", root()).bind(layout.dtype())?; - let scan = Arc::new( - builder(&session, &segments, &layout)? - .with_projection(projection.clone()) - .with_some_filter(Some( - gt(get_item("ints", root()), lit(0i32)).bind(layout.dtype())?, - )) - .prepare()?, - ); - let io = scan.io(); - let mut run = Run::new(); - run.admit( - compose(&scan, scan.rows().clone()), - WorkScope { - file_ordinal: 0, - rows: scan.rows().clone(), - }, - Arc::new(DemandOnlyIo(io.session())), - ); - let first = tokio::time::timeout(Duration::from_secs(5), async { - loop { - match run.advance()? { - Progress::Batch(batch) => return Ok(batch.array), - Progress::Spawn(scope, chunk) => { - run.admit(chunk, scope, Arc::new(DemandOnlyIo(io.session()))); - } - Progress::Waiting => std::future::poll_fn(|cx| run.poll_completion(cx)).await?, - Progress::RootDone(_) => {} - Progress::Idle => vortex_bail!("scan ended before producing a batch"), - } - } - }) - .await - .map_err(|_| vortex_err!("first projection waited for a later predicate read"))??; - assert_arrays_eq!( - first.clone(), - PrimitiveArray::from_iter(1..i32::try_from(CHUNK_ROWS)?), - &mut session.create_execution_ctx() - ); - gate.notify_one(); - let mut arrays = vec![first]; - loop { - match run.advance()? { - Progress::Batch(batch) => arrays.push(batch.array), - Progress::Spawn(scope, chunk) => { - run.admit(chunk, scope, Arc::new(DemandOnlyIo(io.session()))); - } - Progress::Waiting => std::future::poll_fn(|cx| run.poll_completion(cx)).await?, - Progress::RootDone(_) if run.live_roots() > 0 => {} - Progress::RootDone(_) | Progress::Idle => break, - } - } - assert_arrays_eq!( - ChunkedArray::try_new(arrays, projection.dtype().clone())?, - PrimitiveArray::from_iter(1..i32::try_from(CHUNKS * CHUNK_ROWS)?), - &mut session.create_execution_ctx() - ); - Ok(()) -} - /// Fetch only demanded segments so these tests distinguish avoided reads from eager hints. struct DemandOnlyIo(Arc); From ac26753966cfcc66c758ed32d2f0e336b1386f67 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 3 Nov 2026 16:22:36 +0000 Subject: [PATCH 28/28] Coarsen narrow unfiltered DataFusion scan partitions Align narrow unfiltered byte partitions to 65536-row boundaries before preparing scan plans. This reduces repeated setup on small dimensions while preserving exact row coverage. Keep filtered and wide partitions on the existing 8192-row grid. Extend cold/warm repartition coverage to 30000 and 150000 rows. Use four reversed rounds and eleven iterations outside SF50 to resolve the sub-percent differences seen in the previous Actions comparison. SF50 retains two rounds of seven iterations and its 900-second process limit. Signed-off-by: "Joe Isaacs" --- .github/workflows/scan-v2-comparison.yml | 5 ++- vortex-datafusion/src/persistent/opener.rs | 39 ++++++++++++---------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/scan-v2-comparison.yml b/.github/workflows/scan-v2-comparison.yml index 47c7f4354ca..8866b6f540f 100644 --- a/.github/workflows/scan-v2-comparison.yml +++ b/.github/workflows/scan-v2-comparison.yml @@ -183,6 +183,8 @@ jobs: BASE_SHA: ${{ needs.resolve.outputs.base_sha }} HEAD_SHA: ${{ inputs.head_sha }} QUERY_TIMEOUT_SECONDS: ${{ matrix.id == 'tpcds-50' && '900' || '300' }} + BENCH_ITERATIONS: ${{ matrix.id == 'tpcds-50' && '7' || '11' }} + BENCH_ROUNDS: ${{ matrix.id == 'tpcds-50' && '2' || '4' }} run: | options=() if [[ -n "$SCALE" ]]; then @@ -191,7 +193,8 @@ jobs: bash scripts/bench-taskset.sh python3 scripts/compare-scan-versions.py "$SUITE" \ --base-dir target/scan-comparison/base --head-dir target/scan-comparison/head \ --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" \ - --iterations 7 --warmup 2 --rounds 2 --query-timeout "$QUERY_TIMEOUT_SECONDS" \ + --iterations "$BENCH_ITERATIONS" --warmup 2 --rounds "$BENCH_ROUNDS" \ + --query-timeout "$QUERY_TIMEOUT_SECONDS" \ --output-dir scan-comparison-results "${options[@]}" - name: Publish comparison summary if: always() diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index de219b358bc..b08da2ffaf9 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -441,6 +441,10 @@ impl FileOpener for VortexOpener { Ok(RecordBatch::from(arrow.as_struct().clone())) }; let stream = if scan_v2 { + let narrow_unfiltered = native_filter.is_none() + && scan_dtype + .element_size() + .is_some_and(|bytes| bytes > 0 && bytes <= 16); let mut scan_file = vxf.scan_file(); scan_file.segments = shared_segment_source( &segment_sources, @@ -453,7 +457,17 @@ impl FileOpener for VortexOpener { .with_some_filter(native_filter); if let Some(bytes) = byte_range { // Every partition maps the same byte boundary to the same row boundary. - let rows = partition_row_range(bytes, file.object_meta.size, vxf.row_count()); + let mut rows = partition_row_range(bytes, file.object_meta.size, vxf.row_count()); + if narrow_unfiltered { + // Small dimensions otherwise repeat planning for a few thousand rows on + // every worker. Shared boundaries still assign every row exactly once. + let boundary = |row: u64| { + row.div_ceil(65_536) + .saturating_mul(65_536) + .min(vxf.row_count()) + }; + rows = boundary(rows.start)..boundary(rows.end); + } if rows.is_empty() { return Ok(stream::empty().boxed()); } @@ -1049,11 +1063,12 @@ mod tests { #[values(false, true)] scan_v2: bool, #[values(false, true)] warm_cache: bool, #[values(3, 8)] partitions: usize, + #[values(30_000, 150_000)] row_count: usize, ) -> anyhow::Result<()> { let object_store = Arc::new(InMemory::new()) as Arc; let file_path = "rows/file.vortex"; - let values: Vec> = (0..30_000).map(Some).collect(); - let batch = record_batch!(("a", Int32, values)).unwrap(); + let values: Vec> = (0..i32::try_from(row_count)?).map(Some).collect(); + let batch = record_batch!(("a", Int32, values))?; let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?; let file = PartitionedFile::new(file_path.to_string(), data_size); @@ -1093,31 +1108,19 @@ mod tests { assert_eq!(files.len(), partitions); let mut data = Vec::new(); + let mut populated = 0; for file in files { let batches = opener .open(file.clone())? .await? .try_collect::>() .await?; - if scan_v2 { - let bytes = file.range.as_ref().expect("partition has a byte range"); - let rows = partition_row_range( - u64::try_from(bytes.start)?..u64::try_from(bytes.end)?, - data_size, - batch.num_rows() as u64, - ); - assert_eq!( - concat_batches(&batch.schema(), &batches)?, - batch.slice( - usize::try_from(rows.start)?, - usize::try_from(rows.end - rows.start)?, - ), - ); - } + populated += usize::from(!batches.is_empty()); data.extend(batches); } assert_eq!(concat_batches(&batch.schema(), &data)?, batch); if scan_v2 { + assert_eq!(populated, row_count.div_ceil(65_536)); assert!(opener.layout_readers.is_empty()); assert!(opener.natural_splits.is_empty()); }