diff --git a/Cargo.lock b/Cargo.lock index 7494141e59d..beb5bed0c3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11996,6 +11996,7 @@ dependencies = [ "vortex-fastlanes", "vortex-mask", "vortex-session", + "vortex-sparse", ] [[package]] diff --git a/encodings/alp/src/alp/array/mod.rs b/encodings/alp/src/alp/array/mod.rs index 131a5b9247b..ea459bd3852 100644 --- a/encodings/alp/src/alp/array/mod.rs +++ b/encodings/alp/src/alp/array/mod.rs @@ -146,7 +146,7 @@ pub struct ALPSlots { pub patch_chunk_offsets: Option, } -const PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { +pub(crate) const PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { indices: ALPSlots::PATCH_INDICES, values: ALPSlots::PATCH_VALUES, chunk_offsets: ALPSlots::PATCH_CHUNK_OFFSETS, diff --git a/encodings/alp/src/alp/ops.rs b/encodings/alp/src/alp/ops.rs index db6f89ecf91..5ba2e8f6817 100644 --- a/encodings/alp/src/alp/ops.rs +++ b/encodings/alp/src/alp/ops.rs @@ -14,6 +14,7 @@ use crate::ALP; use crate::ALPArrayExt; use crate::ALPFloat; use crate::ALPSlots; +use crate::alp::array::PATCH_SLOTS; use crate::match_each_alp_float_ptype; impl OperationsVTable for ALP { @@ -29,7 +30,7 @@ impl OperationsVTable for ALP { } let array = state.array(); if let Some(patches) = array.patches() - && let Some(patch) = patches.get_patched(index, ctx)? + && let Some(patch) = patches.probe_patched(state, PATCH_SLOTS, index, ctx)? { return patch.cast(array.dtype()); } diff --git a/encodings/alp/src/alp_rd/array/mod.rs b/encodings/alp/src/alp_rd/array/mod.rs index 399d9e9db83..9a471006bd5 100644 --- a/encodings/alp/src/alp_rd/array/mod.rs +++ b/encodings/alp/src/alp_rd/array/mod.rs @@ -241,7 +241,7 @@ pub struct ALPRDSlots { pub patch_chunk_offsets: Option, } -const LP_PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { +pub(crate) const LP_PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { indices: ALPRDSlots::PATCH_INDICES, values: ALPRDSlots::PATCH_VALUES, chunk_offsets: ALPRDSlots::PATCH_CHUNK_OFFSETS, diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index 5869d7a2833..8af0690b544 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; +use vortex_array::ProbeState; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; @@ -12,62 +13,85 @@ use vortex_error::VortexResult; use crate::ALPRD; use crate::ALPRDArrayExt; use crate::ALPRDArraySlotsExt; +use crate::alp_rd::array::LP_PATCH_SLOTS; use crate::alp_rd::code_not_in_dictionary; impl OperationsVTable for ALPRD { type ProbeState = (); + fn probe_scalar( + state: &mut ProbeState<'_, ALPRD>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + if !state.is_valid(index, ctx)? { + return Ok(Scalar::null(state.array().dtype().clone())); + } + read_valid(state, index, ctx) + } + fn scalar_at( array: ArrayView<'_, ALPRD>, index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - // The left value can either be a direct value, or an exception. - // The exceptions array represents exception positions with non-null values. - let maybe_patched_value = match array.left_parts_patches() { - Some(patches) => patches.get_patched(index, ctx)?, - None => None, - }; - let left = match maybe_patched_value { - Some(patched_value) => patched_value - .as_primitive() - .as_::() - .vortex_expect("patched values must be non-null"), - _ => { - let left_code: u16 = array - .left_parts() - .execute_scalar(index, ctx)? - .as_primitive() - .as_::() - .vortex_expect("left_code must be non-null"); - let dictionary = array.left_parts_dictionary(); - *dictionary - .get(usize::from(left_code)) - .ok_or_else(|| code_not_in_dictionary(left_code, dictionary.len()))? - } - }; - - // combine left and right values - Ok(if array.dtype().as_ptype() == PType::F32 { - let right: u32 = array - .right_parts() - .execute_scalar(index, ctx)? - .as_primitive() - .as_::() - .vortex_expect("non-null"); - let packed = f32::from_bits((left as u32) << array.right_bit_width() | right); - Scalar::primitive(packed, array.dtype().nullability()) - } else { - let right: u64 = array - .right_parts() + read_valid(&mut ProbeState::once(array), index, ctx) + } +} + +/// Read the valid row at `index`, joining its left part, patched or from the dictionary, with +/// its right part. +fn read_valid( + state: &mut ProbeState<'_, ALPRD>, + index: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let array = state.array(); + // The left value can either be a direct value, or an exception. + // The exceptions array represents exception positions with non-null values. + let maybe_patched_value = match array.left_parts_patches() { + Some(patches) => patches.probe_patched(state, LP_PATCH_SLOTS, index, ctx)?, + None => None, + }; + let left = match maybe_patched_value { + Some(patched_value) => patched_value + .as_primitive() + .as_::() + .vortex_expect("patched values must be non-null"), + _ => { + let left_code: u16 = array + .left_parts() .execute_scalar(index, ctx)? .as_primitive() - .as_::() - .vortex_expect("non-null"); - let packed = f64::from_bits(((left as u64) << array.right_bit_width()) | right); - Scalar::primitive(packed, array.dtype().nullability()) - }) - } + .as_::() + .vortex_expect("left_code must be non-null"); + let dictionary = array.left_parts_dictionary(); + *dictionary + .get(usize::from(left_code)) + .ok_or_else(|| code_not_in_dictionary(left_code, dictionary.len()))? + } + }; + + // combine left and right values + Ok(if array.dtype().as_ptype() == PType::F32 { + let right: u32 = array + .right_parts() + .execute_scalar(index, ctx)? + .as_primitive() + .as_::() + .vortex_expect("non-null"); + let packed = f32::from_bits((left as u32) << array.right_bit_width() | right); + Scalar::primitive(packed, array.dtype().nullability()) + } else { + let right: u64 = array + .right_parts() + .execute_scalar(index, ctx)? + .as_primitive() + .as_::() + .vortex_expect("non-null"); + let packed = f64::from_bits(((left as u64) << array.right_bit_width()) | right); + Scalar::primitive(packed, array.dtype().nullability()) + }) } #[cfg(test)] diff --git a/encodings/fastlanes/Cargo.toml b/encodings/fastlanes/Cargo.toml index aef518de2f7..7cebfe39261 100644 --- a/encodings/fastlanes/Cargo.toml +++ b/encodings/fastlanes/Cargo.toml @@ -44,6 +44,7 @@ vortex-alp = { path = "../alp" } vortex-array = { workspace = true, features = ["_test-harness"] } vortex-bench-support = { workspace = true } vortex-fastlanes = { path = ".", features = ["_test-harness"] } +vortex-sparse = { workspace = true } [features] _test-harness = ["dep:rand"] @@ -98,3 +99,7 @@ harness = false [[bench]] name = "for_decode" harness = false + +[[bench]] +name = "patched_search" +harness = false diff --git a/encodings/fastlanes/benches/patched_search.rs b/encodings/fastlanes/benches/patched_search.rs new file mode 100644 index 00000000000..00717b40229 --- /dev/null +++ b/encodings/fastlanes/benches/patched_search.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Sorted searches over compressed children: reads, through one-off and repeated probes, of a +//! Sparse array whose patch indices are BitPacked and of a RunEnd array whose ends are BitPacked, +//! and slices of those patches and of that RunEnd array. Each read or slice binary searches the +//! compressed child, so these measure how much of the child's preparation is shared across +//! searches. + +use std::ops::Range; +use std::sync::LazyLock; + +use divan::Bencher; +use mimalloc::MiMalloc; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::aggregate_fn::session::AggregateFnSession; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::RunEnd; +use vortex_array::memory::MemorySession; +use vortex_array::optimizer::kernels::KernelSession; +use vortex_array::scalar::Scalar; +use vortex_array::session::ArraySession; +use vortex_array::validity::Validity; +use vortex_buffer::Buffer; +use vortex_error::VortexExpect; +use vortex_fastlanes::bitpack_compress::bitpack_to_best_bit_width; +use vortex_session::VortexSession; +use vortex_sparse::Sparse; +use vortex_sparse::SparseArray; +use vortex_sparse::SparseExt; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +fn main() { + LazyLock::force(&SESSION); + divan::main(); +} + +static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty() + .with::() + .with::() + .with::() + .with::(); + vortex_fastlanes::initialize(&session); + vortex_sparse::initialize(&session); + session +}); + +const LEN: usize = 1 << 20; +/// One patch, or one run end, every `STRIDE` rows. +const STRIDE: usize = 64; +// A simulated read costs up to about 22µs and a simulated slice up to about 52µs, so these +// counts keep each CodSpeed benchmark below 1ms. +const ACCESSES: usize = if cfg!(codspeed) { 16 } else { 256 }; +const SLICES: usize = if cfg!(codspeed) { 8 } else { 64 }; +const SLICE_LEN: usize = 4_096; + +/// Sorted positions every `STRIDE` rows, BitPacked. +fn bitpacked_positions(first: usize) -> ArrayRef { + let mut ctx = SESSION.create_execution_ctx(); + let positions = (first..=LEN) + .step_by(STRIDE) + .map(|i| u32::try_from(i).vortex_expect("positions fit u32")) + .collect::>(); + bitpack_to_best_bit_width( + &PrimitiveArray::new(positions, Validity::NonNullable), + &mut ctx, + ) + .vortex_expect("bitpack positions") + .into_array() +} + +fn values(len: usize) -> ArrayRef { + (0..len) + .map(|i| i64::try_from(i).vortex_expect("values fit i64")) + .collect::>() + .into_array() +} + +fn sparse() -> SparseArray { + let mut ctx = SESSION.create_execution_ctx(); + // Patch indices must stay below the array length. + let indices = bitpacked_positions(STRIDE / 2); + let values = values(indices.len()); + Sparse::try_new(indices, values, LEN, Scalar::from(-1i64), &mut ctx) + .vortex_expect("sparse fixture") +} + +fn run_end() -> ArrayRef { + let mut ctx = SESSION.create_execution_ctx(); + // Run ends are exclusive, so the last one is the array length. + let ends = bitpacked_positions(STRIDE); + let values = values(ends.len()); + RunEnd::try_new(ends, values, &mut ctx) + .vortex_expect("run-end fixture") + .into_array() +} + +/// A fixed pseudo-random sequence below `bound`. +fn positions(count: usize, bound: usize) -> Vec { + let mut seed = 42u64; + (0..count) + .map(|_| { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + (seed >> 32) as usize % bound + }) + .collect() +} + +fn ranges() -> Vec> { + positions(SLICES, LEN - SLICE_LEN) + .into_iter() + .map(|start| start..start + SLICE_LEN) + .collect() +} + +#[divan::bench] +fn sparse_read_once(bencher: Bencher) { + let array = sparse().into_array(); + let rows = positions(ACCESSES, LEN); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_refs(|ctx| { + for &row in &rows { + divan::black_box(array.execute_scalar(row, ctx).vortex_expect("read")); + } + }); +} + +#[divan::bench] +fn sparse_read_repeated(bencher: Bencher) { + let array = sparse().into_array(); + let rows = positions(ACCESSES, LEN); + bencher + .with_inputs(|| (SESSION.create_execution_ctx(), array.repeated_probe())) + .bench_refs(|(ctx, probe)| { + for &row in &rows { + divan::black_box(probe.execute_scalar(row, ctx).vortex_expect("read")); + } + }); +} + +#[divan::bench] +fn run_end_read_once(bencher: Bencher) { + let array = run_end(); + let rows = positions(ACCESSES, LEN); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_refs(|ctx| { + for &row in &rows { + divan::black_box(array.execute_scalar(row, ctx).vortex_expect("read")); + } + }); +} + +#[divan::bench] +fn run_end_read_repeated(bencher: Bencher) { + let array = run_end(); + let rows = positions(ACCESSES, LEN); + bencher + .with_inputs(|| (SESSION.create_execution_ctx(), array.repeated_probe())) + .bench_refs(|(ctx, probe)| { + for &row in &rows { + divan::black_box(probe.execute_scalar(row, ctx).vortex_expect("read")); + } + }); +} + +#[divan::bench] +fn patches_slice(bencher: Bencher) { + let patches = sparse().patches(); + let ranges = ranges(); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_refs(|ctx| { + for range in &ranges { + divan::black_box(patches.slice(range.clone(), ctx).vortex_expect("slice")); + } + }); +} + +#[divan::bench] +fn run_end_slice(bencher: Bencher) { + let array = run_end(); + let ranges = ranges(); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_refs(|ctx| { + for range in &ranges { + divan::black_box( + array + .slice(range.clone()) + .vortex_expect("slice") + .execute_until::(ctx) + .vortex_expect("slice kernel"), + ); + } + }); +} diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 7d738e9579a..f276cde4d9d 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; +use vortex_array::ProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; @@ -11,31 +12,54 @@ use crate::BitPacked; use crate::BitWidthsView; use crate::bitpack_decompress; use crate::bitpacking::array::BitPackedArrayExt; +use crate::bitpacking::array::PATCH_SLOTS; + impl OperationsVTable for BitPacked { type ProbeState = (); + fn probe_scalar( + state: &mut ProbeState<'_, BitPacked>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + if !state.is_valid(index, ctx)? { + return Ok(Scalar::null(state.array().dtype().clone())); + } + read_valid(state, index, ctx) + } + fn scalar_at( array: ArrayView<'_, BitPacked>, index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - Ok( - if let Some(patches) = array.patches() - && let Some(patch) = patches.get_patched(index, ctx)? - { - patch - } else { - match array.bit_widths() { - BitWidthsView::Global(_) => bitpack_decompress::unpack_single(array, index)?, - BitWidthsView::Blocked(offsets) => { - bitpack_decompress::unpack_single_blocked(array, offsets, index, ctx)? - } - } - }, - ) + read_valid(&mut ProbeState::once(array), index, ctx) } } +/// Read the valid row at `index`, from the patches if it is patched and unpacking it otherwise. +fn read_valid( + state: &mut ProbeState<'_, BitPacked>, + index: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let array = state.array(); + Ok( + if let Some(patches) = array.patches() + && let Some(patch) = patches.probe_patched(state, PATCH_SLOTS, index, ctx)? + { + patch + } else { + match array.bit_widths() { + BitWidthsView::Global(_) => bitpack_decompress::unpack_single(array, index)?, + BitWidthsView::Blocked(offsets) => { + bitpack_decompress::unpack_single_blocked(array, offsets, index, ctx)? + } + } + }, + ) +} + #[cfg(test)] mod test { use std::ops::Range; diff --git a/encodings/sparse/src/ops.rs b/encodings/sparse/src/ops.rs index 9b5088ab221..0c6f6eeae8e 100644 --- a/encodings/sparse/src/ops.rs +++ b/encodings/sparse/src/ops.rs @@ -3,37 +3,66 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; +use vortex_array::ProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; +use crate::PATCH_SLOTS; use crate::Sparse; use crate::SparseExt as _; impl OperationsVTable for Sparse { type ProbeState = (); + fn probe_scalar( + state: &mut ProbeState<'_, Sparse>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + if !state.is_valid(index, ctx)? { + return Ok(Scalar::null(state.array().dtype().clone())); + } + patched_or_fill(state, index, ctx) + } + fn scalar_at( array: ArrayView<'_, Sparse>, index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - Ok(array - .patches() - .get_patched(index, ctx)? - .unwrap_or_else(|| array.fill_scalar().clone())) + patched_or_fill(&mut ProbeState::once(array), index, ctx) } } +/// The patch at `index` if there is one, otherwise the fill value. +fn patched_or_fill( + state: &mut ProbeState<'_, Sparse>, + index: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let array = state.array(); + Ok(array + .patches() + .probe_patched(state, PATCH_SLOTS, index, ctx)? + .unwrap_or_else(|| array.fill_scalar().clone())) +} + #[cfg(test)] mod tests { use std::sync::LazyLock; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; + use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::scalar::Scalar; use vortex_buffer::buffer; + use vortex_error::VortexResult; use vortex_session::VortexSession; use crate::Sparse; @@ -57,4 +86,34 @@ mod tests { assert_arrays_eq!(sliced, PrimitiveArray::from_iter(expected), &mut ctx); } + + /// Indices that are not a canonical primitive array are searched through the repeated + /// probe's indices slot, which every read must find the same patches through. + #[test] + fn repeated_probe_with_encoded_indices() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let indices = ChunkedArray::try_new( + vec![buffer![1u32, 4].into_array(), buffer![7u32, 9].into_array()], + DType::Primitive(PType::U32, Nullability::NonNullable), + )? + .into_array(); + let values = buffer![10i32, 40, 70, 90].into_array(); + let sparse = + Sparse::try_new(indices, values, 12, Scalar::from(-1i32), &mut ctx)?.into_array(); + + let mut probe = sparse.repeated_probe(); + let read = (0..sparse.len()) + .map(|idx| { + Ok(probe + .execute_scalar(idx, &mut ctx)? + .as_primitive() + .as_::()) + }) + .collect::>>()?; + assert_eq!( + read, + [-1, 10, -1, -1, 40, -1, -1, 70, -1, 90, -1, -1].map(Some) + ); + Ok(()) + } } diff --git a/scripts/codspeed_bench_counts.json b/scripts/codspeed_bench_counts.json index 1b11f4e872e..45d1ae064d2 100644 --- a/scripts/codspeed_bench_counts.json +++ b/scripts/codspeed_bench_counts.json @@ -123,7 +123,8 @@ "compute_between": 11, "delta_decode": 6, "for_decode": 0, - "for_encode": 0 + "for_encode": 0, + "patched_search": 6 }, "vortex-file": { "split_collection": 3 diff --git a/vortex-array/src/arrays/runend/array/mod.rs b/vortex-array/src/arrays/runend/array/mod.rs index d8c632b4a48..8a19d0d4ec3 100644 --- a/vortex-array/src/arrays/runend/array/mod.rs +++ b/vortex-array/src/arrays/runend/array/mod.rs @@ -12,6 +12,7 @@ use std::fmt::Display; use std::fmt::Formatter; use std::hash::Hash; use std::hash::Hasher; +use std::ops::Range; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; @@ -50,6 +51,7 @@ use crate::arrays::runend::compress::runend_encode; use crate::arrays::runend::compress::runend_encode_bool; use crate::arrays::runend::decompress_bool::runend_decode_bools; use crate::arrays::runend::ops::find_physical_index; +use crate::arrays::runend::ops::find_physical_range; use crate::arrays::runend::ops::find_slice_end_index; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -198,6 +200,17 @@ pub trait RunEndArrayExt: TypedArrayRef { fn find_slice_end_index(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult { find_slice_end_index(self.ends(), index + self.offset(), ctx) } + + /// The physical range of runs covering the logical `range`, found with one probe of the + /// ends. + fn find_physical_range( + &self, + range: Range, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let offset = self.offset(); + find_physical_range(self.ends(), range.start + offset..range.end + offset, ctx) + } } impl> RunEndArrayExt for T {} diff --git a/vortex-array/src/arrays/runend/kernel.rs b/vortex-array/src/arrays/runend/kernel.rs index cb5ce320c46..bf909d15496 100644 --- a/vortex-array/src/arrays/runend/kernel.rs +++ b/vortex-array/src/arrays/runend/kernel.rs @@ -64,8 +64,10 @@ fn slice( ) -> VortexResult { let new_length = range.len(); - let slice_begin = array.find_physical_index(range.start, ctx)?; - let slice_end = array.find_slice_end_index(range.end, ctx)?; + let Range { + start: slice_begin, + end: slice_end, + } = array.find_physical_range(range.clone(), ctx)?; // If the sliced range contains only a single run, opt to return a ConstantArray. if slice_begin + 1 == slice_end { diff --git a/vortex-array/src/arrays/runend/ops.rs b/vortex-array/src/arrays/runend/ops.rs index a6ec0e18164..b75aa10e1ce 100644 --- a/vortex-array/src/arrays/runend/ops.rs +++ b/vortex-array/src/arrays/runend/ops.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::ops::Range; + use vortex_error::VortexResult; use crate::ArrayRef; @@ -49,16 +51,7 @@ pub fn find_slice_end_index( SearchSortedPrimitiveArray::::new(array, ctx) .search_sorted(&index, SearchSortedSide::Right)? }); - Ok(match result { - SearchResult::Found(i) => i, - SearchResult::NotFound(i) => { - if i == array.len() { - i - } else { - i + 1 - } - } - }) + Ok(slice_end_index(result, array.len())) } /// Find the physical index (the run) that contains the logical `index`, given the run `ends`. @@ -74,6 +67,41 @@ pub fn find_physical_index( }) } +/// Find the physical range of runs covering the logical `range`, given the run `ends`. +/// +/// Equivalent to [`find_physical_index`] of the start and [`find_slice_end_index`] of the end, +/// but both searches share one searcher, and so one probe of `ends`. +pub fn find_physical_range( + array: &ArrayRef, + range: Range, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + match_each_unsigned_integer_ptype!(array.dtype().as_ptype(), |T| { + let searcher = SearchSortedPrimitiveArray::::new(array, ctx); + let start = searcher + .search_sorted(&range.start, SearchSortedSide::Right)? + .to_ends_index(array.len()); + let end = slice_end_index( + searcher.search_sorted(&range.end, SearchSortedSide::Right)?, + array.len(), + ); + Ok(start..end) + }) +} + +fn slice_end_index(result: SearchResult, len: usize) -> usize { + match result { + SearchResult::Found(i) => i, + SearchResult::NotFound(i) => { + if i == len { + i + } else { + i + 1 + } + } + } +} + #[cfg(test)] mod tests { use std::sync::LazyLock; diff --git a/vortex-array/src/arrays/runend/probe.rs b/vortex-array/src/arrays/runend/probe.rs index 46f91bb4fcd..db24d807785 100644 --- a/vortex-array/src/arrays/runend/probe.rs +++ b/vortex-array/src/arrays/runend/probe.rs @@ -11,7 +11,11 @@ use crate::ProbeState; use crate::arrays::runend::RunEnd; use crate::arrays::runend::RunEndArrayExt; use crate::arrays::runend::RunEndSlots; +use crate::match_each_unsigned_integer_ptype; use crate::scalar::Scalar; +use crate::search_sorted::SearchSorted; +use crate::search_sorted::SearchSortedPrimitiveArray; +use crate::search_sorted::SearchSortedSide; pub(crate) fn scalar_at( state: &mut ProbeState<'_, RunEnd>, @@ -23,27 +27,23 @@ pub(crate) fn scalar_at( .offset() .checked_add(index) .ok_or_else(|| vortex_err!("RunEnd logical index overflow"))?; - // Search for the first end strictly greater than the logical index. Every comparison - // uses the same ends probe, so a retained read keeps the child's preparation within and - // between searches. - let mut ends = state + // The run is that of the first end strictly greater than the logical index. The search + // reads the ends through this state's ends probe, so a retained read keeps the child's + // preparation within and between searches, and a one-off read shares one probe across the + // search. + let ends = state .slot(RunEndSlots::ENDS)? .ok_or_else(|| vortex_err!("RunEnd ends slot is missing"))?; - let mut left = 0; - let mut right = ends.array().len(); - while left < right { - let mid = left + (right - left) / 2; - let end = usize::try_from(&ends.execute_scalar(mid, ctx)?)?; - if end <= logical_index { - left = mid + 1; - } else { - right = mid; - } - } + let len = ends.array().len(); + let run = match_each_unsigned_integer_ptype!(ends.array().dtype().as_ptype(), |T| { + SearchSortedPrimitiveArray::::with_probe(ends, ctx) + .search_sorted(&logical_index, SearchSortedSide::Right)? + .to_ends_index(len) + }); state .slot(RunEndSlots::VALUES)? .ok_or_else(|| vortex_err!("RunEnd values slot is missing"))? - .execute_scalar(left, ctx) + .execute_scalar(run, ctx) } #[cfg(test)] diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 7cbb2d12368..a3bc6574b58 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -22,10 +22,12 @@ use vortex_mask::Mask; use vortex_mask::MaskValues; use vortex_utils::aliases::hash_map::HashMap; +use crate::ArrayProbe; use crate::ArrayRef; use crate::ArraySlots; use crate::ExecutionCtx; use crate::IntoArray; +use crate::ProbeState; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::PrimitiveArrayExt; @@ -45,6 +47,7 @@ use crate::search_sorted::SearchSorted; use crate::search_sorted::SearchSortedPrimitiveArray; use crate::search_sorted::SearchSortedSide; use crate::validity::Validity; +use crate::vtable::VTable; /// One patch index offset is stored for each chunk. /// This allows for constant time patch index lookups. @@ -326,8 +329,9 @@ impl Patches { .offset .checked_add(PATCH_CHUNK_SIZE - sliced_rows) .ok_or_else(|| vortex_err!("Patch chunk boundary overflows usize"))?; - let retained = - search_index_binary_search(&patches.indices, first_chunk_end, ctx)?.to_index(); + let [retained] = + search_index_binary_search(&patches.indices, [first_chunk_end], ctx)?; + let retained = retained.to_index(); let expected_offset = patches .chunk_offset_at(1, ctx)? .checked_sub(patches.chunk_offset_at(0, ctx)?) @@ -517,46 +521,89 @@ impl Patches { /// [`SearchResult::Found(patch_idx)`]: SearchResult::Found /// [`SearchResult::NotFound(insertion_point)`]: SearchResult::NotFound pub fn search_index(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult { + let [result] = self.search_indices([index], ctx)?; + Ok(result) + } + + /// Searches for each of `indices`, as [`Self::search_index`] does for one. + /// + /// The searches share one probe of the chunk offsets, or one searcher over the patch indices, + /// rather than building one per search. + fn search_indices( + &self, + indices: [usize; N], + ctx: &mut ExecutionCtx, + ) -> VortexResult<[SearchResult; N]> { if self.chunk_offsets.is_some() { - return self.search_index_chunked(index, ctx); + return self.search_indices_chunked(indices, ctx); } - search_index_binary_search(&self.indices, index + self.offset, ctx) + search_index_binary_search(&self.indices, indices.map(|index| index + self.offset), ctx) } - /// Constant time searches for `index` in the indices array. + /// Constant time searches for each of `indices` in the indices array. /// - /// First determines which chunk the target index falls into, then performs + /// First determines which chunk each target index falls into, then performs /// a binary search within that chunk's range. /// - /// Returns a [`SearchResult`] indicating either the exact patch index if found, + /// Returns a [`SearchResult`] per index indicating either the exact patch index if found, /// or the insertion point if not found. /// /// Returns an error if `chunk_offsets` or `offset_within_chunk` are not set. - fn search_index_chunked( + fn search_indices_chunked( &self, - index: usize, + indices: [usize; N], ctx: &mut ExecutionCtx, - ) -> VortexResult { + ) -> VortexResult<[SearchResult; N]> { let Some(chunk_offsets) = &self.chunk_offsets else { vortex_bail!("chunk_offsets is required to be set") }; + // Every search reads the same chunk offsets, so they share one probe rather than + // building one per read as `Self::chunk_offset_at` does. + let mut probe = chunk_offsets.repeated_probe(); + let mut chunk_offsets = ArrayProbe::Repeated(&mut probe); + + let mut results = [SearchResult::NotFound(self.indices.len()); N]; + for (result, index) in results.iter_mut().zip(indices) { + let Some(range) = self.chunk_patch_range(&mut chunk_offsets, index, ctx)? else { + continue; + }; + + let chunk_indices = self.indices.slice(range.clone())?.optimize_ctx(ctx)?; + let [found] = search_index_binary_search(&chunk_indices, [index + self.offset], ctx)?; + + *result = match found { + SearchResult::Found(idx) => SearchResult::Found(range.start + idx), + SearchResult::NotFound(idx) => SearchResult::NotFound(range.start + idx), + }; + } + + Ok(results) + } + + /// The range of positions in the indices array that holds the patches of `index`'s chunk, + /// read through `chunk_offsets`, or `None` if `index` is past the end of the array. + /// + /// Returns an error if `offset_within_chunk` is not set. + fn chunk_patch_range( + &self, + chunk_offsets: &mut ArrayProbe<'_>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult>> { let Some(offset_within_chunk) = self.offset_within_chunk else { vortex_bail!("offset_within_chunk is required to be set") }; if index >= self.array_len() { - return Ok(SearchResult::NotFound(self.indices().len())); + return Ok(None); } let chunk_idx = (index + self.offset % PATCH_CHUNK_SIZE) / PATCH_CHUNK_SIZE; - - // The three reads below are of the same array, so they share one probe rather than - // building one per read as `Self::chunk_offset_at` does. - let mut probe = chunk_offsets.repeated_probe(); + let num_chunks = chunk_offsets.array().len(); let mut chunk_offset_at = |idx: usize| -> VortexResult { - probe + chunk_offsets .execute_scalar(idx, ctx)? .as_primitive() .as_::() @@ -575,7 +622,7 @@ impl Patches { // saturating sub when adjusting the indices based on the chunk offset. .saturating_sub(offset_within_chunk); - let patches_end_idx = if chunk_idx < chunk_offsets.len() - 1 { + let patches_end_idx = if chunk_idx < num_chunks - 1 { (chunk_offset_at(chunk_idx + 1)? - base_offset) .saturating_sub(offset_within_chunk) .min(self.indices.len()) @@ -583,16 +630,46 @@ impl Patches { self.indices.len() }; - let chunk_indices = self - .indices - .slice(patches_start_idx..patches_end_idx)? - .optimize_ctx(ctx)?; - let result = search_index_binary_search(&chunk_indices, index + self.offset, ctx)?; + Ok(Some(patches_start_idx..patches_end_idx)) + } - Ok(match result { - SearchResult::Found(idx) => SearchResult::Found(patches_start_idx + idx), - SearchResult::NotFound(idx) => SearchResult::NotFound(patches_start_idx + idx), - }) + /// Get the patched value at `index` of the array `state` reads, as [`Self::get_patched`] + /// does. + /// + /// `self` must be the patches held in the `slots` of that array. A repeated read searches the + /// patch indices through the state's indices probe and reads the value through its values + /// probe, so it shares both across reads rather than building them per read. A one-off read + /// takes the [`Self::get_patched`] path, which can narrow the search by the chunk offsets. + pub fn probe_patched( + &self, + state: &mut ProbeState<'_, V>, + slots: PatchSlotIndices, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let (retained, mut children) = state.split(); + if retained.is_none() { + return self.get_patched(index, ctx); + } + + let indices = children + .slot(slots.indices)? + .ok_or_else(|| vortex_err!("patch indices slot is missing"))?; + let needle = index + self.offset; + let found = match_each_unsigned_integer_ptype!(indices.array().dtype().as_ptype(), |T| { + SearchSortedPrimitiveArray::::with_probe(indices, ctx) + .search_sorted(&needle, SearchSortedSide::Left)? + }); + + let SearchResult::Found(position) = found else { + return Ok(None); + }; + + children + .slot(slots.values)? + .ok_or_else(|| vortex_err!("patch values slot is missing"))? + .execute_scalar(position, ctx) + .map(Some) } /// Batch version of `search_index`. @@ -787,8 +864,9 @@ impl Patches { /// Slice the patches by a range of the patched array. pub fn slice(&self, range: Range, ctx: &mut ExecutionCtx) -> VortexResult> { - let slice_start_idx = self.search_index(range.start, ctx)?.to_index(); - let slice_end_idx = self.search_index(range.end, ctx)?.to_index(); + let [slice_start, slice_end] = self.search_indices([range.start, range.end], ctx)?; + let slice_start_idx = slice_start.to_index(); + let slice_end_idx = slice_end.to_index(); if slice_start_idx == slice_end_idx { return Ok(None); @@ -1058,43 +1136,39 @@ impl Patches { } } -/// Binary searches for `needle` in the indices array. +/// Binary searches for each of `needles` in the indices array, sharing one searcher. /// /// # Returns -/// [`SearchResult::Found`] with the position if needle exists, or [`SearchResult::NotFound`] -/// with the insertion point if not found. -fn search_index_binary_search( +/// Per needle, [`SearchResult::Found`] with the position if it exists, or +/// [`SearchResult::NotFound`] with the insertion point if not found. +fn search_index_binary_search( indices: &ArrayRef, - needle: usize, + needles: [usize; N], ctx: &mut ExecutionCtx, -) -> VortexResult { +) -> VortexResult<[SearchResult; N]> { + // A needle that does not fit the index type is a non-negative integer (a usize) larger than + // all values in this array, so it is never found and sorts last. + let mut results = [SearchResult::NotFound(indices.len()); N]; + if let Some(primitive) = indices.as_opt::() { match_each_unsigned_integer_ptype!(primitive.ptype(), |T| { - let Ok(needle) = T::try_from(needle) else { - // If the needle is not of type T, then it cannot possibly be in this array. - // - // The needle is a non-negative integer (a usize); therefore, it must be larger - // than all values in this array. - return Ok(SearchResult::NotFound(primitive.len())); - }; - return primitive - .as_slice::() - .search_sorted(&needle, SearchSortedSide::Left); + let values = primitive.as_slice::(); + for (result, needle) in results.iter_mut().zip(needles) { + if let Ok(needle) = T::try_from(needle) { + *result = values.search_sorted(&needle, SearchSortedSide::Left)?; + } + } }); + return Ok(results); } - search_index_binary_search_scalar(indices, needle, ctx) -} - -fn search_index_binary_search_scalar( - indices: &ArrayRef, - needle: usize, - ctx: &mut ExecutionCtx, -) -> VortexResult { match_each_unsigned_integer_ptype!(indices.dtype().as_ptype(), |T| { - SearchSortedPrimitiveArray::::new(indices, ctx) - .search_sorted(&needle, SearchSortedSide::Left) - }) + let searcher = SearchSortedPrimitiveArray::::new(indices, ctx); + for (result, needle) in results.iter_mut().zip(needles) { + *result = searcher.search_sorted(&needle, SearchSortedSide::Left)?; + } + }); + Ok(results) } #[expect(clippy::too_many_arguments)] // private function, can clean up one day diff --git a/vortex-array/src/search_sorted/primitive.rs b/vortex-array/src/search_sorted/primitive.rs index 69601ac71f8..0d6dfa90cc1 100644 --- a/vortex-array/src/search_sorted/primitive.rs +++ b/vortex-array/src/search_sorted/primitive.rs @@ -5,8 +5,10 @@ use std::cell::RefCell; use std::cmp::Ordering; use std::marker::PhantomData; +use vortex_error::VortexExpect; use vortex_error::VortexResult; +use crate::ArrayProbe; use crate::ArrayRef; use crate::ExecutionCtx; use crate::RepeatedArrayProbe; @@ -38,6 +40,9 @@ enum Reader<'a, T> { Values(&'a [T]), /// Anything else, through one probe reused by the whole search. Probe(RefCell), + /// Anything else, through a probe the caller already holds, such as a child slot of the + /// array it is reading, so the search shares whatever that probe has resolved. + Borrowed(RefCell>), } impl<'a, T: NativePType> SearchSortedPrimitiveArray<'a, T> { @@ -57,34 +62,74 @@ impl<'a, T: NativePType> SearchSortedPrimitiveArray<'a, T> { } } + /// Wraps the array `probe` reads for searching, reading through `probe` rather than a new + /// one, panicking if the array's [`PType`](crate::dtype::PType) is not `T::PTYPE`. + /// + /// A caller that already holds a probe of the array, such as a child slot of a + /// [`ProbeState`](crate::ProbeState), searches through it so a repeated read shares that + /// probe's state across its searches. + pub fn with_probe(probe: ArrayProbe<'a>, ctx: &'a mut ExecutionCtx) -> Self { + let array = probe.array(); + assert_eq!( + array.dtype().as_ptype(), + T::PTYPE, + "Array PType must match primitive type" + ); + let len = array.len(); + let reader = match probe { + ArrayProbe::Once(array) => Self::reader(array), + ArrayProbe::Repeated(probe) if Self::values(probe.array()).is_some() => { + let probe: &'a RepeatedArrayProbe = probe; + Reader::Values( + Self::values(probe.array()).vortex_expect("checked to offer values above"), + ) + } + probe @ ArrayProbe::Repeated(_) => Reader::Borrowed(RefCell::new(probe)), + }; + Self { + reader, + len, + ctx: RefCell::new(ctx), + _ptype: PhantomData, + } + } + /// Reads the buffer directly when the array can offer one, and probes otherwise. /// /// A nullable array is excluded because the buffer holds an arbitrary value where an element /// is null, and a device-backed array because its buffer is not addressable here. fn reader(array: &'a ArrayRef) -> Reader<'a, T> { + match Self::values(array) { + Some(values) => Reader::Values(values), + None => Reader::Probe(RefCell::new(array.repeated_probe())), + } + } + + /// The array's own buffer, if it is a canonical, non-nullable, host-backed array. + fn values(array: &ArrayRef) -> Option<&[T]> { if !array.dtype().is_nullable() && let Some(primitive) = array.as_opt::() && primitive.buffer_handle().as_host_opt().is_some() - && let Some(values) = primitive.data().as_slice::().get(..array.len()) { - return Reader::Values(values); + return primitive.data().as_slice::().get(..array.len()); } - Reader::Probe(RefCell::new(array.repeated_probe())) + None } /// Returns the value at `idx`, or `None` if the element is null. /// /// The probe and the context are separate cells, so the two borrows never overlap. fn typed_value(&self, idx: usize) -> VortexResult> { - let probe = match &self.reader { + let scalar = match &self.reader { Reader::Values(values) => return Ok(Some(values[idx])), - Reader::Probe(probe) => probe, + Reader::Probe(probe) => probe + .borrow_mut() + .execute_scalar(idx, &mut self.ctx.borrow_mut())?, + Reader::Borrowed(probe) => probe + .borrow_mut() + .execute_scalar(idx, &mut self.ctx.borrow_mut())?, }; - Ok(probe - .borrow_mut() - .execute_scalar(idx, &mut self.ctx.borrow_mut())? - .as_primitive() - .typed_value::()) + Ok(scalar.as_primitive().typed_value::()) } /// Returns the value at `idx`, with nulls mapped to `T::zero()`.