From 73b1ff4fa8f59cc1dc7300f5af10fbb82bad18a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 14:34:27 +0000 Subject: [PATCH 1/7] perf: share one searcher across repeated sorted searches Several call sites built a fresh searcher, and with it a fresh probe, for each search of the same sorted array: - RunEnd slice searched `ends` twice through two helpers. Add `find_physical_range`, which finds both bounds with one searcher. - `Patches::slice` called `search_index` twice. Both lookups now go through `search_indices`, which shares the chunk offsets probe and the indices searcher. - Sparse, BitPacked, ALP and ALP-RD read patches with `get_patched`, which builds a searcher over the patch indices per read. Add `Patches::probe_patched`, which reads the patch children through the `ProbeState` slots so a repeated probe keeps them across reads, and use it from each encoding's `probe_scalar`. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jck72aD351kMboqN5g8Ccs --- encodings/alp/src/alp/array.rs | 2 +- encodings/alp/src/alp/ops.rs | 3 +- encodings/alp/src/alp_rd/array.rs | 2 +- encodings/alp/src/alp_rd/ops.rs | 104 ++++---- .../src/bitpacking/vtable/operations.rs | 52 ++-- encodings/runend/src/array.rs | 13 + encodings/runend/src/kernel.rs | 6 +- encodings/runend/src/ops.rs | 48 +++- encodings/sparse/src/ops.rs | 66 +++++- vortex-array/src/patches.rs | 222 ++++++++++++++---- 10 files changed, 395 insertions(+), 123 deletions(-) diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 319a08c7385..8f6dcc5ade1 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -221,7 +221,7 @@ pub struct ALPSlots { pub patch_chunk_offsets: Option, } -const PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { +pub(crate) const PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { indices: ALPSlots::PATCH_INDICES, values: ALPSlots::PATCH_VALUES, chunk_offsets: ALPSlots::PATCH_CHUNK_OFFSETS, diff --git a/encodings/alp/src/alp/ops.rs b/encodings/alp/src/alp/ops.rs index db6f89ecf91..5ba2e8f6817 100644 --- a/encodings/alp/src/alp/ops.rs +++ b/encodings/alp/src/alp/ops.rs @@ -14,6 +14,7 @@ use crate::ALP; use crate::ALPArrayExt; use crate::ALPFloat; use crate::ALPSlots; +use crate::alp::array::PATCH_SLOTS; use crate::match_each_alp_float_ptype; impl OperationsVTable for ALP { @@ -29,7 +30,7 @@ impl OperationsVTable for ALP { } let array = state.array(); if let Some(patches) = array.patches() - && let Some(patch) = patches.get_patched(index, ctx)? + && let Some(patch) = patches.probe_patched(state, PATCH_SLOTS, index, ctx)? { return patch.cast(array.dtype()); } diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index b11b4f8cacc..9637e51091f 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -335,7 +335,7 @@ pub struct ALPRDSlots { pub patch_chunk_offsets: Option, } -const LP_PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { +pub(crate) const LP_PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { indices: ALPRDSlots::PATCH_INDICES, values: ALPRDSlots::PATCH_VALUES, chunk_offsets: ALPRDSlots::PATCH_CHUNK_OFFSETS, diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index 550d5022d80..2a90d8009f7 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; +use vortex_array::ProbeState; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; @@ -12,58 +13,81 @@ use vortex_error::VortexResult; use crate::ALPRD; use crate::ALPRDArrayExt; use crate::ALPRDArraySlotsExt; +use crate::alp_rd::array::LP_PATCH_SLOTS; impl OperationsVTable for ALPRD { type ProbeState = (); + fn probe_scalar( + state: &mut ProbeState<'_, ALPRD>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + if !state.is_valid(index, ctx)? { + return Ok(Scalar::null(state.array().dtype().clone())); + } + read_valid(state, index, ctx) + } + fn scalar_at( array: ArrayView<'_, ALPRD>, index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - // The left value can either be a direct value, or an exception. - // The exceptions array represents exception positions with non-null values. - let maybe_patched_value = match array.left_parts_patches() { - Some(patches) => patches.get_patched(index, ctx)?, - None => None, - }; - let left = match maybe_patched_value { - Some(patched_value) => patched_value - .as_primitive() - .as_::() - .vortex_expect("patched values must be non-null"), - _ => { - let left_code: u16 = array - .left_parts() - .execute_scalar(index, ctx)? - .as_primitive() - .as_::() - .vortex_expect("left_code must be non-null"); - array.left_parts_dictionary()[left_code as usize] - } - }; + read_valid(&mut ProbeState::once(array), index, ctx) + } +} - // combine left and right values - Ok(if array.dtype().as_ptype() == PType::F32 { - let right: u32 = array - .right_parts() - .execute_scalar(index, ctx)? - .as_primitive() - .as_::() - .vortex_expect("non-null"); - let packed = f32::from_bits((left as u32) << array.right_bit_width() | right); - Scalar::primitive(packed, array.dtype().nullability()) - } else { - let right: u64 = array - .right_parts() +/// Read the valid row at `index`, joining its left part, patched or from the dictionary, with +/// its right part. +fn read_valid( + state: &mut ProbeState<'_, ALPRD>, + index: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let array = state.array(); + // The left value can either be a direct value, or an exception. + // The exceptions array represents exception positions with non-null values. + let maybe_patched_value = match array.left_parts_patches() { + Some(patches) => patches.probe_patched(state, LP_PATCH_SLOTS, index, ctx)?, + None => None, + }; + let left = match maybe_patched_value { + Some(patched_value) => patched_value + .as_primitive() + .as_::() + .vortex_expect("patched values must be non-null"), + _ => { + let left_code: u16 = array + .left_parts() .execute_scalar(index, ctx)? .as_primitive() - .as_::() - .vortex_expect("non-null"); - let packed = f64::from_bits(((left as u64) << array.right_bit_width()) | right); - Scalar::primitive(packed, array.dtype().nullability()) - }) - } + .as_::() + .vortex_expect("left_code must be non-null"); + array.left_parts_dictionary()[left_code as usize] + } + }; + + // combine left and right values + Ok(if array.dtype().as_ptype() == PType::F32 { + let right: u32 = array + .right_parts() + .execute_scalar(index, ctx)? + .as_primitive() + .as_::() + .vortex_expect("non-null"); + let packed = f32::from_bits((left as u32) << array.right_bit_width() | right); + Scalar::primitive(packed, array.dtype().nullability()) + } else { + let right: u64 = array + .right_parts() + .execute_scalar(index, ctx)? + .as_primitive() + .as_::() + .vortex_expect("non-null"); + let packed = f64::from_bits(((left as u64) << array.right_bit_width()) | right); + Scalar::primitive(packed, array.dtype().nullability()) + }) } #[cfg(test)] diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 63fab5081d4..6dc4c337837 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; +use vortex_array::ProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; @@ -11,31 +12,54 @@ use crate::BitPacked; use crate::BitWidthsView; use crate::bitpack_decompress; use crate::bitpacking::array::BitPackedArrayExt; +use crate::bitpacking::array::PATCH_SLOTS; + impl OperationsVTable for BitPacked { type ProbeState = (); + fn probe_scalar( + state: &mut ProbeState<'_, BitPacked>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + if !state.is_valid(index, ctx)? { + return Ok(Scalar::null(state.array().dtype().clone())); + } + read_valid(state, index, ctx) + } + fn scalar_at( array: ArrayView<'_, BitPacked>, index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - Ok( - if let Some(patches) = array.patches() - && let Some(patch) = patches.get_patched(index, ctx)? - { - patch - } else { - match array.bit_widths() { - BitWidthsView::Global(_) => bitpack_decompress::unpack_single(array, index)?, - BitWidthsView::Blocked(offsets) => { - bitpack_decompress::unpack_single_blocked(array, offsets, index, ctx)? - } - } - }, - ) + read_valid(&mut ProbeState::once(array), index, ctx) } } +/// Read the valid row at `index`, from the patches if it is patched and unpacking it otherwise. +fn read_valid( + state: &mut ProbeState<'_, BitPacked>, + index: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let array = state.array(); + Ok( + if let Some(patches) = array.patches() + && let Some(patch) = patches.probe_patched(state, PATCH_SLOTS, index, ctx)? + { + patch + } else { + match array.bit_widths() { + BitWidthsView::Global(_) => bitpack_decompress::unpack_single(array, index)?, + BitWidthsView::Blocked(offsets) => { + bitpack_decompress::unpack_single_blocked(array, offsets, index, ctx)? + } + } + }, + ) +} + #[cfg(test)] mod test { use std::ops::Range; diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 4ce0350a37a..7e1352a3998 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -6,6 +6,7 @@ use std::fmt::Display; use std::fmt::Formatter; use std::hash::Hash; use std::hash::Hasher; +use std::ops::Range; use prost::Message; use vortex_array::AnyCanonical; @@ -54,6 +55,7 @@ use crate::compress::runend_decode_varbinview; use crate::compress::runend_encode; use crate::decompress_bool::runend_decode_bools; use crate::ops::find_physical_index; +use crate::ops::find_physical_range; use crate::ops::find_slice_end_index; use crate::rules::RULES; @@ -235,6 +237,17 @@ pub trait RunEndArrayExt: RunEndArraySlotsExt { fn find_slice_end_index(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult { find_slice_end_index(self.ends(), index + self.offset(), ctx) } + + /// The physical range of runs covering the logical `range`, found with one probe of the + /// ends. + fn find_physical_range( + &self, + range: Range, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let offset = self.offset(); + find_physical_range(self.ends(), range.start + offset..range.end + offset, ctx) + } } impl> RunEndArrayExt for T {} diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs index 2eed964e3a9..4e892de0eab 100644 --- a/encodings/runend/src/kernel.rs +++ b/encodings/runend/src/kernel.rs @@ -64,8 +64,10 @@ fn slice( ) -> VortexResult { let new_length = range.len(); - let slice_begin = array.find_physical_index(range.start, ctx)?; - let slice_end = array.find_slice_end_index(range.end, ctx)?; + let Range { + start: slice_begin, + end: slice_end, + } = array.find_physical_range(range.clone(), ctx)?; // If the sliced range contains only a single run, opt to return a ConstantArray. if slice_begin + 1 == slice_end { diff --git a/encodings/runend/src/ops.rs b/encodings/runend/src/ops.rs index 637fa8ecd25..8e8f54b0a6c 100644 --- a/encodings/runend/src/ops.rs +++ b/encodings/runend/src/ops.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::ops::Range; + use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -49,16 +51,7 @@ pub fn find_slice_end_index( SearchSortedPrimitiveArray::::new(array, ctx) .search_sorted(&index, SearchSortedSide::Right)? }); - Ok(match result { - SearchResult::Found(i) => i, - SearchResult::NotFound(i) => { - if i == array.len() { - i - } else { - i + 1 - } - } - }) + Ok(slice_end_index(result, array.len())) } /// Find the physical index (the run) that contains the logical `index`, given the run `ends`. @@ -74,6 +67,41 @@ pub fn find_physical_index( }) } +/// Find the physical range of runs covering the logical `range`, given the run `ends`. +/// +/// Equivalent to [`find_physical_index`] of the start and [`find_slice_end_index`] of the end, +/// but both searches share one searcher, and so one probe of `ends`. +pub fn find_physical_range( + array: &ArrayRef, + range: Range, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + match_each_unsigned_integer_ptype!(array.dtype().as_ptype(), |T| { + let searcher = SearchSortedPrimitiveArray::::new(array, ctx); + let start = searcher + .search_sorted(&range.start, SearchSortedSide::Right)? + .to_ends_index(array.len()); + let end = slice_end_index( + searcher.search_sorted(&range.end, SearchSortedSide::Right)?, + array.len(), + ); + Ok(start..end) + }) +} + +fn slice_end_index(result: SearchResult, len: usize) -> usize { + match result { + SearchResult::Found(i) => i, + SearchResult::NotFound(i) => { + if i == len { + i + } else { + i + 1 + } + } + } +} + #[cfg(test)] mod tests { use std::sync::LazyLock; diff --git a/encodings/sparse/src/ops.rs b/encodings/sparse/src/ops.rs index 41a64379791..ea3f1fac633 100644 --- a/encodings/sparse/src/ops.rs +++ b/encodings/sparse/src/ops.rs @@ -3,37 +3,66 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; +use vortex_array::ProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; +use crate::PATCH_SLOTS; use crate::Sparse; use crate::SparseExt as _; impl OperationsVTable for Sparse { type ProbeState = (); + fn probe_scalar( + state: &mut ProbeState<'_, Sparse>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + if !state.is_valid(index, ctx)? { + return Ok(Scalar::null(state.array().dtype().clone())); + } + patched_or_fill(state, index, ctx) + } + fn scalar_at( array: ArrayView<'_, Sparse>, index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - Ok(array - .patches() - .get_patched(index, ctx)? - .unwrap_or_else(|| array.fill_scalar().clone())) + patched_or_fill(&mut ProbeState::once(array), index, ctx) } } +/// The patch at `index` if there is one, otherwise the fill value. +fn patched_or_fill( + state: &mut ProbeState<'_, Sparse>, + index: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let array = state.array(); + Ok(array + .patches() + .probe_patched(state, PATCH_SLOTS, index, ctx)? + .unwrap_or_else(|| array.fill_scalar().clone())) +} + #[cfg(test)] mod tests { use std::sync::LazyLock; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; + use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::scalar::Scalar; use vortex_buffer::buffer; + use vortex_error::VortexResult; use vortex_session::VortexSession; use crate::Sparse; @@ -58,4 +87,33 @@ mod tests { let values = sliced.execute::(&mut ctx).unwrap(); assert_arrays_eq!(values, PrimitiveArray::from_iter(expected), &mut ctx); } + + /// Indices that are not a canonical primitive array are searched through the repeated + /// probe's indices slot, which every read must find the same patches through. + #[test] + fn repeated_probe_with_encoded_indices() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let indices = ChunkedArray::try_new( + vec![buffer![1u32, 4].into_array(), buffer![7u32, 9].into_array()], + DType::Primitive(PType::U32, Nullability::NonNullable), + )? + .into_array(); + let values = buffer![10i32, 40, 70, 90].into_array(); + let sparse = Sparse::try_new(indices, values, 12, Scalar::from(-1i32))?.into_array(); + + let mut probe = sparse.repeated_probe(); + let read = (0..sparse.len()) + .map(|idx| { + Ok(probe + .execute_scalar(idx, &mut ctx)? + .as_primitive() + .as_::()) + }) + .collect::>>()?; + assert_eq!( + read, + [-1, 10, -1, -1, 40, -1, -1, 70, -1, 90, -1, -1].map(Some) + ); + Ok(()) + } } diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index d75dc587276..f2e838908d5 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::cell::RefCell; +use std::cmp::Ordering; use std::fmt::Debug; use std::hash::Hash; use std::ops::Range; @@ -21,10 +23,12 @@ use vortex_mask::Mask; use vortex_mask::MaskValues; use vortex_utils::aliases::hash_map::HashMap; +use crate::ArrayProbe; use crate::ArrayRef; use crate::ArraySlots; use crate::ExecutionCtx; use crate::IntoArray; +use crate::ProbeState; use crate::VortexSessionExecute; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; @@ -40,11 +44,13 @@ use crate::dtype::UnsignedPType; use crate::legacy_session; use crate::match_each_unsigned_integer_ptype; use crate::scalar::Scalar; +use crate::search_sorted::IndexOrd; use crate::search_sorted::SearchResult; use crate::search_sorted::SearchSorted; use crate::search_sorted::SearchSortedPrimitiveArray; use crate::search_sorted::SearchSortedSide; use crate::validity::Validity; +use crate::vtable::VTable; /// One patch index offset is stored for each chunk. /// This allows for constant time patch index lookups. @@ -469,46 +475,89 @@ impl Patches { /// [`SearchResult::Found(patch_idx)`]: SearchResult::Found /// [`SearchResult::NotFound(insertion_point)`]: SearchResult::NotFound pub fn search_index(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult { + let [result] = self.search_indices([index], ctx)?; + Ok(result) + } + + /// Searches for each of `indices`, as [`Self::search_index`] does for one. + /// + /// The searches share one probe of the chunk offsets, or one searcher over the patch indices, + /// rather than building one per search. + fn search_indices( + &self, + indices: [usize; N], + ctx: &mut ExecutionCtx, + ) -> VortexResult<[SearchResult; N]> { if self.chunk_offsets.is_some() { - return self.search_index_chunked(index, ctx); + return self.search_indices_chunked(indices, ctx); } - search_index_binary_search(&self.indices, index + self.offset, ctx) + search_index_binary_search(&self.indices, indices.map(|index| index + self.offset), ctx) } - /// Constant time searches for `index` in the indices array. + /// Constant time searches for each of `indices` in the indices array. /// - /// First determines which chunk the target index falls into, then performs + /// First determines which chunk each target index falls into, then performs /// a binary search within that chunk's range. /// - /// Returns a [`SearchResult`] indicating either the exact patch index if found, + /// Returns a [`SearchResult`] per index indicating either the exact patch index if found, /// or the insertion point if not found. /// /// Returns an error if `chunk_offsets` or `offset_within_chunk` are not set. - fn search_index_chunked( + fn search_indices_chunked( &self, - index: usize, + indices: [usize; N], ctx: &mut ExecutionCtx, - ) -> VortexResult { + ) -> VortexResult<[SearchResult; N]> { let Some(chunk_offsets) = &self.chunk_offsets else { vortex_bail!("chunk_offsets is required to be set") }; + // Every search reads the same chunk offsets, so they share one probe rather than + // building one per read as `Self::chunk_offset_at` does. + let mut probe = chunk_offsets.repeated_probe(); + let mut chunk_offsets = ArrayProbe::Repeated(&mut probe); + + let mut results = [SearchResult::NotFound(self.indices.len()); N]; + for (result, index) in results.iter_mut().zip(indices) { + let Some(range) = self.chunk_patch_range(&mut chunk_offsets, index, ctx)? else { + continue; + }; + + let chunk_indices = self.indices.slice(range.clone())?; + let [found] = search_index_binary_search(&chunk_indices, [index + self.offset], ctx)?; + + *result = match found { + SearchResult::Found(idx) => SearchResult::Found(range.start + idx), + SearchResult::NotFound(idx) => SearchResult::NotFound(range.start + idx), + }; + } + + Ok(results) + } + + /// The range of positions in the indices array that holds the patches of `index`'s chunk, + /// read through `chunk_offsets`, or `None` if `index` is past the end of the array. + /// + /// Returns an error if `offset_within_chunk` is not set. + fn chunk_patch_range( + &self, + chunk_offsets: &mut ArrayProbe<'_>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult>> { let Some(offset_within_chunk) = self.offset_within_chunk else { vortex_bail!("offset_within_chunk is required to be set") }; if index >= self.array_len() { - return Ok(SearchResult::NotFound(self.indices().len())); + return Ok(None); } let chunk_idx = (index + self.offset % PATCH_CHUNK_SIZE) / PATCH_CHUNK_SIZE; - - // The three reads below are of the same array, so they share one probe rather than - // building one per read as `Self::chunk_offset_at` does. - let mut probe = chunk_offsets.repeated_probe(); + let num_chunks = chunk_offsets.array().len(); let mut chunk_offset_at = |idx: usize| -> VortexResult { - probe + chunk_offsets .execute_scalar(idx, ctx)? .as_primitive() .as_::() @@ -527,7 +576,7 @@ impl Patches { // saturating sub when adjusting the indices based on the chunk offset. .saturating_sub(offset_within_chunk); - let patches_end_idx = if chunk_idx < chunk_offsets.len() - 1 { + let patches_end_idx = if chunk_idx < num_chunks - 1 { (chunk_offset_at(chunk_idx + 1)? - base_offset) .saturating_sub(offset_within_chunk) .min(self.indices.len()) @@ -535,13 +584,64 @@ impl Patches { self.indices.len() }; - let chunk_indices = self.indices.slice(patches_start_idx..patches_end_idx)?; - let result = search_index_binary_search(&chunk_indices, index + self.offset, ctx)?; + Ok(Some(patches_start_idx..patches_end_idx)) + } - Ok(match result { - SearchResult::Found(idx) => SearchResult::Found(patches_start_idx + idx), - SearchResult::NotFound(idx) => SearchResult::NotFound(patches_start_idx + idx), - }) + /// Get the patched value at `index` of the array `state` reads, as [`Self::get_patched`] + /// does. + /// + /// `self` must be the patches held in the `slots` of that array. The patch children are + /// read through `state`, so a [`RepeatedArrayProbe`](crate::RepeatedArrayProbe) keeps their + /// probes across reads instead of building a searcher per read. + pub fn probe_patched( + &self, + state: &mut ProbeState<'_, V>, + slots: PatchSlotIndices, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let range = if self.chunk_offsets.is_some() { + let mut chunk_offsets = state + .slot(slots.chunk_offsets)? + .ok_or_else(|| vortex_err!("patch chunk offsets slot is missing"))?; + match self.chunk_patch_range(&mut chunk_offsets, index, ctx)? { + Some(range) => range, + None => return Ok(None), + } + } else { + 0..self.indices.len() + }; + + let needle = index + self.offset; + let found = if let Some(primitive) = self.indices.as_opt::() { + match_each_unsigned_integer_ptype!(primitive.ptype(), |T| { + // A needle that does not fit the index type is larger than every patch index. + let Ok(needle) = T::try_from(needle) else { + return Ok(None); + }; + primitive.as_slice::()[range.clone()] + .search_sorted(&needle, SearchSortedSide::Left)? + }) + } else { + let indices = state + .slot(slots.indices)? + .ok_or_else(|| vortex_err!("patch indices slot is missing"))?; + ProbedIndices { + probe: RefCell::new(indices), + ctx: RefCell::new(&mut *ctx), + range: range.clone(), + } + .search_sorted(&needle, SearchSortedSide::Left)? + }; + + let Some(found) = found.to_found() else { + return Ok(None); + }; + state + .slot(slots.values)? + .ok_or_else(|| vortex_err!("patch values slot is missing"))? + .execute_scalar(range.start + found, ctx) + .map(Some) } /// Batch version of `search_index`. @@ -743,8 +843,9 @@ impl Patches { #[allow(clippy::disallowed_methods)] pub fn slice(&self, range: Range) -> VortexResult> { let mut ctx = legacy_session().create_execution_ctx(); - let slice_start_idx = self.search_index(range.start, &mut ctx)?.to_index(); - let slice_end_idx = self.search_index(range.end, &mut ctx)?.to_index(); + let [slice_start, slice_end] = self.search_indices([range.start, range.end], &mut ctx)?; + let slice_start_idx = slice_start.to_index(); + let slice_end_idx = slice_end.to_index(); if slice_start_idx == slice_end_idx { return Ok(None); @@ -1006,43 +1107,64 @@ impl Patches { } } -/// Binary searches for `needle` in the indices array. +/// A [`SearchSorted`] adapter over the patch indices in `range`, read through a probe. +struct ProbedIndices<'p, 'c> { + probe: RefCell>, + ctx: RefCell<&'c mut ExecutionCtx>, + range: Range, +} + +impl IndexOrd for ProbedIndices<'_, '_> { + /// The probe and the context are separate cells, so the two borrows never overlap. + fn index_cmp(&self, idx: usize, elem: &usize) -> VortexResult> { + let value = self + .probe + .borrow_mut() + .execute_scalar(self.range.start + idx, &mut self.ctx.borrow_mut())? + .as_primitive() + .as_::() + .ok_or_else(|| vortex_err!("patch index does not fit in usize"))?; + Ok(Some(value.cmp(elem))) + } + + fn index_len(&self) -> usize { + self.range.len() + } +} + +/// Binary searches for each of `needles` in the indices array, sharing one searcher. /// /// # Returns -/// [`SearchResult::Found`] with the position if needle exists, or [`SearchResult::NotFound`] -/// with the insertion point if not found. -fn search_index_binary_search( +/// Per needle, [`SearchResult::Found`] with the position if it exists, or +/// [`SearchResult::NotFound`] with the insertion point if not found. +fn search_index_binary_search( indices: &ArrayRef, - needle: usize, + needles: [usize; N], ctx: &mut ExecutionCtx, -) -> VortexResult { +) -> VortexResult<[SearchResult; N]> { + // A needle that does not fit the index type is a non-negative integer (a usize) larger than + // all values in this array, so it is never found and sorts last. + let mut results = [SearchResult::NotFound(indices.len()); N]; + if let Some(primitive) = indices.as_opt::() { match_each_unsigned_integer_ptype!(primitive.ptype(), |T| { - let Ok(needle) = T::try_from(needle) else { - // If the needle is not of type T, then it cannot possibly be in this array. - // - // The needle is a non-negative integer (a usize); therefore, it must be larger - // than all values in this array. - return Ok(SearchResult::NotFound(primitive.len())); - }; - return primitive - .as_slice::() - .search_sorted(&needle, SearchSortedSide::Left); + let values = primitive.as_slice::(); + for (result, needle) in results.iter_mut().zip(needles) { + if let Ok(needle) = T::try_from(needle) { + *result = values.search_sorted(&needle, SearchSortedSide::Left)?; + } + } }); + return Ok(results); } - search_index_binary_search_scalar(indices, needle, ctx) -} - -fn search_index_binary_search_scalar( - indices: &ArrayRef, - needle: usize, - ctx: &mut ExecutionCtx, -) -> VortexResult { match_each_unsigned_integer_ptype!(indices.dtype().as_ptype(), |T| { - SearchSortedPrimitiveArray::::new(indices, ctx) - .search_sorted(&needle, SearchSortedSide::Left) - }) + let searcher = SearchSortedPrimitiveArray::::new(indices, ctx); + for (result, needle) in results.iter_mut().zip(needles) { + *result = searcher.search_sorted(&needle, SearchSortedSide::Left)?; + } + }); + Ok(results) } #[expect(clippy::too_many_arguments)] // private function, can clean up one day From 47f468a5e4faa1684302a616a9291b13e2608471 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 14:33:24 +0000 Subject: [PATCH 2/7] fix: keep one-off patched reads on get_patched and error on bad ranges A one-off read through `probe_patched` read compressed patch indices with a fresh probe per comparison, which regressed `scalar_at` against the old `get_patched` path, where one probe serves the whole search. One-off reads now take `get_patched`; only repeated probes use the slot probes. Slicing the primitive indices now returns an error for an out-of-range patch range instead of panicking, as `Array::slice` did before. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jck72aD351kMboqN5g8Ccs --- vortex-array/src/patches.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index f2e838908d5..56fb02bc82c 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -592,7 +592,8 @@ impl Patches { /// /// `self` must be the patches held in the `slots` of that array. The patch children are /// read through `state`, so a [`RepeatedArrayProbe`](crate::RepeatedArrayProbe) keeps their - /// probes across reads instead of building a searcher per read. + /// probes across reads instead of building a searcher per read. A one-off read takes the + /// [`Self::get_patched`] path. pub fn probe_patched( &self, state: &mut ProbeState<'_, V>, @@ -600,6 +601,12 @@ impl Patches { index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult> { + // A one-off read keeps nothing between reads, so reading the children through `state` + // would build a probe per comparison. `get_patched` shares one probe across the search. + if state.retained().is_none() { + return self.get_patched(index, ctx); + } + let range = if self.chunk_offsets.is_some() { let mut chunk_offsets = state .slot(slots.chunk_offsets)? @@ -619,7 +626,15 @@ impl Patches { let Ok(needle) = T::try_from(needle) else { return Ok(None); }; - primitive.as_slice::()[range.clone()] + primitive + .as_slice::() + .get(range.clone()) + .ok_or_else(|| { + vortex_err!( + "patch range {range:?} out of bounds for {} indices", + primitive.len() + ) + })? .search_sorted(&needle, SearchSortedSide::Left)? }) } else { From ec505d8e6769411e7422513723f049d77587e3ea Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 17:02:55 +0000 Subject: [PATCH 3/7] bench: sorted searches over compressed children Add `patched_search`, which reads a Sparse array whose patch indices are BitPacked through one-off and repeated probes, and slices those patches and a RunEnd array whose ends are BitPacked. Every case binary searches a compressed child, which no existing benchmark covers. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jck72aD351kMboqN5g8Ccs --- Cargo.lock | 1 + encodings/fastlanes/Cargo.toml | 5 + encodings/fastlanes/benches/patched_search.rs | 175 ++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 encodings/fastlanes/benches/patched_search.rs diff --git a/Cargo.lock b/Cargo.lock index f9b8e501845..eb47977803a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11992,6 +11992,7 @@ dependencies = [ "vortex-fastlanes", "vortex-mask", "vortex-session", + "vortex-sparse", ] [[package]] diff --git a/encodings/fastlanes/Cargo.toml b/encodings/fastlanes/Cargo.toml index c1261092b61..79e2eedfde7 100644 --- a/encodings/fastlanes/Cargo.toml +++ b/encodings/fastlanes/Cargo.toml @@ -43,6 +43,7 @@ vortex-alp = { path = "../alp" } vortex-array = { workspace = true, features = ["_test-harness"] } vortex-bench-support = { workspace = true } vortex-fastlanes = { path = ".", features = ["_test-harness"] } +vortex-sparse = { workspace = true } [features] _test-harness = ["dep:rand"] @@ -93,3 +94,7 @@ harness = false [[bench]] name = "for_decode" harness = false + +[[bench]] +name = "patched_search" +harness = false diff --git a/encodings/fastlanes/benches/patched_search.rs b/encodings/fastlanes/benches/patched_search.rs new file mode 100644 index 00000000000..5301e4b9525 --- /dev/null +++ b/encodings/fastlanes/benches/patched_search.rs @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Sorted searches over compressed children: reads of a Sparse array whose patch indices are +//! BitPacked, through one-off and repeated probes, and slices of those patches and of a RunEnd +//! array whose ends are BitPacked. Each read or slice binary searches the compressed child, so +//! these measure how much of the child's preparation is shared across searches. + +use std::ops::Range; +use std::sync::LazyLock; + +use divan::Bencher; +use mimalloc::MiMalloc; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::aggregate_fn::session::AggregateFnSession; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::RunEnd; +use vortex_array::memory::MemorySession; +use vortex_array::optimizer::kernels::KernelSession; +use vortex_array::scalar::Scalar; +use vortex_array::session::ArraySession; +use vortex_array::validity::Validity; +use vortex_buffer::Buffer; +use vortex_error::VortexExpect; +use vortex_fastlanes::bitpack_compress::bitpack_to_best_bit_width; +use vortex_session::VortexSession; +use vortex_sparse::Sparse; +use vortex_sparse::SparseArray; +use vortex_sparse::SparseExt; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +fn main() { + LazyLock::force(&SESSION); + divan::main(); +} + +static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty() + .with::() + .with::() + .with::() + .with::(); + vortex_fastlanes::initialize(&session); + vortex_sparse::initialize(&session); + session +}); + +const LEN: usize = 1 << 20; +/// One patch, or one run end, every `STRIDE` rows. +const STRIDE: usize = 64; +// Sized to keep the CodSpeed simulation short per benchmark. +const ACCESSES: usize = 256; +const SLICES: usize = 64; +const SLICE_LEN: usize = 4_096; + +/// Sorted positions every `STRIDE` rows, BitPacked. +fn bitpacked_positions(first: usize) -> ArrayRef { + let mut ctx = SESSION.create_execution_ctx(); + let positions = (first..=LEN) + .step_by(STRIDE) + .map(|i| u32::try_from(i).vortex_expect("positions fit u32")) + .collect::>(); + bitpack_to_best_bit_width( + &PrimitiveArray::new(positions, Validity::NonNullable), + &mut ctx, + ) + .vortex_expect("bitpack positions") + .into_array() +} + +fn values(len: usize) -> ArrayRef { + (0..len) + .map(|i| i64::try_from(i).vortex_expect("values fit i64")) + .collect::>() + .into_array() +} + +fn sparse() -> SparseArray { + let mut ctx = SESSION.create_execution_ctx(); + // Patch indices must stay below the array length. + let indices = bitpacked_positions(STRIDE / 2); + let values = values(indices.len()); + Sparse::try_new(indices, values, LEN, Scalar::from(-1i64), &mut ctx) + .vortex_expect("sparse fixture") +} + +fn run_end() -> ArrayRef { + let mut ctx = SESSION.create_execution_ctx(); + // Run ends are exclusive, so the last one is the array length. + let ends = bitpacked_positions(STRIDE); + let values = values(ends.len()); + RunEnd::try_new(ends, values, &mut ctx) + .vortex_expect("run-end fixture") + .into_array() +} + +/// A fixed pseudo-random sequence below `bound`. +fn positions(count: usize, bound: usize) -> Vec { + let mut seed = 42u64; + (0..count) + .map(|_| { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + (seed >> 32) as usize % bound + }) + .collect() +} + +fn ranges() -> Vec> { + positions(SLICES, LEN - SLICE_LEN) + .into_iter() + .map(|start| start..start + SLICE_LEN) + .collect() +} + +#[divan::bench] +fn sparse_read_once(bencher: Bencher) { + let array = sparse().into_array(); + let rows = positions(ACCESSES, LEN); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_refs(|ctx| { + for &row in &rows { + divan::black_box(array.execute_scalar(row, ctx).vortex_expect("read")); + } + }); +} + +#[divan::bench] +fn sparse_read_repeated(bencher: Bencher) { + let array = sparse().into_array(); + let rows = positions(ACCESSES, LEN); + bencher + .with_inputs(|| (SESSION.create_execution_ctx(), array.repeated_probe())) + .bench_refs(|(ctx, probe)| { + for &row in &rows { + divan::black_box(probe.execute_scalar(row, ctx).vortex_expect("read")); + } + }); +} + +#[divan::bench] +fn patches_slice(bencher: Bencher) { + let patches = sparse().patches(); + let ranges = ranges(); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_refs(|ctx| { + for range in &ranges { + divan::black_box(patches.slice(range.clone(), ctx).vortex_expect("slice")); + } + }); +} + +#[divan::bench] +fn run_end_slice(bencher: Bencher) { + let array = run_end(); + let ranges = ranges(); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_refs(|ctx| { + for range in &ranges { + divan::black_box( + array + .slice(range.clone()) + .vortex_expect("slice") + .execute_until::(ctx) + .vortex_expect("slice kernel"), + ); + } + }); +} From b1d0f71c8e2b5b54a06a0ab9c5dc02336c74ac90 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 9 Nov 2026 18:03:59 +0000 Subject: [PATCH 4/7] perf: decode sorted children once per repeated probe A repeated probe of a patched array (Sparse, BitPacked, ALP, ALP-RD) binary searched the patch indices through a child probe, reading one value and building one scalar per comparison, and a RunEnd probe did the same with its ends. Both now keep the decoded child in their encoding probe state: `PatchesProbeState` holds the patch indices and `RunEndProbeState` the run ends, each decoded on the probe's first read, so later reads binary search a plain `u64` buffer. One-off reads still search without decoding the whole child. The benchmark gains repeated and one-off RunEnd reads. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jck72aD351kMboqN5g8Ccs --- encodings/alp/src/alp/ops.rs | 3 +- encodings/alp/src/alp_rd/ops.rs | 3 +- encodings/fastlanes/benches/patched_search.rs | 32 ++++- .../src/bitpacking/vtable/operations.rs | 3 +- encodings/sparse/src/ops.rs | 3 +- vortex-array/src/arrays/runend/ops.rs | 2 +- vortex-array/src/arrays/runend/probe.rs | 63 ++++++--- vortex-array/src/patches.rs | 131 +++++++----------- vortex-array/src/search_sorted/decoded.rs | 32 +++++ vortex-array/src/search_sorted/mod.rs | 2 + 10 files changed, 167 insertions(+), 107 deletions(-) create mode 100644 vortex-array/src/search_sorted/decoded.rs diff --git a/encodings/alp/src/alp/ops.rs b/encodings/alp/src/alp/ops.rs index 5ba2e8f6817..f3ddabd2c91 100644 --- a/encodings/alp/src/alp/ops.rs +++ b/encodings/alp/src/alp/ops.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ProbeState; +use vortex_array::patches::PatchesProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexExpect; @@ -18,7 +19,7 @@ use crate::alp::array::PATCH_SLOTS; use crate::match_each_alp_float_ptype; impl OperationsVTable for ALP { - type ProbeState = (); + type ProbeState = PatchesProbeState; fn probe_scalar( state: &mut ProbeState<'_, ALP>, diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index 8af0690b544..1e3fe5f4757 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -5,6 +5,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ProbeState; use vortex_array::dtype::PType; +use vortex_array::patches::PatchesProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexExpect; @@ -17,7 +18,7 @@ use crate::alp_rd::array::LP_PATCH_SLOTS; use crate::alp_rd::code_not_in_dictionary; impl OperationsVTable for ALPRD { - type ProbeState = (); + type ProbeState = PatchesProbeState; fn probe_scalar( state: &mut ProbeState<'_, ALPRD>, diff --git a/encodings/fastlanes/benches/patched_search.rs b/encodings/fastlanes/benches/patched_search.rs index 5301e4b9525..40eada055bc 100644 --- a/encodings/fastlanes/benches/patched_search.rs +++ b/encodings/fastlanes/benches/patched_search.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Sorted searches over compressed children: reads of a Sparse array whose patch indices are -//! BitPacked, through one-off and repeated probes, and slices of those patches and of a RunEnd -//! array whose ends are BitPacked. Each read or slice binary searches the compressed child, so +//! Sorted searches over compressed children: reads, through one-off and repeated probes, of a +//! Sparse array whose patch indices are BitPacked and of a RunEnd array whose ends are BitPacked, +//! and slices of those patches and of that RunEnd array. Each read or slice binary searches the compressed child, so //! these measure how much of the child's preparation is shared across searches. use std::ops::Range; @@ -142,6 +142,32 @@ fn sparse_read_repeated(bencher: Bencher) { }); } +#[divan::bench] +fn run_end_read_once(bencher: Bencher) { + let array = run_end(); + let rows = positions(ACCESSES, LEN); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_refs(|ctx| { + for &row in &rows { + divan::black_box(array.execute_scalar(row, ctx).vortex_expect("read")); + } + }); +} + +#[divan::bench] +fn run_end_read_repeated(bencher: Bencher) { + let array = run_end(); + let rows = positions(ACCESSES, LEN); + bencher + .with_inputs(|| (SESSION.create_execution_ctx(), array.repeated_probe())) + .bench_refs(|(ctx, probe)| { + for &row in &rows { + divan::black_box(probe.execute_scalar(row, ctx).vortex_expect("read")); + } + }); +} + #[divan::bench] fn patches_slice(bencher: Bencher) { let patches = sparse().patches(); diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index f276cde4d9d..9e4b29268ad 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ProbeState; +use vortex_array::patches::PatchesProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; @@ -15,7 +16,7 @@ use crate::bitpacking::array::BitPackedArrayExt; use crate::bitpacking::array::PATCH_SLOTS; impl OperationsVTable for BitPacked { - type ProbeState = (); + type ProbeState = PatchesProbeState; fn probe_scalar( state: &mut ProbeState<'_, BitPacked>, diff --git a/encodings/sparse/src/ops.rs b/encodings/sparse/src/ops.rs index 0c6f6eeae8e..e444498fca9 100644 --- a/encodings/sparse/src/ops.rs +++ b/encodings/sparse/src/ops.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ProbeState; +use vortex_array::patches::PatchesProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; @@ -13,7 +14,7 @@ use crate::Sparse; use crate::SparseExt as _; impl OperationsVTable for Sparse { - type ProbeState = (); + type ProbeState = PatchesProbeState; fn probe_scalar( state: &mut ProbeState<'_, Sparse>, diff --git a/vortex-array/src/arrays/runend/ops.rs b/vortex-array/src/arrays/runend/ops.rs index b75aa10e1ce..6c833d85787 100644 --- a/vortex-array/src/arrays/runend/ops.rs +++ b/vortex-array/src/arrays/runend/ops.rs @@ -19,7 +19,7 @@ use crate::search_sorted::SearchSortedSide; use crate::vtable::OperationsVTable; impl OperationsVTable for RunEnd { - type ProbeState = (); + type ProbeState = super::probe::RunEndProbeState; fn probe_scalar( state: &mut ProbeState<'_, RunEnd>, diff --git a/vortex-array/src/arrays/runend/probe.rs b/vortex-array/src/arrays/runend/probe.rs index 46f91bb4fcd..80aa7b6696a 100644 --- a/vortex-array/src/arrays/runend/probe.rs +++ b/vortex-array/src/arrays/runend/probe.rs @@ -1,8 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! RunEnd probes retain their ends and values child probes across lookups. +//! RunEnd probes retain their decoded ends and their values child probe across lookups. +use vortex_buffer::Buffer; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; @@ -10,8 +12,17 @@ use crate::ExecutionCtx; use crate::ProbeState; use crate::arrays::runend::RunEnd; use crate::arrays::runend::RunEndArrayExt; +use crate::arrays::runend::RunEndArraySlotsExt; use crate::arrays::runend::RunEndSlots; use crate::scalar::Scalar; +use crate::search_sorted::decode_sorted_positions; + +/// The run ends a repeated probe keeps, decoded on its first read so that later reads binary +/// search a plain buffer instead of reading the ends a value at a time. +#[derive(Debug, Default)] +pub struct RunEndProbeState { + ends: Option>, +} pub(crate) fn scalar_at( state: &mut ProbeState<'_, RunEnd>, @@ -23,27 +34,41 @@ pub(crate) fn scalar_at( .offset() .checked_add(index) .ok_or_else(|| vortex_err!("RunEnd logical index overflow"))?; - // Search for the first end strictly greater than the logical index. Every comparison - // uses the same ends probe, so a retained read keeps the child's preparation within and - // between searches. - let mut ends = state - .slot(RunEndSlots::ENDS)? - .ok_or_else(|| vortex_err!("RunEnd ends slot is missing"))?; - let mut left = 0; - let mut right = ends.array().len(); - while left < right { - let mid = left + (right - left) / 2; - let end = usize::try_from(&ends.execute_scalar(mid, ctx)?)?; - if end <= logical_index { - left = mid + 1; - } else { - right = mid; + let (retained, mut children) = state.split(); + // The physical index of the run is that of the first end strictly greater than the logical + // index. + let run = match retained { + Some(retained) => { + if retained.ends.is_none() { + retained.ends = Some(decode_sorted_positions(array.ends(), ctx)?); + } + let ends = retained.ends.as_ref().vortex_expect("decoded above"); + let logical_index = u64::try_from(logical_index)?; + ends.partition_point(|&end| end <= logical_index) + } + None => { + // A one-off read searches through one ends probe rather than decoding every end. + let mut ends = children + .slot(RunEndSlots::ENDS)? + .ok_or_else(|| vortex_err!("RunEnd ends slot is missing"))?; + let mut left = 0; + let mut right = ends.array().len(); + while left < right { + let mid = left + (right - left) / 2; + let end = usize::try_from(&ends.execute_scalar(mid, ctx)?)?; + if end <= logical_index { + left = mid + 1; + } else { + right = mid; + } + } + left } - } - state + }; + children .slot(RunEndSlots::VALUES)? .ok_or_else(|| vortex_err!("RunEnd values slot is missing"))? - .execute_scalar(left, ctx) + .execute_scalar(run, ctx) } #[cfg(test)] diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 2cd637beef7..2160d372583 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::cell::RefCell; -use std::cmp::Ordering; use std::fmt::Debug; use std::hash::Hash; use std::ops::Range; @@ -10,6 +8,7 @@ use std::ops::Range; use num_traits::NumCast; use vortex_buffer::BitBuffer; use vortex_buffer::BitBufferMut; +use vortex_buffer::Buffer; use vortex_buffer::BufferAllocatorRef; use vortex_buffer::BufferMut; use vortex_error::VortexError; @@ -27,6 +26,7 @@ use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayProbe; use crate::ArrayRef; use crate::ArraySlots; +use crate::EncodingProbeState; use crate::ExecutionCtx; use crate::IntoArray; use crate::ProbeState; @@ -44,11 +44,11 @@ use crate::dtype::UnsignedPType; use crate::match_each_unsigned_integer_ptype; use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; -use crate::search_sorted::IndexOrd; use crate::search_sorted::SearchResult; use crate::search_sorted::SearchSorted; use crate::search_sorted::SearchSortedPrimitiveArray; use crate::search_sorted::SearchSortedSide; +use crate::search_sorted::decode_sorted_positions; use crate::validity::Validity; use crate::vtable::VTable; @@ -141,6 +141,37 @@ pub struct PatchesData { offset_within_chunk: Option, } +/// What a repeated probe keeps for the patches of the array it reads: the patch indices, +/// decoded on its first read so that later reads binary search a plain buffer. +/// +/// Encodings with patches use it as, or inside, their +/// [`OperationsVTable::ProbeState`](crate::vtable::OperationsVTable::ProbeState) and read +/// through [`Patches::probe_patched`]. +#[derive(Debug, Default)] +pub struct PatchesProbeState { + indices: Option>, +} + +impl PatchesProbeState { + /// The decoded patch indices, decoding `indices` on the first call. + fn indices( + &mut self, + indices: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<&Buffer> { + if self.indices.is_none() { + self.indices = Some(decode_sorted_positions(indices, ctx)?); + } + Ok(self.indices.as_ref().vortex_expect("decoded above")) + } +} + +impl AsMut for PatchesProbeState { + fn as_mut(&mut self) -> &mut PatchesProbeState { + self + } +} + /// Slot indices for the three patch components within a slot array. #[derive(Copy, Clone, Debug)] pub struct PatchSlotIndices { @@ -639,72 +670,37 @@ impl Patches { /// Get the patched value at `index` of the array `state` reads, as [`Self::get_patched`] /// does. /// - /// `self` must be the patches held in the `slots` of that array. The patch children are - /// read through `state`, so a [`RepeatedArrayProbe`](crate::RepeatedArrayProbe) keeps their - /// probes across reads instead of building a searcher per read. A one-off read takes the - /// [`Self::get_patched`] path. + /// `self` must be the patches held in the `slots` of that array. A repeated probe decodes the + /// patch indices into its [`PatchesProbeState`] on its first read, so every later read binary + /// searches a plain buffer and reads the value through the probe's values slot. A one-off + /// read takes the [`Self::get_patched`] path, which does not decode the whole index array. pub fn probe_patched( &self, state: &mut ProbeState<'_, V>, slots: PatchSlotIndices, index: usize, ctx: &mut ExecutionCtx, - ) -> VortexResult> { - // A one-off read keeps nothing between reads, so reading the children through `state` - // would build a probe per comparison. `get_patched` shares one probe across the search. - if state.retained().is_none() { + ) -> VortexResult> + where + EncodingProbeState: AsMut, + { + let (retained, mut children) = state.split(); + let Some(retained) = retained else { return self.get_patched(index, ctx); - } - - let range = if self.chunk_offsets.is_some() { - let mut chunk_offsets = state - .slot(slots.chunk_offsets)? - .ok_or_else(|| vortex_err!("patch chunk offsets slot is missing"))?; - match self.chunk_patch_range(&mut chunk_offsets, index, ctx)? { - Some(range) => range, - None => return Ok(None), - } - } else { - 0..self.indices.len() }; + let indices = retained.as_mut().indices(&self.indices, ctx)?; - let needle = index + self.offset; - let found = if let Some(primitive) = self.indices.as_opt::() { - match_each_unsigned_integer_ptype!(primitive.ptype(), |T| { - // A needle that does not fit the index type is larger than every patch index. - let Ok(needle) = T::try_from(needle) else { - return Ok(None); - }; - primitive - .as_slice::() - .get(range.clone()) - .ok_or_else(|| { - vortex_err!( - "patch range {range:?} out of bounds for {} indices", - primitive.len() - ) - })? - .search_sorted(&needle, SearchSortedSide::Left)? - }) - } else { - let indices = state - .slot(slots.indices)? - .ok_or_else(|| vortex_err!("patch indices slot is missing"))?; - ProbedIndices { - probe: RefCell::new(indices), - ctx: RefCell::new(&mut *ctx), - range: range.clone(), - } - .search_sorted(&needle, SearchSortedSide::Left)? + // A needle past `u64` is past every patch index. + let Ok(needle) = u64::try_from(index + self.offset) else { + return Ok(None); }; - - let Some(found) = found.to_found() else { + let Ok(position) = indices.binary_search(&needle) else { return Ok(None); }; - state + children .slot(slots.values)? .ok_or_else(|| vortex_err!("patch values slot is missing"))? - .execute_scalar(range.start + found, ctx) + .execute_scalar(position, ctx) .map(Some) } @@ -1172,31 +1168,6 @@ impl Patches { } } -/// A [`SearchSorted`] adapter over the patch indices in `range`, read through a probe. -struct ProbedIndices<'p, 'c> { - probe: RefCell>, - ctx: RefCell<&'c mut ExecutionCtx>, - range: Range, -} - -impl IndexOrd for ProbedIndices<'_, '_> { - /// The probe and the context are separate cells, so the two borrows never overlap. - fn index_cmp(&self, idx: usize, elem: &usize) -> VortexResult> { - let value = self - .probe - .borrow_mut() - .execute_scalar(self.range.start + idx, &mut self.ctx.borrow_mut())? - .as_primitive() - .as_::() - .ok_or_else(|| vortex_err!("patch index does not fit in usize"))?; - Ok(Some(value.cmp(elem))) - } - - fn index_len(&self) -> usize { - self.range.len() - } -} - /// Binary searches for each of `needles` in the indices array, sharing one searcher. /// /// # Returns diff --git a/vortex-array/src/search_sorted/decoded.rs b/vortex-array/src/search_sorted/decoded.rs new file mode 100644 index 00000000000..0889edf7c6c --- /dev/null +++ b/vortex-array/src/search_sorted/decoded.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use num_traits::AsPrimitive; +use vortex_buffer::Buffer; +use vortex_error::VortexResult; + +use crate::ArrayRef; +use crate::ExecutionCtx; +use crate::arrays::PrimitiveArray; +use crate::arrays::primitive::PrimitiveArrayExt; +use crate::match_each_unsigned_integer_ptype; + +/// Decodes a sorted, non-null array of unsigned positions, such as patch indices or run ends, +/// into a `u64` buffer that can be binary searched directly. +/// +/// A repeated probe decodes such a child once instead of reading it a value at a time on every +/// search. A one-off search should not call this: it costs a full decode. +pub fn decode_sorted_positions( + array: &ArrayRef, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + let primitive = array.clone().execute::(ctx)?; + match_each_unsigned_integer_ptype!(primitive.ptype(), |T| { + // Every unsigned position type widens to `u64` without loss. + Ok(primitive + .as_slice::() + .iter() + .map(|&position| AsPrimitive::::as_(position)) + .collect::>()) + }) +} diff --git a/vortex-array/src/search_sorted/mod.rs b/vortex-array/src/search_sorted/mod.rs index e390fb3b7ae..1294365e8fe 100644 --- a/vortex-array/src/search_sorted/mod.rs +++ b/vortex-array/src/search_sorted/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod decoded; mod primitive; mod scalar; @@ -13,6 +14,7 @@ use std::fmt::Display; use std::fmt::Formatter; use std::hint; +pub use decoded::*; pub use primitive::*; pub use scalar::*; use vortex_error::VortexResult; From 7080a70c1d706f9dcc9fb8047376caa11a051621 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 9 Nov 2026 19:00:40 +0000 Subject: [PATCH 5/7] bench: count patched_search in CodSpeed shards Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jck72aD351kMboqN5g8Ccs --- scripts/codspeed_bench_counts.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/codspeed_bench_counts.json b/scripts/codspeed_bench_counts.json index 8b7757be5ea..5141596fd37 100644 --- a/scripts/codspeed_bench_counts.json +++ b/scripts/codspeed_bench_counts.json @@ -122,7 +122,8 @@ "cast_bitpacked": 6, "compute_between": 11, "for_decode": 0, - "for_encode": 0 + "for_encode": 0, + "patched_search": 6 }, "vortex-file": { "split_collection": 3 From 6bdff3046b22d084bf41c90fd82e7292a6ec1f40 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 13 Nov 2026 22:36:15 +0000 Subject: [PATCH 6/7] perf: search sorted children through the caller's shared probe Split the decode-once caching out into a follow-up PR. This PR now only changes how sorted searches are called: - SearchSortedPrimitiveArray::with_probe searches through a probe the caller already holds, such as a ProbeState child slot. - RunEnd's probe lookup searches its ends slot through that API instead of a hand-rolled binary search, so a one-off read shares one probe across the search and a repeated read keeps its ends probe. - Patches::probe_patched searches the indices slot and reads the values slot through the probe, for repeated reads of Sparse, BitPacked, ALP and ALP-RD. One-off reads keep get_patched. - Encoding probe states go back to (); nothing is decoded ahead. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jck72aD351kMboqN5g8Ccs --- encodings/alp/src/alp/ops.rs | 3 +- encodings/alp/src/alp_rd/ops.rs | 3 +- .../src/bitpacking/vtable/operations.rs | 3 +- encodings/sparse/src/ops.rs | 3 +- vortex-array/src/arrays/runend/ops.rs | 2 +- vortex-array/src/arrays/runend/probe.rs | 63 ++++++----------- vortex-array/src/patches.rs | 67 +++++-------------- vortex-array/src/search_sorted/decoded.rs | 32 --------- vortex-array/src/search_sorted/mod.rs | 2 - vortex-array/src/search_sorted/primitive.rs | 65 +++++++++++++++--- 10 files changed, 95 insertions(+), 148 deletions(-) delete mode 100644 vortex-array/src/search_sorted/decoded.rs diff --git a/encodings/alp/src/alp/ops.rs b/encodings/alp/src/alp/ops.rs index f3ddabd2c91..5ba2e8f6817 100644 --- a/encodings/alp/src/alp/ops.rs +++ b/encodings/alp/src/alp/ops.rs @@ -4,7 +4,6 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ProbeState; -use vortex_array::patches::PatchesProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexExpect; @@ -19,7 +18,7 @@ use crate::alp::array::PATCH_SLOTS; use crate::match_each_alp_float_ptype; impl OperationsVTable for ALP { - type ProbeState = PatchesProbeState; + type ProbeState = (); fn probe_scalar( state: &mut ProbeState<'_, ALP>, diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index 1e3fe5f4757..8af0690b544 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -5,7 +5,6 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ProbeState; use vortex_array::dtype::PType; -use vortex_array::patches::PatchesProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexExpect; @@ -18,7 +17,7 @@ use crate::alp_rd::array::LP_PATCH_SLOTS; use crate::alp_rd::code_not_in_dictionary; impl OperationsVTable for ALPRD { - type ProbeState = PatchesProbeState; + type ProbeState = (); fn probe_scalar( state: &mut ProbeState<'_, ALPRD>, diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 9e4b29268ad..f276cde4d9d 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -4,7 +4,6 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ProbeState; -use vortex_array::patches::PatchesProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; @@ -16,7 +15,7 @@ use crate::bitpacking::array::BitPackedArrayExt; use crate::bitpacking::array::PATCH_SLOTS; impl OperationsVTable for BitPacked { - type ProbeState = PatchesProbeState; + type ProbeState = (); fn probe_scalar( state: &mut ProbeState<'_, BitPacked>, diff --git a/encodings/sparse/src/ops.rs b/encodings/sparse/src/ops.rs index e444498fca9..0c6f6eeae8e 100644 --- a/encodings/sparse/src/ops.rs +++ b/encodings/sparse/src/ops.rs @@ -4,7 +4,6 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ProbeState; -use vortex_array::patches::PatchesProbeState; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; @@ -14,7 +13,7 @@ use crate::Sparse; use crate::SparseExt as _; impl OperationsVTable for Sparse { - type ProbeState = PatchesProbeState; + type ProbeState = (); fn probe_scalar( state: &mut ProbeState<'_, Sparse>, diff --git a/vortex-array/src/arrays/runend/ops.rs b/vortex-array/src/arrays/runend/ops.rs index 6c833d85787..b75aa10e1ce 100644 --- a/vortex-array/src/arrays/runend/ops.rs +++ b/vortex-array/src/arrays/runend/ops.rs @@ -19,7 +19,7 @@ use crate::search_sorted::SearchSortedSide; use crate::vtable::OperationsVTable; impl OperationsVTable for RunEnd { - type ProbeState = super::probe::RunEndProbeState; + type ProbeState = (); fn probe_scalar( state: &mut ProbeState<'_, RunEnd>, diff --git a/vortex-array/src/arrays/runend/probe.rs b/vortex-array/src/arrays/runend/probe.rs index 80aa7b6696a..db24d807785 100644 --- a/vortex-array/src/arrays/runend/probe.rs +++ b/vortex-array/src/arrays/runend/probe.rs @@ -1,10 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! RunEnd probes retain their decoded ends and their values child probe across lookups. +//! RunEnd probes retain their ends and values child probes across lookups. -use vortex_buffer::Buffer; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; @@ -12,17 +10,12 @@ use crate::ExecutionCtx; use crate::ProbeState; use crate::arrays::runend::RunEnd; use crate::arrays::runend::RunEndArrayExt; -use crate::arrays::runend::RunEndArraySlotsExt; use crate::arrays::runend::RunEndSlots; +use crate::match_each_unsigned_integer_ptype; use crate::scalar::Scalar; -use crate::search_sorted::decode_sorted_positions; - -/// The run ends a repeated probe keeps, decoded on its first read so that later reads binary -/// search a plain buffer instead of reading the ends a value at a time. -#[derive(Debug, Default)] -pub struct RunEndProbeState { - ends: Option>, -} +use crate::search_sorted::SearchSorted; +use crate::search_sorted::SearchSortedPrimitiveArray; +use crate::search_sorted::SearchSortedSide; pub(crate) fn scalar_at( state: &mut ProbeState<'_, RunEnd>, @@ -34,38 +27,20 @@ pub(crate) fn scalar_at( .offset() .checked_add(index) .ok_or_else(|| vortex_err!("RunEnd logical index overflow"))?; - let (retained, mut children) = state.split(); - // The physical index of the run is that of the first end strictly greater than the logical - // index. - let run = match retained { - Some(retained) => { - if retained.ends.is_none() { - retained.ends = Some(decode_sorted_positions(array.ends(), ctx)?); - } - let ends = retained.ends.as_ref().vortex_expect("decoded above"); - let logical_index = u64::try_from(logical_index)?; - ends.partition_point(|&end| end <= logical_index) - } - None => { - // A one-off read searches through one ends probe rather than decoding every end. - let mut ends = children - .slot(RunEndSlots::ENDS)? - .ok_or_else(|| vortex_err!("RunEnd ends slot is missing"))?; - let mut left = 0; - let mut right = ends.array().len(); - while left < right { - let mid = left + (right - left) / 2; - let end = usize::try_from(&ends.execute_scalar(mid, ctx)?)?; - if end <= logical_index { - left = mid + 1; - } else { - right = mid; - } - } - left - } - }; - children + // The run is that of the first end strictly greater than the logical index. The search + // reads the ends through this state's ends probe, so a retained read keeps the child's + // preparation within and between searches, and a one-off read shares one probe across the + // search. + let ends = state + .slot(RunEndSlots::ENDS)? + .ok_or_else(|| vortex_err!("RunEnd ends slot is missing"))?; + let len = ends.array().len(); + let run = match_each_unsigned_integer_ptype!(ends.array().dtype().as_ptype(), |T| { + SearchSortedPrimitiveArray::::with_probe(ends, ctx) + .search_sorted(&logical_index, SearchSortedSide::Right)? + .to_ends_index(len) + }); + state .slot(RunEndSlots::VALUES)? .ok_or_else(|| vortex_err!("RunEnd values slot is missing"))? .execute_scalar(run, ctx) diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 2160d372583..36eb65dbf37 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -8,7 +8,6 @@ use std::ops::Range; use num_traits::NumCast; use vortex_buffer::BitBuffer; use vortex_buffer::BitBufferMut; -use vortex_buffer::Buffer; use vortex_buffer::BufferAllocatorRef; use vortex_buffer::BufferMut; use vortex_error::VortexError; @@ -26,7 +25,6 @@ use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayProbe; use crate::ArrayRef; use crate::ArraySlots; -use crate::EncodingProbeState; use crate::ExecutionCtx; use crate::IntoArray; use crate::ProbeState; @@ -48,7 +46,6 @@ use crate::search_sorted::SearchResult; use crate::search_sorted::SearchSorted; use crate::search_sorted::SearchSortedPrimitiveArray; use crate::search_sorted::SearchSortedSide; -use crate::search_sorted::decode_sorted_positions; use crate::validity::Validity; use crate::vtable::VTable; @@ -141,37 +138,6 @@ pub struct PatchesData { offset_within_chunk: Option, } -/// What a repeated probe keeps for the patches of the array it reads: the patch indices, -/// decoded on its first read so that later reads binary search a plain buffer. -/// -/// Encodings with patches use it as, or inside, their -/// [`OperationsVTable::ProbeState`](crate::vtable::OperationsVTable::ProbeState) and read -/// through [`Patches::probe_patched`]. -#[derive(Debug, Default)] -pub struct PatchesProbeState { - indices: Option>, -} - -impl PatchesProbeState { - /// The decoded patch indices, decoding `indices` on the first call. - fn indices( - &mut self, - indices: &ArrayRef, - ctx: &mut ExecutionCtx, - ) -> VortexResult<&Buffer> { - if self.indices.is_none() { - self.indices = Some(decode_sorted_positions(indices, ctx)?); - } - Ok(self.indices.as_ref().vortex_expect("decoded above")) - } -} - -impl AsMut for PatchesProbeState { - fn as_mut(&mut self) -> &mut PatchesProbeState { - self - } -} - /// Slot indices for the three patch components within a slot array. #[derive(Copy, Clone, Debug)] pub struct PatchSlotIndices { @@ -670,31 +636,30 @@ impl Patches { /// Get the patched value at `index` of the array `state` reads, as [`Self::get_patched`] /// does. /// - /// `self` must be the patches held in the `slots` of that array. A repeated probe decodes the - /// patch indices into its [`PatchesProbeState`] on its first read, so every later read binary - /// searches a plain buffer and reads the value through the probe's values slot. A one-off - /// read takes the [`Self::get_patched`] path, which does not decode the whole index array. + /// `self` must be the patches held in the `slots` of that array. A repeated read searches the + /// patch indices through the state's indices probe and reads the value through its values + /// probe, so it shares both across reads rather than building them per read. A one-off read + /// takes the [`Self::get_patched`] path, which can narrow the search by the chunk offsets. pub fn probe_patched( &self, state: &mut ProbeState<'_, V>, slots: PatchSlotIndices, index: usize, ctx: &mut ExecutionCtx, - ) -> VortexResult> - where - EncodingProbeState: AsMut, - { + ) -> VortexResult> { let (retained, mut children) = state.split(); - let Some(retained) = retained else { + if retained.is_none() { return self.get_patched(index, ctx); - }; - let indices = retained.as_mut().indices(&self.indices, ctx)?; - - // A needle past `u64` is past every patch index. - let Ok(needle) = u64::try_from(index + self.offset) else { - return Ok(None); - }; - let Ok(position) = indices.binary_search(&needle) else { + } + let indices = children + .slot(slots.indices)? + .ok_or_else(|| vortex_err!("patch indices slot is missing"))?; + let needle = index + self.offset; + let found = match_each_unsigned_integer_ptype!(indices.array().dtype().as_ptype(), |T| { + SearchSortedPrimitiveArray::::with_probe(indices, ctx) + .search_sorted(&needle, SearchSortedSide::Left)? + }); + let SearchResult::Found(position) = found else { return Ok(None); }; children diff --git a/vortex-array/src/search_sorted/decoded.rs b/vortex-array/src/search_sorted/decoded.rs deleted file mode 100644 index 0889edf7c6c..00000000000 --- a/vortex-array/src/search_sorted/decoded.rs +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use num_traits::AsPrimitive; -use vortex_buffer::Buffer; -use vortex_error::VortexResult; - -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::arrays::PrimitiveArray; -use crate::arrays::primitive::PrimitiveArrayExt; -use crate::match_each_unsigned_integer_ptype; - -/// Decodes a sorted, non-null array of unsigned positions, such as patch indices or run ends, -/// into a `u64` buffer that can be binary searched directly. -/// -/// A repeated probe decodes such a child once instead of reading it a value at a time on every -/// search. A one-off search should not call this: it costs a full decode. -pub fn decode_sorted_positions( - array: &ArrayRef, - ctx: &mut ExecutionCtx, -) -> VortexResult> { - let primitive = array.clone().execute::(ctx)?; - match_each_unsigned_integer_ptype!(primitive.ptype(), |T| { - // Every unsigned position type widens to `u64` without loss. - Ok(primitive - .as_slice::() - .iter() - .map(|&position| AsPrimitive::::as_(position)) - .collect::>()) - }) -} diff --git a/vortex-array/src/search_sorted/mod.rs b/vortex-array/src/search_sorted/mod.rs index 1294365e8fe..e390fb3b7ae 100644 --- a/vortex-array/src/search_sorted/mod.rs +++ b/vortex-array/src/search_sorted/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -mod decoded; mod primitive; mod scalar; @@ -14,7 +13,6 @@ use std::fmt::Display; use std::fmt::Formatter; use std::hint; -pub use decoded::*; pub use primitive::*; pub use scalar::*; use vortex_error::VortexResult; diff --git a/vortex-array/src/search_sorted/primitive.rs b/vortex-array/src/search_sorted/primitive.rs index 69601ac71f8..0d6dfa90cc1 100644 --- a/vortex-array/src/search_sorted/primitive.rs +++ b/vortex-array/src/search_sorted/primitive.rs @@ -5,8 +5,10 @@ use std::cell::RefCell; use std::cmp::Ordering; use std::marker::PhantomData; +use vortex_error::VortexExpect; use vortex_error::VortexResult; +use crate::ArrayProbe; use crate::ArrayRef; use crate::ExecutionCtx; use crate::RepeatedArrayProbe; @@ -38,6 +40,9 @@ enum Reader<'a, T> { Values(&'a [T]), /// Anything else, through one probe reused by the whole search. Probe(RefCell), + /// Anything else, through a probe the caller already holds, such as a child slot of the + /// array it is reading, so the search shares whatever that probe has resolved. + Borrowed(RefCell>), } impl<'a, T: NativePType> SearchSortedPrimitiveArray<'a, T> { @@ -57,34 +62,74 @@ impl<'a, T: NativePType> SearchSortedPrimitiveArray<'a, T> { } } + /// Wraps the array `probe` reads for searching, reading through `probe` rather than a new + /// one, panicking if the array's [`PType`](crate::dtype::PType) is not `T::PTYPE`. + /// + /// A caller that already holds a probe of the array, such as a child slot of a + /// [`ProbeState`](crate::ProbeState), searches through it so a repeated read shares that + /// probe's state across its searches. + pub fn with_probe(probe: ArrayProbe<'a>, ctx: &'a mut ExecutionCtx) -> Self { + let array = probe.array(); + assert_eq!( + array.dtype().as_ptype(), + T::PTYPE, + "Array PType must match primitive type" + ); + let len = array.len(); + let reader = match probe { + ArrayProbe::Once(array) => Self::reader(array), + ArrayProbe::Repeated(probe) if Self::values(probe.array()).is_some() => { + let probe: &'a RepeatedArrayProbe = probe; + Reader::Values( + Self::values(probe.array()).vortex_expect("checked to offer values above"), + ) + } + probe @ ArrayProbe::Repeated(_) => Reader::Borrowed(RefCell::new(probe)), + }; + Self { + reader, + len, + ctx: RefCell::new(ctx), + _ptype: PhantomData, + } + } + /// Reads the buffer directly when the array can offer one, and probes otherwise. /// /// A nullable array is excluded because the buffer holds an arbitrary value where an element /// is null, and a device-backed array because its buffer is not addressable here. fn reader(array: &'a ArrayRef) -> Reader<'a, T> { + match Self::values(array) { + Some(values) => Reader::Values(values), + None => Reader::Probe(RefCell::new(array.repeated_probe())), + } + } + + /// The array's own buffer, if it is a canonical, non-nullable, host-backed array. + fn values(array: &ArrayRef) -> Option<&[T]> { if !array.dtype().is_nullable() && let Some(primitive) = array.as_opt::() && primitive.buffer_handle().as_host_opt().is_some() - && let Some(values) = primitive.data().as_slice::().get(..array.len()) { - return Reader::Values(values); + return primitive.data().as_slice::().get(..array.len()); } - Reader::Probe(RefCell::new(array.repeated_probe())) + None } /// Returns the value at `idx`, or `None` if the element is null. /// /// The probe and the context are separate cells, so the two borrows never overlap. fn typed_value(&self, idx: usize) -> VortexResult> { - let probe = match &self.reader { + let scalar = match &self.reader { Reader::Values(values) => return Ok(Some(values[idx])), - Reader::Probe(probe) => probe, + Reader::Probe(probe) => probe + .borrow_mut() + .execute_scalar(idx, &mut self.ctx.borrow_mut())?, + Reader::Borrowed(probe) => probe + .borrow_mut() + .execute_scalar(idx, &mut self.ctx.borrow_mut())?, }; - Ok(probe - .borrow_mut() - .execute_scalar(idx, &mut self.ctx.borrow_mut())? - .as_primitive() - .typed_value::()) + Ok(scalar.as_primitive().typed_value::()) } /// Returns the value at `idx`, with nulls mapped to `T::zero()`. From 43742ab21411128967fd41084a20fb02ce504095 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sun, 15 Nov 2026 13:32:48 +0000 Subject: [PATCH 7/7] bench: keep patched_search below 1ms per CodSpeed benchmark Under CodSpeed, do 16 reads and 8 slices per iteration instead of 256 and 64. The simulated cost of the earlier counts was 2.1ms to 5.7ms. Also wrap the module doc comment and add blank lines between the blocks of Patches::probe_patched. Signed-off-by: Robert Kruszewski --- encodings/fastlanes/benches/patched_search.rs | 12 +++++++----- vortex-array/src/patches.rs | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/encodings/fastlanes/benches/patched_search.rs b/encodings/fastlanes/benches/patched_search.rs index 40eada055bc..00717b40229 100644 --- a/encodings/fastlanes/benches/patched_search.rs +++ b/encodings/fastlanes/benches/patched_search.rs @@ -3,8 +3,9 @@ //! Sorted searches over compressed children: reads, through one-off and repeated probes, of a //! Sparse array whose patch indices are BitPacked and of a RunEnd array whose ends are BitPacked, -//! and slices of those patches and of that RunEnd array. Each read or slice binary searches the compressed child, so -//! these measure how much of the child's preparation is shared across searches. +//! and slices of those patches and of that RunEnd array. Each read or slice binary searches the +//! compressed child, so these measure how much of the child's preparation is shared across +//! searches. use std::ops::Range; use std::sync::LazyLock; @@ -52,9 +53,10 @@ static SESSION: LazyLock = LazyLock::new(|| { const LEN: usize = 1 << 20; /// One patch, or one run end, every `STRIDE` rows. const STRIDE: usize = 64; -// Sized to keep the CodSpeed simulation short per benchmark. -const ACCESSES: usize = 256; -const SLICES: usize = 64; +// A simulated read costs up to about 22µs and a simulated slice up to about 52µs, so these +// counts keep each CodSpeed benchmark below 1ms. +const ACCESSES: usize = if cfg!(codspeed) { 16 } else { 256 }; +const SLICES: usize = if cfg!(codspeed) { 8 } else { 64 }; const SLICE_LEN: usize = 4_096; /// Sorted positions every `STRIDE` rows, BitPacked. diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 36eb65dbf37..a3bc6574b58 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -651,6 +651,7 @@ impl Patches { if retained.is_none() { return self.get_patched(index, ctx); } + let indices = children .slot(slots.indices)? .ok_or_else(|| vortex_err!("patch indices slot is missing"))?; @@ -659,9 +660,11 @@ impl Patches { SearchSortedPrimitiveArray::::with_probe(indices, ctx) .search_sorted(&needle, SearchSortedSide::Left)? }); + let SearchResult::Found(position) = found else { return Ok(None); }; + children .slot(slots.values)? .ok_or_else(|| vortex_err!("patch values slot is missing"))?