diff --git a/encodings/fsst/src/compress.rs b/encodings/fsst/src/compress.rs index 04051596fe3..fac6e9ebd1d 100644 --- a/encodings/fsst/src/compress.rs +++ b/encodings/fsst/src/compress.rs @@ -393,12 +393,26 @@ where .set(Stat::IsSorted, Precision::Exact(true.into())); let validity = Validity::from_mask(mask.clone(), dtype.nullability()); + // The codes buffer grows by doubling, so up to half of its allocation can be spare. + // Freezing keeps that allocation, which `nbytes` does not count, so copy the codes into an + // exactly sized buffer when the spare capacity is large. + let codes_bytes = if self.codes.capacity() - self.codes.len() > self.codes.len() / 4 { + ByteBufferMut::copy_from_aligned_in( + &self.codes, + self.codes.alignment(), + ctx.allocator().clone(), + ) + .freeze() + } else { + self.codes.freeze() + }; + // SAFETY: the offsets start at zero, grow monotonically, and end at the length of the // codes. The validity is the validity of the input, which has one entry per row. let codes = unsafe { VarBinArray::new_unchecked( codes_offsets.into_array(), - self.codes.freeze(), + codes_bytes, DType::Binary(dtype.nullability()), validity, ) @@ -424,11 +438,13 @@ where mod tests { use std::sync::LazyLock; + use rstest::rstest; use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::varbin::VarBinArraySlotsExt; use vortex_array::dtype::PType; use vortex_error::VortexResult; + use vortex_error::vortex_err; use vortex_session::VortexSession; use super::fsst_compress; @@ -464,4 +480,43 @@ mod tests { assert_eq!(fsst.codes().offsets().dtype().as_ptype(), PType::I32); Ok(()) } + + /// Regression for #11420: the codes buffer grows by doubling, and must not keep the spare + /// capacity of the last doubling once it is frozen. + #[rstest] + #[case::str_len_80(80)] + #[case::str_len_128(128)] + #[case::str_len_160(160)] + fn codes_buffer_has_little_spare_capacity(#[case] str_len: usize) -> VortexResult<()> { + let mut x: u64 = 1; + let strings: Vec = (0..8192) + .map(|_| { + (0..str_len / 16) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + format!("{x:016x}") + }) + .collect() + }) + .collect(); + let array = VarBinViewArray::from_iter_str(strings.iter().map(String::as_str)); + let mut ctx = SESSION.create_execution_ctx(); + let compressor = fsst_train_compressor(array.as_array(), &mut ctx)?; + let fsst = fsst_compress(array.as_array(), &compressor, &mut ctx)?; + + let codes = fsst.codes_bytes().clone(); + drop(fsst); + let codes = codes + .try_into_mut() + .map_err(|_| vortex_err!("codes buffer is still shared"))?; + let spare = codes.capacity() - codes.len(); + assert!( + spare <= codes.len() / 4, + "codes len {} has {spare} bytes of spare capacity", + codes.len() + ); + Ok(()) + } } diff --git a/encodings/onpair/src/compress.rs b/encodings/onpair/src/compress.rs index b46428445d8..220e8168112 100644 --- a/encodings/onpair/src/compress.rs +++ b/encodings/onpair/src/compress.rs @@ -82,7 +82,12 @@ pub fn onpair_compress( total_bytes, }; let column = onpair::compress_rows::<_, u64>(&rows, config); - let (dict, codes, row_offsets) = column.into_raw(); + let (dict, mut codes, row_offsets) = column.into_raw(); + // The library grows the codes from empty, so up to half of their allocation can be spare. + // The buffer keeps that allocation, which `nbytes` does not count. + if codes.capacity() - codes.len() > codes.len() / 4 { + codes.shrink_to_fit(); + } let (dict_bytes, dict_offsets) = dict.into_raw(); let codes_offsets = codes_offsets_array(&row_offsets); let codes = Buffer::from(codes).into_array(); diff --git a/encodings/onpair/src/tests.rs b/encodings/onpair/src/tests.rs index 57223199aa8..58cef222b25 100644 --- a/encodings/onpair/src/tests.rs +++ b/encodings/onpair/src/tests.rs @@ -679,3 +679,48 @@ fn test_sparse_dict_over_onpair_appends_to_builder() -> vortex_error::VortexResu ); Ok(()) } + +/// Regression for #11420: the `onpair` crate grows the codes from empty, and the compressed array +/// must not keep the spare capacity of their last growth. +#[rstest::rstest] +#[case::str_len_64(64)] +#[case::str_len_80(80)] +#[case::str_len_160(160)] +fn codes_buffer_has_little_spare_capacity( + #[case] str_len: usize, +) -> vortex_error::VortexResult<()> { + let mut x: u64 = 1; + let strings: Vec = (0..8192) + .map(|_| { + (0..str_len / 16) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + format!("{x:016x}") + }) + .collect() + }) + .collect(); + let array = VarBinViewArray::from_iter_str(strings.iter().map(String::as_str)).into_array(); + let mut ctx = SESSION.create_execution_ctx(); + let onpair = compress_onpair(&array, &mut ctx)?; + + let codes = onpair + .as_view() + .codes() + .clone() + .execute::(&mut ctx)? + .into_buffer::(); + drop(onpair); + let codes = codes + .try_into_mut() + .map_err(|_| vortex_error::vortex_err!("codes buffer is still shared"))?; + let spare = codes.capacity() - codes.len(); + assert!( + spare <= codes.len() / 4, + "{} codes have {spare} codes of spare capacity", + codes.len() + ); + Ok(()) +}