Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 56 additions & 1 deletion encodings/fsst/src/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -393,12 +393,26 @@ where
.set(Stat::IsSorted, Precision::Exact(true.into()));
let validity = Validity::from_mask(mask.clone(), dtype.nullability());

// The codes buffer grows by doubling, so up to half of its allocation can be spare.
// Freezing keeps that allocation, which `nbytes` does not count, so copy the codes into an
// exactly sized buffer when the spare capacity is large.
let codes_bytes = if self.codes.capacity() - self.codes.len() > self.codes.len() / 4 {
ByteBufferMut::copy_from_aligned_in(
&self.codes,
self.codes.alignment(),
ctx.allocator().clone(),
)
.freeze()
} else {
self.codes.freeze()
};

// SAFETY: the offsets start at zero, grow monotonically, and end at the length of the
// codes. The validity is the validity of the input, which has one entry per row.
let codes = unsafe {
VarBinArray::new_unchecked(
codes_offsets.into_array(),
self.codes.freeze(),
codes_bytes,
DType::Binary(dtype.nullability()),
validity,
)
Expand All @@ -424,11 +438,13 @@ where
mod tests {
use std::sync::LazyLock;

use rstest::rstest;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::VarBinViewArray;
use vortex_array::arrays::varbin::VarBinArraySlotsExt;
use vortex_array::dtype::PType;
use vortex_error::VortexResult;
use vortex_error::vortex_err;
use vortex_session::VortexSession;

use super::fsst_compress;
Expand Down Expand Up @@ -464,4 +480,43 @@ mod tests {
assert_eq!(fsst.codes().offsets().dtype().as_ptype(), PType::I32);
Ok(())
}

/// Regression for #11420: the codes buffer grows by doubling, and must not keep the spare
/// capacity of the last doubling once it is frozen.
#[rstest]
#[case::str_len_80(80)]
#[case::str_len_128(128)]
#[case::str_len_160(160)]
fn codes_buffer_has_little_spare_capacity(#[case] str_len: usize) -> VortexResult<()> {
let mut x: u64 = 1;
let strings: Vec<String> = (0..8192)
.map(|_| {
(0..str_len / 16)
.map(|_| {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
format!("{x:016x}")
})
.collect()
})
.collect();
let array = VarBinViewArray::from_iter_str(strings.iter().map(String::as_str));
let mut ctx = SESSION.create_execution_ctx();
let compressor = fsst_train_compressor(array.as_array(), &mut ctx)?;
let fsst = fsst_compress(array.as_array(), &compressor, &mut ctx)?;

let codes = fsst.codes_bytes().clone();
drop(fsst);
let codes = codes
.try_into_mut()
.map_err(|_| vortex_err!("codes buffer is still shared"))?;
let spare = codes.capacity() - codes.len();
assert!(
spare <= codes.len() / 4,
"codes len {} has {spare} bytes of spare capacity",
codes.len()
);
Ok(())
}
}
7 changes: 6 additions & 1 deletion encodings/onpair/src/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,12 @@ pub fn onpair_compress(
total_bytes,
};
let column = onpair::compress_rows::<_, u64>(&rows, config);
let (dict, codes, row_offsets) = column.into_raw();
let (dict, mut codes, row_offsets) = column.into_raw();
// The library grows the codes from empty, so up to half of their allocation can be spare.
// The buffer keeps that allocation, which `nbytes` does not count.
if codes.capacity() - codes.len() > codes.len() / 4 {
codes.shrink_to_fit();
}
let (dict_bytes, dict_offsets) = dict.into_raw();
let codes_offsets = codes_offsets_array(&row_offsets);
let codes = Buffer::from(codes).into_array();
Expand Down
45 changes: 45 additions & 0 deletions encodings/onpair/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -679,3 +679,48 @@ fn test_sparse_dict_over_onpair_appends_to_builder() -> vortex_error::VortexResu
);
Ok(())
}

/// Regression for #11420: the `onpair` crate grows the codes from empty, and the compressed array
/// must not keep the spare capacity of their last growth.
#[rstest::rstest]
#[case::str_len_64(64)]
#[case::str_len_80(80)]
#[case::str_len_160(160)]
fn codes_buffer_has_little_spare_capacity(
#[case] str_len: usize,
) -> vortex_error::VortexResult<()> {
let mut x: u64 = 1;
let strings: Vec<String> = (0..8192)
.map(|_| {
(0..str_len / 16)
.map(|_| {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
format!("{x:016x}")
})
.collect()
})
.collect();
let array = VarBinViewArray::from_iter_str(strings.iter().map(String::as_str)).into_array();
let mut ctx = SESSION.create_execution_ctx();
let onpair = compress_onpair(&array, &mut ctx)?;

let codes = onpair
.as_view()
.codes()
.clone()
.execute::<PrimitiveArray>(&mut ctx)?
.into_buffer::<u16>();
drop(onpair);
let codes = codes
.try_into_mut()
.map_err(|_| vortex_error::vortex_err!("codes buffer is still shared"))?;
let spare = codes.capacity() - codes.len();
assert!(
spare <= codes.len() / 4,
"{} codes have {spare} codes of spare capacity",
codes.len()
);
Ok(())
}