Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions vortex-buffer/benches/allocation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,3 +159,33 @@ fn from_vec_drop_arrow(bencher: Bencher, size: usize) {
}
});
}

/// Buffer lengths for the shrink benchmarks, from a small chunk to a large column.
const SHRINK_SIZES: &[usize] = &[65_536, 1 << 20, 4 << 20];

/// A byte buffer of `size` bytes whose capacity is double its length, as left by growth.
fn half_full(size: usize) -> BufferMut<u8> {
let mut buffer = BufferMut::<u8>::with_capacity(2 * size);
buffer.extend(std::iter::repeat_n(7u8, size));
buffer
}

#[divan::bench(args = SHRINK_SIZES)]
fn shrink_to_fit(bencher: Bencher, size: usize) {
bencher
.with_inputs(|| half_full(size))
.bench_local_values(|mut buffer| {
buffer.shrink_to_fit();
buffer.freeze()
});
}

/// Copying into an exactly sized buffer, the alternative to [`shrink_to_fit`].
#[divan::bench(args = SHRINK_SIZES)]
fn shrink_by_copy(bencher: Bencher, size: usize) {
bencher
.with_inputs(|| half_full(size))
.bench_local_values(|buffer| {
BufferMut::<u8>::copy_from_aligned(&buffer, buffer.alignment()).freeze()
});
}
Comment on lines +173 to +191

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe put them behind not(codspeed) cfg?

216 changes: 216 additions & 0 deletions vortex-buffer/src/allocation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,27 @@ impl Allocation {
self.ptr = allocation.cast();
self.layout = new_layout;
}

/// Shrink the allocation to `new_size` bytes, keeping its layout alignment.
///
/// The first `new_size` bytes are preserved, but the block may move. Returns `false` and
/// leaves the allocation unchanged if the allocator cannot shrink it.
pub(crate) fn shrink(&mut self, new_size: usize) -> bool {
debug_assert!(new_size > 0 && new_size <= self.layout.size());
let Ok(new_layout) = Layout::from_size_align(new_size, self.layout.align()) else {
return false;
};
// SAFETY: ptr denotes a live block owned by allocator, old_layout fits the block, and the
// new layout keeps the alignment and is no larger than the old one.
match unsafe { self.allocator.shrink(self.ptr, self.layout, new_layout) } {
Ok(allocation) => {
self.ptr = allocation.cast();
self.layout = new_layout;
true
}
Err(_) => false,
}
}
}

impl Drop for Allocation {
Expand Down Expand Up @@ -535,4 +556,199 @@ mod tests {
drop(original);
assert_eq!(state.deallocations.load(Ordering::Relaxed), 2);
}

/// Forwards to [`Global`], but places every block at a varying offset past a 64 KiB boundary.
/// Each shrink moves the block, so a buffer whose allocation is less aligned than the buffer
/// itself lands at a different alignment offset after the shrink.
#[derive(Debug, Default)]
struct MovingAllocator {
calls: AtomicUsize,
shrinks: AtomicUsize,
}

impl MovingAllocator {
const REAL_ALIGN: usize = 1 << 16;

/// The real block behind `ptr`: the shift is below `REAL_ALIGN`, so rounding down finds it.
fn real_block(ptr: NonNull<u8>, layout: Layout) -> (NonNull<u8>, Layout) {
let shift = ptr.addr().get() % Self::REAL_ALIGN;
// SAFETY: ptr was returned by allocate, which placed it shift bytes into a real block.
let real = unsafe { ptr.sub(shift) };
let real_layout = Layout::from_size_align(layout.size() + shift, Self::REAL_ALIGN)
.unwrap_or_else(|_| unreachable!("allocate checked this layout"));
(real, real_layout)
}
}

// SAFETY: every block is a live Global block, offset by a multiple of the requested alignment,
// and deallocation recovers the real block it came from.
unsafe impl Allocator for MovingAllocator {
fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
let call = self.calls.fetch_add(1, Ordering::Relaxed);
let shift = layout.align() * (1 + call % 7);
if shift >= Self::REAL_ALIGN {
return Err(AllocError);
}
let real_layout = Layout::from_size_align(layout.size() + shift, Self::REAL_ALIGN)
.map_err(|_| AllocError)?;
let real = Global.allocate(real_layout)?.cast::<u8>();
// SAFETY: real_layout reserves shift bytes before the returned block.
let ptr = unsafe { real.add(shift) };
Ok(NonNull::slice_from_raw_parts(ptr, layout.size()))
}

unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
let (real, real_layout) = Self::real_block(ptr, layout);
// SAFETY: real and real_layout describe the block from Global.allocate.
unsafe { Global.deallocate(real, real_layout) }
}

unsafe fn shrink(
&self,
ptr: NonNull<u8>,
old_layout: Layout,
new_layout: Layout,
) -> Result<NonNull<[u8]>, AllocError> {
self.shrinks.fetch_add(1, Ordering::Relaxed);
let new = self.allocate(new_layout)?;
// SAFETY: both blocks are live and disjoint, and new_layout is no larger than old.
unsafe {
std::ptr::copy_nonoverlapping(
ptr.as_ptr(),
new.cast::<u8>().as_ptr(),
new_layout.size(),
);
self.deallocate(ptr, old_layout);
}
Ok(new)
}
}

fn assert_shrunk<T: Copy + PartialEq + std::fmt::Debug>(
buffer: &BufferMut<T>,
expected: &[T],
alignment: Alignment,
) {
assert_eq!(buffer.as_slice(), expected);
assert!(alignment.is_ptr_aligned(buffer.as_ptr()));
assert!(buffer.capacity() >= buffer.len());
// The data keeps its offset of up to the over-alignment, plus padding to realign it.
let max_bytes = buffer.len() * size_of::<T>()
+ alignment
.as_usize()
.max(Alignment::DEFAULT_ALIGNMENT.as_usize())
+ alignment.as_usize()
- 1;
assert!(buffer.allocation.size() <= max_bytes.max(1));
}

#[rstest]
fn shrink_to_fit_realigns_moved_blocks(
#[values(8, 64, 4096)] alignment: usize,
#[values(1, 3, 1000)] len: u64,
) {
let allocator = Arc::new(MovingAllocator::default());
let alignment = Alignment::new(alignment);
// Growing from empty makes an allocation that is only byte aligned.
let mut buffer = BufferMut::<u64>::empty_aligned_in(
alignment,
BufferAllocatorRef::new(ArcAllocator(Arc::clone(&allocator))),
);
let expected: Vec<u64> = (0..len).collect();
buffer.reserve(8192);
buffer.extend_from_slice(&expected);
assert!(buffer.allocation.alignment() < alignment.as_usize());

buffer.shrink_to_fit();

assert_shrunk(&buffer, &expected, alignment);
assert_eq!(allocator.shrinks.load(Ordering::Relaxed), 1);
buffer.push(u64::MAX);
assert_eq!(buffer[expected.len()], u64::MAX);
assert!(alignment.is_ptr_aligned(buffer.as_ptr()));
}

#[rstest]
fn shrink_to_fit_with_global(
#[values(2, 8, 64, 4096)] alignment: usize,
#[values(0, 1, 1000)] len: u16,
) {
let alignment = Alignment::new(alignment);
let mut buffer = BufferMut::<u16>::with_capacity_aligned(1 << 16, alignment);
let expected: Vec<u16> = (0..len).collect();
buffer.extend_from_slice(&expected);

buffer.shrink_to_fit();

assert_shrunk(&buffer, &expected, alignment);
assert_eq!(buffer.alignment(), alignment);
}

#[test]
fn shrink_to_fit_drops_slice_prefix() -> VortexResult<()> {
let allocator = Arc::new(MovingAllocator::default());
let alignment = Alignment::new(64);
let values: Vec<u64> = (0..1024).collect();
let buffer = BufferMut::<u64>::copy_from_aligned_in(
&values,
alignment,
BufferAllocatorRef::new(ArcAllocator(Arc::clone(&allocator))),
)
.freeze()
.slice(512..520);
let mut buffer = buffer
.try_into_mut()
.map_err(|_| vortex_err!("unique buffer"))?;

buffer.shrink_to_fit();

assert_shrunk(&buffer, &values[512..520], alignment);
// The data sits after a prefix, so it is copied once into a new allocation.
assert_eq!(allocator.shrinks.load(Ordering::Relaxed), 0);
assert_eq!(allocator.calls.load(Ordering::Relaxed), 2);
Ok(())
}

#[rstest]
fn shrink_to_fit_is_idempotent(#[values(1, 64)] alignment: usize) {
let allocator = Arc::new(MovingAllocator::default());
let alignment = Alignment::new(alignment);
let mut buffer = BufferAllocatorRef::new(ArcAllocator(Arc::clone(&allocator)))
.with_capacity_aligned::<u8>(1024, alignment);
buffer.extend_from_slice(&[1, 2, 3]);

buffer.shrink_to_fit();
let ptr = buffer.as_ptr();
buffer.shrink_to_fit();

assert_eq!(buffer.as_slice(), [1, 2, 3]);
assert_eq!(buffer.as_ptr(), ptr);
assert_eq!(allocator.shrinks.load(Ordering::Relaxed), 1);
}

/// Shares a [`MovingAllocator`] between the buffer and the test.
#[derive(Debug)]
struct ArcAllocator(Arc<MovingAllocator>);

// SAFETY: forwards every call to the shared allocator.
unsafe impl Allocator for ArcAllocator {
fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
self.0.allocate(layout)
}

unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
// SAFETY: the caller upholds the Allocator contract.
unsafe { self.0.deallocate(ptr, layout) }
}

unsafe fn shrink(
&self,
ptr: NonNull<u8>,
old_layout: Layout,
new_layout: Layout,
) -> Result<NonNull<[u8]>, AllocError> {
// SAFETY: the caller upholds the Allocator contract.
unsafe { self.0.shrink(ptr, old_layout, new_layout) }
}
}
}
119 changes: 119 additions & 0 deletions vortex-buffer/src/buffer_mut.rs
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,125 @@ impl<T> BufferMut<T> {
}
}

/// Shrinks the capacity of the buffer as much as possible while keeping its alignment.
///
/// Has the same intent as [`Vec::shrink_to_fit`]: the allocator is asked to shrink the
/// allocation in place (a `realloc`), which usually avoids copying the data. If the allocation
/// does not guarantee the buffer's alignment, up to `alignment - 1` bytes of padding are kept so
/// the data can be realigned when the allocator moves the block. The data also keeps its offset
/// within the allocation, so the capacity may remain slightly larger than the length, and a
/// buffer whose slack is already that small is left unchanged. A buffer whose data starts after a prefix, such as a slice
/// turned back into a `BufferMut`, is copied once into a new allocation instead.
///
/// ```
/// use vortex_buffer::BufferMut;
///
/// let mut buffer = BufferMut::<u32>::with_capacity(1024);
/// buffer.extend_from_slice(&[1, 2, 3]);
/// buffer.shrink_to_fit();
/// assert_eq!(buffer.as_slice(), &[1, 2, 3]);
/// assert!(buffer.capacity() < 1024);
/// ```
pub fn shrink_to_fit(&mut self) {
if self.allocation.size() == 0 {
return;
}

let alignment = self.alignment.as_usize();
if self.length == 0 {
*self = Self::empty_aligned_in(self.alignment, self.allocation.allocator().clone());
return;
}

let data_bytes = self.length * size_of::<T>();
// A block aligned to the buffer's alignment keeps the data aligned at the same offset
// wherever the allocator moves it. A less aligned block needs padding to realign the data
// afterwards.
let padding = if self.allocation.alignment() >= alignment {
0
} else {
alignment - 1
};
let offset = self.ptr.cast::<u8>().addr().get() - self.allocation.ptr().addr().get();

// Constructors place the data at an offset of up to the over-alignment. Leave a buffer
// with no more slack than that, which also makes a second call a no-op.
let max_offset = alignment.max(Alignment::DEFAULT_ALIGNMENT.as_usize());
if self.allocation.size() <= max_offset + data_bytes + padding {
return;
}

// A larger offset is a prefix left by slicing, which a shrink would keep, so copy the data
// once into a new allocation instead.
if offset > max_offset {
let new_size = data_bytes + padding;
let offset = self.reallocate_exact(new_size, data_bytes);
// SAFETY: offset is aligned to the buffer's alignment and the data lies within the
// allocation from it.
self.ptr = unsafe { self.allocation.ptr().add(offset).cast() };
self.capacity = (self.allocation.size() - offset) / size_of::<T>();
return;
}

// Keep the data where it is, so a shrink in place copies nothing.
let new_size = offset + data_bytes + padding;

// `align_offset` rather than the address, so Miri's symbolic alignment check agrees.
let offset = if self.allocation.shrink(new_size)
&& self
.allocation
.ptr()
.as_ptr()
.wrapping_add(offset)
.align_offset(alignment)
!= 0
{
// The block moved to an address that misaligns the data, which needs padding.
let new_offset = self.allocation.ptr().as_ptr().align_offset(alignment);
// SAFETY: the shrink preserved the first new_size bytes, which hold the data at
// offset. new_offset < alignment, so new_offset + data_bytes <= padding + data_bytes
// <= new_size; copy permits overlap.
unsafe {
std::ptr::copy(
self.allocation.ptr().as_ptr().add(offset),
self.allocation.ptr().as_ptr().add(new_offset),
data_bytes,
);
}
new_offset
} else {
offset
};

// SAFETY: offset is aligned to the buffer's alignment and the data lies within the
// allocation from it.
self.ptr = unsafe { self.allocation.ptr().add(offset).cast() };
self.capacity = (self.allocation.size() - offset) / size_of::<T>();
}

/// Copies the data into a new allocation of `size` bytes with the same layout alignment and
/// allocator, and returns the aligned offset of the data in it.
fn reallocate_exact(&mut self, size: usize, data_bytes: usize) -> usize {
let layout = Layout::from_size_align(size, self.allocation.alignment())
.unwrap_or_else(|_| vortex_panic!("buffer capacity exceeds maximum allocation size"));
let allocation = Allocation::allocate(layout, self.allocation.allocator().clone());
let offset = allocation
.ptr()
.as_ptr()
.align_offset(self.alignment.as_usize());
// SAFETY: self.ptr addresses data_bytes of live data. The new allocation is disjoint and
// its padding leaves room for those bytes after offset.
unsafe {
std::ptr::copy_nonoverlapping(
self.ptr.cast::<u8>().as_ptr(),
allocation.ptr().as_ptr().add(offset),
data_bytes,
);
}
self.allocation = allocation;
offset
}

/// Reserves capacity for at least `additional` more elements to be inserted in the buffer.
#[inline]
pub fn reserve(&mut self, additional: usize) {
Expand Down
Loading