From 5be101375027089a25fd872720b722f735c43c97 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 26 Oct 2026 23:28:43 +0000 Subject: [PATCH] Aggregate nested columns under nullable parents with a definedness mask Struct fields are stored unmasked, so aggregates over them saw values under null struct rows. Each chunk of a sequential write stream now carries a definedness mask, false in the rows an ancestor nulls out, and DynAccumulator gains accumulate_defined to aggregate the logical column without masking first. Aggregates give definedness meaning through AggregateFnVTable::try_accumulate_defined. By default undefined rows read as null, which requires the same partial dtype for a nullable input as for the non-nullable one, and a test asserts this for every built-in aggregate that keeps the default. Combined aggregates hand the definedness to their children, and UncompressedSizeInBytes measures the stored batch, which definedness does not change. The struct writer intersects the definedness with its validity for each field, the list writer drops the elements of null and undefined lists so every element written is defined. Fixed-size lists give their elements the same semantics: helpers return the elements of defined, valid rows only, since their fixed slots cannot be dropped from storage. Re-chunking writers carry the mask with the rows. Zone maps, in either layout, and file statistics are now written for children of nullable structs, and the fuzzer generates nullable structs. Signed-off-by: Robert Kruszewski Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X1HnweRGxeDWgQwXXM6f5s --- fuzz/fuzz_targets/file_io.rs | 14 +- vortex-array/Cargo.toml | 4 + vortex-array/benches/aggregate_definedness.rs | 200 ++++ vortex-array/src/aggregate_fn/accumulator.rs | 75 ++ vortex-array/src/aggregate_fn/combined.rs | 16 + vortex-array/src/aggregate_fn/definedness.rs | 864 ++++++++++++++++++ .../fns/uncompressed_size_in_bytes/mod.rs | 17 + vortex-array/src/aggregate_fn/mod.rs | 1 + vortex-array/src/aggregate_fn/vtable.rs | 27 + vortex-cuda/src/layout.rs | 2 +- vortex-duckdb/src/copy.rs | 6 +- vortex-file/src/tests.rs | 65 +- vortex-file/src/writer.rs | 3 +- vortex-layout/src/layouts/buffered.rs | 12 +- vortex-layout/src/layouts/chunked/reader.rs | 16 +- vortex-layout/src/layouts/collect.rs | 7 +- vortex-layout/src/layouts/compressed.rs | 8 +- vortex-layout/src/layouts/dict/writer.rs | 85 +- vortex-layout/src/layouts/file_stats.rs | 209 +++-- vortex-layout/src/layouts/flat/writer.rs | 3 +- vortex-layout/src/layouts/indexed/writer.rs | 4 +- vortex-layout/src/layouts/list/writer.rs | 256 +++++- vortex-layout/src/layouts/repartition.rs | 100 +- vortex-layout/src/layouts/struct_/writer.rs | 279 +++--- vortex-layout/src/layouts/table.rs | 8 +- vortex-layout/src/layouts/zoned/builder.rs | 19 + vortex-layout/src/layouts/zoned/mod.rs | 1 + vortex-layout/src/layouts/zoned/writer.rs | 216 +++-- vortex-layout/src/scan/v2/tests.rs | 23 +- vortex-layout/src/sequence.rs | 35 +- vortex-layout/src/strategy.rs | 19 - vortex-layout/src/test.rs | 45 +- .../slt/duckdb/copy_return_stats.slt | 5 +- 33 files changed, 2249 insertions(+), 395 deletions(-) create mode 100644 vortex-array/benches/aggregate_definedness.rs create mode 100644 vortex-array/src/aggregate_fn/definedness.rs diff --git a/fuzz/fuzz_targets/file_io.rs b/fuzz/fuzz_targets/file_io.rs index 543ea76db0e..050f1ceab5b 100644 --- a/fuzz/fuzz_targets/file_io.rs +++ b/fuzz/fuzz_targets/file_io.rs @@ -38,7 +38,7 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus { } = fuzz; let array_data = array; - if has_nullable_struct(array_data.dtype()) || has_duplicate_field_names(array_data.dtype()) { + if has_duplicate_field_names(array_data.dtype()) { return Corpus::Reject; } @@ -115,18 +115,6 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus { Corpus::Keep }); -fn has_nullable_struct(dtype: &DType) -> bool { - dtype.is_struct() && dtype.is_nullable() - || dtype - .as_struct_fields_opt() - .map(|sdt| sdt.fields().any(|dtype| has_nullable_struct(&dtype))) - .unwrap_or(false) - || dtype - .as_list_element_opt() - .map(|e| has_nullable_struct(e.as_ref())) - .unwrap_or(false) -} - fn has_duplicate_field_names(dtype: &DType) -> bool { if let Some(struct_dtype) = dtype.as_struct_fields_opt() { struct_has_duplicate_names(struct_dtype) diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index 5a29e064667..b2e74f195ef 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -111,6 +111,10 @@ harness = false name = "aggregate_grouped" harness = false +[[bench]] +name = "aggregate_definedness" +harness = false + [[bench]] name = "cast_primitive" harness = false diff --git a/vortex-array/benches/aggregate_definedness.rs b/vortex-array/benches/aggregate_definedness.rs new file mode 100644 index 00000000000..cc1f43b21f3 --- /dev/null +++ b/vortex-array/benches/aggregate_definedness.rs @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Accumulating a nested field with the definedness of its nullable parent, compared with accumulating the +//! logical field, whose validity already excludes the rows the parent nulls out. + +#![allow(clippy::unwrap_used)] + +use std::sync::LazyLock; + +use divan::Bencher; +use mimalloc::MiMalloc; +use rand::prelude::*; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::aggregate_fn::AggregateFnRef; +use vortex_array::aggregate_fn::AggregateFnVTableExt; +use vortex_array::aggregate_fn::EmptyOptions; +use vortex_array::aggregate_fn::NumericalAggregateOpts; +use vortex_array::aggregate_fn::definedness::list_elements_definedness; +use vortex_array::aggregate_fn::fns::null_count::NullCount; +use vortex_array::aggregate_fn::fns::sum::Sum; +use vortex_array::arrays::ListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::dtype::Nullability; +use vortex_array::scalar::Scalar; +use vortex_array::validity::Validity; +use vortex_buffer::Buffer; +use vortex_mask::Mask; +use vortex_session::VortexSession; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +fn main() { + LazyLock::force(&SESSION); + check_results(); + divan::main(); +} + +// Sized to keep the CodSpeed simulation under 1ms per benchmark. +const N: usize = 15_000; + +static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + +/// An aggregate that skips nulls and one that counts them. +const AGGREGATES: &[&str] = &["sum", "null_count"]; + +fn aggregate(name: &str) -> AggregateFnRef { + match name { + "sum" => Sum.bind(NumericalAggregateOpts::default()), + "null_count" => NullCount.bind(EmptyOptions), + _ => unreachable!("unknown aggregate {name}"), + } +} + +fn values() -> Buffer { + let mut rng = StdRng::seed_from_u64(1); + (0..N).map(|_| rng.random_range(-1000..1000)).collect() +} + +/// The field's own validity: a tenth of its values are null. +fn field_validity() -> Mask { + let mut rng = StdRng::seed_from_u64(5); + Mask::from_iter((0..N).map(|_| rng.random_bool(0.9))) +} + +/// A nullable struct field, holding values in every row including those the struct nulls out. +fn field() -> ArrayRef { + PrimitiveArray::new( + values(), + Validity::from_mask(field_validity(), Nullability::Nullable), + ) + .into_array() +} + +/// Nine in ten rows, chosen at random, as the struct validity of a nested field would be. +fn definedness() -> Mask { + let mut rng = StdRng::seed_from_u64(2); + Mask::from_iter((0..N).map(|_| rng.random_bool(0.9))) +} + +/// The logical field: null wherever the struct or the field is null. +fn masked_field() -> ArrayRef { + PrimitiveArray::new( + values(), + Validity::from_mask(&field_validity() & &definedness(), Nullability::Nullable), + ) + .into_array() +} + +fn run(aggregate_fn: &AggregateFnRef, batch: &ArrayRef, defined: Option<&Mask>) -> Scalar { + let mut ctx = SESSION.create_execution_ctx(); + let mut acc = aggregate_fn.accumulator(batch.dtype()).unwrap(); + match defined { + Some(defined) => acc.accumulate_defined(batch, defined, &mut ctx).unwrap(), + None => acc.accumulate(batch, &mut ctx).unwrap(), + } + acc.finish().unwrap() +} + +/// Check that every benchmark computes the right result before timing it. +/// +/// Undefined rows are null, so the defined field gives the same result as the logical field for +/// every aggregate. The null count counts the field's own nulls and the rows the struct nulls out. +fn check_results() { + let (field, defined) = (field(), definedness()); + for name in AGGREGATES { + let aggregate_fn = aggregate(name); + assert_eq!( + run(&aggregate_fn, &field, Some(&defined)), + run(&aggregate_fn, &masked_field(), None), + "{name}" + ); + } + + let expected_nulls = (!&(&field_validity() & &defined)).true_count() as u64; + let null_count = run(&aggregate("null_count"), &field, Some(&defined)); + assert_eq!( + null_count.as_primitive().typed_value::(), + Some(expected_nulls) + ); +} + +/// Accumulate the field with the definedness of its parent. +/// +/// Both this and [`nullable`] build their input outside the timed region, a fresh batch per +/// iteration so that no statistic cached by a previous iteration is reused, and run the same +/// accumulator over the same logical values. The defined side also pays for merging the definedness +/// into the field's validity, which the nullable side receives already merged. +#[divan::bench(args = AGGREGATES)] +fn defined(bencher: Bencher, name: &str) { + let aggregate_fn = aggregate(name); + let defined = definedness(); + bencher + .with_inputs(|| { + let batch = field(); + ( + aggregate_fn.accumulator(batch.dtype()).unwrap(), + batch, + SESSION.create_execution_ctx(), + ) + }) + .bench_values(|(mut acc, batch, mut ctx)| { + acc.accumulate_defined(&batch, &defined, &mut ctx).unwrap(); + acc.flush().unwrap() + }); +} + +/// Accumulate the logical field, whose validity already excludes the undefined rows. +#[divan::bench(args = AGGREGATES)] +fn nullable(bencher: Bencher, name: &str) { + let aggregate_fn = aggregate(name); + bencher + .with_inputs(|| { + let batch = masked_field(); + ( + aggregate_fn.accumulator(batch.dtype()).unwrap(), + batch, + SESSION.create_execution_ctx(), + ) + }) + .bench_values(|(mut acc, batch, mut ctx)| { + acc.accumulate(&batch, &mut ctx).unwrap(); + acc.flush().unwrap() + }); +} + +const VALID_DENSITIES: &[f64] = &[0.5, 0.99]; + +/// A list column of `N` lists of up to eight elements, whose given fraction of lists is valid. +fn lists_with_validity(density: f64) -> ListArray { + let mut rng = StdRng::seed_from_u64(7); + let mut offsets = Vec::with_capacity(N + 1); + offsets.push(0u32); + for _ in 0..N { + let last = *offsets.last().unwrap(); + offsets.push(last + rng.random_range(0..8u32)); + } + let elements: Buffer = (0..*offsets.last().unwrap()).collect(); + let validity = Mask::from_iter((0..N).map(|_| rng.random_bool(density))); + ListArray::try_new( + elements.into_array(), + Buffer::from(offsets).into_array(), + Validity::from_mask(validity, Nullability::Nullable), + &mut SESSION.create_execution_ctx(), + ) + .unwrap() +} + +/// Derive the definedness on the elements of a list column from the lists' validity. +#[divan::bench(args = VALID_DENSITIES)] +fn list_elements_definedness_by_validity(bencher: Bencher, density: f64) { + let lists = lists_with_validity(density); + let defined = Mask::new_true(N); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_values(|mut ctx| list_elements_definedness(&lists, &defined, &mut ctx).unwrap()); +} diff --git a/vortex-array/src/aggregate_fn/accumulator.rs b/vortex-array/src/aggregate_fn/accumulator.rs index 388d92ffd2f..13c596ae8bb 100644 --- a/vortex-array/src/aggregate_fn/accumulator.rs +++ b/vortex-array/src/aggregate_fn/accumulator.rs @@ -8,14 +8,17 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; +use vortex_mask::Mask; use crate::ArrayRef; use crate::Columnar; use crate::ExecutionCtx; +use crate::IntoArray; use crate::aggregate_fn::AggregateDTypes; use crate::aggregate_fn::AggregateFn; use crate::aggregate_fn::AggregateFnRef; use crate::aggregate_fn::AggregateFnVTable; +use crate::builtins::ArrayBuiltins; use crate::columnar::AnyColumnar; use crate::dtype::DType; use crate::executor::max_iterations; @@ -181,6 +184,20 @@ pub trait DynAccumulator: 'static + Send { /// Accumulate a new array into the accumulator's state. fn accumulate(&mut self, batch: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()>; + /// Accumulate the logical column that `defined` describes over `batch`. + /// + /// `defined` must have the length of `batch`. See the [`definedness`](crate::aggregate_fn::definedness) + /// module for what definedness means and the semantics aggregates give it. An aggregate defines + /// its own semantics with [`AggregateFnVTable::try_accumulate_defined`]; by default the + /// undefined rows are null whatever they hold, so the result equals accumulating + /// `batch.mask(defined)`. + fn accumulate_defined( + &mut self, + batch: &ArrayRef, + defined: &Mask, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()>; + /// Drain another accumulator's state into this one, resetting `other`. /// /// The other accumulator must have been constructed for the same aggregate function, @@ -283,6 +300,64 @@ impl DynAccumulator for Accumulator { self.fold_partial(batch_partial) } + fn accumulate_defined( + &mut self, + batch: &ArrayRef, + defined: &Mask, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + vortex_ensure_eq!( + defined.len(), + batch.len(), + "Definedness mask length must match the batch length", + ); + if defined.all_true() { + return self.accumulate(batch, ctx); + } + if self.is_saturated() { + return Ok(()); + } + vortex_ensure_eq!(batch.dtype(), &self.dtypes.dtype, "Input DType mismatch"); + + self.ensure_partial()?; + let partial = self.partial.as_mut().vortex_expect("partial materialized"); + if self.vtable.try_accumulate_defined( + self.dtypes.args(&self.options), + partial, + batch, + defined, + ctx, + )? { + return Ok(()); + } + + // The default semantics: undefined rows read as null. The masked batch is a temporary + // array, so no statistics are read from or cached on it. + let masked = batch.clone().mask(defined.clone().into_array())?; + if masked.dtype() == &self.dtypes.dtype { + return self.accumulate_batch(&masked, ctx); + } + + // Masking widened a non-nullable input to nullable. Accumulate it as the nullable input, + // whose partial state must be the same, and fold the result into this state. + let mut nullable = Accumulator::try_new( + self.vtable.clone(), + self.options.clone(), + masked.dtype().clone(), + )?; + vortex_ensure_eq!( + nullable.dtypes.partial_dtype, + self.dtypes.partial_dtype, + "Aggregate {} has a different partial dtype for a nullable input", + self.aggregate_fn, + ); + nullable.accumulate_batch(&masked, ctx)?; + match nullable.take_partial() { + Some(partial) => self.fold_partial(partial), + None => Ok(()), + } + } + fn merge_from(&mut self, other: &mut dyn DynAccumulator) -> VortexResult<()> { let Some(other) = other.downcast_mut::() else { vortex_bail!( diff --git a/vortex-array/src/aggregate_fn/combined.rs b/vortex-array/src/aggregate_fn/combined.rs index 842603c672e..ac710550609 100644 --- a/vortex-array/src/aggregate_fn/combined.rs +++ b/vortex-array/src/aggregate_fn/combined.rs @@ -13,6 +13,7 @@ use std::hash::Hash; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_mask::Mask; use vortex_session::VortexSession; use crate::ArrayRef; @@ -316,6 +317,21 @@ impl AggregateFnVTable for Combined { Ok(true) } + /// Hand the definedness to each child, which gives it the child's own semantics. The children + /// keep the parent's input dtype, so their states merge with those of undefined batches. + fn try_accumulate_defined( + &self, + _args: AggregateArgs<'_, Self::Options>, + state: &mut Self::Partial, + batch: &ArrayRef, + defined: &Mask, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + state.0.accumulate_defined(batch, defined, ctx)?; + state.1.accumulate_defined(batch, defined, ctx)?; + Ok(true) + } + fn accumulate( &self, _args: AggregateArgs<'_, Self::Options>, diff --git a/vortex-array/src/aggregate_fn/definedness.rs b/vortex-array/src/aggregate_fn/definedness.rs new file mode 100644 index 00000000000..668118c7d75 --- /dev/null +++ b/vortex-array/src/aggregate_fn/definedness.rs @@ -0,0 +1,864 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Definedness: which rows of an array belong to the logical column an aggregate describes. +//! +//! A nested array stores its children unmasked: a struct field holds a value in every row, including +//! the rows where the struct is null. Aggregating such a child directly folds in values the logical +//! column does not contain. +//! +//! *Definedness* is a [`Mask`] with one bit per row of a batch. A defined row belongs to the logical +//! column with the value the batch holds there, which may itself be null. An undefined row is +//! null in the logical column, because an ancestor nulls it out, whatever the batch holds there. +//! A batch is aggregated under a definedness mask with [`DynAccumulator::accumulate_defined`]. +//! +//! # Semantics +//! +//! An aggregate gives definedness meaning through [`AggregateFnVTable::try_accumulate_defined`]: +//! +//! - By default, undefined rows read as null: the result equals aggregating the batch masked by +//! the definedness. This suits every aggregate of the logical values, such as a minimum, a sum or a +//! null count. The masked batch has the input dtype made nullable, so an aggregate that keeps +//! the default must have the same partial dtype for a nullable input as for the non-nullable +//! input. +//! - An aggregate that describes the stored array rather than its logical values defines its own +//! semantics. [`UncompressedSizeInBytes`] measures the bytes the batch holds, which definedness +//! does not change, so it ignores the definedness. +//! +//! # Deriving definedness +//! +//! The helpers here derive the definedness on a child from the definedness on its parent: +//! +//! - a struct field is defined in the rows where the struct is defined and valid, see +//! [`struct_field_definedness`]; +//! - a list element belongs to the logical elements column only if a defined, valid list +//! references it. Elements no such list references are not null but absent, so they cannot be +//! described by definedness: a writer drops them instead, keeping only the elements that +//! [`list_elements_definedness`] selects, so that the remaining elements are all defined; +//! - a fixed-size list's elements follow the same rule: the elements of a null or undefined row are +//! absent, not null. They hold fixed slots, so they cannot be dropped from storage without +//! breaking the row stride; a consumer that aggregates the elements on their own filters them +//! out first, with [`fixed_size_list_logical_elements`], or with the mask that +//! [`fixed_size_list_elements_definedness`] returns. Element aggregates then agree with those of +//! the same data stored as a variable-size list. +//! +//! [`DynAccumulator::accumulate_defined`]: crate::aggregate_fn::DynAccumulator::accumulate_defined +//! [`AggregateFnVTable::try_accumulate_defined`]: crate::aggregate_fn::AggregateFnVTable::try_accumulate_defined +//! [`UncompressedSizeInBytes`]: crate::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeInBytes + +use num_traits::AsPrimitive; +use vortex_buffer::BitBufferMut; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure_eq; +use vortex_mask::AllOr; +use vortex_mask::Mask; + +use crate::ArrayRef; +use crate::ExecutionCtx; +use crate::arrays::FixedSizeListArray; +use crate::arrays::ListArray; +use crate::arrays::PrimitiveArray; +use crate::arrays::StructArray; +use crate::arrays::fixed_size_list::FixedSizeListArrayExt; +use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt; +use crate::arrays::list::ListArraySlotsExt; +use crate::arrays::primitive::PrimitiveArrayExt; +use crate::match_each_unsigned_integer_ptype; +use crate::validity::Validity; + +/// Intersect `defined` with `validity`: the defined rows that hold a value. +pub fn valid_definedness( + validity: &Validity, + defined: &Mask, + ctx: &mut ExecutionCtx, +) -> VortexResult { + if validity.definitely_no_nulls() || defined.all_false() { + return Ok(defined.clone()); + } + let valid = validity.execute_mask(defined.len(), ctx)?; + Ok(&valid & defined) +} + +/// The definedness on each field of `array`: the rows defined by `defined` where the struct is valid. +pub fn struct_field_definedness( + array: &StructArray, + defined: &Mask, + ctx: &mut ExecutionCtx, +) -> VortexResult { + vortex_ensure_eq!( + defined.len(), + array.len(), + "Definedness mask length must match the struct length", + ); + valid_definedness(&array.validity()?, defined, ctx) +} + +/// The elements of `array` that belong to its logical elements column: the elements of each list +/// that `defined` selects and that is valid. +/// +/// Elements outside such lists, including the elements of null lists and any elements before the +/// first or after the last offset, are not part of the elements column. +pub fn list_elements_definedness( + array: &ListArray, + defined: &Mask, + ctx: &mut ExecutionCtx, +) -> VortexResult { + vortex_ensure_eq!( + defined.len(), + array.len(), + "Definedness mask length must match the list length", + ); + let elements_len = array.elements().len(); + let lists = valid_definedness(&array.validity()?, defined, ctx)?; + if lists.all_false() || elements_len == 0 { + return Ok(Mask::new_false(elements_len)); + } + + let offsets = array.offsets().clone().execute::(ctx)?; + // Offsets are non-negative, so they can be read as unsigned. + let offsets = offsets.reinterpret_cast(offsets.ptype().to_unsigned()); + + let mut buffer = BitBufferMut::new_unset(elements_len); + match_each_unsigned_integer_ptype!(offsets.ptype(), |O| { + let offsets = offsets.as_slice::(); + // Offsets are monotonic, so a run of defined lists covers one contiguous element range. + for_each_defined_run(&lists, |start, end| { + fill_elements(&mut buffer, offsets[start].as_(), offsets[end].as_()); + }); + }); + Ok(Mask::from_buffer(buffer.freeze())) +} + +/// The elements of `array` that belong to its logical elements column: the `list_size` elements of +/// each row that `defined` selects and that is valid. +/// +/// The elements of null or undefined rows are not part of the elements column, as for a variable +/// size list, see [`list_elements_definedness`]. +pub fn fixed_size_list_elements_definedness( + array: &FixedSizeListArray, + defined: &Mask, + ctx: &mut ExecutionCtx, +) -> VortexResult { + vortex_ensure_eq!( + defined.len(), + array.len(), + "Definedness mask length must match the list length", + ); + let list_size = array.list_size() as usize; + let elements_len = array.elements().len(); + let rows = valid_definedness(&array.validity()?, defined, ctx)?; + Ok(match rows.slices() { + AllOr::All => Mask::new_true(elements_len), + AllOr::None => Mask::new_false(elements_len), + AllOr::Some(slices) => Mask::from_slices( + elements_len, + slices + .iter() + .map(|&(start, end)| (start * list_size, end * list_size)) + .filter(|(start, end)| start < end) + .collect(), + ), + }) +} + +/// The logical elements column of `array`: its elements without those of null or undefined rows, +/// as [`fixed_size_list_elements_definedness`] selects them. Every row it keeps contributes its +/// `list_size` elements, in row order. +pub fn fixed_size_list_logical_elements( + array: &FixedSizeListArray, + defined: &Mask, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let elements = fixed_size_list_elements_definedness(array, defined, ctx)?; + if elements.all_true() { + return Ok(array.elements().clone()); + } + array.elements().filter(elements) +} + +fn fill_elements(buffer: &mut BitBufferMut, start: usize, end: usize) { + if start < end { + buffer.fill_range(start, end, true); + } +} + +/// Call `run` with the `start..end` of every run of consecutive lists selected by `lists`. +fn for_each_defined_run(lists: &Mask, mut run: impl FnMut(usize, usize)) { + match lists.slices() { + AllOr::All => run(0, lists.len()), + AllOr::None => {} + AllOr::Some(slices) => { + for &(start, end) in slices { + run(start, end); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroUsize; + use std::sync::LazyLock; + + use rand::prelude::*; + use rstest::rstest; + use vortex_buffer::Buffer; + use vortex_buffer::buffer; + use vortex_error::VortexResult; + use vortex_mask::Mask; + use vortex_session::SessionExt; + use vortex_session::VortexSession; + + use super::*; + use crate::ArrayRef; + use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::aggregate_fn::AggregateDTypes; + use crate::aggregate_fn::AggregateFnRef; + use crate::aggregate_fn::AggregateFnVTable; + use crate::aggregate_fn::AggregateFnVTableExt; + use crate::aggregate_fn::EmptyOptions; + use crate::aggregate_fn::NumericalAggregateOpts; + use crate::aggregate_fn::combined::PairOptions; + use crate::aggregate_fn::fns::all_nan::AllNan; + use crate::aggregate_fn::fns::all_non_distinct::AllNonDistinct; + use crate::aggregate_fn::fns::all_non_nan::AllNonNan; + use crate::aggregate_fn::fns::all_non_null::AllNonNull; + use crate::aggregate_fn::fns::all_null::AllNull; + use crate::aggregate_fn::fns::bounded_max::BoundedMax; + use crate::aggregate_fn::fns::bounded_max::BoundedMaxOptions; + use crate::aggregate_fn::fns::bounded_min::BoundedMin; + use crate::aggregate_fn::fns::bounded_min::BoundedMinOptions; + use crate::aggregate_fn::fns::count::Count; + use crate::aggregate_fn::fns::first::First; + use crate::aggregate_fn::fns::is_constant::IsConstant; + use crate::aggregate_fn::fns::is_sorted::IsSorted; + use crate::aggregate_fn::fns::is_sorted::IsSortedOptions; + use crate::aggregate_fn::fns::last::Last; + use crate::aggregate_fn::fns::max::Max; + use crate::aggregate_fn::fns::mean::Mean; + use crate::aggregate_fn::fns::min::Min; + use crate::aggregate_fn::fns::min_max::MinMax; + use crate::aggregate_fn::fns::nan_count::NanCount; + use crate::aggregate_fn::fns::null_count::NullCount; + use crate::aggregate_fn::fns::sum::Sum; + use crate::aggregate_fn::fns::sum_v2::SumV2; + use crate::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeInBytes; + use crate::aggregate_fn::kernels::DynAggregateKernel; + use crate::aggregate_fn::session::AggregateFnSession; + use crate::array::VTable; + use crate::arrays::BoolArray; + use crate::arrays::ConstantArray; + use crate::arrays::DecimalArray; + use crate::arrays::Dict; + use crate::arrays::DictArray; + use crate::arrays::FixedSizeListArray; + use crate::arrays::VarBinViewArray; + use crate::assert_arrays_eq; + use crate::builtins::ArrayBuiltins; + use crate::dtype::DecimalDType; + use crate::dtype::Nullability; + use crate::scalar::Scalar; + + static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + + fn ctx() -> ExecutionCtx { + SESSION.create_execution_ctx() + } + + /// Aggregates that keep the default definedness semantics. `UncompressedSizeInBytes` defines its own, + /// see `uncompressed_size_ignores_definedness`. + fn numeric_aggregates() -> Vec { + let opts = NumericalAggregateOpts::default(); + vec![ + Sum.bind(opts), + SumV2.bind(opts), + Min.bind(opts), + Max.bind(opts), + MinMax.bind(opts), + Count.bind(opts), + Count.bind(NumericalAggregateOpts::include_nans()), + Mean::combined().bind(PairOptions(opts, opts)), + NullCount.bind(EmptyOptions), + AllNull.bind(EmptyOptions), + AllNonNull.bind(EmptyOptions), + First.bind(EmptyOptions), + Last.bind(EmptyOptions), + IsConstant.bind(EmptyOptions), + IsSorted.bind(IsSortedOptions { strict: false }), + IsSorted.bind(IsSortedOptions { strict: true }), + ] + } + + fn float_aggregates() -> Vec { + let mut aggregates = numeric_aggregates(); + aggregates.extend([ + NanCount.bind(EmptyOptions), + AllNan.bind(EmptyOptions), + AllNonNan.bind(EmptyOptions), + Sum.bind(NumericalAggregateOpts::include_nans()), + Min.bind(NumericalAggregateOpts::include_nans()), + ]); + aggregates + } + + fn string_aggregates() -> Vec { + let max_bytes = NonZeroUsize::new(2).expect("non zero"); + vec![ + Min.bind(NumericalAggregateOpts::default()), + Max.bind(NumericalAggregateOpts::default()), + BoundedMin.bind(BoundedMinOptions { max_bytes }), + BoundedMax.bind(BoundedMaxOptions { max_bytes }), + NullCount.bind(EmptyOptions), + First.bind(EmptyOptions), + IsConstant.bind(EmptyOptions), + ] + } + + /// Accumulating a defined batch gives the same partial as accumulating the batch masked to null + /// outside the definedness. + fn assert_definedness_matches_mask( + aggregates: &[AggregateFnRef], + batch: &ArrayRef, + defined: &Mask, + ) -> VortexResult<()> { + let mut ctx = ctx(); + for aggregate_fn in aggregates { + let mut defined_acc = aggregate_fn.accumulator(batch.dtype())?; + defined_acc.accumulate_defined(batch, defined, &mut ctx)?; + + let masked = batch.clone().mask(defined.clone().into_array())?; + let mut logical = aggregate_fn.accumulator(masked.dtype())?; + logical.accumulate(&masked, &mut ctx)?; + + assert_eq!( + defined_acc.flush()?, + logical.flush()?, + "{aggregate_fn} over {} defined by {defined:?}", + batch.display_values(&SESSION.create_execution_ctx()), + ); + } + Ok(()) + } + + fn defined_masks(len: usize) -> Vec { + vec![ + Mask::new_true(len), + Mask::new_false(len), + Mask::from_iter((0..len).map(|i| i % 2 == 0)), + Mask::from_iter((0..len).map(|i| i % 3 != 1)), + Mask::from_indices(len, [len - 1]), + ] + } + + #[rstest] + #[case::non_nullable(buffer![5i32, -9, 3, 3, 7, 1].into_array())] + #[case::nullable( + PrimitiveArray::from_option_iter([Some(5i64), None, Some(3), Some(3), None, Some(1)]) + .into_array() + )] + #[case::sorted_where_defined(buffer![1u8, 9, 2, 0, 3, 0].into_array())] + #[case::constant(ConstantArray::new(4i32, 6).into_array())] + #[case::dict( + DictArray::new(buffer![0u32, 1, 0, 2, 2, 1].into_array(), buffer![7i32, -2, 4].into_array()) + .into_array() + )] + #[case::bool(BoolArray::from_iter([true, false, true, true, false, false]).into_array())] + fn definedness_matches_mask(#[case] batch: ArrayRef) -> VortexResult<()> { + let aggregates: Vec<_> = numeric_aggregates() + .into_iter() + .filter(|aggregate_fn| aggregate_fn.return_dtype(batch.dtype()).is_some()) + .collect(); + for defined in defined_masks(batch.len()) { + assert_definedness_matches_mask(&aggregates, &batch, &defined)?; + } + Ok(()) + } + + #[test] + fn definedness_matches_mask_over_floats() -> VortexResult<()> { + let batch = PrimitiveArray::from_option_iter([ + Some(f64::NAN), + Some(2.5), + None, + Some(f64::NAN), + Some(-1.0), + None, + ]) + .into_array(); + for defined in defined_masks(batch.len()) { + assert_definedness_matches_mask(&float_aggregates(), &batch, &defined)?; + } + Ok(()) + } + + #[test] + fn definedness_matches_mask_over_strings() -> VortexResult<()> { + let batch = VarBinViewArray::from_iter_nullable_str([ + Some("zebra"), + Some("apple"), + None, + Some("mango"), + Some("apple"), + ]) + .into_array(); + for defined in defined_masks(batch.len()) { + assert_definedness_matches_mask(&string_aggregates(), &batch, &defined)?; + } + Ok(()) + } + + /// Batches with and without partial definedness accumulate into one state, whose result is that + /// of the batches masked by their definedness masks. + #[rstest] + #[case::non_nullable(buffer![5i32, -9, 3, 3, 7, 1].into_array())] + #[case::nullable( + PrimitiveArray::from_option_iter([Some(5i64), None, Some(3), Some(3), None, Some(1)]) + .into_array() + )] + fn defined_and_undefined_batches_accumulate_together( + #[case] batch: ArrayRef, + ) -> VortexResult<()> { + let mut ctx = ctx(); + let len = batch.len(); + let partial = Mask::from_iter((0..len).map(|i| i % 3 != 1)); + let defined_masks = [ + Mask::new_true(len), + partial.clone(), + Mask::new_true(len), + partial, + ]; + for aggregate_fn in numeric_aggregates() + .into_iter() + .filter(|aggregate_fn| aggregate_fn.return_dtype(batch.dtype()).is_some()) + { + let mut defined_acc = aggregate_fn.accumulator(batch.dtype())?; + let mut logical = aggregate_fn.accumulator(&batch.dtype().as_nullable())?; + for defined in &defined_masks { + defined_acc.accumulate_defined(&batch, defined, &mut ctx)?; + logical.accumulate(&batch.clone().mask(defined.clone().into_array())?, &mut ctx)?; + } + assert_eq!(defined_acc.finish()?, logical.finish()?, "{aggregate_fn}"); + } + Ok(()) + } + + /// A non-nullable batch of each kind of dtype. + fn non_nullable_batches(ctx: &mut ExecutionCtx) -> VortexResult> { + let i32s = buffer![5i32, -9, 3, 3, 7, 1].into_array(); + Ok(vec![ + i32s.clone(), + buffer![2.5f64, -1.0, 0.0, 9.0, 1.5, 3.0].into_array(), + BoolArray::from_iter([true, false, true, true, false, false]).into_array(), + VarBinViewArray::from_iter_str(["a", "bb", "a", "ccc", "d", "bb"]).into_array(), + DecimalArray::new( + buffer![100i64, 250, -75, 0, 13, 999], + DecimalDType::new(10, 2), + Validity::NonNullable, + ) + .into_array(), + StructArray::try_from_iter([("a", i32s.clone())])?.into_array(), + ListArray::try_new( + i32s.clone(), + buffer![0u32, 1, 1, 3, 4, 5, 6].into_array(), + Validity::NonNullable, + ctx, + )? + .into_array(), + FixedSizeListArray::try_new(i32s, 1, Validity::NonNullable, 6)?.into_array(), + ]) + } + + /// An aggregate that keeps the default definedness semantics accumulates a batch masked to the + /// nullable input dtype into its partial state, so its partial dtype must not depend on the + /// nullability of its input. + fn assert_default_definedness_keeps_partial_dtype( + vtable: V, + options: V::Options, + ) -> VortexResult<()> { + let mut ctx = ctx(); + for batch in non_nullable_batches(&mut ctx)? { + let Ok(dtypes) = AggregateDTypes::try_new(&vtable, &options, batch.dtype().clone()) + else { + continue; + }; + let args = dtypes.args(&options); + let mut partial = vtable.empty_partial(args)?; + let defined = Mask::from_iter((0..batch.len()).map(|i| i % 2 == 0)); + if vtable.try_accumulate_defined(args, &mut partial, &batch, &defined, &mut ctx)? { + continue; + } + assert_eq!( + vtable.partial_dtype(&options, &batch.dtype().as_nullable()), + Some(dtypes.partial_dtype.clone()), + "{} over {}", + vtable.id(), + batch.dtype(), + ); + } + Ok(()) + } + + #[test] + fn default_definedness_keeps_partial_dtype_of_every_aggregate() -> VortexResult<()> { + let opts = NumericalAggregateOpts::default(); + let nans = NumericalAggregateOpts::include_nans(); + let max_bytes = NonZeroUsize::new(2).expect("non zero"); + assert_default_definedness_keeps_partial_dtype(AllNan, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype(AllNonDistinct, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype(AllNonNan, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype(AllNonNull, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype(AllNull, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype( + BoundedMax, + BoundedMaxOptions { max_bytes }, + )?; + assert_default_definedness_keeps_partial_dtype( + BoundedMin, + BoundedMinOptions { max_bytes }, + )?; + assert_default_definedness_keeps_partial_dtype(Count, opts)?; + assert_default_definedness_keeps_partial_dtype(Count, nans)?; + assert_default_definedness_keeps_partial_dtype(First, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype(IsConstant, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype( + IsSorted, + IsSortedOptions { strict: false }, + )?; + assert_default_definedness_keeps_partial_dtype(IsSorted, IsSortedOptions { strict: true })?; + assert_default_definedness_keeps_partial_dtype(Last, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype(Max, opts)?; + assert_default_definedness_keeps_partial_dtype(Mean::combined(), PairOptions(opts, opts))?; + assert_default_definedness_keeps_partial_dtype(Min, opts)?; + assert_default_definedness_keeps_partial_dtype(MinMax, opts)?; + assert_default_definedness_keeps_partial_dtype(MinMax, nans)?; + assert_default_definedness_keeps_partial_dtype(NanCount, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype(NullCount, EmptyOptions)?; + assert_default_definedness_keeps_partial_dtype(Sum, opts)?; + assert_default_definedness_keeps_partial_dtype(Sum, nans)?; + assert_default_definedness_keeps_partial_dtype(SumV2, opts)?; + assert_default_definedness_keeps_partial_dtype(UncompressedSizeInBytes, EmptyOptions)?; + Ok(()) + } + + /// The uncompressed size measures the stored batch, which definedness does not change. + #[test] + fn uncompressed_size_ignores_definedness() -> VortexResult<()> { + let mut ctx = ctx(); + let size = UncompressedSizeInBytes.bind(EmptyOptions); + for batch in non_nullable_batches(&mut ctx)? { + let mut defined_acc = size.accumulator(batch.dtype())?; + defined_acc.accumulate_defined( + &batch, + &Mask::from_iter((0..batch.len()).map(|i| i % 2 == 0)), + &mut ctx, + )?; + let mut stored = size.accumulator(batch.dtype())?; + stored.accumulate(&batch, &mut ctx)?; + assert_eq!(defined_acc.finish()?, stored.finish()?, "{}", batch.dtype()); + } + Ok(()) + } + + #[test] + fn definedness_length_must_match_batch() -> VortexResult<()> { + let batch = buffer![1i32, 2, 3].into_array(); + let mut acc = Sum + .bind(NumericalAggregateOpts::default()) + .accumulator(batch.dtype())?; + assert!( + acc.accumulate_defined(&batch, &Mask::new_true(2), &mut ctx()) + .is_err() + ); + Ok(()) + } + + /// Fully defined batches take the plain path, which caches the result on the batch; partially + /// defined batches never cache a statistic on the batch. + #[test] + fn only_fully_defined_batches_cache_statistics() -> VortexResult<()> { + use crate::expr::stats::Precision; + use crate::expr::stats::Stat; + use crate::expr::stats::StatsProviderExt; + + let mut ctx = ctx(); + let min = Min.bind(NumericalAggregateOpts::default()); + + let partially = buffer![5i32, 1, 3].into_array(); + let mut acc = min.accumulator(partially.dtype())?; + acc.accumulate_defined(&partially, &Mask::from_iter([true, false, true]), &mut ctx)?; + assert_eq!( + acc.finish()?, + Scalar::primitive(3i32, Nullability::Nullable) + ); + assert_eq!( + partially.statistics().get_as::(Stat::Min), + Precision::Absent + ); + + let fully = buffer![5i32, 1, 3].into_array(); + acc.accumulate_defined(&fully, &Mask::new_true(3), &mut ctx)?; + assert_eq!( + acc.finish()?, + Scalar::primitive(1i32, Nullability::Nullable) + ); + assert_eq!( + fully.statistics().get_as::(Stat::Min), + Precision::exact(1i32) + ); + Ok(()) + } + + /// A sentinel sum, distinguishable from any sum the test batches produce. + #[derive(Debug)] + struct SentinelSumKernel; + impl DynAggregateKernel for SentinelSumKernel { + fn aggregate( + &self, + _aggregate_fn: &AggregateFnRef, + _batch: &ArrayRef, + _ctx: &mut ExecutionCtx, + ) -> VortexResult> { + Ok(Some(Scalar::primitive(42i64, Nullability::Nullable))) + } + } + + /// The defined rows are filtered lazily, and a dictionary filters only its codes, so the + /// dictionary's kernel still fires. + #[test] + fn definedness_keeps_encoding_kernels() -> VortexResult<()> { + static KERNEL: SentinelSumKernel = SentinelSumKernel; + let session = VortexSession::empty(); + session + .get::() + .register_aggregate_kernel(Dict.id(), Some(Sum.id()), &KERNEL); + let mut ctx = session.create_execution_ctx(); + + let batch = DictArray::try_new( + buffer![0u32, 1, 0].into_array(), + buffer![7i32, 9].into_array(), + )? + .into_array(); + let mut acc = Sum + .bind(NumericalAggregateOpts::default()) + .accumulator(batch.dtype())?; + acc.accumulate_defined(&batch, &Mask::from_iter([true, false, true]), &mut ctx)?; + + assert_eq!( + acc.finish()?, + Scalar::primitive(42i64, Nullability::Nullable) + ); + Ok(()) + } + + /// Aggregating a struct field under the field definedness matches aggregating the logical field, + /// whose values are null wherever the struct is null. + #[test] + fn struct_field_aggregates_ignore_values_under_null_parents() -> VortexResult<()> { + let mut ctx = ctx(); + // The field holds 100 and -100 under the struct's null rows. + let field = + PrimitiveArray::from_option_iter([Some(1i32), Some(100), None, Some(3), Some(-100)]) + .into_array(); + let array = StructArray::try_from_iter_with_validity( + [("a", field.clone())], + Validity::from_iter([true, false, true, true, false]), + )?; + let logical_field = array.clone().into_array().get_item("a")?; + + let field_definedness = + struct_field_definedness(&array, &Mask::new_true(array.len()), &mut ctx)?; + let aggregates = [ + Min.bind(NumericalAggregateOpts::default()), + Max.bind(NumericalAggregateOpts::default()), + Sum.bind(NumericalAggregateOpts::default()), + NullCount.bind(EmptyOptions), + IsConstant.bind(EmptyOptions), + ]; + for aggregate_fn in &aggregates { + let mut defined_acc = aggregate_fn.accumulator(field.dtype())?; + defined_acc.accumulate_defined(&field, &field_definedness, &mut ctx)?; + let mut logical = aggregate_fn.accumulator(logical_field.dtype())?; + logical.accumulate(&logical_field, &mut ctx)?; + assert_eq!(defined_acc.finish()?, logical.finish()?, "{aggregate_fn}"); + } + + let mut min = aggregates[0].accumulator(field.dtype())?; + min.accumulate_defined(&field, &field_definedness, &mut ctx)?; + assert_eq!( + min.finish()?, + Scalar::primitive(1i32, Nullability::Nullable) + ); + // The field's own null, and the two rows the struct nulls out. + let mut null_count = aggregates[3].accumulator(field.dtype())?; + null_count.accumulate_defined(&field, &field_definedness, &mut ctx)?; + assert_eq!( + null_count.finish()?, + Scalar::primitive(3u64, Nullability::NonNullable) + ); + Ok(()) + } + + #[test] + fn non_nullable_parents_define_every_row() -> VortexResult<()> { + let mut ctx = ctx(); + let array = StructArray::try_from_iter([("a", buffer![1i32, 2].into_array())])?; + assert!(struct_field_definedness(&array, &Mask::new_true(2), &mut ctx)?.all_true()); + Ok(()) + } + + #[test] + fn struct_field_definedness_intersects_validity() -> VortexResult<()> { + let array = StructArray::try_from_iter_with_validity( + [("a", buffer![1i32, 2, 3, 4].into_array())], + Validity::from_iter([true, false, true, true]), + )?; + let defined = Mask::from_iter([true, true, false, true]); + + let field_definedness = struct_field_definedness(&array, &defined, &mut ctx())?; + + assert_eq!( + field_definedness, + Mask::from_iter([true, false, false, true]) + ); + Ok(()) + } + + /// A fixed-size list's elements under null and undefined rows are absent, so its element + /// aggregates match those of the same rows stored as a variable-size list. + #[test] + fn fixed_size_list_elements_match_list_semantics() -> VortexResult<()> { + let mut ctx = ctx(); + // Rows [1, null], [7, 8] (null), [null, 3] (undefined) and [4, 5]. + let elements = PrimitiveArray::from_option_iter([ + Some(1i32), + None, + Some(7), + Some(8), + None, + Some(3), + Some(4), + Some(5), + ]) + .into_array(); + let validity = Validity::from_iter([true, false, true, true]); + let defined = Mask::from_iter([true, true, false, true]); + let fixed = FixedSizeListArray::try_new(elements.clone(), 2, validity.clone(), 4)?; + let list = ListArray::try_new( + elements, + buffer![0u32, 2, 4, 6, 8].into_array(), + validity, + &mut ctx, + )?; + + assert_eq!( + fixed_size_list_elements_definedness(&fixed, &defined, &mut ctx)?, + list_elements_definedness(&list, &defined, &mut ctx)? + ); + let logical = fixed_size_list_logical_elements(&fixed, &defined, &mut ctx)?; + assert_arrays_eq!( + logical, + PrimitiveArray::from_option_iter([Some(1i32), None, Some(4), Some(5)]), + &mut ctx + ); + + // One null from the first row; the null under the undefined row is absent. + let mut null_count = NullCount.bind(EmptyOptions).accumulator(logical.dtype())?; + null_count.accumulate(&logical, &mut ctx)?; + assert_eq!( + null_count.finish()?, + Scalar::primitive(1u64, Nullability::NonNullable) + ); + Ok(()) + } + + #[test] + fn fixed_size_list_logical_elements_of_defined_valid_rows_are_all_elements() -> VortexResult<()> + { + let mut ctx = ctx(); + let fixed = FixedSizeListArray::try_new( + buffer![1i32, 2, 3, 4].into_array(), + 2, + Validity::NonNullable, + 2, + )?; + let logical = fixed_size_list_logical_elements(&fixed, &Mask::new_true(2), &mut ctx)?; + assert_arrays_eq!(logical, buffer![1i32, 2, 3, 4].into_array(), &mut ctx); + Ok(()) + } + + #[test] + fn list_elements_definedness_skips_null_lists_and_unreferenced_elements() -> VortexResult<()> { + // Lists [1, 2], [3] (null) and [4, 5] over elements that start with an unreferenced 0. + let array = ListArray::try_new( + buffer![0i32, 1, 2, 3, 4, 5, 6].into_array(), + buffer![1u16, 3, 4, 6].into_array(), + Validity::from_iter([true, false, true]), + &mut SESSION.create_execution_ctx(), + )?; + + let elements_definedness = + list_elements_definedness(&array, &Mask::new_true(3), &mut ctx())?; + + assert_eq!( + elements_definedness, + Mask::from_iter([false, true, true, false, true, true, false]) + ); + Ok(()) + } + + /// The elements of the defined, valid lists, computed one element at a time. + fn reference_elements_definedness( + ranges: &[(u32, u32)], + lists: &Mask, + elements_end: u32, + ) -> Mask { + Mask::from_iter((0..elements_end).map(|element| { + ranges + .iter() + .enumerate() + .any(|(row, &(start, end))| lists.value(row) && start <= element && element < end) + })) + } + + /// Run-based list element definedness agrees with the element-by-element reference, across empty + /// and null lists and partial definedness masks. + #[rstest] + #[case(1)] + #[case(2)] + #[case(3)] + #[case(4)] + fn elements_definedness_matches_reference(#[case] seed: u64) -> VortexResult<()> { + let mut ctx = ctx(); + let mut rng = StdRng::seed_from_u64(seed); + let len = 64; + let mut offsets = vec![0u32]; + for _ in 0..len { + let last = *offsets.last().expect("non-empty"); + offsets.push(last + rng.random_range(0..4u32)); + } + let elements_end = *offsets.last().expect("non-empty"); + let ranges: Vec<(u32, u32)> = offsets.windows(2).map(|w| (w[0], w[1])).collect(); + let valid = Mask::from_iter((0..len).map(|_| rng.random_bool(0.7))); + let defined = Mask::from_iter((0..len).map(|_| rng.random_bool(0.8))); + let expected = reference_elements_definedness(&ranges, &(&valid & &defined), elements_end); + let elements: ArrayRef = (0..elements_end).collect::>().into_array(); + let validity = Validity::from_mask(valid, Nullability::Nullable); + + let list = ListArray::try_new( + elements, + offsets + .iter() + .copied() + .collect::>() + .into_array(), + validity, + &mut ctx, + )?; + assert_eq!( + list_elements_definedness(&list, &defined, &mut ctx)?, + expected + ); + Ok(()) + } +} diff --git a/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs b/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs index 7c5ac4d44c1..1d18e5677d3 100644 --- a/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs @@ -183,6 +183,23 @@ impl AggregateFnVTable for UncompressedSizeInBytes { false } + /// The size of the stored batch, which holds a value in every row, defined or not: a + /// definedness mask changes which rows are null in the logical column, not the bytes the batch holds. + fn try_accumulate_defined( + &self, + _args: AggregateArgs<'_, Self::Options>, + partial: &mut Self::Partial, + batch: &ArrayRef, + _defined: &Mask, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let size = uncompressed_size_in_bytes_u64(batch, ctx)?; + *partial = partial + .checked_add(size) + .ok_or_else(|| vortex_err!("uncompressed size in bytes overflowed u64"))?; + Ok(true) + } + fn accumulate( &self, _args: AggregateArgs<'_, Self::Options>, diff --git a/vortex-array/src/aggregate_fn/mod.rs b/vortex-array/src/aggregate_fn/mod.rs index a6610b010da..00b4e9685cc 100644 --- a/vortex-array/src/aggregate_fn/mod.rs +++ b/vortex-array/src/aggregate_fn/mod.rs @@ -39,6 +39,7 @@ mod options; pub use options::*; pub mod combined; +pub mod definedness; pub mod fns; pub mod kernels; pub mod proto; diff --git a/vortex-array/src/aggregate_fn/vtable.rs b/vortex-array/src/aggregate_fn/vtable.rs index ba236ab1d5f..bc51d208de6 100644 --- a/vortex-array/src/aggregate_fn/vtable.rs +++ b/vortex-array/src/aggregate_fn/vtable.rs @@ -11,6 +11,7 @@ use prost::Message; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_mask::Mask; use vortex_session::VortexSession; use crate::ArrayRef; @@ -250,6 +251,32 @@ pub trait AggregateFnVTable: 'static + Sized + Clone + Send + Sync { Ok(false) } + /// Try to accumulate `batch` under a [definedness](crate::aggregate_fn::definedness) mask, + /// handling the rows the mask leaves undefined itself. + /// + /// `defined` has the length of `batch` and leaves at least one row undefined. Returns `true` if the + /// batch was handled, or `false` to accumulate it under the default definedness semantics: the + /// undefined rows read as null, as if `batch` were masked by `defined`. When returning + /// `false`, the partial state must be unchanged. + /// + /// The default semantics accumulate a masked batch, whose dtype is the input dtype made + /// nullable, into the same partial state. An aggregate that keeps them must therefore have the + /// same [`partial_dtype`] for a nullable input as for the non-nullable input. An aggregate + /// whose result is not a function of the logical values, such as the size of the stored + /// array, overrides this to define its own semantics. + /// + /// [`partial_dtype`]: AggregateFnVTable::partial_dtype + fn try_accumulate_defined( + &self, + _args: AggregateArgs<'_, Self::Options>, + _state: &mut Self::Partial, + _batch: &ArrayRef, + _defined: &Mask, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(false) + } + /// Accumulate a new canonical array into the partial state. fn accumulate( &self, diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index 919ea2393b5..bf6a856e43b 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -437,7 +437,7 @@ impl LayoutStrategy for CudaFlatLayoutStrategy { let Some(chunk) = stream.next().await else { vortex_bail!("CudaFlatLayoutStrategy needs a single chunk"); }; - let (sequence_id, chunk) = chunk?; + let (sequence_id, chunk, _) = chunk?; let row_count = chunk.len() as u64; match chunk.dtype() { diff --git a/vortex-duckdb/src/copy.rs b/vortex-duckdb/src/copy.rs index 5209f5e37bf..d8b2b5d4e91 100644 --- a/vortex-duckdb/src/copy.rs +++ b/vortex-duckdb/src/copy.rs @@ -473,8 +473,10 @@ mod tests { assert_eq!(leaf_stats_columns(file_stats).len(), 2); let nested = column_stats(&summary, 0)?; assert_eq!(nested.column_key, "\"s\".\"b\""); - assert!(nested.min.is_none()); - assert!(nested.null_count.is_none()); + // `s` is nullable, and its field's statistics describe `s.b` as null wherever `s` is. + assert!(nested.min.is_some()); + assert!(nested.max.is_some()); + assert_eq!(nested.null_count, Some(0)); assert!(nested.column_size_bytes.is_some_and(|size| size > 0)); let top_level = column_stats(&summary, 1)?; diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index 1dcf89a916e..58baffe14b6 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -1509,9 +1509,11 @@ async fn file_take() -> VortexResult<()> { Ok(()) } +/// The fields of a nullable top-level struct are null wherever the struct is null, so `age`'s 99 +/// under the struct's last null row is left out of its statistics and counted as null. #[tokio::test] async fn write_nullable_top_level_struct() -> VortexResult<()> { - let ages = PrimitiveArray::from_option_iter([Some(25), Some(31), None, Some(57), None]); + let ages = PrimitiveArray::from_option_iter([Some(25), Some(31), None, Some(57), Some(99)]); let row_validity = BoolArray::from_iter([true, true, false, true, false]).into_array(); let array = StructArray::try_new( @@ -1542,18 +1544,34 @@ async fn write_nullable_top_level_struct() -> VortexResult<()> { Precision::exact(ScalarValue::from(2u64)) ); let (age, _) = stats.get_by_path(&field_path!(age)).expect("age entry"); - assert!(age.is_empty()); + assert_logical_age_stats(age); let file = SESSION.open_options().open_buffer(ByteBuffer::from(buf))?; let stats = file .file_stats() .expect("file statistics should be present"); let (age, _) = stats.get_by_path(&field_path!(age)).expect("age entry"); - assert!(age.is_empty()); + assert_logical_age_stats(age); Ok(()) } +/// `age` is `[25, 31, null, 57, null]` once the struct's null rows are null. +fn assert_logical_age_stats(age: &AggregateStats) { + assert_eq!( + stat_value(age, Stat::NullCount), + Precision::exact(ScalarValue::from(2u64)) + ); + assert_eq!( + stat_value(age, Stat::Min), + Precision::exact(ScalarValue::from(25i32)) + ); + assert_eq!( + stat_value(age, Stat::Max), + Precision::exact(ScalarValue::from(57i32)) + ); +} + #[tokio::test] async fn exclude_legacy_statistics_empties_legacy_but_keeps_nested() -> VortexResult<()> { let inner = StructArray::try_new( @@ -2330,6 +2348,47 @@ async fn nested_empty_list_roundtrip(#[case] validity: Validity) -> VortexResult Ok(()) } +/// The list writer drops the elements of null lists, of lists under null struct rows, and +/// elements no list references, and writes those lists as empty. The column reads back +/// unchanged. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn lists_with_absent_elements_roundtrip() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + // Lists [1, 2], [null, 3] (null), [4], [5, 6] over elements that start and end with an + // unreferenced element. + let elements = PrimitiveArray::from_option_iter([ + Some(0i32), + Some(1), + Some(2), + None, + Some(3), + Some(4), + Some(5), + Some(6), + Some(7), + ]) + .into_array(); + let lists = ListArray::try_new( + elements, + buffer![1u32, 3, 5, 6, 8].into_array(), + Validity::from_iter([true, false, true, true]), + &mut ctx, + )? + .into_array(); + let array = StructArray::try_new( + ["lists"].into(), + vec![lists], + 4, + Validity::from_iter([true, true, false, true]), + )? + .into_array(); + + let result = write_read_roundtrip(array.clone()).await?; + assert_arrays_eq!(result, array, &mut SESSION.create_execution_ctx()); + Ok(()) +} + type MapEntryFixture<'a> = (i32, Option<&'a str>); type MapRowFixture<'a> = Option>>; diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 659fe70b656..d737a363aee 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -55,6 +55,7 @@ use vortex_layout::layouts::file_stats::accumulate_stats; use vortex_layout::sequence::SequenceId; use vortex_layout::sequence::SequentialStreamAdapter; use vortex_layout::sequence::SequentialStreamExt; +use vortex_layout::sequence::fully_defined; use vortex_session::SessionExt; use vortex_session::VortexSession; use vortex_session::registry::Id; @@ -310,7 +311,7 @@ impl VortexWriteOptions { dtype.clone(), stream .try_filter(|chunk| ready(!chunk.is_empty())) - .map(move |result| result.map(|chunk| (ptr.advance(), chunk))), + .map(move |result| result.map(|chunk| fully_defined(ptr.advance(), chunk))), ) .sendable(); let omit_statistics = matches!(&self.file_statistics, Some(stats) if stats.is_empty()) diff --git a/vortex-layout/src/layouts/buffered.rs b/vortex-layout/src/layouts/buffered.rs index b3a3e4652d9..40c8b19960e 100644 --- a/vortex-layout/src/layouts/buffered.rs +++ b/vortex-layout/src/layouts/buffered.rs @@ -57,17 +57,17 @@ impl LayoutStrategy for BufferedStrategy { let mut chunks = VecDeque::new(); while let Some(chunk) = stream.as_mut().next().await { - let (sequence_id, chunk) = chunk?; + let (sequence_id, chunk, defined) = chunk?; let chunk_size = chunk.nbytes(); nbytes += chunk_size; - chunks.push_back((chunk, buffered_bytes.reserve(chunk_size))); + chunks.push_back((chunk, defined, buffered_bytes.reserve(chunk_size))); // If this is the last element, flush everything. if stream.as_mut().peek().await.is_none() { let mut sequence_ptr = sequence_id.descend(); - while let Some((chunk, reservation)) = chunks.pop_front() { + while let Some((chunk, defined, reservation)) = chunks.pop_front() { drop(reservation); - yield (sequence_ptr.advance(), chunk) + yield (sequence_ptr.advance(), chunk, defined) } break; } @@ -80,12 +80,12 @@ impl LayoutStrategy for BufferedStrategy { // This avoids small tail stragglers being flushed at the end of the file. let mut sequence_ptr = sequence_id.descend(); while nbytes > buffer_size { - let Some((chunk, reservation)) = chunks.pop_front() else { + let Some((chunk, defined, reservation)) = chunks.pop_front() else { break; }; nbytes -= reservation.bytes(); drop(reservation); - yield (sequence_ptr.advance(), chunk) + yield (sequence_ptr.advance(), chunk, defined) } } }; diff --git a/vortex-layout/src/layouts/chunked/reader.rs b/vortex-layout/src/layouts/chunked/reader.rs index c658a83bb46..4f97df4bf47 100644 --- a/vortex-layout/src/layouts/chunked/reader.rs +++ b/vortex-layout/src/layouts/chunked/reader.rs @@ -474,6 +474,7 @@ mod test { use crate::sequence::SequenceId; use crate::sequence::SequentialStreamAdapter; use crate::sequence::SequentialStreamExt as _; + use crate::sequence::fully_defined; use crate::test::SESSION; use crate::test::new_session; @@ -495,9 +496,18 @@ mod test { SequentialStreamAdapter::new( DType::Primitive(PType::I32, NonNullable), stream::iter([ - Ok((sequence_id.advance(), buffer![1, 2, 3].into_array())), - Ok((sequence_id.advance(), buffer![4, 5, 6].into_array())), - Ok((sequence_id.advance(), buffer![7, 8, 9].into_array())), + Ok(fully_defined( + sequence_id.advance(), + buffer![1, 2, 3].into_array(), + )), + Ok(fully_defined( + sequence_id.advance(), + buffer![4, 5, 6].into_array(), + )), + Ok(fully_defined( + sequence_id.advance(), + buffer![7, 8, 9].into_array(), + )), ]), ) .sendable(), diff --git a/vortex-layout/src/layouts/collect.rs b/vortex-layout/src/layouts/collect.rs index 1fad03ef0a7..ec10183b77f 100644 --- a/vortex-layout/src/layouts/collect.rs +++ b/vortex-layout/src/layouts/collect.rs @@ -10,6 +10,7 @@ use futures::pin_mut; use vortex_array::IntoArray; use vortex_array::arrays::ChunkedArray; use vortex_error::VortexResult; +use vortex_mask::Mask; use vortex_session::VortexSession; use crate::LayoutRef; @@ -53,11 +54,13 @@ impl LayoutStrategy for CollectStrategy { pin_mut!(stream); let mut chunks = Vec::new(); + let mut defined_masks = Vec::new(); let mut latest_sequence_id = None; while let Some(chunk) = stream.next().await { - let (sequence_id, chunk) = chunk?; + let (sequence_id, chunk, defined) = chunk?; latest_sequence_id = Some(sequence_id); chunks.push(chunk); + defined_masks.push(defined); } // an empty input yields no chunk; the child layout handles it. @@ -66,7 +69,7 @@ impl LayoutStrategy for CollectStrategy { }; let collected = ChunkedArray::try_new(chunks, _dtype)?.into_array(); - yield (sequence_id, collected); + yield (sequence_id, collected, Mask::concat(defined_masks.iter())?); }; let adapted = Box::pin(SequentialStreamAdapter::new(dtype, collected_stream)); diff --git a/vortex-layout/src/layouts/compressed.rs b/vortex-layout/src/layouts/compressed.rs index 07d83bf41c7..962710e83de 100644 --- a/vortex-layout/src/layouts/compressed.rs +++ b/vortex-layout/src/layouts/compressed.rs @@ -110,7 +110,7 @@ impl LayoutStrategy for CompressingStrategy { let stats = Arc::clone(&stats); let session = compute_session.clone(); handle.spawn_cpu(move || { - let (sequence_id, chunk) = chunk?; + let (sequence_id, chunk, defined) = chunk?; let mut ctx = session.create_execution_ctx(); // Compute the stats for the chunk prior to compression. The exact // uncompressed size is measured once here and cached for the compressor. @@ -118,7 +118,11 @@ impl LayoutStrategy for CompressingStrategy { if UncompressedSizeInBytes::supports_dtype(chunk.dtype()) { uncompressed_size_in_bytes(&chunk, &mut ctx)?; } - Ok((sequence_id, compressor.compress_chunk(&chunk, &mut ctx)?)) + Ok(( + sequence_id, + compressor.compress_chunk(&chunk, &mut ctx)?, + defined, + )) }) }) .buffered(self.concurrency); diff --git a/vortex-layout/src/layouts/dict/writer.rs b/vortex-layout/src/layouts/dict/writer.rs index 0e854a924af..73c8230e383 100644 --- a/vortex-layout/src/layouts/dict/writer.rs +++ b/vortex-layout/src/layouts/dict/writer.rs @@ -40,6 +40,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_io::session::RuntimeSessionExt; +use vortex_mask::Mask; use vortex_session::VortexSession; use crate::LayoutRef; @@ -53,9 +54,11 @@ use crate::segments::SegmentSinkRef; use crate::sequence::SendableSequentialStream; use crate::sequence::SequenceId; use crate::sequence::SequencePointer; +use crate::sequence::SequentialChunk; use crate::sequence::SequentialStream; use crate::sequence::SequentialStreamAdapter; use crate::sequence::SequentialStreamExt; +use crate::sequence::fully_defined; /// Constraints for dictionary layout encoding. /// @@ -271,9 +274,11 @@ enum DictionaryChunk { Codes { seq_id: SequenceId, codes: ArrayRef, + /// The definedness on the rows of the codes, which are the rows of the input. + defined: Mask, codes_ptype: PType, }, - Values((SequenceId, ArrayRef)), + Values(SequentialChunk), } type DictionaryStream = BoxStream<'static, VortexResult>; @@ -293,7 +298,7 @@ fn dict_encode_stream( pin_mut!(input); while let Some(item) = input.next().await { - let (sequence_id, chunk) = item?; + let (sequence_id, chunk, defined) = item?; // labeler potentially creates sub sequences, we must // create it on both arms to avoid having a SequencePointer @@ -301,7 +306,7 @@ fn dict_encode_stream( match input.as_mut().peek().await { Some(_) => { let mut labeler = DictChunkLabeler::new(sequence_id); - let chunks = state.encode(&mut labeler, chunk, &mut exec_ctx)?; + let chunks = state.encode(&mut labeler, chunk, defined, &mut exec_ctx)?; drop(labeler); for dict_chunk in chunks { yield dict_chunk; @@ -310,7 +315,7 @@ fn dict_encode_stream( None => { // this is the last element, encode and drain chunks let mut labeler = DictChunkLabeler::new(sequence_id); - let encoded = state.encode(&mut labeler, chunk, &mut exec_ctx)?; + let encoded = state.encode(&mut labeler, chunk, defined, &mut exec_ctx)?; let drained = state.drain_values(&mut labeler); drop(labeler); for dict_chunk in encoded.into_iter().chain(drained.into_iter()) { @@ -332,23 +337,30 @@ impl DictStreamState { &mut self, labeler: &mut DictChunkLabeler, chunk: ArrayRef, + mut defined: Mask, exec_ctx: &mut ExecutionCtx, ) -> VortexResult> { let mut res = Vec::new(); let mut to_be_encoded = Some(chunk); + // The codes of each encoded prefix take the definedness on its rows. + let mut codes_definedness = |encoded: &ArrayRef| { + let prefix = defined.slice(..encoded.len()); + defined = defined.slice(encoded.len()..); + prefix + }; while let Some(remaining) = to_be_encoded.take() { match self.encoder.take() { None => match start_encoding(&self.constraints, &remaining, exec_ctx)? { EncodingState::Continue((encoder, encoded)) => { let ptype = encoder.codes_ptype(); - res.push(labeler.codes(encoded, ptype)); + res.push(labeler.codes(codes_definedness(&encoded), encoded, ptype)); self.encoder = Some(encoder); } EncodingState::Done((values, encoded, unencoded)) => { // Encoder was created and consumed within start_encoding let ptype = PType::try_from(encoded.dtype()) .vortex_expect("codes should be primitive"); - res.push(labeler.codes(encoded, ptype)); + res.push(labeler.codes(codes_definedness(&encoded), encoded, ptype)); res.push(labeler.values(values)); to_be_encoded = Some(unencoded); } @@ -357,11 +369,11 @@ impl DictStreamState { let ptype = encoder.codes_ptype(); match encode_chunk(encoder, &remaining, exec_ctx)? { EncodingState::Continue((encoder, encoded)) => { - res.push(labeler.codes(encoded, ptype)); + res.push(labeler.codes(codes_definedness(&encoded), encoded, ptype)); self.encoder = Some(encoder); } EncodingState::Done((values, encoded, unencoded)) => { - res.push(labeler.codes(encoded, ptype)); + res.push(labeler.codes(codes_definedness(&encoded), encoded, ptype)); res.push(labeler.values(values)); to_be_encoded = Some(unencoded); } @@ -390,20 +402,22 @@ impl DictChunkLabeler { Self { sequence_pointer } } - fn codes(&mut self, chunk: ArrayRef, ptype: PType) -> DictionaryChunk { + fn codes(&mut self, defined: Mask, chunk: ArrayRef, ptype: PType) -> DictionaryChunk { DictionaryChunk::Codes { seq_id: self.sequence_pointer.advance(), codes: chunk, + defined, codes_ptype: ptype, } } + /// Every value is part of the dictionary, so every value is defined. fn values(&mut self, chunk: ArrayRef) -> DictionaryChunk { - DictionaryChunk::Values((self.sequence_pointer.advance(), chunk)) + DictionaryChunk::Values(fully_defined(self.sequence_pointer.advance(), chunk)) } } -type SequencedChunk = VortexResult<(SequenceId, ArrayRef)>; +type SequencedChunk = VortexResult; struct DictionaryTransformer { input: DictionaryStream, @@ -453,6 +467,7 @@ impl Stream for DictionaryTransformer { Poll::Ready(Some(Ok(DictionaryChunk::Codes { seq_id, codes, + defined, codes_ptype, }))) => { if self.active_codes_tx.is_none() { @@ -467,10 +482,9 @@ impl Stream for DictionaryTransformer { let codes_dtype = DType::Primitive(codes_ptype, Nullability::NonNullable); // Send first codes. - self.pending_send = - Some(Box::pin( - async move { codes_tx.send(Ok((seq_id, codes))).await }, - )); + self.pending_send = Some(Box::pin(async move { + codes_tx.send(Ok((seq_id, codes, defined))).await + })); // Create output streams. let codes_stream = @@ -490,8 +504,9 @@ impl Stream for DictionaryTransformer { // Continue streaming codes to existing group if let Some(tx) = &self.active_codes_tx { let tx = tx.clone(); - self.pending_send = - Some(Box::pin(async move { tx.send(Ok((seq_id, codes))).await })); + self.pending_send = Some(Box::pin(async move { + tx.send(Ok((seq_id, codes, defined))).await + })); } } Poll::Ready(Some(Ok(DictionaryChunk::Values(values)))) => { @@ -536,16 +551,16 @@ async fn peek_probe_sample( let Some(item) = stream.next().await else { break; }; - let (sequence_id, chunk) = item?; + let (sequence_id, chunk, defined) = item?; peeked_len += chunk.len(); - peeked.push((sequence_id, chunk)); + peeked.push((sequence_id, chunk, defined)); } let sample_len = peeked_len.min(probe_len); let sample = match peeked.as_slice() { [] => return Ok((stream, None)), - [(_, chunk)] => chunk.slice(0..sample_len)?, - [(_, first), ..] => ChunkedArray::try_new( - peeked.iter().map(|(_, chunk)| chunk.clone()), + [(_, chunk, _)] => chunk.slice(0..sample_len)?, + [(_, first, _), ..] => ChunkedArray::try_new( + peeked.iter().map(|(_, chunk, _)| chunk.clone()), first.dtype().clone(), )? .into_array() @@ -626,6 +641,7 @@ mod tests { use vortex_error::VortexResult; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; + use vortex_mask::Mask; use vortex_session::VortexSession; use super::DictChunkLabeler; @@ -644,6 +660,7 @@ mod tests { use crate::sequence::SequentialStream; use crate::sequence::SequentialStreamAdapter; use crate::sequence::SequentialStreamExt; + use crate::sequence::fully_defined; use crate::test::new_session; static SESSION: LazyLock = @@ -668,7 +685,7 @@ mod tests { let mut pointer = SequenceId::root(); let input_stream = SequentialStreamAdapter::new( arr.dtype().clone(), - futures::stream::once(async move { Ok((pointer.advance(), arr)) }), + futures::stream::once(async move { Ok(fully_defined(pointer.advance(), arr)) }), ) .sendable(); @@ -711,7 +728,7 @@ mod tests { let mut pointer = SequenceId::root(); let input_stream = SequentialStreamAdapter::new( arr.dtype().clone(), - futures::stream::once(async move { Ok((pointer.advance(), arr)) }), + futures::stream::once(async move { Ok(fully_defined(pointer.advance(), arr)) }), ) .sendable(); @@ -761,7 +778,13 @@ mod tests { let value = "x".repeat(49); let chunk = VarBinArray::from(vec![value.as_str()]).into_array(); let mut labeler = DictChunkLabeler::new(SequenceId::root().advance()); - let encoded = state.encode(&mut labeler, chunk, &mut SESSION.create_execution_ctx())?; + let defined = Mask::new_true(chunk.len()); + let encoded = state.encode( + &mut labeler, + chunk, + defined, + &mut SESSION.create_execution_ctx(), + )?; assert_eq!(encoded_row_count(&encoded), 1); Ok(()) @@ -780,7 +803,13 @@ mod tests { let big = "x".repeat(49); let chunk = VarBinArray::from(vec!["a", big.as_str(), "b"]).into_array(); let mut labeler = DictChunkLabeler::new(SequenceId::root().advance()); - let encoded = state.encode(&mut labeler, chunk, &mut SESSION.create_execution_ctx())?; + let defined = Mask::new_true(chunk.len()); + let encoded = state.encode( + &mut labeler, + chunk, + defined, + &mut SESSION.create_execution_ctx(), + )?; let drained = state.drain_values(&mut labeler); assert_eq!(encoded_row_count(&encoded) + encoded_row_count(&drained), 3); @@ -788,7 +817,7 @@ mod tests { .iter() .chain(&drained) .filter_map(|chunk| match chunk { - DictionaryChunk::Values((_, values)) => Some(values), + DictionaryChunk::Values((_, values, _)) => Some(values), DictionaryChunk::Codes { .. } => None, }) .collect::>(); @@ -841,7 +870,7 @@ mod tests { let chunks = (0..nchunks) .map(|i| { let chunk = values.slice(i * chunk_len..(i + 1) * chunk_len)?; - Ok((sequence_id.advance(), chunk)) + Ok(fully_defined(sequence_id.advance(), chunk)) }) .collect::>>(); let segments = Arc::new(TestSegments::default()); diff --git a/vortex-layout/src/layouts/file_stats.rs b/vortex-layout/src/layouts/file_stats.rs index 46d4e5ab7c1..ce030b67f35 100644 --- a/vortex-layout/src/layouts/file_stats.rs +++ b/vortex-layout/src/layouts/file_stats.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::iter; use std::sync::Arc; use futures::StreamExt; @@ -16,6 +15,7 @@ use vortex_array::aggregate_fn::AggregateFnSatisfaction; use vortex_array::aggregate_fn::AggregateFnVTableExt; use vortex_array::aggregate_fn::EmptyOptions; use vortex_array::aggregate_fn::NumericalAggregateOpts; +use vortex_array::aggregate_fn::definedness::struct_field_definedness; use vortex_array::aggregate_fn::fns::bounded_max::BoundedMax; use vortex_array::aggregate_fn::fns::bounded_max::BoundedMaxOptions; use vortex_array::aggregate_fn::fns::bounded_min::BoundedMin; @@ -39,6 +39,7 @@ use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_io::session::RuntimeSessionExt; +use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_utils::parallelism::get_available_parallelism; @@ -92,9 +93,9 @@ pub fn accumulate_stats( let accumulator = chunk_accumulator.clone(); let mut exec_ctx = exec_ctx.clone(); handle.spawn_cpu(move || { - let (sequence_id, chunk) = item?; - let stats = accumulator.chunk_stats(&chunk, &mut exec_ctx)?; - Ok::<_, VortexError>((sequence_id, chunk, stats)) + let (sequence_id, chunk, defined) = item?; + let stats = accumulator.chunk_stats(&chunk, &defined, &mut exec_ctx)?; + Ok::<_, VortexError>((sequence_id, chunk, defined, stats)) }) }) .buffered(concurrency); @@ -104,9 +105,9 @@ pub fn accumulate_stats( let stream = SequentialStreamAdapter::new( dtype, stream.map(move |item| { - let (sequence_id, chunk, stats) = item?; + let (sequence_id, chunk, defined, stats) = item?; merge_accumulator.merge(stats)?; - Ok((sequence_id, chunk)) + Ok((sequence_id, chunk, defined)) }), ) .sendable(); @@ -306,9 +307,15 @@ impl StatsAccumulator { Self { aggregates } } - fn push_chunk(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { + /// Accumulate `array` as null wherever `defined` is false. + fn push_chunk( + &mut self, + array: &ArrayRef, + defined: &Mask, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { for (_, accumulator) in &mut self.aggregates { - accumulator.accumulate(array, ctx)?; + accumulator.accumulate_defined(array, defined, ctx)?; } Ok(()) } @@ -404,9 +411,8 @@ fn default_aggregate_fns(dtype: &DType) -> Vec { /// [`DType::Variant`], are skipped entirely: no entry is emitted for them or anything beneath /// them. /// -/// Entries beneath a nullable struct are still emitted, but the writer records no aggregates for -/// them until struct nulls are propagated into children -/// (). +/// Entries beneath a nullable struct describe the logical column, which is null wherever any +/// ancestor struct is null, whatever the unmasked child holds there. /// /// This function is the single source of truth for the number and order of stats entries, used /// both when accumulating stats at write time and when reconstructing them from the footer at @@ -443,18 +449,6 @@ enum StatsNode { /// Used for the own stats of a non-nullable top-level struct field, which the nested layout /// doesn't record but the legacy layout does. LegacyOnly(StatsAccumulator), - /// A subtree beneath a nullable struct, which contributes `entries` entries that record no - /// aggregates. - /// - /// Struct fields are stored unmasked, so a row where the struct is null holds an unspecified - /// value in each child that the child's own validity may report as valid. Stats over those - /// children would count that value, e.g. undercount nulls, which can prune a file that - /// `is_null(s.a)` matches. Until nulls are propagated from structs into their children, these - /// entries are left empty. See . - /// - /// The entries are kept rather than dropped so the footer layout from - /// [`postorder_stats_layout`] doesn't depend on whether they're recorded. - Unrecorded { entries: usize }, /// A dtype with addressable children. Currently only built for `DType::Struct`. Container { /// One child per addressable sub-dtype, in declaration order (one per struct field, @@ -514,20 +508,17 @@ impl StatsNode { let Some(struct_fields) = dtype.as_struct_fields_opt() else { return Self::LegacyOnly(StatsAccumulator::new(dtype, stats, allowed)); }; - // Like `build`, the fields of a nullable root aren't recorded, since they're stored - // unmasked. - let nullable = dtype.nullability() == Nullability::Nullable; + // The fields of a nullable root are pushed with the root's validity as their definedness, so + // they are recorded as their logical columns, as in `build`. let children = struct_fields .names() .iter() .zip(struct_fields.fields()) .map(|(name, field_dtype)| { - let child = if nullable { - Self::Skipped - } else { - Self::LegacyOnly(StatsAccumulator::new(&field_dtype, stats, allowed)) - }; - (Field::Name(name.clone()), child) + ( + Field::Name(name.clone()), + Self::LegacyOnly(StatsAccumulator::new(&field_dtype, stats, allowed)), + ) }) .collect(); Self::Container { @@ -549,18 +540,12 @@ impl StatsNode { .iter() .zip(struct_fields.fields()) .map(|(name, field_dtype)| { - let child = if nullable { - Self::Unrecorded { - entries: postorder_stats_layout(&field_dtype).len(), - } - } else { - Self::build(&field_dtype, stats, allowed) - }; - (Field::Name(name.clone()), child) + ( + Field::Name(name.clone()), + Self::build(&field_dtype, stats, allowed), + ) }) .collect(); - // Only reached through non-nullable ancestors, so the struct's own validity is - // its full logical validity. let own = if nullable { Self::Leaf(StatsAccumulator::new(dtype, stats, allowed)) } else { @@ -582,21 +567,31 @@ impl StatsNode { /// [`Field::Name`] and extracted via [`StructArrayExt::iter_unmasked_fields`]. A future `List`/ /// `Map` container would need a different extraction here (e.g. flattened elements, or /// derived per-row shape arrays), dispatched per child's [`Field`] kind. - fn push_chunk(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { + /// + /// The chunk is null wherever `defined` is false, i.e. wherever an ancestor struct is null. + fn push_chunk( + &mut self, + array: &ArrayRef, + defined: &Mask, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { match self { - Self::Skipped | Self::Unrecorded { .. } => Ok(()), - Self::Leaf(acc) | Self::LegacyOnly(acc) => acc.push_chunk(array, ctx), + Self::Skipped => Ok(()), + Self::Leaf(acc) | Self::LegacyOnly(acc) => acc.push_chunk(array, defined, ctx), Self::Container { children, own } => { // The container's own `ArrayRef` already carries the validity needed to compute // its own stats (e.g. null count), so we push it directly rather than building a // synthetic array. - own.push_chunk(array, ctx)?; + own.push_chunk(array, defined, ctx)?; let struct_array = array.clone().execute::(ctx)?; + // Fields are stored unmasked, so they are pushed with the struct's validity merged + // into the definedness: each field is null wherever the struct or an ancestor is null. + let field_definedness = struct_field_definedness(&struct_array, defined, ctx)?; for ((_, child), field) in children .iter_mut() .zip_eq(struct_array.iter_unmasked_fields()) { - child.push_chunk(field, ctx)?; + child.push_chunk(field, &field_definedness, ctx)?; } Ok(()) } @@ -606,9 +601,7 @@ impl StatsNode { /// Drains the state of `other`, a node built for the same dtype and aggregates, into `self`. fn merge_from(&mut self, other: &mut Self) -> VortexResult<()> { match (self, other) { - (Self::Skipped, Self::Skipped) | (Self::Unrecorded { .. }, Self::Unrecorded { .. }) => { - Ok(()) - } + (Self::Skipped, Self::Skipped) => Ok(()), (Self::Leaf(acc), Self::Leaf(other)) | (Self::LegacyOnly(acc), Self::LegacyOnly(other)) => acc.merge_from(other), ( @@ -639,10 +632,6 @@ impl StatsNode { ) -> VortexResult<()> { match self { Self::Skipped | Self::LegacyOnly(_) => Ok(()), - Self::Unrecorded { entries } => { - out.extend(iter::repeat_n(AggregateStats::default(), *entries)); - Ok(()) - } Self::Leaf(acc) => { out.push(acc.aggregate_stats(allowed)?); Ok(()) @@ -663,7 +652,7 @@ impl StatsNode { allowed: impl Fn(&AggregateFnRef) -> bool, ) -> VortexResult { match self { - Self::Skipped | Self::Unrecorded { .. } => Ok(AggregateStats::default()), + Self::Skipped => Ok(AggregateStats::default()), Self::Leaf(acc) | Self::LegacyOnly(acc) => acc.aggregate_stats(allowed), Self::Container { own, .. } => own.legacy_aggregate_stats(allowed), } @@ -732,7 +721,12 @@ impl FileStatsAccumulator { /// Computes the stats of one chunk into a new tree of accumulators, to [merge](Self::merge) /// later. - fn chunk_stats(&self, chunk: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + fn chunk_stats( + &self, + chunk: &ArrayRef, + defined: &Mask, + ctx: &mut ExecutionCtx, + ) -> VortexResult { let mut node = StatsNode::build_root( &self.dtype, self.stats.as_deref(), @@ -747,7 +741,7 @@ impl FileStatsAccumulator { self.write_nested_stats, self.write_legacy_stats, ); - node.push_chunk(chunk, ctx)?; + node.push_chunk(chunk, defined, ctx)?; Ok(node) } @@ -810,8 +804,7 @@ impl FileStatsAccumulator { }; let root = self.root.lock(); // Readers that predate nested stats reject a file whose legacy layout doesn't have one - // entry per top-level field, so entries that aren't recorded are still written, empty. The - // fields of a nullable root are `StatsNode::Unrecorded`, so they come out empty too. + // entry per top-level field, so entries that aren't recorded are still written, empty. let fields = match &*root { StatsNode::Container { children, .. } => { children.iter().map(|(_, child)| child).collect() @@ -857,6 +850,23 @@ mod tests { use super::*; + /// A stats node or accumulator a chunk can be pushed into with every row defined. + trait PushAll { + fn push_all(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()>; + } + + impl PushAll for StatsAccumulator { + fn push_all(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { + self.push_chunk(array, &Mask::new_true(array.len()), ctx) + } + } + + impl PushAll for StatsNode { + fn push_all(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { + self.push_chunk(array, &Mask::new_true(array.len()), ctx) + } + } + static SESSION: LazyLock = LazyLock::new(VortexSession::empty); /// Pushes `chunk` through `acc` the way the write stream does. @@ -865,7 +875,7 @@ mod tests { chunk: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - acc.merge(acc.chunk_stats(chunk, ctx)?) + acc.merge(acc.chunk_stats(chunk, &Mask::new_true(chunk.len()), ctx)?) } fn stats_set(acc: &StatsAccumulator) -> VortexResult { @@ -914,7 +924,7 @@ mod tests { vortex_buffer::BufferAllocatorRef::statically_allocated(), ); builder.append_value(value); - acc.push_chunk(&builder.finish(), &mut ctx)?; + acc.push_all(&builder.finish(), &mut ctx)?; } let stats = stats_set(&acc)?; @@ -970,7 +980,7 @@ mod tests { let mut acc = StatsAccumulator::new(&dtype, Some(&[agg(Stat::Max), agg(Stat::Min)]), &|_| true); - acc.push_chunk(&builder.finish(), &mut ctx)?; + acc.push_all(&builder.finish(), &mut ctx)?; let stats = stats_set(&acc)?; assert!(matches!(stats.get(Stat::Max), Precision::Exact(_))); @@ -993,9 +1003,9 @@ mod tests { &|_| true, ); - acc.push_chunk(&buffer![0, 5, 2].into_array(), &mut ctx)?; - acc.push_chunk(&buffer![7, 1, 3].into_array(), &mut ctx)?; - acc.push_chunk(&buffer![-4, 9].into_array(), &mut ctx)?; + acc.push_all(&buffer![0, 5, 2].into_array(), &mut ctx)?; + acc.push_all(&buffer![7, 1, 3].into_array(), &mut ctx)?; + acc.push_all(&buffer![-4, 9].into_array(), &mut ctx)?; let stats = stats_set(&acc)?; assert_eq!( @@ -1143,11 +1153,10 @@ mod tests { } #[test] - fn three_level_nested_struct_records_only_entries_without_nullable_ancestors() - -> VortexResult<()> { + fn three_level_nested_struct_records_entries_beneath_nullable_ancestors() -> VortexResult<()> { // Same shape as `postorder_layout_three_level_nesting_with_mixed_nullability`, but - // exercises actual accumulation. The root is nullable, so every entry beneath it is left - // empty; only the root's own null-count entry is recorded. + // exercises actual accumulation. Every entry describes its logical column, which is null + // wherever any ancestor struct is null. let mut ctx = SESSION.create_execution_ctx(); let leaf_c = buffer![10i32, 20, 30].into_array(); @@ -1167,14 +1176,30 @@ mod tests { let requested = [agg(Stat::NullCount), agg(Stat::Min), agg(Stat::Max)]; let mut node = StatsNode::build(root.dtype(), Some(&requested), &|_| true); - node.push_chunk(&root, &mut ctx)?; + node.push_all(&root, &mut ctx)?; let stats_sets = node_stats_sets(&node)?; // Post-order: `a.b.c`, then `a.b`'s null-count entry, then the root's. assert_eq!(stats_sets.len(), 3); - assert!(stats_sets[0].is_empty()); - assert!(stats_sets[1].is_empty()); + // `a.b.c` is valid only where both the root and `a.b` are, so 20 and 30 are left out. + assert_eq!( + stats_sets[0].get(Stat::NullCount).as_exact(), + Some(ScalarValue::from(2u64)) + ); + assert_eq!( + stats_sets[0].get(Stat::Min).as_exact(), + Some(ScalarValue::from(10i32)) + ); + assert_eq!( + stats_sets[0].get(Stat::Max).as_exact(), + Some(ScalarValue::from(10i32)) + ); + // `a.b` is null in its own null row and in the root's. + assert_eq!( + stats_sets[1].get(Stat::NullCount).as_exact(), + Some(ScalarValue::from(2u64)) + ); assert_eq!( stats_sets[2].get(Stat::NullCount).as_exact(), Some(ScalarValue::from(1u64)) @@ -1183,8 +1208,9 @@ mod tests { } #[test] - fn children_of_nullable_struct_are_unrecorded() -> VortexResult<()> { - // https://github.com/vortex-data/vortex/issues/9205 + fn children_of_nullable_struct_are_null_under_its_null_rows() -> VortexResult<()> { + // https://github.com/vortex-data/vortex/issues/9205: `s.a` holds 999 in the row where `s` + // is null, which its stats leave out and count as null. let mut ctx = SESSION.create_execution_ctx(); let a = buffer![1i32, 999, 3].into_array(); @@ -1195,14 +1221,25 @@ mod tests { let requested = [agg(Stat::NullCount), agg(Stat::Min), agg(Stat::Max)]; let mut node = StatsNode::build(root.dtype(), Some(&requested), &|_| true); - node.push_chunk(&root, &mut ctx)?; + node.push_all(&root, &mut ctx)?; let mut aggregates = Vec::new(); node.collect_aggregate_stats(&|_| true, &mut aggregates)?; // `s.a`, then `s`'s own null-count entry. assert_eq!(aggregates.len(), 2); - assert!(aggregates[0].is_empty()); + assert_eq!( + aggregates[0].get(&agg(Stat::NullCount)), + Precision::exact(Scalar::from(1u64)) + ); + assert_eq!( + aggregates[0].get(&agg(Stat::Min)), + Precision::exact(Scalar::from(1i32)) + ); + assert_eq!( + aggregates[0].get(&agg(Stat::Max)), + Precision::exact(Scalar::from(3i32)) + ); assert_eq!( aggregates[1].get(&agg(Stat::NullCount)), Precision::exact(Scalar::from(1u64)) @@ -1223,7 +1260,7 @@ mod tests { let requested = [agg(Stat::NullCount), agg(Stat::Max)]; let mut node = StatsNode::build(root.dtype(), Some(&requested), &|_| true); - node.push_chunk(&root, &mut ctx)?; + node.push_all(&root, &mut ctx)?; let stats_sets = node_stats_sets(&node)?; assert_eq!(stats_sets.len(), 1); @@ -1251,7 +1288,7 @@ mod tests { let requested = [agg(Stat::NullCount), agg(Stat::Min), agg(Stat::Max)]; let mut node = StatsNode::build(outer.dtype(), Some(&requested), &|_| true); - node.push_chunk(&outer, &mut ctx)?; + node.push_all(&outer, &mut ctx)?; let stats_sets = node_stats_sets(&node)?; @@ -1340,7 +1377,7 @@ mod tests { } #[test] - fn legacy_aggregate_stats_of_nullable_root_are_unrecorded() -> VortexResult<()> { + fn legacy_aggregate_stats_of_nullable_root_are_null_under_its_null_rows() -> VortexResult<()> { let mut exec_ctx = SESSION.create_execution_ctx(); let dtype = DType::struct_([("a", i32_dtype())], Nullability::Nullable); @@ -1360,7 +1397,15 @@ mod tests { let legacy = acc.legacy_aggregate_stats()?; assert_eq!(legacy.len(), 1); - assert!(legacy[0].is_empty()); + // `a` holds 999 in the row the root nulls out, which counts as null and is left out. + assert_eq!( + legacy[0].get(&agg(Stat::NullCount)), + Precision::exact(Scalar::from(1u64)) + ); + assert_eq!( + legacy[0].get(&agg(Stat::Max)), + Precision::exact(Scalar::from(3i32)) + ); Ok(()) } @@ -1453,7 +1498,7 @@ mod tests { max_bytes: NonZeroUsize::new(8).vortex_expect("non-zero"), }); let mut acc = StatsAccumulator::new(&dtype, Some(std::slice::from_ref(&max)), &|_| true); - acc.push_chunk( + acc.push_all( &utf8_chunk(&["short", "a value longer than the bound"]), &mut ctx, )?; @@ -1644,7 +1689,7 @@ mod tests { let sum = agg(Stat::Sum); let mut acc = StatsAccumulator::new(&dtype, Some(std::slice::from_ref(&sum)), &|_| true); for chunk in chunks { - acc.push_chunk( + acc.push_all( &PrimitiveArray::from_option_iter(chunk).into_array(), &mut ctx, )?; diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index 1228c06f61c..f153dde7a02 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -98,7 +98,8 @@ impl LayoutStrategy for FlatLayoutStrategy { ) .into_layout()); }; - let (sequence_id, chunk) = chunk?; + // The flat writer records no aggregates, so it has no use for the definedness. + let (sequence_id, chunk, _) = chunk?; let row_count = chunk.len() as u64; diff --git a/vortex-layout/src/layouts/indexed/writer.rs b/vortex-layout/src/layouts/indexed/writer.rs index 67769334f96..95aa566f2ca 100644 --- a/vortex-layout/src/layouts/indexed/writer.rs +++ b/vortex-layout/src/layouts/indexed/writer.rs @@ -516,9 +516,9 @@ impl LayoutStrategy for IndexedStrategy { let stream = SequentialStreamAdapter::new( dtype, stream.map(move |item| { - let (sequence_id, chunk) = item?; + let (sequence_id, chunk, defined) = item?; feed_state.lock().push(&chunk, &feed_tracker)?; - Ok((sequence_id, chunk)) + Ok((sequence_id, chunk, defined)) }), ) .sendable(); diff --git a/vortex-layout/src/layouts/list/writer.rs b/vortex-layout/src/layouts/list/writer.rs index dfd7c856e06..9b19bdacd4f 100644 --- a/vortex-layout/src/layouts/list/writer.rs +++ b/vortex-layout/src/layouts/list/writer.rs @@ -11,14 +11,18 @@ use futures::StreamExt; use futures::future::join; use futures::future::try_join_all; use vortex_array::ArrayRef; +use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; +use vortex_array::aggregate_fn::definedness::list_elements_definedness; +use vortex_array::aggregate_fn::definedness::valid_definedness; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::List; +use vortex_array::arrays::ListArray; use vortex_array::arrays::ListView; use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::list::ListArrayExt; +use vortex_array::arrays::list::ListArraySlotsExt; use vortex_array::arrays::list::ListDataParts; use vortex_array::arrays::listview::list_from_list_view; use vortex_array::builtins::ArrayBuiltins; @@ -27,10 +31,12 @@ use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::matcher::Matcher; use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_io::session::RuntimeSessionExt; +use vortex_mask::Mask; use vortex_session::VortexSession; use crate::LayoutRef; @@ -42,14 +48,15 @@ use crate::layouts::flat::writer::FlatLayoutStrategy; use crate::layouts::list::ListLayout; use crate::segments::SegmentSinkRef; use crate::sequence::SendableSequentialStream; -use crate::sequence::SequenceId; use crate::sequence::SequencePointer; +use crate::sequence::SequentialChunk; use crate::sequence::SequentialStream; use crate::sequence::SequentialStreamAdapter; use crate::sequence::SequentialStreamExt; +use crate::sequence::fully_defined; /// Item carried on each child sub-stream: a sequenced, materialized chunk. -type ChildChunk = VortexResult<(SequenceId, ArrayRef)>; +type ChildChunk = VortexResult; /// Strategy for writing list-typed arrays, with a fallback for non-list dtypes. /// @@ -168,6 +175,9 @@ impl LayoutStrategy for ListLayoutStrategy { (None, None) }; + // The elements are normalised to those of the lists its ancestors define and that are + // valid, so every element written belongs to the logical elements column, and every row + // of every child is defined. // Transpose the list column into its child sub-streams and rebase offsets to global // positions. Kept joined with the child writers below so producer errors surface rather // than being hidden as an early channel close. @@ -233,7 +243,7 @@ impl LayoutStrategy for ListLayoutStrategy { /// `validity_tx` is `Some` exactly when the list is nullable. Errors surface to the caller, which /// joins this against the child writers, rather than being hidden as an early channel close. async fn transpose_list_column( - first_chunk: (SequenceId, ArrayRef), + first_chunk: SequentialChunk, stream: SendableSequentialStream, session: VortexSession, elements_tx: Sender, @@ -245,14 +255,16 @@ async fn transpose_list_column( let mut first = true; let mut chunks = futures::stream::iter([Ok(first_chunk)]).chain(stream); while let Some(chunk) = chunks.next().await { - let (sequence_id, array) = chunk?; + let (sequence_id, array, defined) = chunk?; let mut sp = sequence_id.descend(); + let list = canonicalize_to_list(array, &mut exec_ctx)?; + let list = drop_absent_elements(list, &defined, &mut exec_ctx)?; let ListDataParts { elements, offsets, validity, .. - } = canonicalize_to_list_parts(array, &mut exec_ctx)?; + } = list.into_data_parts(); let n_elements = elements.len() as u64; let row_count = offsets.len().saturating_sub(1); let offsets = global_offsets(offsets, element_base, first, &mut exec_ctx)?; @@ -260,10 +272,13 @@ async fn transpose_list_column( first = false; if elements_tx - .send(Ok((sp.advance(), elements))) + .send(Ok(fully_defined(sp.advance(), elements))) .await .is_err() - || offsets_tx.send(Ok((sp.advance(), offsets))).await.is_err() + || offsets_tx + .send(Ok(fully_defined(sp.advance(), offsets))) + .await + .is_err() { vortex_bail!("list child writer finished before all chunks were sent"); } @@ -272,7 +287,7 @@ async fn transpose_list_column( .execute_mask(row_count, &mut exec_ctx)? .into_array(); if validity_tx - .send(Ok((sp.advance(), validity))) + .send(Ok(fully_defined(sp.advance(), validity))) .await .is_err() { @@ -283,18 +298,50 @@ async fn transpose_list_column( Ok(()) } -/// Canonicalize a list-dtype array into [`ListDataParts`]. -fn canonicalize_to_list_parts( - array: ArrayRef, +/// Drop the elements that do not belong to the logical elements column: those of null lists, of +/// lists `defined` does not select, and elements no list references. Dropped lists become empty. +fn drop_absent_elements( + list: ListArray, + defined: &Mask, exec_ctx: &mut ExecutionCtx, -) -> VortexResult { +) -> VortexResult { + let validity = list.validity()?; + let lists = valid_definedness(&validity, defined, exec_ctx)?; + let elements = list_elements_definedness(&list, &lists, exec_ctx)?; + if elements.all_true() { + return Ok(list); + } + + let offsets = list + .offsets() + .clone() + .cast(DType::Primitive(PType::U64, Nullability::NonNullable))? + .execute::(exec_ctx)?; + let mut kept = BufferMut::::with_capacity(offsets.len()); + kept.push(0); + let mut end = 0u64; + for (row, bounds) in offsets.as_slice::().windows(2).enumerate() { + if lists.value(row) { + end += bounds[1] - bounds[0]; + } + kept.push(end); + } + // Writers serialize what they receive, so the filter is executed rather than left lazy. + let elements = list + .elements() + .filter(elements)? + .execute::(exec_ctx)? + .into_array(); + ListArray::try_new(elements, kept.freeze().into_array(), validity, exec_ctx) +} + +/// Canonicalize a list-dtype array into a [`ListArray`]. +fn canonicalize_to_list(array: ArrayRef, exec_ctx: &mut ExecutionCtx) -> VortexResult { let canonical = array.execute_until::(exec_ctx)?; if let Some(list) = canonical.as_opt::() { - // A sliced list keeps its parent's whole `elements`, but `global_offsets` assumes each - // chunk's offsets span exactly `[0, elements.len()]`. - Ok(list.reset_offsets(false, exec_ctx)?.into_data_parts()) + Ok(list.into_owned()) } else if let Some(view) = canonical.as_opt::() { - Ok(list_from_list_view(view.into_owned(), exec_ctx)?.into_data_parts()) + list_from_list_view(view.into_owned(), exec_ctx) } else { unreachable!("AnyList matcher guarantees List or ListView") } @@ -340,20 +387,28 @@ impl Matcher for AnyList { #[cfg(test)] mod tests { + use std::num::NonZeroUsize; use std::sync::LazyLock; use futures::stream; use rstest::rstest; use vortex_array::ArrayContext; use vortex_array::VortexSessionExecute; + use vortex_array::aggregate_fn::AggregateFnVTableExt; + use vortex_array::aggregate_fn::EmptyOptions; + use vortex_array::aggregate_fn::fns::max::Max; + use vortex_array::aggregate_fn::fns::min::Min; + use vortex_array::aggregate_fn::fns::null_count::NullCount; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::StructArray; + use vortex_array::assert_arrays_eq; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::validity::Validity; use vortex_buffer::buffer; + use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSession; use vortex_session::VortexSession; @@ -361,12 +416,18 @@ mod tests { use crate::layouts::chunked::writer::ChunkedLayoutStrategy; use crate::layouts::flat::writer::FlatLayoutStrategy; use crate::layouts::table::TableStrategy; + use crate::layouts::zoned::Zoned; + use crate::layouts::zoned::writer::ZonedLayoutOptions; + use crate::layouts::zoned::writer::ZonedStrategy; use crate::segments::TestSegments; + use crate::sequence::SequenceId; use crate::sequence::SequentialArrayStreamExt; use crate::session::LayoutSession; use crate::test::FailingStrategy; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + use crate::test::new_session; + use crate::test::read_zone_stats; fn layout_test_session() -> VortexSession { VortexSession::empty() @@ -418,6 +479,165 @@ mod tests { .into_array() } + fn zoned_leaf() -> Arc { + Arc::new(ZonedStrategy::new( + ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()), + FlatLayoutStrategy::default(), + ZonedLayoutOptions { + block_size: NonZeroUsize::new(3).vortex_expect("non zero"), + ..Default::default() + }, + )) + } + + /// Write `array` through `strategy` on a single-threaded runtime, returning the layout and the + /// segments it was written to. + fn write_blocking( + strategy: S, + array: ArrayRef, + ) -> VortexResult<(LayoutRef, Arc)> { + let (ptr, eof) = SequenceId::root().split(); + let stream = array.to_array_stream().sequenced(ptr); + let segments = Arc::new(TestSegments::default()); + let segments2 = Arc::clone(&segments); + let layout = block_on(|handle| async move { + let session = new_session().with_handle(handle); + strategy + .write_stream( + ArrayContext::empty().into(), + segments2, + stream, + eof, + &session, + ) + .await + })?; + Ok((layout, segments)) + } + + /// The per-zone minimum and maximum of a zoned elements layout. + fn zone_bounds( + elements: &LayoutRef, + segments: &Arc, + ) -> VortexResult<(ArrayRef, ArrayRef)> { + assert!(elements.is::(), "{elements} has no zone map"); + Ok(( + read_zone_stats(elements, segments, &Min.bind(Default::default()))?, + read_zone_stats(elements, segments, &Max.bind(Default::default()))?, + )) + } + + /// The elements of a null list are dropped, so the elements' zone map covers only the + /// elements of valid lists, and does not count the null list's null element. + #[test] + fn elements_of_null_lists_are_dropped() -> VortexResult<()> { + let list = ListArray::try_new( + PrimitiveArray::from_option_iter([Some(1i32), Some(100), None, Some(2), Some(3)]) + .into_array(), + buffer![0u32, 1, 3, 5].into_array(), + Validity::from_iter([true, false, true]), + &mut SESSION.create_execution_ctx(), + )? + .into_array(); + + let (layout, segments) = write_blocking( + ListLayoutStrategy::default().with_elements(zoned_leaf()), + list, + )?; + + let elements = &layout.children()?[0]; + assert_eq!(elements.row_count(), 3); + let (min, max) = zone_bounds(elements, &segments)?; + let null_count = read_zone_stats(elements, &segments, &NullCount.bind(EmptyOptions))?; + let mut ctx = new_session().create_execution_ctx(); + assert_arrays_eq!( + min, + PrimitiveArray::from_option_iter([Some(1i32)]), + &mut ctx + ); + assert_arrays_eq!( + max, + PrimitiveArray::from_option_iter([Some(3i32)]), + &mut ctx + ); + assert_arrays_eq!( + null_count, + PrimitiveArray::from_option_iter([Some(0u64)]), + &mut ctx + ); + Ok(()) + } + + /// Lists that are already normalised are written unchanged. + #[test] + fn normalised_lists_keep_their_elements() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let list = ListArray::try_new( + buffer![1i32, 2, 3].into_array(), + buffer![0u32, 1, 1, 3].into_array(), + Validity::from_iter([true, false, true]), + &mut ctx, + )?; + let normalised = drop_absent_elements(list.clone(), &Mask::new_true(3), &mut ctx)?; + assert_arrays_eq!(normalised, list, &mut ctx); + assert_eq!(normalised.elements().len(), 3); + Ok(()) + } + + /// A list zoned in its outer-row space under a nullable struct hands the struct's definedness on + /// to the list writer, which keeps only the elements of the struct's valid rows. + #[test] + fn list_under_nullable_struct_keeps_elements_of_valid_rows() -> VortexResult<()> { + let list = ListArray::try_new( + buffer![1i32, 60, 50, 2].into_array(), + buffer![0u32, 1, 3, 4].into_array(), + Validity::NonNullable, + &mut SESSION.create_execution_ctx(), + )?; + let array = StructArray::try_from_iter_with_validity( + [("l", list.into_array())], + Validity::from_iter([true, false, true]), + )? + .into_array(); + + let flat: Arc = Arc::new(FlatLayoutStrategy::default()); + let strategy = TableStrategy::new(Arc::clone(&flat), Arc::clone(&flat)) + .with_list_layout_factory(move |list_layout: ListLayoutStrategy| { + Arc::new(ZonedStrategy::new( + list_layout.with_elements(zoned_leaf()), + Arc::clone(&flat), + ZonedLayoutOptions { + block_size: NonZeroUsize::new(3).vortex_expect("non zero"), + ..Default::default() + }, + )) as Arc + }); + let (layout, segments) = write_blocking(strategy, array)?; + + // Skip the struct's validity child, then the list's zone map to reach the list layout. + let list_layout = Arc::clone(&layout.children()?[1]); + let list_layout = if list_layout.is::() { + Arc::clone(&list_layout.children()?[0]) + } else { + list_layout + }; + // The list in the struct's null row is written empty, without its 60 and 50. + assert_eq!(list_layout.children()?[0].row_count(), 2); + let (min, max) = zone_bounds(&list_layout.children()?[0], &segments)?; + let mut ctx = new_session().create_execution_ctx(); + assert_arrays_eq!( + min, + PrimitiveArray::from_option_iter([Some(1i32)]), + &mut ctx + ); + assert_arrays_eq!( + max, + PrimitiveArray::from_option_iter([Some(2i32)]), + &mut ctx + ); + Ok(()) + } + #[tokio::test] async fn basic_non_nullable_input() -> VortexResult<()> { let list = create_basic_list(Validity::NonNullable); @@ -469,7 +689,7 @@ mod tests { ) -> VortexResult<()> { let segments = Arc::new(TestSegments::default()); let (_, eof) = SequenceId::root().split(); - let empty = stream::empty::>().boxed(); + let empty = stream::empty::>().boxed(); let stream = SequentialStreamAdapter::new(i32_list_dtype(nullable), empty).sendable(); let session = layout_test_session(); diff --git a/vortex-layout/src/layouts/repartition.rs b/vortex-layout/src/layouts/repartition.rs index 7098b0d547c..35d4575ec10 100644 --- a/vortex-layout/src/layouts/repartition.rs +++ b/vortex-layout/src/layouts/repartition.rs @@ -20,6 +20,7 @@ use vortex_array::dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; +use vortex_mask::Mask; use vortex_session::VortexSession; use crate::LayoutRef; @@ -113,10 +114,10 @@ impl LayoutStrategy for RepartitionStrategy { SequentialStreamAdapter::new( dtype.clone(), stream.map(move |chunk| { - let (sequence_id, chunk) = chunk?; + let (sequence_id, chunk, defined) = chunk?; let mut ctx = canonicalize_session.create_execution_ctx(); let canonical = chunk.execute::(&mut ctx)?.into_array(); - VortexResult::Ok((sequence_id, canonical)) + VortexResult::Ok((sequence_id, canonical, defined)) }), ) .sendable() @@ -141,17 +142,18 @@ impl LayoutStrategy for RepartitionStrategy { let mut ctx = repartition_session.create_execution_ctx(); let mut chunks = ChunksBuffer::new(block_size_minimum, block_len); while let Some(chunk) = canonical_stream.as_mut().next().await { - let (sequence_id, chunk) = chunk?; + let (sequence_id, chunk, defined) = chunk?; let mut sequence_pointer = sequence_id.descend(); let mut offset = 0; while offset < chunk.len() { let end = (offset + block_len).min(chunk.len()); let sliced = chunk.slice(offset..end)?; - chunks.push_back(sliced, &mut ctx)?; + chunks.push_back(sliced, defined.slice(offset..end), &mut ctx)?; offset = end; if chunks.have_enough() { - let output_chunks = chunks.collect_exact_blocks(&mut ctx)?; + let (output_chunks, output_defined): (Vec<_>, Vec<_>) = + chunks.collect_exact_blocks(&mut ctx)?.into_iter().unzip(); assert!(!output_chunks.is_empty()); let chunked = ChunkedArray::try_new(output_chunks, dtype_clone.clone())?; @@ -160,20 +162,24 @@ impl LayoutStrategy for RepartitionStrategy { yield ( sequence_pointer.advance(), canonical, + Mask::concat(output_defined.iter())?, ) } } } if canonical_stream.as_mut().peek().await.is_none() { - let to_flush = ChunkedArray::try_new( - chunks.data.drain(..).map(|(arr, _)| arr), - dtype_clone.clone(), - )?; + let (to_flush, flush_defined): (Vec<_>, Vec<_>) = chunks + .data + .drain(..) + .map(|(arr, defined, _)| (arr, defined)) + .unzip(); + let to_flush = ChunkedArray::try_new(to_flush, dtype_clone.clone())?; if !to_flush.is_empty() { let canonical = to_flush.into_array().execute::(&mut ctx)?.into_array(); yield ( sequence_pointer.advance(), canonical, + Mask::concat(flush_defined.iter())?, ) } } @@ -197,10 +203,11 @@ impl LayoutStrategy for RepartitionStrategy { /// Bytes are the exact uncompressed size of each block, not the buffers it holds: a view array /// read from Arrow points into whole decompressed pages, which `nbytes()` would count in full. struct ChunksBuffer { - /// Each entry stores the chunk and the size snapshot taken at push time. + /// Each entry stores the chunk, the definedness on its rows, and the size snapshot taken at push + /// time. /// This avoids accounting mismatches when interior-mutable arrays (e.g. `SharedArray`) /// change their reported size after being pushed. - data: VecDeque<(ArrayRef, u64)>, + data: VecDeque<(ArrayRef, Mask, u64)>, row_count: usize, nbytes: u64, block_size_minimum: u64, @@ -222,12 +229,15 @@ impl ChunksBuffer { self.nbytes >= self.block_size_minimum && self.row_count >= self.block_len_multiple } - fn collect_exact_blocks(&mut self, ctx: &mut ExecutionCtx) -> VortexResult> { + fn collect_exact_blocks( + &mut self, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { let nblocks = self.row_count / self.block_len_multiple; let mut res = Vec::with_capacity(self.data.len()); let mut remaining = nblocks * self.block_len_multiple; while remaining > 0 { - let (chunk, _) = self + let (chunk, defined, _) = self .pop_front() .vortex_expect("must have at least one chunk"); let len = chunk.len(); @@ -235,36 +245,46 @@ impl ChunksBuffer { if len > remaining { let left = chunk.slice(0..remaining)?; let right = chunk.slice(remaining..len)?; - self.push_front(right, ctx)?; - res.push(left); + self.push_front(right, defined.slice(remaining..len), ctx)?; + res.push((left, defined.slice(0..remaining))); remaining = 0; } else { - res.push(chunk); + res.push((chunk, defined)); remaining -= len; } } Ok(res) } - fn push_back(&mut self, chunk: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { + fn push_back( + &mut self, + chunk: ArrayRef, + defined: Mask, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { let nb = exact_size(&chunk, ctx)?; self.row_count += chunk.len(); self.nbytes += nb; - self.data.push_back((chunk, nb)); + self.data.push_back((chunk, defined, nb)); Ok(()) } - fn push_front(&mut self, chunk: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { + fn push_front( + &mut self, + chunk: ArrayRef, + defined: Mask, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { let nb = exact_size(&chunk, ctx)?; self.row_count += chunk.len(); self.nbytes += nb; - self.data.push_front((chunk, nb)); + self.data.push_front((chunk, defined, nb)); Ok(()) } - fn pop_front(&mut self) -> Option<(ArrayRef, u64)> { + fn pop_front(&mut self) -> Option<(ArrayRef, Mask, u64)> { let res = self.data.pop_front(); - if let Some((chunk, nb)) = res.as_ref() { + if let Some((chunk, _, nb)) = res.as_ref() { self.row_count -= chunk.len(); self.nbytes -= nb; } @@ -302,6 +322,7 @@ mod tests { use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::ByteBuffer; + use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; @@ -577,8 +598,9 @@ mod tests { let s2 = arr.slice(block_len..n)?; let mut buf = ChunksBuffer::new(0, block_len); - buf.push_back(s1, &mut ctx)?; - buf.push_back(s2, &mut ctx)?; + let (d1, d2) = (Mask::new_true(s1.len()), Mask::new_true(s2.len())); + buf.push_back(s1, d1, &mut ctx)?; + buf.push_back(s2, d2, &mut ctx)?; let _output = buf.pop_front().unwrap(); @@ -594,4 +616,34 @@ mod tests { Ok(()) } + + /// Blocks cut across input chunks keep the definedness on each of their rows. + #[test] + fn exact_blocks_carry_the_definedness_on_their_rows() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let mut buf = ChunksBuffer::new(0, 3); + buf.push_back( + buffer![1i32, 2].into_array(), + Mask::from_iter([true, false]), + &mut ctx, + )?; + buf.push_back( + buffer![3i32, 4, 5, 6].into_array(), + Mask::from_iter([false, true, true, false]), + &mut ctx, + )?; + + let blocks = buf.collect_exact_blocks(&mut ctx)?; + let defined_masks: Vec<_> = blocks.iter().map(|(_, defined)| defined.clone()).collect(); + assert_eq!( + Mask::concat(defined_masks.iter())?, + Mask::from_iter([true, false, false, true, true, false]) + ); + assert!( + blocks + .iter() + .all(|(block, defined)| block.len() == defined.len()) + ); + Ok(()) + } } diff --git a/vortex-layout/src/layouts/struct_/writer.rs b/vortex-layout/src/layouts/struct_/writer.rs index b842c121168..f74d6fcbdf8 100644 --- a/vortex-layout/src/layouts/struct_/writer.rs +++ b/vortex-layout/src/layouts/struct_/writer.rs @@ -21,7 +21,6 @@ use futures::future::join; use futures::future::try_join_all; use futures::pin_mut; use itertools::Itertools; -use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::StructArray; @@ -44,10 +43,11 @@ use crate::LayoutWriterContext; use crate::layouts::struct_::StructLayout; use crate::segments::SegmentSinkRef; use crate::sequence::SendableSequentialStream; -use crate::sequence::SequenceId; use crate::sequence::SequencePointer; +use crate::sequence::SequentialChunk; use crate::sequence::SequentialStreamAdapter; use crate::sequence::SequentialStreamExt; +use crate::sequence::fully_defined; /// Writes struct-typed arrays into a [`StructLayout`], one child layout per field. /// @@ -129,10 +129,9 @@ impl LayoutStrategy for StructStrategy { // StructLayout. if struct_dtype.nfields() == 0 && !is_nullable { let row_count = stream - .try_fold( - 0u64, - |acc, (_, arr)| async move { Ok(acc + arr.len() as u64) }, - ) + .try_fold(0u64, |acc, (_, arr, _)| async move { + Ok(acc + arr.len() as u64) + }) .await?; return Ok(StructLayout::new(row_count, dtype, vec![]).into_layout()); } @@ -140,26 +139,37 @@ impl LayoutStrategy for StructStrategy { // stream -> stream> let columns_session = session.clone(); let columns_vec_stream = stream.map(move |chunk| { - let (sequence_id, chunk) = chunk?; + let (sequence_id, chunk, defined) = chunk?; let mut sequence_pointer = sequence_id.descend(); let mut ctx = columns_session.create_execution_ctx(); let struct_chunk = chunk.clone().execute::(&mut ctx)?; - let mut columns: Vec<(SequenceId, ArrayRef)> = Vec::new(); - if is_nullable { - columns.push(( - sequence_pointer.advance(), - chunk - .validity()? - .execute_mask(chunk.len(), &mut ctx)? - .into_array(), - )); - } + let mut columns: Vec = Vec::new(); + let validity = is_nullable + .then(|| chunk.validity()?.execute_mask(chunk.len(), &mut ctx)) + .transpose()?; + + // Fields are written unmasked, so each field is defined only where its ancestors + // define the struct and the struct is valid. Every row of the validity is part of + // the column. + let field_definedness = match &validity { + Some(validity) => { + let field_definedness = &defined & validity; + columns.push(fully_defined( + sequence_pointer.advance(), + validity.clone().into_array(), + )); + field_definedness + } + None => defined, + }; - columns.extend( - struct_chunk - .iter_unmasked_fields() - .map(|field| (sequence_pointer.advance(), field.clone())), - ); + columns.extend(struct_chunk.iter_unmasked_fields().map(|field| { + ( + sequence_pointer.advance(), + field.clone(), + field_definedness.clone(), + ) + })); Ok(columns) }); @@ -227,14 +237,7 @@ impl LayoutStrategy for StructStrategy { let child_eof = eof.split_off(); let session = session.clone(); let is_validity = index == 0 && is_nullable; - // Fields are written unmasked, so a nullable struct hides some of their values - // behind its own validity. Tell the field writers so they omit statistics that - // those hidden values would corrupt. - let ctx = if is_nullable && !is_validity { - ctx.clone().with_nullable_ancestor() - } else { - ctx.clone() - }; + let ctx = ctx.clone(); let segment_sink = Arc::clone(&segment_sink); handle.spawn_nested(move |_| { // Validity is written through the validity strategy; every other field @@ -274,8 +277,13 @@ mod tests { use std::num::NonZeroUsize; use vortex_array::ArrayContext; + use vortex_array::ArrayRef; + use vortex_array::aggregate_fn::AggregateFnVTableExt; + use vortex_array::aggregate_fn::fns::max::Max; + use vortex_array::aggregate_fn::fns::min::Min; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ChunkedArray; + use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexExpect; @@ -290,103 +298,112 @@ mod tests { use crate::layouts::zoned::writer::ZonedLayoutOptions; use crate::layouts::zoned::writer::ZonedStrategy; use crate::segments::TestSegments; + use crate::sequence::SequenceId; use crate::sequence::SequentialArrayStreamExt; use crate::test::FailingStrategy; use crate::test::SESSION; use crate::test::new_session; + use crate::test::read_zone_stats; + + fn zoned_leaf() -> Arc { + Arc::new(ZonedStrategy::new( + ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()), + FlatLayoutStrategy::default(), + ZonedLayoutOptions { + block_size: NonZeroUsize::new(3).vortex_expect("non zero"), + ..Default::default() + }, + )) + } - /// Write a single-field struct with zoned fields, returning whether the field got a zone map. - fn field_has_zone_map(validity: Validity) -> VortexResult { - let strategy = StructStrategy::new( - Arc::new(FlatLayoutStrategy::default()), - Arc::new(ZonedStrategy::new( - ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()), - FlatLayoutStrategy::default(), - ZonedLayoutOptions { - block_size: NonZeroUsize::new(3).vortex_expect("non zero"), - ..Default::default() - }, - )), - ); - let is_nullable = validity.nullability().is_nullable(); + /// Write `array` through `strategy`, returning the layout and the segments it was written to. + fn write( + strategy: S, + array: ArrayRef, + ) -> VortexResult<(LayoutRef, Arc)> { let (ptr, eof) = SequenceId::root().split(); - let stream = StructArray::try_from_iter_with_validity( - [("a", buffer![1, 2, 3].into_array())], - validity, - )? - .into_array() - .to_array_stream() - .sequenced(ptr); - + let stream = array.to_array_stream().sequenced(ptr); + let segments = Arc::new(TestSegments::default()); + let segments2 = Arc::clone(&segments); let layout = block_on(|handle| async move { let session = new_session().with_handle(handle); strategy .write_stream( LayoutWriterContext::new(ArrayContext::empty()), - Arc::new(TestSegments::default()), + segments2, stream, eof, &session, ) .await })?; + Ok((layout, segments)) + } + /// Write a single-field struct of `[1, 2, 3]` with zoned fields, returning the field's layout. + fn write_field(validity: Validity) -> VortexResult<(LayoutRef, Arc)> { + let strategy = StructStrategy::new(Arc::new(FlatLayoutStrategy::default()), zoned_leaf()); + let is_nullable = validity.nullability().is_nullable(); + let array = StructArray::try_from_iter_with_validity( + [("a", buffer![1, 2, 3].into_array())], + validity, + )?; + let (layout, segments) = write(strategy, array.into_array())?; // The validity child, when present, precedes the fields. - Ok(layout.children()?[usize::from(is_nullable)].is::()) + Ok(( + Arc::clone(&layout.children()?[usize::from(is_nullable)]), + segments, + )) } - /// Write `array` through a [`TableStrategy`] with zoned leaves, returning the layout. - fn write_table(array: StructArray) -> VortexResult { - let strategy = TableStrategy::new( - Arc::new(FlatLayoutStrategy::default()), - Arc::new(ZonedStrategy::new( - ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()), - FlatLayoutStrategy::default(), - ZonedLayoutOptions { - block_size: NonZeroUsize::new(3).vortex_expect("non zero"), - ..Default::default() - }, - )), - ); - let (ptr, eof) = SequenceId::root().split(); - let stream = array.into_array().to_array_stream().sequenced(ptr); - block_on(|handle| async move { - let session = new_session().with_handle(handle); - strategy - .write_stream( - LayoutWriterContext::new(ArrayContext::empty()), - Arc::new(TestSegments::default()), - stream, - eof, - &session, - ) - .await - }) + /// Write `array` through a [`TableStrategy`] with zoned leaves. + fn write_table(array: StructArray) -> VortexResult<(LayoutRef, Arc)> { + let strategy = TableStrategy::new(Arc::new(FlatLayoutStrategy::default()), zoned_leaf()); + write(strategy, array.into_array()) + } + + /// The minimum and maximum a zoned leaf of at most three rows records for its single zone. + fn zone_bounds(leaf: &LayoutRef, segments: &Arc) -> VortexResult<(i32, i32)> { + assert!(leaf.is::(), "{leaf} has no zone map"); + let mut ctx = new_session().create_execution_ctx(); + let mut bound = |aggregate_fn| -> VortexResult { + let scalar: Scalar = + read_zone_stats(leaf, segments, &aggregate_fn)?.execute_scalar(0, &mut ctx)?; + Ok(scalar + .as_primitive() + .typed_value::() + .vortex_expect("defined zone has a bound")) + }; + Ok(( + bound(Min.bind(Default::default()))?, + bound(Max.bind(Default::default()))?, + )) + } + + fn some_nulls() -> Validity { + Validity::Array(BoolArray::from_iter([false, true, true]).into_array()) } #[test] fn non_nullable_struct_fields_get_a_zone_map() -> VortexResult<()> { - assert!(field_has_zone_map(Validity::NonNullable)?); + let (field, segments) = write_field(Validity::NonNullable)?; + assert_eq!(zone_bounds(&field, &segments)?, (1, 3)); Ok(()) } - /// The field holds values in rows the struct nulls out, so no aggregate over it is exact for - /// the logical column and no zone map is written. + /// The field holds a value in the row the struct nulls out, and its zone map leaves it out. #[test] - fn nullable_struct_fields_get_no_zone_map() -> VortexResult<()> { - assert!(!field_has_zone_map(Validity::Array( - BoolArray::from_iter([false, true, true]).into_array(), - ))?); + fn nullable_struct_fields_zone_map_only_valid_rows() -> VortexResult<()> { + let (field, segments) = write_field(some_nulls())?; + assert_eq!(zone_bounds(&field, &segments)?, (2, 3)); Ok(()) } /// A nullable struct anywhere above a leaf hides its values, however deep the leaf sits, while /// leaves beside a nullable struct are unaffected. #[test] - fn nullable_ancestor_propagates_through_nested_structs() -> VortexResult<()> { - let some_nulls = || Validity::Array(BoolArray::from_iter([false, true, true]).into_array()); - - // Nullable outer struct, non-nullable inner struct: the inner leaf is still hidden. + fn definedness_propagates_through_nested_structs() -> VortexResult<()> { + // Nullable outer struct, non-nullable inner struct: the inner leaf still skips row 0. let outer_nullable = StructArray::try_from_iter_with_validity( [( "inner", @@ -394,12 +411,12 @@ mod tests { )], some_nulls(), )?; - let layout = write_table(outer_nullable)?; + let (layout, segments) = write_table(outer_nullable)?; let inner = &layout.children()?[1]; - assert!(!inner.children()?[0].is::()); + assert_eq!(zone_bounds(&inner.children()?[0], &segments)?, (2, 3)); - // Non-nullable outer struct: a leaf beside the nullable inner struct keeps its zone map, - // the leaf below it loses it. + // Non-nullable outer struct: a leaf beside the nullable inner struct covers every row, + // the leaf below it skips row 0. let inner_nullable = StructArray::try_from_iter([ ("sibling", buffer![1, 2, 3].into_array()), ( @@ -411,10 +428,55 @@ mod tests { .into_array(), ), ])?; - let layout = write_table(inner_nullable)?; + let (layout, segments) = write_table(inner_nullable)?; let children = layout.children()?; - assert!(children[0].is::()); - assert!(!children[1].children()?[1].is::()); + assert_eq!(zone_bounds(&children[0], &segments)?, (1, 3)); + assert_eq!(zone_bounds(&children[1].children()?[1], &segments)?, (2, 3)); + Ok(()) + } + + /// The definedness on a leaf under two nullable structs is the rows where both are valid. + #[test] + fn nested_nullable_structs_compose_their_validity() -> VortexResult<()> { + let inner = StructArray::try_from_iter_with_validity( + [("a", buffer![1, 2, 3].into_array())], + some_nulls(), + )?; + let outer = StructArray::try_from_iter_with_validity( + [("inner", inner.into_array())], + Validity::Array(BoolArray::from_iter([true, true, false]).into_array()), + )?; + let (layout, segments) = write_table(outer)?; + let leaf = &layout.children()?[1].children()?[1]; + assert_eq!(zone_bounds(leaf, &segments)?, (2, 2)); + Ok(()) + } + + /// A chunked writer below the struct writes each chunk through its own child, and hands each + /// child the definedness on its chunk. + #[test] + fn chunked_fields_take_each_chunks_definedness() -> VortexResult<()> { + let chunk = |values: ArrayRef, validity: [bool; 3]| -> VortexResult { + Ok(StructArray::try_from_iter_with_validity( + [("a", values)], + Validity::Array(BoolArray::from_iter(validity).into_array()), + )? + .into_array()) + }; + let array = ChunkedArray::from_iter([ + chunk(buffer![1, 2, 3].into_array(), [false, true, true])?, + chunk(buffer![10, 20, 30].into_array(), [true, false, true])?, + ]) + .into_array(); + let strategy = StructStrategy::new( + Arc::new(ChunkedLayoutStrategy::new(FlatLayoutStrategy::default())), + Arc::new(ChunkedLayoutStrategy::new(zoned_leaf())), + ); + + let (layout, segments) = write(strategy, array)?; + let chunks = layout.children()?[1].children()?; + assert_eq!(zone_bounds(&chunks[0], &segments)?, (2, 3)); + assert_eq!(zone_bounds(&chunks[1], &segments)?, (10, 30)); Ok(()) } @@ -432,25 +494,24 @@ mod tests { }) } - /// The flag survives any number of non-nullable struct levels below the nullable one, and is - /// only set at the nullable level, not above it. + /// The definedness survives any number of non-nullable struct levels below the nullable one, and + /// only applies below the nullable level, not above it. #[test] - fn nullable_ancestor_propagates_through_deep_nesting() -> VortexResult<()> { + fn definedness_propagates_through_deep_nesting() -> VortexResult<()> { const DEPTH: usize = 6; - let some_nulls = || Validity::Array(BoolArray::from_iter([false, true, true]).into_array()); // Nullable root over DEPTH non-nullable levels. let root = StructArray::try_from_iter_with_validity( [("top", nest(buffer![1, 2, 3].into_array(), DEPTH)?)], some_nulls(), )?; - let layout = write_table(root)?; + let (layout, segments) = write_table(root)?; // Skip the root's validity child, then walk every non-nullable level to the leaf. let leaf = descend(&layout.children()?[1], DEPTH)?; - assert!(!leaf.is::()); + assert_eq!(zone_bounds(&leaf, &segments)?, (2, 3)); // Non-nullable levels above, a nullable struct in the middle, non-nullable levels below. - // A leaf above the nullable level keeps its zone map, the leaf below it does not. + // A leaf above the nullable level covers every row, the leaf below it skips row 0. let middle = StructArray::try_from_iter_with_validity( [("below", nest(buffer![1, 2, 3].into_array(), DEPTH)?)], some_nulls(), @@ -464,13 +525,13 @@ mod tests { DEPTH, )? .execute::(&mut new_session().create_execution_ctx())?; - let layout = write_table(root)?; + let (layout, segments) = write_table(root)?; let holder = descend(&layout, DEPTH)?; let holder_children = holder.children()?; - assert!(holder_children[0].is::()); + assert_eq!(zone_bounds(&holder_children[0], &segments)?, (1, 3)); // The nullable middle struct's children are `[validity, below]`. let leaf = descend(&holder_children[1].children()?[1], DEPTH)?; - assert!(!leaf.is::()); + assert_eq!(zone_bounds(&leaf, &segments)?, (2, 3)); Ok(()) } diff --git a/vortex-layout/src/layouts/table.rs b/vortex-layout/src/layouts/table.rs index 08575a80515..9dee58325c8 100644 --- a/vortex-layout/src/layouts/table.rs +++ b/vortex-layout/src/layouts/table.rs @@ -355,6 +355,7 @@ mod tests { use crate::segments::TestSegments; use crate::sequence::SequenceId; use crate::sequence::SequentialArrayStreamExt; + use crate::sequence::SequentialChunk; use crate::sequence::SequentialStreamAdapter; use crate::sequence::SequentialStreamExt; use crate::test::SESSION; @@ -724,10 +725,9 @@ mod tests { )]), Nullability::NonNullable, ); - let stream = - futures::stream::poll_fn(|_| -> Poll>> { - panic!("panic while transposing table stream"); - }); + let stream = futures::stream::poll_fn(|_| -> Poll>> { + panic!("panic while transposing table stream"); + }); let strategy = TableStrategy::new( Arc::new(FlatLayoutStrategy::default()), Arc::new(FlatLayoutStrategy::default()), diff --git a/vortex-layout/src/layouts/zoned/builder.rs b/vortex-layout/src/layouts/zoned/builder.rs index a5d3250310e..88fa917d2be 100644 --- a/vortex-layout/src/layouts/zoned/builder.rs +++ b/vortex-layout/src/layouts/zoned/builder.rs @@ -18,6 +18,7 @@ use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; +use vortex_mask::Mask; use crate::layouts::file_stats::contains_variant; use crate::layouts::zoned::schema::aggregate_state_dtype; @@ -122,6 +123,24 @@ pub(crate) fn aggregate_partials( .collect() } +/// Aggregate `array` as null wherever `defined` is false, e.g. a struct field as null wherever the +/// struct is null. +pub(crate) fn aggregate_defined_partials( + array: &ArrayRef, + defined: &Mask, + aggregate_fns: &[AggregateFnRef], + ctx: &mut ExecutionCtx, +) -> VortexResult> { + aggregate_fns + .iter() + .map(|aggregate_fn| { + let mut accumulator = aggregate_fn.accumulator(array.dtype())?; + accumulator.accumulate_defined(array, defined, ctx)?; + accumulator.partial_scalar() + }) + .collect() +} + struct AggregateStatsArrayBuilder { aggregate_fn: AggregateFnRef, dtype: DType, diff --git a/vortex-layout/src/layouts/zoned/mod.rs b/vortex-layout/src/layouts/zoned/mod.rs index e61f8588aa1..1d136f80b8a 100644 --- a/vortex-layout/src/layouts/zoned/mod.rs +++ b/vortex-layout/src/layouts/zoned/mod.rs @@ -24,6 +24,7 @@ use std::num::NonZeroUsize; use std::sync::Arc; pub(crate) use builder::AggregateStatsAccumulator; +pub(crate) use builder::aggregate_defined_partials; pub(crate) use builder::aggregate_partials; pub(crate) use legacy_builder::LegacyStatsAccumulator; pub(crate) use legacy_builder::legacy_stat_values; diff --git a/vortex-layout/src/layouts/zoned/writer.rs b/vortex-layout/src/layouts/zoned/writer.rs index 09044047e73..9b9ac7430ad 100644 --- a/vortex-layout/src/layouts/zoned/writer.rs +++ b/vortex-layout/src/layouts/zoned/writer.rs @@ -24,6 +24,7 @@ use vortex_array::aggregate_fn::fns::min::Min; use vortex_array::aggregate_fn::fns::nan_count::NanCount; use vortex_array::aggregate_fn::fns::null_count::NullCount; use vortex_array::aggregate_fn::session::AggregateFnSessionExt; +use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_error::VortexError; use vortex_error::VortexResult; @@ -42,6 +43,7 @@ use crate::layouts::zoned::LegacyStatsAccumulator; use crate::layouts::zoned::LegacyStatsLayout; use crate::layouts::zoned::Zoned; use crate::layouts::zoned::ZonedLayout; +use crate::layouts::zoned::aggregate_defined_partials; use crate::layouts::zoned::aggregate_partials; use crate::layouts::zoned::legacy_stat_values; use crate::layouts::zoned::schema::default_bounded_stat_max_bytes; @@ -115,21 +117,9 @@ impl LayoutStrategy for ZonedStrategy { .aggregate_fns .clone() .unwrap_or_else(|| default_zoned_aggregate_fns(stream.dtype(), session)); - // Fields of a nullable struct are written unmasked, so every aggregate over them sees - // values the logical column does not contain: counts come out wrong and min/max come out - // wider than the truth. Zone maps hold exact values only, so until the aggregates are - // computed over the masked values no zone map is written for such a field. - // - // TODO(joe): write zone maps for fields under a nullable struct. The struct validity has - // to travel with each chunk, since repartitioning below the struct writer re-chunks the - // stream, and the aggregates then run over `chunk.mask(validity)` while the data child - // still receives the unmasked chunk. - if ctx.has_nullable_ancestor() { - return self - .child - .write_stream(ctx, segment_sink, stream, eof, session) - .await; - } + // Fields of a nullable struct are written unmasked, so an aggregate over all their rows + // sees values the logical column does not contain. Each chunk carries the definedness on its + // rows, and each zone aggregates its logical column, null in the undefined rows. // Readers before `vortex.zoned` only know the legacy `vortex.stats` layout. If the write // does not permit either layout, the column gets no zone map. @@ -170,13 +160,14 @@ impl LayoutStrategy for ZonedStrategy { let aggregate_fns = Arc::clone(&aggregate_fns); let session = compute_session.clone(); session.handle().spawn_cpu(move || { - let (sequence_id, chunk) = item?; - let partials = aggregate_partials( - &chunk, - &aggregate_fns, - &mut session.create_execution_ctx(), - )?; - Ok::<_, VortexError>((sequence_id, chunk, partials)) + let (sequence_id, chunk, defined) = item?; + let mut ctx = session.create_execution_ctx(); + let partials = if defined.all_true() { + aggregate_partials(&chunk, &aggregate_fns, &mut ctx)? + } else { + aggregate_defined_partials(&chunk, &defined, &aggregate_fns, &mut ctx)? + }; + Ok::<_, VortexError>((sequence_id, chunk, defined, partials)) }) }) .buffered(concurrency); @@ -187,9 +178,11 @@ impl LayoutStrategy for ZonedStrategy { let stream = SequentialStreamAdapter::new( stream_dtype, stream.map(move |item| { - let (sequence_id, chunk, partials) = item?; + let (sequence_id, chunk, defined, partials) = item?; stats_accumulator2.lock().push_partials(partials)?; - Ok((sequence_id, chunk)) + // The data child receives the same rows, and their definedness, e.g. for a list + // writer to drop the elements of undefined lists. + Ok((sequence_id, chunk, defined)) }), ) .sendable(); @@ -258,10 +251,18 @@ impl ZonedStrategy { let stats = Arc::clone(&stats); let session = compute_session.clone(); session.handle().spawn_cpu(move || { - let (sequence_id, chunk) = item?; + let (sequence_id, chunk, defined) = item?; + // The legacy stats all describe the logical values, which are null in the + // undefined rows. The masked chunk is temporary, so the stats it caches are + // not cached on the stored chunk. + let logical = if defined.all_true() { + chunk.clone() + } else { + chunk.clone().mask(defined.clone().into_array())? + }; let values = - legacy_stat_values(&chunk, &stats, &mut session.create_execution_ctx())?; - Ok::<_, VortexError>((sequence_id, chunk, values)) + legacy_stat_values(&logical, &stats, &mut session.create_execution_ctx())?; + Ok::<_, VortexError>((sequence_id, chunk, defined, values)) }) }) .buffered(concurrency); @@ -272,9 +273,9 @@ impl ZonedStrategy { let stream = SequentialStreamAdapter::new( stream_dtype, stream.map(move |item| { - let (sequence_id, chunk, values) = item?; + let (sequence_id, chunk, defined, values) = item?; stats_accumulator2.lock().push_values(values)?; - Ok((sequence_id, chunk)) + Ok((sequence_id, chunk, defined)) }), ) .sendable(); @@ -353,6 +354,7 @@ mod tests { use rstest::rstest; use vortex_array::ArrayContext; + use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::bounded_max::BoundedMax; @@ -360,10 +362,12 @@ mod tests { use vortex_array::aggregate_fn::fns::max::Max; use vortex_array::aggregate_fn::fns::min::Min; use vortex_array::aggregate_fn::fns::sum::Sum; - use vortex_array::arrays::ChunkedArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; + use vortex_array::expr::stats::Stat; use vortex_array::extension::datetime::TimeUnit; use vortex_array::extension::datetime::Timestamp; use vortex_buffer::buffer; @@ -372,6 +376,7 @@ mod tests { use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSession; use vortex_io::session::RuntimeSessionExt; + use vortex_mask::Mask; use vortex_utils::aliases::hash_set::HashSet; use super::*; @@ -382,8 +387,9 @@ mod tests { use crate::layouts::zoned::schema::aggregate_state_dtype; use crate::segments::TestSegments; use crate::sequence::SequenceId; - use crate::sequence::SequentialArrayStreamExt; use crate::session::LayoutSession; + use crate::test::read_zone_field; + use crate::test::read_zone_stats; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); @@ -416,6 +422,20 @@ mod tests { /// Write three zones of primitives through `ctx`, returning the resulting layout. fn write_layout(ctx: LayoutWriterContext) -> VortexResult { + let chunks = [ + buffer![1, 2, 3].into_array(), + buffer![4, 5, 6].into_array(), + buffer![7, 8, 9].into_array(), + ]; + Ok(write_chunks(ctx, chunks.map(|chunk| (chunk, Mask::new_true(3))))?.0) + } + + /// Write `chunks`, each with the definedness on its rows, as zones of three rows through `ctx`, + /// returning the resulting layout and the segments it was written to. + fn write_chunks( + ctx: LayoutWriterContext, + chunks: impl IntoIterator, + ) -> VortexResult<(LayoutRef, Arc)> { let strategy = ZonedStrategy::new( ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()), FlatLayoutStrategy::default(), @@ -424,31 +444,117 @@ mod tests { ..Default::default() }, ); - let (ptr, eof) = SequenceId::root().split(); - let stream = ChunkedArray::from_iter([ - buffer![1, 2, 3].into_array(), - buffer![4, 5, 6].into_array(), - buffer![7, 8, 9].into_array(), - ]) - .into_array() - .to_array_stream() - .sequenced(ptr); - - block_on(|handle: Handle| async move { + let (mut ptr, eof) = SequenceId::root().split(); + let chunks: Vec<_> = chunks.into_iter().collect(); + let dtype = chunks[0].0.dtype().clone(); + let chunks: Vec<_> = chunks + .into_iter() + .map(|(chunk, defined)| Ok((ptr.advance(), chunk, defined))) + .collect(); + let stream = SequentialStreamAdapter::new(dtype, futures::stream::iter(chunks)).sendable(); + + let segments = Arc::new(TestSegments::default()); + let segments2 = Arc::clone(&segments); + let layout = block_on(|handle: Handle| async move { let session = VortexSession::empty() .with::() .with::() .with_handle(handle); strategy - .write_stream( - ctx, - Arc::new(TestSegments::default()), - stream, - eof, - &session, - ) + .write_stream(ctx, segments2, stream, eof, &session) .await - }) + })?; + Ok((layout, segments)) + } + + /// Under a nullable ancestor, each zone aggregates its logical column, null in the rows the + /// ancestor nulls out, so undefined values never widen the zone's bounds and count as nulls. + #[test] + fn defined_zones_aggregate_only_defined_rows() -> VortexResult<()> { + let (layout, segments) = write_chunks( + LayoutWriterContext::new(ArrayContext::empty()), + [ + ( + PrimitiveArray::from_option_iter([Some(1), Some(2), Some(3)]).into_array(), + Mask::new_true(3), + ), + ( + PrimitiveArray::from_option_iter([Some(-40), None, Some(6)]).into_array(), + Mask::from_iter([false, true, true]), + ), + ( + PrimitiveArray::from_option_iter([Some(7), Some(80), None]).into_array(), + Mask::from_iter([true, false, false]), + ), + ], + )?; + assert!(layout.is::()); + + let mut exec_ctx = VortexSession::empty().create_execution_ctx(); + let min = read_zone_stats(&layout, &segments, &Min.bind(Default::default()))?; + assert_arrays_eq!( + min, + PrimitiveArray::from_option_iter([Some(1i32), Some(6), Some(7)]), + &mut exec_ctx + ); + let max = read_zone_stats(&layout, &segments, &Max.bind(Default::default()))?; + assert_arrays_eq!( + max, + PrimitiveArray::from_option_iter([Some(3i32), Some(6), Some(7)]), + &mut exec_ctx + ); + // Undefined rows are null: the second zone has its own null and one undefined row, the + // third zone two undefined rows, one of which is also null. + let null_count = read_zone_stats(&layout, &segments, &NullCount.bind(EmptyOptions))?; + assert_arrays_eq!( + null_count, + PrimitiveArray::from_option_iter([Some(0u64), Some(2), Some(2)]), + &mut exec_ctx + ); + Ok(()) + } + + /// A write that permits only the legacy stats layout records the logical values too. + #[test] + fn legacy_zones_aggregate_only_defined_rows() -> VortexResult<()> { + let ctx = LayoutWriterContext::new(ArrayContext::empty()) + .with_allowed_layouts(HashSet::from_iter([LegacyStats.id()])); + let (layout, segments) = write_chunks( + ctx, + [ + ( + PrimitiveArray::from_option_iter([Some(1), Some(2), Some(3)]).into_array(), + Mask::new_true(3), + ), + ( + PrimitiveArray::from_option_iter([Some(-40), None, Some(6)]).into_array(), + Mask::from_iter([false, true, true]), + ), + ( + PrimitiveArray::from_option_iter([Some(7), Some(80), None]).into_array(), + Mask::from_iter([true, false, false]), + ), + ], + )?; + assert!(layout.is::()); + + let mut exec_ctx = VortexSession::empty().create_execution_ctx(); + assert_arrays_eq!( + read_zone_field(&layout, &segments, Stat::Min.name())?, + PrimitiveArray::from_option_iter([Some(1i32), Some(6), Some(7)]), + &mut exec_ctx + ); + assert_arrays_eq!( + read_zone_field(&layout, &segments, Stat::Max.name())?, + PrimitiveArray::from_option_iter([Some(3i32), Some(6), Some(7)]), + &mut exec_ctx + ); + assert_arrays_eq!( + read_zone_field(&layout, &segments, Stat::NullCount.name())?, + PrimitiveArray::from_option_iter([Some(0u64), Some(2), Some(2)]), + &mut exec_ctx + ); + Ok(()) } #[test] @@ -463,16 +569,6 @@ mod tests { Ok(()) } - /// A field of a nullable struct is written unmasked, so no aggregate over it is exact and no - /// zone map is written. - #[test] - fn a_nullable_ancestor_writes_no_zone_map() -> VortexResult<()> { - let layout = - write_layout(LayoutWriterContext::new(ArrayContext::empty()).with_nullable_ancestor())?; - assert!(!layout.is::()); - Ok(()) - } - #[test] fn a_permitted_set_covering_the_defaults_writes_them() -> VortexResult<()> { let ctx = LayoutWriterContext::new(ArrayContext::empty()).with_allowed_aggregates( diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index ea88dd5ba40..c23c80cba38 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -84,6 +84,7 @@ use crate::segments::TestSegments; use crate::sequence::SequenceId; use crate::sequence::SequentialStreamAdapter; use crate::sequence::SequentialStreamExt as _; +use crate::sequence::fully_defined; use crate::test::new_session; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); @@ -137,7 +138,12 @@ async fn write_chunks( let segments = Arc::new(TestSegments::default()); let (mut sequence_id, eof) = SequenceId::root().split(); let chunks = (0..count) - .map(|chunk| Ok((sequence_id.advance(), table_chunk(chunk * CHUNK_ROWS)?))) + .map(|chunk| { + Ok(fully_defined( + sequence_id.advance(), + table_chunk(chunk * CHUNK_ROWS)?, + )) + }) .collect::>(); let dtype = table_chunk(0)?.dtype().clone(); let layout = strategy @@ -364,7 +370,7 @@ async fn large_filtered_chunks_match_v1(#[case] cutoff: i32) -> VortexResult<()> Arc::::clone(&segments), SequentialStreamAdapter::new( dtype.clone(), - stream::iter([Ok((sequence.advance(), values))]), + stream::iter([Ok(fully_defined(sequence.advance(), values))]), ) .sendable(), eof, @@ -583,8 +589,15 @@ async fn dictionary_values_are_reused_between_splits() -> VortexResult<()> { .write_stream( ArrayContext::empty().into(), Arc::::clone(&segments), - SequentialStreamAdapter::new(dtype, stream::iter(arrays.into_iter().map(Ok))) - .sendable(), + SequentialStreamAdapter::new( + dtype, + stream::iter( + arrays + .into_iter() + .map(|(id, array)| Ok(fully_defined(id, array))), + ), + ) + .sendable(), eof, &session, ) @@ -744,7 +757,7 @@ async fn chunked_ints( let (mut sequence, eof) = SequenceId::root().split(); let arrays = (0..3i32) .map(|index| { - Ok(( + Ok(fully_defined( sequence.advance(), PrimitiveArray::from_iter(index * 10..index * 10 + 10).into_array(), )) diff --git a/vortex-layout/src/sequence.rs b/vortex-layout/src/sequence.rs index bd531ba6612..f46248de28d 100644 --- a/vortex-layout/src/sequence.rs +++ b/vortex-layout/src/sequence.rs @@ -22,6 +22,7 @@ use vortex_array::dtype::DType; use vortex_array::stream::ArrayStream; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_mask::Mask; use vortex_utils::aliases::hash_map::HashMap; /// A hierarchical sequence identifier that exists within a shared universe. @@ -285,7 +286,24 @@ impl Drop for WaitSequenceFuture<'_> { } } -pub trait SequentialStream: Stream> { +/// A chunk of a [`SequentialStream`]: its position, its rows, and the definedness on its rows. +/// +/// Container writers store their children unmasked: a struct field holds a value even in rows +/// where the struct is null. The definedness is false in the rows an ancestor nulls out, so that +/// writers recording statistics aggregate the logical column, null in those rows (see +/// [`DynAccumulator::accumulate_defined`]). It has the length of the chunk, and is all true +/// unless an ancestor nulls rows out. +/// +/// [`DynAccumulator::accumulate_defined`]: vortex_array::aggregate_fn::DynAccumulator::accumulate_defined +pub type SequentialChunk = (SequenceId, ArrayRef, Mask); + +/// A chunk whose every row is defined. +pub fn fully_defined(sequence_id: SequenceId, array: ArrayRef) -> SequentialChunk { + let defined = Mask::new_true(array.len()); + (sequence_id, array, defined) +} + +pub trait SequentialStream: Stream> { fn dtype(&self) -> &DType; } @@ -325,7 +343,7 @@ impl SequentialStreamAdapter { impl SequentialStream for SequentialStreamAdapter where - S: Stream>, + S: Stream>, { fn dtype(&self) -> &DType { &self.dtype @@ -334,14 +352,14 @@ where impl Stream for SequentialStreamAdapter where - S: Stream>, + S: Stream>, { - type Item = VortexResult<(SequenceId, ArrayRef)>; + type Item = VortexResult; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.project(); let array = futures::ready!(this.inner.poll_next(cx)); - if let Some(Ok((_, array))) = array.as_ref() { + if let Some(Ok((_, array, defined))) = array.as_ref() { assert_eq!( array.dtype(), this.dtype, @@ -349,6 +367,11 @@ where array.dtype(), this.dtype ); + debug_assert_eq!( + defined.len(), + array.len(), + "Definedness must have the length of its chunk" + ); } Poll::Ready(array) @@ -368,7 +391,7 @@ pub trait SequentialArrayStreamExt: ArrayStream { Box::pin(SequentialStreamAdapter::new( self.dtype().clone(), StreamExt::map(self, move |item| { - item.map(|array| (pointer.advance(), array)) + item.map(|array| fully_defined(pointer.advance(), array)) }), )) } diff --git a/vortex-layout/src/strategy.rs b/vortex-layout/src/strategy.rs index 3e5bcedd5ec..000e5f8335c 100644 --- a/vortex-layout/src/strategy.rs +++ b/vortex-layout/src/strategy.rs @@ -77,7 +77,6 @@ pub struct LayoutWriterContext { array_ctx: ArrayContext, allowed_aggregates: Option>>, allowed_layouts: Option>>, - nullable_ancestor: bool, buffered_bytes: BufferedBytesTracker, } @@ -88,7 +87,6 @@ impl LayoutWriterContext { array_ctx, allowed_aggregates: None, allowed_layouts: None, - nullable_ancestor: false, buffered_bytes: BufferedBytesTracker::new(), } } @@ -130,23 +128,6 @@ impl LayoutWriterContext { .is_none_or(|allowed| allowed.contains(layout)) } - /// Marks this write as nested under a nullable container, such as a field of a nullable - /// struct. - /// - /// Container writers store their children unmasked: a field holds a value even in rows where - /// the containing struct is null, and the struct's validity is written separately. Statistics - /// computed over such a child therefore see values the logical column does not contain, so - /// writers that record statistics must omit them rather than record inexact values. - pub fn with_nullable_ancestor(mut self) -> Self { - self.nullable_ancestor = true; - self - } - - /// Returns whether a nullable container above this write may hide some of its values. - pub fn has_nullable_ancestor(&self) -> bool { - self.nullable_ancestor - } - /// Replaces the buffered bytes tracker, so callers can observe the counter from outside the /// strategy tree. pub fn with_buffered_bytes_tracker(mut self, tracker: BufferedBytesTracker) -> Self { diff --git a/vortex-layout/src/test.rs b/vortex-layout/src/test.rs index 29eeb8c3eba..aaac4a388c8 100644 --- a/vortex-layout/src/test.rs +++ b/vortex-layout/src/test.rs @@ -1,12 +1,23 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::sync::Arc; use std::sync::LazyLock; +use vortex_array::ArrayRef; +use vortex_array::MaskFuture; +use vortex_array::aggregate_fn::AggregateFnRef; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::expr::root; +use vortex_error::VortexResult; +use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSession; use vortex_io::session::RuntimeSessionExt; use vortex_session::VortexSession; +use crate::LayoutRef; +use crate::segments::SegmentSource; +use crate::segments::TestSegments; use crate::session::LayoutSession; pub static SESSION: LazyLock = LazyLock::new(|| new_session().with_tokio()); @@ -34,7 +45,39 @@ impl crate::LayoutStrategy for FailingStrategy { _stream: crate::sequence::SendableSequentialStream, _eof: crate::sequence::SequencePointer, _session: &VortexSession, - ) -> vortex_error::VortexResult { + ) -> VortexResult { vortex_error::vortex_bail!("{}", Self::MESSAGE) } } + +/// Read the zone map column that a zoned `layout` stores for `aggregate_fn`, one partial per zone. +pub fn read_zone_stats( + layout: &LayoutRef, + segments: &Arc, + aggregate_fn: &AggregateFnRef, +) -> VortexResult { + read_zone_field(layout, segments, &aggregate_fn.to_string()) +} + +/// Read the zone table column `name` of a zoned layout, such as a legacy statistic's name. +pub fn read_zone_field( + layout: &LayoutRef, + segments: &Arc, + name: &str, +) -> VortexResult { + let zones = Arc::clone(&layout.children()?[1]); + let segments: Arc = Arc::::clone(segments); + let row_count = zones.row_count(); + let table = block_on(|handle| async move { + let session = new_session().with_handle(handle); + let reader = zones.new_reader("".into(), segments, &session, &Default::default())?; + reader + .projection_evaluation( + &(0..row_count), + &root().bind(reader.dtype())?, + MaskFuture::new_true(usize::try_from(row_count)?), + )? + .await + })?; + table.get_item(name) +} diff --git a/vortex-sqllogictest/slt/duckdb/copy_return_stats.slt b/vortex-sqllogictest/slt/duckdb/copy_return_stats.slt index ca06123f805..c912b31a991 100644 --- a/vortex-sqllogictest/slt/duckdb/copy_return_stats.slt +++ b/vortex-sqllogictest/slt/duckdb/copy_return_stats.slt @@ -12,10 +12,9 @@ TO '${WORK_DIR}/copy_return_stats.vortex' (FORMAT vortex, RETURN_STATS); # As in parquet, statistics are reported per leaf column: struct fields are keyed by their full # quoted path and the struct itself gets no entry. Lists are reported as a single column. -# DuckDB structs are nullable, so their fields report no min, max or null count until struct -# nulls are propagated into children (https://github.com/vortex-data/vortex/issues/9205). +# DuckDB structs are nullable, and their fields' statistics treat the struct's null rows as null. query TIII??? COPY (SELECT {'a': 1, 'b': 2} AS st, [1, 2, 3] AS lst, 7 AS i) TO '${WORK_DIR}/copy_return_stats_nested.vortex' (FORMAT vortex, RETURN_STATS); ---- -:\$\{WORK_DIR\}/copy_return_stats_nested\.vortex 1 \d+ 0 \{'"i"'=\{column_size_bytes=\d+, max=7, min=7, null_count=0, num_values=1\}, '"lst"'=\{column_size_bytes=\d+, null_count=0, num_values=1\}, '"st"\."a"'=\{column_size_bytes=\d+, num_values=1\}, '"st"\."b"'=\{column_size_bytes=\d+, num_values=1\}\} NULL \{\} +:\$\{WORK_DIR\}/copy_return_stats_nested\.vortex 1 \d+ 0 \{'"i"'=\{column_size_bytes=\d+, max=7, min=7, null_count=0, num_values=1\}, '"lst"'=\{column_size_bytes=\d+, null_count=0, num_values=1\}, '"st"\."a"'=\{column_size_bytes=\d+, max=1, min=1, null_count=0, num_values=1\}, '"st"\."b"'=\{column_size_bytes=\d+, max=2, min=2, null_count=0, num_values=1\}\} NULL \{\}