diff --git a/vortex-datafusion/src/convert/stats/mod.rs b/vortex-datafusion/src/convert/stats/mod.rs index dfb11983f87..2cf8d70e126 100644 --- a/vortex-datafusion/src/convert/stats/mod.rs +++ b/vortex-datafusion/src/convert/stats/mod.rs @@ -18,13 +18,16 @@ use crate::PrecisionExt; use crate::convert::TryToDataFusion; /// Convert a stats set for an array with the given dtype. +/// +/// The uncompressed size is left out of `byte_size`. DataFusion's join selection picks +/// `CollectLeft` below `hash_join_single_partition_threshold` (1 MiB) when a byte size is known +/// and below `hash_join_single_partition_threshold_rows` (128 Ki rows) otherwise, so known sizes +/// turn small build sides into partitioned joins. DataFusion's `project_statistics` also keeps +/// the whole table's size for any projection with a variable-width column. pub(crate) fn stats_set_to_df( stats_set: &StatsSet, dtype: &DType, ) -> VortexResult { - // Update the total size in bytes. - let column_size = stats_set.get_as::(Stat::UncompressedSizeInBytes, &PType::U64.into()); - let null_count = stats_set .get_as::(Stat::NullCount, &PType::U64.into()) .to_df(); @@ -39,7 +42,7 @@ pub(crate) fn stats_set_to_df( max_value, sum_value: scalar_stat_to_df(stats_set, Stat::Sum, dtype).to_df(), distinct_count, - byte_size: column_size.to_df(), + byte_size: Precision::Absent, }) } diff --git a/vortex-datafusion/src/convert/stats/tests.rs b/vortex-datafusion/src/convert/stats/tests.rs index e2e7e0fc67f..bfb165b5e21 100644 --- a/vortex-datafusion/src/convert/stats/tests.rs +++ b/vortex-datafusion/src/convert/stats/tests.rs @@ -204,3 +204,17 @@ fn float_statistics_require_no_nans( assert_eq!(stats.sum_value, expected(3.0)); Ok(()) } + +#[test] +fn uncompressed_size_is_not_reported_as_byte_size() -> VortexResult<()> { + let stats = StatsSet::of( + Stat::UncompressedSizeInBytes, + VortexPrecision::exact(4096u64), + ); + let dtype = DType::Utf8(Nullability::NonNullable); + assert_eq!( + stats_set_to_df(&stats, &dtype)?.byte_size, + Precision::Absent + ); + Ok(()) +} diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index 1444b1879ae..fb50ed26d17 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -686,10 +686,6 @@ impl FileFormat for VortexFormat { } let stats_set = &stats_set; - // Update the total size in bytes. - let column_size = - stats_set.get_as::(Stat::UncompressedSizeInBytes, &PType::U64.into()); - let nan_free = is_nan_free(stats_set, stats_dtype); let bound = |stat| { if nan_free { @@ -713,21 +709,18 @@ impl FileFormat for VortexFormat { max_value: max, sum_value: DFPrecision::Absent, distinct_count, - byte_size: column_size.to_df(), + // See `stats_set_to_df` for why the uncompressed size is not reported. + byte_size: DFPrecision::Absent, }) } - let total_byte_size = column_statistics - .iter() - .fold(DFPrecision::Exact(0), |acc, cs| acc.add(&cs.byte_size)); - Ok(Statistics { num_rows: DFPrecision::Exact( usize::try_from(row_count) .map_err(|_| vortex_err!("Row count overflow")) .vortex_expect("Row count overflow"), ), - total_byte_size, + total_byte_size: DFPrecision::Absent, column_statistics, }) }) diff --git a/vortex-duckdb/src/column_statistics.rs b/vortex-duckdb/src/column_statistics.rs index 36d0ce9b3ab..6a3660ed73a 100644 --- a/vortex-duckdb/src/column_statistics.rs +++ b/vortex-duckdb/src/column_statistics.rs @@ -91,9 +91,10 @@ impl ColumnStatisticsAggregate { let max = exact(Max.bind(NumericalAggregateOpts::skip_nans())); // DuckDB's string length is u32 - #[allow(clippy::cast_possible_truncation)] - let max_string_length = exact(UncompressedSizeInBytes.bind(EmptyOptions)) - .map(|value| value.as_primitive().as_u64().vortex_expect("not a u64") as u32); + let max_string_length = + exact(UncompressedSizeInBytes.bind(EmptyOptions)).and_then(|value| { + u32::try_from(value.as_primitive().as_u64().vortex_expect("not a u64")).ok() + }); let has_null = exact(NullCount.bind(EmptyOptions)) .is_none_or(|count| count.as_primitive().as_u64().vortex_expect("not a u64") > 0); diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index 51386aca2d4..dcfcfeb7f6d 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -91,6 +91,7 @@ use vortex_edition::EDITION_DECLARATIONS; use vortex_edition::EditionSession; use vortex_edition::EditionSessionExt; use vortex_edition::declarations::core::CORE_2026_08_3; +use vortex_edition::declarations::preview::PREVIEW_2026_08_0; use vortex_edition::declarations::unstable::UNSTABLE; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -1587,6 +1588,64 @@ async fn exclude_legacy_statistics_empties_legacy_but_keeps_nested() -> VortexRe Ok(()) } +/// Both the nested and the legacy file statistics record the uncompressed size only when the +/// enabled editions include it, or when editions are disabled. +#[rstest] +#[case::core(false, false, false)] +#[case::preview(true, false, true)] +#[case::editions_disabled(false, true, true)] +#[tokio::test] +async fn uncompressed_size_follows_editions( + #[case] preview: bool, + #[case] disable_editions: bool, + #[case] recorded: bool, +) -> VortexResult<()> { + let session = VortexSession::empty() + .with::() + .with::() + .with::(); + crate::register_default_encodings(&session); + for declaration in EDITION_DECLARATIONS { + session.register_edition(declaration)?; + } + session.enable_edition(CORE_2026_08_3)?; + if preview { + session.enable_edition(PREVIEW_2026_08_0)?; + } + + let c = PrimitiveArray::from_iter([4i32, 5, 6]).into_array(); + let array = StructArray::try_new(["c"].into(), vec![c], 3, Validity::NonNullable)?.into_array(); + let options = session.write_options().include_nested_statistics(); + let options = if disable_editions { + options.disable_editions() + } else { + options + }; + let mut buf = ByteBufferMut::empty(); + let summary = options.write(&mut buf, array.to_array_stream()).await?; + + let three_i32 = recorded.then_some(12u64); + let stats = summary + .footer() + .statistics() + .expect("file statistics should be present"); + let (c_stats, _) = stats.get_by_path(&field_path!(c)).expect("c entry"); + assert_eq!( + stat_value(c_stats, Stat::UncompressedSizeInBytes), + three_i32.map_or(Precision::Absent, |size| Precision::exact( + ScalarValue::from(size) + )) + ); + + let bytes = stats.write_flatbuffer_bytes()?; + let fb = + flatbuffers::root::(bytes.as_ref()).vortex_expect("valid flatbuffer"); + let field_stats = fb.field_stats().vortex_expect("field_stats is written"); + assert_eq!(field_stats.get(0).uncompressed_size_in_bytes(), three_i32); + + Ok(()) +} + #[tokio::test] async fn nested_statistics_are_excluded_by_default() -> VortexResult<()> { let inner = StructArray::try_new( diff --git a/vortex-layout/src/layouts/file_stats.rs b/vortex-layout/src/layouts/file_stats.rs index ed86d163fb3..06a10742df8 100644 --- a/vortex-layout/src/layouts/file_stats.rs +++ b/vortex-layout/src/layouts/file_stats.rs @@ -24,6 +24,7 @@ use vortex_array::aggregate_fn::fns::max::Max; use vortex_array::aggregate_fn::fns::min::Min; use vortex_array::aggregate_fn::fns::nan_count::NanCount; use vortex_array::aggregate_fn::fns::null_count::NullCount; +use vortex_array::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeInBytes; use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; @@ -50,7 +51,7 @@ use crate::sequence::SequentialStreamExt; /// Accumulates file statistics over `stream` as it passes through. /// /// `stats` are the aggregates to record for every field, or `None` for each field's default (see -/// `default_pruning_aggregate_fns`). Only aggregates `ctx` allows are recorded in the nested +/// `default_aggregate_fns`). Only aggregates `ctx` allows are recorded in the nested /// statistics: an explicitly requested aggregate it forbids fails, like it does for zone maps, /// while forbidden defaults are left out. /// @@ -248,7 +249,13 @@ impl StatsAccumulator { } } - fn new(dtype: &DType, stats: Option<&[AggregateFnRef]>) -> Self { + /// Builds the accumulators for `stats`, or for the dtype's defaults when `None`, skipping any + /// aggregate `allowed` rejects so that it is not computed only to be dropped. + fn new( + dtype: &DType, + stats: Option<&[AggregateFnRef]>, + allowed: &dyn Fn(&AggregateFnRef) -> bool, + ) -> Self { if !supports_file_stats(dtype) { return Self::empty(); } @@ -260,13 +267,13 @@ impl StatsAccumulator { let stats = match stats { Some(stats) => stats, None => { - default_stats = default_pruning_aggregate_fns(dtype); + default_stats = default_aggregate_fns(dtype); default_stats.as_slice() } }; let mut aggregates = Vec::new(); - for aggregate_fn in stats { + for aggregate_fn in stats.iter().filter(|aggregate_fn| allowed(aggregate_fn)) { // A dtype that doesn't support a given aggregate is silently skipped, matching this // stat's absence from the result. Check the dtypes first: a failed `accumulator` call // builds an error, and an error captures a backtrace, which is expensive for wide @@ -328,11 +335,11 @@ fn is_varlen_dtype(dtype: &DType) -> bool { matches!(dtype, DType::Utf8(_) | DType::Binary(_)) } -/// Default file-level pruning aggregates for `dtype`, chosen the way `default_zoned_aggregate_fns` -/// picks zoned aggregates: a byte-bounded min/max (capped at [`default_bounded_stat_max_bytes`]) -/// for variable-length columns, and exact min/max otherwise. Callers wanting a different bound pass -/// their own `BoundedMax`/`BoundedMin` instead. -fn default_pruning_aggregate_fns(dtype: &DType) -> Vec { +/// Default file-level aggregates for `dtype`. The pruning aggregates are chosen the way +/// `default_zoned_aggregate_fns` picks zoned aggregates: a byte-bounded min/max (capped at +/// [`default_bounded_stat_max_bytes`]) for variable-length columns, and exact min/max otherwise. +/// Callers wanting a different bound pass their own `BoundedMax`/`BoundedMin` instead. +fn default_aggregate_fns(dtype: &DType) -> Vec { let (max, min) = if is_varlen_dtype(dtype) { // These read back as inexact even when nothing was truncated, until a V2 of the bounded // aggregates records that in its partial state. @@ -354,6 +361,7 @@ fn default_pruning_aggregate_fns(dtype: &DType) -> Vec { min, NullCount.bind(EmptyOptions), NanCount.bind(EmptyOptions), + UncompressedSizeInBytes.bind(EmptyOptions), ] } @@ -437,14 +445,15 @@ impl StatsNode { fn build_root( dtype: &DType, stats: Option<&[AggregateFnRef]>, + allowed: &dyn Fn(&AggregateFnRef) -> bool, write_nested_stats: bool, write_legacy_stats: bool, ) -> Self { if !write_nested_stats { - return Self::build_legacy_root(dtype, stats); + return Self::build_legacy_root(dtype, stats, allowed); } - let mut root = Self::build(dtype, stats); + let mut root = Self::build(dtype, stats, allowed); // The legacy layout records each top-level field's stats over its own dtype. The nested // layout already accumulates those for every top-level field except a non-nullable struct, @@ -457,7 +466,7 @@ impl StatsNode { if let Self::Container { own, .. } = child && matches!(**own, Self::Skipped) { - **own = Self::LegacyOnly(StatsAccumulator::new(&field_dtype, stats)); + **own = Self::LegacyOnly(StatsAccumulator::new(&field_dtype, stats, allowed)); } } } @@ -468,9 +477,13 @@ impl StatsNode { /// Builds a root node that only accumulates the legacy layout's stats: each top-level field's /// stats over its own dtype, or the whole file's for a non-struct root. It contributes no /// nested entries. - fn build_legacy_root(dtype: &DType, stats: Option<&[AggregateFnRef]>) -> Self { + fn build_legacy_root( + dtype: &DType, + stats: Option<&[AggregateFnRef]>, + allowed: &dyn Fn(&AggregateFnRef) -> bool, + ) -> Self { let Some(struct_fields) = dtype.as_struct_fields_opt() else { - return Self::LegacyOnly(StatsAccumulator::new(dtype, stats)); + return Self::LegacyOnly(StatsAccumulator::new(dtype, stats, allowed)); }; // Like `build`, the fields of a nullable root aren't recorded, since they're stored // unmasked. @@ -483,7 +496,7 @@ impl StatsNode { let child = if nullable { Self::Skipped } else { - Self::LegacyOnly(StatsAccumulator::new(&field_dtype, stats)) + Self::LegacyOnly(StatsAccumulator::new(&field_dtype, stats, allowed)) }; (Field::Name(name.clone()), child) }) @@ -494,7 +507,11 @@ impl StatsNode { } } - fn build(dtype: &DType, stats: Option<&[AggregateFnRef]>) -> Self { + fn build( + dtype: &DType, + stats: Option<&[AggregateFnRef]>, + allowed: &dyn Fn(&AggregateFnRef) -> bool, + ) -> Self { match dtype.as_struct_fields_opt() { Some(struct_fields) => { let nullable = dtype.nullability() == Nullability::Nullable; @@ -508,7 +525,7 @@ impl StatsNode { entries: postorder_stats_layout(&field_dtype).len(), } } else { - Self::build(&field_dtype, stats) + Self::build(&field_dtype, stats, allowed) }; (Field::Name(name.clone()), child) }) @@ -516,7 +533,7 @@ impl StatsNode { // Only reached through non-nullable ancestors, so the struct's own validity is // its full logical validity. let own = if nullable { - Self::Leaf(StatsAccumulator::new(dtype, stats)) + Self::Leaf(StatsAccumulator::new(dtype, stats, allowed)) } else { Self::Skipped }; @@ -526,7 +543,7 @@ impl StatsNode { } } None if !supports_file_stats(dtype) => Self::Skipped, - None => Self::Leaf(StatsAccumulator::new(dtype, stats)), + None => Self::Leaf(StatsAccumulator::new(dtype, stats, allowed)), } } @@ -611,12 +628,15 @@ impl StatsNode { } /// Returns this node's stats over its own, undecomposed dtype, as the legacy layout records - /// them for a top-level field. - fn legacy_aggregate_stats(&self) -> VortexResult { + /// them for a top-level field, leaving out the aggregates `allowed` rejects. + fn legacy_aggregate_stats( + &self, + allowed: impl Fn(&AggregateFnRef) -> bool, + ) -> VortexResult { match self { Self::Skipped | Self::Unrecorded { .. } => Ok(AggregateStats::default()), - Self::Leaf(acc) | Self::LegacyOnly(acc) => acc.aggregate_stats(|_| true), - Self::Container { own, .. } => own.legacy_aggregate_stats(), + Self::Leaf(acc) | Self::LegacyOnly(acc) => acc.aggregate_stats(allowed), + Self::Container { own, .. } => own.legacy_aggregate_stats(allowed), } } } @@ -648,18 +668,16 @@ impl FileStatsAccumulator { write_legacy_stats: bool, writer_ctx: &LayoutWriterContext, ) -> VortexResult { - // The legacy layout has a fixed slot per `Stat`, so only the nested one is restricted. - if write_nested_stats { - for aggregate_fn in stats.iter().flat_map(|stats| stats.iter()) { - if !writer_ctx.allows_aggregate(&aggregate_fn.id()) { - vortex_bail!("Aggregate {} not permitted by ctx", aggregate_fn.id()); - } + for aggregate_fn in stats.iter().flat_map(|stats| stats.iter()) { + if !writer_ctx.allows_aggregate(&aggregate_fn.id()) { + vortex_bail!("Aggregate {} not permitted by ctx", aggregate_fn.id()); } } let root = StatsNode::build_root( dtype, stats.as_deref(), + &|aggregate_fn| writer_ctx.allows_aggregate(&aggregate_fn.id()), write_nested_stats, write_legacy_stats, ); @@ -680,6 +698,7 @@ impl FileStatsAccumulator { let mut node = StatsNode::build_root( &self.dtype, self.stats.as_deref(), + &|aggregate_fn| self.writer_ctx.allows_aggregate(&aggregate_fn.id()), self.write_nested_stats, self.write_legacy_stats, ); @@ -712,12 +731,11 @@ impl FileStatsAccumulator { } /// Returns the legacy top-level-fields-only aggregates (one per top-level struct field, or a - /// single entry for a non-struct root dtype). Each entry is empty if `write_legacy_stats` was - /// `false`. - /// - /// The legacy `ArrayStats` format has a fixed slot per [`Stat`] rather than open-ended - /// aggregates, so it isn't restricted by the writer context. + /// single entry for a non-struct root dtype), leaving out the aggregates the writer context + /// forbids. Each entry is empty if `write_legacy_stats` was `false`. pub fn legacy_aggregate_stats(&self) -> VortexResult> { + let allowed = + |aggregate_fn: &AggregateFnRef| self.writer_ctx.allows_aggregate(&aggregate_fn.id()); let root = self.root.lock(); // Readers that predate nested stats reject a file whose legacy layout doesn't have one // entry per top-level field, so entries that aren't recorded are still written, empty. The @@ -732,7 +750,7 @@ impl FileStatsAccumulator { .into_iter() .map(|node| { if self.write_legacy_stats { - node.legacy_aggregate_stats() + node.legacy_aggregate_stats(allowed) } else { Ok(AggregateStats::default()) } @@ -815,7 +833,7 @@ mod tests { BoundedMax.bind(BoundedMaxOptions { max_bytes }), BoundedMin.bind(BoundedMinOptions { max_bytes }), ]; - let mut acc = StatsAccumulator::new(&dtype, Some(&bounded)); + let mut acc = StatsAccumulator::new(&dtype, Some(&bounded), &|_| true); for value in values { let mut builder = VarBinViewBuilder::with_capacity_in( dtype.clone(), @@ -833,9 +851,9 @@ mod tests { } #[test] - fn default_pruning_aggregate_fns_uses_bounded_min_max_for_varlen() { + fn default_aggregate_fns_uses_bounded_min_max_for_varlen() { let dtype = DType::Utf8(Nullability::NonNullable); - let fns = default_pruning_aggregate_fns(&dtype); + let fns = default_aggregate_fns(&dtype); assert!(fns.iter().any(|f| f.is::())); assert!(fns.iter().any(|f| f.is::())); assert!(!fns.iter().any(|f| f.is::())); @@ -843,14 +861,26 @@ mod tests { } #[test] - fn default_pruning_aggregate_fns_uses_plain_min_max_for_fixed_width() { - let fns = default_pruning_aggregate_fns(&i32_dtype()); + fn default_aggregate_fns_uses_plain_min_max_for_fixed_width() { + let fns = default_aggregate_fns(&i32_dtype()); assert!(fns.iter().any(|f| f.is::())); assert!(fns.iter().any(|f| f.is::())); assert!(!fns.iter().any(|f| f.is::())); assert!(!fns.iter().any(|f| f.is::())); } + #[test] + fn disallowed_default_aggregates_are_not_accumulated() { + let acc = + StatsAccumulator::new(&i32_dtype(), None, &|f| !f.is::()); + assert!( + !acc.aggregates + .iter() + .any(|(f, _)| f.is::()) + ); + assert!(acc.aggregates.iter().any(|(f, _)| f.is::())); + } + #[test] fn explicit_min_max_on_varlen_is_exact_and_untruncated() -> VortexResult<()> { // An explicit override applies literally: unlike the default, plain `Max`/`Min` on a @@ -865,7 +895,8 @@ mod tests { builder.append_value("a long value that would have been truncated before"); builder.append_value("short"); - let mut acc = StatsAccumulator::new(&dtype, Some(&[agg(Stat::Max), agg(Stat::Min)])); + let mut acc = + StatsAccumulator::new(&dtype, Some(&[agg(Stat::Max), agg(Stat::Min)]), &|_| true); acc.push_chunk(&builder.finish(), &mut ctx)?; let stats = stats_set(&acc)?; @@ -886,6 +917,7 @@ mod tests { agg(Stat::Sum), agg(Stat::NullCount), ]), + &|_| true, ); acc.push_chunk(&buffer![0, 5, 2].into_array(), &mut ctx)?; @@ -1046,7 +1078,7 @@ mod tests { StructArray::new(FieldNames::from(["a"]), [struct_a], 3, root_validity).into_array(); let requested = [agg(Stat::NullCount), agg(Stat::Min), agg(Stat::Max)]; - let mut node = StatsNode::build(root.dtype(), Some(&requested)); + let mut node = StatsNode::build(root.dtype(), Some(&requested), &|_| true); node.push_chunk(&root, &mut ctx)?; let stats_sets = node_stats_sets(&node)?; @@ -1074,7 +1106,7 @@ mod tests { StructArray::new(FieldNames::from(["s"]), [s], 3, Validity::NonNullable).into_array(); let requested = [agg(Stat::NullCount), agg(Stat::Min), agg(Stat::Max)]; - let mut node = StatsNode::build(root.dtype(), Some(&requested)); + let mut node = StatsNode::build(root.dtype(), Some(&requested), &|_| true); node.push_chunk(&root, &mut ctx)?; let mut aggregates = Vec::new(); @@ -1102,7 +1134,7 @@ mod tests { StructArray::new(FieldNames::from(["s"]), [s], 3, Validity::NonNullable).into_array(); let requested = [agg(Stat::NullCount), agg(Stat::Max)]; - let mut node = StatsNode::build(root.dtype(), Some(&requested)); + let mut node = StatsNode::build(root.dtype(), Some(&requested), &|_| true); node.push_chunk(&root, &mut ctx)?; let stats_sets = node_stats_sets(&node)?; @@ -1130,7 +1162,7 @@ mod tests { .into_array(); let requested = [agg(Stat::NullCount), agg(Stat::Min), agg(Stat::Max)]; - let mut node = StatsNode::build(outer.dtype(), Some(&requested)); + let mut node = StatsNode::build(outer.dtype(), Some(&requested), &|_| true); node.push_chunk(&outer, &mut ctx)?; let stats_sets = node_stats_sets(&node)?; @@ -1332,7 +1364,7 @@ mod tests { let max = BoundedMax.bind(BoundedMaxOptions { max_bytes: NonZeroUsize::new(8).vortex_expect("non-zero"), }); - let mut acc = StatsAccumulator::new(&dtype, Some(std::slice::from_ref(&max))); + let mut acc = StatsAccumulator::new(&dtype, Some(std::slice::from_ref(&max)), &|_| true); acc.push_chunk( &utf8_chunk(&["short", "a value longer than the bound"]), &mut ctx, @@ -1383,6 +1415,53 @@ mod tests { Ok(()) } + #[rstest] + #[case::allowed(true)] + #[case::forbidden(false)] + fn default_uncompressed_size_follows_ctx(#[case] allowed: bool) -> VortexResult<()> { + let mut exec_ctx = SESSION.create_execution_ctx(); + let dtype = DType::struct_([("a", i32_dtype())], Nullability::NonNullable); + let mut aggregates = HashSet::from_iter([Min.id(), Max.id(), NullCount.id()]); + if allowed { + aggregates.insert(UncompressedSizeInBytes.id()); + } + let ctx = + LayoutWriterContext::new(ArrayContext::empty()).with_allowed_aggregates(aggregates); + let acc = FileStatsAccumulator::try_new(&dtype, None, true, true, &ctx)?; + for values in [buffer![1i32, 2, 3], buffer![4i32, 5]] { + let len = values.len(); + let chunk = StructArray::new( + FieldNames::from(["a"]), + [values.into_array()], + len, + Validity::NonNullable, + ) + .into_array(); + push(&acc, &chunk, &mut exec_ctx)?; + } + + // Five i32 values, which both layouts record only if the editions allow it. + let size = agg(Stat::UncompressedSizeInBytes); + let expected = if allowed { + Precision::exact(ScalarValue::from(20u64)) + } else { + Precision::Absent + }; + assert_eq!( + acc.legacy_aggregate_stats()?[0] + .get(&size) + .and_then(Scalar::into_value), + expected + ); + assert_eq!( + acc.aggregate_stats()?[0] + .get(&size) + .and_then(Scalar::into_value), + expected + ); + Ok(()) + } + #[test] fn forbidden_aggregates_are_gated() -> VortexResult<()> { let mut exec_ctx = SESSION.create_execution_ctx(); @@ -1402,8 +1481,7 @@ mod tests { .vortex_expect("Sum is not permitted"); assert!(error.to_string().contains("not permitted by ctx")); - // The defaults leave a forbidden aggregate out of the nested aggregates, but the legacy - // stats keep it. + // The defaults leave a forbidden aggregate out of both the nested and the legacy stats. let acc = FileStatsAccumulator::try_new(&dtype, None, true, true, &ctx)?; let chunk = StructArray::new( FieldNames::from(["a"]), @@ -1429,7 +1507,7 @@ mod tests { acc.legacy_aggregate_stats()?[0] .get(&agg(Stat::NullCount)) .and_then(Scalar::into_value), - Precision::exact(ScalarValue::from(0u64)) + Precision::Absent ); Ok(()) } @@ -1476,7 +1554,7 @@ mod tests { let mut ctx = SESSION.create_execution_ctx(); let dtype = DType::Primitive(PType::I64, Nullability::Nullable); let sum = agg(Stat::Sum); - let mut acc = StatsAccumulator::new(&dtype, Some(std::slice::from_ref(&sum))); + let mut acc = StatsAccumulator::new(&dtype, Some(std::slice::from_ref(&sum)), &|_| true); for chunk in chunks { acc.push_chunk( &PrimitiveArray::from_option_iter(chunk).into_array(),