Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions vortex-datafusion/src/convert/stats/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,16 @@ use crate::PrecisionExt;
use crate::convert::TryToDataFusion;

/// Convert a stats set for an array with the given dtype.
///
/// The uncompressed size is left out of `byte_size`. DataFusion's join selection picks
/// `CollectLeft` below `hash_join_single_partition_threshold` (1 MiB) when a byte size is known
/// and below `hash_join_single_partition_threshold_rows` (128 Ki rows) otherwise, so known sizes
/// turn small build sides into partitioned joins. DataFusion's `project_statistics` also keeps
/// the whole table's size for any projection with a variable-width column.
pub(crate) fn stats_set_to_df(
stats_set: &StatsSet,
dtype: &DType,
) -> VortexResult<ColumnStatistics> {
// Update the total size in bytes.
let column_size = stats_set.get_as::<usize>(Stat::UncompressedSizeInBytes, &PType::U64.into());

let null_count = stats_set
.get_as::<usize>(Stat::NullCount, &PType::U64.into())
.to_df();
Expand All @@ -39,7 +42,7 @@ pub(crate) fn stats_set_to_df(
max_value,
sum_value: scalar_stat_to_df(stats_set, Stat::Sum, dtype).to_df(),
distinct_count,
byte_size: column_size.to_df(),
byte_size: Precision::Absent,
})
}

Expand Down
14 changes: 14 additions & 0 deletions vortex-datafusion/src/convert/stats/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,3 +204,17 @@ fn float_statistics_require_no_nans(
assert_eq!(stats.sum_value, expected(3.0));
Ok(())
}

#[test]
fn uncompressed_size_is_not_reported_as_byte_size() -> VortexResult<()> {
let stats = StatsSet::of(
Stat::UncompressedSizeInBytes,
VortexPrecision::exact(4096u64),
);
let dtype = DType::Utf8(Nullability::NonNullable);
assert_eq!(
stats_set_to_df(&stats, &dtype)?.byte_size,
Precision::Absent
);
Ok(())
}
13 changes: 3 additions & 10 deletions vortex-datafusion/src/persistent/format.rs
Original file line number Diff line number Diff line change
Expand Up @@ -686,10 +686,6 @@ impl FileFormat for VortexFormat {
}
let stats_set = &stats_set;

// Update the total size in bytes.
let column_size =
stats_set.get_as::<usize>(Stat::UncompressedSizeInBytes, &PType::U64.into());

let nan_free = is_nan_free(stats_set, stats_dtype);
let bound = |stat| {
if nan_free {
Expand All @@ -713,21 +709,18 @@ impl FileFormat for VortexFormat {
max_value: max,
sum_value: DFPrecision::Absent,
distinct_count,
byte_size: column_size.to_df(),
// See `stats_set_to_df` for why the uncompressed size is not reported.
byte_size: DFPrecision::Absent,
})
}

let total_byte_size = column_statistics
.iter()
.fold(DFPrecision::Exact(0), |acc, cs| acc.add(&cs.byte_size));

Ok(Statistics {
num_rows: DFPrecision::Exact(
usize::try_from(row_count)
.map_err(|_| vortex_err!("Row count overflow"))
.vortex_expect("Row count overflow"),
),
total_byte_size,
total_byte_size: DFPrecision::Absent,
column_statistics,
})
})
Expand Down
7 changes: 4 additions & 3 deletions vortex-duckdb/src/column_statistics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,10 @@ impl ColumnStatisticsAggregate {
let max = exact(Max.bind(NumericalAggregateOpts::skip_nans()));

// DuckDB's string length is u32
#[allow(clippy::cast_possible_truncation)]
let max_string_length = exact(UncompressedSizeInBytes.bind(EmptyOptions))
.map(|value| value.as_primitive().as_u64().vortex_expect("not a u64") as u32);
let max_string_length =
exact(UncompressedSizeInBytes.bind(EmptyOptions)).and_then(|value| {
u32::try_from(value.as_primitive().as_u64().vortex_expect("not a u64")).ok()
});

let has_null = exact(NullCount.bind(EmptyOptions))
.is_none_or(|count| count.as_primitive().as_u64().vortex_expect("not a u64") > 0);
Expand Down
59 changes: 59 additions & 0 deletions vortex-file/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ use vortex_edition::EDITION_DECLARATIONS;
use vortex_edition::EditionSession;
use vortex_edition::EditionSessionExt;
use vortex_edition::declarations::core::CORE_2026_08_3;
use vortex_edition::declarations::preview::PREVIEW_2026_08_0;
use vortex_edition::declarations::unstable::UNSTABLE;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
Expand Down Expand Up @@ -1587,6 +1588,64 @@ async fn exclude_legacy_statistics_empties_legacy_but_keeps_nested() -> VortexRe
Ok(())
}

/// Both the nested and the legacy file statistics record the uncompressed size only when the
/// enabled editions include it, or when editions are disabled.
#[rstest]
#[case::core(false, false, false)]
#[case::preview(true, false, true)]
#[case::editions_disabled(false, true, true)]
#[tokio::test]
async fn uncompressed_size_follows_editions(
#[case] preview: bool,
#[case] disable_editions: bool,
#[case] recorded: bool,
) -> VortexResult<()> {
let session = VortexSession::empty()
.with::<EditionSession>()
.with::<LayoutSession>()
.with::<RuntimeSession>();
crate::register_default_encodings(&session);
for declaration in EDITION_DECLARATIONS {
session.register_edition(declaration)?;
}
session.enable_edition(CORE_2026_08_3)?;
if preview {
session.enable_edition(PREVIEW_2026_08_0)?;
}

let c = PrimitiveArray::from_iter([4i32, 5, 6]).into_array();
let array = StructArray::try_new(["c"].into(), vec![c], 3, Validity::NonNullable)?.into_array();
let options = session.write_options().include_nested_statistics();
let options = if disable_editions {
options.disable_editions()
} else {
options
};
let mut buf = ByteBufferMut::empty();
let summary = options.write(&mut buf, array.to_array_stream()).await?;

let three_i32 = recorded.then_some(12u64);
let stats = summary
.footer()
.statistics()
.expect("file statistics should be present");
let (c_stats, _) = stats.get_by_path(&field_path!(c)).expect("c entry");
assert_eq!(
stat_value(c_stats, Stat::UncompressedSizeInBytes),
three_i32.map_or(Precision::Absent, |size| Precision::exact(
ScalarValue::from(size)
))
);

let bytes = stats.write_flatbuffer_bytes()?;
let fb =
flatbuffers::root::<fb::FileStatistics>(bytes.as_ref()).vortex_expect("valid flatbuffer");
let field_stats = fb.field_stats().vortex_expect("field_stats is written");
assert_eq!(field_stats.get(0).uncompressed_size_in_bytes(), three_i32);

Ok(())
}

#[tokio::test]
async fn nested_statistics_are_excluded_by_default() -> VortexResult<()> {
let inner = StructArray::try_new(
Expand Down
Loading
Loading