From b6b6b485545dc253711b23cbc6d812cbc04d788e Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Thu, 5 Nov 2026 23:26:05 +0100 Subject: [PATCH] fix(writer): the root node of a flat segment carries the array stats Rust keeps (#583) Rust's flat layout computes no stats; it keeps what the compressor had cached on the array, and each encoding passes on only some. Java wrote min, max and null_count on every root. Now an integer root under vortex.primitive or fastlanes.bitpacked carries min, max, is_constant (only without nulls), null_count and the uncompressed size, a float root the flag, count and size, a bool root the flag and size, and every other encoding none. As in a Rust-written file, a string column then has no flat min/max, so columnStats() reports none for it. Child nodes and vortex.sequence's sorted flags are still to do. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011fRbYsrGtpKHp9APhU8qDV --- CHANGELOG.md | 1 + docs/compatibility.md | 20 +++ .../vortex/integration/EncodingTrees.java | 74 ++++++++- .../JavaWritesRustReadsIntegrationTest.java | 22 +-- .../NodeStatsInteropIntegrationTest.java | 155 ++++++++++++++++++ .../VortexInspectorIntegrationTest.java | 15 +- .../dfa1/vortex/writer/RootNodeStats.java | 76 +++++++++ .../dfa1/vortex/writer/VortexWriter.java | 88 ++++++---- .../dfa1/vortex/writer/encode/EncodeNode.java | 24 ++- .../dfa1/vortex/writer/encode/NodeStats.java | 40 +++++ .../vortex/writer/GlobalDictUtf8Test.java | 7 +- .../writer/NullableDecimalWriteTest.java | 34 ++-- .../dfa1/vortex/writer/RootNodeStatsTest.java | 88 ++++++++++ 13 files changed, 559 insertions(+), 85 deletions(-) create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java create mode 100644 writer/src/main/java/io/github/dfa1/vortex/writer/RootNodeStats.java create mode 100644 writer/src/main/java/io/github/dfa1/vortex/writer/encode/NodeStats.java create mode 100644 writer/src/test/java/io/github/dfa1/vortex/writer/RootNodeStatsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e2fa461b6..1d4209986 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed +- The array stats on a flat segment's root node are the ones Rust keeps: integers under `vortex.primitive` and `fastlanes.bitpacked` carry min, max, `is_constant`, `null_count` and the uncompressed size, and every other encoding carries none, so a Java-written string column reports no flat `min`/`max`, as a Rust-written one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The compact preset writes text columns as zstd frames of 8192 values, as Rust does, so a row-range scan decompresses only the frames it overlaps. ([#636](https://github.com/dfa1/vortex-java/issues/636)) - A nullable decimal column keeps its validity in its own encoding, or has none when every row is valid, instead of a `vortex.masked` wrapper, and a column of equal decimals is one constant, as Rust writes them: decimal zone-stats tables are smaller. ([#646](https://github.com/dfa1/vortex-java/issues/646)) - `VortexAggregates` summarizes a `U64` column exactly: `MIN`, `MAX` and `SUM` compare and add unsigned instead of failing past 2^63. ([#665](https://github.com/dfa1/vortex-java/issues/665)) diff --git a/docs/compatibility.md b/docs/compatibility.md index b438f3d2a..83a296306 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -152,6 +152,26 @@ decimals ([#430](https://github.com/dfa1/vortex-java/pull/430)) and nulls in nul | `vortex.zstd_buffers` | `ZstdBuffersEncodingDecoder` | — | ✅ | ❌ | Read only (`zstd2026.02.0`, opt-in in Rust): each buffer of any wrapped encoding is Zstd-decompressed at its declared alignment, then the inner encoding decodes with its own children. Needs the optional zstd binding, like `vortex.zstd`. Interop-tested against a Rust-written fixture (`scripts/fixtures/zstd-buffers`) | | `vortex.onpair` | `OnPairEncodingDecoder` | `OnPairEncodingEncoder` | ✅ | ✅ | Utf8, Binary; `core2026.08.1`, a cascade candidate under the default edition (competes with FSST, as in Rust) | +### Array stats on flat segments + +A flat segment's `ArrayNode` tree carries array stats, and Rust's flat layout does not compute them: it keeps +the ones the compressor had cached on the array, and each encoding passes on only some +(#583). The writer mirrors that for the **root node**: + +| Column | Root encoding | Stats written | +|---|---|---| +| integer | `vortex.primitive`, `fastlanes.bitpacked` | `min`, `max`, `is_constant` (only without nulls), `null_count`, `uncompressed_size_in_bytes` | +| float | `vortex.primitive` | `is_constant` (only without nulls), `null_count`, `uncompressed_size_in_bytes`; no bounds | +| bool | `vortex.bool` | `is_constant`, `uncompressed_size_in_bytes` | +| any | any other encoding (`vortex.alp`, `vortex.alprd`, `vortex.fsst`, `vortex.onpair`, `vortex.constant`, `fastlanes.for`, `vortex.dict`, strings, ...) | none | + +So a Java-written string column has no bound in its flat segments, as a Rust-written one has none: +`VortexReader.columnStats()` reports no string `min`/`max` for either, and a string bound lives only in the +zoned layout's stats table. The decimal bounds the reader can read come from the same two places, and the +zone table's are not decoded yet. Child nodes carry no stats yet, and `vortex.sequence` does not write its sorted +flags yet ([#583](https://github.com/dfa1/vortex-java/issues/583)); `NodeStatsInteropIntegrationTest` compares +every flat root against vortex-jni. + ### Decode shape Per [ADR 0010](../adr/0010-lazy-decode.md) and [ADR 0012](../adr/0012-zero-copy-layout-decoding.md), each diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/EncodingTrees.java b/integration/src/test/java/io/github/dfa1/vortex/integration/EncodingTrees.java index 2f4ada076..da315b741 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/EncodingTrees.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/EncodingTrees.java @@ -2,6 +2,8 @@ import io.github.dfa1.vortex.core.fbs.FbsArray; import io.github.dfa1.vortex.core.fbs.FbsArrayNode; +import io.github.dfa1.vortex.core.fbs.FbsArrayStats; +import io.github.dfa1.vortex.core.fbs.FbsPrecision; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.reader.SegmentSpec; import io.github.dfa1.vortex.reader.VortexReader; @@ -25,16 +27,26 @@ private EncodingTrees() { /// layout around a single chunk is left out: Java keeps one where Rust writes the flat layout /// directly, and that layout shape is not what the encoding comparisons test. static List of(Path file) throws IOException { + return of(file, false); + } + + /// [#of(Path)] with the array stats each node carries after its id, as `[min max null_count ..]` + /// naming the stats present and their precision: what Rust and Java write into a flat segment. + static List withStats(Path file) throws IOException { + return of(file, true); + } + + private static List of(Path file, boolean stats) throws IOException { List out = new ArrayList<>(); try (VortexReader reader = VortexReader.open(file)) { - layoutTree(reader, reader.layout(), 0, out); + layoutTree(reader, reader.layout(), 0, out, stats); } return out; } - private static void layoutTree(VortexReader reader, Layout layout, int depth, List out) { + private static void layoutTree(VortexReader reader, Layout layout, int depth, List out, boolean stats) { if (layout.isChunked() && layout.children().size() == 1) { - layoutTree(reader, layout.children().getFirst(), depth, out); + layoutTree(reader, layout.children().getFirst(), depth, out, stats); return; } out.add(" ".repeat(depth) + layout.layoutId().id()); @@ -43,18 +55,64 @@ private static void layoutTree(VortexReader reader, Layout layout, int depth, Li MemorySegment seg = reader.rawSegment(spec); int fbLen = seg.get(LE_INT, seg.byteSize() - 4L); FbsArray array = FbsArray.getRootAsFbsArray(seg.asSlice(seg.byteSize() - 4L - fbLen, fbLen)); - arrayTree(array.root(), reader.footer().arraySpecs(), depth + 1, out); + arrayTree(array.root(), reader.footer().arraySpecs(), depth + 1, out, stats); return; } for (Layout child : layout.children()) { - layoutTree(reader, child, depth + 1, out); + layoutTree(reader, child, depth + 1, out, stats); } } - private static void arrayTree(FbsArrayNode node, List specs, int depth, List out) { - out.add(" ".repeat(depth) + specs.get(node.encoding()).id()); + private static void arrayTree(FbsArrayNode node, List specs, int depth, List out, + boolean stats) { + String line = " ".repeat(depth) + specs.get(node.encoding()).id(); + if (stats) { + line += " " + statNames(node.stats()); + } + out.add(line); for (int i = 0; i < node.childrenLength(); i++) { - arrayTree(node.children(i), specs, depth + 1, out); + arrayTree(node.children(i), specs, depth + 1, out, stats); + } + } + + /// The stats a node records, by Rust's names, with their values: bounds as hex of the scalar + /// bytes (`~` after the name when inexact), flags and counts as numbers. + private static String statNames(FbsArrayStats stats) { + if (stats == null) { + return "[]"; + } + List names = new ArrayList<>(); + if (stats.hasMin()) { + names.add("min" + (stats.minPrecision() == FbsPrecision.Exact ? "" : "~") + "=" + hex(stats.minAsSegment())); + } + if (stats.hasMax()) { + names.add("max" + (stats.maxPrecision() == FbsPrecision.Exact ? "" : "~") + "=" + hex(stats.maxAsSegment())); } + if (stats.hasSum()) { + names.add("sum=" + hex(stats.sumAsSegment())); + } + if (stats.hasIsConstant()) { + names.add("is_constant=" + stats.isConstant()); + } + if (stats.hasIsSorted()) { + names.add("is_sorted=" + stats.isSorted()); + } + if (stats.hasIsStrictSorted()) { + names.add("is_strict_sorted=" + stats.isStrictSorted()); + } + if (stats.hasNullCount()) { + names.add("null_count=" + stats.nullCount()); + } + if (stats.hasNanCount()) { + names.add("nan_count=" + stats.nanCount()); + } + if (stats.hasUncompressedSizeInBytes()) { + names.add("uncompressed_size=" + stats.uncompressedSizeInBytes()); + } + return names.toString(); + } + + private static String hex(java.lang.foreign.MemorySegment segment) { + return java.util.HexFormat.of().formatHex(segment.toArray(java.lang.foreign.ValueLayout.JAVA_BYTE)); } } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java index 45ec80cdd..a98ee7564 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java @@ -918,8 +918,8 @@ static Stream stringEncoders() { @MethodSource("stringEncoders") void javaWriter_jniReader_utf8ChunkStats_inUtf8ByteOrder(EncodingEncoder encoder, @TempDir Path tmp) throws IOException { - // Given — each string encoder writes the chunk's min/max, and with zone maps off those are - // the only bounds in the file. The emoji is a surrogate pair: below U+E000 in UTF-16, above + // Given — a string column's bounds live in the zone map (Rust keeps none in a flat segment's + // array stats), and a filtered read prunes against them. The emoji is a surrogate pair: below U+E000 in UTF-16, above // it in UTF-8, the order Rust compares in, so a UTF-16 max (U+E000) would let a reader prune // the chunk holding it. The shared-prefix ids made the FSST table keep two symbols in one of // Rust's hash slots, and vortex-jni aborted the JVM rebuilding it to filter. @@ -932,23 +932,16 @@ void javaWriter_jniReader_utf8ChunkStats_inUtf8ByteOrder(EncodingEncoder encoder keys[500] = "\uE000"; Path file = tmp.resolve("java_utf8_chunk_stats.vtx"); try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); - var sut = VortexWriter.create(ch, STRING_SCHEMA, WriteOptions.defaults().withZoneMaps(false), - List.of(encoder))) { + var sut = VortexWriter.create(ch, STRING_SCHEMA, WriteOptions.defaults(), List.of(encoder))) { sut.writeChunk(Map.of(ColumnName.of("s"), keys)); } // When List result = readColumnFiltered(file, "s", Expression.binary(Expression.BinaryOp.EQ, Expression.column("s"), Expression.literal(emoji))); - io.github.dfa1.vortex.reader.ArrayStats stats; - try (var reader = io.github.dfa1.vortex.reader.VortexReader.open(file)) { - stats = reader.columnStats().get(ColumnName.of("s")); - } // Then assertThat(result).map(Object::toString).containsExactly(emoji); - assertThat(stats.min()).isEqualTo("EVT0000000000"); - assertThat(stats.max()).isEqualTo(emoji); } @Test @@ -980,11 +973,12 @@ void javaWriter_jniReader_filteredUtf8_prunesOnTruncatedBounds(@TempDir Path tmp stats = reader.columnStats().get(ColumnName.of("k")); } - // Then — vortex-jni accepts the flat stats' precision fields, and the column's min/max are - // the full keys: only the zone map truncates, so SQL MIN/MAX may answer from these stats. + // Then — the matching row survives the truncated zone bounds. As in a Rust-written file, no + // flat array stats hold a string bound, and the zone map's are truncated, so the column + // reports no exact min/max. assertThat(result).extracting(Object::toString).containsExactly(target); - assertThat(stats.min()).isEqualTo(prefix + String.format("%08d", 0)); - assertThat(stats.max()).isEqualTo(prefix + String.format("%08d", 3 * batch - 1)); + assertThat(stats.min()).isNull(); + assertThat(stats.max()).isNull(); } @Test diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java new file mode 100644 index 000000000..4a26d8c2d --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java @@ -0,0 +1,155 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.writer.WriteOptions; +import io.github.dfa1.vortex.writer.encode.NullableData; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +/// The array stats on the root node of every flat segment, against what vortex-jni writes for the same +/// data (#583). Rust's flat layout keeps only the stats the compressor had cached on the array, and each +/// encoding passes on some of them: integers keep min, max, `is_constant`, `null_count` and the +/// uncompressed size under `vortex.primitive` and `fastlanes.bitpacked`; floats and strings keep none +/// under ALP, FSST or onpair. `vortex.sequence` is compared by encoding only: its sorted flags are not +/// written yet. +class NodeStatsInteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + private static final int N = 20_000; + + static { + NativeLoader.loadJni(); + } + + @Test + void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { + // Given — the same rows through both writers: sorted, random and small-range integers, floats, + // low- and high-cardinality strings, bools, and a nullable integer + Random r = new Random(7); + long[] sorted = new long[N]; + long[] random = new long[N]; + int[] small = new int[N]; + double[] rf = new double[N]; + double[] sf = new double[N]; + String[] low = new String[N]; + String[] high = new String[N]; + boolean[] bool = new boolean[N]; + int[] nullable = new int[N]; + boolean[] valid = new boolean[N]; + for (int i = 0; i < N; i++) { + sorted[i] = i; + random[i] = r.nextLong(); + small[i] = r.nextInt(100); + rf[i] = r.nextDouble() * 1000; + sf[i] = i * 0.5; + low[i] = "k" + r.nextInt(10); + high[i] = "value-" + Long.toString(r.nextLong() & 0xffffffffL, 36); + bool[i] = r.nextBoolean(); + valid[i] = i % 7 != 0; + nullable[i] = valid[i] ? r.nextInt(5_000) : 0; + } + Schema arrow = new Schema(List.of( + Field.notNullable("sorted", new ArrowType.Int(64, true)), Field.notNullable("random", new ArrowType.Int(64, true)), + Field.notNullable("small", new ArrowType.Int(32, true)), + Field.notNullable("rf", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + Field.notNullable("sf", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + Field.notNullable("low", ArrowType.Utf8.INSTANCE), Field.notNullable("high", ArrowType.Utf8.INSTANCE), + Field.notNullable("bool", ArrowType.Bool.INSTANCE), + Field.nullable("nullable", new ArrowType.Int(32, true)))); + Path jni = tmp.resolve("jni.vortex"); + try (VortexWriter w = VortexWriter.builder(SESSION, jni.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { + BigIntVector a = (BigIntVector) root.getVector("sorted"); + BigIntVector b = (BigIntVector) root.getVector("random"); + IntVector c = (IntVector) root.getVector("small"); + Float8Vector d = (Float8Vector) root.getVector("rf"); + Float8Vector e = (Float8Vector) root.getVector("sf"); + VarCharVector f = (VarCharVector) root.getVector("low"); + VarCharVector g = (VarCharVector) root.getVector("high"); + BitVector h = (BitVector) root.getVector("bool"); + IntVector nv = (IntVector) root.getVector("nullable"); + root.allocateNew(); + for (int i = 0; i < N; i++) { + a.setSafe(i, sorted[i]); + b.setSafe(i, random[i]); + c.setSafe(i, small[i]); + d.setSafe(i, rf[i]); + e.setSafe(i, sf[i]); + f.setSafe(i, low[i].getBytes(StandardCharsets.UTF_8)); + g.setSafe(i, high[i].getBytes(StandardCharsets.UTF_8)); + h.setSafe(i, bool[i] ? 1 : 0); + if (valid[i]) { + nv.setSafe(i, nullable[i]); + } else { + nv.setNull(i); + } + } + root.setRowCount(N); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema as = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, as); + w.writeBatch(array.memoryAddress(), as.memoryAddress()); + } + } + Path java = tmp.resolve("java.vortex"); + DType.Struct schema = new DType.Struct( + List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable").stream().map(ColumnName::of).toList(), + List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable()), false); + try (FileChannel ch = FileChannel.open(java, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) { + w.writeChunk(Map.of(ColumnName.of("sorted"), sorted, ColumnName.of("random"), random, + ColumnName.of("small"), small, ColumnName.of("rf"), rf, ColumnName.of("sf"), sf, + ColumnName.of("low"), low, ColumnName.of("high"), high, ColumnName.of("bool"), bool, + ColumnName.of("nullable"), new NullableData(nullable, valid))); + } + List expected = flatRoots(EncodingTrees.withStats(jni)); + + // When + List result = flatRoots(EncodingTrees.withStats(java)); + + // Then + assertThat(result).isEqualTo(expected); + } + + /// The root node line of every flat segment, in file order; a sequence's stats are blanked. + private static List flatRoots(List tree) { + List roots = new java.util.ArrayList<>(); + for (int i = 0; i + 1 < tree.size(); i++) { + if (tree.get(i).strip().equals("vortex.flat")) { + String root = tree.get(i + 1).strip(); + roots.add(root.startsWith("vortex.sequence") ? "vortex.sequence" : root); + } + } + return roots; + } +} diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java index 402adfa27..f5f270e1d 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java @@ -202,11 +202,11 @@ void columnProfile_rustWrittenFile_matchesTheData(@TempDir Path tmp) throws IOEx } @Test - void columnProfile_globalDictColumn_rangeFromValuesNotCodes(@TempDir Path tmp) throws IOException { - // Given — the default vortex.zoned map records only truncated bounds for a string, so the - // profile takes min/max from each chunk's own stats; under a dict layout those are the - // values pool's. The pool's sum and null count describe the dictionary, not the rows, and - // must not leak in: four distinct values against 2,000 rows + void columnProfile_globalDictColumn_hasNoRangeAndNoDictionaryTotals(@TempDir Path tmp) throws IOException { + // Given — the default vortex.zoned map records only truncated bounds for a string, and Rust + // keeps no bound in a flat segment's array stats, so the profile has no range. The pool's + // sum and null count describe the dictionary, not the rows, and must not leak in: four + // distinct values against 2,000 rows Path file = tmp.resolve("dict.vtx"); ColumnName fruit = ColumnName.of("fruit"); var schema = new DType.Struct(List.of(fruit), List.of(DTypes.UTF8), false); @@ -223,9 +223,8 @@ void columnProfile_globalDictColumn_rangeFromValuesNotCodes(@TempDir Path tmp) t } // Then - assertThat(result.min()).isEqualTo("apple"); - assertThat(result.max()).isEqualTo("date"); - assertThat(result.nullCount()).isZero(); + assertThat(result.min()).isNull(); + assertThat(result.max()).isNull(); assertThat(result.sum()).isNull(); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/RootNodeStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/RootNodeStats.java new file mode 100644 index 000000000..7301130bd --- /dev/null +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/RootNodeStats.java @@ -0,0 +1,76 @@ +package io.github.dfa1.vortex.writer; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.writer.encode.NodeStats; + +import java.util.Arrays; + +/// The stats Rust leaves on the root node of a flat segment, by the column's dtype and the encoding +/// the compressor chose. +/// +/// Rust's flat layout does not compute stats; it keeps what the compressor had cached on the array +/// (`flat/writer.rs`). The compressor computes, on the canonical array it is about to compress, the +/// integer min and max, the null count, the exact uncompressed size (`compressor/cascade.rs`) and +/// whether the array is constant. A scheme's output keeps those only when the scheme passes them +/// on: a canonical primitive or bool does, as does `fastlanes.bitpacked` +/// (`bitpacking/array/bitpack_compress`); `vortex.alp`, `vortex.alprd`, `vortex.fsst`, +/// `vortex.onpair`, `vortex.constant` and the rest do not, so their roots carry nothing. +final class RootNodeStats { + + private RootNodeStats() { + } + + /// The stats the root node of a segment encoded as `root` carries. + /// + /// @param dtype the column's dtype + /// @param root the root encoding the compressor chose + /// @param min the exact minimum of the input, or `null` when unknown + /// @param max the exact maximum of the input, or `null` when unknown + /// @param rows the number of rows + /// @param nullCount the number of null rows + /// @return the root's stats, [NodeStats#NONE] when the encoding passes none on + static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoScalarValue max, long rows, + long nullCount) { + return switch (dtype) { + case DType.Primitive p when p.ptype().isFloating() && root.equals(EncodingId.VORTEX_PRIMITIVE) -> + new NodeStats(null, null, notConstant(nullCount), null, null, nullCount, canonicalSize(p, rows, nullCount), null); + case DType.Primitive p when !p.ptype().isFloating() && retainsIntegerStats(root) -> + new NodeStats(min, max, isConstant(min, max, nullCount), null, null, nullCount, + canonicalSize(p, rows, nullCount), null); + case DType.Bool _ when root.equals(EncodingId.VORTEX_BOOL) -> + new NodeStats(null, null, false, null, null, null, (rows + 7) / 8, null); + default -> NodeStats.NONE; + }; + } + + private static boolean retainsIntegerStats(EncodingId root) { + return root.equals(EncodingId.VORTEX_PRIMITIVE) || root.equals(EncodingId.FASTLANES_BITPACKED); + } + + /// Rust records `is_constant` only for an array without nulls: its constant check does not run + /// on one with invalid rows, so the stat stays absent. + private static Boolean isConstant(ProtoScalarValue min, ProtoScalarValue max, long nullCount) { + if (nullCount > 0) { + return null; + } + return min != null && max != null && Arrays.equals(min.encode(), max.encode()); + } + + private static Boolean notConstant(long nullCount) { + if (nullCount > 0) { + return null; + } + return false; + } + + /// Rust's `exact_uncompressed_size` of a primitive: the values, plus a packed bitmap when some row is null. + private static long canonicalSize(DType.Primitive dtype, long rows, long nullCount) { + long size = rows * dtype.ptype().byteSize(); + if (nullCount > 0) { + size += (rows + 7) / 8; + } + return size; + } +} diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index 2bbd2298c..d9f861581 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -23,6 +23,7 @@ import io.github.dfa1.vortex.writer.encode.EncodeResult; import io.github.dfa1.vortex.writer.encode.EncodedBuffer; import io.github.dfa1.vortex.writer.encode.MinMax; +import io.github.dfa1.vortex.writer.encode.NodeStats; import io.github.dfa1.vortex.writer.encode.NullableData; import io.github.dfa1.vortex.writer.encode.StructData; import io.github.dfa1.vortex.writer.encode.StructEncodingEncoder; @@ -957,6 +958,10 @@ private EncodedSegment encodeSegment(DType dtype, Object data, EncodingEncoder e max = fallback != null ? fallback.max() : null; } SegmentStats stats = new SegmentStats(min, max, ZoneMapStatCodec.columnSum(dtype, data), nullCount); + // the stats the flat segment's root node carries are Rust's, not the zone map's + EncodeNode root = result.rootNode().withStats(RootNodeStats.of(dtype, result.rootNode().encodingId(), + min, max, arrayLength(data), nullCount)); + result = new EncodeResult(root, result.encodedBuffers(), result.statsMin(), result.statsMax()); return new EncodedSegment(result, arena, stats); } catch (RuntimeException | Error e) { arena.close(); @@ -979,7 +984,7 @@ private void writeEncoded(EncodedSegment encoded) throws IOException { long offset = bytesWritten; List buffers = result.encodedBuffers(); int[] paddings = bufferPaddings(buffers); - ByteBuffer fbBuf = buildArrayFlatBuffer(result, paddings, encoded.stats().nullCount()); + ByteBuffer fbBuf = buildArrayFlatBuffer(result, paddings); // Segment format: [padding, buffer]... [FlatBuffer Array bytes] [4-byte LE u32 = fbLen] int fbLen = fbBuf.remaining(); @@ -1095,36 +1100,10 @@ private static int[] bufferPaddings(List buffers) { return paddings; } - private ByteBuffer buildArrayFlatBuffer(EncodeResult result, int[] paddings, long nullCount) { + private ByteBuffer buildArrayFlatBuffer(EncodeResult result, int[] paddings) { var fbb = new FbsBuilder(256); - // Stats for the root node only (build vectors before the ArrayStats table). null_count is - // always recorded; min/max only when the encoder produced them. Sum is not embedded per-flat - // (Rust's flat writer doesn't either — flat/writer.rs retains only pre-computed stats); the - // per-zone sum lives in the vortex.stats zone-map table emitted by flushZoneMaps(). - int minVec = result.hasStats() - ? io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMinVector(fbb, result.statsMin().encode()) : 0; - int maxVec = result.hasStats() - ? io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMaxVector(fbb, result.statsMax().encode()) : 0; - // forceDefaults only while building ArrayStats, so null_count = 0 is serialized (flatbuffers - // omits a scalar equal to its default otherwise) — matching the Rust writer and letting the - // reader prune IS NULL on zero-null chunks. Reset immediately so the Array/ArrayNode tables - // keep their normal (offset-default-omitting) layout. - fbb.forceDefaults(true); - io.github.dfa1.vortex.core.fbs.FbsArrayStats.startFbsArrayStats(fbb); - if (result.hasStats()) { - // The encoder's min/max are the exact extremes, never truncated. An absent precision - // is the FlatBuffer default Inexact, which is what Rust read every Java bound as. - io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMin(fbb, minVec); - io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMinPrecision(fbb, io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); - io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMax(fbb, maxVec); - io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMaxPrecision(fbb, io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); - } - io.github.dfa1.vortex.core.fbs.FbsArrayStats.addNullCount(fbb, nullCount); - int statsOff = io.github.dfa1.vortex.core.fbs.FbsArrayStats.endFbsArrayStats(fbb); - fbb.forceDefaults(false); - - int rootNodeOff = buildArrayNodeFlatBuffer(fbb, result.rootNode(), statsOff); + int rootNodeOff = buildArrayNodeFlatBuffer(fbb, result.rootNode()); // Buffer struct vector — one entry per buffer in result. // FbsLayout (LE): padding(u16) | alignment_exponent(u8) | compression(u8) | length(u32) @@ -1151,12 +1130,59 @@ private ByteBuffer buildArrayFlatBuffer(EncodeResult result, int[] paddings, lon // ── FbsFooter / metadata serialization ────────────────────────────────────── - private int buildArrayNodeFlatBuffer(FbsBuilder fbb, EncodeNode node, int statsOff) { + /// The `ArrayStats` table of a node's stats, or `0` when it carries none. Every stat present is + /// written, a `false` flag or a zero count included, as Rust writes an exact stat of any value. + private static int buildStats(FbsBuilder fbb, NodeStats stats) { + if (stats == null || stats.isEmpty()) { + return 0; + } + // vectors first: FlatBuffers builds bottom-up + int minVec = stats.min() == null ? 0 + : io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMinVector(fbb, stats.min().encode()); + int maxVec = stats.max() == null ? 0 + : io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMaxVector(fbb, stats.max().encode()); + // forceDefaults only while building ArrayStats, so a scalar equal to its default is still + // serialized; reset immediately so the Array/ArrayNode tables keep their normal layout + fbb.forceDefaults(true); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.startFbsArrayStats(fbb); + if (stats.min() != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMin(fbb, minVec); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMinPrecision(fbb, io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); + } + if (stats.max() != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMax(fbb, maxVec); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMaxPrecision(fbb, io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); + } + if (stats.isSorted() != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsSorted(fbb, stats.isSorted()); + } + if (stats.isStrictSorted() != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsStrictSorted(fbb, stats.isStrictSorted()); + } + if (stats.isConstant() != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsConstant(fbb, stats.isConstant()); + } + if (stats.nullCount() != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addNullCount(fbb, stats.nullCount()); + } + if (stats.uncompressedSize() != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addUncompressedSizeInBytes(fbb, stats.uncompressedSize()); + } + if (stats.nanCount() != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addNanCount(fbb, stats.nanCount()); + } + int off = io.github.dfa1.vortex.core.fbs.FbsArrayStats.endFbsArrayStats(fbb); + fbb.forceDefaults(false); + return off; + } + + private int buildArrayNodeFlatBuffer(FbsBuilder fbb, EncodeNode node) { // Build children first (FlatBuffers bottom-up: nested objects before parent table) int[] childOffsets = new int[node.children().length]; for (int i = 0; i < childOffsets.length; i++) { - childOffsets[i] = buildArrayNodeFlatBuffer(fbb, node.children()[i], 0); + childOffsets[i] = buildArrayNodeFlatBuffer(fbb, node.children()[i]); } + int statsOff = buildStats(fbb, node.stats()); int metaOff = 0; if (node.metadata() != null && node.metadata().byteSize() > 0) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeNode.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeNode.java index be4af6f20..1dad9a9ba 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeNode.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeNode.java @@ -11,13 +11,33 @@ /// @param metadata optional encoding-specific metadata bytes, or `null` /// @param children child encode nodes (empty for leaf nodes) /// @param bufferIndices indices into the flat buffer list for data owned by this node +/// @param stats the array stats this node carries, or `null` for none @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. public record EncodeNode( EncodingId encodingId, MemorySegment metadata, EncodeNode[] children, - int[] bufferIndices + int[] bufferIndices, + NodeStats stats ) { + /// Creates a node that carries no stats. + /// + /// @param encodingId encoding id for this node + /// @param metadata optional encoding-specific metadata bytes, or `null` + /// @param children child encode nodes (empty for leaf nodes) + /// @param bufferIndices indices into the flat buffer list for data owned by this node + public EncodeNode(EncodingId encodingId, MemorySegment metadata, EncodeNode[] children, int[] bufferIndices) { + this(encodingId, metadata, children, bufferIndices, null); + } + + /// A copy of this node carrying `stats`. + /// + /// @param stats the array stats the copy carries + /// @return the copy + public EncodeNode withStats(NodeStats stats) { + return new EncodeNode(encodingId, metadata, children, bufferIndices, stats); + } + /// Creates a leaf node with a single buffer and no children or metadata. /// /// @param encodingId the encoding identifier for this leaf node @@ -46,6 +66,6 @@ public static EncodeNode remapBufferIndices(EncodeNode node, int offset) { for (int i = 0; i < oldChildren.length; i++) { newChildren[i] = remapBufferIndices(oldChildren[i], offset); } - return new EncodeNode(node.encodingId(), node.metadata(), newChildren, newIdx); + return new EncodeNode(node.encodingId(), node.metadata(), newChildren, newIdx, node.stats()); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/NodeStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/NodeStats.java new file mode 100644 index 000000000..8f1dcd6e9 --- /dev/null +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/NodeStats.java @@ -0,0 +1,40 @@ +package io.github.dfa1.vortex.writer.encode; + +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; + +/// The array stats Rust records on one node of a flat segment's `ArrayNode` tree. A `null` component +/// is a stat the node does not carry: Rust's flat layout retains only what the compressor had +/// computed on the array, and each encoding passes on only some of it. +/// +/// Every stat written is exact. +/// +/// @param min exact minimum of the array's values, or `null` +/// @param max exact maximum of the array's values, or `null` +/// @param isConstant whether every value is equal, or `null` +/// @param isSorted whether the values are sorted, or `null` +/// @param isStrictSorted whether the values are strictly sorted, or `null` +/// @param nullCount number of null rows, or `null` +/// @param uncompressedSize size of the canonical array in bytes, or `null` +/// @param nanCount number of NaN values, or `null` +public record NodeStats( + ProtoScalarValue min, + ProtoScalarValue max, + Boolean isConstant, + Boolean isSorted, + Boolean isStrictSorted, + Long nullCount, + Long uncompressedSize, + Long nanCount +) { + + /// A node that carries no stats. + public static final NodeStats NONE = new NodeStats(null, null, null, null, null, null, null, null); + + /// Whether this node carries no stat at all. + /// + /// @return `true` if every component is `null` + public boolean isEmpty() { + return min == null && max == null && isConstant == null && isSorted == null && isStrictSorted == null + && nullCount == null && uncompressedSize == null && nanCount == null; + } +} diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java index 0124f6f81..006adc6b3 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java @@ -216,9 +216,10 @@ void globalDict_utf8_reportsColumnStats(@TempDir Path tmp) throws IOException { try (var vf = VortexReader.open(file, ReadRegistry.loadAll())) { var result = vf.columnStats().get(ColumnName.of("status")); - // Then — lexicographic bounds over the three values, not over the codes - assertThat(result.min()).isEqualTo("closed"); - assertThat(result.max()).isEqualTo("open"); + // Then — Rust keeps no string bounds in a flat array's stats, so none are reported; the + // null count is still known + assertThat(result.min()).isNull(); + assertThat(result.max()).isNull(); assertThat(result.nullCount()).isZero(); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/NullableDecimalWriteTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/NullableDecimalWriteTest.java index a6605e5c2..12f3d2eb9 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/NullableDecimalWriteTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/NullableDecimalWriteTest.java @@ -21,7 +21,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.channels.FileChannel; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; @@ -42,18 +41,19 @@ class NullableDecimalWriteTest { @ParameterizedTest(name = "DECIMAL({0}, 2), cascade {1}") @CsvSource({"20, 3", "20, 0", "15, 3", "15, 0", "38, 3", "5, 0"}) - void nullableDecimal_boundsSkipTheNullRows(int precision, int cascade) throws Exception { - // Given — the null row's slot holds a placeholder zero that is below every valid value + void nullableDecimal_withNulls_roundTripsAndCountsItsNulls(int precision, int cascade) throws Exception { + // Given — the null row's slot holds a placeholder zero that is below every valid value; the + // flat segment keeps no bounds (Rust's does not), and the zone table's are checked against + // vortex-jni by DecimalBytePartsInteropIntegrationTest's filtered read BigDecimal[] values = {dec("0.05"), null, dec("0.07"), dec("0.09")}; Path file = write(precision, WriteOptions.cascading(cascade), values); // When - ArrayStats result = stats(file); + List result = zoneStats(file); - // Then — the minimum is 0.05, not the placeholder - assertThat(unscaled(result.min())).isEqualTo(5); - assertThat(unscaled(result.max())).isEqualTo(9); - assertThat(result.nullCount()).isEqualTo(1); + // Then + assertThat(result).hasSize(1); + assertThat(result.getFirst().nullCount()).isEqualTo(1); assertThat(read(file)).containsExactly(values); } @@ -174,6 +174,13 @@ private static ArrayStats stats(Path file) throws IOException { } } + private static List zoneStats(Path file) throws IOException { + try (VortexReader reader = VortexReader.open(file); + ScanIterator scan = reader.scan(ScanOptions.all())) { + return scan.columnZoneStats(D); + } + } + private static BigDecimal[] read(Path file) throws IOException { List out = new ArrayList<>(); try (VortexReader reader = VortexReader.open(file); @@ -199,15 +206,4 @@ private static long fileSize(Path file) throws IOException { private static BigDecimal dec(String value) { return new BigDecimal(value); } - - /// The unscaled value of a decimal bound as `ArrayStats` reads it: a UTF-8 string of the - /// little-endian two's-complement bytes, at whatever width the encoding wrote them. - private static long unscaled(Object bound) { - byte[] le = ((String) bound).getBytes(StandardCharsets.UTF_8); - byte[] be = new byte[le.length]; - for (int i = 0; i < le.length; i++) { - be[le.length - 1 - i] = le[i]; - } - return new BigInteger(be).longValueExact(); - } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/RootNodeStatsTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/RootNodeStatsTest.java new file mode 100644 index 000000000..326fd5a12 --- /dev/null +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/RootNodeStatsTest.java @@ -0,0 +1,88 @@ +package io.github.dfa1.vortex.writer; + +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.writer.encode.NodeStats; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; + +class RootNodeStatsTest { + + private static final ProtoScalarValue MIN = ProtoScalarValue.ofInt64Value(3L); + private static final ProtoScalarValue MAX = ProtoScalarValue.ofInt64Value(90L); + + @ParameterizedTest + @ValueSource(strings = {"vortex.primitive", "fastlanes.bitpacked"}) + void integerRoot_thatPassesStatsOn_keepsBoundsFlagCountAndSize(String root) { + // Given / When + NodeStats result = RootNodeStats.of(DTypes.I32, EncodingId.parse(root), MIN, MAX, 1_000, 0); + + // Then + assertThat(result).isEqualTo(new NodeStats(MIN, MAX, false, null, null, 0L, 4_000L, null)); + } + + @Test + void integerRoot_withNulls_hasNoConstantFlag_andCountsItsValidityBitmap() { + // Given / When + NodeStats result = RootNodeStats.of(DTypes.I64.asNullable(), EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 1_000, 7); + + // Then — Rust does not run its constant check on an array with invalid rows + assertThat(result.isConstant()).isNull(); + assertThat(result.nullCount()).isEqualTo(7); + assertThat(result.uncompressedSize()).isEqualTo(1_000 * 8 + 125); + } + + @Test + void integerRoot_whoseBoundsAreEqual_isConstant() { + // Given / When + NodeStats result = RootNodeStats.of(DTypes.I32, EncodingId.VORTEX_PRIMITIVE, MIN, MIN, 10, 0); + + // Then + assertThat(result.isConstant()).isTrue(); + } + + @ParameterizedTest + @ValueSource(strings = {"fastlanes.for", "vortex.constant", "vortex.runend", "fastlanes.delta", "vortex.dict"}) + void integerRoot_thatDoesNotPassStatsOn_carriesNone(String root) { + // Given / When + NodeStats result = RootNodeStats.of(DTypes.I64, EncodingId.parse(root), MIN, MAX, 1_000, 0); + + // Then + assertThat(result).isEqualTo(NodeStats.NONE); + } + + @Test + void floatRoot_keepsNoBounds_andAlpKeepsNothing() { + // Given / When + NodeStats primitive = RootNodeStats.of(DTypes.F64, EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 100, 0); + NodeStats alp = RootNodeStats.of(DTypes.F64, EncodingId.VORTEX_ALP, MIN, MAX, 100, 0); + + // Then — the compressor computes only the null count for a float + assertThat(primitive).isEqualTo(new NodeStats(null, null, false, null, null, 0L, 800L, null)); + assertThat(alp).isEqualTo(NodeStats.NONE); + } + + @Test + void boolRoot_keepsTheFlagAndThePackedSize_butNoNullCount() { + // Given / When + NodeStats result = RootNodeStats.of(DTypes.BOOL, EncodingId.VORTEX_BOOL, null, null, 20_000, 0); + + // Then + assertThat(result).isEqualTo(new NodeStats(null, null, false, null, null, null, 2_500L, null)); + } + + @Test + void stringRoots_carryNone() { + // Given / When + NodeStats varbin = RootNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_VARBIN, MIN, MAX, 100, 0); + NodeStats fsst = RootNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_FSST, MIN, MAX, 100, 0); + + // Then — Rust keeps no string bound in a flat segment's array stats + assertThat(varbin).isEqualTo(NodeStats.NONE); + assertThat(fsst).isEqualTo(NodeStats.NONE); + } +}