From ebb6ced7c26d8cdca7a3d49559c59cebbd2eb2fb Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Fri, 6 Nov 2026 19:25:08 +0100 Subject: [PATCH] refactor(writer): narrow the node-stats API and cover the child stats with unit tests (#583) Review of #674: RootNodeStats is now the package-private CachedNodeStats, reached from VortexWriter through one public EncodeResult#withRootStats. Unit tests cover the cached stats on dict children, the validity bitmaps of a nullable integer and float, patch values and the fill-only-when-empty rule; the interop test gains a nullable f64 column. Docs: three shapes, and bit-packed patch values. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011fRbYsrGtpKHp9APhU8qDV --- docs/compatibility.md | 8 +- .../NodeStatsInteropIntegrationTest.java | 22 ++++-- .../dfa1/vortex/writer/VortexWriter.java | 8 +- ...ootNodeStats.java => CachedNodeStats.java} | 8 +- .../writer/encode/CascadingCompressor.java | 4 +- .../vortex/writer/encode/EncodeResult.java | 22 +++++- ...tatsTest.java => CachedNodeStatsTest.java} | 56 ++++++++++---- .../encode/CascadingCompressorTest.java | 74 +++++++++++++++++++ .../writer/encode/PatchBuffersTest.java | 30 ++++++++ .../encode/SequenceEncodingEncoderTest.java | 20 ++--- 10 files changed, 205 insertions(+), 47 deletions(-) rename writer/src/main/java/io/github/dfa1/vortex/writer/encode/{RootNodeStats.java => CachedNodeStats.java} (94%) rename writer/src/test/java/io/github/dfa1/vortex/writer/encode/{RootNodeStatsTest.java => CachedNodeStatsTest.java} (57%) diff --git a/docs/compatibility.md b/docs/compatibility.md index 2a21b99c8..83c7eb245 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -171,10 +171,10 @@ So a Java-written string column has no bound in its flat segments, as a Rust-wri zoned layout's stats table. The decimal bounds the reader can read come from the same two places, and the zone table's are not decoded yet. -The nodes below the root follow the same table, plus four shapes of their own: a `vortex.sequence` carries -`is_sorted` and `is_strict_sorted` (false for a falling step), the primitive values of ALP and ALP-RD patches -carry `is_constant = false`, and the `vortex.bool` validity bitmap of a nullable primitive carries `is_constant` -when every row is valid and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`. +The nodes below the root follow the same table, plus three shapes of their own: a `vortex.sequence` carries +`is_sorted` and `is_strict_sorted` (false for a falling step), the primitive values of ALP, ALP-RD and bit-packed +patches carry `is_constant = false`, and the `vortex.bool` validity bitmap of a nullable primitive carries `is_constant` +when every row is valid (or none is) and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`. `NodeStatsInteropIntegrationTest` compares every flat root, and the whole tree of the numeric columns, against vortex-jni. The string columns' children (FSST lengths and codes, whose values depend on the trained symbol table) and the zone-table segments of string and bool columns are not covered yet diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java index 41e5a194d..8c0e775c3 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java @@ -55,7 +55,7 @@ class NodeStatsInteropIntegrationTest { @Test void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { // Given — the same rows through both writers: sorted, random and small-range integers, floats, - // low- and high-cardinality strings, bools, and a nullable integer + // low- and high-cardinality strings, bools, and a nullable integer and float Random r = new Random(7); long[] sorted = new long[N]; long[] random = new long[N]; @@ -69,6 +69,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { boolean[] valid = new boolean[N]; double[] patched = new double[N]; long[] descending = new long[N]; + double[] nullableFloat = new double[N]; + boolean[] validFloat = new boolean[N]; for (int i = 0; i < N; i++) { sorted[i] = i; random[i] = r.nextLong(); @@ -81,6 +83,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { high[i] = "value-" + Long.toString(r.nextLong() & 0xffffffffL, 36); bool[i] = r.nextBoolean(); valid[i] = i % 7 != 0; + validFloat[i] = i % 5 != 0; + nullableFloat[i] = validFloat[i] ? r.nextDouble() * 100 : 0; nullable[i] = valid[i] ? r.nextInt(5_000) : 0; } Schema arrow = new Schema(List.of( @@ -92,7 +96,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { Field.notNullable("bool", ArrowType.Bool.INSTANCE), Field.nullable("nullable", new ArrowType.Int(32, true)), Field.notNullable("patched", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), - Field.notNullable("descending", new ArrowType.Int(64, true)))); + Field.notNullable("descending", new ArrowType.Int(64, true)), + Field.nullable("nullf", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)))); Path jni = tmp.resolve("jni.vortex"); try (VortexWriter w = VortexWriter.builder(SESSION, jni.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { @@ -107,6 +112,7 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { IntVector nv = (IntVector) root.getVector("nullable"); Float8Vector pv = (Float8Vector) root.getVector("patched"); BigIntVector dv = (BigIntVector) root.getVector("descending"); + Float8Vector nf = (Float8Vector) root.getVector("nullf"); root.allocateNew(); for (int i = 0; i < N; i++) { a.setSafe(i, sorted[i]); @@ -119,6 +125,11 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { h.setSafe(i, bool[i] ? 1 : 0); pv.setSafe(i, patched[i]); dv.setSafe(i, descending[i]); + if (validFloat[i]) { + nf.setSafe(i, nullableFloat[i]); + } else { + nf.setNull(i); + } if (valid[i]) { nv.setSafe(i, nullable[i]); } else { @@ -133,8 +144,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { } Path java = tmp.resolve("java.vortex"); DType.Struct schema = new DType.Struct( - List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable", "patched", "descending").stream().map(ColumnName::of).toList(), - List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable(), DTypes.F64, DTypes.I64), false); + List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable", "patched", "descending", "nullf").stream().map(ColumnName::of).toList(), + List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable(), DTypes.F64, DTypes.I64, DTypes.F64.asNullable()), false); try (FileChannel ch = FileChannel.open(java, StandardOpenOption.CREATE, StandardOpenOption.WRITE); var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) { Map chunk = new java.util.LinkedHashMap<>(); @@ -149,6 +160,7 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { chunk.put(ColumnName.of("nullable"), new NullableData(nullable, valid)); chunk.put(ColumnName.of("patched"), patched); chunk.put(ColumnName.of("descending"), descending); + chunk.put(ColumnName.of("nullf"), new NullableData(nullableFloat, validFloat)); w.writeChunk(chunk); } List expected = EncodingTrees.withStats(jni); @@ -162,7 +174,7 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { List> expectedColumns = columns(expected); List> resultColumns = columns(result); assertThat(resultColumns).hasSameSizeAs(expectedColumns); - for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10}) { + for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10, 11}) { assertThat(resultColumns.get(c)).as("column %d", c).isEqualTo(expectedColumns.get(c)); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index 0cdfbc462..1d345f188 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -24,7 +24,6 @@ import io.github.dfa1.vortex.writer.encode.EncodedBuffer; import io.github.dfa1.vortex.writer.encode.MinMax; import io.github.dfa1.vortex.writer.encode.NodeStats; -import io.github.dfa1.vortex.writer.encode.RootNodeStats; import io.github.dfa1.vortex.writer.encode.NullableData; import io.github.dfa1.vortex.writer.encode.StructData; import io.github.dfa1.vortex.writer.encode.StructEncodingEncoder; @@ -960,12 +959,7 @@ private EncodedSegment encodeSegment(DType dtype, Object data, EncodingEncoder e } SegmentStats stats = new SegmentStats(min, max, ZoneMapStatCodec.columnSum(dtype, data), nullCount); // the stats the flat segment's root node carries are Rust's, not the zone map's - NodeStats own = result.rootNode().stats(); - if (own == null || own.isEmpty()) { - EncodeNode root = result.rootNode().withStats(RootNodeStats.of(dtype, result.rootNode().encodingId(), - min, max, arrayLength(data), nullCount)); - result = new EncodeResult(root, result.encodedBuffers(), result.statsMin(), result.statsMax()); - } + result = result.withRootStats(dtype, min, max, arrayLength(data), nullCount); return new EncodedSegment(result, arena, stats); } catch (RuntimeException | Error e) { arena.close(); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RootNodeStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java similarity index 94% rename from writer/src/main/java/io/github/dfa1/vortex/writer/encode/RootNodeStats.java rename to writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java index f1d75e9e1..b456de854 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RootNodeStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java @@ -16,9 +16,9 @@ /// on: a canonical primitive or bool does, as does `fastlanes.bitpacked` /// (`bitpacking/array/bitpack_compress`); `vortex.alp`, `vortex.alprd`, `vortex.fsst`, /// `vortex.onpair`, `vortex.constant` and the rest do not, so their roots carry nothing. -public final class RootNodeStats { +final class CachedNodeStats { - private RootNodeStats() { + private CachedNodeStats() { } /// The stats the root node of a segment encoded as `root` carries. @@ -30,7 +30,7 @@ private RootNodeStats() { /// @param rows the number of rows /// @param nullCount the number of null rows /// @return the root's stats, [NodeStats#NONE] when the encoding passes none on - public static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoScalarValue max, long rows, + static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoScalarValue max, long rows, long nullCount) { return switch (dtype) { case DType.Primitive p when p.ptype().isFloating() && root.equals(EncodingId.VORTEX_PRIMITIVE) -> @@ -50,7 +50,7 @@ public static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, P /// /// @param validity per-row validity /// @return the validity child's stats - public static NodeStats ofValidity(boolean[] validity) { + static NodeStats ofValidity(boolean[] validity) { long valid = 0; for (boolean v : validity) { if (v) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java index fe097294e..43c03f687 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java @@ -268,14 +268,14 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx) } /// The stats Rust keeps on the node of an array it compressed, here a child of the segment's root - /// as much as the root itself ([RootNodeStats]). + /// as much as the root itself ([CachedNodeStats]). private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) { if (!(dtype instanceof DType.Primitive || dtype instanceof DType.Bool)) { return NodeStats.NONE; } MinMax bounds = dtype instanceof DType.Primitive p && !p.ptype().isFloating() ? ZoneMapStats.of(dtype, data) : null; long nullCount = data instanceof NullableData nullable ? nullable.nullCount() : 0L; - return RootNodeStats.of(dtype, root, bounds == null ? null : bounds.min(), bounds == null ? null : bounds.max(), + return CachedNodeStats.of(dtype, root, bounds == null ? null : bounds.min(), bounds == null ? null : bounds.max(), dataLength(data), nullCount); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java index 3db1af72d..c8fe2aff9 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.writer.encode; +import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.EncodingId; @@ -62,6 +63,25 @@ public EncodeResult withStats(MinMax stats) { : new EncodeResult(rootNode, encodedBuffers, stats.min(), stats.max()); } + /// This result with the array stats Rust keeps on the root of a flat segment, unless the encoder + /// already put its own there. + /// + /// @param dtype the column's dtype + /// @param min the exact minimum of the input, or `null` when unknown + /// @param max the exact maximum of the input, or `null` when unknown + /// @param rows the number of rows + /// @param nullCount the number of null rows + /// @return this result, or a copy whose root carries the stats + public EncodeResult withRootStats(DType dtype, ProtoScalarValue min, ProtoScalarValue max, long rows, + long nullCount) { + NodeStats own = rootNode.stats(); + if (own != null && !own.isEmpty()) { + return this; + } + EncodeNode root = rootNode.withStats(CachedNodeStats.of(dtype, rootNode.encodingId(), min, max, rows, nullCount)); + return new EncodeResult(root, encodedBuffers, statsMin, statsMax); + } + /// This result with `validity` appended to the root as its last child, a plain `vortex.bool`: /// where Rust's primitive and bit-packed arrays keep the validity of a nullable array, which /// the encodings stacked above them (FoR, ZigZag, dict codes) pass down unchanged. @@ -73,7 +93,7 @@ EncodeResult withValidity(boolean[] validity, EncodeContext ctx) { EncodeResult bool = new BoolEncodingEncoder().encode(DTypes.BOOL, validity, ctx); EncodeNode[] children = Arrays.copyOf(rootNode.children(), rootNode.children().length + 1); children[children.length - 1] = EncodeNode.remapBufferIndices(bool.rootNode(), encodedBuffers.size()) - .withStats(RootNodeStats.ofValidity(validity)); + .withStats(CachedNodeStats.ofValidity(validity)); List buffers = new ArrayList<>(encodedBuffers); buffers.addAll(bool.encodedBuffers()); EncodeNode root = new EncodeNode(rootNode.encodingId(), rootNode.metadata(), children, rootNode.bufferIndices()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RootNodeStatsTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java similarity index 57% rename from writer/src/test/java/io/github/dfa1/vortex/writer/encode/RootNodeStatsTest.java rename to writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java index cb32c36db..269307e2f 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RootNodeStatsTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java @@ -9,7 +9,7 @@ import static org.assertj.core.api.Assertions.assertThat; -class RootNodeStatsTest { +class CachedNodeStatsTest { private static final ProtoScalarValue MIN = ProtoScalarValue.ofInt64Value(3L); private static final ProtoScalarValue MAX = ProtoScalarValue.ofInt64Value(90L); @@ -18,7 +18,7 @@ class RootNodeStatsTest { @ValueSource(strings = {"vortex.primitive", "fastlanes.bitpacked"}) void integerRoot_thatPassesStatsOn_keepsBoundsFlagCountAndSize(String root) { // Given / When - NodeStats result = RootNodeStats.of(DTypes.I32, EncodingId.parse(root), MIN, MAX, 1_000, 0); + NodeStats result = CachedNodeStats.of(DTypes.I32, EncodingId.parse(root), MIN, MAX, 1_000, 0); // Then assertThat(result).isEqualTo(new NodeStats(MIN, MAX, false, null, null, 0L, 4_000L, null)); @@ -27,7 +27,7 @@ void integerRoot_thatPassesStatsOn_keepsBoundsFlagCountAndSize(String root) { @Test void integerRoot_withNulls_hasNoConstantFlag_andCountsItsValidityBitmap() { // Given / When - NodeStats result = RootNodeStats.of(DTypes.I64.asNullable(), EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 1_000, 7); + NodeStats result = CachedNodeStats.of(DTypes.I64.asNullable(), EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 1_000, 7); // Then — Rust does not run its constant check on an array with invalid rows assertThat(result.isConstant()).isNull(); @@ -38,7 +38,7 @@ void integerRoot_withNulls_hasNoConstantFlag_andCountsItsValidityBitmap() { @Test void integerRoot_whoseBoundsAreEqual_isConstant() { // Given / When - NodeStats result = RootNodeStats.of(DTypes.I32, EncodingId.VORTEX_PRIMITIVE, MIN, MIN, 10, 0); + NodeStats result = CachedNodeStats.of(DTypes.I32, EncodingId.VORTEX_PRIMITIVE, MIN, MIN, 10, 0); // Then assertThat(result.isConstant()).isTrue(); @@ -48,7 +48,7 @@ void integerRoot_whoseBoundsAreEqual_isConstant() { @ValueSource(strings = {"fastlanes.for", "vortex.constant", "vortex.runend", "fastlanes.delta", "vortex.dict"}) void integerRoot_thatDoesNotPassStatsOn_carriesNone(String root) { // Given / When - NodeStats result = RootNodeStats.of(DTypes.I64, EncodingId.parse(root), MIN, MAX, 1_000, 0); + NodeStats result = CachedNodeStats.of(DTypes.I64, EncodingId.parse(root), MIN, MAX, 1_000, 0); // Then assertThat(result).isEqualTo(NodeStats.NONE); @@ -57,8 +57,8 @@ void integerRoot_thatDoesNotPassStatsOn_carriesNone(String root) { @Test void floatRoot_keepsNoBounds_andAlpKeepsNothing() { // Given / When - NodeStats primitive = RootNodeStats.of(DTypes.F64, EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 100, 0); - NodeStats alp = RootNodeStats.of(DTypes.F64, EncodingId.VORTEX_ALP, MIN, MAX, 100, 0); + NodeStats primitive = CachedNodeStats.of(DTypes.F64, EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 100, 0); + NodeStats alp = CachedNodeStats.of(DTypes.F64, EncodingId.VORTEX_ALP, MIN, MAX, 100, 0); // Then — the compressor computes only the null count for a float assertThat(primitive).isEqualTo(new NodeStats(null, null, false, null, null, 0L, 800L, null)); @@ -68,7 +68,7 @@ void floatRoot_keepsNoBounds_andAlpKeepsNothing() { @Test void boolRoot_keepsTheFlagAndThePackedSize_butNoNullCount() { // Given / When - NodeStats result = RootNodeStats.of(DTypes.BOOL, EncodingId.VORTEX_BOOL, null, null, 20_000, 0); + NodeStats result = CachedNodeStats.of(DTypes.BOOL, EncodingId.VORTEX_BOOL, null, null, 20_000, 0); // Then assertThat(result).isEqualTo(new NodeStats(null, null, false, null, null, null, 2_500L, null)); @@ -77,8 +77,8 @@ void boolRoot_keepsTheFlagAndThePackedSize_butNoNullCount() { @Test void stringRoots_carryNone() { // Given / When - NodeStats varbin = RootNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_VARBIN, MIN, MAX, 100, 0); - NodeStats fsst = RootNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_FSST, MIN, MAX, 100, 0); + NodeStats varbin = CachedNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_VARBIN, MIN, MAX, 100, 0); + NodeStats fsst = CachedNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_FSST, MIN, MAX, 100, 0); // Then — Rust keeps no string bound in a flat segment's array stats assertThat(varbin).isEqualTo(NodeStats.NONE); @@ -88,7 +88,7 @@ void stringRoots_carryNone() { @Test void validityBitmap_mixedCarriesBoundsAndTheValidCount() { // Given / When - NodeStats result = RootNodeStats.ofValidity(new boolean[]{true, false, true, true}); + NodeStats result = CachedNodeStats.ofValidity(new boolean[]{true, false, true, true}); // Then assertThat(result).isEqualTo(new NodeStats(ProtoScalarValue.ofBoolValue(false), @@ -98,11 +98,39 @@ void validityBitmap_mixedCarriesBoundsAndTheValidCount() { @Test void validityBitmap_ofOneValueIsConstant() { // Given / When - NodeStats allValid = RootNodeStats.ofValidity(new boolean[]{true, true}); - NodeStats noneValid = RootNodeStats.ofValidity(new boolean[]{false, false}); + NodeStats allValid = CachedNodeStats.ofValidity(new boolean[]{true, true}); + NodeStats result = CachedNodeStats.ofValidity(new boolean[]{false, false}); // Then assertThat(allValid).isEqualTo(new NodeStats(null, null, true, null, null, null, null, null)); - assertThat(noneValid).isEqualTo(allValid); + assertThat(result).isEqualTo(allValid); + } + + @Test + void withRootStats_keepsTheStatsAnEncoderAlreadyPut() { + // Given + NodeStats own = new NodeStats(null, null, true, null, null, null, null, null); + EncodeResult sut = new EncodeResult(EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0).withStats(own), + java.util.List.of(), null, null); + + // When + EncodeResult result = sut.withRootStats(DTypes.I64, MIN, MAX, 100, 0); + + // Then + assertThat(result).isSameAs(sut); + assertThat(result.rootNode().stats()).isEqualTo(own); + } + + @Test + void withRootStats_fillsAnEmptyRootWithTheCachedStats() { + // Given + EncodeResult sut = new EncodeResult(EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0), java.util.List.of(), null, null); + + // When + EncodeResult result = sut.withRootStats(DTypes.I64, MIN, MAX, 100, 0); + + // Then + assertThat(result.rootNode().stats()) + .isEqualTo(CachedNodeStats.of(DTypes.I64, EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 100, 0)); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java index a86575c61..2a0edd1c6 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java @@ -17,6 +17,7 @@ import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.reader.ReadRegistry; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -160,6 +161,79 @@ private boolean containsEncoding(EncodeNode node, EncodingId id) { } } + @Nested + class ChildNodeStats { + + @Test + void dictChildren_carryTheirBoundsAndNullCount() { + // Given + int[] values = new int[2048]; + for (int i = 0; i < values.length; i++) { + values[i] = (i % 10) * 1000; + } + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + + // When + EncodeResult result = sut.encode(DTypes.I32, values, ctx(3)); + + // Then — the dict itself passes nothing on; its codes and its values are arrays of their own + assertThat(result.rootNode().encodingId()).isEqualTo(EncodingId.VORTEX_DICT); + assertThat(result.rootNode().stats().isEmpty()).isTrue(); + NodeStats codes = result.rootNode().children()[0].stats(); + assertThat(codes.min()).isEqualTo(ProtoScalarValue.ofUint64Value(0L)); + assertThat(codes.max()).isEqualTo(ProtoScalarValue.ofUint64Value(9L)); + assertThat(codes.nullCount()).isZero(); + assertThat(codes.isConstant()).isFalse(); + NodeStats distinct = result.rootNode().children()[1].stats(); + assertThat(distinct.min()).isEqualTo(ProtoScalarValue.ofInt64Value(0L)); + assertThat(distinct.max()).isEqualTo(ProtoScalarValue.ofInt64Value(9000L)); + assertThat(distinct.nullCount()).isZero(); + } + + @Test + void nullableIntegerValidity_isABoolChildCarryingTheValidCount() { + // Given + int[] values = new int[1024]; + boolean[] validity = new boolean[values.length]; + for (int i = 0; i < values.length; i++) { + values[i] = i * 3; + validity[i] = i % 5 != 0; + } + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + + // When + EncodeResult result = sut.encode(new DType.Primitive(PType.I32, true), new NullableData(values, validity), ctx(0)); + + // Then + EncodeNode[] children = result.rootNode().children(); + EncodeNode bool = children[children.length - 1]; + assertThat(bool.encodingId()).isEqualTo(EncodingId.VORTEX_BOOL); + assertThat(bool.stats()).isEqualTo(CachedNodeStats.ofValidity(validity)); + } + + @Test + void nullableFloatValidity_isABoolChildCarryingTheValidCount() { + // Given + double[] values = new double[1024]; + boolean[] validity = new boolean[values.length]; + for (int i = 0; i < values.length; i++) { + values[i] = i * 0.5; + validity[i] = i % 7 != 0; + } + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + + // When + EncodeResult result = sut.encode(new DType.Primitive(PType.F64, true), new NullableData(values, validity), ctx(0)); + + // Then + EncodeNode[] children = result.rootNode().children(); + EncodeNode bool = children[children.length - 1]; + assertThat(bool.encodingId()).isEqualTo(EncodingId.VORTEX_BOOL); + assertThat(bool.stats()).isEqualTo(CachedNodeStats.ofValidity(validity)); + } + + } + @Nested class RoundTrip { diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PatchBuffersTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PatchBuffersTest.java index 3780a6e52..14dafcbd2 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PatchBuffersTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PatchBuffersTest.java @@ -1,12 +1,15 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.model.PType; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import java.lang.foreign.ValueLayout; +import java.util.ArrayList; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -55,6 +58,33 @@ void of_noPositions_fails() { .isInstanceOf(IllegalArgumentException.class); } + @Test + void valuesNode_unequalValues_arePrimitiveMarkedNotConstant() { + // Given + List buffers = new ArrayList<>(); + + // When + EncodeNode result = PatchBuffers.valuesNode(new long[]{3, 9}, PType.I64, buffers, EncodeTestHelper.testCtx()); + + // Then — the constant check that found them unequal leaves its answer on the array + assertThat(result.encodingId()).isEqualTo(EncodingId.VORTEX_PRIMITIVE); + assertThat(result.stats()).isEqualTo(new NodeStats(null, null, false, null, null, null, null, null)); + assertThat(buffers).hasSize(1); + } + + @Test + void valuesNode_equalValues_areAConstantCarryingNoNotConstantFlag() { + // Given + List buffers = new ArrayList<>(); + + // When + EncodeNode result = PatchBuffers.valuesNode(new long[]{7, 7}, PType.I64, buffers, EncodeTestHelper.testCtx()); + + // Then + assertThat(result.encodingId()).isEqualTo(EncodingId.VORTEX_CONSTANT); + assertThat(result.stats() == null || result.stats().isEmpty()).isTrue(); + } + @ParameterizedTest(name = "{0} -> {1}") @CsvSource({"0, U8", "255, U8", "256, U16", "65535, U16", "65536, U32", "4294967295, U32", "4294967296, U64", "-1, U64"}) diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java index 8430521e5..7d417ebf0 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java @@ -76,31 +76,31 @@ void accepts_float_false(PType ptype) { @Test void encode_risingStep_isSortedAndStrictlySorted() { // Given / When - NodeStats stats = ENCODER.encode(DTypes.I64, new long[]{5, 8, 11}, EncodeTestHelper.testCtx()).rootNode().stats(); + NodeStats result = ENCODER.encode(DTypes.I64, new long[]{5, 8, 11}, EncodeTestHelper.testCtx()).rootNode().stats(); - // Then — Rust's Sequence::stats - assertThat(stats.isSorted()).isTrue(); - assertThat(stats.isStrictSorted()).isTrue(); + // Then — Rust's Sequence::result + assertThat(result.isSorted()).isTrue(); + assertThat(result.isStrictSorted()).isTrue(); } @Test void encode_fallingStep_isNeitherSorted() { // Given / When - NodeStats stats = ENCODER.encode(DTypes.I64, new long[]{11, 8, 5}, EncodeTestHelper.testCtx()).rootNode().stats(); + NodeStats result = ENCODER.encode(DTypes.I64, new long[]{11, 8, 5}, EncodeTestHelper.testCtx()).rootNode().stats(); // Then - assertThat(stats.isSorted()).isFalse(); - assertThat(stats.isStrictSorted()).isFalse(); + assertThat(result.isSorted()).isFalse(); + assertThat(result.isStrictSorted()).isFalse(); } @Test void encode_singleValue_isSortedWhateverTheStep() { // Given / When - NodeStats stats = ENCODER.encode(DTypes.I64, new long[]{7}, EncodeTestHelper.testCtx()).rootNode().stats(); + NodeStats result = ENCODER.encode(DTypes.I64, new long[]{7}, EncodeTestHelper.testCtx()).rootNode().stats(); // Then — no adjacent pair to compare - assertThat(stats.isSorted()).isTrue(); - assertThat(stats.isStrictSorted()).isTrue(); + assertThat(result.isSorted()).isTrue(); + assertThat(result.isStrictSorted()).isTrue(); } @Test