Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions docs/compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -171,10 +171,10 @@ So a Java-written string column has no bound in its flat segments, as a Rust-wri
zoned layout's stats table. The decimal bounds the reader can read come from the same two places, and the
zone table's are not decoded yet.

The nodes below the root follow the same table, plus four shapes of their own: a `vortex.sequence` carries
`is_sorted` and `is_strict_sorted` (false for a falling step), the primitive values of ALP and ALP-RD patches
carry `is_constant = false`, and the `vortex.bool` validity bitmap of a nullable primitive carries `is_constant`
when every row is valid and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`.
The nodes below the root follow the same table, plus three shapes of their own: a `vortex.sequence` carries
`is_sorted` and `is_strict_sorted` (false for a falling step), the primitive values of ALP, ALP-RD and bit-packed
patches carry `is_constant = false`, and the `vortex.bool` validity bitmap of a nullable primitive carries `is_constant`
when every row is valid (or none is) and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`.
`NodeStatsInteropIntegrationTest` compares every flat root, and the whole tree of the numeric columns, against
vortex-jni. The string columns' children (FSST lengths and codes, whose values depend on the trained symbol
table) and the zone-table segments of string and bool columns are not covered yet
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class NodeStatsInteropIntegrationTest {
@Test
void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
// Given — the same rows through both writers: sorted, random and small-range integers, floats,
// low- and high-cardinality strings, bools, and a nullable integer
// low- and high-cardinality strings, bools, and a nullable integer and float
Random r = new Random(7);
long[] sorted = new long[N];
long[] random = new long[N];
Expand All @@ -69,6 +69,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
boolean[] valid = new boolean[N];
double[] patched = new double[N];
long[] descending = new long[N];
double[] nullableFloat = new double[N];
boolean[] validFloat = new boolean[N];
for (int i = 0; i < N; i++) {
sorted[i] = i;
random[i] = r.nextLong();
Expand All @@ -81,6 +83,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
high[i] = "value-" + Long.toString(r.nextLong() & 0xffffffffL, 36);
bool[i] = r.nextBoolean();
valid[i] = i % 7 != 0;
validFloat[i] = i % 5 != 0;
nullableFloat[i] = validFloat[i] ? r.nextDouble() * 100 : 0;
nullable[i] = valid[i] ? r.nextInt(5_000) : 0;
}
Schema arrow = new Schema(List.of(
Expand All @@ -92,7 +96,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
Field.notNullable("bool", ArrowType.Bool.INSTANCE),
Field.nullable("nullable", new ArrowType.Int(32, true)),
Field.notNullable("patched", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)),
Field.notNullable("descending", new ArrowType.Int(64, true))));
Field.notNullable("descending", new ArrowType.Int(64, true)),
Field.nullable("nullf", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))));
Path jni = tmp.resolve("jni.vortex");
try (VortexWriter w = VortexWriter.builder(SESSION, jni.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build();
VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) {
Expand All @@ -107,6 +112,7 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
IntVector nv = (IntVector) root.getVector("nullable");
Float8Vector pv = (Float8Vector) root.getVector("patched");
BigIntVector dv = (BigIntVector) root.getVector("descending");
Float8Vector nf = (Float8Vector) root.getVector("nullf");
root.allocateNew();
for (int i = 0; i < N; i++) {
a.setSafe(i, sorted[i]);
Expand All @@ -119,6 +125,11 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
h.setSafe(i, bool[i] ? 1 : 0);
pv.setSafe(i, patched[i]);
dv.setSafe(i, descending[i]);
if (validFloat[i]) {
nf.setSafe(i, nullableFloat[i]);
} else {
nf.setNull(i);
}
if (valid[i]) {
nv.setSafe(i, nullable[i]);
} else {
Expand All @@ -133,8 +144,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
}
Path java = tmp.resolve("java.vortex");
DType.Struct schema = new DType.Struct(
List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable", "patched", "descending").stream().map(ColumnName::of).toList(),
List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable(), DTypes.F64, DTypes.I64), false);
List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable", "patched", "descending", "nullf").stream().map(ColumnName::of).toList(),
List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable(), DTypes.F64, DTypes.I64, DTypes.F64.asNullable()), false);
try (FileChannel ch = FileChannel.open(java, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) {
Map<ColumnName, Object> chunk = new java.util.LinkedHashMap<>();
Expand All @@ -149,6 +160,7 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
chunk.put(ColumnName.of("nullable"), new NullableData(nullable, valid));
chunk.put(ColumnName.of("patched"), patched);
chunk.put(ColumnName.of("descending"), descending);
chunk.put(ColumnName.of("nullf"), new NullableData(nullableFloat, validFloat));
w.writeChunk(chunk);
}
List<String> expected = EncodingTrees.withStats(jni);
Expand All @@ -162,7 +174,7 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
List<List<String>> expectedColumns = columns(expected);
List<List<String>> resultColumns = columns(result);
assertThat(resultColumns).hasSameSizeAs(expectedColumns);
for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10}) {
for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10, 11}) {
assertThat(resultColumns.get(c)).as("column %d", c).isEqualTo(expectedColumns.get(c));
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import io.github.dfa1.vortex.writer.encode.EncodedBuffer;
import io.github.dfa1.vortex.writer.encode.MinMax;
import io.github.dfa1.vortex.writer.encode.NodeStats;
import io.github.dfa1.vortex.writer.encode.RootNodeStats;
import io.github.dfa1.vortex.writer.encode.NullableData;
import io.github.dfa1.vortex.writer.encode.StructData;
import io.github.dfa1.vortex.writer.encode.StructEncodingEncoder;
Expand Down Expand Up @@ -960,12 +959,7 @@ private EncodedSegment encodeSegment(DType dtype, Object data, EncodingEncoder e
}
SegmentStats stats = new SegmentStats(min, max, ZoneMapStatCodec.columnSum(dtype, data), nullCount);
// the stats the flat segment's root node carries are Rust's, not the zone map's
NodeStats own = result.rootNode().stats();
if (own == null || own.isEmpty()) {
EncodeNode root = result.rootNode().withStats(RootNodeStats.of(dtype, result.rootNode().encodingId(),
min, max, arrayLength(data), nullCount));
result = new EncodeResult(root, result.encodedBuffers(), result.statsMin(), result.statsMax());
}
result = result.withRootStats(dtype, min, max, arrayLength(data), nullCount);
return new EncodedSegment(result, arena, stats);
} catch (RuntimeException | Error e) {
arena.close();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@
/// on: a canonical primitive or bool does, as does `fastlanes.bitpacked`
/// (`bitpacking/array/bitpack_compress`); `vortex.alp`, `vortex.alprd`, `vortex.fsst`,
/// `vortex.onpair`, `vortex.constant` and the rest do not, so their roots carry nothing.
public final class RootNodeStats {
final class CachedNodeStats {

private RootNodeStats() {
private CachedNodeStats() {
}

/// The stats the root node of a segment encoded as `root` carries.
Expand All @@ -30,7 +30,7 @@ private RootNodeStats() {
/// @param rows the number of rows
/// @param nullCount the number of null rows
/// @return the root's stats, [NodeStats#NONE] when the encoding passes none on
public static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoScalarValue max, long rows,
static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoScalarValue max, long rows,
long nullCount) {
return switch (dtype) {
case DType.Primitive p when p.ptype().isFloating() && root.equals(EncodingId.VORTEX_PRIMITIVE) ->
Expand All @@ -50,7 +50,7 @@ public static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, P
///
/// @param validity per-row validity
/// @return the validity child's stats
public static NodeStats ofValidity(boolean[] validity) {
static NodeStats ofValidity(boolean[] validity) {
long valid = 0;
for (boolean v : validity) {
if (v) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -268,14 +268,14 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx)
}

/// The stats Rust keeps on the node of an array it compressed, here a child of the segment's root
/// as much as the root itself ([RootNodeStats]).
/// as much as the root itself ([CachedNodeStats]).
private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) {
if (!(dtype instanceof DType.Primitive || dtype instanceof DType.Bool)) {
return NodeStats.NONE;
}
MinMax bounds = dtype instanceof DType.Primitive p && !p.ptype().isFloating() ? ZoneMapStats.of(dtype, data) : null;
long nullCount = data instanceof NullableData nullable ? nullable.nullCount() : 0L;
return RootNodeStats.of(dtype, root, bounds == null ? null : bounds.min(), bounds == null ? null : bounds.max(),
return CachedNodeStats.of(dtype, root, bounds == null ? null : bounds.min(), bounds == null ? null : bounds.max(),
dataLength(data), nullCount);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.github.dfa1.vortex.writer.encode;

import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.proto.ProtoScalarValue;
import io.github.dfa1.vortex.core.model.DTypes;
import io.github.dfa1.vortex.core.model.EncodingId;
Expand Down Expand Up @@ -62,6 +63,25 @@ public EncodeResult withStats(MinMax stats) {
: new EncodeResult(rootNode, encodedBuffers, stats.min(), stats.max());
}

/// This result with the array stats Rust keeps on the root of a flat segment, unless the encoder
/// already put its own there.
///
/// @param dtype the column's dtype
/// @param min the exact minimum of the input, or `null` when unknown
/// @param max the exact maximum of the input, or `null` when unknown
/// @param rows the number of rows
/// @param nullCount the number of null rows
/// @return this result, or a copy whose root carries the stats
public EncodeResult withRootStats(DType dtype, ProtoScalarValue min, ProtoScalarValue max, long rows,
long nullCount) {
NodeStats own = rootNode.stats();
if (own != null && !own.isEmpty()) {
return this;
}
EncodeNode root = rootNode.withStats(CachedNodeStats.of(dtype, rootNode.encodingId(), min, max, rows, nullCount));
return new EncodeResult(root, encodedBuffers, statsMin, statsMax);
}

/// This result with `validity` appended to the root as its last child, a plain `vortex.bool`:
/// where Rust's primitive and bit-packed arrays keep the validity of a nullable array, which
/// the encodings stacked above them (FoR, ZigZag, dict codes) pass down unchanged.
Expand All @@ -73,7 +93,7 @@ EncodeResult withValidity(boolean[] validity, EncodeContext ctx) {
EncodeResult bool = new BoolEncodingEncoder().encode(DTypes.BOOL, validity, ctx);
EncodeNode[] children = Arrays.copyOf(rootNode.children(), rootNode.children().length + 1);
children[children.length - 1] = EncodeNode.remapBufferIndices(bool.rootNode(), encodedBuffers.size())
.withStats(RootNodeStats.ofValidity(validity));
.withStats(CachedNodeStats.ofValidity(validity));
List<EncodedBuffer> buffers = new ArrayList<>(encodedBuffers);
buffers.addAll(bool.encodedBuffers());
EncodeNode root = new EncodeNode(rootNode.encodingId(), rootNode.metadata(), children, rootNode.bufferIndices());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import static org.assertj.core.api.Assertions.assertThat;

class RootNodeStatsTest {
class CachedNodeStatsTest {

private static final ProtoScalarValue MIN = ProtoScalarValue.ofInt64Value(3L);
private static final ProtoScalarValue MAX = ProtoScalarValue.ofInt64Value(90L);
Expand All @@ -18,7 +18,7 @@ class RootNodeStatsTest {
@ValueSource(strings = {"vortex.primitive", "fastlanes.bitpacked"})
void integerRoot_thatPassesStatsOn_keepsBoundsFlagCountAndSize(String root) {
// Given / When
NodeStats result = RootNodeStats.of(DTypes.I32, EncodingId.parse(root), MIN, MAX, 1_000, 0);
NodeStats result = CachedNodeStats.of(DTypes.I32, EncodingId.parse(root), MIN, MAX, 1_000, 0);

// Then
assertThat(result).isEqualTo(new NodeStats(MIN, MAX, false, null, null, 0L, 4_000L, null));
Expand All @@ -27,7 +27,7 @@ void integerRoot_thatPassesStatsOn_keepsBoundsFlagCountAndSize(String root) {
@Test
void integerRoot_withNulls_hasNoConstantFlag_andCountsItsValidityBitmap() {
// Given / When
NodeStats result = RootNodeStats.of(DTypes.I64.asNullable(), EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 1_000, 7);
NodeStats result = CachedNodeStats.of(DTypes.I64.asNullable(), EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 1_000, 7);

// Then — Rust does not run its constant check on an array with invalid rows
assertThat(result.isConstant()).isNull();
Expand All @@ -38,7 +38,7 @@ void integerRoot_withNulls_hasNoConstantFlag_andCountsItsValidityBitmap() {
@Test
void integerRoot_whoseBoundsAreEqual_isConstant() {
// Given / When
NodeStats result = RootNodeStats.of(DTypes.I32, EncodingId.VORTEX_PRIMITIVE, MIN, MIN, 10, 0);
NodeStats result = CachedNodeStats.of(DTypes.I32, EncodingId.VORTEX_PRIMITIVE, MIN, MIN, 10, 0);

// Then
assertThat(result.isConstant()).isTrue();
Expand All @@ -48,7 +48,7 @@ void integerRoot_whoseBoundsAreEqual_isConstant() {
@ValueSource(strings = {"fastlanes.for", "vortex.constant", "vortex.runend", "fastlanes.delta", "vortex.dict"})
void integerRoot_thatDoesNotPassStatsOn_carriesNone(String root) {
// Given / When
NodeStats result = RootNodeStats.of(DTypes.I64, EncodingId.parse(root), MIN, MAX, 1_000, 0);
NodeStats result = CachedNodeStats.of(DTypes.I64, EncodingId.parse(root), MIN, MAX, 1_000, 0);

// Then
assertThat(result).isEqualTo(NodeStats.NONE);
Expand All @@ -57,8 +57,8 @@ void integerRoot_thatDoesNotPassStatsOn_carriesNone(String root) {
@Test
void floatRoot_keepsNoBounds_andAlpKeepsNothing() {
// Given / When
NodeStats primitive = RootNodeStats.of(DTypes.F64, EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 100, 0);
NodeStats alp = RootNodeStats.of(DTypes.F64, EncodingId.VORTEX_ALP, MIN, MAX, 100, 0);
NodeStats primitive = CachedNodeStats.of(DTypes.F64, EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 100, 0);
NodeStats alp = CachedNodeStats.of(DTypes.F64, EncodingId.VORTEX_ALP, MIN, MAX, 100, 0);

// Then — the compressor computes only the null count for a float
assertThat(primitive).isEqualTo(new NodeStats(null, null, false, null, null, 0L, 800L, null));
Expand All @@ -68,7 +68,7 @@ void floatRoot_keepsNoBounds_andAlpKeepsNothing() {
@Test
void boolRoot_keepsTheFlagAndThePackedSize_butNoNullCount() {
// Given / When
NodeStats result = RootNodeStats.of(DTypes.BOOL, EncodingId.VORTEX_BOOL, null, null, 20_000, 0);
NodeStats result = CachedNodeStats.of(DTypes.BOOL, EncodingId.VORTEX_BOOL, null, null, 20_000, 0);

// Then
assertThat(result).isEqualTo(new NodeStats(null, null, false, null, null, null, 2_500L, null));
Expand All @@ -77,8 +77,8 @@ void boolRoot_keepsTheFlagAndThePackedSize_butNoNullCount() {
@Test
void stringRoots_carryNone() {
// Given / When
NodeStats varbin = RootNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_VARBIN, MIN, MAX, 100, 0);
NodeStats fsst = RootNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_FSST, MIN, MAX, 100, 0);
NodeStats varbin = CachedNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_VARBIN, MIN, MAX, 100, 0);
NodeStats fsst = CachedNodeStats.of(DTypes.UTF8, EncodingId.VORTEX_FSST, MIN, MAX, 100, 0);

// Then — Rust keeps no string bound in a flat segment's array stats
assertThat(varbin).isEqualTo(NodeStats.NONE);
Expand All @@ -88,7 +88,7 @@ void stringRoots_carryNone() {
@Test
void validityBitmap_mixedCarriesBoundsAndTheValidCount() {
// Given / When
NodeStats result = RootNodeStats.ofValidity(new boolean[]{true, false, true, true});
NodeStats result = CachedNodeStats.ofValidity(new boolean[]{true, false, true, true});

// Then
assertThat(result).isEqualTo(new NodeStats(ProtoScalarValue.ofBoolValue(false),
Expand All @@ -98,11 +98,39 @@ void validityBitmap_mixedCarriesBoundsAndTheValidCount() {
@Test
void validityBitmap_ofOneValueIsConstant() {
// Given / When
NodeStats allValid = RootNodeStats.ofValidity(new boolean[]{true, true});
NodeStats noneValid = RootNodeStats.ofValidity(new boolean[]{false, false});
NodeStats allValid = CachedNodeStats.ofValidity(new boolean[]{true, true});
NodeStats result = CachedNodeStats.ofValidity(new boolean[]{false, false});

// Then
assertThat(allValid).isEqualTo(new NodeStats(null, null, true, null, null, null, null, null));
assertThat(noneValid).isEqualTo(allValid);
assertThat(result).isEqualTo(allValid);
}

@Test
void withRootStats_keepsTheStatsAnEncoderAlreadyPut() {
// Given
NodeStats own = new NodeStats(null, null, true, null, null, null, null, null);
EncodeResult sut = new EncodeResult(EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0).withStats(own),
java.util.List.of(), null, null);

// When
EncodeResult result = sut.withRootStats(DTypes.I64, MIN, MAX, 100, 0);

// Then
assertThat(result).isSameAs(sut);
assertThat(result.rootNode().stats()).isEqualTo(own);
}

@Test
void withRootStats_fillsAnEmptyRootWithTheCachedStats() {
// Given
EncodeResult sut = new EncodeResult(EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0), java.util.List.of(), null, null);

// When
EncodeResult result = sut.withRootStats(DTypes.I64, MIN, MAX, 100, 0);

// Then
assertThat(result.rootNode().stats())
.isEqualTo(CachedNodeStats.of(DTypes.I64, EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 100, 0));
}
}
Loading
Loading