Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

50 changes: 43 additions & 7 deletions java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,19 @@
import com.google.common.base.Preconditions;
import dev.vortex.VortexCleaner;
import dev.vortex.io.NativeWritable;
import dev.vortex.jni.NativeBuffer;
import dev.vortex.jni.NativeWriter;
import java.io.Closeable;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.arrow.c.ArrowArray;
import org.apache.arrow.c.ArrowSchema;
import org.apache.arrow.c.Data;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.types.pojo.Schema;

/**
Expand All @@ -24,22 +27,25 @@
* <p>Open one with {@link #builder(Session, String, Schema, BufferAllocator)} to write to a URI, or
* {@link #builder(Session, NativeWritable, Schema, BufferAllocator)} to write into a caller-provided byte sink.
*
* <p>Batches are accepted via the Arrow C Data Interface: callers export an Arrow record batch to an {@code ArrowArray}
* / {@code ArrowSchema} pair and pass the addresses to {@link #writeBatch(long, long)}. The writer accepts up to four
* in-flight batches on the session's runtime thread before back-pressuring the caller.
* <p>Use {@link #writeBatch(VectorSchemaRoot, BufferAllocator)} for zero-copy Java batches with independent buffer
* lifetimes. Batches are also accepted via the Arrow C Data Interface: callers export an Arrow record batch to an
* {@code ArrowArray} / {@code ArrowSchema} pair and pass the addresses to {@link #writeBatch(long, long)}. The writer
* accepts up to four in-flight batches on the session's runtime thread before back-pressuring the caller.
*
* <p>Call {@link #close()} to flush remaining batches and finalize the file. If the writer becomes unreachable without
* an explicit {@code close()}, {@link VortexCleaner} will flush and release native resources as a backstop — but
* callers should always finalize explicitly so that I/O errors surface through the normal call path.
*/
public final class VortexWriter implements AutoCloseable {
private final long pointer;
private final Schema arrowSchema;
private final AtomicBoolean closed = new AtomicBoolean(false);
private volatile VortexWriteSummary summary;

private VortexWriter(long pointer) {
private VortexWriter(long pointer, Schema arrowSchema) {
Preconditions.checkArgument(pointer != 0, "invalid writer pointer");
this.pointer = pointer;
this.arrowSchema = arrowSchema;
AtomicBoolean closedRef = this.closed;
VortexCleaner.register(this, () -> {
if (closedRef.compareAndSet(false, true)) {
Expand Down Expand Up @@ -170,7 +176,7 @@ public VortexWriter build() throws IOException {
if (pointer <= 0) {
throw new IOException("failed to create writer for " + destination() + " (ptr=" + pointer + ")");
}
return new VortexWriter(pointer);
return new VortexWriter(pointer, arrowSchema);
}
}

Expand All @@ -183,8 +189,38 @@ private void checkOptionsApply() {
}
}

/** Write a batch directly from Arrow C Data Interface addresses. */
public void writeBatch(long arrowArrayAddr, long arrowSchemaAddr) throws IOException {
/**
* Write a Java Arrow batch without copying its data. Native buffers retain their allocations independently, so a
* retained column does not pin the entire batch. The caller may close or clear the root after this method returns,
* but must not overwrite its retained buffers.
*
* @param root batch to write; dictionary-encoded fields require the address-based API
* @param allocator allocator for temporary C Data Interface metadata
*/
public synchronized void writeBatch(VectorSchemaRoot root, BufferAllocator allocator) throws IOException {
Preconditions.checkState(!closed.get(), "writer already closed");
if (!arrowSchema.equals(root.getSchema())) {
throw new IOException("batch schema does not match writer schema");
}
try (ArrowArray array = ArrowArray.allocateNew(allocator)) {
try {
Data.exportVectorSchemaRoot(allocator, root, null, array);
if (!NativeWriter.writeBatchBuffers(pointer, array.memoryAddress(), NativeBuffer.collect(root))) {
throw new IOException("failed to write batch");
}
} finally {
array.release();
}
} catch (RuntimeException e) {
throw new IOException("failed to write batch", e);
}
}

/**
* Write a batch directly from Arrow C Data Interface addresses. These buffers share the producer's batch release
* callback; use {@link #writeBatch(VectorSchemaRoot, BufferAllocator)} for independently retained Java allocations.
*/
public synchronized void writeBatch(long arrowArrayAddr, long arrowSchemaAddr) throws IOException {
Preconditions.checkState(!closed.get(), "writer already closed");
final boolean ok;
try {
Expand Down
64 changes: 64 additions & 0 deletions java/vortex-jni/src/main/java/dev/vortex/jni/NativeBuffer.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

package dev.vortex.jni;

import java.util.ArrayList;
import java.util.List;
import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.ReferenceManager;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.VectorSchemaRoot;

/** Per-allocation ownership used by the native zero-copy importer. */
public final class NativeBuffer {
private final ReferenceManager manager;
private final long address;
private final long capacity;

private NativeBuffer(ArrowBuf buffer) {
manager = buffer.getReferenceManager();
address = buffer.memoryAddress();
capacity = buffer.capacity();
}

/** Capture buffer ownership metadata while the root remains alive. */
public static NativeBuffer[] collect(VectorSchemaRoot root) {
List<NativeBuffer> buffers = new ArrayList<>();
for (FieldVector vector : root.getFieldVectors()) {
collect(vector, buffers);
}
return buffers.toArray(new NativeBuffer[0]);
}

private static void collect(FieldVector vector, List<NativeBuffer> buffers) {
for (ArrowBuf buffer : vector.getFieldBuffers()) {
if (buffer.capacity() != 0) {
buffers.add(new NativeBuffer(buffer));
}
}
for (FieldVector child : vector.getChildrenFromFields()) {
collect(child, buffers);
}
}

/** Buffer address, called by JNI. */
public long address() {
return address;
}

/** Buffer capacity, called by JNI. */
public long capacity() {
return capacity;
}

/** Acquire an independent native reference, called by JNI. */
public void retain() {
manager.retain();
}

/** Release the native reference, possibly from a writer worker thread. */
public void release() {
manager.release();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ public static native long createStream(
*/
public static native boolean writeBatch(long writerPointer, long arrowArrayAddress, long arrowSchemaAddress);

/**
* Import a Java batch with independently retained buffer allocations. The exported array must match the schema
* passed to {@link #create}; the caller must serialize writes and close.
*/
public static native boolean writeBatchBuffers(long writerPointer, long arrowArrayAddress, NativeBuffer[] buffers);

/** Number of bytes successfully written to the underlying sink so far. */
public static native long bytesWritten(long writerPointer);

Expand Down
Loading
Loading