Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 81 additions & 47 deletions java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import dev.vortex.VortexCleaner;
import dev.vortex.io.NativeReadable;
import dev.vortex.jni.NativeDataSource;
import java.lang.ref.Cleaner;
import java.lang.ref.Reference;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
Expand All @@ -22,17 +24,21 @@
* A set of Vortex files opened through a {@link Session}. Data sources are cheap to open (only the first file is read
* eagerly, to determine the schema) and can be scanned multiple times.
*
* <p>Native resources are released automatically via {@link VortexCleaner} when the data source becomes unreachable.
* <p>Call {@link #close()} to release this handle and its cached file resources promptly. Scans and partitions already
* created from it remain valid and retain the resources they still need. {@link VortexCleaner} releases an unclosed
* handle when it becomes unreachable.
*/
public final class DataSource {
public final class DataSource implements AutoCloseable {
private final Session session;
private final long pointer;
private final Cleaner.Cleanable cleanable;
private boolean closed;

private DataSource(Session session, long pointer) {
Preconditions.checkArgument(pointer != 0, "invalid data source pointer");
this.session = Objects.requireNonNull(session, "session");
this.pointer = pointer;
VortexCleaner.register(this, () -> NativeDataSource.free(pointer));
cleanable = VortexCleaner.register(this, () -> NativeDataSource.free(pointer));
}

/** Open a single URI. */
Expand Down Expand Up @@ -88,8 +94,8 @@ public static DataSource open(Session session, List<NativeReadable> readables) {
* for external I/O abstractions (for example Iceberg's {@code FileIO}). Each file is identified by
* {@link NativeReadable#name()}, which must be unique within {@code readables}.
*
* <p>The readables must remain open until this data source and all scans created from it are closed; native code
* never closes them.
* <p>The readables must remain open until this data source is closed and every derived scan, partition, and Arrow
* reader has finished reading. Native code never closes them.
*
* @param session open session
* @param readables byte sources
Expand Down Expand Up @@ -118,10 +124,13 @@ public static DataSource open(Session session, List<NativeReadable> readables, i
}

/** Arrow schema of the data source (and of scans produced from it). */
public Schema arrowSchema(BufferAllocator allocator) {
public synchronized Schema arrowSchema(BufferAllocator allocator) {
Preconditions.checkState(!closed, "data source is closed");
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {
NativeDataSource.arrowSchema(session.nativePointer(), pointer, schema.memoryAddress());
return Data.importSchema(allocator, schema, null);
} finally {
Reference.reachabilityFence(this);
}
}

Expand All @@ -130,14 +139,19 @@ public Schema arrowSchema(BufferAllocator allocator) {
* {@code DataSource::row_count}: {@link RowCount.Unknown} when no estimate is available, {@link RowCount.Estimate}
* for an inexact hint, {@link RowCount.Exact} when the count is authoritative.
*/
public RowCount rowCount() {
long[] out = new long[2];
NativeDataSource.rowCount(pointer, out);
return switch ((int) out[1]) {
case 1 -> new RowCount.Estimate(out[0]);
case 2 -> new RowCount.Exact(out[0]);
default -> RowCount.Unknown.INSTANCE;
};
public synchronized RowCount rowCount() {
Preconditions.checkState(!closed, "data source is closed");
try {
long[] out = new long[2];
NativeDataSource.rowCount(pointer, out);
return switch ((int) out[1]) {
case 1 -> new RowCount.Estimate(out[0]);
case 2 -> new RowCount.Exact(out[0]);
default -> RowCount.Unknown.INSTANCE;
};
} finally {
Reference.reachabilityFence(this);
}
}

/** Precision-aware row count. See {@link #rowCount()}. */
Expand Down Expand Up @@ -181,14 +195,19 @@ public OptionalLong asOptional() {
* sizes), {@link ByteSize.Estimate} for an inexact hint (some files contribute extrapolated sizes), and
* {@link ByteSize.Exact} when every file has a known size.
*/
public ByteSize byteSize() {
long[] out = new long[2];
NativeDataSource.byteSize(pointer, out);
return switch ((int) out[1]) {
case 1 -> new ByteSize.Estimate(out[0]);
case 2 -> new ByteSize.Exact(out[0]);
default -> ByteSize.Unknown.INSTANCE;
};
public synchronized ByteSize byteSize() {
Preconditions.checkState(!closed, "data source is closed");
try {
long[] out = new long[2];
NativeDataSource.byteSize(pointer, out);
return switch ((int) out[1]) {
case 1 -> new ByteSize.Estimate(out[0]);
case 2 -> new ByteSize.Exact(out[0]);
default -> ByteSize.Unknown.INSTANCE;
};
} finally {
Reference.reachabilityFence(this);
}
}

/** Precision-aware byte size. See {@link #byteSize()}. */
Expand Down Expand Up @@ -226,33 +245,48 @@ public OptionalLong asOptional() {
}

/** Submit a scan. */
public Scan scan(ScanOptions options) {
Objects.requireNonNull(options, "options");
public synchronized Scan scan(ScanOptions options) {
Preconditions.checkState(!closed, "data source is closed");
try {
Objects.requireNonNull(options, "options");

long projectionPtr = options.projection().map(Expression::nativePointer).orElse(0L);
long filterPtr = options.filter().map(Expression::nativePointer).orElse(0L);
long begin = options.rowRangeBegin().orElse(0L);
long end = options.rowRangeEnd().orElse(0L);
ScanOptions.SelectionMode selectionMode = options.selectionMode();
long[] selectionIndices = selectionIndices(options);
byte[] selectionRoaringBitmap = selectionRoaringBitmap(options);
long limit = options.limit().orElse(0L);
boolean ordered = options.ordered();
int splitConcurrency = options.splitConcurrency().orElse(0);
long projectionPtr =
options.projection().map(Expression::nativePointer).orElse(0L);
long filterPtr = options.filter().map(Expression::nativePointer).orElse(0L);
long begin = options.rowRangeBegin().orElse(0L);
long end = options.rowRangeEnd().orElse(0L);
ScanOptions.SelectionMode selectionMode = options.selectionMode();
long[] selectionIndices = selectionIndices(options);
byte[] selectionRoaringBitmap = selectionRoaringBitmap(options);
long limit = options.limit().orElse(0L);
boolean ordered = options.ordered();
int splitConcurrency = options.splitConcurrency().orElse(0);

long scanPtr = dev.vortex.jni.NativeScan.create(
pointer,
projectionPtr,
filterPtr,
begin,
end,
selectionIndices,
selectionRoaringBitmap,
selectionMode.code(),
limit,
ordered,
splitConcurrency);
return Scan.fromPointer(session, scanPtr);
long scanPtr = dev.vortex.jni.NativeScan.create(
pointer,
projectionPtr,
filterPtr,
begin,
end,
selectionIndices,
selectionRoaringBitmap,
selectionMode.code(),
limit,
ordered,
splitConcurrency);
return Scan.fromPointer(session, scanPtr);
} finally {
Reference.reachabilityFence(this);
}
}

/** Releases this handle once; existing scans, partitions, and batches remain valid. */
@Override
public synchronized void close() {
if (!closed) {
closed = true;
cleanable.clean();
}
}

private static long[] selectionIndices(ScanOptions options) {
Expand Down
47 changes: 33 additions & 14 deletions java/vortex-jni/src/main/java/dev/vortex/api/Partition.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.google.common.base.Preconditions;
import dev.vortex.VortexCleaner;
import dev.vortex.jni.NativePartition;
import java.lang.ref.Reference;
import java.util.OptionalLong;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.arrow.c.ArrowArrayStream;
Expand Down Expand Up @@ -43,12 +44,16 @@ static Partition fromPointer(Session session, long pointer) {
/** Estimated row count of the partition. Empty when unknown. */
public OptionalLong rowCount() {
Preconditions.checkState(!consumed.get(), "partition already consumed");
long[] out = new long[2];
NativePartition.rowCount(pointer, out);
if (out[1] == 0) {
return OptionalLong.empty();
try {
long[] out = new long[2];
NativePartition.rowCount(pointer, out);
if (out[1] == 0) {
return OptionalLong.empty();
}
return OptionalLong.of(out[0]);
} finally {
Reference.reachabilityFence(this);
}
return OptionalLong.of(out[0]);
}

/**
Expand All @@ -59,16 +64,30 @@ public ArrowReader scanArrow(BufferAllocator allocator) {
if (!consumed.compareAndSet(false, true)) {
throw new IllegalStateException("partition already consumed");
}
// Native side unconditionally takes ownership of the partition, regardless of
// whether the call subsequently throws, so it is correct to flip `consumed`
// before invoking JNI. The cleaner then always skips free for this handle.
ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator);
try {
NativePartition.scanArrow(session.nativePointer(), pointer, stream.memoryAddress());
} catch (RuntimeException ex) {
stream.close();
throw ex;
ArrowArrayStream stream;
try {
stream = ArrowArrayStream.allocateNew(allocator);
} catch (RuntimeException | Error failure) {
// JNI has not taken the partition yet, and the Cleaner skips consumed partitions.
NativePartition.free(pointer);
throw failure;
}
try {
// JNI consumes the partition even when it throws.
NativePartition.scanArrow(session.nativePointer(), pointer, stream.memoryAddress());
// Closes the stream struct once its reader has taken the stream.
return Data.importArrayStream(allocator, stream);
} catch (RuntimeException | Error failure) {
// Import can fail before moving the stream into its reader.
if (stream.snapshot().release != 0) {
stream.release();
}
stream.close();
throw failure;
}
} finally {
Reference.reachabilityFence(this);
}
return Data.importArrayStream(allocator, stream);
}
}
64 changes: 49 additions & 15 deletions java/vortex-jni/src/main/java/dev/vortex/api/Scan.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@

import com.google.common.base.Preconditions;
import dev.vortex.VortexCleaner;
import dev.vortex.jni.NativePartition;
import dev.vortex.jni.NativeScan;
import java.lang.ref.Cleaner;
import java.lang.ref.Reference;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.arrow.c.ArrowSchema;
import org.apache.arrow.c.Data;
import org.apache.arrow.memory.BufferAllocator;
Expand All @@ -16,21 +20,34 @@
/**
* A lazy handle to a set of {@link Partition partitions}.
*
* <p>Once a scan has produced its last partition it is effectively exhausted; native resources are released
* automatically via {@link VortexCleaner} when the scan becomes unreachable.
* <p>Call {@link #close()} to release the scan and any partition prefetched by {@link #hasNext()} but not yet returned
* by {@link #next()}. Partitions already returned remain valid. {@link VortexCleaner} releases an unclosed scan when it
* becomes unreachable.
*/
public final class Scan implements Iterator<Partition> {
public final class Scan implements Iterator<Partition>, AutoCloseable {
private final Session session;
private final long pointer;

private long nextPartitionPointer;
private final AtomicLong nextPartitionPointer = new AtomicLong();
private final Cleaner.Cleanable cleanable;
private boolean primed;
private boolean closed;

private Scan(Session session, long pointer) {
Preconditions.checkArgument(pointer != 0, "invalid scan pointer");
this.session = session;
this.pointer = pointer;
VortexCleaner.register(this, () -> NativeScan.free(pointer));
AtomicLong pending = nextPartitionPointer;
cleanable = VortexCleaner.register(this, () -> {
try {
long partition = pending.getAndSet(0);
if (partition != 0) {
NativePartition.free(partition);
}
} finally {
NativeScan.free(pointer);
}
});
}

static Scan fromPointer(Session session, long pointer) {
Expand All @@ -40,31 +57,48 @@ static Scan fromPointer(Session session, long pointer) {
/**
* Arrow schema produced by this scan. Must be called before the first call to {@link #hasNext()}/{@link #next()}.
*/
public Schema arrowSchema(BufferAllocator allocator) {
public synchronized Schema arrowSchema(BufferAllocator allocator) {
Preconditions.checkState(!closed, "scan is closed");
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {
NativeScan.arrowSchema(session.nativePointer(), pointer, schema.memoryAddress());
return Data.importSchema(allocator, schema, null);
} finally {
Reference.reachabilityFence(this);
}
}

@Override
public boolean hasNext() {
if (primed) {
return nextPartitionPointer != 0;
public synchronized boolean hasNext() {
if (closed) {
return false;
}
try {
if (!primed) {
nextPartitionPointer.set(NativeScan.nextPartition(pointer));
primed = true;
}
return nextPartitionPointer.get() != 0;
} finally {
Reference.reachabilityFence(this);
}
nextPartitionPointer = NativeScan.nextPartition(pointer);
primed = true;
return nextPartitionPointer != 0;
}

@Override
public Partition next() {
public synchronized Partition next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
long ptr = nextPartitionPointer;
nextPartitionPointer = 0;
long ptr = nextPartitionPointer.getAndSet(0);
primed = false;
return Partition.fromPointer(session, ptr);
}

/** Releases this scan once, without closing partitions already returned to the caller. */
@Override
public synchronized void close() {
if (!closed) {
closed = true;
cleanable.clean();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* opening one underlying stream per concurrent call, or delegating to a positional-read API).
*
* <p>Lifecycle: the creator owns this object. Native code never calls {@link #close()}; close it only after every data
* source or scan built on top of it has been closed.
* source built on top of it has been closed and every derived scan, partition, and Arrow reader has finished reading.
*/
public interface NativeReadable extends Closeable {
/**
Expand Down
Loading
Loading