Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 60 additions & 33 deletions java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import dev.vortex.VortexCleaner;
import dev.vortex.io.NativeReadable;
import dev.vortex.jni.NativeDataSource;
import java.lang.ref.Cleaner;
import java.lang.ref.Reference;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
Expand All @@ -22,17 +24,24 @@
* A set of Vortex files opened through a {@link Session}. Data sources are cheap to open (only the first file is read
* eagerly, to determine the schema) and can be scanned multiple times.
*
* <p>Native resources are released automatically via {@link VortexCleaner} when the data source becomes unreachable.
* <p>Call {@link #close()} to release this handle and its cached file resources promptly. Scans and partitions already
* created from it remain valid and retain the resources they still need. {@link VortexCleaner} releases an unclosed
* handle when it becomes unreachable.
*
* <p>Native calls are synchronized with {@code close()} to prevent another thread from freeing the handle during an
* operation.
*/
public final class DataSource {
public final class DataSource implements AutoCloseable {
private final Session session;
private final long pointer;
private final Cleaner.Cleanable cleanable;
private boolean closed;

private DataSource(Session session, long pointer) {
Preconditions.checkArgument(pointer != 0, "invalid data source pointer");
this.session = Objects.requireNonNull(session, "session");
this.pointer = pointer;
VortexCleaner.register(this, () -> NativeDataSource.free(pointer));
cleanable = VortexCleaner.register(this, () -> NativeDataSource.free(pointer));
}

/** Open a single URI. */
Expand Down Expand Up @@ -88,8 +97,8 @@ public static DataSource open(Session session, List<NativeReadable> readables) {
* for external I/O abstractions (for example Iceberg's {@code FileIO}). Each file is identified by
* {@link NativeReadable#name()}, which must be unique within {@code readables}.
*
* <p>The readables must remain open until this data source and all scans created from it are closed; native code
* never closes them.
* <p>The readables must remain open until this data source is closed and every derived scan, partition, and Arrow
* reader has finished reading. Native code never closes them.
*
* @param session open session
* @param readables byte sources
Expand Down Expand Up @@ -118,7 +127,8 @@ public static DataSource open(Session session, List<NativeReadable> readables, i
}

/** Arrow schema of the data source (and of scans produced from it). */
public Schema arrowSchema(BufferAllocator allocator) {
public synchronized Schema arrowSchema(BufferAllocator allocator) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need all the synchronised blocks? Reference.reachabilityFence(this) is a noop for synchronised. I think we should add Reference.reachabilityFence and remove synchronised unless I am missing something

Preconditions.checkState(!closed, "data source is closed");
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {
NativeDataSource.arrowSchema(session.nativePointer(), pointer, schema.memoryAddress());
return Data.importSchema(allocator, schema, null);
Expand All @@ -130,7 +140,8 @@ public Schema arrowSchema(BufferAllocator allocator) {
* {@code DataSource::row_count}: {@link RowCount.Unknown} when no estimate is available, {@link RowCount.Estimate}
* for an inexact hint, {@link RowCount.Exact} when the count is authoritative.
*/
public RowCount rowCount() {
public synchronized RowCount rowCount() {
Preconditions.checkState(!closed, "data source is closed");
long[] out = new long[2];
NativeDataSource.rowCount(pointer, out);
return switch ((int) out[1]) {
Expand Down Expand Up @@ -181,7 +192,8 @@ public OptionalLong asOptional() {
* sizes), {@link ByteSize.Estimate} for an inexact hint (some files contribute extrapolated sizes), and
* {@link ByteSize.Exact} when every file has a known size.
*/
public ByteSize byteSize() {
public synchronized ByteSize byteSize() {
Preconditions.checkState(!closed, "data source is closed");
long[] out = new long[2];
NativeDataSource.byteSize(pointer, out);
return switch ((int) out[1]) {
Expand Down Expand Up @@ -226,33 +238,48 @@ public OptionalLong asOptional() {
}

/** Submit a scan. */
public Scan scan(ScanOptions options) {
Objects.requireNonNull(options, "options");
public synchronized Scan scan(ScanOptions options) {
Preconditions.checkState(!closed, "data source is closed");
try {
Objects.requireNonNull(options, "options");

long projectionPtr = options.projection().map(Expression::nativePointer).orElse(0L);
long filterPtr = options.filter().map(Expression::nativePointer).orElse(0L);
long begin = options.rowRangeBegin().orElse(0L);
long end = options.rowRangeEnd().orElse(0L);
ScanOptions.SelectionMode selectionMode = options.selectionMode();
long[] selectionIndices = selectionIndices(options);
byte[] selectionRoaringBitmap = selectionRoaringBitmap(options);
long limit = options.limit().orElse(0L);
boolean ordered = options.ordered();
int splitConcurrency = options.splitConcurrency().orElse(0);
long projectionPtr =
options.projection().map(Expression::nativePointer).orElse(0L);
long filterPtr = options.filter().map(Expression::nativePointer).orElse(0L);
long begin = options.rowRangeBegin().orElse(0L);
long end = options.rowRangeEnd().orElse(0L);
ScanOptions.SelectionMode selectionMode = options.selectionMode();
long[] selectionIndices = selectionIndices(options);
byte[] selectionRoaringBitmap = selectionRoaringBitmap(options);
long limit = options.limit().orElse(0L);
boolean ordered = options.ordered();
int splitConcurrency = options.splitConcurrency().orElse(0);

long scanPtr = dev.vortex.jni.NativeScan.create(
pointer,
projectionPtr,
filterPtr,
begin,
end,
selectionIndices,
selectionRoaringBitmap,
selectionMode.code(),
limit,
ordered,
splitConcurrency);
return Scan.fromPointer(session, scanPtr);
long scanPtr = dev.vortex.jni.NativeScan.create(
pointer,
projectionPtr,
filterPtr,
begin,
end,
selectionIndices,
selectionRoaringBitmap,
selectionMode.code(),
limit,
ordered,
splitConcurrency);
return Scan.fromPointer(session, scanPtr);
} finally {
Reference.reachabilityFence(options);
}
}

/** Releases this handle once; existing scans, partitions, and batches remain valid. */
@Override
public synchronized void close() {
if (!closed) {
closed = true;
cleanable.clean();
}
}

private static long[] selectionIndices(ScanOptions options) {
Expand Down
47 changes: 33 additions & 14 deletions java/vortex-jni/src/main/java/dev/vortex/api/Partition.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.google.common.base.Preconditions;
import dev.vortex.VortexCleaner;
import dev.vortex.jni.NativePartition;
import java.lang.ref.Reference;
import java.util.OptionalLong;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.arrow.c.ArrowArrayStream;
Expand Down Expand Up @@ -43,12 +44,16 @@ static Partition fromPointer(Session session, long pointer) {
/** Estimated row count of the partition. Empty when unknown. */
public OptionalLong rowCount() {
Preconditions.checkState(!consumed.get(), "partition already consumed");
long[] out = new long[2];
NativePartition.rowCount(pointer, out);
if (out[1] == 0) {
return OptionalLong.empty();
try {
long[] out = new long[2];
NativePartition.rowCount(pointer, out);
if (out[1] == 0) {
return OptionalLong.empty();
}
return OptionalLong.of(out[0]);
} finally {
Reference.reachabilityFence(this);
}
return OptionalLong.of(out[0]);
}

/**
Expand All @@ -59,16 +64,30 @@ public ArrowReader scanArrow(BufferAllocator allocator) {
if (!consumed.compareAndSet(false, true)) {
throw new IllegalStateException("partition already consumed");
}
// Native side unconditionally takes ownership of the partition, regardless of
// whether the call subsequently throws, so it is correct to flip `consumed`
// before invoking JNI. The cleaner then always skips free for this handle.
ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator);
try {
NativePartition.scanArrow(session.nativePointer(), pointer, stream.memoryAddress());
} catch (RuntimeException ex) {
stream.close();
throw ex;
ArrowArrayStream stream;
try {
stream = ArrowArrayStream.allocateNew(allocator);
} catch (RuntimeException | Error failure) {
// JNI has not taken the partition yet, and the Cleaner skips consumed partitions.
NativePartition.free(pointer);
throw failure;
}
try {
// JNI consumes the partition even when it throws.
NativePartition.scanArrow(session.nativePointer(), pointer, stream.memoryAddress());
// Closes the stream struct once its reader has taken the stream.
return Data.importArrayStream(allocator, stream);
} catch (RuntimeException | Error failure) {
// Import can fail before moving the stream into its reader.
if (stream.snapshot().release != 0) {
stream.release();
}
stream.close();
throw failure;
}
} finally {
Reference.reachabilityFence(this);
}
return Data.importArrayStream(allocator, stream);
}
}
68 changes: 53 additions & 15 deletions java/vortex-jni/src/main/java/dev/vortex/api/Scan.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@

import com.google.common.base.Preconditions;
import dev.vortex.VortexCleaner;
import dev.vortex.jni.NativePartition;
import dev.vortex.jni.NativeScan;
import java.lang.ref.Cleaner;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.arrow.c.ArrowSchema;
import org.apache.arrow.c.Data;
import org.apache.arrow.memory.BufferAllocator;
Expand All @@ -16,21 +19,37 @@
/**
* A lazy handle to a set of {@link Partition partitions}.
*
* <p>Once a scan has produced its last partition it is effectively exhausted; native resources are released
* automatically via {@link VortexCleaner} when the scan becomes unreachable.
* <p>Call {@link #close()} to release the scan and any partition prefetched by {@link #hasNext()} but not yet returned
* by {@link #next()}. Partitions already returned remain valid. {@link VortexCleaner} releases an unclosed scan when it
* becomes unreachable.
*
* <p>Native calls are synchronized with {@code close()} to prevent another thread from freeing the handle during an
* operation.
*/
public final class Scan implements Iterator<Partition> {
public final class Scan implements Iterator<Partition>, AutoCloseable {
private final Session session;
private final long pointer;

private long nextPartitionPointer;
private final AtomicLong nextPartitionPointer = new AtomicLong();
private final Cleaner.Cleanable cleanable;
private boolean primed;
private boolean closed;

private Scan(Session session, long pointer) {
Preconditions.checkArgument(pointer != 0, "invalid scan pointer");
this.session = session;
this.pointer = pointer;
VortexCleaner.register(this, () -> NativeScan.free(pointer));
AtomicLong pending = nextPartitionPointer;
cleanable = VortexCleaner.register(this, () -> {
try {
long partition = pending.getAndSet(0);
if (partition != 0) {
NativePartition.free(partition);
}
} finally {
NativeScan.free(pointer);
}
});
}

static Scan fromPointer(Session session, long pointer) {
Expand All @@ -40,31 +59,50 @@ static Scan fromPointer(Session session, long pointer) {
/**
* Arrow schema produced by this scan. Must be called before the first call to {@link #hasNext()}/{@link #next()}.
*/
public Schema arrowSchema(BufferAllocator allocator) {
public synchronized Schema arrowSchema(BufferAllocator allocator) {
Preconditions.checkState(!closed, "scan is closed");
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {
NativeScan.arrowSchema(session.nativePointer(), pointer, schema.memoryAddress());
return Data.importSchema(allocator, schema, null);
}
}

/**
* {@inheritDoc}
*
* @throws IllegalStateException if this scan has been closed
*/
@Override
public boolean hasNext() {
if (primed) {
return nextPartitionPointer != 0;
public synchronized boolean hasNext() {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this likely shouldn't be synchronised like datasource?

Preconditions.checkState(!closed, "scan is closed");
if (!primed) {
nextPartitionPointer.set(NativeScan.nextPartition(pointer));
primed = true;
}
nextPartitionPointer = NativeScan.nextPartition(pointer);
primed = true;
return nextPartitionPointer != 0;
return nextPartitionPointer.get() != 0;
}

/**
* {@inheritDoc}
*
* @throws IllegalStateException if this scan has been closed
*/
@Override
public Partition next() {
public synchronized Partition next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
long ptr = nextPartitionPointer;
nextPartitionPointer = 0;
long ptr = nextPartitionPointer.getAndSet(0);
primed = false;
return Partition.fromPointer(session, ptr);
}

/** Releases this scan once, without closing partitions already returned to the caller. */
@Override
public synchronized void close() {
if (!closed) {
closed = true;
cleanable.clean();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* opening one underlying stream per concurrent call, or delegating to a positional-read API).
*
* <p>Lifecycle: the creator owns this object. Native code never calls {@link #close()}; close it only after every data
* source or scan built on top of it has been closed.
* source built on top of it has been closed and every derived scan, partition, and Arrow reader has finished reading.
*/
public interface NativeReadable extends Closeable {
/**
Expand Down
Loading
Loading