Repository navigation
fix(spark): release native read resources deterministically #11367
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
robert3005
merged 4 commits into
vortex-data:develop
from
quantcast:cwirt/upstream-native-read-cleanup
Nov 11, 2026
Merged
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
566803c
fix(spark): release native read resources deterministically
ChristopherWirt a49a62c
Merge branch 'develop' into cwirt/upstream-native-read-cleanup
ChristopherWirt 7731fbd
Merge branch 'develop' into cwirt/upstream-native-read-cleanup
ChristopherWirt 19a2635
fix(jni): refine read handle lifecycle review follow-up
ChristopherWirt File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,9 +5,12 @@ | |
|
|
||
| import com.google.common.base.Preconditions; | ||
| import dev.vortex.VortexCleaner; | ||
| import dev.vortex.jni.NativePartition; | ||
| import dev.vortex.jni.NativeScan; | ||
| import java.lang.ref.Cleaner; | ||
| import java.util.Iterator; | ||
| import java.util.NoSuchElementException; | ||
| import java.util.concurrent.atomic.AtomicLong; | ||
| import org.apache.arrow.c.ArrowSchema; | ||
| import org.apache.arrow.c.Data; | ||
| import org.apache.arrow.memory.BufferAllocator; | ||
|
|
@@ -16,21 +19,37 @@ | |
| /** | ||
| * A lazy handle to a set of {@link Partition partitions}. | ||
| * | ||
| * <p>Once a scan has produced its last partition it is effectively exhausted; native resources are released | ||
| * automatically via {@link VortexCleaner} when the scan becomes unreachable. | ||
| * <p>Call {@link #close()} to release the scan and any partition prefetched by {@link #hasNext()} but not yet returned | ||
| * by {@link #next()}. Partitions already returned remain valid. {@link VortexCleaner} releases an unclosed scan when it | ||
| * becomes unreachable. | ||
| * | ||
| * <p>Native calls are synchronized with {@code close()} to prevent another thread from freeing the handle during an | ||
| * operation. | ||
| */ | ||
| public final class Scan implements Iterator<Partition> { | ||
| public final class Scan implements Iterator<Partition>, AutoCloseable { | ||
| private final Session session; | ||
| private final long pointer; | ||
|
|
||
| private long nextPartitionPointer; | ||
| private final AtomicLong nextPartitionPointer = new AtomicLong(); | ||
| private final Cleaner.Cleanable cleanable; | ||
| private boolean primed; | ||
| private boolean closed; | ||
|
|
||
| private Scan(Session session, long pointer) { | ||
| Preconditions.checkArgument(pointer != 0, "invalid scan pointer"); | ||
| this.session = session; | ||
| this.pointer = pointer; | ||
| VortexCleaner.register(this, () -> NativeScan.free(pointer)); | ||
| AtomicLong pending = nextPartitionPointer; | ||
| cleanable = VortexCleaner.register(this, () -> { | ||
| try { | ||
| long partition = pending.getAndSet(0); | ||
| if (partition != 0) { | ||
| NativePartition.free(partition); | ||
| } | ||
| } finally { | ||
| NativeScan.free(pointer); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| static Scan fromPointer(Session session, long pointer) { | ||
|
|
@@ -40,31 +59,50 @@ static Scan fromPointer(Session session, long pointer) { | |
| /** | ||
| * Arrow schema produced by this scan. Must be called before the first call to {@link #hasNext()}/{@link #next()}. | ||
| */ | ||
| public Schema arrowSchema(BufferAllocator allocator) { | ||
| public synchronized Schema arrowSchema(BufferAllocator allocator) { | ||
| Preconditions.checkState(!closed, "scan is closed"); | ||
| try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) { | ||
| NativeScan.arrowSchema(session.nativePointer(), pointer, schema.memoryAddress()); | ||
| return Data.importSchema(allocator, schema, null); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * {@inheritDoc} | ||
| * | ||
| * @throws IllegalStateException if this scan has been closed | ||
| */ | ||
| @Override | ||
| public boolean hasNext() { | ||
| if (primed) { | ||
| return nextPartitionPointer != 0; | ||
| public synchronized boolean hasNext() { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this likely shouldn't be synchronised like datasource? |
||
| Preconditions.checkState(!closed, "scan is closed"); | ||
| if (!primed) { | ||
| nextPartitionPointer.set(NativeScan.nextPartition(pointer)); | ||
| primed = true; | ||
| } | ||
| nextPartitionPointer = NativeScan.nextPartition(pointer); | ||
| primed = true; | ||
| return nextPartitionPointer != 0; | ||
| return nextPartitionPointer.get() != 0; | ||
| } | ||
|
|
||
| /** | ||
| * {@inheritDoc} | ||
| * | ||
| * @throws IllegalStateException if this scan has been closed | ||
| */ | ||
| @Override | ||
| public Partition next() { | ||
| public synchronized Partition next() { | ||
| if (!hasNext()) { | ||
| throw new NoSuchElementException(); | ||
| } | ||
| long ptr = nextPartitionPointer; | ||
| nextPartitionPointer = 0; | ||
| long ptr = nextPartitionPointer.getAndSet(0); | ||
| primed = false; | ||
| return Partition.fromPointer(session, ptr); | ||
| } | ||
|
|
||
| /** Releases this scan once, without closing partitions already returned to the caller. */ | ||
| @Override | ||
| public synchronized void close() { | ||
| if (!closed) { | ||
| closed = true; | ||
| cleanable.clean(); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we need all the synchronised blocks?
Reference.reachabilityFence(this)is a noop forsynchronised. I think we should addReference.reachabilityFenceand remove synchronised unless I am missing something