From 8b534dbc5e70d18d91025d6eb024a59147201ff2 Mon Sep 17 00:00:00 2001 From: Christopher Wirt Date: Wed, 4 Nov 2026 08:19:59 +0000 Subject: [PATCH 1/2] fix(jni): preserve literal keys in file listing and deletion Signed-off-by: Christopher Wirt --- .../main/java/dev/vortex/jni/NativeFiles.java | 10 +- .../java/dev/vortex/jni/NativeFilesTest.java | 71 +++++++++++ vortex-jni/src/file.rs | 114 ++++++++++++++++-- 3 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java index 445b0ac7691..4a875be89f8 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java @@ -22,12 +22,18 @@ public final class NativeFiles { private NativeFiles() {} - /** List all Vortex files reachable under {@code uri}. */ + /** + * List all Vortex files reachable under {@code uri}, returning percent-encoded URIs that can be passed to + * {@link #delete(Session, String[], Map)} or the URI-based reader and writer APIs. + */ public static List listFiles(Session session, String uri, Map options) { return listFiles(session.nativePointer(), uri, options); } - /** Delete files at the given URIs. Silently tolerates an empty list. */ + /** + * Delete files at the given URIs or local paths. Each target is resolved independently using {@code options}. + * Silently tolerates an empty list. + */ public static void delete(Session session, String[] uris, Map options) { delete(session.nativePointer(), uris, options); } diff --git a/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java b/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java new file mode 100644 index 00000000000..db7ceec4fc6 --- /dev/null +++ b/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public final class NativeFilesTest { + @TempDir + Path tempDir; + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void listedUrisAddressTheWrittenFiles(boolean useUri) throws IOException { + Path directory = Files.createDirectories(tempDir.resolve("list delete %20 Côte+")); + Path literalPercent = directory.resolve("literal%20name.vortex"); + Path decodedSibling = directory.resolve("literal name.vortex"); + Set files = Set.of(literalPercent, decodedSibling, directory.resolve("Côte+%?#.vortex")); + Session session = Session.create(); + BufferAllocator allocator = ArrowAllocation.rootAllocator(); + Schema schema = new Schema(List.of(Field.notNullable("value", new ArrowType.Int(64, true)))); + for (Path file : files) { + VortexWriter.builder(session, pathOrUri(file, useUri), schema, allocator) + .build() + .close(); + } + + Map listed = NativeFiles.listFiles(session, pathOrUri(directory, useUri), Map.of()).stream() + .collect(Collectors.toMap(uri -> Path.of(URI.create(uri)), uri -> uri)); + assertEquals(files, listed.keySet()); + + NativeFiles.delete(session, new String[] {listed.get(literalPercent)}, Map.of()); + assertFalse(Files.exists(literalPercent)); + assertTrue(Files.exists(decodedSibling), "deleting a percent-literal key must preserve its decoded sibling"); + assertTrue(Files.exists(directory.resolve("Côte+%?#.vortex"))); + + String[] remaining = files.stream() + .filter(file -> !file.equals(literalPercent)) + .map(file -> pathOrUri(file, useUri)) + .toArray(String[]::new); + NativeFiles.delete(session, remaining, Map.of()); + for (Path file : files) { + assertFalse(Files.exists(file), "delete must address the same file as the writer"); + } + } + + private static String pathOrUri(Path path, boolean useUri) { + return useUri ? path.toUri().toString() : path.toString(); + } +} diff --git a/vortex-jni/src/file.rs b/vortex-jni/src/file.rs index 9799b7d9139..e4c7a31d61a 100644 --- a/vortex-jni/src/file.rs +++ b/vortex-jni/src/file.rs @@ -16,6 +16,8 @@ use jni::objects::JObjectArray; use jni::objects::JString; use jni::sys::jlong; use jni::sys::jobject; +use object_store::path::Path; +use url::Url; use vortex::buffer::ByteBuffer; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -297,6 +299,29 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_readStatisticsFromReadabl }) } +/// Recover the store's URL mount without decoding its namespace (for example an HF revision). +fn listing_base_url(url: &Url, prefix: &str) -> VortexResult { + let segments: Vec<_> = url.path().split('/').filter(|s| !s.is_empty()).collect(); + let depth = (0..=segments.len()) + .find(|&depth| { + Path::from_url_path(segments[depth..].join("/")) + .is_ok_and(|path| path.as_ref() == prefix) + }) + .ok_or_else(|| vortex_err!("Resolved listing prefix does not match its URL path"))?; + let mut base = url.clone(); + base.set_path(&format!("/{}", segments[..depth].join("/"))); + Ok(base) +} + +fn listed_file_url(base: &Url, key: &str) -> VortexResult { + let mut url = base.clone(); + url.path_segments_mut() + .map_err(|_| vortex_err!("Listing URL cannot contain path segments"))? + .pop_if_empty() + .extend(key.split('/')); + Ok(url) +} + /// List Vortex files under the given URI prefix. Returns a `java.util.ArrayList`. #[unsafe(no_mangle)] pub extern "system" fn Java_dev_vortex_jni_NativeFiles_listFiles( @@ -314,15 +339,14 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_listFiles( let properties = extract_properties(env, &options)?; let (fs, prefix) = object_store_fs(&url, &properties, session.handle())?; + let base = listing_base_url(&url, &prefix)?; let mut stream = fs.list(&prefix); let paths_vec = RUNTIME.block_on(async move { let mut paths = Vec::new(); while let Some(file) = stream.next().await { - let mut found = url.clone(); - found.set_path(&file?.path); - paths.push(found.to_string()); + paths.push(listed_file_url(&base, &file?.path)?.to_string()); } VortexResult::Ok(paths) @@ -368,16 +392,15 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_delete( return Ok(()); } - let store_url = parse_uri_or_path(&delete_uris[0])?; - let properties = extract_properties(env, &options)?; - - let (fs, _path) = object_store_fs(&store_url, &properties, session.handle())?; + let targets = delete_uris + .iter() + .map(|uri| object_store_fs(&parse_uri_or_path(uri)?, &properties, session.handle())) + .collect::>>()?; RUNTIME.block_on(async { - for uri in delete_uris { - let url = parse_uri_or_path(&uri)?; - fs.delete(url.path()).await?; + for (fs, path) in targets { + fs.delete(&path).await?; } VortexResult::Ok(()) })?; @@ -385,3 +408,74 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_delete( Ok(()) }); } + +#[cfg(test)] +mod tests { + use rstest::rstest; + use vortex::error::VortexResult; + use vortex::file::multi::parse_uri_or_path; + + use super::listed_file_url; + use super::listing_base_url; + + #[rstest] + #[case::file( + "file:///tmp/with%20space", + "tmp/with space", + "tmp/with space/literal%20name.vortex", + "file:///tmp/with%20space/literal%2520name.vortex" + )] + #[case::s3( + "s3://bucket/dir", + "dir", + "dir/Côte+%?#.vortex", + "s3://bucket/dir/C%C3%B4te+%25%3F%23.vortex" + )] + #[case::path_style_bucket( + "https://s3.us-east-1.amazonaws.com/bucket/dir", + "dir", + "dir/literal%20name.vortex", + "https://s3.us-east-1.amazonaws.com/bucket/dir/literal%2520name.vortex" + )] + #[case::bucket_root( + "https://s3.us-east-1.amazonaws.com/bucket/", + "", + "with space.vortex", + "https://s3.us-east-1.amazonaws.com/bucket/with%20space.vortex" + )] + #[case::repeated_separators( + "https://s3.us-east-1.amazonaws.com/bucket//dir//", + "dir", + "dir/file.vortex", + "https://s3.us-east-1.amazonaws.com/bucket/dir/file.vortex" + )] + #[case::encoded_key_separator( + "s3://bucket/dir%2Fsubdir", + "dir/subdir", + "dir/subdir/file.vortex", + "s3://bucket/dir/subdir/file.vortex" + )] + #[case::hf_revision( + "hf://datasets/org/name@refs%2Fconvert%2Fparquet/dir", + "dir", + "dir/with space.vortex", + "hf://datasets/org/name@refs%2Fconvert%2Fparquet/dir/with%20space.vortex" + )] + fn listed_urls_preserve_keys_and_mounts( + #[case] input: &str, + #[case] prefix: &str, + #[case] key: &str, + #[case] expected: &str, + ) -> VortexResult<()> { + let base = listing_base_url(&parse_uri_or_path(input)?, prefix)?; + assert_eq!(listed_file_url(&base, key)?.as_str(), expected); + Ok(()) + } + + #[test] + fn listing_rejects_an_unmatched_prefix() -> VortexResult<()> { + let url = parse_uri_or_path("s3://bucket/dir")?; + assert!(listing_base_url(&url, "other").is_err()); + Ok(()) + } +} From cb1eda20b4621937c2dc15052c5b33680b131dde Mon Sep 17 00:00:00 2001 From: Christopher Wirt Date: Wed, 4 Nov 2026 09:41:05 +0000 Subject: [PATCH 2/2] fix(jni): tighten file utility cleanup and regression coverage Signed-off-by: Christopher Wirt --- .../java/dev/vortex/jni/NativeFilesTest.java | 33 +++++++++++++--- vortex-jni/src/file.rs | 39 ++++++++++++++----- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java b/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java index db7ceec4fc6..4cdc1402c77 100644 --- a/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java +++ b/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import dev.vortex.api.Session; @@ -14,14 +15,16 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -46,11 +49,20 @@ public void listedUrisAddressTheWrittenFiles(boolean useUri) throws IOException .close(); } - Map listed = NativeFiles.listFiles(session, pathOrUri(directory, useUri), Map.of()).stream() - .collect(Collectors.toMap(uri -> Path.of(URI.create(uri)), uri -> uri)); - assertEquals(files, listed.keySet()); + // macOS can normalize Unicode differently in paths constructed from strings and URIs. + Set expected = new HashSet<>(); + for (Path file : files) { + expected.add(file.toRealPath()); + } + List listedUris = NativeFiles.listFiles(session, pathOrUri(directory, useUri), Map.of()); + assertEquals(files.size(), listedUris.size()); + Map listed = new HashMap<>(); + for (String uri : listedUris) { + listed.put(Path.of(URI.create(uri)).toRealPath(), uri); + } + assertEquals(expected, listed.keySet()); - NativeFiles.delete(session, new String[] {listed.get(literalPercent)}, Map.of()); + NativeFiles.delete(session, new String[] {listed.get(literalPercent.toRealPath())}, Map.of()); assertFalse(Files.exists(literalPercent)); assertTrue(Files.exists(decodedSibling), "deleting a percent-literal key must preserve its decoded sibling"); assertTrue(Files.exists(directory.resolve("Côte+%?#.vortex"))); @@ -65,6 +77,17 @@ public void listedUrisAddressTheWrittenFiles(boolean useUri) throws IOException } } + @Test + public void invalidTargetIsRejectedBeforeDeletingFiles() throws IOException { + Path barePath = Files.createFile(tempDir.resolve("bare path.vortex")); + Path uriPath = Files.createFile(tempDir.resolve("uri%20path.vortex")); + String[] targets = {barePath.toString(), uriPath.toUri().toString(), "unsupported://bucket/file.vortex"}; + + assertThrows(RuntimeException.class, () -> NativeFiles.delete(Session.create(), targets, Map.of())); + assertTrue(Files.exists(barePath), "all targets must resolve before any file is deleted"); + assertTrue(Files.exists(uriPath), "all targets must resolve before any file is deleted"); + } + private static String pathOrUri(Path path, boolean useUri) { return useUri ? path.toUri().toString() : path.toString(); } diff --git a/vortex-jni/src/file.rs b/vortex-jni/src/file.rs index e4c7a31d61a..fe7b7d48cd4 100644 --- a/vortex-jni/src/file.rs +++ b/vortex-jni/src/file.rs @@ -362,6 +362,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_listFiles( for path in paths_vec.into_iter() { let path_string = env.new_string(path)?; paths_list.add(env, path_string.as_ref())?; + env.delete_local_ref(path_string); } Ok(raw) @@ -386,6 +387,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_delete( let uri = uris.get_element(env, idx)?; let uri_str = env.cast_local::(uri)?; delete_uris.push(uri_str.try_to_string(env)?); + env.delete_local_ref(uri_str); } if delete_uris.is_empty() { @@ -411,9 +413,13 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_delete( #[cfg(test)] mod tests { + use std::sync::Arc; + + use object_store::registry::ObjectStoreRegistry; use rstest::rstest; use vortex::error::VortexResult; use vortex::file::multi::parse_uri_or_path; + use vortex_cloud::Registry; use super::listed_file_url; use super::listing_base_url; @@ -443,12 +449,6 @@ mod tests { "with space.vortex", "https://s3.us-east-1.amazonaws.com/bucket/with%20space.vortex" )] - #[case::repeated_separators( - "https://s3.us-east-1.amazonaws.com/bucket//dir//", - "dir", - "dir/file.vortex", - "https://s3.us-east-1.amazonaws.com/bucket/dir/file.vortex" - )] #[case::encoded_key_separator( "s3://bucket/dir%2Fsubdir", "dir/subdir", @@ -463,12 +463,33 @@ mod tests { )] fn listed_urls_preserve_keys_and_mounts( #[case] input: &str, - #[case] prefix: &str, + #[case] expected_prefix: &str, #[case] key: &str, #[case] expected: &str, ) -> VortexResult<()> { - let base = listing_base_url(&parse_uri_or_path(input)?, prefix)?; - assert_eq!(listed_file_url(&base, key)?.as_str(), expected); + let url = parse_uri_or_path(input)?; + let registry = Registry::with_vars([("aws_region".to_string(), "us-east-1".to_string())]); + let (store, prefix) = registry.resolve(&url)?; + assert_eq!(prefix.as_ref(), expected_prefix); + + let base = listing_base_url(&url, prefix.as_ref())?; + let listed = listed_file_url(&base, key)?; + assert_eq!(listed.as_str(), expected); + + let (listed_store, listed_path) = registry.resolve(&listed)?; + assert!(Arc::ptr_eq(&store, &listed_store)); + assert_eq!(listed_path.as_ref(), key); + Ok(()) + } + + #[test] + fn listing_base_accepts_a_normalized_prefix() -> VortexResult<()> { + let url = parse_uri_or_path("https://s3.us-east-1.amazonaws.com/bucket//dir//")?; + let base = listing_base_url(&url, "dir")?; + assert_eq!( + listed_file_url(&base, "dir/file.vortex")?.as_str(), + "https://s3.us-east-1.amazonaws.com/bucket/dir/file.vortex" + ); Ok(()) }