diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java index 445b0ac7691..4a875be89f8 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java @@ -22,12 +22,18 @@ public final class NativeFiles { private NativeFiles() {} - /** List all Vortex files reachable under {@code uri}. */ + /** + * List all Vortex files reachable under {@code uri}, returning percent-encoded URIs that can be passed to + * {@link #delete(Session, String[], Map)} or the URI-based reader and writer APIs. + */ public static List listFiles(Session session, String uri, Map options) { return listFiles(session.nativePointer(), uri, options); } - /** Delete files at the given URIs. Silently tolerates an empty list. */ + /** + * Delete files at the given URIs or local paths. Each target is resolved independently using {@code options}. + * Silently tolerates an empty list. + */ public static void delete(Session session, String[] uris, Map options) { delete(session.nativePointer(), uris, options); } diff --git a/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java b/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java new file mode 100644 index 00000000000..4cdc1402c77 --- /dev/null +++ b/java/vortex-jni/src/test/java/dev/vortex/jni/NativeFilesTest.java @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public final class NativeFilesTest { + @TempDir + Path tempDir; + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void listedUrisAddressTheWrittenFiles(boolean useUri) throws IOException { + Path directory = Files.createDirectories(tempDir.resolve("list delete %20 Côte+")); + Path literalPercent = directory.resolve("literal%20name.vortex"); + Path decodedSibling = directory.resolve("literal name.vortex"); + Set files = Set.of(literalPercent, decodedSibling, directory.resolve("Côte+%?#.vortex")); + Session session = Session.create(); + BufferAllocator allocator = ArrowAllocation.rootAllocator(); + Schema schema = new Schema(List.of(Field.notNullable("value", new ArrowType.Int(64, true)))); + for (Path file : files) { + VortexWriter.builder(session, pathOrUri(file, useUri), schema, allocator) + .build() + .close(); + } + + // macOS can normalize Unicode differently in paths constructed from strings and URIs. + Set expected = new HashSet<>(); + for (Path file : files) { + expected.add(file.toRealPath()); + } + List listedUris = NativeFiles.listFiles(session, pathOrUri(directory, useUri), Map.of()); + assertEquals(files.size(), listedUris.size()); + Map listed = new HashMap<>(); + for (String uri : listedUris) { + listed.put(Path.of(URI.create(uri)).toRealPath(), uri); + } + assertEquals(expected, listed.keySet()); + + NativeFiles.delete(session, new String[] {listed.get(literalPercent.toRealPath())}, Map.of()); + assertFalse(Files.exists(literalPercent)); + assertTrue(Files.exists(decodedSibling), "deleting a percent-literal key must preserve its decoded sibling"); + assertTrue(Files.exists(directory.resolve("Côte+%?#.vortex"))); + + String[] remaining = files.stream() + .filter(file -> !file.equals(literalPercent)) + .map(file -> pathOrUri(file, useUri)) + .toArray(String[]::new); + NativeFiles.delete(session, remaining, Map.of()); + for (Path file : files) { + assertFalse(Files.exists(file), "delete must address the same file as the writer"); + } + } + + @Test + public void invalidTargetIsRejectedBeforeDeletingFiles() throws IOException { + Path barePath = Files.createFile(tempDir.resolve("bare path.vortex")); + Path uriPath = Files.createFile(tempDir.resolve("uri%20path.vortex")); + String[] targets = {barePath.toString(), uriPath.toUri().toString(), "unsupported://bucket/file.vortex"}; + + assertThrows(RuntimeException.class, () -> NativeFiles.delete(Session.create(), targets, Map.of())); + assertTrue(Files.exists(barePath), "all targets must resolve before any file is deleted"); + assertTrue(Files.exists(uriPath), "all targets must resolve before any file is deleted"); + } + + private static String pathOrUri(Path path, boolean useUri) { + return useUri ? path.toUri().toString() : path.toString(); + } +} diff --git a/vortex-jni/src/file.rs b/vortex-jni/src/file.rs index 9799b7d9139..fe7b7d48cd4 100644 --- a/vortex-jni/src/file.rs +++ b/vortex-jni/src/file.rs @@ -16,6 +16,8 @@ use jni::objects::JObjectArray; use jni::objects::JString; use jni::sys::jlong; use jni::sys::jobject; +use object_store::path::Path; +use url::Url; use vortex::buffer::ByteBuffer; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -297,6 +299,29 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_readStatisticsFromReadabl }) } +/// Recover the store's URL mount without decoding its namespace (for example an HF revision). +fn listing_base_url(url: &Url, prefix: &str) -> VortexResult { + let segments: Vec<_> = url.path().split('/').filter(|s| !s.is_empty()).collect(); + let depth = (0..=segments.len()) + .find(|&depth| { + Path::from_url_path(segments[depth..].join("/")) + .is_ok_and(|path| path.as_ref() == prefix) + }) + .ok_or_else(|| vortex_err!("Resolved listing prefix does not match its URL path"))?; + let mut base = url.clone(); + base.set_path(&format!("/{}", segments[..depth].join("/"))); + Ok(base) +} + +fn listed_file_url(base: &Url, key: &str) -> VortexResult { + let mut url = base.clone(); + url.path_segments_mut() + .map_err(|_| vortex_err!("Listing URL cannot contain path segments"))? + .pop_if_empty() + .extend(key.split('/')); + Ok(url) +} + /// List Vortex files under the given URI prefix. Returns a `java.util.ArrayList`. #[unsafe(no_mangle)] pub extern "system" fn Java_dev_vortex_jni_NativeFiles_listFiles( @@ -314,15 +339,14 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_listFiles( let properties = extract_properties(env, &options)?; let (fs, prefix) = object_store_fs(&url, &properties, session.handle())?; + let base = listing_base_url(&url, &prefix)?; let mut stream = fs.list(&prefix); let paths_vec = RUNTIME.block_on(async move { let mut paths = Vec::new(); while let Some(file) = stream.next().await { - let mut found = url.clone(); - found.set_path(&file?.path); - paths.push(found.to_string()); + paths.push(listed_file_url(&base, &file?.path)?.to_string()); } VortexResult::Ok(paths) @@ -338,6 +362,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_listFiles( for path in paths_vec.into_iter() { let path_string = env.new_string(path)?; paths_list.add(env, path_string.as_ref())?; + env.delete_local_ref(path_string); } Ok(raw) @@ -362,22 +387,22 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_delete( let uri = uris.get_element(env, idx)?; let uri_str = env.cast_local::(uri)?; delete_uris.push(uri_str.try_to_string(env)?); + env.delete_local_ref(uri_str); } if delete_uris.is_empty() { return Ok(()); } - let store_url = parse_uri_or_path(&delete_uris[0])?; - let properties = extract_properties(env, &options)?; - - let (fs, _path) = object_store_fs(&store_url, &properties, session.handle())?; + let targets = delete_uris + .iter() + .map(|uri| object_store_fs(&parse_uri_or_path(uri)?, &properties, session.handle())) + .collect::>>()?; RUNTIME.block_on(async { - for uri in delete_uris { - let url = parse_uri_or_path(&uri)?; - fs.delete(url.path()).await?; + for (fs, path) in targets { + fs.delete(&path).await?; } VortexResult::Ok(()) })?; @@ -385,3 +410,93 @@ pub extern "system" fn Java_dev_vortex_jni_NativeFiles_delete( Ok(()) }); } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use object_store::registry::ObjectStoreRegistry; + use rstest::rstest; + use vortex::error::VortexResult; + use vortex::file::multi::parse_uri_or_path; + use vortex_cloud::Registry; + + use super::listed_file_url; + use super::listing_base_url; + + #[rstest] + #[case::file( + "file:///tmp/with%20space", + "tmp/with space", + "tmp/with space/literal%20name.vortex", + "file:///tmp/with%20space/literal%2520name.vortex" + )] + #[case::s3( + "s3://bucket/dir", + "dir", + "dir/Côte+%?#.vortex", + "s3://bucket/dir/C%C3%B4te+%25%3F%23.vortex" + )] + #[case::path_style_bucket( + "https://s3.us-east-1.amazonaws.com/bucket/dir", + "dir", + "dir/literal%20name.vortex", + "https://s3.us-east-1.amazonaws.com/bucket/dir/literal%2520name.vortex" + )] + #[case::bucket_root( + "https://s3.us-east-1.amazonaws.com/bucket/", + "", + "with space.vortex", + "https://s3.us-east-1.amazonaws.com/bucket/with%20space.vortex" + )] + #[case::encoded_key_separator( + "s3://bucket/dir%2Fsubdir", + "dir/subdir", + "dir/subdir/file.vortex", + "s3://bucket/dir/subdir/file.vortex" + )] + #[case::hf_revision( + "hf://datasets/org/name@refs%2Fconvert%2Fparquet/dir", + "dir", + "dir/with space.vortex", + "hf://datasets/org/name@refs%2Fconvert%2Fparquet/dir/with%20space.vortex" + )] + fn listed_urls_preserve_keys_and_mounts( + #[case] input: &str, + #[case] expected_prefix: &str, + #[case] key: &str, + #[case] expected: &str, + ) -> VortexResult<()> { + let url = parse_uri_or_path(input)?; + let registry = Registry::with_vars([("aws_region".to_string(), "us-east-1".to_string())]); + let (store, prefix) = registry.resolve(&url)?; + assert_eq!(prefix.as_ref(), expected_prefix); + + let base = listing_base_url(&url, prefix.as_ref())?; + let listed = listed_file_url(&base, key)?; + assert_eq!(listed.as_str(), expected); + + let (listed_store, listed_path) = registry.resolve(&listed)?; + assert!(Arc::ptr_eq(&store, &listed_store)); + assert_eq!(listed_path.as_ref(), key); + Ok(()) + } + + #[test] + fn listing_base_accepts_a_normalized_prefix() -> VortexResult<()> { + let url = parse_uri_or_path("https://s3.us-east-1.amazonaws.com/bucket//dir//")?; + let base = listing_base_url(&url, "dir")?; + assert_eq!( + listed_file_url(&base, "dir/file.vortex")?.as_str(), + "https://s3.us-east-1.amazonaws.com/bucket/dir/file.vortex" + ); + Ok(()) + } + + #[test] + fn listing_rejects_an_unmatched_prefix() -> VortexResult<()> { + let url = parse_uri_or_path("s3://bucket/dir")?; + assert!(listing_base_url(&url, "other").is_err()); + Ok(()) + } +}