From d4b41184ea187827d65c8fe1e80d812d529a1fb4 Mon Sep 17 00:00:00 2001 From: KK <37928252+dragonknightk@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:28:51 +0800 Subject: [PATCH 1/3] [core] Delete files of aborted writes with bounded concurrency ### Purpose Closes issue 11111 `FileStoreCommitImpl.abort` now deletes the files of an aborted write concurrently, bounded by `file-operation.thread-num`. Before this change, each delete ran on the caller thread one after another, so abort time grew with the file count times the per-delete latency. - Abort first collects the paths with the existing rules, then deletes them as one batch. The set of files owned by the aborted write is unchanged, including the PR 10582 rule that keeps files reused by compaction. - The deletes run on the shared `FileOperationThreadPool`, the same pool and option that snapshot expiration (PR 8241) uses. No option or pool is added. Each delete runs with the caller's context class loader, as the sequential abort did. - Every path gets a delete attempt, and the caller blocks until all attempts have finished, including when a delete fails or the caller is interrupted. - Each delete still goes through `FileIO.deleteQuietly`, so an IOException is logged and does not throw. Any other exception is rethrown as is after all attempts, with later failures suppressed into the first in input order. - An interrupt on the caller thread does not cancel the cleanup. Abort keeps waiting for all attempts and restores the interrupt flag before returning or throwing. ### Tests - With `file-operation.thread-num = 4` and a FileIO that delays each operation, the deletes of one abort overlap, with 2 to 4 in flight. On the code before this change at most one delete is in flight, so this test fails there. Each delete sees the caller's context class loader, and the pool threads keep their own context class loaders after abort. - All files owned by the aborted write are deleted, and a file reused by compaction stays. - When one delete fails with an IOException, abort does not throw and still deletes every other file. - When the caller is interrupted before abort and `file-operation.thread-num` is below the shared pool size, abort still deletes every file and the interrupt flag stays set. - When the caller is interrupted while abort waits for the deletes, abort still waits for and deletes every file and the interrupt flag stays set. - In the same interrupted case with two deletes failing with unchecked exceptions, abort deletes every other file, throws the first failure in input order with its original type, and carries the later one as its only suppressed exception. The interrupt flag stays set. ## AI Disclosure Human-led contribution. Grok Bot, working for dragonknightk, assisted with this change, including the code, tests, and description. All submitted content was reviewed by a human before submission. Signed-off-by: KK <37928252+dragonknightk@users.noreply.github.com> --- .../paimon/operation/FileStoreCommitImpl.java | 72 ++++- .../apache/paimon/TestAppendFileStore.java | 8 +- .../paimon/operation/FileStoreCommitTest.java | 291 ++++++++++++++++++ .../org/apache/paimon/utils/SlowFileIO.java | 30 ++ 4 files changed, 397 insertions(+), 4 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 7e7c14da49ec..3f4ca107e2a0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -80,6 +80,8 @@ import org.apache.paimon.table.sink.CommitPreCallback; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.DataFilePathFactories; +import org.apache.paimon.utils.ExceptionUtils; +import org.apache.paimon.utils.FileOperationThreadPool; import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.IndexFilePathFactories; @@ -107,6 +109,9 @@ import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.FutureTask; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -732,6 +737,7 @@ public void truncateTable(long commitIdentifier) { public void abort(List commitMessages) { DataFilePathFactories dataFactories = new DataFilePathFactories(pathFactory); IndexFilePathFactories indexFactories = new IndexFilePathFactories(pathFactory); + List filesToDelete = new ArrayList<>(); for (CommitMessage message : commitMessages) { DataFilePathFactory dataPathFactory = dataFactories.get(message.partition(), message.bucket()); @@ -745,7 +751,7 @@ public void abort(List commitMessages) { // New files may also be compaction inputs, but abort still owns their cleanup. for (DataFileMeta file : dataFilesToDelete) { - file.collectFiles(dataPathFactory).forEach(fileIO::deleteQuietly); + filesToDelete.addAll(file.collectFiles(dataPathFactory)); } // A level upgrade may reuse a main file while adding new auxiliary files. @@ -757,16 +763,76 @@ public void abort(List commitMessages) { for (DataFileMeta file : commitMessage.compactIncrement().compactAfter()) { file.collectFiles(dataPathFactory).stream() .filter(path -> !compactInputPaths.contains(path)) - .forEach(fileIO::deleteQuietly); + .forEach(filesToDelete::add); } List indexFilesToDelete = new ArrayList<>(); indexFilesToDelete.addAll(commitMessage.newFilesIncrement().newIndexFiles()); indexFilesToDelete.addAll(commitMessage.compactIncrement().newIndexFiles()); for (IndexFileMeta file : indexFilesToDelete) { - fileIO.deleteQuietly(indexPathFactory.toPath(file)); + filesToDelete.add(indexPathFactory.toPath(file)); } } + deleteConcurrently(filesToDelete); + } + + /** + * Attempts every delete and waits for all of them even when the caller is interrupted. The + * first unchecked failure in input order is rethrown as is, later ones are suppressed into it. + * Do not call this from a {@link FileOperationThreadPool} thread, it may wait for itself. + */ + private void deleteConcurrently(List files) { + if (files.isEmpty()) { + return; + } + + ExecutorService executor = + FileOperationThreadPool.getExecutorService(options.fileOperationThreadNum()); + boolean interrupted = false; + // Run each delete with the caller's context class loader, as the sequential abort did. + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + List> deletes = new ArrayList<>(files.size()); + for (Path file : files) { + FutureTask delete = + new FutureTask<>( + () -> { + Thread thread = Thread.currentThread(); + ClassLoader previous = thread.getContextClassLoader(); + thread.setContextClassLoader(classLoader); + try { + fileIO.deleteQuietly(file); + } finally { + thread.setContextClassLoader(previous); + } + }, + null); + // Clear the flag first, otherwise the bounded executor skips its permit and the bound. + interrupted |= Thread.interrupted(); + executor.execute(delete); + deletes.add(delete); + } + + Throwable failure = null; + for (FutureTask delete : deletes) { + while (true) { + try { + delete.get(); + break; + } catch (InterruptedException e) { + interrupted = true; + } catch (ExecutionException e) { + failure = ExceptionUtils.firstOrSuppressed(e.getCause(), failure); + break; + } + } + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + if (failure != null) { + ExceptionUtils.rethrow(failure); + } } @Override diff --git a/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java b/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java index 827ec9d499c3..c843fba39992 100644 --- a/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java +++ b/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java @@ -203,7 +203,13 @@ public CommitMessageImpl writeDVIndexFiles( public static TestAppendFileStore createAppendStore( java.nio.file.Path tempDir, Map options) throws Exception { - String root = TraceableFileIO.SCHEME + "://" + tempDir.toString(); + return createAppendStore(tempDir, options, TraceableFileIO.SCHEME); + } + + public static TestAppendFileStore createAppendStore( + java.nio.file.Path tempDir, Map options, String scheme) + throws Exception { + String root = scheme + "://" + tempDir.toString(); Path path = new Path(tempDir.toUri()); FileIO fileIO = FileIOFinder.find(new Path(root)); SchemaManager schemaManage = new FileSystemSchemaManager(new LocalFileIO(), path); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index b85329fd9025..0215f6eee991 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -75,8 +75,10 @@ import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FailingFileIO; +import org.apache.paimon.utils.FileOperationThreadPool; import org.apache.paimon.utils.JsonSerdeUtil; import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.SlowFileIO; import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TraceableFileIO; @@ -95,6 +97,8 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -107,7 +111,11 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -1327,6 +1335,289 @@ public void testAbortCleansUncommittedCompactionInputs(String changelogProducer) } } + @Test + public void testAbortDeletesFilesConcurrently() throws Exception { + Map options = new HashMap<>(); + options.put(CoreOptions.FILE_OPERATION_THREAD_NUM.key(), "4"); + TestAppendFileStore store = + TestAppendFileStore.createAppendStore(tempDir, options, SlowFileIO.SCHEME); + BinaryRow partition = gen.getPartition(gen.next()); + DataFilePathFactory pathFactory = + store.pathFactory().createDataFilePathFactory(partition, 0); + List newFiles = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + newFiles.add(createDataFileWithExtraFile(store, pathFactory.newPath(), false)); + } + DataFileMeta reused = createDataFileWithExtraFile(store, pathFactory.newPath(), false); + DataFileMeta compactNew = createDataFileWithExtraFile(store, pathFactory.newPath(), false); + CommitMessage commitMessage = + new CommitMessageImpl( + partition, + 0, + store.options().bucket(), + new DataIncrement( + newFiles, Collections.emptyList(), Collections.emptyList()), + new CompactIncrement( + Collections.singletonList(reused), + Arrays.asList(reused.upgrade(1), compactNew), + Collections.emptyList())); + + // start every pool thread now, so no delete runs on a thread created by the caller below + // that inherited its context class loader; size the shared pool for 4 first, as a pool + // with fewer threads is replaced by a new one on the abort below + FileOperationThreadPool.getExecutorService(4); + List poolLoaders = poolThreadContextClassLoaders(); + Thread caller = Thread.currentThread(); + ClassLoader original = caller.getContextClassLoader(); + try (URLClassLoader callerLoader = new URLClassLoader(new URL[0], original)) { + SlowFileIO.reset(); + SlowFileIO.setDelayMillis(50); + caller.setContextClassLoader(callerLoader); + try (FileStoreCommitImpl commit = store.newCommit()) { + commit.abort(Collections.singletonList(commitMessage)); + } finally { + caller.setContextClassLoader(original); + } + // issue 11111: abort deleted one file at a time, so at most one delete was in flight; + // the deletes now overlap and stay within file-operation.thread-num + assertThat(SlowFileIO.maxActiveOperations()).isBetween(2, 4); + // the sequential abort deleted with the caller's context class loader, the pool + // threads use it only while deleting + assertThat(SlowFileIO.deleteContextClassLoaders()) + .isNotEmpty() + .allMatch(loader -> loader == callerLoader); + assertThat(poolThreadContextClassLoaders()) + .containsExactlyInAnyOrderElementsOf(poolLoaders); + } finally { + SlowFileIO.reset(); + } + + newFiles.add(compactNew); + for (DataFileMeta file : newFiles) { + for (Path path : file.collectFiles(pathFactory)) { + assertThat(store.fileIO().exists(path)).as("aborted file %s", path).isFalse(); + } + } + // PR 10582: a file reused by compaction is not owned by the aborted write + for (Path path : reused.collectFiles(pathFactory)) { + assertThat(store.fileIO().exists(path)).as("reused file %s", path).isTrue(); + } + } + + @Test + public void testAbortContinuesWhenOneDeleteFails() throws Exception { + Map options = new HashMap<>(); + options.put(CoreOptions.FILE_OPERATION_THREAD_NUM.key(), "4"); + TestAppendFileStore store = TestAppendFileStore.createAppendStore(tempDir, options); + BinaryRow partition = gen.getPartition(gen.next()); + DataFilePathFactory pathFactory = + store.pathFactory().createDataFilePathFactory(partition, 0); + List newFiles = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + newFiles.add(createDataFileWithExtraFile(store, pathFactory.newPath(), false)); + } + // A non-empty directory cannot be deleted non-recursively, so this delete fails for real. + DataFileMeta first = newFiles.get(0); + Path undeletable = new Path(pathFactory.newPath().getParent(), "undeletable-extra"); + store.fileIO().newOutputStream(new Path(undeletable, "child"), false).close(); + List extraFiles = new ArrayList<>(first.extraFiles()); + extraFiles.add(undeletable.getName()); + newFiles.set(0, first.copy(extraFiles)); + CommitMessage commitMessage = + new CommitMessageImpl( + partition, + 0, + store.options().bucket(), + new DataIncrement( + newFiles, Collections.emptyList(), Collections.emptyList()), + new CompactIncrement( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList())); + + // issue 11111: a failed delete is logged and neither throws nor skips the other files + try (FileStoreCommitImpl commit = store.newCommit()) { + commit.abort(Collections.singletonList(commitMessage)); + } + + assertThat(store.fileIO().exists(undeletable)).isTrue(); + for (DataFileMeta file : newFiles) { + for (Path path : file.collectFiles(pathFactory)) { + if (!path.equals(undeletable)) { + assertThat(store.fileIO().exists(path)).as("aborted file %s", path).isFalse(); + } + } + } + } + + @Test + public void testAbortDeletesAllFilesWhenCallerIsInterrupted() throws Exception { + // thread-num 1 is below the shared pool size, so deletes go through the bounded executor + Map options = new HashMap<>(); + options.put(CoreOptions.FILE_OPERATION_THREAD_NUM.key(), "1"); + TestAppendFileStore store = TestAppendFileStore.createAppendStore(tempDir, options); + BinaryRow partition = gen.getPartition(gen.next()); + DataFilePathFactory pathFactory = + store.pathFactory().createDataFilePathFactory(partition, 0); + List newFiles = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + newFiles.add(createDataFileWithExtraFile(store, pathFactory.newPath(), false)); + } + CommitMessage commitMessage = newFilesCommitMessage(store, partition, newFiles); + + boolean interruptedAfterAbort; + try (FileStoreCommitImpl commit = store.newCommit()) { + Thread.currentThread().interrupt(); + try { + commit.abort(Collections.singletonList(commitMessage)); + } finally { + interruptedAfterAbort = Thread.interrupted(); + } + } + + // issue 11111: the sequential abort deleted every file under an interrupt, and the + // caller's interrupt flag must survive the cleanup + assertThat(interruptedAfterAbort).isTrue(); + assertAllDeleted(store, pathFactory, newFiles); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testAbortWaitsForAllDeletesWhenInterruptedWhileWaiting(boolean failDeletes) + throws Exception { + // 2 data files with one extra file each, all 4 deletes can run at once + Map options = new HashMap<>(); + options.put(CoreOptions.FILE_OPERATION_THREAD_NUM.key(), "4"); + TestAppendFileStore store = + TestAppendFileStore.createAppendStore(tempDir, options, SlowFileIO.SCHEME); + BinaryRow partition = gen.getPartition(gen.next()); + DataFilePathFactory pathFactory = + store.pathFactory().createDataFilePathFactory(partition, 0); + List newFiles = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + newFiles.add(createDataFileWithExtraFile(store, pathFactory.newPath(), false)); + } + CommitMessage commitMessage = + newFilesCommitMessage(store, partition, new ArrayList<>(newFiles)); + List failing = new ArrayList<>(); + if (failDeletes) { + failing.add(newFiles.remove(0)); + failing.add(newFiles.remove(0)); + } + + Thread caller = Thread.currentThread(); + Throwable abortFailure = null; + boolean interruptedAfterAbort; + AtomicBoolean allDeletesStarted = new AtomicBoolean(); + try (FileStoreCommitImpl commit = store.newCommit()) { + SlowFileIO.reset(); + SlowFileIO.setDelayMillis(50); + SlowFileIO.failDeletesUnchecked( + failing.stream().map(DataFileMeta::fileName).toArray(String[]::new)); + // every delete has started, so all were submitted and abort is waiting for them + Thread interrupter = + new Thread( + () -> { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (SlowFileIO.delayedOperations() < 4) { + if (System.nanoTime() > deadline) { + return; + } + Thread.yield(); + } + allDeletesStarted.set(true); + caller.interrupt(); + }); + interrupter.start(); + try { + commit.abort(Collections.singletonList(commitMessage)); + } catch (Throwable t) { + abortFailure = t; + } finally { + interruptedAfterAbort = Thread.interrupted(); + interrupter.join(TimeUnit.SECONDS.toMillis(20)); + // drops an interrupt that arrives only after a broken abort returned early + Thread.interrupted(); + } + assertThat(interrupter.isAlive()).as("interrupter finished").isFalse(); + } finally { + SlowFileIO.reset(); + } + assertThat(allDeletesStarted).as("all deletes started within 10 seconds").isTrue(); + + // issue 11111: the sequential abort was not cut short by an interrupt, so every delete is + // still attempted and awaited, and the caller's interrupt flag survives the cleanup + assertThat(interruptedAfterAbort).isTrue(); + assertAllDeleted(store, pathFactory, newFiles); + if (failDeletes) { + // only IOException is swallowed by deleteQuietly; unchecked failures keep their type, + // the first in input order is thrown and the later one is suppressed into it + assertThat(abortFailure) + .isExactlyInstanceOf(IllegalStateException.class) + .hasMessageContaining(failing.get(0).fileName()); + assertThat(abortFailure.getSuppressed()) + .singleElement() + .isInstanceOf(IllegalStateException.class) + .extracting(Throwable::getMessage) + .asString() + .contains(failing.get(1).fileName()); + for (DataFileMeta file : failing) { + assertThat(store.fileIO().exists(pathFactory.toPath(file))).isTrue(); + for (String extraFile : file.extraFiles()) { + assertThat(store.fileIO().exists(pathFactory.toAlignedPath(extraFile, file))) + .as("extra file %s", extraFile) + .isFalse(); + } + } + } else { + assertThat(abortFailure).isNull(); + } + } + + private static CommitMessage newFilesCommitMessage( + TestAppendFileStore store, BinaryRow partition, List newFiles) { + return new CommitMessageImpl( + partition, + 0, + store.options().bucket(), + new DataIncrement(newFiles, Collections.emptyList(), Collections.emptyList()), + new CompactIncrement( + Collections.emptyList(), Collections.emptyList(), Collections.emptyList())); + } + + /** Occupies every thread of the shared file operation pool and returns their loaders. */ + private static List poolThreadContextClassLoaders() throws Exception { + ThreadPoolExecutor pool = + (ThreadPoolExecutor) FileOperationThreadPool.getExecutorService(0); + int poolSize = pool.getMaximumPoolSize(); + CountDownLatch allRunning = new CountDownLatch(poolSize); + List> loaders = new ArrayList<>(); + for (int i = 0; i < poolSize; i++) { + loaders.add( + pool.submit( + () -> { + allRunning.countDown(); + assertThat(allRunning.await(10, TimeUnit.SECONDS)).isTrue(); + return Thread.currentThread().getContextClassLoader(); + })); + } + List result = new ArrayList<>(); + for (Future loader : loaders) { + result.add(loader.get(20, TimeUnit.SECONDS)); + } + return result; + } + + private static void assertAllDeleted( + TestAppendFileStore store, DataFilePathFactory pathFactory, List files) + throws IOException { + for (DataFileMeta file : files) { + for (Path path : file.collectFiles(pathFactory)) { + assertThat(store.fileIO().exists(path)).as("aborted file %s", path).isFalse(); + } + } + } + private static DataFileMeta createDataFileWithExtraFile( TestAppendFileStore store, Path path, boolean external) throws Exception { store.fileIO().newOutputStream(path, false).close(); diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/SlowFileIO.java b/paimon-core/src/test/java/org/apache/paimon/utils/SlowFileIO.java index a8f68cd24d5e..1c6ea9e19a66 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/SlowFileIO.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/SlowFileIO.java @@ -27,8 +27,16 @@ import org.apache.paimon.fs.local.LocalFileIO; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; /** A local {@link FileIO} which adds configurable latency to each file operation. */ public class SlowFileIO extends LocalFileIO { @@ -39,12 +47,28 @@ public class SlowFileIO extends LocalFileIO { private static final AtomicLong DELAYED_OPERATIONS = new AtomicLong(); private static final AtomicInteger ACTIVE_OPERATIONS = new AtomicInteger(); private static final AtomicInteger MAX_ACTIVE_OPERATIONS = new AtomicInteger(); + private static final AtomicReference> UNCHECKED_DELETE_FAILURE_NAMES = + new AtomicReference<>(Collections.emptySet()); + private static final ConcurrentLinkedQueue DELETE_CONTEXT_CLASS_LOADERS = + new ConcurrentLinkedQueue<>(); public static void reset() { DELAY_MILLIS.set(0); DELAYED_OPERATIONS.set(0); ACTIVE_OPERATIONS.set(0); MAX_ACTIVE_OPERATIONS.set(0); + UNCHECKED_DELETE_FAILURE_NAMES.set(Collections.emptySet()); + DELETE_CONTEXT_CLASS_LOADERS.clear(); + } + + /** Context class loaders seen by delayed deletes since the last {@link #reset()}. */ + public static List deleteContextClassLoaders() { + return new ArrayList<>(DELETE_CONTEXT_CLASS_LOADERS); + } + + /** Makes deletes of files with the given names throw an unchecked exception. */ + public static void failDeletesUnchecked(String... fileNames) { + UNCHECKED_DELETE_FAILURE_NAMES.set(new HashSet<>(Arrays.asList(fileNames))); } public static void setDelayMillis(long delayMillis) { @@ -96,7 +120,13 @@ public boolean exists(Path path) throws IOException { @Override public boolean delete(Path path, boolean recursive) throws IOException { + if (DELAY_MILLIS.get() > 0) { + DELETE_CONTEXT_CLASS_LOADERS.add(Thread.currentThread().getContextClassLoader()); + } delay(); + if (UNCHECKED_DELETE_FAILURE_NAMES.get().contains(path.getName())) { + throw new IllegalStateException("Injected delete failure for " + path); + } return super.delete(path, recursive); } From dbd8c62e1d2248b651eb9a010e5ee9debfc8e7ae Mon Sep 17 00:00:00 2001 From: KK <37928252+dragonknightk@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:04:36 +0800 Subject: [PATCH 2/3] [core] Keep the abort delete bound when the caller is interrupted Signed-off-by: KK <37928252+dragonknightk@users.noreply.github.com> --- .../paimon/operation/FileStoreCommitImpl.java | 13 +- .../paimon/operation/FileStoreCommitTest.java | 113 ++++++++++++++++++ .../org/apache/paimon/utils/SlowFileIO.java | 48 ++++++-- 3 files changed, 164 insertions(+), 10 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 3f4ca107e2a0..4cb35ad64668 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -112,6 +112,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.FutureTask; +import java.util.concurrent.Semaphore; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -786,8 +787,12 @@ private void deleteConcurrently(List files) { return; } - ExecutorService executor = - FileOperationThreadPool.getExecutorService(options.fileOperationThreadNum()); + int threadNum = options.fileOperationThreadNum(); + ExecutorService executor = FileOperationThreadPool.getExecutorService(threadNum); + // The bounded executor runs a task without a permit if the caller is interrupted while + // waiting for one, so abort bounds the deletes with its own permits and keeps waiting for + // them under interrupts. A thread-num of 0 or below is bounded by the pool size only. + Semaphore permits = new Semaphore(threadNum > 0 ? threadNum : files.size()); boolean interrupted = false; // Run each delete with the caller's context class loader, as the sequential abort did. ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); @@ -803,10 +808,12 @@ private void deleteConcurrently(List files) { fileIO.deleteQuietly(file); } finally { thread.setContextClassLoader(previous); + permits.release(); } }, null); - // Clear the flag first, otherwise the bounded executor skips its permit and the bound. + permits.acquireUninterruptibly(); + // Remember and clear the flag, so the bounded executor takes its permit as usual. interrupted |= Thread.interrupted(); executor.execute(delete); deletes.add(delete); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index 0215f6eee991..53b7b3bb096f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -128,6 +128,7 @@ import static org.apache.paimon.stats.SimpleStats.EMPTY_STATS; import static org.apache.paimon.table.BucketMode.POSTPONE_BUCKET; import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; +import static org.apache.paimon.utils.CommonTestUtils.waitUtil; import static org.apache.paimon.utils.HintFileUtils.LATEST; import static org.apache.paimon.utils.Preconditions.checkNotNull; import static org.assertj.core.api.Assertions.assertThat; @@ -1574,6 +1575,118 @@ public void testAbortWaitsForAllDeletesWhenInterruptedWhileWaiting(boolean failD } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testAbortKeepsDeleteBoundWhenInterruptedWhileSubmitting(boolean interrupt) + throws Exception { + // file-operation.thread-num is "the maximum number of concurrent file operations"; 1 is + // below the shared pool size, so the 4 deletes (2 data files with one extra file each) + // must run one at a time + Map options = new HashMap<>(); + options.put(CoreOptions.FILE_OPERATION_THREAD_NUM.key(), "1"); + TestAppendFileStore store = + TestAppendFileStore.createAppendStore(tempDir, options, SlowFileIO.SCHEME); + BinaryRow partition = gen.getPartition(gen.next()); + DataFilePathFactory pathFactory = + store.pathFactory().createDataFilePathFactory(partition, 0); + List newFiles = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + newFiles.add(createDataFileWithExtraFile(store, pathFactory.newPath(), false)); + } + CommitMessage commitMessage = newFilesCommitMessage(store, partition, newFiles); + + // Start every thread of the shared pool and keep all but two busy: one runs the first + // delete, and the other is the only idle thread, so it runs queued tasks in order. + FileOperationThreadPool.getExecutorService(4); + ThreadPoolExecutor pool = + (ThreadPoolExecutor) FileOperationThreadPool.getExecutorService(0); + int poolSize = pool.getMaximumPoolSize(); + CountDownLatch allRunning = new CountDownLatch(poolSize); + CountDownLatch freeTwo = new CountDownLatch(1); + CountDownLatch freeRest = new CountDownLatch(1); + List> blockers = new ArrayList<>(); + SlowFileIO.reset(); + CountDownLatch firstDelete = SlowFileIO.holdNextDelete(); + Thread caller = null; + try { + for (int i = 0; i < poolSize; i++) { + CountDownLatch free = i < 2 ? freeTwo : freeRest; + blockers.add( + pool.submit( + () -> { + allRunning.countDown(); + return free.await(30, TimeUnit.SECONDS); + })); + } + assertThat(allRunning.await(10, TimeUnit.SECONDS)).isTrue(); + freeTwo.countDown(); + blockers.get(0).get(10, TimeUnit.SECONDS); + blockers.get(1).get(10, TimeUnit.SECONDS); + + AtomicReference abortFailure = new AtomicReference<>(); + AtomicBoolean interruptedAfterAbort = new AtomicBoolean(); + FileStoreCommitImpl commit = store.newCommit(); + caller = + new Thread( + () -> { + try { + commit.abort(Collections.singletonList(commitMessage)); + } catch (Throwable t) { + abortFailure.set(t); + } finally { + interruptedAfterAbort.set(Thread.interrupted()); + commit.close(); + } + }); + caller.start(); + Thread abortCaller = caller; + // the first delete holds the only permit, so the caller waits to submit the next one + waitUtil( + () -> + SlowFileIO.activeDeletes() == 1 + && abortCaller.getState() == Thread.State.WAITING, + Duration.ofSeconds(10), + Duration.ofMillis(1)); + // Interrupt twice. If abort stopped waiting for its own permit on the first interrupt, + // it would go on to wait in the bounded executor, which keeps the bound on that + // interrupt and only runs a task without a permit on the next one. + if (interrupt) { + for (int i = 0; i < 2; i++) { + abortCaller.interrupt(); + // the caller clears the flag when it handles the interrupt, then waits again + waitUtil( + () -> + !abortCaller.isInterrupted() + && abortCaller.getState() == Thread.State.WAITING, + Duration.ofSeconds(10), + Duration.ofMillis(1)); + } + } + // the idle thread runs queued tasks in order, so after this probe any delete + // submitted while the first one is held has run + pool.submit(() -> {}).get(10, TimeUnit.SECONDS); + firstDelete.countDown(); + caller.join(TimeUnit.SECONDS.toMillis(20)); + assertThat(caller.isAlive()).as("abort finished").isFalse(); + + // issue 11111: an interrupt must not let a delete run without a permit, the bound + // holds, every file is deleted and the caller's interrupt flag survives the cleanup + assertThat(SlowFileIO.maxActiveDeletes()).isEqualTo(1); + assertThat(abortFailure.get()).isNull(); + assertThat(interruptedAfterAbort.get()).isEqualTo(interrupt); + assertAllDeleted(store, pathFactory, newFiles); + } finally { + firstDelete.countDown(); + freeTwo.countDown(); + freeRest.countDown(); + if (caller != null) { + caller.interrupt(); + caller.join(TimeUnit.SECONDS.toMillis(20)); + } + SlowFileIO.reset(); + } + } + private static CommitMessage newFilesCommitMessage( TestAppendFileStore store, BinaryRow partition, List newFiles) { return new CommitMessageImpl( diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/SlowFileIO.java b/paimon-core/src/test/java/org/apache/paimon/utils/SlowFileIO.java index 1c6ea9e19a66..522e881098dd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/SlowFileIO.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/SlowFileIO.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -51,6 +52,9 @@ public class SlowFileIO extends LocalFileIO { new AtomicReference<>(Collections.emptySet()); private static final ConcurrentLinkedQueue DELETE_CONTEXT_CLASS_LOADERS = new ConcurrentLinkedQueue<>(); + private static final AtomicReference NEXT_DELETE_GATE = new AtomicReference<>(); + private static final AtomicInteger ACTIVE_DELETES = new AtomicInteger(); + private static final AtomicInteger MAX_ACTIVE_DELETES = new AtomicInteger(); public static void reset() { DELAY_MILLIS.set(0); @@ -59,6 +63,24 @@ public static void reset() { MAX_ACTIVE_OPERATIONS.set(0); UNCHECKED_DELETE_FAILURE_NAMES.set(Collections.emptySet()); DELETE_CONTEXT_CLASS_LOADERS.clear(); + NEXT_DELETE_GATE.set(null); + ACTIVE_DELETES.set(0); + MAX_ACTIVE_DELETES.set(0); + } + + /** Makes the next delete wait until the returned latch is counted down. */ + public static CountDownLatch holdNextDelete() { + CountDownLatch gate = new CountDownLatch(1); + NEXT_DELETE_GATE.set(gate); + return gate; + } + + public static int activeDeletes() { + return ACTIVE_DELETES.get(); + } + + public static int maxActiveDeletes() { + return MAX_ACTIVE_DELETES.get(); } /** Context class loaders seen by delayed deletes since the last {@link #reset()}. */ @@ -120,14 +142,26 @@ public boolean exists(Path path) throws IOException { @Override public boolean delete(Path path, boolean recursive) throws IOException { - if (DELAY_MILLIS.get() > 0) { - DELETE_CONTEXT_CLASS_LOADERS.add(Thread.currentThread().getContextClassLoader()); - } - delay(); - if (UNCHECKED_DELETE_FAILURE_NAMES.get().contains(path.getName())) { - throw new IllegalStateException("Injected delete failure for " + path); + MAX_ACTIVE_DELETES.accumulateAndGet(ACTIVE_DELETES.incrementAndGet(), Math::max); + try { + CountDownLatch gate = NEXT_DELETE_GATE.getAndSet(null); + if (gate != null) { + gate.await(); + } + if (DELAY_MILLIS.get() > 0) { + DELETE_CONTEXT_CLASS_LOADERS.add(Thread.currentThread().getContextClassLoader()); + } + delay(); + if (UNCHECKED_DELETE_FAILURE_NAMES.get().contains(path.getName())) { + throw new IllegalStateException("Injected delete failure for " + path); + } + return super.delete(path, recursive); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } finally { + ACTIVE_DELETES.decrementAndGet(); } - return super.delete(path, recursive); } @Override From 1dacf2128ba862730061292ae3b34f2fea49d0e1 Mon Sep 17 00:00:00 2001 From: KK <37928252+dragonknightk@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:42:07 +0800 Subject: [PATCH 3/3] [core] Delete files of aborted writes with bounded concurrency ### Purpose Closes issue 11111 `FileStoreCommitImpl.abort` now deletes the files of an aborted write concurrently, bounded by `file-operation.thread-num`. Before this change, each delete ran on the caller thread one after another, so abort time grew with the file count times the per-delete latency. - Abort first collects the paths with the existing rules, then deletes them as one batch. The set of files owned by the aborted write is unchanged, including the PR 10582 rule that keeps files reused by compaction. - The deletes run on the shared `FileOperationThreadPool`, the same pool and option that snapshot expiration (PR 8241) uses, and no option or pool is added. `TableCommitImpl.abort` passes its `threadNum` as the bound. `AbstractFileStoreTable` sets it from `file-operation.thread-num`, and it already sizes the executor of `checkFilesExistence`, so the bound has one source. Each delete runs with the caller's context class loader, as the sequential abort did. - Every path gets a delete attempt, and the caller blocks until all attempts have finished, including when a delete fails. An interrupt on the caller thread neither cancels the cleanup nor raises the bound: abort keeps waiting for permits and for all attempts, and restores the interrupt flag before returning or throwing. - Each delete still goes through `FileIO.deleteQuietly`, so an IOException is logged and does not throw. Any other exception is rethrown as is after all attempts, with later failures suppressed into the first in input order. - Abort bounds the deletes with its own permits, `threadNum` of them (all paths when it is 0 or below, where the shared pool size is the only bound), and waits for a permit with `Semaphore.acquireUninterruptibly` before each submission. `SemaphoredDelegatingExecutor.execute` runs a task without a permit when the caller is interrupted while waiting for one, so with its bound alone an interrupt let more deletes run at once than the limit allows. ### Tests - With `file-operation.thread-num = 4` and a FileIO that delays each operation, the deletes of one abort overlap, with 2 to 4 in flight. On the code before this change at most one delete is in flight, so this test fails there. Each delete sees the caller's context class loader, and the pool threads keep their own context class loaders after abort. - All files owned by the aborted write are deleted, and a file reused by compaction stays. When one delete fails with an IOException, abort does not throw and still deletes every other file. - When the caller is interrupted before abort and `file-operation.thread-num` is below the shared pool size, abort still deletes every file and the interrupt flag stays set. - With `file-operation.thread-num = 1`, the caller is interrupted twice while it waits for a permit with the first delete held. At most one delete is in flight, every file is deleted and the interrupt flag stays set. Two interrupts are needed because an abort that stops waiting for its own permit on the first interrupt still waits in the bounded executor, which runs a task without a permit only on the second. Such an abort and one that relies on the bounded executor alone both fail this test with two deletes in flight. Without an interrupt, one delete is in flight either way. - When the caller is interrupted while abort waits for the deletes, abort still waits for and deletes every file and the interrupt flag stays set. If two of those deletes also fail with unchecked exceptions, abort still deletes every other file, throws the first failure in input order with its original type, and carries the later one as its only suppressed exception. ## AI Disclosure Human-led contribution. Grok Bot, working for dragonknightk, assisted with this change, including the code, tests, and description. All submitted content was reviewed by a human before submission. Signed-off-by: KK <37928252+dragonknightk@users.noreply.github.com> --- .../paimon/operation/FileStoreCommit.java | 7 +++- .../paimon/operation/FileStoreCommitImpl.java | 7 ++-- .../paimon/table/sink/TableCommitImpl.java | 4 +- .../paimon/operation/FileStoreCommitTest.java | 40 ++++++++++++++----- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java index 050a345a0e3b..c17d74e2e622 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java @@ -82,8 +82,11 @@ public interface FileStoreCommit extends AutoCloseable { /** Roll back to the target snapshot and materialize it as the latest snapshot. */ boolean rollbackToAsLatest(Snapshot targetSnapshot); - /** Abort an unsuccessful commit. The data files will be deleted. */ - void abort(List commitMessages); + /** + * Abort an unsuccessful commit. The data files will be deleted, with {@code threadNum} bounding + * the concurrent deletes as {@code file-operation.thread-num} does. + */ + void abort(List commitMessages, int threadNum); /** With metrics to measure commits. */ FileStoreCommit withMetrics(CommitMetrics metrics); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 4cb35ad64668..a47609ea812e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -735,7 +735,7 @@ public void truncateTable(long commitIdentifier) { } @Override - public void abort(List commitMessages) { + public void abort(List commitMessages, int threadNum) { DataFilePathFactories dataFactories = new DataFilePathFactories(pathFactory); IndexFilePathFactories indexFactories = new IndexFilePathFactories(pathFactory); List filesToDelete = new ArrayList<>(); @@ -774,7 +774,7 @@ public void abort(List commitMessages) { filesToDelete.add(indexPathFactory.toPath(file)); } } - deleteConcurrently(filesToDelete); + deleteConcurrently(filesToDelete, threadNum); } /** @@ -782,12 +782,11 @@ public void abort(List commitMessages) { * first unchecked failure in input order is rethrown as is, later ones are suppressed into it. * Do not call this from a {@link FileOperationThreadPool} thread, it may wait for itself. */ - private void deleteConcurrently(List files) { + private void deleteConcurrently(List files, int threadNum) { if (files.isEmpty()) { return; } - int threadNum = options.fileOperationThreadNum(); ExecutorService executor = FileOperationThreadPool.getExecutorService(threadNum); // The bounded executor runs a task without a permit if the caller is interrupted while // waiting for one, so abort bounds the deletes with its own permits and keeps waiting for diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java index 23f1132fd795..b38809aecb39 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java @@ -88,6 +88,7 @@ public class TableCommitImpl implements InnerTableCommit { private final AtomicReference maintainError; private final String tableName; private final boolean forceCreatingSnapshot; + private final int threadNum; private final ExecutorService fileCheckExecutor; @Nullable private Map overwritePartitionSpec = null; @@ -128,6 +129,7 @@ public TableCommitImpl( this.tableName = tableName; this.forceCreatingSnapshot = forceCreatingSnapshot; + this.threadNum = threadNum; this.fileCheckExecutor = FileOperationThreadPool.getExecutorService(threadNum); } @@ -461,7 +463,7 @@ public void close() throws Exception { @Override public void abort(List commitMessages) { - commit.abort(commitMessages); + commit.abort(commitMessages, threadNum); } @VisibleForTesting diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index 53b7b3bb096f..5cc4309339f7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -1188,7 +1188,9 @@ public void testAbortIndexFiles() throws Exception { Collections.singletonList(compactDeleted))); try (FileStoreCommitImpl commit = store.newCommit()) { - commit.abort(Collections.singletonList(commitMessage)); + commit.abort( + Collections.singletonList(commitMessage), + store.options().fileOperationThreadNum()); } assertThat(store.fileIO().exists(dataNewPath)).isFalse(); @@ -1224,7 +1226,9 @@ public void testAbortDataFileWithExtraFiles() throws Exception { Collections.emptyList())); try (FileStoreCommitImpl commit = store.newCommit()) { - commit.abort(Collections.singletonList(commitMessage)); + commit.abort( + Collections.singletonList(commitMessage), + store.options().fileOperationThreadNum()); } for (Path path : dataNew.collectFiles(pathFactory)) { @@ -1270,8 +1274,12 @@ public void testAbortPreservesReusedCompactionFiles(boolean external) throws Exc Collections.emptyList())); try (FileStoreCommitImpl commit = store.newCommit()) { - commit.abort(Collections.singletonList(commitMessage)); - commit.abort(Collections.singletonList(commitMessage)); + commit.abort( + Collections.singletonList(commitMessage), + store.options().fileOperationThreadNum()); + commit.abort( + Collections.singletonList(commitMessage), + store.options().fileOperationThreadNum()); } for (Path path : reused.collectFiles(pathFactory)) { @@ -1324,8 +1332,8 @@ public void testAbortCleansUncommittedCompactionInputs(String changelogProducer) } assertThat(store.snapshotManager().latestSnapshotId()).isNull(); - commit.abort(messages); - commit.abort(messages); + commit.abort(messages, store.options().fileOperationThreadNum()); + commit.abort(messages, store.options().fileOperationThreadNum()); for (Path path : pendingPaths) { assertThat(store.fileIO().exists(path)).as("aborted file %s", path).isFalse(); @@ -1375,7 +1383,9 @@ public void testAbortDeletesFilesConcurrently() throws Exception { SlowFileIO.setDelayMillis(50); caller.setContextClassLoader(callerLoader); try (FileStoreCommitImpl commit = store.newCommit()) { - commit.abort(Collections.singletonList(commitMessage)); + commit.abort( + Collections.singletonList(commitMessage), + store.options().fileOperationThreadNum()); } finally { caller.setContextClassLoader(original); } @@ -1438,7 +1448,9 @@ public void testAbortContinuesWhenOneDeleteFails() throws Exception { // issue 11111: a failed delete is logged and neither throws nor skips the other files try (FileStoreCommitImpl commit = store.newCommit()) { - commit.abort(Collections.singletonList(commitMessage)); + commit.abort( + Collections.singletonList(commitMessage), + store.options().fileOperationThreadNum()); } assertThat(store.fileIO().exists(undeletable)).isTrue(); @@ -1470,7 +1482,9 @@ public void testAbortDeletesAllFilesWhenCallerIsInterrupted() throws Exception { try (FileStoreCommitImpl commit = store.newCommit()) { Thread.currentThread().interrupt(); try { - commit.abort(Collections.singletonList(commitMessage)); + commit.abort( + Collections.singletonList(commitMessage), + store.options().fileOperationThreadNum()); } finally { interruptedAfterAbort = Thread.interrupted(); } @@ -1531,7 +1545,9 @@ public void testAbortWaitsForAllDeletesWhenInterruptedWhileWaiting(boolean failD }); interrupter.start(); try { - commit.abort(Collections.singletonList(commitMessage)); + commit.abort( + Collections.singletonList(commitMessage), + store.options().fileOperationThreadNum()); } catch (Throwable t) { abortFailure = t; } finally { @@ -1630,7 +1646,9 @@ public void testAbortKeepsDeleteBoundWhenInterruptedWhileSubmitting(boolean inte new Thread( () -> { try { - commit.abort(Collections.singletonList(commitMessage)); + commit.abort( + Collections.singletonList(commitMessage), + store.options().fileOperationThreadNum()); } catch (Throwable t) { abortFailure.set(t); } finally {