Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,11 @@ public interface FileStoreCommit extends AutoCloseable {
/** Roll back to the target snapshot and materialize it as the latest snapshot. */
boolean rollbackToAsLatest(Snapshot targetSnapshot);

/** Abort an unsuccessful commit. The data files will be deleted. */
void abort(List<CommitMessage> commitMessages);
/**
* Abort an unsuccessful commit. The data files will be deleted, with {@code threadNum} bounding
* the concurrent deletes as {@code file-operation.thread-num} does.
*/
void abort(List<CommitMessage> commitMessages, int threadNum);

/** With metrics to measure commits. */
FileStoreCommit withMetrics(CommitMetrics metrics);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@
import org.apache.paimon.table.sink.CommitPreCallback;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.DataFilePathFactories;
import org.apache.paimon.utils.ExceptionUtils;
import org.apache.paimon.utils.FileOperationThreadPool;
import org.apache.paimon.utils.FileStorePathFactory;
import org.apache.paimon.utils.IOUtils;
import org.apache.paimon.utils.IndexFilePathFactories;
Expand Down Expand Up @@ -107,6 +109,10 @@
import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.FutureTask;
import java.util.concurrent.Semaphore;
import java.util.function.Supplier;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -729,9 +735,10 @@ public void truncateTable(long commitIdentifier) {
}

@Override
public void abort(List<CommitMessage> commitMessages) {
public void abort(List<CommitMessage> commitMessages, int threadNum) {
DataFilePathFactories dataFactories = new DataFilePathFactories(pathFactory);
IndexFilePathFactories indexFactories = new IndexFilePathFactories(pathFactory);
List<Path> filesToDelete = new ArrayList<>();
for (CommitMessage message : commitMessages) {
DataFilePathFactory dataPathFactory =
dataFactories.get(message.partition(), message.bucket());
Expand All @@ -745,7 +752,7 @@ public void abort(List<CommitMessage> commitMessages) {

// New files may also be compaction inputs, but abort still owns their cleanup.
for (DataFileMeta file : dataFilesToDelete) {
file.collectFiles(dataPathFactory).forEach(fileIO::deleteQuietly);
filesToDelete.addAll(file.collectFiles(dataPathFactory));
}

// A level upgrade may reuse a main file while adding new auxiliary files.
Expand All @@ -757,16 +764,81 @@ public void abort(List<CommitMessage> commitMessages) {
for (DataFileMeta file : commitMessage.compactIncrement().compactAfter()) {
file.collectFiles(dataPathFactory).stream()
.filter(path -> !compactInputPaths.contains(path))
.forEach(fileIO::deleteQuietly);
.forEach(filesToDelete::add);
}

List<IndexFileMeta> indexFilesToDelete = new ArrayList<>();
indexFilesToDelete.addAll(commitMessage.newFilesIncrement().newIndexFiles());
indexFilesToDelete.addAll(commitMessage.compactIncrement().newIndexFiles());
for (IndexFileMeta file : indexFilesToDelete) {
fileIO.deleteQuietly(indexPathFactory.toPath(file));
filesToDelete.add(indexPathFactory.toPath(file));
}
}
deleteConcurrently(filesToDelete, threadNum);
}

/**
* Attempts every delete and waits for all of them even when the caller is interrupted. The
* first unchecked failure in input order is rethrown as is, later ones are suppressed into it.
* Do not call this from a {@link FileOperationThreadPool} thread, it may wait for itself.
*/
private void deleteConcurrently(List<Path> files, int threadNum) {
if (files.isEmpty()) {
return;
}

ExecutorService executor = FileOperationThreadPool.getExecutorService(threadNum);
// The bounded executor runs a task without a permit if the caller is interrupted while
// waiting for one, so abort bounds the deletes with its own permits and keeps waiting for
// them under interrupts. A thread-num of 0 or below is bounded by the pool size only.
Semaphore permits = new Semaphore(threadNum > 0 ? threadNum : files.size());
boolean interrupted = false;
// Run each delete with the caller's context class loader, as the sequential abort did.
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
List<FutureTask<?>> deletes = new ArrayList<>(files.size());
for (Path file : files) {
FutureTask<?> delete =
new FutureTask<>(
() -> {
Thread thread = Thread.currentThread();
ClassLoader previous = thread.getContextClassLoader();
thread.setContextClassLoader(classLoader);
try {
fileIO.deleteQuietly(file);
} finally {
thread.setContextClassLoader(previous);
permits.release();
}
},
null);
permits.acquireUninterruptibly();
// Remember and clear the flag, so the bounded executor takes its permit as usual.
interrupted |= Thread.interrupted();
executor.execute(delete);
Comment on lines +816 to +817

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Preserve the delete bound when interrupted while acquiring a permit

Clearing the interrupt flag before execute does not cover an interrupt arriving while SemaphoredDelegatingExecutor.execute is blocked in Semaphore.acquire(). That executor catches the interruption and dispatches the command without a permit. A table limit below the shared pool size is a normal configuration, so cancellation can bypass the advertised maximum concurrent file operations.

I reproduced this through public abort using four real owned file paths, file-operation.thread-num=1, and a shared pool with at least four threads. Holding the first delete, waiting until the caller is blocked in semaphore acquisition, then interrupting it once produces two simultaneous deletes; the identical no-interrupt control produces one. Both finish all deletes and preserve the interrupt flag.

Please make bounded submission continue waiting for a permit under interruption (while remembering and restoring the flag), or use a fixed number of delete workers, and cover interruption during submission. The existing tests interrupt before submission or after every delete has already been submitted.

deletes.add(delete);
}

Throwable failure = null;
for (FutureTask<?> delete : deletes) {
while (true) {
try {
delete.get();
break;
} catch (InterruptedException e) {
interrupted = true;
} catch (ExecutionException e) {
failure = ExceptionUtils.firstOrSuppressed(e.getCause(), failure);
break;
}
}
}

if (interrupted) {
Thread.currentThread().interrupt();
}
if (failure != null) {
ExceptionUtils.rethrow(failure);
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ public class TableCommitImpl implements InnerTableCommit {
private final AtomicReference<Throwable> maintainError;
private final String tableName;
private final boolean forceCreatingSnapshot;
private final int threadNum;
private final ExecutorService fileCheckExecutor;

@Nullable private Map<String, String> overwritePartitionSpec = null;
Expand Down Expand Up @@ -128,6 +129,7 @@ public TableCommitImpl(

this.tableName = tableName;
this.forceCreatingSnapshot = forceCreatingSnapshot;
this.threadNum = threadNum;
this.fileCheckExecutor = FileOperationThreadPool.getExecutorService(threadNum);
}

Expand Down Expand Up @@ -461,7 +463,7 @@ public void close() throws Exception {

@Override
public void abort(List<CommitMessage> commitMessages) {
commit.abort(commitMessages);
commit.abort(commitMessages, threadNum);
}

@VisibleForTesting
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,13 @@ public CommitMessageImpl writeDVIndexFiles(

public static TestAppendFileStore createAppendStore(
java.nio.file.Path tempDir, Map<String, String> options) throws Exception {
String root = TraceableFileIO.SCHEME + "://" + tempDir.toString();
return createAppendStore(tempDir, options, TraceableFileIO.SCHEME);
}

public static TestAppendFileStore createAppendStore(
java.nio.file.Path tempDir, Map<String, String> options, String scheme)
throws Exception {
String root = scheme + "://" + tempDir.toString();
Path path = new Path(tempDir.toUri());
FileIO fileIO = FileIOFinder.find(new Path(root));
SchemaManager schemaManage = new FileSystemSchemaManager(new LocalFileIO(), path);
Expand Down
Loading
Loading