From 01707a01bdfd7bdfe2498daac704cb29cadcc031 Mon Sep 17 00:00:00 2001 From: KK <37928252+dragonknightk@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:34:38 +0800 Subject: [PATCH] [core] Compact leftover small files in batch append compaction ### Purpose Closes issue 5130 In batch mode, `AppendCompactCoordinator` now compacts the small files that packing leaves in a partition when there are at least 2 of them. Before this change, such files were compacted only after `COMPACT_AGE` further planning rounds without new files in the partition. A batch compaction stops after planning its last scanned files, so the files left over in that plan stayed uncompacted. - The coordinator already knows the mode from its `isStreaming` constructor argument, which the Flink compact source, the Flink multi-table compact scan and the Spark compact procedure pass. No caller or option changes. - The leftover files form one extra task per partition, even when they are fewer than `compaction.min.file-num`. A single file is not compacted, because rewriting it alone does not reduce the file count. - In batch mode this applies to every plan. When the Flink batch source scans more files than `compaction.file-num-limit`, it plans once per chunk, so a partition can get one leftover task per chunk. - Streaming is unchanged: leftover files still wait for `COMPACT_AGE`. ### Tests - With deletion vectors enabled and `compaction.min.file-num = 3`, 14 files of a third of the target size pack into 2 tasks and leave 2 files. A streaming coordinator plans the 2 tasks and keeps the 2 files. A batch coordinator plans a third task with the 2 files, so the tasks cover all 14 files and nothing is kept. - On a table without deletion vectors, a batch compaction of 2 committed files below `compaction.min.file-num` produces 1 file with both rows, while a streaming coordinator plans nothing. - A batch coordinator does not compact a single file. - The first two scenarios fail on the code before this change. ## AI Disclosure Human-led contribution. Grok Bot, working for dragonknightk, assisted with this change, including the code, tests, and description. All submitted content was reviewed by a human before submission. Signed-off-by: KK <37928252+dragonknightk@users.noreply.github.com> --- .../append/AppendCompactCoordinator.java | 8 +++++ .../append/AppendCompactCoordinatorTest.java | 36 +++++++++++++++++++ .../append/AppendOnlyTableCompactionTest.java | 23 ++++++++++++ 3 files changed, 67 insertions(+) diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java index 8ca96640939f..b2b874cf4459 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java @@ -266,6 +266,14 @@ private List> agePack() { } } + // a batch job plans each file once, so files left over by packing never reach + // COMPACT_AGE; compact them now (issue 5130) + if (!isStreaming && toCompact.size() > 1) { + packed = new ArrayList<>(packed); + packed.add(new ArrayList<>(toCompact)); + toCompact.clear(); + } + return packed; } diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java index 703bcd8814c0..96222c5cf4d7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java @@ -167,6 +167,42 @@ public void testAgeGrowUp() { assertThat(compactionCoordinator.subCoordinators.get(partition).age).isEqualTo(0); } + @Test + public void testBatchCompactsFilesLeftOverByPacking() { + // 14 files of a third of the target size: packing fills 2 tasks of 6 files and leaves 2 + // files, fewer than compaction.min.file-num + List files = + generateNewFiles( + 14, appendOnlyFileStoreTable.coreOptions().targetFileSize(false) / 3 + 1); + + // streaming keeps the 2 files until they reach COMPACT_AGE + compactionCoordinator.notifyNewFiles(partition, files); + assertThat(compactionCoordinator.compactPlan()).hasSize(2); + assertThat(compactionCoordinator.listRestoredFiles()).hasSize(2); + + // issue 5130: a batch job plans once, so it compacts the 2 files in a third task + AppendCompactCoordinator batchCoordinator = + new AppendCompactCoordinator(appendOnlyFileStoreTable, false); + batchCoordinator.notifyNewFiles(partition, files); + List tasks = batchCoordinator.compactPlan(); + assertThat(tasks).hasSize(3); + assertThat(tasks.get(2).compactBefore()).hasSize(2); + assertThat(tasks.stream().flatMap(task -> task.compactBefore().stream())) + .containsExactlyInAnyOrderElementsOf(files); + assertThat(batchCoordinator.subCoordinators).isEmpty(); + } + + @Test + public void testBatchKeepsSingleFile() { + AppendCompactCoordinator batchCoordinator = + new AppendCompactCoordinator(appendOnlyFileStoreTable, false); + batchCoordinator.notifyNewFiles(partition, generateNewFiles(1, 0)); + + // compacting a single file alone does not reduce the file count + assertThat(batchCoordinator.compactPlan()).isEmpty(); + assertThat(batchCoordinator.listRestoredFiles()).hasSize(1); + } + @Test public void testBatchScanEmptyTable() { compactionCoordinator = new AppendCompactCoordinator(appendOnlyFileStoreTable, false); diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java index 6c3246ae0b12..687998215d93 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.operation.BaseAppendFileStoreWrite; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; @@ -50,6 +51,7 @@ import java.util.Map; import java.util.Random; import java.util.UUID; +import java.util.stream.Collectors; import static java.util.Collections.singletonMap; import static org.apache.paimon.SnapshotTest.newSnapshotManager; @@ -276,6 +278,27 @@ public void testBatchScanAllFilesFilteredOut() throws Exception { assertThat(batchCoordinator.batchRemainFiles()).isFalse(); } + @Test + public void testBatchCompactsLeftoverFiles() throws Exception { + // 2 files, fewer than compaction.min.file-num = 3 + commit(writeCommit(2)); + + // streaming waits for more files or for COMPACT_AGE + assertThat(coordinator.run()).isEmpty(); + + // issue 5130: a batch compaction plans once, so it compacts the 2 files right away + List tasks = createBatchCoordinator(Collections.emptyMap()).run(); + assertThat(tasks).hasSize(1); + assertThat(tasks.get(0).compactBefore()).hasSize(2); + commit(doCompact(tasks)); + List files = + appendOnlyFileStoreTable.store().newScan().plan().files().stream() + .map(ManifestEntry::file) + .collect(Collectors.toList()); + assertThat(files).hasSize(1); + assertThat(files.get(0).rowCount()).isEqualTo(2); + } + private AppendCompactCoordinator createBatchCoordinator(Map extraOptions) { TableSchema schema = tableSchema.copy(extraOptions); FileStoreTable table =