diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java index 8ca96640939f..b2b874cf4459 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java @@ -266,6 +266,14 @@ private List> agePack() { } } + // a batch job plans each file once, so files left over by packing never reach + // COMPACT_AGE; compact them now (issue 5130) + if (!isStreaming && toCompact.size() > 1) { + packed = new ArrayList<>(packed); + packed.add(new ArrayList<>(toCompact)); + toCompact.clear(); + } + return packed; } diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java index 703bcd8814c0..96222c5cf4d7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java @@ -167,6 +167,42 @@ public void testAgeGrowUp() { assertThat(compactionCoordinator.subCoordinators.get(partition).age).isEqualTo(0); } + @Test + public void testBatchCompactsFilesLeftOverByPacking() { + // 14 files of a third of the target size: packing fills 2 tasks of 6 files and leaves 2 + // files, fewer than compaction.min.file-num + List files = + generateNewFiles( + 14, appendOnlyFileStoreTable.coreOptions().targetFileSize(false) / 3 + 1); + + // streaming keeps the 2 files until they reach COMPACT_AGE + compactionCoordinator.notifyNewFiles(partition, files); + assertThat(compactionCoordinator.compactPlan()).hasSize(2); + assertThat(compactionCoordinator.listRestoredFiles()).hasSize(2); + + // issue 5130: a batch job plans once, so it compacts the 2 files in a third task + AppendCompactCoordinator batchCoordinator = + new AppendCompactCoordinator(appendOnlyFileStoreTable, false); + batchCoordinator.notifyNewFiles(partition, files); + List tasks = batchCoordinator.compactPlan(); + assertThat(tasks).hasSize(3); + assertThat(tasks.get(2).compactBefore()).hasSize(2); + assertThat(tasks.stream().flatMap(task -> task.compactBefore().stream())) + .containsExactlyInAnyOrderElementsOf(files); + assertThat(batchCoordinator.subCoordinators).isEmpty(); + } + + @Test + public void testBatchKeepsSingleFile() { + AppendCompactCoordinator batchCoordinator = + new AppendCompactCoordinator(appendOnlyFileStoreTable, false); + batchCoordinator.notifyNewFiles(partition, generateNewFiles(1, 0)); + + // compacting a single file alone does not reduce the file count + assertThat(batchCoordinator.compactPlan()).isEmpty(); + assertThat(batchCoordinator.listRestoredFiles()).hasSize(1); + } + @Test public void testBatchScanEmptyTable() { compactionCoordinator = new AppendCompactCoordinator(appendOnlyFileStoreTable, false); diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java index 6c3246ae0b12..687998215d93 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.operation.BaseAppendFileStoreWrite; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; @@ -50,6 +51,7 @@ import java.util.Map; import java.util.Random; import java.util.UUID; +import java.util.stream.Collectors; import static java.util.Collections.singletonMap; import static org.apache.paimon.SnapshotTest.newSnapshotManager; @@ -276,6 +278,27 @@ public void testBatchScanAllFilesFilteredOut() throws Exception { assertThat(batchCoordinator.batchRemainFiles()).isFalse(); } + @Test + public void testBatchCompactsLeftoverFiles() throws Exception { + // 2 files, fewer than compaction.min.file-num = 3 + commit(writeCommit(2)); + + // streaming waits for more files or for COMPACT_AGE + assertThat(coordinator.run()).isEmpty(); + + // issue 5130: a batch compaction plans once, so it compacts the 2 files right away + List tasks = createBatchCoordinator(Collections.emptyMap()).run(); + assertThat(tasks).hasSize(1); + assertThat(tasks.get(0).compactBefore()).hasSize(2); + commit(doCompact(tasks)); + List files = + appendOnlyFileStoreTable.store().newScan().plan().files().stream() + .map(ManifestEntry::file) + .collect(Collectors.toList()); + assertThat(files).hasSize(1); + assertThat(files.get(0).rowCount()).isEqualTo(2); + } + private AppendCompactCoordinator createBatchCoordinator(Map extraOptions) { TableSchema schema = tableSchema.copy(extraOptions); FileStoreTable table =