Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,14 @@ private List<List<DataFileMeta>> agePack() {
}
}

// a batch job plans each file once, so files left over by packing never reach
// COMPACT_AGE; compact them now (issue 5130)
if (!isStreaming && toCompact.size() > 1) {
packed = new ArrayList<>(packed);
packed.add(new ArrayList<>(toCompact));
toCompact.clear();
}

return packed;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,42 @@ public void testAgeGrowUp() {
assertThat(compactionCoordinator.subCoordinators.get(partition).age).isEqualTo(0);
}

@Test
public void testBatchCompactsFilesLeftOverByPacking() {
// 14 files of a third of the target size: packing fills 2 tasks of 6 files and leaves 2
// files, fewer than compaction.min.file-num
List<DataFileMeta> files =
generateNewFiles(
14, appendOnlyFileStoreTable.coreOptions().targetFileSize(false) / 3 + 1);

// streaming keeps the 2 files until they reach COMPACT_AGE
compactionCoordinator.notifyNewFiles(partition, files);
assertThat(compactionCoordinator.compactPlan()).hasSize(2);
assertThat(compactionCoordinator.listRestoredFiles()).hasSize(2);

// issue 5130: a batch job plans once, so it compacts the 2 files in a third task
AppendCompactCoordinator batchCoordinator =
new AppendCompactCoordinator(appendOnlyFileStoreTable, false);
batchCoordinator.notifyNewFiles(partition, files);
List<AppendCompactTask> tasks = batchCoordinator.compactPlan();
assertThat(tasks).hasSize(3);
assertThat(tasks.get(2).compactBefore()).hasSize(2);
assertThat(tasks.stream().flatMap(task -> task.compactBefore().stream()))
.containsExactlyInAnyOrderElementsOf(files);
assertThat(batchCoordinator.subCoordinators).isEmpty();
}

@Test
public void testBatchKeepsSingleFile() {
AppendCompactCoordinator batchCoordinator =
new AppendCompactCoordinator(appendOnlyFileStoreTable, false);
batchCoordinator.notifyNewFiles(partition, generateNewFiles(1, 0));

// compacting a single file alone does not reduce the file count
assertThat(batchCoordinator.compactPlan()).isEmpty();
assertThat(batchCoordinator.listRestoredFiles()).hasSize(1);
}

@Test
public void testBatchScanEmptyTable() {
compactionCoordinator = new AppendCompactCoordinator(appendOnlyFileStoreTable, false);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.operation.BaseAppendFileStoreWrite;
import org.apache.paimon.schema.FileSystemSchemaManager;
import org.apache.paimon.schema.Schema;
Expand All @@ -50,6 +51,7 @@
import java.util.Map;
import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;

import static java.util.Collections.singletonMap;
import static org.apache.paimon.SnapshotTest.newSnapshotManager;
Expand Down Expand Up @@ -276,6 +278,27 @@ public void testBatchScanAllFilesFilteredOut() throws Exception {
assertThat(batchCoordinator.batchRemainFiles()).isFalse();
}

@Test
public void testBatchCompactsLeftoverFiles() throws Exception {
// 2 files, fewer than compaction.min.file-num = 3
commit(writeCommit(2));

// streaming waits for more files or for COMPACT_AGE
assertThat(coordinator.run()).isEmpty();

// issue 5130: a batch compaction plans once, so it compacts the 2 files right away
List<AppendCompactTask> tasks = createBatchCoordinator(Collections.emptyMap()).run();
assertThat(tasks).hasSize(1);
assertThat(tasks.get(0).compactBefore()).hasSize(2);
commit(doCompact(tasks));
List<DataFileMeta> files =
appendOnlyFileStoreTable.store().newScan().plan().files().stream()
.map(ManifestEntry::file)
.collect(Collectors.toList());
assertThat(files).hasSize(1);
assertThat(files.get(0).rowCount()).isEqualTo(2);
}

private AppendCompactCoordinator createBatchCoordinator(Map<String, String> extraOptions) {
TableSchema schema = tableSchema.copy(extraOptions);
FileStoreTable table =
Expand Down
Loading