diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java
index 6c5493765e13f0..8e8e4c4f627030 100644
--- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java
+++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java
@@ -59,6 +59,7 @@
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataOutputViewStreamWrapper;
import org.apache.paimon.options.Options;
@@ -86,6 +87,7 @@
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.InstantiationUtil;
@@ -990,6 +992,15 @@ && hasTrustworthyLimitAccounting(table)
// $ro wraps the pinned file-store table; resolve its schema dictionary once, only if native is considered.
java.util.function.Supplier
legacyOrcSchemaTable = com.google.common.base.Suppliers.memoize(
() -> resolveSchemaDictTable(table, paimonHandle));
+ // The wrappers read through JNI report the options of the main branch, but their splits come from a branch
+ // of the pinned source, which the timestamp-history checks resolve the same way.
+ java.util.function.Supplier binaryPartitionSourceTable = paimonHandle.isForceJni()
+ ? com.google.common.base.Suppliers.memoize(() -> withSchemaHistoryAuthentication(
+ () -> resolveTimestampHistoryTable(table, paimonHandle)))
+ : legacyOrcSchemaTable;
+ int[] binaryPartitionFields = binaryPartitionFields(table);
+ Map> legacyPartitionNameSchemas = new java.util.IdentityHashMap<>();
+ Map fileExistence = new HashMap<>();
// Process DataSplits
for (DataSplit dataSplit : dataSplits) {
if (isCountPushdownSplit(countPushdown, dataSplit)) {
@@ -999,6 +1010,10 @@ && hasTrustworthyLimitAccounting(table)
}
continue;
}
+ if (binaryPartitionFields.length > 0) {
+ checkBinaryPartitionFiles(paimonHandle, table, dataSplit, binaryPartitionFields,
+ binaryPartitionSourceTable, legacyPartitionNameSchemas, fileExistence);
+ }
Map partitionValues = getPartitionInfoMap(
table, dataSplit.partition(), session.getTimeZone());
@@ -1429,7 +1444,7 @@ private T withSchemaHistoryAuthentication(Supplier action) {
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
- throw new RuntimeException("Failed to read Paimon timestamp schema history", e);
+ throw new RuntimeException("Failed to access Paimon table files", e);
}
}
@@ -2285,6 +2300,11 @@ static FileStoreTable legacyOrcSplitTable(FileStoreTable table, DataSplit split)
FileStoreTable base = PaimonTableDecorators.unwrapToFallbackOrBase(table);
if (base instanceof FallbackReadFileStoreTable) {
FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) base;
+ if (!(split instanceof FallbackReadFileStoreTable.FallbackSplit)) {
+ // A split planned through the pair's SnapshotReader (a file-creation-time read) comes from its
+ // wrapped branch.
+ return pair.wrapped();
+ }
boolean fallback = ((FallbackReadFileStoreTable.FallbackSplit) split).isFallback();
// Match the SDK reader's branch selection before reading either options or historical schemas.
return PaimonReaderOptions.isWrappedFirst(pair) != fallback ? pair.wrapped() : pair.other();
@@ -2292,6 +2312,109 @@ static FileStoreTable legacyOrcSplitTable(FileStoreTable table, DataSplit split)
return base;
}
+ /** Positions in the partition row of the partition columns that hold binary values. */
+ private static int[] binaryPartitionFields(Table table) {
+ RowType partitionType = table.rowType().project(table.partitionKeys());
+ return java.util.stream.IntStream.range(0, partitionType.getFieldCount()).filter(field -> {
+ DataTypeRoot typeRoot = partitionType.getTypeAt(field).getTypeRoot();
+ return typeRoot == DataTypeRoot.BINARY || typeRoot == DataTypeRoot.VARBINARY;
+ }).toArray();
+ }
+
+ /**
+ * Paimon finds the files of a partition under the directory its values name. With {@code partition.legacy-name}
+ * = true, the default, it names a partition after the string of each value, which for a non-NULL binary value
+ * is the Java array object ({@code [B@1b6d3586}) rather than its bytes, and every resolution of the name
+ * differs. A file written under such a name, or looked up under one, is never found: Paimon's own read fails,
+ * while the BE skips a missing file of an external table and would silently drop the partition's rows. A file
+ * with an external path is found by the path its metadata records. A system wrapper reports the options of the
+ * main branch while a fallback split is planned and read under its own branch, so the options and schemas come
+ * from the branch that produced the split; for the wrappers read through JNI that branch is resolved from the
+ * pinned source of the wrapper.
+ *
+ * The schema a file refers to does not prove how the file was named, because dynamic options
+ * ({@code spark.paimon.*}, Flink hints, {@code Table.copy}) keep the schema ID: a file whose schema uses legacy
+ * names fails only if it is also missing at its planned path. With {@code index-file-in-data-file-dir} = true, a
+ * deletion vector written without an external data path is looked up under the directory of its partition too,
+ * whatever the path of its data file, and the BE skips the whole data file when it cannot open the deletion
+ * vector, so every deletion vector must exist. Each path is checked once per scan.
+ */
+ private void checkBinaryPartitionFiles(PaimonTableHandle handle, Table table, DataSplit split,
+ int[] binaryPartitionFields, Supplier
schemaTable,
+ Map> legacyPartitionNameSchemas, Map fileExistence) {
+ BinaryRow partition = split.partition();
+ OptionalInt nonNullField = Arrays.stream(binaryPartitionFields)
+ .filter(field -> !partition.isNullAt(field)).findFirst();
+ if (!nonNullField.isPresent()) {
+ return;
+ }
+ int binaryField = nonNullField.getAsInt();
+ FileStoreTable splitTable = schemaTable.get() == null
+ ? null : legacyOrcSplitTable((FileStoreTable) schemaTable.get(), split);
+ Table optionsTable = splitTable == null ? table : splitTable;
+ CoreOptions options = CoreOptions.fromMap(optionsTable.options());
+ boolean legacyLookup = options.legacyPartitionName();
+ // Deletion vectors under the table's index directory do not depend on partition names. The option is
+ // immutable, so every writer of the table used this value.
+ List deletionFiles = options.indexFileInDataFileDir() ? split.deletionFiles().orElse(null) : null;
+ List files = split.dataFiles();
+ for (int i = 0; i < files.size(); i++) {
+ DataFileMeta file = files.get(i);
+ if (!file.externalPath().isPresent()) {
+ if (legacyLookup) {
+ throw binaryPartitionReadError(handle, "the table uses legacy partition names"
+ + " ('partition.legacy-name' = 'true'), under which Paimon names partition "
+ + binaryPartitionName(table, partition, binaryField) + " after a Java array object"
+ + " rather than its bytes, so the files of the partition are never found");
+ }
+ if (splitTable != null && legacyPartitionNameSchemas
+ .computeIfAbsent(splitTable, ignored -> new HashMap<>())
+ .computeIfAbsent(file.schemaId(), id -> withSchemaHistoryAuthentication(() -> CoreOptions
+ .fromMap(splitTable.schemaManager().schema(id).options()).legacyPartitionName()))
+ && !fileExists(optionsTable.fileIO(), split.bucketPath() + "/" + file.fileName(),
+ fileExistence)) {
+ throw binaryPartitionReadError(handle, "data file " + file.fileName() + " of partition "
+ + binaryPartitionName(table, partition, binaryField) + " was written while the table"
+ + " used legacy partition names ('partition.legacy-name' = 'true'), under which Paimon"
+ + " names the partition after a Java array object rather than its bytes, so the file is"
+ + " not under the directory of the partition");
+ }
+ }
+ DeletionFile deletionFile = deletionFiles == null ? null : deletionFiles.get(i);
+ if (deletionFile != null && !fileExists(optionsTable.fileIO(), deletionFile.path(), fileExistence)) {
+ throw binaryPartitionReadError(handle, "deletion vector file "
+ + new Path(deletionFile.path()).getName() + " of data file " + file.fileName()
+ + " of partition " + binaryPartitionName(table, partition, binaryField)
+ + " does not exist: the table keeps index files with its data files"
+ + " ('index-file-in-data-file-dir' = 'true'), so a deletion vector written without an external"
+ + " path is looked up under the directory of its partition, and legacy partition names"
+ + " ('partition.legacy-name' = 'true') name the partition after a Java array object rather"
+ + " than its bytes");
+ }
+ }
+ }
+
+ private static String binaryPartitionName(Table table, BinaryRow partition, int binaryField) {
+ return table.partitionKeys().get(binaryField) + " = 0x"
+ + java.util.HexFormat.of().withUpperCase().formatHex(partition.getBinary(binaryField));
+ }
+
+ private static DorisConnectorException binaryPartitionReadError(PaimonTableHandle handle, String reason) {
+ return new DorisConnectorException("Cannot read Paimon table " + handle.getDatabaseName() + "."
+ + handle.getTableName() + ": " + reason + ". Paimon cannot read the partition either; a table with a"
+ + " binary partition column needs 'partition.legacy-name' = 'false' before its data is written");
+ }
+
+ private boolean fileExists(FileIO fileIO, String path, Map fileExistence) {
+ return fileExistence.computeIfAbsent(path, ignored -> withSchemaHistoryAuthentication(() -> {
+ try {
+ return fileIO.exists(new Path(path));
+ } catch (IOException e) {
+ throw new DorisConnectorException("Failed to check whether Paimon file " + path + " exists", e);
+ }
+ }));
+ }
+
private static boolean requiresCrossFamilyTimestampReader(FileStoreTable table, Optional> rawFiles,
Set readFieldIds, Map schemaTimestamps) {
RowType readType = new RowType(table.rowType().getFields().stream()
diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java
index d29cae9ba8d1ef..bbb0513a29a376 100644
--- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java
+++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java
@@ -50,9 +50,14 @@
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.deletionvectors.BitmapDeletionVector;
+import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
import org.apache.paimon.fs.FileStatus;
import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataIncrement;
import org.apache.paimon.io.DataInputViewStreamWrapper;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.reader.RecordReader;
@@ -69,11 +74,13 @@
import org.apache.paimon.table.sink.BatchTableWrite;
import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.DeletionFile;
import org.apache.paimon.table.source.RawFile;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.system.ReadOptimizedTable;
+import org.apache.paimon.table.system.SystemTableLoader;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
@@ -892,9 +899,11 @@ public void nativeBinaryPartitionsKeepAllPathValues(@TempDir Path warehouse) thr
new org.apache.paimon.fs.Path(warehouse.resolve(format).toUri()))) {
catalog.createDatabase("db", false);
Identifier id = Identifier.create("db", "binary_parts");
+ // Legacy partition names cannot name a binary partition (legacyNamedBinaryPartitionsFailScanPlanning).
catalog.createTable(id, Schema.newBuilder().column("id", DataTypes.INT())
.column("key", DataTypes.BYTES()).column("region", DataTypes.STRING())
.partitionKeys("key", "region").option("bucket", "-1")
+ .option("partition.legacy-name", "false")
.option("file.format", format).build(), false);
Table table = catalog.getTable(id);
BatchWriteBuilder writer = table.newBatchWriteBuilder();
@@ -925,6 +934,323 @@ public void nativeBinaryPartitionsKeepAllPathValues(@TempDir Path warehouse) thr
}
}
+ @Test
+ public void legacyNamedBinaryPartitionsFailScanPlanning(@TempDir Path warehouse) throws Exception {
+ // With partition.legacy-name = true, the default, Paimon looks a non-NULL binary partition up under the name
+ // of a Java array object, so the planned files never exist and the BE would skip them as missing files.
+ try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
+ new org.apache.paimon.fs.Path(warehouse.toUri()))) {
+ catalog.createDatabase("db", false);
+ Table legacy = binaryPartitionTable(catalog, "legacy", Collections.emptyMap(),
+ new byte[] {0x41, 0x42}, null);
+ DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
+ () -> planBinaryPartitionScan(legacy, false));
+ Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.legacy: the table uses"
+ + " legacy partition names ('partition.legacy-name' = 'true'), under which Paimon names partition"
+ + " key = 0x4142 after a Java array object"), error.getMessage());
+
+ // COUNT(*) answers from the metadata row counts and reads no file.
+ List count = planBinaryPartitionScan(legacy, true);
+ Assertions.assertEquals(1, count.size());
+ Assertions.assertEquals(2, ((PaimonScanRange) count.get(0)).getPushDownRowCount());
+
+ // The NULL partition is named after partition.default-name, which Paimon finds again.
+ Table nullOnly = binaryPartitionTable(catalog, "null_only", Collections.emptyMap(), (byte[]) null);
+ Assertions.assertEquals(1, planBinaryPartitionScan(nullOnly, false).size());
+ }
+ }
+
+ @Test
+ public void binaryPartitionFilesOfAnotherNamingFailScanPlanning(@TempDir Path warehouse) throws Exception {
+ // Paimon writes a file under the name its partition has when written and looks it up under the name the
+ // partition has when read, so a file written under a legacy name stays lost after the option changes.
+ try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
+ new org.apache.paimon.fs.Path(warehouse.toUri()))) {
+ catalog.createDatabase("db", false);
+ Table legacy = binaryPartitionTable(catalog, "renamed", Collections.emptyMap(), new byte[] {0x41, 0x42});
+ String legacyFile = ((DataSplit) legacy.newReadBuilder().newScan().plan().splits().get(0))
+ .dataFiles().get(0).fileName();
+ catalog.alterTable(Identifier.create("db", "renamed"), Collections.singletonList(
+ SchemaChange.setOption("partition.legacy-name", "false")), false);
+ writeBinaryPartitions(catalog.getTable(Identifier.create("db", "renamed")), new byte[] {0x43, 0x44});
+ Table current = catalog.getTable(Identifier.create("db", "renamed"));
+ DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
+ () -> planBinaryPartitionScan(current, false));
+ Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.renamed: data file "
+ + legacyFile + " of partition key = 0x4142 was written while the table used legacy partition"
+ + " names ('partition.legacy-name' = 'true')"), error.getMessage());
+
+ // A file written under the bytes of its value is looked up under a legacy name once those are back on.
+ binaryPartitionTable(catalog, "restored", Collections.singletonMap("partition.legacy-name", "false"),
+ new byte[] {0x41, 0x42});
+ Table byBytes = catalog.getTable(Identifier.create("db", "restored"));
+ Assertions.assertEquals(1, planBinaryPartitionScan(byBytes, false).size());
+ catalog.alterTable(Identifier.create("db", "restored"), Collections.singletonList(
+ SchemaChange.setOption("partition.legacy-name", "true")), false);
+ Table restored = catalog.getTable(Identifier.create("db", "restored"));
+ error = Assertions.assertThrows(DorisConnectorException.class,
+ () -> planBinaryPartitionScan(restored, false));
+ Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.restored: the table"
+ + " uses legacy partition names"), error.getMessage());
+ }
+ }
+
+ @Test
+ public void binaryPartitionFilesWrittenWithDynamicNamesStayReadable(@TempDir Path warehouse) throws Exception {
+ // Dynamic options keep the schema ID, so a file written with a dynamic partition.legacy-name = false refers
+ // to a schema with legacy names while it sits under the bytes of its value, where a lookup by bytes finds it.
+ try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
+ new org.apache.paimon.fs.Path(warehouse.toUri()))) {
+ catalog.createDatabase("db", false);
+ Identifier id = Identifier.create("db", "dynamic_names");
+ Table legacy = binaryPartitionTable(catalog, "dynamic_names", Collections.emptyMap());
+ writeBinaryPartitions(legacy.copy(Collections.singletonMap("partition.legacy-name", "false")),
+ new byte[] {0x41, 0x42});
+ Table current = catalog.getTable(id);
+ DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
+ () -> planBinaryPartitionScan(current, false));
+ Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.dynamic_names: the"
+ + " table uses legacy partition names"), error.getMessage());
+
+ catalog.alterTable(id, Collections.singletonList(
+ SchemaChange.setOption("partition.legacy-name", "false")), false);
+ List ranges = planBinaryPartitionScan(catalog.getTable(id), false);
+ Assertions.assertEquals(1, ranges.size());
+ String path = ranges.get(0).getPath().orElseThrow(AssertionError::new);
+ Assertions.assertTrue(path.contains("/key=AB/"), path);
+ Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(path)), path);
+ }
+ }
+
+ @Test
+ public void legacyNamedBinaryPartitionFilesAtExternalPathsStayReadable(@TempDir Path warehouse) throws Exception {
+ // A file at an external path is read from the path its metadata records, whatever its partition is named.
+ try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
+ new org.apache.paimon.fs.Path(warehouse.resolve("warehouse").toUri()))) {
+ catalog.createDatabase("db", false);
+ String externalRoot = warehouse.resolve("external").toUri().toString();
+ Map options = new HashMap<>();
+ options.put("data-file.external-paths", externalRoot);
+ options.put("data-file.external-paths.strategy", "round-robin");
+ Table legacy = binaryPartitionTable(catalog, "external_files", options, new byte[] {0x41, 0x42});
+ List ranges = planBinaryPartitionScan(legacy, false);
+ Assertions.assertEquals(1, ranges.size());
+ String path = ranges.get(0).getPath().orElseThrow(AssertionError::new);
+ Assertions.assertTrue(path.contains(warehouse.resolve("external").toString()), path);
+ Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(path)), path);
+ }
+ }
+
+ @Test
+ public void externalBinaryPartitionFileWithLocalDeletionVectorFailsScanPlanning(@TempDir Path warehouse)
+ throws Exception {
+ // With index-file-in-data-file-dir, a deletion vector written while no external data path is configured
+ // records only its name and is looked up under the directory of its partition, which a legacy name never
+ // finds, while its data file keeps its external path. The BE skips the whole data file when it cannot open
+ // the deletion vector, rows that were not deleted included.
+ try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
+ new org.apache.paimon.fs.Path(warehouse.resolve("warehouse").toUri()))) {
+ catalog.createDatabase("db", false);
+ Identifier id = Identifier.create("db", "local_dv");
+ deletionVectorBinaryPartitionTable(catalog, "local_dv", warehouse);
+ catalog.alterTable(id, Arrays.asList(SchemaChange.removeOption("data-file.external-paths"),
+ SchemaChange.removeOption("data-file.external-paths.strategy")), false);
+ Table table = catalog.getTable(id);
+ DataFileMeta file = ((DataSplit) table.newReadBuilder().newScan().plan().splits().get(0))
+ .dataFiles().get(0);
+ Assertions.assertTrue(file.externalPath().isPresent());
+ String deletionVector = deleteFirstRow(table);
+ DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
+ () -> planBinaryPartitionScan(table, false));
+ Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.local_dv: deletion vector"
+ + " file " + deletionVector + " of data file " + file.fileName() + " of partition key = 0x4142"
+ + " does not exist"), error.getMessage());
+ }
+ }
+
+ @Test
+ public void externalBinaryPartitionDeletionVectorAtExternalPathStaysReadable(@TempDir Path warehouse)
+ throws Exception {
+ // A deletion vector written while an external data path is configured records its absolute path.
+ try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
+ new org.apache.paimon.fs.Path(warehouse.resolve("warehouse").toUri()))) {
+ catalog.createDatabase("db", false);
+ Table table = deletionVectorBinaryPartitionTable(catalog, "external_dv", warehouse);
+ deleteFirstRow(table);
+ List ranges = planBinaryPartitionScan(table, false);
+ Assertions.assertEquals(1, ranges.size());
+ String deletionVector = ranges.get(0).getProperties().get("paimon.deletion_file.path");
+ Assertions.assertTrue(deletionVector.contains(warehouse.resolve("external").toString()), deletionVector);
+ Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(deletionVector)),
+ deletionVector);
+ }
+ }
+
+ @Test
+ public void fallbackBinaryPartitionsUseTheNamingOfTheirBranch(@TempDir Path warehouse) throws Exception {
+ // A partition missing on the main branch is planned and read under the options of the fallback branch,
+ // while the table and its $audit_log wrapper report the options of the main branch.
+ for (boolean fallbackUsesBytes : new boolean[] {true, false}) {
+ try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
+ new org.apache.paimon.fs.Path(warehouse.resolve("fallback_bytes_" + fallbackUsesBytes).toUri()))) {
+ catalog.createDatabase("db", false);
+ Map byBytes = Collections.singletonMap("partition.legacy-name", "false");
+ // The main branch only holds the NULL partition, which every naming finds.
+ FileStoreTable main = (FileStoreTable) binaryPartitionTable(catalog, "main",
+ fallbackUsesBytes ? Collections.emptyMap() : byBytes, (byte[]) null);
+ FileStoreTable fallback = (FileStoreTable) binaryPartitionTable(catalog, "fallback",
+ fallbackUsesBytes ? byBytes : Collections.emptyMap(), new byte[] {0x41, 0x42});
+ FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true);
+ if (!fallbackUsesBytes) {
+ Map>> plans = new LinkedHashMap<>();
+ plans.put("table", () -> planBinaryPartitionScan(pair, false));
+ plans.put("$audit_log", () -> planSystemTableScan("audit_log", pair));
+ plans.forEach((name, plan) -> {
+ DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
+ plan::get, name);
+ Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.main: the"
+ + " table uses legacy partition names ('partition.legacy-name' = 'true'), under which"
+ + " Paimon names partition key = 0x4142"), error.getMessage());
+ });
+ continue;
+ }
+ List ranges = planBinaryPartitionScan(pair, false);
+ Assertions.assertEquals(2, ranges.size());
+ for (ConnectorScanRange range : ranges) {
+ String path = range.getPath().orElseThrow(AssertionError::new);
+ Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(path)), path);
+ }
+ Assertions.assertTrue(ranges.stream().anyMatch(range -> range.getPath().get().contains("/key=AB/")));
+
+ List auditLog = planSystemTableScan("audit_log", pair);
+ Assertions.assertEquals(2, auditLog.size());
+ List rows = new ArrayList<>();
+ SystemTableLoader.load("audit_log", pair).newReadBuilder().newRead()
+ .createReader(deserializeJniSplits(auditLog))
+ .forEachRemaining(row -> rows.add(row.getString(0) + " " + (row.isNullAt(2) ? "NULL"
+ : java.util.HexFormat.of().withUpperCase().formatHex(row.getBinary(2)))));
+ rows.sort(Comparator.naturalOrder());
+ Assertions.assertEquals(Arrays.asList("+I 4142", "+I NULL"), rows);
+ }
+ }
+ }
+
+ @Test
+ public void fileCreationTimeScanOfFallbackTableChecksBinaryPartitionsOfTheMainBranch(@TempDir Path warehouse)
+ throws Exception {
+ // A scan.file-creation-time-millis read plans a fallback table through its SnapshotReader, which reads the
+ // main branch and returns plain DataSplits rather than the FallbackSplits of the fallback scan.
+ try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),
+ new org.apache.paimon.fs.Path(warehouse.toUri()))) {
+ catalog.createDatabase("db", false);
+ Map byBytes = Collections.singletonMap("partition.legacy-name", "false");
+ FileStoreTable main = (FileStoreTable) binaryPartitionTable(catalog, "main", byBytes,
+ new byte[] {0x41, 0x42});
+ FileStoreTable fallback = (FileStoreTable) binaryPartitionTable(catalog, "fallback", byBytes,
+ new byte[] {0x43, 0x44});
+ FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true);
+ RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps();
+ ops.table = pair;
+ PaimonScanPlanProvider provider = new PaimonScanPlanProvider(
+ PaimonCatalogProperties.of(Collections.emptyMap()), ops);
+ PaimonTableHandle handle = new PaimonTableHandle("db", "main", Collections.emptyList(),
+ Collections.emptyList()).withScanOptions(PaimonScanParams.markAsOptions(
+ PaimonScanParams.resolveOptions(pair, Collections.singletonMap(
+ CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key(), "0"))));
+ List ranges = provider.planScan(
+ sessionWithProps(Collections.emptyMap(), new TestStatementScope()),
+ ConnectorScanRequest.builder(handle, Arrays.asList(new PaimonColumnHandle("id", 0),
+ new PaimonColumnHandle("key", 1))).build());
+ Assertions.assertEquals(1, ranges.size());
+ String path = ranges.get(0).getPath().orElseThrow(AssertionError::new);
+ Assertions.assertTrue(path.contains("/key=AB/"), path);
+
+ DataSplit split = (DataSplit) pair.newSnapshotReader().read().splits().get(0);
+ Assertions.assertFalse(split instanceof FallbackReadFileStoreTable.FallbackSplit);
+ Assertions.assertSame(main, PaimonScanPlanProvider.legacyOrcSplitTable(pair, split));
+ }
+ }
+
+ private static Table binaryPartitionTable(Catalog catalog, String name, Map options,
+ byte[]... keys) throws Exception {
+ Identifier id = Identifier.create("db", name);
+ Schema.Builder schema = Schema.newBuilder().column("id", DataTypes.INT()).column("key", DataTypes.BYTES())
+ .partitionKeys("key").option("bucket", "-1");
+ options.forEach(schema::option);
+ catalog.createTable(id, schema.build(), false);
+ writeBinaryPartitions(catalog.getTable(id), keys);
+ return catalog.getTable(id);
+ }
+
+ private static void writeBinaryPartitions(Table table, byte[]... keys) throws Exception {
+ BatchWriteBuilder writer = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = writer.newWrite()) {
+ for (int i = 0; i < keys.length; i++) {
+ write.write(GenericRow.of(i, keys[i]));
+ }
+ try (BatchTableCommit commit = writer.newCommit()) {
+ commit.commit(write.prepareCommit());
+ }
+ }
+ }
+
+ private static List planBinaryPartitionScan(Table table, boolean countPushdown) {
+ RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps();
+ ops.table = table;
+ PaimonScanPlanProvider provider = new PaimonScanPlanProvider(
+ PaimonCatalogProperties.of(Collections.emptyMap()), ops);
+ PaimonTableHandle handle = new PaimonTableHandle("db", table.name(),
+ Collections.emptyList(), Collections.emptyList());
+ return provider.planScan(sessionWithProps(Collections.emptyMap(), new TestStatementScope()),
+ ConnectorScanRequest.builder(handle, Arrays.asList(new PaimonColumnHandle("id", 0),
+ new PaimonColumnHandle("key", 1))).countPushdown(countPushdown).build());
+ }
+
+ /** Plans a scan of {@code source$kind} the way the connector metadata builds its system table handle. */
+ private static List planSystemTableScan(String kind, FileStoreTable source) {
+ RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps();
+ ops.table = source;
+ PaimonScanPlanProvider provider = new PaimonScanPlanProvider(
+ PaimonCatalogProperties.of(Collections.emptyMap()), ops);
+ PaimonTableHandle handle = PaimonTableHandle.forSystemTable("db", source.name(), kind,
+ PaimonScanParams.requiresPaimonReader(kind));
+ handle.setPaimonTable(SystemTableLoader.load(kind, source));
+ handle.setSysBaseTable(source);
+ handle.setSystemTableSource(source);
+ return provider.planScan(sessionWithProps(Collections.emptyMap(), new TestStatementScope()),
+ ConnectorScanRequest.builder(handle, Collections.emptyList()).build());
+ }
+
+ private static Table deletionVectorBinaryPartitionTable(Catalog catalog, String name, Path warehouse)
+ throws Exception {
+ Map options = new HashMap<>();
+ options.put("deletion-vectors.enabled", "true");
+ options.put("index-file-in-data-file-dir", "true");
+ options.put("data-file.external-paths", warehouse.resolve("external").toUri().toString());
+ options.put("data-file.external-paths.strategy", "round-robin");
+ return binaryPartitionTable(catalog, name, options, new byte[] {0x41, 0x42}, new byte[] {0x41, 0x42});
+ }
+
+ /** Deletes the first row of the first data file the way a Spark DELETE persists its deletion vectors. */
+ private static String deleteFirstRow(Table table) throws Exception {
+ DataSplit split = (DataSplit) table.newReadBuilder().newScan().plan().splits().get(0);
+ // PaimonSparkWriter.persistDeletionVectors writes the index files under the table's current options.
+ BaseAppendDeleteFileMaintainer maintainer = BaseAppendDeleteFileMaintainer.forUnawareAppend(
+ ((FileStoreTable) table).store().newIndexFileHandler(), table.latestSnapshot().get(),
+ split.partition());
+ BitmapDeletionVector deletionVector = new BitmapDeletionVector();
+ deletionVector.delete(0);
+ maintainer.notifyNewDeletionVector(split.dataFiles().get(0).fileName(), deletionVector);
+ List indexFiles = new ArrayList<>();
+ maintainer.persist().forEach(entry -> indexFiles.add(entry.indexFile()));
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(Collections.singletonList(new CommitMessageImpl(maintainer.getPartition(),
+ maintainer.getBucket(), null, DataIncrement.indexIncrement(indexFiles),
+ CompactIncrement.emptyIncrement())));
+ }
+ return indexFiles.get(0).fileName();
+ }
+
@Test
public void planScanPushesLimitIntoPaimonSplitPlanning(@TempDir Path warehouse) throws Exception {
try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),