From a77603c0a0efba7aaed96099afd2692d8b696a55 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 24 Oct 2026 16:37:39 +0800 Subject: [PATCH 1/3] [fix](paimon) Fail scans of binary partitions with legacy partition names ### What problem does this PR solve? Issue Number: None Related PR: #69567 Problem Summary: Paimon finds the files of a partition under the directory its partition values name. With partition.legacy-name = true, the default, Paimon names a partition after the string of each value, which for a non-NULL BINARY / VARBINARY value is the Java array object ([B@1b6d3586) rather than its bytes, so every resolution of the name differs. The paths the connector planned for such a partition never existed: Paimon's own read fails with FileNotFoundException, while the BE skips missing files of an external table (ignore_not_found_file_in_external_table, default true), so a Doris query silently returned only the rows of the NULL partition (SELECT * of a table with rows in X'4142', X'' and NULL returned 1 row, while COUNT(*), answered from metadata, returned 3). The same happens to files written under a legacy name after the table switches to 'partition.legacy-name' = 'false', and to files written under the new names after it switches back. Scan planning now fails for a split whose binary partition value is not NULL when the split is looked up under legacy partition names, or when one of its data files was written under a schema whose options use legacy names, naming the table, the partition and the option. A split of the NULL partition, a file at an external path (found by the path its metadata records) and a COUNT(*) answered from metadata still plan as before. ### Release note Fix: a query of a Paimon table with a binary partition column whose partitions use legacy partition names ('partition.legacy-name' = 'true', the default) fails with an error naming the option, instead of silently returning only the rows of the NULL partition. ### Check List (For Author) - Test: Unit Test, Manual test - PaimonScanPlanProviderTest: legacy names fail planning while the NULL partition and COUNT(*) do not; a file written under legacy names fails after the option changes, and a file written under the new names fails after it changes back; a file at an external path still plans - Manual: local file:// Paimon catalog with tables created by the Paimon API - Behavior changed: Yes. A query that reads a non-NULL binary partition of a Paimon table with legacy partition names fails instead of returning a partial result. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) --- .../paimon/PaimonScanPlanProvider.java | 66 +++++++++- .../paimon/PaimonScanPlanProviderTest.java | 117 ++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 6c5493765e13f0..b037ca7ed064a3 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -86,6 +86,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InstantiationUtil; @@ -990,6 +991,8 @@ && hasTrustworthyLimitAccounting(table) // $ro wraps the pinned file-store table; resolve its schema dictionary once, only if native is considered. java.util.function.Supplier legacyOrcSchemaTable = com.google.common.base.Suppliers.memoize( () -> resolveSchemaDictTable(table, paimonHandle)); + int[] binaryPartitionFields = binaryPartitionFields(table); + Map> legacyPartitionNameSchemas = new java.util.IdentityHashMap<>(); // Process DataSplits for (DataSplit dataSplit : dataSplits) { if (isCountPushdownSplit(countPushdown, dataSplit)) { @@ -999,6 +1002,10 @@ && hasTrustworthyLimitAccounting(table) } continue; } + if (binaryPartitionFields.length > 0) { + checkBinaryPartitionFiles(paimonHandle, table, dataSplit, binaryPartitionFields, + legacyOrcSchemaTable, legacyPartitionNameSchemas); + } Map partitionValues = getPartitionInfoMap( table, dataSplit.partition(), session.getTimeZone()); @@ -1429,7 +1436,7 @@ private T withSchemaHistoryAuthentication(Supplier action) { } catch (RuntimeException e) { throw e; } catch (Exception e) { - throw new RuntimeException("Failed to read Paimon timestamp schema history", e); + throw new RuntimeException("Failed to read Paimon schema history", e); } } @@ -2292,6 +2299,63 @@ static FileStoreTable legacyOrcSplitTable(FileStoreTable table, DataSplit split) return base; } + /** Positions in the partition row of the partition columns that hold binary values. */ + private static int[] binaryPartitionFields(Table table) { + RowType partitionType = table.rowType().project(table.partitionKeys()); + return java.util.stream.IntStream.range(0, partitionType.getFieldCount()).filter(field -> { + DataTypeRoot typeRoot = partitionType.getTypeAt(field).getTypeRoot(); + return typeRoot == DataTypeRoot.BINARY || typeRoot == DataTypeRoot.VARBINARY; + }).toArray(); + } + + /** + * Paimon finds the files of a partition under the directory its values name. With {@code partition.legacy-name} + * = true, the default, it names a partition after the string of each value, which for a non-NULL binary value + * is the Java array object ({@code [B@1b6d3586}) rather than its bytes, and every resolution of the name + * differs. A file written under such a name, or looked up under one, is never found: Paimon's own read fails, + * while the BE skips a missing file of an external table and would silently drop the partition's rows. A file + * with an external path is found by the path its metadata records. Without a schema dictionary (system tables + * other than {@code $ro}, which the JNI reader reads with Paimon's own reader) only the lookup is checked. + */ + private void checkBinaryPartitionFiles(PaimonTableHandle handle, Table table, DataSplit split, + int[] binaryPartitionFields, Supplier
schemaTable, + Map> legacyPartitionNameSchemas) { + BinaryRow partition = split.partition(); + OptionalInt nonNullField = Arrays.stream(binaryPartitionFields) + .filter(field -> !partition.isNullAt(field)).findFirst(); + if (!nonNullField.isPresent()) { + return; + } + int binaryField = nonNullField.getAsInt(); + FileStoreTable splitTable = schemaTable.get() == null + ? null : legacyOrcSplitTable((FileStoreTable) schemaTable.get(), split); + boolean legacyLookup = CoreOptions.fromMap((splitTable == null ? table : splitTable).options()) + .legacyPartitionName(); + for (DataFileMeta file : split.dataFiles()) { + if (file.externalPath().isPresent()) { + continue; + } + if (legacyLookup || (splitTable != null && legacyPartitionNameSchemas + .computeIfAbsent(splitTable, ignored -> new HashMap<>()) + .computeIfAbsent(file.schemaId(), id -> withSchemaHistoryAuthentication(() -> CoreOptions + .fromMap(splitTable.schemaManager().schema(id).options()).legacyPartitionName())))) { + String partitionName = table.partitionKeys().get(binaryField) + " = 0x" + + java.util.HexFormat.of().withUpperCase().formatHex(partition.getBinary(binaryField)); + throw new DorisConnectorException("Cannot read Paimon table " + handle.getDatabaseName() + "." + + handle.getTableName() + ": " + (legacyLookup + ? "the table uses legacy partition names ('partition.legacy-name' = 'true'), under which" + + " Paimon names partition " + partitionName + " after a Java array object rather" + + " than its bytes, so the files of the partition are never found" + : "data file " + file.fileName() + " of partition " + partitionName + " was written while" + + " the table used legacy partition names ('partition.legacy-name' = 'true')," + + " under which Paimon names the partition after a Java array object rather than" + + " its bytes, so the file is not under the directory of the partition") + + ". Paimon cannot read the partition either; a table with a binary partition column needs" + + " 'partition.legacy-name' = 'false' before its data is written"); + } + } + } + private static boolean requiresCrossFamilyTimestampReader(FileStoreTable table, Optional> rawFiles, Set readFieldIds, Map schemaTimestamps) { RowType readType = new RowType(table.rowType().getFields().stream() diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index d29cae9ba8d1ef..5228d063037d2e 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -892,9 +892,11 @@ public void nativeBinaryPartitionsKeepAllPathValues(@TempDir Path warehouse) thr new org.apache.paimon.fs.Path(warehouse.resolve(format).toUri()))) { catalog.createDatabase("db", false); Identifier id = Identifier.create("db", "binary_parts"); + // Legacy partition names cannot name a binary partition (legacyNamedBinaryPartitionsFailScanPlanning). catalog.createTable(id, Schema.newBuilder().column("id", DataTypes.INT()) .column("key", DataTypes.BYTES()).column("region", DataTypes.STRING()) .partitionKeys("key", "region").option("bucket", "-1") + .option("partition.legacy-name", "false") .option("file.format", format).build(), false); Table table = catalog.getTable(id); BatchWriteBuilder writer = table.newBatchWriteBuilder(); @@ -925,6 +927,121 @@ public void nativeBinaryPartitionsKeepAllPathValues(@TempDir Path warehouse) thr } } + @Test + public void legacyNamedBinaryPartitionsFailScanPlanning(@TempDir Path warehouse) throws Exception { + // With partition.legacy-name = true, the default, Paimon looks a non-NULL binary partition up under the name + // of a Java array object, so the planned files never exist and the BE would skip them as missing files. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Table legacy = binaryPartitionTable(catalog, "legacy", Collections.emptyMap(), + new byte[] {0x41, 0x42}, null); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(legacy, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.legacy: the table uses" + + " legacy partition names ('partition.legacy-name' = 'true'), under which Paimon names partition" + + " key = 0x4142 after a Java array object"), error.getMessage()); + + // COUNT(*) answers from the metadata row counts and reads no file. + List count = planBinaryPartitionScan(legacy, true); + Assertions.assertEquals(1, count.size()); + Assertions.assertEquals(2, ((PaimonScanRange) count.get(0)).getPushDownRowCount()); + + // The NULL partition is named after partition.default-name, which Paimon finds again. + Table nullOnly = binaryPartitionTable(catalog, "null_only", Collections.emptyMap(), (byte[]) null); + Assertions.assertEquals(1, planBinaryPartitionScan(nullOnly, false).size()); + } + } + + @Test + public void binaryPartitionFilesOfAnotherNamingFailScanPlanning(@TempDir Path warehouse) throws Exception { + // Paimon writes a file under the name its partition has when written and looks it up under the name the + // partition has when read, so a file written under a legacy name stays lost after the option changes. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Table legacy = binaryPartitionTable(catalog, "renamed", Collections.emptyMap(), new byte[] {0x41, 0x42}); + String legacyFile = ((DataSplit) legacy.newReadBuilder().newScan().plan().splits().get(0)) + .dataFiles().get(0).fileName(); + catalog.alterTable(Identifier.create("db", "renamed"), Collections.singletonList( + SchemaChange.setOption("partition.legacy-name", "false")), false); + writeBinaryPartitions(catalog.getTable(Identifier.create("db", "renamed")), new byte[] {0x43, 0x44}); + Table current = catalog.getTable(Identifier.create("db", "renamed")); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(current, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.renamed: data file " + + legacyFile + " of partition key = 0x4142 was written while the table used legacy partition" + + " names ('partition.legacy-name' = 'true')"), error.getMessage()); + + // A file written under the bytes of its value is looked up under a legacy name once those are back on. + binaryPartitionTable(catalog, "restored", Collections.singletonMap("partition.legacy-name", "false"), + new byte[] {0x41, 0x42}); + Table byBytes = catalog.getTable(Identifier.create("db", "restored")); + Assertions.assertEquals(1, planBinaryPartitionScan(byBytes, false).size()); + catalog.alterTable(Identifier.create("db", "restored"), Collections.singletonList( + SchemaChange.setOption("partition.legacy-name", "true")), false); + Table restored = catalog.getTable(Identifier.create("db", "restored")); + error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(restored, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.restored: the table" + + " uses legacy partition names"), error.getMessage()); + } + } + + @Test + public void legacyNamedBinaryPartitionFilesAtExternalPathsStayReadable(@TempDir Path warehouse) throws Exception { + // A file at an external path is read from the path its metadata records, whatever its partition is named. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.resolve("warehouse").toUri()))) { + catalog.createDatabase("db", false); + String externalRoot = warehouse.resolve("external").toUri().toString(); + Map options = new HashMap<>(); + options.put("data-file.external-paths", externalRoot); + options.put("data-file.external-paths.strategy", "round-robin"); + Table legacy = binaryPartitionTable(catalog, "external_files", options, new byte[] {0x41, 0x42}); + List ranges = planBinaryPartitionScan(legacy, false); + Assertions.assertEquals(1, ranges.size()); + String path = ranges.get(0).getPath().orElseThrow(AssertionError::new); + Assertions.assertTrue(path.contains(warehouse.resolve("external").toString()), path); + Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(path)), path); + } + } + + private static Table binaryPartitionTable(Catalog catalog, String name, Map options, + byte[]... keys) throws Exception { + Identifier id = Identifier.create("db", name); + Schema.Builder schema = Schema.newBuilder().column("id", DataTypes.INT()).column("key", DataTypes.BYTES()) + .partitionKeys("key").option("bucket", "-1"); + options.forEach(schema::option); + catalog.createTable(id, schema.build(), false); + writeBinaryPartitions(catalog.getTable(id), keys); + return catalog.getTable(id); + } + + private static void writeBinaryPartitions(Table table, byte[]... keys) throws Exception { + BatchWriteBuilder writer = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writer.newWrite()) { + for (int i = 0; i < keys.length; i++) { + write.write(GenericRow.of(i, keys[i])); + } + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + } + + private static List planBinaryPartitionScan(Table table, boolean countPushdown) { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = new PaimonTableHandle("db", table.name(), + Collections.emptyList(), Collections.emptyList()); + return provider.planScan(sessionWithProps(Collections.emptyMap(), new TestStatementScope()), + ConnectorScanRequest.builder(handle, Arrays.asList(new PaimonColumnHandle("id", 0), + new PaimonColumnHandle("key", 1))).countPushdown(countPushdown).build()); + } + @Test public void planScanPushesLimitIntoPaimonSplitPlanning(@TempDir Path warehouse) throws Exception { try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), From 65a1682d33409db5bcb2146bfbb4ac7ac66dbccf Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 24 Oct 2026 21:38:05 +0800 Subject: [PATCH 2/3] [fix](paimon) Check deletion vectors and confirm missing files of binary partitions ### What problem does this PR solve? Issue Number: None Related PR: #69571 Problem Summary: Follow-up to the scan-planning check of binary partitions in this PR, for two cases found in review. With 'index-file-in-data-file-dir' = 'true', a deletion vector records an absolute path only if an external data path is configured when it is written; otherwise Paimon looks it up under the directory of its partition when the split is planned, which under legacy partition names is named after a fresh Java array object. The check exempted every data file with an external path without looking at its deletion vector, so once the external data paths were removed from such a table, a deletion vector written by a DELETE was never found while its data file kept its absolute path: the native readers get NOT_FOUND when they open the deletion vector and the BE skips the whole data file, surviving rows included. For a split with a non-NULL binary partition value on such a table, every deletion vector the split carries must now exist, whether or not its data file has an external path; a missing one fails planning, naming the deletion vector file, the data file, the partition and the option. Deletion vectors under the table's index/ directory do not depend on partition names and are not checked. A writer's dynamic options (spark.paimon.*, Flink hints, Table.copy) keep the schema ID, so a file written with a dynamic 'partition.legacy-name' = 'false' on a table that keeps the default refers to a schema with legacy names while it sits under the bytes of its value. The check rejected such files after the table was altered to 'false', although Paimon and master read them. A file whose schema uses legacy names now fails planning only if it is also missing at its planned path (bucket path / file name). Both existence checks run inside the catalog authentication and are cached per path for the scan. A table that keeps its index files under index/ and whose files were written under its current naming makes no extra filesystem calls. ### Release note None (follow-up inside the PR) ### Check List (For Author) - Test: Unit Test - PaimonScanPlanProviderTest: a data file at an external path whose deletion vector was written after the external paths were removed fails planning; a deletion vector written while the external paths are configured still plans; a file written through table.copy({"partition.legacy-name": "false"}) on a table that keeps the default fails as a legacy lookup, and plans after the option is altered to false - fe-connector-paimon module tests: 851 run, 0 failures, 0 errors, 1 skipped (pre-existing) - Behavior changed: Yes. A scan of a non-NULL binary partition fails when a deletion vector it carries is missing, and a file whose schema uses legacy names fails it only when the file is missing at its planned path. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) --- .../paimon/PaimonScanPlanProvider.java | 99 +++++++++++----- .../paimon/PaimonScanPlanProviderTest.java | 108 ++++++++++++++++++ 2 files changed, 181 insertions(+), 26 deletions(-) diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index b037ca7ed064a3..ddb111647fc8a4 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -59,6 +59,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.Timestamp; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataOutputViewStreamWrapper; import org.apache.paimon.options.Options; @@ -993,6 +994,7 @@ && hasTrustworthyLimitAccounting(table) () -> resolveSchemaDictTable(table, paimonHandle)); int[] binaryPartitionFields = binaryPartitionFields(table); Map> legacyPartitionNameSchemas = new java.util.IdentityHashMap<>(); + Map fileExistence = new HashMap<>(); // Process DataSplits for (DataSplit dataSplit : dataSplits) { if (isCountPushdownSplit(countPushdown, dataSplit)) { @@ -1004,7 +1006,7 @@ && hasTrustworthyLimitAccounting(table) } if (binaryPartitionFields.length > 0) { checkBinaryPartitionFiles(paimonHandle, table, dataSplit, binaryPartitionFields, - legacyOrcSchemaTable, legacyPartitionNameSchemas); + legacyOrcSchemaTable, legacyPartitionNameSchemas, fileExistence); } Map partitionValues = getPartitionInfoMap( @@ -1436,7 +1438,7 @@ private T withSchemaHistoryAuthentication(Supplier action) { } catch (RuntimeException e) { throw e; } catch (Exception e) { - throw new RuntimeException("Failed to read Paimon schema history", e); + throw new RuntimeException("Failed to access Paimon table files", e); } } @@ -2315,11 +2317,19 @@ private static int[] binaryPartitionFields(Table table) { * differs. A file written under such a name, or looked up under one, is never found: Paimon's own read fails, * while the BE skips a missing file of an external table and would silently drop the partition's rows. A file * with an external path is found by the path its metadata records. Without a schema dictionary (system tables - * other than {@code $ro}, which the JNI reader reads with Paimon's own reader) only the lookup is checked. + * other than {@code $ro}, which the JNI reader reads with Paimon's own reader) the schemas of the files are not + * consulted. + * + *

The schema a file refers to does not prove how the file was named, because dynamic options + * ({@code spark.paimon.*}, Flink hints, {@code Table.copy}) keep the schema ID: a file whose schema uses legacy + * names fails only if it is also missing at its planned path. With {@code index-file-in-data-file-dir} = true, a + * deletion vector written without an external data path is looked up under the directory of its partition too, + * whatever the path of its data file, and the BE skips the whole data file when it cannot open the deletion + * vector, so every deletion vector must exist. Each path is checked once per scan. */ private void checkBinaryPartitionFiles(PaimonTableHandle handle, Table table, DataSplit split, int[] binaryPartitionFields, Supplier

schemaTable, - Map> legacyPartitionNameSchemas) { + Map> legacyPartitionNameSchemas, Map fileExistence) { BinaryRow partition = split.partition(); OptionalInt nonNullField = Arrays.stream(binaryPartitionFields) .filter(field -> !partition.isNullAt(field)).findFirst(); @@ -2329,33 +2339,70 @@ private void checkBinaryPartitionFiles(PaimonTableHandle handle, Table table, Da int binaryField = nonNullField.getAsInt(); FileStoreTable splitTable = schemaTable.get() == null ? null : legacyOrcSplitTable((FileStoreTable) schemaTable.get(), split); - boolean legacyLookup = CoreOptions.fromMap((splitTable == null ? table : splitTable).options()) - .legacyPartitionName(); - for (DataFileMeta file : split.dataFiles()) { - if (file.externalPath().isPresent()) { - continue; + Table optionsTable = splitTable == null ? table : splitTable; + CoreOptions options = CoreOptions.fromMap(optionsTable.options()); + boolean legacyLookup = options.legacyPartitionName(); + // Deletion vectors under the table's index directory do not depend on partition names. The option is + // immutable, so every writer of the table used this value. + List deletionFiles = options.indexFileInDataFileDir() ? split.deletionFiles().orElse(null) : null; + List files = split.dataFiles(); + for (int i = 0; i < files.size(); i++) { + DataFileMeta file = files.get(i); + if (!file.externalPath().isPresent()) { + if (legacyLookup) { + throw binaryPartitionReadError(handle, "the table uses legacy partition names" + + " ('partition.legacy-name' = 'true'), under which Paimon names partition " + + binaryPartitionName(table, partition, binaryField) + " after a Java array object" + + " rather than its bytes, so the files of the partition are never found"); + } + if (splitTable != null && legacyPartitionNameSchemas + .computeIfAbsent(splitTable, ignored -> new HashMap<>()) + .computeIfAbsent(file.schemaId(), id -> withSchemaHistoryAuthentication(() -> CoreOptions + .fromMap(splitTable.schemaManager().schema(id).options()).legacyPartitionName())) + && !fileExists(optionsTable.fileIO(), split.bucketPath() + "/" + file.fileName(), + fileExistence)) { + throw binaryPartitionReadError(handle, "data file " + file.fileName() + " of partition " + + binaryPartitionName(table, partition, binaryField) + " was written while the table" + + " used legacy partition names ('partition.legacy-name' = 'true'), under which Paimon" + + " names the partition after a Java array object rather than its bytes, so the file is" + + " not under the directory of the partition"); + } } - if (legacyLookup || (splitTable != null && legacyPartitionNameSchemas - .computeIfAbsent(splitTable, ignored -> new HashMap<>()) - .computeIfAbsent(file.schemaId(), id -> withSchemaHistoryAuthentication(() -> CoreOptions - .fromMap(splitTable.schemaManager().schema(id).options()).legacyPartitionName())))) { - String partitionName = table.partitionKeys().get(binaryField) + " = 0x" - + java.util.HexFormat.of().withUpperCase().formatHex(partition.getBinary(binaryField)); - throw new DorisConnectorException("Cannot read Paimon table " + handle.getDatabaseName() + "." - + handle.getTableName() + ": " + (legacyLookup - ? "the table uses legacy partition names ('partition.legacy-name' = 'true'), under which" - + " Paimon names partition " + partitionName + " after a Java array object rather" - + " than its bytes, so the files of the partition are never found" - : "data file " + file.fileName() + " of partition " + partitionName + " was written while" - + " the table used legacy partition names ('partition.legacy-name' = 'true')," - + " under which Paimon names the partition after a Java array object rather than" - + " its bytes, so the file is not under the directory of the partition") - + ". Paimon cannot read the partition either; a table with a binary partition column needs" - + " 'partition.legacy-name' = 'false' before its data is written"); + DeletionFile deletionFile = deletionFiles == null ? null : deletionFiles.get(i); + if (deletionFile != null && !fileExists(optionsTable.fileIO(), deletionFile.path(), fileExistence)) { + throw binaryPartitionReadError(handle, "deletion vector file " + + new Path(deletionFile.path()).getName() + " of data file " + file.fileName() + + " of partition " + binaryPartitionName(table, partition, binaryField) + + " does not exist: the table keeps index files with its data files" + + " ('index-file-in-data-file-dir' = 'true'), so a deletion vector written without an external" + + " path is looked up under the directory of its partition, and legacy partition names" + + " ('partition.legacy-name' = 'true') name the partition after a Java array object rather" + + " than its bytes"); } } } + private static String binaryPartitionName(Table table, BinaryRow partition, int binaryField) { + return table.partitionKeys().get(binaryField) + " = 0x" + + java.util.HexFormat.of().withUpperCase().formatHex(partition.getBinary(binaryField)); + } + + private static DorisConnectorException binaryPartitionReadError(PaimonTableHandle handle, String reason) { + return new DorisConnectorException("Cannot read Paimon table " + handle.getDatabaseName() + "." + + handle.getTableName() + ": " + reason + ". Paimon cannot read the partition either; a table with a" + + " binary partition column needs 'partition.legacy-name' = 'false' before its data is written"); + } + + private boolean fileExists(FileIO fileIO, String path, Map fileExistence) { + return fileExistence.computeIfAbsent(path, ignored -> withSchemaHistoryAuthentication(() -> { + try { + return fileIO.exists(new Path(path)); + } catch (IOException e) { + throw new DorisConnectorException("Failed to check whether Paimon file " + path + " exists", e); + } + })); + } + private static boolean requiresCrossFamilyTimestampReader(FileStoreTable table, Optional> rawFiles, Set readFieldIds, Map schemaTimestamps) { RowType readType = new RowType(table.rowType().getFields().stream() diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index 5228d063037d2e..6c5eee3b52ee0f 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -50,9 +50,14 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.deletionvectors.BitmapDeletionVector; +import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataIncrement; import org.apache.paimon.io.DataInputViewStreamWrapper; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; @@ -69,6 +74,7 @@ import org.apache.paimon.table.sink.BatchTableWrite; import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.DeletionFile; import org.apache.paimon.table.source.RawFile; @@ -988,6 +994,33 @@ public void binaryPartitionFilesOfAnotherNamingFailScanPlanning(@TempDir Path wa } } + @Test + public void binaryPartitionFilesWrittenWithDynamicNamesStayReadable(@TempDir Path warehouse) throws Exception { + // Dynamic options keep the schema ID, so a file written with a dynamic partition.legacy-name = false refers + // to a schema with legacy names while it sits under the bytes of its value, where a lookup by bytes finds it. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "dynamic_names"); + Table legacy = binaryPartitionTable(catalog, "dynamic_names", Collections.emptyMap()); + writeBinaryPartitions(legacy.copy(Collections.singletonMap("partition.legacy-name", "false")), + new byte[] {0x41, 0x42}); + Table current = catalog.getTable(id); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(current, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.dynamic_names: the" + + " table uses legacy partition names"), error.getMessage()); + + catalog.alterTable(id, Collections.singletonList( + SchemaChange.setOption("partition.legacy-name", "false")), false); + List ranges = planBinaryPartitionScan(catalog.getTable(id), false); + Assertions.assertEquals(1, ranges.size()); + String path = ranges.get(0).getPath().orElseThrow(AssertionError::new); + Assertions.assertTrue(path.contains("/key=AB/"), path); + Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(path)), path); + } + } + @Test public void legacyNamedBinaryPartitionFilesAtExternalPathsStayReadable(@TempDir Path warehouse) throws Exception { // A file at an external path is read from the path its metadata records, whatever its partition is named. @@ -1007,6 +1040,51 @@ public void legacyNamedBinaryPartitionFilesAtExternalPathsStayReadable(@TempDir } } + @Test + public void externalBinaryPartitionFileWithLocalDeletionVectorFailsScanPlanning(@TempDir Path warehouse) + throws Exception { + // With index-file-in-data-file-dir, a deletion vector written while no external data path is configured + // records only its name and is looked up under the directory of its partition, which a legacy name never + // finds, while its data file keeps its external path. The BE skips the whole data file when it cannot open + // the deletion vector, rows that were not deleted included. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.resolve("warehouse").toUri()))) { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "local_dv"); + deletionVectorBinaryPartitionTable(catalog, "local_dv", warehouse); + catalog.alterTable(id, Arrays.asList(SchemaChange.removeOption("data-file.external-paths"), + SchemaChange.removeOption("data-file.external-paths.strategy")), false); + Table table = catalog.getTable(id); + DataFileMeta file = ((DataSplit) table.newReadBuilder().newScan().plan().splits().get(0)) + .dataFiles().get(0); + Assertions.assertTrue(file.externalPath().isPresent()); + String deletionVector = deleteFirstRow(table); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(table, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.local_dv: deletion vector" + + " file " + deletionVector + " of data file " + file.fileName() + " of partition key = 0x4142" + + " does not exist"), error.getMessage()); + } + } + + @Test + public void externalBinaryPartitionDeletionVectorAtExternalPathStaysReadable(@TempDir Path warehouse) + throws Exception { + // A deletion vector written while an external data path is configured records its absolute path. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.resolve("warehouse").toUri()))) { + catalog.createDatabase("db", false); + Table table = deletionVectorBinaryPartitionTable(catalog, "external_dv", warehouse); + deleteFirstRow(table); + List ranges = planBinaryPartitionScan(table, false); + Assertions.assertEquals(1, ranges.size()); + String deletionVector = ranges.get(0).getProperties().get("paimon.deletion_file.path"); + Assertions.assertTrue(deletionVector.contains(warehouse.resolve("external").toString()), deletionVector); + Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(deletionVector)), + deletionVector); + } + } + private static Table binaryPartitionTable(Catalog catalog, String name, Map options, byte[]... keys) throws Exception { Identifier id = Identifier.create("db", name); @@ -1042,6 +1120,36 @@ private static List planBinaryPartitionScan(Table table, boo new PaimonColumnHandle("key", 1))).countPushdown(countPushdown).build()); } + private static Table deletionVectorBinaryPartitionTable(Catalog catalog, String name, Path warehouse) + throws Exception { + Map options = new HashMap<>(); + options.put("deletion-vectors.enabled", "true"); + options.put("index-file-in-data-file-dir", "true"); + options.put("data-file.external-paths", warehouse.resolve("external").toUri().toString()); + options.put("data-file.external-paths.strategy", "round-robin"); + return binaryPartitionTable(catalog, name, options, new byte[] {0x41, 0x42}, new byte[] {0x41, 0x42}); + } + + /** Deletes the first row of the first data file the way a Spark DELETE persists its deletion vectors. */ + private static String deleteFirstRow(Table table) throws Exception { + DataSplit split = (DataSplit) table.newReadBuilder().newScan().plan().splits().get(0); + // PaimonSparkWriter.persistDeletionVectors writes the index files under the table's current options. + BaseAppendDeleteFileMaintainer maintainer = BaseAppendDeleteFileMaintainer.forUnawareAppend( + ((FileStoreTable) table).store().newIndexFileHandler(), table.latestSnapshot().get(), + split.partition()); + BitmapDeletionVector deletionVector = new BitmapDeletionVector(); + deletionVector.delete(0); + maintainer.notifyNewDeletionVector(split.dataFiles().get(0).fileName(), deletionVector); + List indexFiles = new ArrayList<>(); + maintainer.persist().forEach(entry -> indexFiles.add(entry.indexFile())); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.commit(Collections.singletonList(new CommitMessageImpl(maintainer.getPartition(), + maintainer.getBucket(), null, DataIncrement.indexIncrement(indexFiles), + CompactIncrement.emptyIncrement()))); + } + return indexFiles.get(0).fileName(); + } + @Test public void planScanPushesLimitIntoPaimonSplitPlanning(@TempDir Path warehouse) throws Exception { try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), From cf79a1ac2286e679617b819016acf7c06a3f59fa Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 25 Oct 2026 00:52:08 +0800 Subject: [PATCH 3/3] [fix](paimon) Check binary partitions of fallback splits under the options of their branch ### What problem does this PR solve? Issue Number: None Related PR: #69571 Problem Summary: Follow-up to the scan-planning check of binary partitions in this PR, for two cases found in review. The check takes the naming option and the file schemas from the branch that produced each split, but only when the table has a schema dictionary. For $audit_log and $row_tracking, which are read through JNI, there is none, so the check used the wrapper's own options, and an AuditLogTable reports the options of the fallback pair, which are the main branch's. With scan.fallback-branch set, a partition missing on the main branch is still planned and read under the fallback branch's options. A table whose main branch keeps the default 'partition.legacy-name' = 'true' and holds only a NULL partition, while its fallback branch uses 'false' and holds a non-NULL binary partition, was therefore rejected for $audit_log although Paimon reads the fallback rows; in the reverse layout the scan planned and Paimon's reader failed with FileNotFoundException at execution. For the wrappers read through JNI the check now resolves the table that produced the splits from the wrapper's pinned source, as the timestamp-history checks already do, and picks each split's branch from it as it does for data tables. $binlog exposes its partition columns as arrays and is not checked. A scan.file-creation-time-millis read plans a fallback table through the pair's SnapshotReader, which reads the wrapped branch and returns plain DataSplits. Picking the branch cast every split to FallbackSplit, so such a scan of a table with a non-NULL binary partition failed with a ClassCastException (the same cast on the native timestamp path predates this PR). A split that is not a FallbackSplit now maps to the wrapped branch. ### Release note None (follow-up inside the PR) ### Check List (For Author) - Test: Unit Test - PaimonScanPlanProviderTest: a fallback pair whose main branch holds only a NULL partition and whose fallback branch holds a non-NULL binary partition plans the data table and $audit_log, and the $audit_log splits read back both rows through Paimon, when the fallback branch names partitions by bytes; both fail at planning with the legacy-name error when only the main branch does; a scan.file-creation-time-millis read of a fallback pair with a non-NULL binary partition plans the main branch's file - fe-connector-paimon module tests: 853 run, 0 failures, 0 errors, 1 skipped (pre-existing) - Behavior changed: Yes. $audit_log and $row_tracking of a fallback table check binary partitions under the options of the branch that produced each split, and a file-creation-time read of a fallback table with a binary partition no longer fails with a ClassCastException. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) --- .../paimon/PaimonScanPlanProvider.java | 20 +++- .../paimon/PaimonScanPlanProviderTest.java | 101 ++++++++++++++++++ 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index ddb111647fc8a4..8e8e4c4f627030 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -992,6 +992,12 @@ && hasTrustworthyLimitAccounting(table) // $ro wraps the pinned file-store table; resolve its schema dictionary once, only if native is considered. java.util.function.Supplier
legacyOrcSchemaTable = com.google.common.base.Suppliers.memoize( () -> resolveSchemaDictTable(table, paimonHandle)); + // The wrappers read through JNI report the options of the main branch, but their splits come from a branch + // of the pinned source, which the timestamp-history checks resolve the same way. + java.util.function.Supplier
binaryPartitionSourceTable = paimonHandle.isForceJni() + ? com.google.common.base.Suppliers.memoize(() -> withSchemaHistoryAuthentication( + () -> resolveTimestampHistoryTable(table, paimonHandle))) + : legacyOrcSchemaTable; int[] binaryPartitionFields = binaryPartitionFields(table); Map> legacyPartitionNameSchemas = new java.util.IdentityHashMap<>(); Map fileExistence = new HashMap<>(); @@ -1006,7 +1012,7 @@ && hasTrustworthyLimitAccounting(table) } if (binaryPartitionFields.length > 0) { checkBinaryPartitionFiles(paimonHandle, table, dataSplit, binaryPartitionFields, - legacyOrcSchemaTable, legacyPartitionNameSchemas, fileExistence); + binaryPartitionSourceTable, legacyPartitionNameSchemas, fileExistence); } Map partitionValues = getPartitionInfoMap( @@ -2294,6 +2300,11 @@ static FileStoreTable legacyOrcSplitTable(FileStoreTable table, DataSplit split) FileStoreTable base = PaimonTableDecorators.unwrapToFallbackOrBase(table); if (base instanceof FallbackReadFileStoreTable) { FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) base; + if (!(split instanceof FallbackReadFileStoreTable.FallbackSplit)) { + // A split planned through the pair's SnapshotReader (a file-creation-time read) comes from its + // wrapped branch. + return pair.wrapped(); + } boolean fallback = ((FallbackReadFileStoreTable.FallbackSplit) split).isFallback(); // Match the SDK reader's branch selection before reading either options or historical schemas. return PaimonReaderOptions.isWrappedFirst(pair) != fallback ? pair.wrapped() : pair.other(); @@ -2316,9 +2327,10 @@ private static int[] binaryPartitionFields(Table table) { * is the Java array object ({@code [B@1b6d3586}) rather than its bytes, and every resolution of the name * differs. A file written under such a name, or looked up under one, is never found: Paimon's own read fails, * while the BE skips a missing file of an external table and would silently drop the partition's rows. A file - * with an external path is found by the path its metadata records. Without a schema dictionary (system tables - * other than {@code $ro}, which the JNI reader reads with Paimon's own reader) the schemas of the files are not - * consulted. + * with an external path is found by the path its metadata records. A system wrapper reports the options of the + * main branch while a fallback split is planned and read under its own branch, so the options and schemas come + * from the branch that produced the split; for the wrappers read through JNI that branch is resolved from the + * pinned source of the wrapper. * *

The schema a file refers to does not prove how the file was named, because dynamic options * ({@code spark.paimon.*}, Flink hints, {@code Table.copy}) keep the schema ID: a file whose schema uses legacy diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index 6c5eee3b52ee0f..bbb0513a29a376 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -80,6 +80,7 @@ import org.apache.paimon.table.source.RawFile; import org.apache.paimon.table.source.Split; import org.apache.paimon.table.system.ReadOptimizedTable; +import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; @@ -1085,6 +1086,91 @@ public void externalBinaryPartitionDeletionVectorAtExternalPathStaysReadable(@Te } } + @Test + public void fallbackBinaryPartitionsUseTheNamingOfTheirBranch(@TempDir Path warehouse) throws Exception { + // A partition missing on the main branch is planned and read under the options of the fallback branch, + // while the table and its $audit_log wrapper report the options of the main branch. + for (boolean fallbackUsesBytes : new boolean[] {true, false}) { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.resolve("fallback_bytes_" + fallbackUsesBytes).toUri()))) { + catalog.createDatabase("db", false); + Map byBytes = Collections.singletonMap("partition.legacy-name", "false"); + // The main branch only holds the NULL partition, which every naming finds. + FileStoreTable main = (FileStoreTable) binaryPartitionTable(catalog, "main", + fallbackUsesBytes ? Collections.emptyMap() : byBytes, (byte[]) null); + FileStoreTable fallback = (FileStoreTable) binaryPartitionTable(catalog, "fallback", + fallbackUsesBytes ? byBytes : Collections.emptyMap(), new byte[] {0x41, 0x42}); + FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); + if (!fallbackUsesBytes) { + Map>> plans = new LinkedHashMap<>(); + plans.put("table", () -> planBinaryPartitionScan(pair, false)); + plans.put("$audit_log", () -> planSystemTableScan("audit_log", pair)); + plans.forEach((name, plan) -> { + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + plan::get, name); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.main: the" + + " table uses legacy partition names ('partition.legacy-name' = 'true'), under which" + + " Paimon names partition key = 0x4142"), error.getMessage()); + }); + continue; + } + List ranges = planBinaryPartitionScan(pair, false); + Assertions.assertEquals(2, ranges.size()); + for (ConnectorScanRange range : ranges) { + String path = range.getPath().orElseThrow(AssertionError::new); + Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(path)), path); + } + Assertions.assertTrue(ranges.stream().anyMatch(range -> range.getPath().get().contains("/key=AB/"))); + + List auditLog = planSystemTableScan("audit_log", pair); + Assertions.assertEquals(2, auditLog.size()); + List rows = new ArrayList<>(); + SystemTableLoader.load("audit_log", pair).newReadBuilder().newRead() + .createReader(deserializeJniSplits(auditLog)) + .forEachRemaining(row -> rows.add(row.getString(0) + " " + (row.isNullAt(2) ? "NULL" + : java.util.HexFormat.of().withUpperCase().formatHex(row.getBinary(2))))); + rows.sort(Comparator.naturalOrder()); + Assertions.assertEquals(Arrays.asList("+I 4142", "+I NULL"), rows); + } + } + } + + @Test + public void fileCreationTimeScanOfFallbackTableChecksBinaryPartitionsOfTheMainBranch(@TempDir Path warehouse) + throws Exception { + // A scan.file-creation-time-millis read plans a fallback table through its SnapshotReader, which reads the + // main branch and returns plain DataSplits rather than the FallbackSplits of the fallback scan. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Map byBytes = Collections.singletonMap("partition.legacy-name", "false"); + FileStoreTable main = (FileStoreTable) binaryPartitionTable(catalog, "main", byBytes, + new byte[] {0x41, 0x42}); + FileStoreTable fallback = (FileStoreTable) binaryPartitionTable(catalog, "fallback", byBytes, + new byte[] {0x43, 0x44}); + FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = pair; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = new PaimonTableHandle("db", "main", Collections.emptyList(), + Collections.emptyList()).withScanOptions(PaimonScanParams.markAsOptions( + PaimonScanParams.resolveOptions(pair, Collections.singletonMap( + CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key(), "0")))); + List ranges = provider.planScan( + sessionWithProps(Collections.emptyMap(), new TestStatementScope()), + ConnectorScanRequest.builder(handle, Arrays.asList(new PaimonColumnHandle("id", 0), + new PaimonColumnHandle("key", 1))).build()); + Assertions.assertEquals(1, ranges.size()); + String path = ranges.get(0).getPath().orElseThrow(AssertionError::new); + Assertions.assertTrue(path.contains("/key=AB/"), path); + + DataSplit split = (DataSplit) pair.newSnapshotReader().read().splits().get(0); + Assertions.assertFalse(split instanceof FallbackReadFileStoreTable.FallbackSplit); + Assertions.assertSame(main, PaimonScanPlanProvider.legacyOrcSplitTable(pair, split)); + } + } + private static Table binaryPartitionTable(Catalog catalog, String name, Map options, byte[]... keys) throws Exception { Identifier id = Identifier.create("db", name); @@ -1120,6 +1206,21 @@ private static List planBinaryPartitionScan(Table table, boo new PaimonColumnHandle("key", 1))).countPushdown(countPushdown).build()); } + /** Plans a scan of {@code source$kind} the way the connector metadata builds its system table handle. */ + private static List planSystemTableScan(String kind, FileStoreTable source) { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = source; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = PaimonTableHandle.forSystemTable("db", source.name(), kind, + PaimonScanParams.requiresPaimonReader(kind)); + handle.setPaimonTable(SystemTableLoader.load(kind, source)); + handle.setSysBaseTable(source); + handle.setSystemTableSource(source); + return provider.planScan(sessionWithProps(Collections.emptyMap(), new TestStatementScope()), + ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); + } + private static Table deletionVectorBinaryPartitionTable(Catalog catalog, String name, Path warehouse) throws Exception { Map options = new HashMap<>();