diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 6c5493765e13f0..8e8e4c4f627030 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -59,6 +59,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.Timestamp; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataOutputViewStreamWrapper; import org.apache.paimon.options.Options; @@ -86,6 +87,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InstantiationUtil; @@ -990,6 +992,15 @@ && hasTrustworthyLimitAccounting(table) // $ro wraps the pinned file-store table; resolve its schema dictionary once, only if native is considered. java.util.function.Supplier legacyOrcSchemaTable = com.google.common.base.Suppliers.memoize( () -> resolveSchemaDictTable(table, paimonHandle)); + // The wrappers read through JNI report the options of the main branch, but their splits come from a branch + // of the pinned source, which the timestamp-history checks resolve the same way. + java.util.function.Supplier
binaryPartitionSourceTable = paimonHandle.isForceJni() + ? com.google.common.base.Suppliers.memoize(() -> withSchemaHistoryAuthentication( + () -> resolveTimestampHistoryTable(table, paimonHandle))) + : legacyOrcSchemaTable; + int[] binaryPartitionFields = binaryPartitionFields(table); + Map> legacyPartitionNameSchemas = new java.util.IdentityHashMap<>(); + Map fileExistence = new HashMap<>(); // Process DataSplits for (DataSplit dataSplit : dataSplits) { if (isCountPushdownSplit(countPushdown, dataSplit)) { @@ -999,6 +1010,10 @@ && hasTrustworthyLimitAccounting(table) } continue; } + if (binaryPartitionFields.length > 0) { + checkBinaryPartitionFiles(paimonHandle, table, dataSplit, binaryPartitionFields, + binaryPartitionSourceTable, legacyPartitionNameSchemas, fileExistence); + } Map partitionValues = getPartitionInfoMap( table, dataSplit.partition(), session.getTimeZone()); @@ -1429,7 +1444,7 @@ private T withSchemaHistoryAuthentication(Supplier action) { } catch (RuntimeException e) { throw e; } catch (Exception e) { - throw new RuntimeException("Failed to read Paimon timestamp schema history", e); + throw new RuntimeException("Failed to access Paimon table files", e); } } @@ -2285,6 +2300,11 @@ static FileStoreTable legacyOrcSplitTable(FileStoreTable table, DataSplit split) FileStoreTable base = PaimonTableDecorators.unwrapToFallbackOrBase(table); if (base instanceof FallbackReadFileStoreTable) { FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) base; + if (!(split instanceof FallbackReadFileStoreTable.FallbackSplit)) { + // A split planned through the pair's SnapshotReader (a file-creation-time read) comes from its + // wrapped branch. + return pair.wrapped(); + } boolean fallback = ((FallbackReadFileStoreTable.FallbackSplit) split).isFallback(); // Match the SDK reader's branch selection before reading either options or historical schemas. return PaimonReaderOptions.isWrappedFirst(pair) != fallback ? pair.wrapped() : pair.other(); @@ -2292,6 +2312,109 @@ static FileStoreTable legacyOrcSplitTable(FileStoreTable table, DataSplit split) return base; } + /** Positions in the partition row of the partition columns that hold binary values. */ + private static int[] binaryPartitionFields(Table table) { + RowType partitionType = table.rowType().project(table.partitionKeys()); + return java.util.stream.IntStream.range(0, partitionType.getFieldCount()).filter(field -> { + DataTypeRoot typeRoot = partitionType.getTypeAt(field).getTypeRoot(); + return typeRoot == DataTypeRoot.BINARY || typeRoot == DataTypeRoot.VARBINARY; + }).toArray(); + } + + /** + * Paimon finds the files of a partition under the directory its values name. With {@code partition.legacy-name} + * = true, the default, it names a partition after the string of each value, which for a non-NULL binary value + * is the Java array object ({@code [B@1b6d3586}) rather than its bytes, and every resolution of the name + * differs. A file written under such a name, or looked up under one, is never found: Paimon's own read fails, + * while the BE skips a missing file of an external table and would silently drop the partition's rows. A file + * with an external path is found by the path its metadata records. A system wrapper reports the options of the + * main branch while a fallback split is planned and read under its own branch, so the options and schemas come + * from the branch that produced the split; for the wrappers read through JNI that branch is resolved from the + * pinned source of the wrapper. + * + *

The schema a file refers to does not prove how the file was named, because dynamic options + * ({@code spark.paimon.*}, Flink hints, {@code Table.copy}) keep the schema ID: a file whose schema uses legacy + * names fails only if it is also missing at its planned path. With {@code index-file-in-data-file-dir} = true, a + * deletion vector written without an external data path is looked up under the directory of its partition too, + * whatever the path of its data file, and the BE skips the whole data file when it cannot open the deletion + * vector, so every deletion vector must exist. Each path is checked once per scan. + */ + private void checkBinaryPartitionFiles(PaimonTableHandle handle, Table table, DataSplit split, + int[] binaryPartitionFields, Supplier

schemaTable, + Map> legacyPartitionNameSchemas, Map fileExistence) { + BinaryRow partition = split.partition(); + OptionalInt nonNullField = Arrays.stream(binaryPartitionFields) + .filter(field -> !partition.isNullAt(field)).findFirst(); + if (!nonNullField.isPresent()) { + return; + } + int binaryField = nonNullField.getAsInt(); + FileStoreTable splitTable = schemaTable.get() == null + ? null : legacyOrcSplitTable((FileStoreTable) schemaTable.get(), split); + Table optionsTable = splitTable == null ? table : splitTable; + CoreOptions options = CoreOptions.fromMap(optionsTable.options()); + boolean legacyLookup = options.legacyPartitionName(); + // Deletion vectors under the table's index directory do not depend on partition names. The option is + // immutable, so every writer of the table used this value. + List deletionFiles = options.indexFileInDataFileDir() ? split.deletionFiles().orElse(null) : null; + List files = split.dataFiles(); + for (int i = 0; i < files.size(); i++) { + DataFileMeta file = files.get(i); + if (!file.externalPath().isPresent()) { + if (legacyLookup) { + throw binaryPartitionReadError(handle, "the table uses legacy partition names" + + " ('partition.legacy-name' = 'true'), under which Paimon names partition " + + binaryPartitionName(table, partition, binaryField) + " after a Java array object" + + " rather than its bytes, so the files of the partition are never found"); + } + if (splitTable != null && legacyPartitionNameSchemas + .computeIfAbsent(splitTable, ignored -> new HashMap<>()) + .computeIfAbsent(file.schemaId(), id -> withSchemaHistoryAuthentication(() -> CoreOptions + .fromMap(splitTable.schemaManager().schema(id).options()).legacyPartitionName())) + && !fileExists(optionsTable.fileIO(), split.bucketPath() + "/" + file.fileName(), + fileExistence)) { + throw binaryPartitionReadError(handle, "data file " + file.fileName() + " of partition " + + binaryPartitionName(table, partition, binaryField) + " was written while the table" + + " used legacy partition names ('partition.legacy-name' = 'true'), under which Paimon" + + " names the partition after a Java array object rather than its bytes, so the file is" + + " not under the directory of the partition"); + } + } + DeletionFile deletionFile = deletionFiles == null ? null : deletionFiles.get(i); + if (deletionFile != null && !fileExists(optionsTable.fileIO(), deletionFile.path(), fileExistence)) { + throw binaryPartitionReadError(handle, "deletion vector file " + + new Path(deletionFile.path()).getName() + " of data file " + file.fileName() + + " of partition " + binaryPartitionName(table, partition, binaryField) + + " does not exist: the table keeps index files with its data files" + + " ('index-file-in-data-file-dir' = 'true'), so a deletion vector written without an external" + + " path is looked up under the directory of its partition, and legacy partition names" + + " ('partition.legacy-name' = 'true') name the partition after a Java array object rather" + + " than its bytes"); + } + } + } + + private static String binaryPartitionName(Table table, BinaryRow partition, int binaryField) { + return table.partitionKeys().get(binaryField) + " = 0x" + + java.util.HexFormat.of().withUpperCase().formatHex(partition.getBinary(binaryField)); + } + + private static DorisConnectorException binaryPartitionReadError(PaimonTableHandle handle, String reason) { + return new DorisConnectorException("Cannot read Paimon table " + handle.getDatabaseName() + "." + + handle.getTableName() + ": " + reason + ". Paimon cannot read the partition either; a table with a" + + " binary partition column needs 'partition.legacy-name' = 'false' before its data is written"); + } + + private boolean fileExists(FileIO fileIO, String path, Map fileExistence) { + return fileExistence.computeIfAbsent(path, ignored -> withSchemaHistoryAuthentication(() -> { + try { + return fileIO.exists(new Path(path)); + } catch (IOException e) { + throw new DorisConnectorException("Failed to check whether Paimon file " + path + " exists", e); + } + })); + } + private static boolean requiresCrossFamilyTimestampReader(FileStoreTable table, Optional> rawFiles, Set readFieldIds, Map schemaTimestamps) { RowType readType = new RowType(table.rowType().getFields().stream() diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index d29cae9ba8d1ef..bbb0513a29a376 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -50,9 +50,14 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.deletionvectors.BitmapDeletionVector; +import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataIncrement; import org.apache.paimon.io.DataInputViewStreamWrapper; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; @@ -69,11 +74,13 @@ import org.apache.paimon.table.sink.BatchTableWrite; import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.DeletionFile; import org.apache.paimon.table.source.RawFile; import org.apache.paimon.table.source.Split; import org.apache.paimon.table.system.ReadOptimizedTable; +import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; @@ -892,9 +899,11 @@ public void nativeBinaryPartitionsKeepAllPathValues(@TempDir Path warehouse) thr new org.apache.paimon.fs.Path(warehouse.resolve(format).toUri()))) { catalog.createDatabase("db", false); Identifier id = Identifier.create("db", "binary_parts"); + // Legacy partition names cannot name a binary partition (legacyNamedBinaryPartitionsFailScanPlanning). catalog.createTable(id, Schema.newBuilder().column("id", DataTypes.INT()) .column("key", DataTypes.BYTES()).column("region", DataTypes.STRING()) .partitionKeys("key", "region").option("bucket", "-1") + .option("partition.legacy-name", "false") .option("file.format", format).build(), false); Table table = catalog.getTable(id); BatchWriteBuilder writer = table.newBatchWriteBuilder(); @@ -925,6 +934,323 @@ public void nativeBinaryPartitionsKeepAllPathValues(@TempDir Path warehouse) thr } } + @Test + public void legacyNamedBinaryPartitionsFailScanPlanning(@TempDir Path warehouse) throws Exception { + // With partition.legacy-name = true, the default, Paimon looks a non-NULL binary partition up under the name + // of a Java array object, so the planned files never exist and the BE would skip them as missing files. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Table legacy = binaryPartitionTable(catalog, "legacy", Collections.emptyMap(), + new byte[] {0x41, 0x42}, null); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(legacy, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.legacy: the table uses" + + " legacy partition names ('partition.legacy-name' = 'true'), under which Paimon names partition" + + " key = 0x4142 after a Java array object"), error.getMessage()); + + // COUNT(*) answers from the metadata row counts and reads no file. + List count = planBinaryPartitionScan(legacy, true); + Assertions.assertEquals(1, count.size()); + Assertions.assertEquals(2, ((PaimonScanRange) count.get(0)).getPushDownRowCount()); + + // The NULL partition is named after partition.default-name, which Paimon finds again. + Table nullOnly = binaryPartitionTable(catalog, "null_only", Collections.emptyMap(), (byte[]) null); + Assertions.assertEquals(1, planBinaryPartitionScan(nullOnly, false).size()); + } + } + + @Test + public void binaryPartitionFilesOfAnotherNamingFailScanPlanning(@TempDir Path warehouse) throws Exception { + // Paimon writes a file under the name its partition has when written and looks it up under the name the + // partition has when read, so a file written under a legacy name stays lost after the option changes. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Table legacy = binaryPartitionTable(catalog, "renamed", Collections.emptyMap(), new byte[] {0x41, 0x42}); + String legacyFile = ((DataSplit) legacy.newReadBuilder().newScan().plan().splits().get(0)) + .dataFiles().get(0).fileName(); + catalog.alterTable(Identifier.create("db", "renamed"), Collections.singletonList( + SchemaChange.setOption("partition.legacy-name", "false")), false); + writeBinaryPartitions(catalog.getTable(Identifier.create("db", "renamed")), new byte[] {0x43, 0x44}); + Table current = catalog.getTable(Identifier.create("db", "renamed")); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(current, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.renamed: data file " + + legacyFile + " of partition key = 0x4142 was written while the table used legacy partition" + + " names ('partition.legacy-name' = 'true')"), error.getMessage()); + + // A file written under the bytes of its value is looked up under a legacy name once those are back on. + binaryPartitionTable(catalog, "restored", Collections.singletonMap("partition.legacy-name", "false"), + new byte[] {0x41, 0x42}); + Table byBytes = catalog.getTable(Identifier.create("db", "restored")); + Assertions.assertEquals(1, planBinaryPartitionScan(byBytes, false).size()); + catalog.alterTable(Identifier.create("db", "restored"), Collections.singletonList( + SchemaChange.setOption("partition.legacy-name", "true")), false); + Table restored = catalog.getTable(Identifier.create("db", "restored")); + error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(restored, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.restored: the table" + + " uses legacy partition names"), error.getMessage()); + } + } + + @Test + public void binaryPartitionFilesWrittenWithDynamicNamesStayReadable(@TempDir Path warehouse) throws Exception { + // Dynamic options keep the schema ID, so a file written with a dynamic partition.legacy-name = false refers + // to a schema with legacy names while it sits under the bytes of its value, where a lookup by bytes finds it. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "dynamic_names"); + Table legacy = binaryPartitionTable(catalog, "dynamic_names", Collections.emptyMap()); + writeBinaryPartitions(legacy.copy(Collections.singletonMap("partition.legacy-name", "false")), + new byte[] {0x41, 0x42}); + Table current = catalog.getTable(id); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(current, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.dynamic_names: the" + + " table uses legacy partition names"), error.getMessage()); + + catalog.alterTable(id, Collections.singletonList( + SchemaChange.setOption("partition.legacy-name", "false")), false); + List ranges = planBinaryPartitionScan(catalog.getTable(id), false); + Assertions.assertEquals(1, ranges.size()); + String path = ranges.get(0).getPath().orElseThrow(AssertionError::new); + Assertions.assertTrue(path.contains("/key=AB/"), path); + Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(path)), path); + } + } + + @Test + public void legacyNamedBinaryPartitionFilesAtExternalPathsStayReadable(@TempDir Path warehouse) throws Exception { + // A file at an external path is read from the path its metadata records, whatever its partition is named. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.resolve("warehouse").toUri()))) { + catalog.createDatabase("db", false); + String externalRoot = warehouse.resolve("external").toUri().toString(); + Map options = new HashMap<>(); + options.put("data-file.external-paths", externalRoot); + options.put("data-file.external-paths.strategy", "round-robin"); + Table legacy = binaryPartitionTable(catalog, "external_files", options, new byte[] {0x41, 0x42}); + List ranges = planBinaryPartitionScan(legacy, false); + Assertions.assertEquals(1, ranges.size()); + String path = ranges.get(0).getPath().orElseThrow(AssertionError::new); + Assertions.assertTrue(path.contains(warehouse.resolve("external").toString()), path); + Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(path)), path); + } + } + + @Test + public void externalBinaryPartitionFileWithLocalDeletionVectorFailsScanPlanning(@TempDir Path warehouse) + throws Exception { + // With index-file-in-data-file-dir, a deletion vector written while no external data path is configured + // records only its name and is looked up under the directory of its partition, which a legacy name never + // finds, while its data file keeps its external path. The BE skips the whole data file when it cannot open + // the deletion vector, rows that were not deleted included. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.resolve("warehouse").toUri()))) { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "local_dv"); + deletionVectorBinaryPartitionTable(catalog, "local_dv", warehouse); + catalog.alterTable(id, Arrays.asList(SchemaChange.removeOption("data-file.external-paths"), + SchemaChange.removeOption("data-file.external-paths.strategy")), false); + Table table = catalog.getTable(id); + DataFileMeta file = ((DataSplit) table.newReadBuilder().newScan().plan().splits().get(0)) + .dataFiles().get(0); + Assertions.assertTrue(file.externalPath().isPresent()); + String deletionVector = deleteFirstRow(table); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> planBinaryPartitionScan(table, false)); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.local_dv: deletion vector" + + " file " + deletionVector + " of data file " + file.fileName() + " of partition key = 0x4142" + + " does not exist"), error.getMessage()); + } + } + + @Test + public void externalBinaryPartitionDeletionVectorAtExternalPathStaysReadable(@TempDir Path warehouse) + throws Exception { + // A deletion vector written while an external data path is configured records its absolute path. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.resolve("warehouse").toUri()))) { + catalog.createDatabase("db", false); + Table table = deletionVectorBinaryPartitionTable(catalog, "external_dv", warehouse); + deleteFirstRow(table); + List ranges = planBinaryPartitionScan(table, false); + Assertions.assertEquals(1, ranges.size()); + String deletionVector = ranges.get(0).getProperties().get("paimon.deletion_file.path"); + Assertions.assertTrue(deletionVector.contains(warehouse.resolve("external").toString()), deletionVector); + Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(deletionVector)), + deletionVector); + } + } + + @Test + public void fallbackBinaryPartitionsUseTheNamingOfTheirBranch(@TempDir Path warehouse) throws Exception { + // A partition missing on the main branch is planned and read under the options of the fallback branch, + // while the table and its $audit_log wrapper report the options of the main branch. + for (boolean fallbackUsesBytes : new boolean[] {true, false}) { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.resolve("fallback_bytes_" + fallbackUsesBytes).toUri()))) { + catalog.createDatabase("db", false); + Map byBytes = Collections.singletonMap("partition.legacy-name", "false"); + // The main branch only holds the NULL partition, which every naming finds. + FileStoreTable main = (FileStoreTable) binaryPartitionTable(catalog, "main", + fallbackUsesBytes ? Collections.emptyMap() : byBytes, (byte[]) null); + FileStoreTable fallback = (FileStoreTable) binaryPartitionTable(catalog, "fallback", + fallbackUsesBytes ? byBytes : Collections.emptyMap(), new byte[] {0x41, 0x42}); + FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); + if (!fallbackUsesBytes) { + Map>> plans = new LinkedHashMap<>(); + plans.put("table", () -> planBinaryPartitionScan(pair, false)); + plans.put("$audit_log", () -> planSystemTableScan("audit_log", pair)); + plans.forEach((name, plan) -> { + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + plan::get, name); + Assertions.assertTrue(error.getMessage().startsWith("Cannot read Paimon table db.main: the" + + " table uses legacy partition names ('partition.legacy-name' = 'true'), under which" + + " Paimon names partition key = 0x4142"), error.getMessage()); + }); + continue; + } + List ranges = planBinaryPartitionScan(pair, false); + Assertions.assertEquals(2, ranges.size()); + for (ConnectorScanRange range : ranges) { + String path = range.getPath().orElseThrow(AssertionError::new); + Assertions.assertTrue(LocalFileIO.create().exists(new org.apache.paimon.fs.Path(path)), path); + } + Assertions.assertTrue(ranges.stream().anyMatch(range -> range.getPath().get().contains("/key=AB/"))); + + List auditLog = planSystemTableScan("audit_log", pair); + Assertions.assertEquals(2, auditLog.size()); + List rows = new ArrayList<>(); + SystemTableLoader.load("audit_log", pair).newReadBuilder().newRead() + .createReader(deserializeJniSplits(auditLog)) + .forEachRemaining(row -> rows.add(row.getString(0) + " " + (row.isNullAt(2) ? "NULL" + : java.util.HexFormat.of().withUpperCase().formatHex(row.getBinary(2))))); + rows.sort(Comparator.naturalOrder()); + Assertions.assertEquals(Arrays.asList("+I 4142", "+I NULL"), rows); + } + } + } + + @Test + public void fileCreationTimeScanOfFallbackTableChecksBinaryPartitionsOfTheMainBranch(@TempDir Path warehouse) + throws Exception { + // A scan.file-creation-time-millis read plans a fallback table through its SnapshotReader, which reads the + // main branch and returns plain DataSplits rather than the FallbackSplits of the fallback scan. + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + catalog.createDatabase("db", false); + Map byBytes = Collections.singletonMap("partition.legacy-name", "false"); + FileStoreTable main = (FileStoreTable) binaryPartitionTable(catalog, "main", byBytes, + new byte[] {0x41, 0x42}); + FileStoreTable fallback = (FileStoreTable) binaryPartitionTable(catalog, "fallback", byBytes, + new byte[] {0x43, 0x44}); + FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = pair; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = new PaimonTableHandle("db", "main", Collections.emptyList(), + Collections.emptyList()).withScanOptions(PaimonScanParams.markAsOptions( + PaimonScanParams.resolveOptions(pair, Collections.singletonMap( + CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key(), "0")))); + List ranges = provider.planScan( + sessionWithProps(Collections.emptyMap(), new TestStatementScope()), + ConnectorScanRequest.builder(handle, Arrays.asList(new PaimonColumnHandle("id", 0), + new PaimonColumnHandle("key", 1))).build()); + Assertions.assertEquals(1, ranges.size()); + String path = ranges.get(0).getPath().orElseThrow(AssertionError::new); + Assertions.assertTrue(path.contains("/key=AB/"), path); + + DataSplit split = (DataSplit) pair.newSnapshotReader().read().splits().get(0); + Assertions.assertFalse(split instanceof FallbackReadFileStoreTable.FallbackSplit); + Assertions.assertSame(main, PaimonScanPlanProvider.legacyOrcSplitTable(pair, split)); + } + } + + private static Table binaryPartitionTable(Catalog catalog, String name, Map options, + byte[]... keys) throws Exception { + Identifier id = Identifier.create("db", name); + Schema.Builder schema = Schema.newBuilder().column("id", DataTypes.INT()).column("key", DataTypes.BYTES()) + .partitionKeys("key").option("bucket", "-1"); + options.forEach(schema::option); + catalog.createTable(id, schema.build(), false); + writeBinaryPartitions(catalog.getTable(id), keys); + return catalog.getTable(id); + } + + private static void writeBinaryPartitions(Table table, byte[]... keys) throws Exception { + BatchWriteBuilder writer = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writer.newWrite()) { + for (int i = 0; i < keys.length; i++) { + write.write(GenericRow.of(i, keys[i])); + } + try (BatchTableCommit commit = writer.newCommit()) { + commit.commit(write.prepareCommit()); + } + } + } + + private static List planBinaryPartitionScan(Table table, boolean countPushdown) { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = new PaimonTableHandle("db", table.name(), + Collections.emptyList(), Collections.emptyList()); + return provider.planScan(sessionWithProps(Collections.emptyMap(), new TestStatementScope()), + ConnectorScanRequest.builder(handle, Arrays.asList(new PaimonColumnHandle("id", 0), + new PaimonColumnHandle("key", 1))).countPushdown(countPushdown).build()); + } + + /** Plans a scan of {@code source$kind} the way the connector metadata builds its system table handle. */ + private static List planSystemTableScan(String kind, FileStoreTable source) { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = source; + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(Collections.emptyMap()), ops); + PaimonTableHandle handle = PaimonTableHandle.forSystemTable("db", source.name(), kind, + PaimonScanParams.requiresPaimonReader(kind)); + handle.setPaimonTable(SystemTableLoader.load(kind, source)); + handle.setSysBaseTable(source); + handle.setSystemTableSource(source); + return provider.planScan(sessionWithProps(Collections.emptyMap(), new TestStatementScope()), + ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); + } + + private static Table deletionVectorBinaryPartitionTable(Catalog catalog, String name, Path warehouse) + throws Exception { + Map options = new HashMap<>(); + options.put("deletion-vectors.enabled", "true"); + options.put("index-file-in-data-file-dir", "true"); + options.put("data-file.external-paths", warehouse.resolve("external").toUri().toString()); + options.put("data-file.external-paths.strategy", "round-robin"); + return binaryPartitionTable(catalog, name, options, new byte[] {0x41, 0x42}, new byte[] {0x41, 0x42}); + } + + /** Deletes the first row of the first data file the way a Spark DELETE persists its deletion vectors. */ + private static String deleteFirstRow(Table table) throws Exception { + DataSplit split = (DataSplit) table.newReadBuilder().newScan().plan().splits().get(0); + // PaimonSparkWriter.persistDeletionVectors writes the index files under the table's current options. + BaseAppendDeleteFileMaintainer maintainer = BaseAppendDeleteFileMaintainer.forUnawareAppend( + ((FileStoreTable) table).store().newIndexFileHandler(), table.latestSnapshot().get(), + split.partition()); + BitmapDeletionVector deletionVector = new BitmapDeletionVector(); + deletionVector.delete(0); + maintainer.notifyNewDeletionVector(split.dataFiles().get(0).fileName(), deletionVector); + List indexFiles = new ArrayList<>(); + maintainer.persist().forEach(entry -> indexFiles.add(entry.indexFile())); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.commit(Collections.singletonList(new CommitMessageImpl(maintainer.getPartition(), + maintainer.getBucket(), null, DataIncrement.indexIncrement(indexFiles), + CompactIncrement.emptyIncrement()))); + } + return indexFiles.get(0).fileName(); + } + @Test public void planScanPushesLimitIntoPaimonSplitPlanning(@TempDir Path warehouse) throws Exception { try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(),