Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataOutputViewStreamWrapper;
import org.apache.paimon.options.Options;
Expand Down Expand Up @@ -86,6 +87,7 @@
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.InstantiationUtil;
Expand Down Expand Up @@ -990,6 +992,15 @@ && hasTrustworthyLimitAccounting(table)
// $ro wraps the pinned file-store table; resolve its schema dictionary once, only if native is considered.
java.util.function.Supplier<Table> legacyOrcSchemaTable = com.google.common.base.Suppliers.memoize(
() -> resolveSchemaDictTable(table, paimonHandle));
// The wrappers read through JNI report the options of the main branch, but their splits come from a branch
// of the pinned source, which the timestamp-history checks resolve the same way.
java.util.function.Supplier<Table> binaryPartitionSourceTable = paimonHandle.isForceJni()
? com.google.common.base.Suppliers.memoize(() -> withSchemaHistoryAuthentication(
() -> resolveTimestampHistoryTable(table, paimonHandle)))
: legacyOrcSchemaTable;
int[] binaryPartitionFields = binaryPartitionFields(table);
Map<FileStoreTable, Map<Long, Boolean>> legacyPartitionNameSchemas = new java.util.IdentityHashMap<>();
Map<String, Boolean> fileExistence = new HashMap<>();
// Process DataSplits
for (DataSplit dataSplit : dataSplits) {
if (isCountPushdownSplit(countPushdown, dataSplit)) {
Expand All @@ -999,6 +1010,10 @@ && hasTrustworthyLimitAccounting(table)
}
continue;
}
if (binaryPartitionFields.length > 0) {
checkBinaryPartitionFiles(paimonHandle, table, dataSplit, binaryPartitionFields,
binaryPartitionSourceTable, legacyPartitionNameSchemas, fileExistence);
}

Map<String, String> partitionValues = getPartitionInfoMap(
table, dataSplit.partition(), session.getTimeZone());
Expand Down Expand Up @@ -1429,7 +1444,7 @@ private <T> T withSchemaHistoryAuthentication(Supplier<T> action) {
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException("Failed to read Paimon timestamp schema history", e);
throw new RuntimeException("Failed to access Paimon table files", e);
}
}

Expand Down Expand Up @@ -2285,13 +2300,121 @@ static FileStoreTable legacyOrcSplitTable(FileStoreTable table, DataSplit split)
FileStoreTable base = PaimonTableDecorators.unwrapToFallbackOrBase(table);
if (base instanceof FallbackReadFileStoreTable) {
FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) base;
if (!(split instanceof FallbackReadFileStoreTable.FallbackSplit)) {
// A split planned through the pair's SnapshotReader (a file-creation-time read) comes from its
// wrapped branch.
return pair.wrapped();
}
boolean fallback = ((FallbackReadFileStoreTable.FallbackSplit) split).isFallback();
// Match the SDK reader's branch selection before reading either options or historical schemas.
return PaimonReaderOptions.isWrappedFirst(pair) != fallback ? pair.wrapped() : pair.other();
}
return base;
}

/** Positions in the partition row of the partition columns that hold binary values. */
private static int[] binaryPartitionFields(Table table) {
RowType partitionType = table.rowType().project(table.partitionKeys());
return java.util.stream.IntStream.range(0, partitionType.getFieldCount()).filter(field -> {
DataTypeRoot typeRoot = partitionType.getTypeAt(field).getTypeRoot();
return typeRoot == DataTypeRoot.BINARY || typeRoot == DataTypeRoot.VARBINARY;
}).toArray();
}

/**
* Paimon finds the files of a partition under the directory its values name. With {@code partition.legacy-name}
* = true, the default, it names a partition after the string of each value, which for a non-NULL binary value
* is the Java array object ({@code [B@1b6d3586}) rather than its bytes, and every resolution of the name
* differs. A file written under such a name, or looked up under one, is never found: Paimon's own read fails,
* while the BE skips a missing file of an external table and would silently drop the partition's rows. A file
* with an external path is found by the path its metadata records. A system wrapper reports the options of the
* main branch while a fallback split is planned and read under its own branch, so the options and schemas come
* from the branch that produced the split; for the wrappers read through JNI that branch is resolved from the
* pinned source of the wrapper.
*
* <p>The schema a file refers to does not prove how the file was named, because dynamic options
* ({@code spark.paimon.*}, Flink hints, {@code Table.copy}) keep the schema ID: a file whose schema uses legacy
* names fails only if it is also missing at its planned path. With {@code index-file-in-data-file-dir} = true, a
* deletion vector written without an external data path is looked up under the directory of its partition too,
* whatever the path of its data file, and the BE skips the whole data file when it cannot open the deletion
* vector, so every deletion vector must exist. Each path is checked once per scan.
*/
private void checkBinaryPartitionFiles(PaimonTableHandle handle, Table table, DataSplit split,
int[] binaryPartitionFields, Supplier<Table> schemaTable,
Map<FileStoreTable, Map<Long, Boolean>> legacyPartitionNameSchemas, Map<String, Boolean> fileExistence) {
BinaryRow partition = split.partition();
OptionalInt nonNullField = Arrays.stream(binaryPartitionFields)
.filter(field -> !partition.isNullAt(field)).findFirst();
if (!nonNullField.isPresent()) {
return;
}
int binaryField = nonNullField.getAsInt();
FileStoreTable splitTable = schemaTable.get() == null
? null : legacyOrcSplitTable((FileStoreTable) schemaTable.get(), split);
Table optionsTable = splitTable == null ? table : splitTable;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Resolve fallback options for audit-log binary partitions

For $audit_log, resolveSchemaDictTable returns null, so this selects the system wrapper's options. Paimon's audit wrapper exposes the main branch's options while still planning and reading the fallback branch. For example, write only a NULL binary partition on main with the default partition.legacy-name=true; create a backup branch, set its persisted naming option to false, and write a non-NULL binary partition there. After setting main's scan.fallback-branch=backup, SELECT id FROM t$audit_log now throws the legacy-name error for the backup split, even though its byte-named file exists and the JNI reader selects backup correctly. Resolve the pinned system source and the split's producing branch independently of native-schema-dictionary availability, and add this wrapper/fallback case to the tests. This uses ordinary persisted branch options, with no dynamic writer override or schema-ID provenance ambiguity.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed, and fixed in cf79a1ac228. resolveSchemaDictTable has no answer for $audit_log, since an AuditLogTable is a DataTable rather than a FileStoreTable. The check then fell back to the wrapper's own options, and AuditLogTable.options() returns those of the fallback pair, which are the main branch's. The splits still come from FallbackReadScan, which plans a partition that is missing on main from the fallback branch and reads it with that branch's read. I replayed your history with the Paimon 2.1.0 API. Main keeps the default partition.legacy-name = true and holds only a NULL partition, and backup has false and a row in key = 0x4142. $audit_log reports true, while the fallback split's file is at key=AB/bucket-0/…, exists, and Paimon's own read returns the row. The check rejected that scan although master reads it through JNI.

For the system tables read through JNI, the check now resolves the table that produced the splits from the wrapper's pinned source, the same source the timestamp-history checks already use (resolveTimestampHistoryTable). It then picks each split's branch as it does for data tables, so a fallback split of $audit_log or $row_tracking is judged by the naming and schemas of its own branch. In the reverse history (main false, backup true) the scan now fails at planning with the legacy-name error. Before, it planned and Paimon's reader failed at execution with a FileNotFoundException. $binlog is not affected: it exposes its partition columns as arrays, so it has no binary partition column to check. fallbackBinaryPartitionsUseTheNamingOfTheirBranch covers both directions for the data table and $audit_log, and reads the planned $audit_log splits back through the wrapper.

While there: a scan.file-creation-time-millis read plans a fallback table through the pair's SnapshotReader, which returns plain DataSplits of the wrapped branch. legacyOrcSplitTable cast every split to FallbackSplit, so such a scan of a table with a non-NULL binary partition failed with a ClassCastException. A split that is not a FallbackSplit now maps to the wrapped branch. The same cast on the native timestamp path predates this PR and is fixed with it. fileCreationTimeScanOfFallbackTableChecksBinaryPartitionsOfTheMainBranch plans such a read.

CoreOptions options = CoreOptions.fromMap(optionsTable.options());
boolean legacyLookup = options.legacyPartitionName();
// Deletion vectors under the table's index directory do not depend on partition names. The option is
// immutable, so every writer of the table used this value.
List<DeletionFile> deletionFiles = options.indexFileInDataFileDir() ? split.deletionFiles().orElse(null) : null;
List<DataFileMeta> files = split.dataFiles();
for (int i = 0; i < files.size(); i++) {
DataFileMeta file = files.get(i);
if (!file.externalPath().isPresent()) {
if (legacyLookup) {
throw binaryPartitionReadError(handle, "the table uses legacy partition names"
+ " ('partition.legacy-name' = 'true'), under which Paimon names partition "
+ binaryPartitionName(table, partition, binaryField) + " after a Java array object"
+ " rather than its bytes, so the files of the partition are never found");
}
if (splitTable != null && legacyPartitionNameSchemas
.computeIfAbsent(splitTable, ignored -> new HashMap<>())
.computeIfAbsent(file.schemaId(), id -> withSchemaHistoryAuthentication(() -> CoreOptions
.fromMap(splitTable.schemaManager().schema(id).options()).legacyPartitionName()))
&& !fileExists(optionsTable.fileIO(), split.bucketPath() + "/" + file.fileName(),
fileExistence)) {
throw binaryPartitionReadError(handle, "data file " + file.fileName() + " of partition "
+ binaryPartitionName(table, partition, binaryField) + " was written while the table"
+ " used legacy partition names ('partition.legacy-name' = 'true'), under which Paimon"
+ " names the partition after a Java array object rather than its bytes, so the file is"
+ " not under the directory of the partition");
}
}
DeletionFile deletionFile = deletionFiles == null ? null : deletionFiles.get(i);
if (deletionFile != null && !fileExists(optionsTable.fileIO(), deletionFile.path(), fileExistence)) {
throw binaryPartitionReadError(handle, "deletion vector file "
+ new Path(deletionFile.path()).getName() + " of data file " + file.fileName()
+ " of partition " + binaryPartitionName(table, partition, binaryField)
+ " does not exist: the table keeps index files with its data files"
+ " ('index-file-in-data-file-dir' = 'true'), so a deletion vector written without an external"
+ " path is looked up under the directory of its partition, and legacy partition names"
+ " ('partition.legacy-name' = 'true') name the partition after a Java array object rather"
+ " than its bytes");
}
}
}

private static String binaryPartitionName(Table table, BinaryRow partition, int binaryField) {
return table.partitionKeys().get(binaryField) + " = 0x"
+ java.util.HexFormat.of().withUpperCase().formatHex(partition.getBinary(binaryField));
}

private static DorisConnectorException binaryPartitionReadError(PaimonTableHandle handle, String reason) {
return new DorisConnectorException("Cannot read Paimon table " + handle.getDatabaseName() + "."
+ handle.getTableName() + ": " + reason + ". Paimon cannot read the partition either; a table with a"
+ " binary partition column needs 'partition.legacy-name' = 'false' before its data is written");
}

private boolean fileExists(FileIO fileIO, String path, Map<String, Boolean> fileExistence) {
return fileExistence.computeIfAbsent(path, ignored -> withSchemaHistoryAuthentication(() -> {
try {
return fileIO.exists(new Path(path));
} catch (IOException e) {
throw new DorisConnectorException("Failed to check whether Paimon file " + path + " exists", e);
}
}));
}

private static boolean requiresCrossFamilyTimestampReader(FileStoreTable table, Optional<List<RawFile>> rawFiles,
Set<Integer> readFieldIds, Map<Long, Boolean> schemaTimestamps) {
RowType readType = new RowType(table.rowType().getFields().stream()
Expand Down
Loading
Loading