Repository navigation
[fix](paimon) Pad CHAR values written to Paimon tables as Spark and Flink do #69083
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
f6b658b
b584341
45e6e5d
f2c45d7
1740cfa
3592718
c4acf1b
41d351b
1012317
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,7 +21,10 @@ | |
| import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; | ||
|
|
||
| import org.apache.paimon.CoreOptions; | ||
| import org.apache.paimon.schema.TableSchema; | ||
| import org.apache.paimon.table.FileStoreTable; | ||
| import org.apache.paimon.types.CharType; | ||
| import org.apache.paimon.types.DataType; | ||
| import org.apache.paimon.types.DataTypeRoot; | ||
| import org.apache.paimon.utils.InstantiationUtil; | ||
|
|
||
|
|
@@ -32,14 +35,24 @@ | |
| import java.time.format.DateTimeFormatter; | ||
| import java.util.Base64; | ||
| import java.util.Collections; | ||
| import java.util.HashMap; | ||
| import java.util.LinkedHashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.TreeMap; | ||
| import java.util.function.Predicate; | ||
|
|
||
| /** Statement-scoped Paimon write target shared by sink planning and transaction commit. */ | ||
| final class PaimonWriteBinding { | ||
|
|
||
| /** | ||
| * Option on the table that the BE writer deserializes: whether it pads the CHAR values of the rows it inserts | ||
| * (session variable enable_paimon_write_char_padding). Keep in sync with PaimonWriteSchema | ||
| * .DORIS_WRITE_CHAR_PADDING in be-java-extensions/paimon-scanner. | ||
| */ | ||
| static final String DORIS_WRITE_CHAR_PADDING = "doris.write.char-padding"; | ||
|
|
||
| private final String tableName; | ||
| private final FileStoreTable table; | ||
| private final String serializedTable; | ||
|
|
@@ -49,27 +62,42 @@ final class PaimonWriteBinding { | |
|
|
||
| private PaimonWriteBinding(String tableName, FileStoreTable table, | ||
| Map<String, String> hadoopConfig, boolean overwrite, | ||
| Map<String, String> staticPartition) { | ||
| Map<String, String> staticPartition, boolean charPadding) { | ||
| this.tableName = tableName; | ||
| this.table = table; | ||
| // The FE committer keeps the catalog-aware table, but the BE writer must not deserialize | ||
| // an HMS/DLF catalog loader whose metastore classes are absent from the BE plugin. | ||
| this.serializedTable = serialize(PaimonScanPlanProvider.dropCatalogLoader(table)); | ||
| this.serializedTable = serialize(withCharPadding(PaimonScanPlanProvider.dropCatalogLoader(table), | ||
| charPadding)); | ||
| this.hadoopConfig = Collections.unmodifiableMap(new LinkedHashMap<>(hadoopConfig)); | ||
| this.overwrite = overwrite; | ||
| this.staticPartition = Collections.unmodifiableMap(new LinkedHashMap<>(staticPartition)); | ||
| } | ||
|
|
||
| /** | ||
| * @param charPadding whether the BE writer pads the CHAR values of the rows it inserts, and the static | ||
| * partition values are padded to match them | ||
| */ | ||
| static PaimonWriteBinding create(PaimonTableHandle handle, FileStoreTable table, | ||
| Map<String, String> hadoopConfig, ConnectorWriteHandle writeHandle) { | ||
| Map<String, String> hadoopConfig, ConnectorWriteHandle writeHandle, boolean charPadding) { | ||
| boolean overwrite = writeHandle.isOverwrite(); | ||
| // Only a static overwrite reads the partition values. An INSERT writes each row to the partition its own | ||
| // values name, so it leaves them alone: casting them could only fail a write the rows would accept. | ||
| Map<String, String> staticPartition = overwrite | ||
| ? resolveStaticPartition(table, writeHandle) : Collections.emptyMap(); | ||
| ? resolveStaticPartition(table, writeHandle, charPadding) : Collections.emptyMap(); | ||
| FileStoreTable writeTable = configureTableForWrite(table, overwrite, staticPartition); | ||
| return new PaimonWriteBinding(handle.getDatabaseName() + "." + handle.getTableName(), | ||
| writeTable, hadoopConfig, overwrite, staticPartition); | ||
| writeTable, hadoopConfig, overwrite, staticPartition, charPadding); | ||
| } | ||
|
|
||
| /** | ||
| * The table the BE writer gets: the FE table with {@link #DORIS_WRITE_CHAR_PADDING} among its options. Only | ||
| * the schema is replaced, so nothing else of the table changes, and the FE committer's table goes without it. | ||
| */ | ||
| private static FileStoreTable withCharPadding(FileStoreTable table, boolean charPadding) { | ||
| Map<String, String> options = new HashMap<>(table.schema().options()); | ||
| options.put(DORIS_WRITE_CHAR_PADDING, String.valueOf(charPadding)); | ||
| return table.copy(table.schema().copy(options)); | ||
| } | ||
|
|
||
| static FileStoreTable configureTableForWrite(FileStoreTable table, boolean overwrite, | ||
|
|
@@ -89,9 +117,11 @@ static FileStoreTable configureTableForWrite(FileStoreTable table, boolean overw | |
| /** | ||
| * The static partition of an overwrite, as Paimon's static overwrite parses it: SQL NULL becomes the table's | ||
| * {@code partition.default-name}, and every other value is the one the written rows carry, cast to the | ||
| * column type. A binary column can only be statically overwritten with NULL. | ||
| * column type, and padded as the BE writer pads the CHAR values of the rows ({@link #charPaddedFields}): Paimon | ||
| * parses a partition value without padding it. A binary column can only be statically overwritten with NULL. | ||
| */ | ||
| static Map<String, String> resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle) { | ||
| static Map<String, String> resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle, | ||
| boolean charPadding) { | ||
| Map<String, String> canonicalNames = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); | ||
| for (String partitionKey : table.partitionKeys()) { | ||
| canonicalNames.put(partitionKey, partitionKey); | ||
|
|
@@ -114,7 +144,9 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto | |
| } | ||
| requestedColumns.put(key, canonicalName); | ||
| } | ||
| String defaultPartitionName = CoreOptions.fromMap(table.options()).partitionDefaultName(); | ||
| CoreOptions options = CoreOptions.fromMap(table.options()); | ||
| String defaultPartitionName = options.partitionDefaultName(); | ||
| Predicate<String> charPaddedFields = charPaddedFields(table.schema(), charPadding); | ||
| Map<String, String> castValues = writeHandle.getCastStaticPartitionSpec(); | ||
| Map<String, String> result = new LinkedHashMap<>(); | ||
| for (Map.Entry<String, String> requested : requestedColumns.entrySet()) { | ||
|
|
@@ -126,10 +158,13 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto | |
| } | ||
| String value = Objects.requireNonNull(castValues.get(key), | ||
| () -> "missing the cast value of static partition column " + key); | ||
| if (table.rowType().getField(canonicalName).type().getTypeRoot() | ||
| == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { | ||
| DataType type = table.rowType().getField(canonicalName).type(); | ||
| if (type.getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { | ||
| value = toSdkLocalTime(canonicalName, value); | ||
| } | ||
| if (type.getTypeRoot() == DataTypeRoot.CHAR && charPaddedFields.test(canonicalName)) { | ||
| value = padChar(value, ((CharType) type).getLength()); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] Keep +U overwrite rows aligned with the partition filter For a table
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in Tests: |
||
| } | ||
| if (defaultPartitionName.equals(value)) { | ||
| // Paimon's static overwrite reads this string as the NULL partition of any partition type, so | ||
| // a value equal to it would overwrite the NULL partition instead. | ||
|
|
@@ -142,6 +177,25 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto | |
| return result; | ||
| } | ||
|
|
||
| /** | ||
| * The fields whose CHAR values the BE writer pads, as PaimonWriteSchema.create decides it: with {@code charPadding} | ||
| * on, every field but the table's {@code rowkind.field}, from whose value Paimon decodes the row kind, and of a | ||
| * table that writes ORC files only the partition fields. ORC pads every CHAR value it writes and strips the | ||
| * trailing spaces from one it reads, so the writer leaves the other fields of such a table alone. Keep in sync | ||
| * with PaimonWriteSchema.create and PaimonWriteSchema.writesOrcFiles in be-java-extensions/paimon-scanner. | ||
| */ | ||
| static Predicate<String> charPaddedFields(TableSchema schema, boolean charPadding) { | ||
| if (!charPadding) { | ||
| return field -> false; | ||
| } | ||
| CoreOptions options = CoreOptions.fromMap(schema.options()); | ||
| String rowKindField = options.rowkindField().orElse(null); | ||
| boolean orcFiles = CoreOptions.FILE_FORMAT_ORC.equals(options.formatType()) | ||
| || options.fileFormatPerLevel().containsValue(CoreOptions.FILE_FORMAT_ORC); | ||
| List<String> partitionKeys = schema.partitionKeys(); | ||
| return field -> !field.equals(rowKindField) && (!orcFiles || partitionKeys.contains(field)); | ||
| } | ||
|
|
||
| /** | ||
| * Paimon parses a static overwrite value of a TIMESTAMP WITH LOCAL TIME ZONE column as local time in the FE | ||
| * JVM's default zone. Doris binds such a column as TIMESTAMPTZ, so the cast value is the instant in UTC with | ||
|
|
@@ -161,6 +215,15 @@ private static String toSdkLocalTime(String column, String value) { | |
| return sdkLocal.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME).replace('T', ' '); | ||
| } | ||
|
|
||
| /** | ||
| * Pads a CHAR value with spaces to the column length in code points, as PaimonWriteSchema pads the rows' CHAR | ||
| * values; a longer value is kept, as it is there. | ||
| */ | ||
| private static String padChar(String value, int length) { | ||
| int padLength = length - value.codePointCount(0, value.length()); | ||
| return padLength > 0 ? value + " ".repeat(padLength) : value; | ||
| } | ||
|
|
||
| private static String serialize(FileStoreTable table) { | ||
| try { | ||
| return Base64.getEncoder().encodeToString(InstantiationUtil.serializeObject(table)); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P2] Mirror the rowkind.field exemption in static partition padding
The writer now leaves the column named by
rowkind.fieldunpadded, but this branch still pads it when it is a CHAR partition column. For a valid table(id INT, rk CHAR(3))with primary key(id,rk), partition keyrk,bucket=1androwkind.field=rk,INSERT OVERWRITE TABLE t PARTITION(rk='+I') SELECT 1puts'+I 'in the FE overwrite filter while the BE writes partition'+I'. Paimon rejects that file as outside the specified partition; a SELECT that returns no rows at runtime instead succeeds without clearing the existing'+I'partition. Apply the writer's control-field exemption here too, and cover nonempty and empty static overwrites. This is distinct from the earlier token-padding finding: its BE fix is present, but the FE overwrite filter now disagrees with it.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed in 45e6e5d. This reproduced too: in a table
(id INT, rk CHAR(3))keyed by(id, rk), partitioned byrk, with'rowkind.field' = 'rk'and two'+I'rows,INSERT OVERWRITE TABLE t PARTITION (rk = '+I') SELECT 3failed withTrying to overwrite partition {rk=+I }, but the changes in rk=+I/ does not belong to this partition, and an overwrite whose source returned no rows at run time returned OK and left both rows.resolveStaticPartitionnow leaves therowkind.fieldvalue unpadded, as the writer does (both compare the name case-sensitively), so the first overwrite replaces the partition with the row of id 3 and the empty one clears it.A source the planner already proves empty, such as
SELECT 3 LIMIT 0, behaves differently on this table, and not because of padding: with padding on the table is written by one writer, the sink then sits directly on the empty set, and fe-core skips the write altogether, as it does for any connector table with such a plan (a dynamic-bucket primary-key table, for one). The regression therefore uses a source that is empty at run time.Tests:
PaimonWriteBindingTest.staticRowKindFieldPartitionIsLeftUnpaddedAsTheRowsAre(therowkind.fieldpartition value stays'+I'while another CHAR partition value is padded), andtest_paimon_write_char_padding, which overwrites the'+I'partition of such a table with one row and then with a source that is empty at run time.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
A correction to the second paragraph of my reply above. With
1012317776athis table is no longer written by one writer: its CHAR partition column is therowkind.field, which the writer never pads, so it is routed natively. And since #69010, fe-core runs the transaction of an externalINSERT OVERWRITEeven when planning proves its source empty. On a cluster with both,INSERT OVERWRITE TABLE t PARTITION (rk = '+I') SELECT 3 LIMIT 0now commits an OVERWRITE snapshot and clears the partition, as a source that is empty at run time does; the regression still uses the latter.