Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,10 @@ public void open(String serializedTable, Map<String, String> hadoopConfig,
this.bucketMode = table.bucketMode();

CoreOptions coreOptions = CoreOptions.fromMap(table.options());
this.writeSchema = PaimonWriteSchema.create(
table.rowType(), columnNames, changelogWrite);
this.writeSchema = PaimonWriteSchema.create(table.rowType(), columnNames, changelogWrite,
PaimonWriteSchema.charPadding(table.options()), coreOptions.rowkindField().orElse(null),
table.primaryKeys(), table.partitionKeys(),
PaimonWriteSchema.writesOrcFiles(table.options()), overwrite);
this.arrowAdapter = new PaimonArrowBatchAdapter(
writeSchema.inputType(), ZoneId.of(timeZone), allocator);
validateWriteColumnsForMergeEngine(
Expand Down

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@
import org.apache.doris.connector.spi.handle.ConnectorWriteHandle;

import org.apache.paimon.CoreOptions;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.types.CharType;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.utils.InstantiationUtil;

Expand All @@ -32,14 +35,24 @@
import java.time.format.DateTimeFormatter;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.function.Predicate;

/** Statement-scoped Paimon write target shared by sink planning and transaction commit. */
final class PaimonWriteBinding {

/**
* Option on the table that the BE writer deserializes: whether it pads the CHAR values of the rows it inserts
* (session variable enable_paimon_write_char_padding). Keep in sync with PaimonWriteSchema
* .DORIS_WRITE_CHAR_PADDING in be-java-extensions/paimon-scanner.
*/
static final String DORIS_WRITE_CHAR_PADDING = "doris.write.char-padding";

private final String tableName;
private final FileStoreTable table;
private final String serializedTable;
Expand All @@ -49,27 +62,42 @@ final class PaimonWriteBinding {

private PaimonWriteBinding(String tableName, FileStoreTable table,
Map<String, String> hadoopConfig, boolean overwrite,
Map<String, String> staticPartition) {
Map<String, String> staticPartition, boolean charPadding) {
this.tableName = tableName;
this.table = table;
// The FE committer keeps the catalog-aware table, but the BE writer must not deserialize
// an HMS/DLF catalog loader whose metastore classes are absent from the BE plugin.
this.serializedTable = serialize(PaimonScanPlanProvider.dropCatalogLoader(table));
this.serializedTable = serialize(withCharPadding(PaimonScanPlanProvider.dropCatalogLoader(table),
charPadding));
this.hadoopConfig = Collections.unmodifiableMap(new LinkedHashMap<>(hadoopConfig));
this.overwrite = overwrite;
this.staticPartition = Collections.unmodifiableMap(new LinkedHashMap<>(staticPartition));
}

/**
* @param charPadding whether the BE writer pads the CHAR values of the rows it inserts, and the static
* partition values are padded to match them
*/
static PaimonWriteBinding create(PaimonTableHandle handle, FileStoreTable table,
Map<String, String> hadoopConfig, ConnectorWriteHandle writeHandle) {
Map<String, String> hadoopConfig, ConnectorWriteHandle writeHandle, boolean charPadding) {
boolean overwrite = writeHandle.isOverwrite();
// Only a static overwrite reads the partition values. An INSERT writes each row to the partition its own
// values name, so it leaves them alone: casting them could only fail a write the rows would accept.
Map<String, String> staticPartition = overwrite
? resolveStaticPartition(table, writeHandle) : Collections.emptyMap();
? resolveStaticPartition(table, writeHandle, charPadding) : Collections.emptyMap();
FileStoreTable writeTable = configureTableForWrite(table, overwrite, staticPartition);
return new PaimonWriteBinding(handle.getDatabaseName() + "." + handle.getTableName(),
writeTable, hadoopConfig, overwrite, staticPartition);
writeTable, hadoopConfig, overwrite, staticPartition, charPadding);
}

/**
* The table the BE writer gets: the FE table with {@link #DORIS_WRITE_CHAR_PADDING} among its options. Only
* the schema is replaced, so nothing else of the table changes, and the FE committer's table goes without it.
*/
private static FileStoreTable withCharPadding(FileStoreTable table, boolean charPadding) {
Map<String, String> options = new HashMap<>(table.schema().options());
options.put(DORIS_WRITE_CHAR_PADDING, String.valueOf(charPadding));
return table.copy(table.schema().copy(options));
}

static FileStoreTable configureTableForWrite(FileStoreTable table, boolean overwrite,
Expand All @@ -89,9 +117,11 @@ static FileStoreTable configureTableForWrite(FileStoreTable table, boolean overw
/**
* The static partition of an overwrite, as Paimon's static overwrite parses it: SQL NULL becomes the table's
* {@code partition.default-name}, and every other value is the one the written rows carry, cast to the
* column type. A binary column can only be statically overwritten with NULL.
* column type, and padded as the BE writer pads the CHAR values of the rows ({@link #charPaddedFields}): Paimon
* parses a partition value without padding it. A binary column can only be statically overwritten with NULL.
*/
static Map<String, String> resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle) {
static Map<String, String> resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle,
boolean charPadding) {
Map<String, String> canonicalNames = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
for (String partitionKey : table.partitionKeys()) {
canonicalNames.put(partitionKey, partitionKey);
Expand All @@ -114,7 +144,9 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
}
requestedColumns.put(key, canonicalName);
}
String defaultPartitionName = CoreOptions.fromMap(table.options()).partitionDefaultName();
CoreOptions options = CoreOptions.fromMap(table.options());
String defaultPartitionName = options.partitionDefaultName();
Predicate<String> charPaddedFields = charPaddedFields(table.schema(), charPadding);
Map<String, String> castValues = writeHandle.getCastStaticPartitionSpec();
Map<String, String> result = new LinkedHashMap<>();
for (Map.Entry<String, String> requested : requestedColumns.entrySet()) {
Expand All @@ -126,10 +158,13 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
}
String value = Objects.requireNonNull(castValues.get(key),
() -> "missing the cast value of static partition column " + key);
if (table.rowType().getField(canonicalName).type().getTypeRoot()
== DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) {
DataType type = table.rowType().getField(canonicalName).type();
if (type.getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) {
value = toSdkLocalTime(canonicalName, value);
}
if (type.getTypeRoot() == DataTypeRoot.CHAR && charPaddedFields.test(canonicalName)) {
value = padChar(value, ((CharType) type).getLength());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Mirror the rowkind.field exemption in static partition padding

The writer now leaves the column named by rowkind.field unpadded, but this branch still pads it when it is a CHAR partition column. For a valid table (id INT, rk CHAR(3)) with primary key (id,rk), partition key rk, bucket=1 and rowkind.field=rk, INSERT OVERWRITE TABLE t PARTITION(rk='+I') SELECT 1 puts '+I ' in the FE overwrite filter while the BE writes partition '+I'. Paimon rejects that file as outside the specified partition; a SELECT that returns no rows at runtime instead succeeds without clearing the existing '+I' partition. Apply the writer's control-field exemption here too, and cover nonempty and empty static overwrites. This is distinct from the earlier token-padding finding: its BE fix is present, but the FE overwrite filter now disagrees with it.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 45e6e5d. This reproduced too: in a table (id INT, rk CHAR(3)) keyed by (id, rk), partitioned by rk, with 'rowkind.field' = 'rk' and two '+I' rows, INSERT OVERWRITE TABLE t PARTITION (rk = '+I') SELECT 3 failed with Trying to overwrite partition {rk=+I }, but the changes in rk=+I/ does not belong to this partition, and an overwrite whose source returned no rows at run time returned OK and left both rows. resolveStaticPartition now leaves the rowkind.field value unpadded, as the writer does (both compare the name case-sensitively), so the first overwrite replaces the partition with the row of id 3 and the empty one clears it.

A source the planner already proves empty, such as SELECT 3 LIMIT 0, behaves differently on this table, and not because of padding: with padding on the table is written by one writer, the sink then sits directly on the empty set, and fe-core skips the write altogether, as it does for any connector table with such a plan (a dynamic-bucket primary-key table, for one). The regression therefore uses a source that is empty at run time.

Tests: PaimonWriteBindingTest.staticRowKindFieldPartitionIsLeftUnpaddedAsTheRowsAre (the rowkind.field partition value stays '+I' while another CHAR partition value is padded), and test_paimon_write_char_padding, which overwrites the '+I' partition of such a table with one row and then with a source that is empty at run time.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A correction to the second paragraph of my reply above. With 1012317776a this table is no longer written by one writer: its CHAR partition column is the rowkind.field, which the writer never pads, so it is routed natively. And since #69010, fe-core runs the transaction of an external INSERT OVERWRITE even when planning proves its source empty. On a cluster with both, INSERT OVERWRITE TABLE t PARTITION (rk = '+I') SELECT 3 LIMIT 0 now commits an OVERWRITE snapshot and clears the partition, as a source that is empty at run time does; the regression still uses the latter.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Keep +U overwrite rows aligned with the partition filter

For a table (id INT, p CHAR(3), rk STRING, v INT) with primary key (id,p), partition p, bucket=1 and rowkind.field=rk, the supported INSERT OVERWRITE TABLE t PARTITION(p='x') SELECT 1, '+U', 9 pads the FE overwrite filter to 'x ' here. The new PaimonWriteSchema.insertsRow predicate skips all row padding for +U, leaving the materialized partition value 'x'; the SDK then rejects the appended file as outside the overwrite partition. This worked with the previous unpadded path. Use a consistent static partition policy for the filter and every accepted event kind, or reject the combination before writing, and add a +U overwrite regression. Unlike the earlier control-column partition issue, p is an ordinary CHAR partition and rk is a separate STRING column.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 1740cfa4637. This reproduces, and for more than '+U': in a table keyed by (id, p), partitioned by p CHAR(3) and with rowkind.field = rk, a static overwrite of p = 'x' failed with Trying to overwrite partition {p=x }, but the changes in p=x/ does not belong to this partition for a '+U' row, and likewise for a '-D' row, in an ORC table, and for a '+I' and a '-D' of one key in one statement; with padding off all of them worked. An overwrite replaces the partitions it writes (withIgnorePreviousFiles), so none of its rows matches a stored key, and the reason the previous commit gave for leaving '-D', '-U' and '+U' rows unpadded does not hold there. The writer now pads every row of an overwrite, as FE pads the static partition value of its filter: the '+U' row lands in 'x ', and a '+I' and a '-D' of one key in the overwrite still cancel out. Writes other than overwrites are unchanged.

Tests: PaimonWriteSchemaTest.testOverwritePadsEveryRowOfARowKindFieldTable (the '+I', '+U', '-D' and '-U' rows of an overwrite are padded, only the partition value of an ORC table, and the '+U' row of a plain INSERT is not); padding by the decoded row kind in an overwrite turns it red. test_paimon_write_char_padding overwrites p = 'x' with a '+U' row, which reads back as [x ], and p = 'y' with a '+I' and a '-D' of one key, which leaves the partition empty.

}
if (defaultPartitionName.equals(value)) {
// Paimon's static overwrite reads this string as the NULL partition of any partition type, so
// a value equal to it would overwrite the NULL partition instead.
Expand All @@ -142,6 +177,25 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
return result;
}

/**
* The fields whose CHAR values the BE writer pads, as PaimonWriteSchema.create decides it: with {@code charPadding}
* on, every field but the table's {@code rowkind.field}, from whose value Paimon decodes the row kind, and of a
* table that writes ORC files only the partition fields. ORC pads every CHAR value it writes and strips the
* trailing spaces from one it reads, so the writer leaves the other fields of such a table alone. Keep in sync
* with PaimonWriteSchema.create and PaimonWriteSchema.writesOrcFiles in be-java-extensions/paimon-scanner.
*/
static Predicate<String> charPaddedFields(TableSchema schema, boolean charPadding) {
if (!charPadding) {
return field -> false;
}
CoreOptions options = CoreOptions.fromMap(schema.options());
String rowKindField = options.rowkindField().orElse(null);
boolean orcFiles = CoreOptions.FILE_FORMAT_ORC.equals(options.formatType())
|| options.fileFormatPerLevel().containsValue(CoreOptions.FILE_FORMAT_ORC);
List<String> partitionKeys = schema.partitionKeys();
return field -> !field.equals(rowKindField) && (!orcFiles || partitionKeys.contains(field));
}

/**
* Paimon parses a static overwrite value of a TIMESTAMP WITH LOCAL TIME ZONE column as local time in the FE
* JVM's default zone. Doris binds such a column as TIMESTAMPTZ, so the cast value is the instant in UTC with
Expand All @@ -161,6 +215,15 @@ private static String toSdkLocalTime(String column, String value) {
return sdkLocal.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME).replace('T', ' ');
}

/**
* Pads a CHAR value with spaces to the column length in code points, as PaimonWriteSchema pads the rows' CHAR
* values; a longer value is kept, as it is there.
*/
private static String padChar(String value, int length) {
int padLength = length - value.codePointCount(0, value.length());
return padLength > 0 ? value + " ".repeat(padLength) : value;
}

private static String serialize(FileStoreTable table) {
try {
return Base64.getEncoder().encodeToString(InstantiationUtil.serializeObject(table));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Predicate;
import java.util.stream.Collectors;

/** Builds the JNI-backed Paimon sink and binds it to the active connector transaction. */
Expand All @@ -72,6 +73,8 @@ public class PaimonWritePlanProvider implements ConnectorWritePlanProvider {
static final byte INSERT_OPERATION = 0;
static final byte UPDATE_OPERATION = 1;
static final byte DELETE_OPERATION = 2;
// The session variable, read by its name: a connector cannot import the fe-core SessionVariable constant.
static final String ENABLE_CHAR_PADDING = "enable_paimon_write_char_padding";

private final PaimonCatalogProperties catalogProperties;
private final PaimonCatalogOps catalogOps;
Expand Down Expand Up @@ -152,7 +155,7 @@ public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandl
}
PaimonConnectorTransaction transaction = currentTransaction(session);
PaimonWriteBinding binding = PaimonWriteBinding.create(
tableHandle, table, buildHadoopConfig(), handle);
tableHandle, table, buildHadoopConfig(), handle, charPaddingEnabled(session));
transaction.bind(binding);

TPaimonTableSink sink = new TPaimonTableSink();
Expand All @@ -169,6 +172,16 @@ public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandl
return new ConnectorSinkPlan(dataSink);
}

/**
* Reads the enable_paimon_write_char_padding session variable (default true), which ConnectorSessionBuilder
* passes on with every other session variable: whether a write pads CHAR values with spaces to the column
* length, as Spark and Flink do when they write a Paimon table.
*/
static boolean charPaddingEnabled(ConnectorSession session) {
Boolean enabled = session.getProperty(ENABLE_CHAR_PADDING, Boolean.class);
return enabled == null || enabled;
}

static void requireBeExecVersion(int version) {
if (version < MIN_BE_EXEC_VERSION) {
throw new DorisConnectorException("Paimon writes require BE execution version "
Expand Down Expand Up @@ -335,7 +348,7 @@ public boolean requiresMaterializeStaticPartitionValues() {
public ConnectorWriteDistribution getWriteDistribution(ConnectorSession session,
ConnectorTableHandle connectorHandle) {
FileStoreTable table = resolveTable((PaimonTableHandle) connectorHandle);
ConnectorWriteDistribution fixedBucket = fixedBucketDistribution(table);
ConnectorWriteDistribution fixedBucket = fixedBucketDistribution(table, charPaddingEnabled(session));
if (fixedBucket != null) {
return fixedBucket;
}
Expand All @@ -353,7 +366,16 @@ public ConnectorWriteDistribution getWriteDistribution(ConnectorSession session,
ConnectorWriteDistribution.Mode.EXTERNAL_UNPARTITIONED);
}

private ConnectorWriteDistribution fixedBucketDistribution(FileStoreTable table) {
/**
* Routes each row to the writer that owns its partition and bucket, which BE computes from the row's routing
* values. The writer pads the CHAR values of the fields {@link PaimonWriteBinding#charPaddedFields} names before
* Paimon computes the partition and the bucket of the row, while BE would hash the unpadded values, and several
* writers would write and compact the same bucket. Such a routing field cannot be routed natively: this returns
* null, and the table gets the distribution of a table whose routing fields BE cannot hash, a single writer for
* a primary-key table. A CHAR field that the writer leaves alone, such as the bucket key of a table that writes
* ORC files, is routed natively.
*/
private ConnectorWriteDistribution fixedBucketDistribution(FileStoreTable table, boolean charPadding) {
if (table.bucketMode() != BucketMode.HASH_FIXED) {
return null;
}
Expand All @@ -369,10 +391,11 @@ private ConnectorWriteDistribution fixedBucketDistribution(FileStoreTable table)
}
List<String> routeColumns = new ArrayList<>();
Map<String, Integer> routeIndexes = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
Predicate<String> charPaddedFields = PaimonWriteBinding.charPaddedFields(schema, charPadding);
List<Integer> partitionIndexes = appendRouteFields(
schema.partitionKeys(), fields, routeColumns, routeIndexes);
schema.partitionKeys(), fields, routeColumns, routeIndexes, charPaddedFields);
List<Integer> bucketIndexes = appendRouteFields(
schema.bucketKeys(), fields, routeColumns, routeIndexes);
schema.bucketKeys(), fields, routeColumns, routeIndexes, charPaddedFields);
if (partitionIndexes == null || bucketIndexes == null || bucketIndexes.isEmpty()) {
return null;
}
Expand All @@ -390,12 +413,12 @@ private String joinIndexes(List<Integer> indexes) {

private List<Integer> appendRouteFields(List<String> fieldNames,
Map<String, DataField> fields, List<String> routeColumns,
Map<String, Integer> routeIndexes) {
Map<String, Integer> routeIndexes, Predicate<String> charPaddedFields) {
List<Integer> indexes = new ArrayList<>();
for (String fieldName : fieldNames) {
DataField field = fields.get(fieldName);
if (field == null || field.defaultValue() != null
|| !supportsNativeRouting(field.type().getTypeRoot())) {
|| !supportsNativeRouting(field.type().getTypeRoot(), charPaddedFields.test(field.name()))) {
return null;
}
Integer index = routeIndexes.get(fieldName);
Expand All @@ -409,16 +432,17 @@ private List<Integer> appendRouteFields(List<String> fieldNames,
return indexes;
}

private boolean supportsNativeRouting(DataTypeRoot type) {
private boolean supportsNativeRouting(DataTypeRoot type, boolean charPadded) {
switch (type) {
case CHAR:
return !charPadded;
case BOOLEAN:
case TINYINT:
case SMALLINT:
case INTEGER:
case BIGINT:
case FLOAT:
case DOUBLE:
case CHAR:
case VARCHAR:
case BINARY:
case VARBINARY:
Expand Down
Loading
Loading