diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWriteBinding.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWriteBinding.java index 1a83c6f5a77b59..66e7c6febf6633 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWriteBinding.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWriteBinding.java @@ -26,9 +26,14 @@ import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.VarBinaryType; import org.apache.paimon.utils.InstantiationUtil; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneId; @@ -36,6 +41,7 @@ import java.util.Base64; import java.util.Collections; import java.util.HashMap; +import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -80,14 +86,39 @@ private PaimonWriteBinding(String tableName, FileStoreTable table, */ static PaimonWriteBinding create(PaimonTableHandle handle, FileStoreTable table, Map hadoopConfig, ConnectorWriteHandle writeHandle, boolean charPadding) { + String tableName = handle.getDatabaseName() + "." + handle.getTableName(); + checkBinaryPartitionNames(tableName, table); boolean overwrite = writeHandle.isOverwrite(); // Only a static overwrite reads the partition values. An INSERT writes each row to the partition its own // values name, so it leaves them alone: casting them could only fail a write the rows would accept. Map staticPartition = overwrite ? resolveStaticPartition(table, writeHandle, charPadding) : Collections.emptyMap(); FileStoreTable writeTable = configureTableForWrite(table, overwrite, staticPartition); - return new PaimonWriteBinding(handle.getDatabaseName() + "." + handle.getTableName(), - writeTable, hadoopConfig, overwrite, staticPartition, charPadding); + return new PaimonWriteBinding(tableName, writeTable, hadoopConfig, overwrite, staticPartition, charPadding); + } + + /** + * With {@code partition.legacy-name} = true, the default, Paimon names a partition after the string of each + * value, which for a binary value is the Java array object ({@code [B@1b6d3586}) rather than its bytes. Every + * resolution of the name differs, so a write fails its commit after the BE wrote its files under one of them, + * and Paimon itself cannot read such a partition back. + */ + static void checkBinaryPartitionNames(String tableName, FileStoreTable table) { + if (!CoreOptions.fromMap(table.options()).legacyPartitionName()) { + return; + } + for (String partitionKey : table.partitionKeys()) { + if (isBinary(table.rowType().getField(partitionKey).type())) { + throw new DorisConnectorException("Cannot write Paimon table " + tableName + ": its binary" + + " partition column '" + partitionKey + "' needs the table option 'partition.legacy-name'" + + " = 'false'. With legacy partition names Paimon names a binary partition after a Java" + + " array object rather than its bytes and cannot read the partition back"); + } + } + } + + private static boolean isBinary(DataType type) { + return type.getTypeRoot() == DataTypeRoot.BINARY || type.getTypeRoot() == DataTypeRoot.VARBINARY; } /** @@ -118,7 +149,8 @@ static FileStoreTable configureTableForWrite(FileStoreTable table, boolean overw * The static partition of an overwrite, as Paimon's static overwrite parses it: SQL NULL becomes the table's * {@code partition.default-name}, and every other value is the one the written rows carry, cast to the * column type, and padded as the BE writer pads the CHAR values of the rows ({@link #charPaddedFields}): Paimon - * parses a partition value without padding it. A binary column can only be statically overwritten with NULL. + * parses a partition value without padding it. A binary value is the text of its bytes ({@link + * #binaryStaticValue}). */ static Map resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle, boolean charPadding) { @@ -133,15 +165,6 @@ static Map resolveStaticPartition(FileStoreTable table, Connecto throw new DorisConnectorException("Column '" + key + "' is not a partition column of Paimon table"); } - DataTypeRoot typeRoot = table.rowType().getField(canonicalName).type().getTypeRoot(); - if ((typeRoot == DataTypeRoot.BINARY || typeRoot == DataTypeRoot.VARBINARY) - && !writeHandle.getStaticPartitionNullKeys().contains(key)) { - // Paimon's static overwrite takes the UTF-8 bytes of a value's string, which cannot name every - // binary partition, and a wrong name overwrites another partition without an error. Checked - // before the values are cast. - throw new DorisConnectorException("Static overwrite of binary partition column '" - + canonicalName + "' is not supported"); - } requestedColumns.put(key, canonicalName); } CoreOptions options = CoreOptions.fromMap(table.options()); @@ -165,6 +188,9 @@ static Map resolveStaticPartition(FileStoreTable table, Connecto if (type.getTypeRoot() == DataTypeRoot.CHAR && charPaddedFields.test(canonicalName)) { value = padChar(value, ((CharType) type).getLength()); } + if (isBinary(type)) { + value = binaryStaticValue(canonicalName, value, type); + } if (defaultPartitionName.equals(value)) { // Paimon's static overwrite reads this string as the NULL partition of any partition type, so // a value equal to it would overwrite the NULL partition instead. @@ -215,6 +241,38 @@ private static String toSdkLocalTime(String column, String value) { return sdkLocal.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME).replace('T', ' '); } + /** + * Paimon's static overwrite takes the UTF-8 bytes of a partition value's string, so the bytes the rows carry, + * which reach it as {@code 0x} and their hex digits, name their partition only as the text they decode to. A + * wrong name would overwrite another partition without an error, so bytes that are not UTF-8 are rejected; + * strict decoding also rejects overlong and surrogate sequences, so the text encodes back to the same bytes. + * Paimon keeps those bytes for a BINARY column whatever their length, as the rows keep them, and fails the + * commit for a VARBINARY value longer than its column, so such a value is rejected here, before any file is + * written. + */ + private static String binaryStaticValue(String column, String value, DataType type) { + if (!value.startsWith("0x")) { + throw new IllegalStateException("Binary static partition value of column " + column + + " lacks the 0x prefix: " + value); + } + byte[] bytes = HexFormat.of().parseHex(value.substring(2)); + if (type.getTypeRoot() == DataTypeRoot.VARBINARY && bytes.length > ((VarBinaryType) type).getLength()) { + throw new DorisConnectorException("Static partition value " + value + " of binary column '" + column + + "' has " + bytes.length + " bytes, more than its type " + type + " holds, which a static" + + " overwrite of a Paimon table rejects"); + } + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString(); + } catch (CharacterCodingException e) { + throw new DorisConnectorException("Static partition value " + value + " of binary column '" + column + + "' is not UTF-8 text, which a static overwrite of a Paimon table needs to name its partition"); + } + } + /** * Pads a CHAR value with spaces to the column length in code points, as PaimonWriteSchema pads the rows' CHAR * values; a longer value is kept, as it is there. diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonWriteBindingTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonWriteBindingTest.java index 7cef1818c1de11..63d6fe5f564a7f 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonWriteBindingTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonWriteBindingTest.java @@ -44,6 +44,7 @@ import java.time.OffsetDateTime; import java.util.Arrays; import java.util.Collections; +import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -61,6 +62,8 @@ public class PaimonWriteBindingTest { private static final String DEFAULT_NAME = "__CUSTOM_DEFAULT__"; private static final String DYNAMIC_OVERWRITE = CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key(); + private static final Map NEW_PARTITION_NAMES = + Collections.singletonMap("partition.legacy-name", "false"); private TimeZone jvmZone; @@ -197,25 +200,91 @@ public void staticLtzPartitionOutsideAnOverlapRoundTripsThroughPaimon(@TempDir P } @Test - public void staticOverwriteRejectsABinaryPartitionValue(@TempDir Path warehouse) { - // Paimon's static overwrite takes the UTF-8 bytes of a value's string: "0x4142" names the partition of - // those six bytes, not of the two bytes 41 42 the rows carry, so an overwrite without rows would report - // success and leave the partition it was given in place. The check comes before the values are cast, - // which a binary value may not survive. MUTATION: dropping the check -> red. + public void staticBinaryPartitionIsTheTextOfTheBytesTheRowsCarry(@TempDir Path warehouse) { + // Paimon's static overwrite takes the UTF-8 bytes of a value's string: "0x4142", as the cast value of a + // VARBINARY column reaches the connector, names the partition of those six bytes, not of the two bytes + // 41 42 the rows carry, so an overwrite without rows would report success and leave the partition it was + // given in place. MUTATION: passing the cast value through -> red. Assertions.assertArrayEquals("0x4142".getBytes(StandardCharsets.UTF_8), (byte[]) TypeUtils.castFromString("0x4142", DataTypes.BYTES())); for (DataType type : Arrays.asList(DataTypes.BYTES(), DataTypes.VARBINARY(2), DataTypes.BINARY(2))) { - FileStoreTable table = partitionedTable(warehouse, Collections.emptyMap(), - new DataField(1, "part", type)); - ConnectorWriteHandle overwrite = handle(true, Collections.singletonMap("part", "0x4142"), () -> { - throw new AssertionError("a binary static partition value must be rejected before it is cast"); - }, Collections.emptySet()); + FileStoreTable table = partitionedTable(warehouse, NEW_PARTITION_NAMES, new DataField(1, "part", type)); + + String value = PaimonWriteBinding.resolveStaticPartition(table, write(true, "part", "AB", "0x4142"), + true).get("part"); + + Assertions.assertArrayEquals(new byte[] {0x41, 0x42}, (byte[]) TypeUtils.castFromString(value, type), + type.toString()); + } + } + @Test + public void staticBinaryPartitionRejectsBytesThatAreNotUtf8(@TempDir Path warehouse) { + // No string's UTF-8 bytes are a lone 0xEF, an overlong '/' or an encoded surrogate, so Paimon's static + // overwrite cannot name such a partition; a lenient decoder would name another one through U+FFFD. + // MUTATION: decoding with replacement -> red. + FileStoreTable table = partitionedTable(warehouse, NEW_PARTITION_NAMES, + new DataField(1, "part", DataTypes.BYTES())); + for (String castValue : Arrays.asList("0xEF", "0xC0AF", "0xEDA080")) { DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, - () -> PaimonWriteBinding.resolveStaticPartition(table, overwrite, true)); - Assertions.assertTrue(error.getMessage().contains("binary partition column 'part'"), - error.getMessage()); + () -> PaimonWriteBinding.resolveStaticPartition(table, write(true, "part", castValue, castValue), + true)); + Assertions.assertTrue(error.getMessage().contains("is not UTF-8 text"), error.getMessage()); + } + } + + @Test + public void staticBinaryPartitionKeepsTheLengthOfTheBytes(@TempDir Path warehouse) { + // The rows keep their bytes whatever the column length, and Paimon's static overwrite keeps them for a + // BINARY(n) column too. For a VARBINARY(n) column it fails the commit, after the files are written, when the + // bytes are longer, so such a value fails while the write is planned. MUTATION: no length check -> red. + FileStoreTable binary = partitionedTable(warehouse, NEW_PARTITION_NAMES, + new DataField(1, "part", DataTypes.BINARY(2))); + for (String castValue : Arrays.asList("0x41", "0x414243")) { + String value = PaimonWriteBinding.resolveStaticPartition(binary, + write(true, "part", castValue, castValue), true).get("part"); + Assertions.assertArrayEquals(HexFormat.of().parseHex(castValue.substring(2)), + (byte[]) TypeUtils.castFromString(value, DataTypes.BINARY(2)), castValue); + } + + FileStoreTable varbinary = partitionedTable(warehouse, NEW_PARTITION_NAMES, + new DataField(1, "part", DataTypes.VARBINARY(2))); + Assertions.assertEquals("A", PaimonWriteBinding.resolveStaticPartition(varbinary, + write(true, "part", "0x41", "0x41"), true).get("part")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> TypeUtils.castFromString("ABC", DataTypes.VARBINARY(2))); + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> PaimonWriteBinding.resolveStaticPartition(varbinary, + write(true, "part", "0x414243", "0x414243"), true)); + Assertions.assertTrue(error.getMessage().contains("has 3 bytes, more than its type VARBINARY(2) holds"), + error.getMessage()); + } + + @Test + public void writeOfABinaryPartitionNeedsNewPartitionNames(@TempDir Path warehouse) { + // With partition.legacy-name = true, the default, Paimon names a binary partition after the Java array + // object, so the commit of the written files fails and Paimon cannot read the partition back. The write + // fails while it is planned; a table with new partition names, or without a binary partition, is written. + // MUTATION: dropping the check -> red. + PaimonTableHandle handle = new PaimonTableHandle("db", "tbl", Collections.emptyList(), + Collections.emptyList()); + for (DataType type : Arrays.asList(DataTypes.BYTES(), DataTypes.VARBINARY(2), DataTypes.BINARY(2))) { + FileStoreTable legacy = partitionedTable(warehouse, Collections.emptyMap(), new DataField(1, "part", type)); + for (boolean overwrite : new boolean[] {false, true}) { + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> PaimonWriteBinding.create(handle, legacy, Collections.emptyMap(), + write(overwrite, "part", "AB", "0x4142"), true)); + Assertions.assertTrue(error.getMessage().contains("'partition.legacy-name' = 'false'"), + error.getMessage()); + } + FileStoreTable renamed = partitionedTable(warehouse, NEW_PARTITION_NAMES, new DataField(1, "part", type)); + Assertions.assertEquals("AB", PaimonWriteBinding.create(handle, renamed, Collections.emptyMap(), + write(true, "part", "AB", "0x4142"), true).getStaticPartition().get("part")); } + FileStoreTable legacyString = partitionedTable(warehouse, Collections.emptyMap(), + new DataField(1, "part", DataTypes.STRING())); + Assertions.assertFalse(PaimonWriteBinding.create(handle, legacyString, Collections.emptyMap(), + write(false, "part", "AB", "AB"), true).isOverwrite()); } @Test diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertCommandContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertCommandContext.java index 7d96149086a444..9db5b66674f35f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertCommandContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertCommandContext.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.VarBinaryType; import org.apache.doris.nereids.types.coercion.CharacterType; import com.google.common.base.Preconditions; @@ -139,6 +140,11 @@ private static String specValue(Literal literal) { } private static Literal writtenValue(Literal value, DataType columnType) { + if (value instanceof VarBinaryLiteral && columnType instanceof VarBinaryType) { + // A binary literal is VARBINARY of the maximum length, which Literal casts to no other length. BindSink + // leaves that cast of the rows to BE, which keeps the bytes: a VARBINARY length is metadata only. + return new VarBinaryLiteral((VarBinaryType) columnType, (byte[]) value.getValue()); + } if (!value.getDataType().isStringLikeType() || !columnType.isStringLikeType()) { Expression cast = value.checkedCastTo(columnType); Preconditions.checkState(cast instanceof Literal, "cast of literal %s is not a literal", value); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java index 2da58e50325399..d77c489afd5c66 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java @@ -208,13 +208,28 @@ public void staticPartitionBinaryValueKeepsItsBytesWhenCast() throws AnalysisExc // byte array ([B@...), which names no partition. MUTATION: getStringValue() for a cast binary value -> red. Map partition = Collections.singletonMap("b", new VarBinaryLiteral(new byte[] {0x41, 0x42})); - // The type Paimon's VARBINARY maps to. + // The type Paimon's BYTES maps to. List schema = Collections.singletonList( new Column("b", ScalarType.createVarbinaryType(ScalarType.MAX_VARBINARY_LENGTH))); Assertions.assertEquals(Collections.singletonMap("b", "0x4142"), castStaticPartitionSpec(partition, schema)); } + @Test + public void staticPartitionBinaryValueKeepsItsBytesForAColumnOfAnyLength() throws AnalysisException { + // Paimon's BINARY(n) and VARBINARY(n) map to VARBINARY(n). A binary literal is VARBINARY of the maximum + // length, which Literal casts to no other length; the rows keep their bytes in such a column, shorter or + // longer than it, and so does the cast value. MUTATION: no binary case in writtenValue -> red ("Cast from + // 4142 to VARBINARY(2) not supported"). + List schema = Collections.singletonList(new Column("b", ScalarType.createVarbinaryType(2))); + for (String hex : Arrays.asList("41", "4142", "414243")) { + Map partition = Collections.singletonMap("b", new VarBinaryLiteral(hex)); + + Assertions.assertEquals(Collections.singletonMap("b", "0x" + hex), + castStaticPartitionSpec(partition, schema)); + } + } + @Test public void staticPartitionValueThatDoesNotFitItsTypeFailsOnlyWhenCast() throws AnalysisException { // Only a connector that reads the cast values may fail on them; the others keep reading the literal.