Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,22 @@
import org.apache.paimon.types.CharType;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.VarBinaryType;
import org.apache.paimon.utils.InstantiationUtil;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.HexFormat;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -80,14 +86,39 @@ private PaimonWriteBinding(String tableName, FileStoreTable table,
*/
static PaimonWriteBinding create(PaimonTableHandle handle, FileStoreTable table,
Map<String, String> hadoopConfig, ConnectorWriteHandle writeHandle, boolean charPadding) {
String tableName = handle.getDatabaseName() + "." + handle.getTableName();
checkBinaryPartitionNames(tableName, table);
boolean overwrite = writeHandle.isOverwrite();
// Only a static overwrite reads the partition values. An INSERT writes each row to the partition its own
// values name, so it leaves them alone: casting them could only fail a write the rows would accept.
Map<String, String> staticPartition = overwrite
? resolveStaticPartition(table, writeHandle, charPadding) : Collections.emptyMap();
FileStoreTable writeTable = configureTableForWrite(table, overwrite, staticPartition);
return new PaimonWriteBinding(handle.getDatabaseName() + "." + handle.getTableName(),
writeTable, hadoopConfig, overwrite, staticPartition, charPadding);
return new PaimonWriteBinding(tableName, writeTable, hadoopConfig, overwrite, staticPartition, charPadding);
}

/**
* With {@code partition.legacy-name} = true, the default, Paimon names a partition after the string of each
* value, which for a binary value is the Java array object ({@code [B@1b6d3586}) rather than its bytes. Every
* resolution of the name differs, so a write fails its commit after the BE wrote its files under one of them,
* and Paimon itself cannot read such a partition back.
*/
static void checkBinaryPartitionNames(String tableName, FileStoreTable table) {
if (!CoreOptions.fromMap(table.options()).legacyPartitionName()) {
return;
}
for (String partitionKey : table.partitionKeys()) {
if (isBinary(table.rowType().getField(partitionKey).type())) {
throw new DorisConnectorException("Cannot write Paimon table " + tableName + ": its binary"
+ " partition column '" + partitionKey + "' needs the table option 'partition.legacy-name'"
+ " = 'false'. With legacy partition names Paimon names a binary partition after a Java"
+ " array object rather than its bytes and cannot read the partition back");
}
}
}

private static boolean isBinary(DataType type) {
return type.getTypeRoot() == DataTypeRoot.BINARY || type.getTypeRoot() == DataTypeRoot.VARBINARY;
}

/**
Expand Down Expand Up @@ -118,7 +149,8 @@ static FileStoreTable configureTableForWrite(FileStoreTable table, boolean overw
* The static partition of an overwrite, as Paimon's static overwrite parses it: SQL NULL becomes the table's
* {@code partition.default-name}, and every other value is the one the written rows carry, cast to the
* column type, and padded as the BE writer pads the CHAR values of the rows ({@link #charPaddedFields}): Paimon
* parses a partition value without padding it. A binary column can only be statically overwritten with NULL.
* parses a partition value without padding it. A binary value is the text of its bytes ({@link
* #binaryStaticValue}).
*/
static Map<String, String> resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle,
boolean charPadding) {
Expand All @@ -133,15 +165,6 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
throw new DorisConnectorException("Column '" + key
+ "' is not a partition column of Paimon table");
}
DataTypeRoot typeRoot = table.rowType().getField(canonicalName).type().getTypeRoot();
if ((typeRoot == DataTypeRoot.BINARY || typeRoot == DataTypeRoot.VARBINARY)
&& !writeHandle.getStaticPartitionNullKeys().contains(key)) {
// Paimon's static overwrite takes the UTF-8 bytes of a value's string, which cannot name every
// binary partition, and a wrong name overwrites another partition without an error. Checked
// before the values are cast.
throw new DorisConnectorException("Static overwrite of binary partition column '"
+ canonicalName + "' is not supported");
}
requestedColumns.put(key, canonicalName);
}
CoreOptions options = CoreOptions.fromMap(table.options());
Expand All @@ -165,6 +188,9 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
if (type.getTypeRoot() == DataTypeRoot.CHAR && charPaddedFields.test(canonicalName)) {
value = padChar(value, ((CharType) type).getLength());
}
if (isBinary(type)) {
value = binaryStaticValue(canonicalName, value, type);
}
if (defaultPartitionName.equals(value)) {
// Paimon's static overwrite reads this string as the NULL partition of any partition type, so
// a value equal to it would overwrite the NULL partition instead.
Expand Down Expand Up @@ -215,6 +241,38 @@ private static String toSdkLocalTime(String column, String value) {
return sdkLocal.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME).replace('T', ' ');
}

/**
* Paimon's static overwrite takes the UTF-8 bytes of a partition value's string, so the bytes the rows carry,
* which reach it as {@code 0x} and their hex digits, name their partition only as the text they decode to. A
* wrong name would overwrite another partition without an error, so bytes that are not UTF-8 are rejected;
* strict decoding also rejects overlong and surrogate sequences, so the text encodes back to the same bytes.
* Paimon keeps those bytes for a BINARY column whatever their length, as the rows keep them, and fails the
* commit for a VARBINARY value longer than its column, so such a value is rejected here, before any file is
* written.
*/
private static String binaryStaticValue(String column, String value, DataType type) {
if (!value.startsWith("0x")) {
throw new IllegalStateException("Binary static partition value of column " + column
+ " lacks the 0x prefix: " + value);
}
byte[] bytes = HexFormat.of().parseHex(value.substring(2));
if (type.getTypeRoot() == DataTypeRoot.VARBINARY && bytes.length > ((VarBinaryType) type).getLength()) {
throw new DorisConnectorException("Static partition value " + value + " of binary column '" + column
+ "' has " + bytes.length + " bytes, more than its type " + type + " holds, which a static"
+ " overwrite of a Paimon table rejects");
}
try {
return StandardCharsets.UTF_8.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT)
.decode(ByteBuffer.wrap(bytes))
.toString();
} catch (CharacterCodingException e) {
throw new DorisConnectorException("Static partition value " + value + " of binary column '" + column
+ "' is not UTF-8 text, which a static overwrite of a Paimon table needs to name its partition");
}
}

/**
* Pads a CHAR value with spaces to the column length in code points, as PaimonWriteSchema pads the rows' CHAR
* values; a longer value is kept, as it is there.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import java.time.OffsetDateTime;
import java.util.Arrays;
import java.util.Collections;
import java.util.HexFormat;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -61,6 +62,8 @@ public class PaimonWriteBindingTest {

private static final String DEFAULT_NAME = "__CUSTOM_DEFAULT__";
private static final String DYNAMIC_OVERWRITE = CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key();
private static final Map<String, String> NEW_PARTITION_NAMES =
Collections.singletonMap("partition.legacy-name", "false");

private TimeZone jvmZone;

Expand Down Expand Up @@ -197,25 +200,91 @@ public void staticLtzPartitionOutsideAnOverlapRoundTripsThroughPaimon(@TempDir P
}

@Test
public void staticOverwriteRejectsABinaryPartitionValue(@TempDir Path warehouse) {
// Paimon's static overwrite takes the UTF-8 bytes of a value's string: "0x4142" names the partition of
// those six bytes, not of the two bytes 41 42 the rows carry, so an overwrite without rows would report
// success and leave the partition it was given in place. The check comes before the values are cast,
// which a binary value may not survive. MUTATION: dropping the check -> red.
public void staticBinaryPartitionIsTheTextOfTheBytesTheRowsCarry(@TempDir Path warehouse) {
// Paimon's static overwrite takes the UTF-8 bytes of a value's string: "0x4142", as the cast value of a
// VARBINARY column reaches the connector, names the partition of those six bytes, not of the two bytes
// 41 42 the rows carry, so an overwrite without rows would report success and leave the partition it was
// given in place. MUTATION: passing the cast value through -> red.
Assertions.assertArrayEquals("0x4142".getBytes(StandardCharsets.UTF_8),
(byte[]) TypeUtils.castFromString("0x4142", DataTypes.BYTES()));
for (DataType type : Arrays.asList(DataTypes.BYTES(), DataTypes.VARBINARY(2), DataTypes.BINARY(2))) {
FileStoreTable table = partitionedTable(warehouse, Collections.emptyMap(),
new DataField(1, "part", type));
ConnectorWriteHandle overwrite = handle(true, Collections.singletonMap("part", "0x4142"), () -> {
throw new AssertionError("a binary static partition value must be rejected before it is cast");
}, Collections.emptySet());
FileStoreTable table = partitionedTable(warehouse, NEW_PARTITION_NAMES, new DataField(1, "part", type));

String value = PaimonWriteBinding.resolveStaticPartition(table, write(true, "part", "AB", "0x4142"),
true).get("part");

Assertions.assertArrayEquals(new byte[] {0x41, 0x42}, (byte[]) TypeUtils.castFromString(value, type),
type.toString());
}
}

@Test
public void staticBinaryPartitionRejectsBytesThatAreNotUtf8(@TempDir Path warehouse) {
// No string's UTF-8 bytes are a lone 0xEF, an overlong '/' or an encoded surrogate, so Paimon's static
// overwrite cannot name such a partition; a lenient decoder would name another one through U+FFFD.
// MUTATION: decoding with replacement -> red.
FileStoreTable table = partitionedTable(warehouse, NEW_PARTITION_NAMES,
new DataField(1, "part", DataTypes.BYTES()));
for (String castValue : Arrays.asList("0xEF", "0xC0AF", "0xEDA080")) {
DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
() -> PaimonWriteBinding.resolveStaticPartition(table, overwrite, true));
Assertions.assertTrue(error.getMessage().contains("binary partition column 'part'"),
error.getMessage());
() -> PaimonWriteBinding.resolveStaticPartition(table, write(true, "part", castValue, castValue),
true));
Assertions.assertTrue(error.getMessage().contains("is not UTF-8 text"), error.getMessage());
}
}

@Test
public void staticBinaryPartitionKeepsTheLengthOfTheBytes(@TempDir Path warehouse) {
// The rows keep their bytes whatever the column length, and Paimon's static overwrite keeps them for a
// BINARY(n) column too. For a VARBINARY(n) column it fails the commit, after the files are written, when the
// bytes are longer, so such a value fails while the write is planned. MUTATION: no length check -> red.
FileStoreTable binary = partitionedTable(warehouse, NEW_PARTITION_NAMES,
new DataField(1, "part", DataTypes.BINARY(2)));
for (String castValue : Arrays.asList("0x41", "0x414243")) {
String value = PaimonWriteBinding.resolveStaticPartition(binary,
write(true, "part", castValue, castValue), true).get("part");
Assertions.assertArrayEquals(HexFormat.of().parseHex(castValue.substring(2)),
(byte[]) TypeUtils.castFromString(value, DataTypes.BINARY(2)), castValue);
}

FileStoreTable varbinary = partitionedTable(warehouse, NEW_PARTITION_NAMES,
new DataField(1, "part", DataTypes.VARBINARY(2)));
Assertions.assertEquals("A", PaimonWriteBinding.resolveStaticPartition(varbinary,
write(true, "part", "0x41", "0x41"), true).get("part"));
Assertions.assertThrows(IllegalArgumentException.class,
() -> TypeUtils.castFromString("ABC", DataTypes.VARBINARY(2)));
DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
() -> PaimonWriteBinding.resolveStaticPartition(varbinary,
write(true, "part", "0x414243", "0x414243"), true));
Assertions.assertTrue(error.getMessage().contains("has 3 bytes, more than its type VARBINARY(2) holds"),
error.getMessage());
}

@Test
public void writeOfABinaryPartitionNeedsNewPartitionNames(@TempDir Path warehouse) {
// With partition.legacy-name = true, the default, Paimon names a binary partition after the Java array
// object, so the commit of the written files fails and Paimon cannot read the partition back. The write
// fails while it is planned; a table with new partition names, or without a binary partition, is written.
// MUTATION: dropping the check -> red.
PaimonTableHandle handle = new PaimonTableHandle("db", "tbl", Collections.emptyList(),
Collections.emptyList());
for (DataType type : Arrays.asList(DataTypes.BYTES(), DataTypes.VARBINARY(2), DataTypes.BINARY(2))) {
FileStoreTable legacy = partitionedTable(warehouse, Collections.emptyMap(), new DataField(1, "part", type));
for (boolean overwrite : new boolean[] {false, true}) {
DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
() -> PaimonWriteBinding.create(handle, legacy, Collections.emptyMap(),
write(overwrite, "part", "AB", "0x4142"), true));
Assertions.assertTrue(error.getMessage().contains("'partition.legacy-name' = 'false'"),
error.getMessage());
}
FileStoreTable renamed = partitionedTable(warehouse, NEW_PARTITION_NAMES, new DataField(1, "part", type));
Assertions.assertEquals("AB", PaimonWriteBinding.create(handle, renamed, Collections.emptyMap(),
write(true, "part", "AB", "0x4142"), true).getStaticPartition().get("part"));
}
FileStoreTable legacyString = partitionedTable(warehouse, Collections.emptyMap(),
new DataField(1, "part", DataTypes.STRING()));
Assertions.assertFalse(PaimonWriteBinding.create(handle, legacyString, Collections.emptyMap(),
write(false, "part", "AB", "AB"), true).isOverwrite());
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.VarBinaryType;
import org.apache.doris.nereids.types.coercion.CharacterType;

import com.google.common.base.Preconditions;
Expand Down Expand Up @@ -139,6 +140,11 @@ private static String specValue(Literal literal) {
}

private static Literal writtenValue(Literal value, DataType columnType) {
if (value instanceof VarBinaryLiteral && columnType instanceof VarBinaryType) {
// A binary literal is VARBINARY of the maximum length, which Literal casts to no other length. BindSink
// leaves that cast of the rows to BE, which keeps the bytes: a VARBINARY length is metadata only.
return new VarBinaryLiteral((VarBinaryType) columnType, (byte[]) value.getValue());
}
if (!value.getDataType().isStringLikeType() || !columnType.isStringLikeType()) {
Expression cast = value.checkedCastTo(columnType);
Preconditions.checkState(cast instanceof Literal, "cast of literal %s is not a literal", value);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -208,13 +208,28 @@ public void staticPartitionBinaryValueKeepsItsBytesWhenCast() throws AnalysisExc
// byte array ([B@...), which names no partition. MUTATION: getStringValue() for a cast binary value -> red.
Map<String, Expression> partition = Collections.singletonMap("b",
new VarBinaryLiteral(new byte[] {0x41, 0x42}));
// The type Paimon's VARBINARY maps to.
// The type Paimon's BYTES maps to.
List<Column> schema = Collections.singletonList(
new Column("b", ScalarType.createVarbinaryType(ScalarType.MAX_VARBINARY_LENGTH)));

Assertions.assertEquals(Collections.singletonMap("b", "0x4142"), castStaticPartitionSpec(partition, schema));
}

@Test
public void staticPartitionBinaryValueKeepsItsBytesForAColumnOfAnyLength() throws AnalysisException {
// Paimon's BINARY(n) and VARBINARY(n) map to VARBINARY(n). A binary literal is VARBINARY of the maximum
// length, which Literal casts to no other length; the rows keep their bytes in such a column, shorter or
// longer than it, and so does the cast value. MUTATION: no binary case in writtenValue -> red ("Cast from
// 4142 to VARBINARY(2) not supported").
List<Column> schema = Collections.singletonList(new Column("b", ScalarType.createVarbinaryType(2)));
for (String hex : Arrays.asList("41", "4142", "414243")) {
Map<String, Expression> partition = Collections.singletonMap("b", new VarBinaryLiteral(hex));

Assertions.assertEquals(Collections.singletonMap("b", "0x" + hex),
castStaticPartitionSpec(partition, schema));
}
}

@Test
public void staticPartitionValueThatDoesNotFitItsTypeFailsOnlyWhenCast() throws AnalysisException {
// Only a connector that reads the cast values may fail on them; the others keep reading the literal.
Expand Down
Loading