diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java index 65cc58ab1db621..a26d4468e87857 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java @@ -209,13 +209,10 @@ private static ConnectorType toStructType(RowType rowType, Options options) { * Convert a Doris {@link ConnectorType} (as produced by the CREATE TABLE request path) * to a Paimon {@link DataType}. * - *
This is the faithful reverse of the legacy fe-core - * {@code DorisToPaimonTypeVisitor}: the scalar set is intentionally narrow (it mirrors - * the visitor's {@code atomic} branches and NOT MaxCompute's richer set), CHAR/VARCHAR/STRING - * all collapse to {@code VarChar(MAX)} (declared length dropped), DATETIME/DATETIMEV2 map to a - * plain {@code TimestampType()} (scale dropped), and the MAP key is forced non-null. Types the - * legacy visitor did not handle (TINYINT, SMALLINT, LARGEINT, TIME, IPV4/6, JSON, ...) throw, - * preserving the legacy gap.
+ *This is the connector-SPI equivalent of the former fe-core + * {@code DorisToPaimonTypeVisitor}. It preserves declared CHAR/VARCHAR bounds and timestamp + * precision because Paimon uses them when applying casts and interpreting numeric timestamp + * values. The MAP key remains non-null, matching Paimon's type contract.
* *The returned type carries Paimon's default (nullable) flag; column-level nullability is * applied by the caller via {@code .copy(nullable)} (mirroring legacy @@ -243,10 +240,11 @@ public static DataType toPaimonType(ConnectorType type) { case "DOUBLE": return new DoubleType(); case "CHAR": + // DDL must retain bounds: widening to STRING changes historical narrowing casts. + return new CharType(type.getPrecision()); case "VARCHAR": + return new VarCharType(type.getPrecision()); case "STRING": - // Legacy parity: all char-family types collapse to VarChar(MAX); declared - // length is intentionally dropped (DorisToPaimonTypeVisitor.atomic isCharFamily). return new VarCharType(VarCharType.MAX_LENGTH); case "DATE": case "DATEV2": @@ -259,9 +257,9 @@ public static DataType toPaimonType(ConnectorType type) { case "DECIMAL256": return new DecimalType(type.getPrecision(), type.getScale()); case "DATETIME": - return new TimestampType(); case "DATETIMEV2": - return new TimestampType(type.getPrecision()); + // Paimon interprets numeric timestamp strings in units derived from this precision. + return new TimestampType("DATETIMEV2".equals(name) ? type.getPrecision() : 0); case "TIMESTAMPTZ": // Preserve the instant contract when deriving an external schema. return new LocalZonedTimestampType(type.getPrecision()); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java index af0aad304742cb..055709ac090ceb 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java @@ -20,9 +20,13 @@ import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.paimon.casting.CastExecutor; +import org.apache.paimon.casting.CastExecutors; +import org.apache.paimon.data.BinaryString; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DateType; @@ -46,7 +50,7 @@ /** * P5-T11 — pins the Doris->Paimon reverse type mapping in - * {@link PaimonTypeMapping#toPaimonType} to byte-parity with the legacy fe-core + * {@link PaimonTypeMapping#toPaimonType}, the connector equivalent of the former fe-core * {@code DorisToPaimonTypeVisitor}. * *
The CREATE TABLE path produces these {@link ConnectorType} descriptors; this mapping is
@@ -74,26 +78,56 @@ public void scalarPrimitivesMapExactly() {
}
@Test
- public void charFamilyCollapsesToVarcharMaxDroppingLength() {
- // WHY: legacy isCharFamily -> VarCharType(MAX_LENGTH) unconditionally; the declared length
- // is intentionally dropped and CHAR is NOT mapped to paimon CharType. MUTATION: honoring
- // the declared length (e.g. new VarCharType(10)) or mapping CHAR -> CharType makes these red.
- DataType expected = new VarCharType(VarCharType.MAX_LENGTH);
- Assertions.assertEquals(expected, PaimonTypeMapping.toPaimonType(ConnectorType.of("CHAR", 10, 0)));
- Assertions.assertEquals(expected, PaimonTypeMapping.toPaimonType(ConnectorType.of("VARCHAR", 20, 0)));
- Assertions.assertEquals(expected, PaimonTypeMapping.toPaimonType(ConnectorType.of("STRING")));
+ public void characterTypesKeepDeclaredBounds() {
+ // WHY: the declared length is part of the Paimon schema; Paimon truncates to it when it casts
+ // a historical data file to the current column type. MUTATION: collapsing CHAR/VARCHAR to
+ // VarChar(MAX), or mapping CHAR to VarCharType, makes these red.
+ Assertions.assertEquals(new CharType(3), PaimonTypeMapping.toPaimonType(ConnectorType.of("CHAR", 3, 0)));
+ Assertions.assertEquals(new VarCharType(3),
+ PaimonTypeMapping.toPaimonType(ConnectorType.of("VARCHAR", 3, 0)));
+ Assertions.assertEquals(new VarCharType(VarCharType.MAX_LENGTH),
+ PaimonTypeMapping.toPaimonType(ConnectorType.of("STRING")));
}
@Test
public void datetimePreservesPrecisionAndTimestampKind() {
- Assertions.assertEquals(new TimestampType(3),
- PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIMEV2", 3, 0)));
- Assertions.assertEquals(new TimestampType(),
+ for (int precision : new int[] {0, 3, 6}) {
+ Assertions.assertEquals(new TimestampType(precision),
+ PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIMEV2", precision, 0)));
+ }
+ // DATETIME (v1) has no fractional seconds.
+ Assertions.assertEquals(new TimestampType(0),
PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIME")));
Assertions.assertEquals(new org.apache.paimon.types.LocalZonedTimestampType(6),
PaimonTypeMapping.toPaimonType(ConnectorType.of("TIMESTAMPTZ", 6, 0)));
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void mappedTypesDriveHistoricalCasts() {
+ // WHY: Paimon reads a data file written under an older schema by casting the file type to the
+ // current column type, so the mapped bounds decide the values a reader sees.
+ CastExecutor