diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java index 65cc58ab1db621..a26d4468e87857 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java @@ -209,13 +209,10 @@ private static ConnectorType toStructType(RowType rowType, Options options) { * Convert a Doris {@link ConnectorType} (as produced by the CREATE TABLE request path) * to a Paimon {@link DataType}. * - *

This is the faithful reverse of the legacy fe-core - * {@code DorisToPaimonTypeVisitor}: the scalar set is intentionally narrow (it mirrors - * the visitor's {@code atomic} branches and NOT MaxCompute's richer set), CHAR/VARCHAR/STRING - * all collapse to {@code VarChar(MAX)} (declared length dropped), DATETIME/DATETIMEV2 map to a - * plain {@code TimestampType()} (scale dropped), and the MAP key is forced non-null. Types the - * legacy visitor did not handle (TINYINT, SMALLINT, LARGEINT, TIME, IPV4/6, JSON, ...) throw, - * preserving the legacy gap.

+ *

This is the connector-SPI equivalent of the former fe-core + * {@code DorisToPaimonTypeVisitor}. It preserves declared CHAR/VARCHAR bounds and timestamp + * precision because Paimon uses them when applying casts and interpreting numeric timestamp + * values. The MAP key remains non-null, matching Paimon's type contract.

* *

The returned type carries Paimon's default (nullable) flag; column-level nullability is * applied by the caller via {@code .copy(nullable)} (mirroring legacy @@ -243,10 +240,11 @@ public static DataType toPaimonType(ConnectorType type) { case "DOUBLE": return new DoubleType(); case "CHAR": + // DDL must retain bounds: widening to STRING changes historical narrowing casts. + return new CharType(type.getPrecision()); case "VARCHAR": + return new VarCharType(type.getPrecision()); case "STRING": - // Legacy parity: all char-family types collapse to VarChar(MAX); declared - // length is intentionally dropped (DorisToPaimonTypeVisitor.atomic isCharFamily). return new VarCharType(VarCharType.MAX_LENGTH); case "DATE": case "DATEV2": @@ -259,9 +257,9 @@ public static DataType toPaimonType(ConnectorType type) { case "DECIMAL256": return new DecimalType(type.getPrecision(), type.getScale()); case "DATETIME": - return new TimestampType(); case "DATETIMEV2": - return new TimestampType(type.getPrecision()); + // Paimon interprets numeric timestamp strings in units derived from this precision. + return new TimestampType("DATETIMEV2".equals(name) ? type.getPrecision() : 0); case "TIMESTAMPTZ": // Preserve the instant contract when deriving an external schema. return new LocalZonedTimestampType(type.getPrecision()); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java index af0aad304742cb..055709ac090ceb 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java @@ -20,9 +20,13 @@ import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.paimon.casting.CastExecutor; +import org.apache.paimon.casting.CastExecutors; +import org.apache.paimon.data.BinaryString; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DateType; @@ -46,7 +50,7 @@ /** * P5-T11 — pins the Doris->Paimon reverse type mapping in - * {@link PaimonTypeMapping#toPaimonType} to byte-parity with the legacy fe-core + * {@link PaimonTypeMapping#toPaimonType}, the connector equivalent of the former fe-core * {@code DorisToPaimonTypeVisitor}. * *

The CREATE TABLE path produces these {@link ConnectorType} descriptors; this mapping is @@ -74,26 +78,56 @@ public void scalarPrimitivesMapExactly() { } @Test - public void charFamilyCollapsesToVarcharMaxDroppingLength() { - // WHY: legacy isCharFamily -> VarCharType(MAX_LENGTH) unconditionally; the declared length - // is intentionally dropped and CHAR is NOT mapped to paimon CharType. MUTATION: honoring - // the declared length (e.g. new VarCharType(10)) or mapping CHAR -> CharType makes these red. - DataType expected = new VarCharType(VarCharType.MAX_LENGTH); - Assertions.assertEquals(expected, PaimonTypeMapping.toPaimonType(ConnectorType.of("CHAR", 10, 0))); - Assertions.assertEquals(expected, PaimonTypeMapping.toPaimonType(ConnectorType.of("VARCHAR", 20, 0))); - Assertions.assertEquals(expected, PaimonTypeMapping.toPaimonType(ConnectorType.of("STRING"))); + public void characterTypesKeepDeclaredBounds() { + // WHY: the declared length is part of the Paimon schema; Paimon truncates to it when it casts + // a historical data file to the current column type. MUTATION: collapsing CHAR/VARCHAR to + // VarChar(MAX), or mapping CHAR to VarCharType, makes these red. + Assertions.assertEquals(new CharType(3), PaimonTypeMapping.toPaimonType(ConnectorType.of("CHAR", 3, 0))); + Assertions.assertEquals(new VarCharType(3), + PaimonTypeMapping.toPaimonType(ConnectorType.of("VARCHAR", 3, 0))); + Assertions.assertEquals(new VarCharType(VarCharType.MAX_LENGTH), + PaimonTypeMapping.toPaimonType(ConnectorType.of("STRING"))); } @Test public void datetimePreservesPrecisionAndTimestampKind() { - Assertions.assertEquals(new TimestampType(3), - PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIMEV2", 3, 0))); - Assertions.assertEquals(new TimestampType(), + for (int precision : new int[] {0, 3, 6}) { + Assertions.assertEquals(new TimestampType(precision), + PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIMEV2", precision, 0))); + } + // DATETIME (v1) has no fractional seconds. + Assertions.assertEquals(new TimestampType(0), PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIME"))); Assertions.assertEquals(new org.apache.paimon.types.LocalZonedTimestampType(6), PaimonTypeMapping.toPaimonType(ConnectorType.of("TIMESTAMPTZ", 6, 0))); } + @Test + @SuppressWarnings("unchecked") + public void mappedTypesDriveHistoricalCasts() { + // WHY: Paimon reads a data file written under an older schema by casting the file type to the + // current column type, so the mapped bounds decide the values a reader sees. + CastExecutor varcharCast = (CastExecutor) CastExecutors.resolve( + new VarCharType(10), PaimonTypeMapping.toPaimonType(ConnectorType.of("VARCHAR", 3, 0))); + Assertions.assertNotNull(varcharCast); + Assertions.assertEquals("abc", varcharCast.cast(BinaryString.fromString("abcdef")).toString()); + CastExecutor timestampCast = (CastExecutor) CastExecutors.resolve( + new VarCharType(VarCharType.MAX_LENGTH), + PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIMEV2", 3, 0))); + Assertions.assertNotNull(timestampCast); + Assertions.assertEquals("1970-01-01T00:00:00.042", + timestampCast.cast(BinaryString.fromString("42")).toString()); + } + + @Test + public void nestedTypesKeepBoundsAndPrecision() { + Assertions.assertEquals(new ArrayType(new VarCharType(3)), + PaimonTypeMapping.toPaimonType(ConnectorType.arrayOf(ConnectorType.of("VARCHAR", 3, 0)))); + Assertions.assertEquals(new MapType(new CharType(false, 2), new TimestampType(3)), + PaimonTypeMapping.toPaimonType(ConnectorType.mapOf( + ConnectorType.of("CHAR", 2, 0), ConnectorType.of("DATETIMEV2", 3, 0)))); + } + @Test public void decimalCarriesPrecisionAndScale() { // WHY: every decimal family member carries precision/scale through verbatim. MUTATION: