From bd8676232a50c3d6b1c60fb57274f81e2d9d176f Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 13 Oct 2026 22:42:17 +0100 Subject: [PATCH] [fix](paimon) Keep CHAR/VARCHAR lengths and DATETIME precision when creating Paimon tables ### What problem does this PR solve? Issue Number: None Related PR: #66227 Problem Summary: Ported from the DDL type mapping change in branch-4.1 #66227. When Doris creates or alters a Paimon table, PaimonTypeMapping.toPaimonType turns the declared column type into the Paimon schema. On master CHAR(n), VARCHAR(n) and STRING all became VARCHAR(2147483647), and DATETIME became TIMESTAMP(6). Paimon uses these bounds when it reads a data file written under an older schema: it casts the file's column type to the current one, truncating to the declared length and reading numeric timestamp strings in units of the declared precision. With the bounds dropped, the Paimon table did not have the schema the user declared, and Doris read the column back as STRING. CHAR(n) now maps to CHAR(n), VARCHAR(n) to VARCHAR(n), STRING to VARCHAR(2147483647), DATETIME to TIMESTAMP(0), and DATETIMEV2(p) to TIMESTAMP(p) as before, also inside ARRAY, MAP and STRUCT, as on branch-4.1 and branch-4.2. ### Release note Paimon tables created by Doris keep the declared CHAR(n) / VARCHAR(n) length; DATETIME maps to TIMESTAMP(0). ### Check List (For Author) - Test: Unit Test - PaimonTypeMappingToPaimonTest: bounds, precision, nested types, and Paimon's own casts on the mapped types - Behavior changed: Yes. CHAR(n) / VARCHAR(n) columns of Paimon tables created by Doris keep their length instead of becoming STRING. - Does this need documentation: No Co-authored-by: Fangyuan Deng <982092332@qq.com> Co-Authored-By: Claude Opus 5.5 (1M context) --- .../connector/paimon/PaimonTypeMapping.java | 20 +++---- .../paimon/PaimonTypeMappingToPaimonTest.java | 58 +++++++++++++++---- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java index 65cc58ab1db621..a26d4468e87857 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonTypeMapping.java @@ -209,13 +209,10 @@ private static ConnectorType toStructType(RowType rowType, Options options) { * Convert a Doris {@link ConnectorType} (as produced by the CREATE TABLE request path) * to a Paimon {@link DataType}. * - *

This is the faithful reverse of the legacy fe-core - * {@code DorisToPaimonTypeVisitor}: the scalar set is intentionally narrow (it mirrors - * the visitor's {@code atomic} branches and NOT MaxCompute's richer set), CHAR/VARCHAR/STRING - * all collapse to {@code VarChar(MAX)} (declared length dropped), DATETIME/DATETIMEV2 map to a - * plain {@code TimestampType()} (scale dropped), and the MAP key is forced non-null. Types the - * legacy visitor did not handle (TINYINT, SMALLINT, LARGEINT, TIME, IPV4/6, JSON, ...) throw, - * preserving the legacy gap.

+ *

This is the connector-SPI equivalent of the former fe-core + * {@code DorisToPaimonTypeVisitor}. It preserves declared CHAR/VARCHAR bounds and timestamp + * precision because Paimon uses them when applying casts and interpreting numeric timestamp + * values. The MAP key remains non-null, matching Paimon's type contract.

* *

The returned type carries Paimon's default (nullable) flag; column-level nullability is * applied by the caller via {@code .copy(nullable)} (mirroring legacy @@ -243,10 +240,11 @@ public static DataType toPaimonType(ConnectorType type) { case "DOUBLE": return new DoubleType(); case "CHAR": + // DDL must retain bounds: widening to STRING changes historical narrowing casts. + return new CharType(type.getPrecision()); case "VARCHAR": + return new VarCharType(type.getPrecision()); case "STRING": - // Legacy parity: all char-family types collapse to VarChar(MAX); declared - // length is intentionally dropped (DorisToPaimonTypeVisitor.atomic isCharFamily). return new VarCharType(VarCharType.MAX_LENGTH); case "DATE": case "DATEV2": @@ -259,9 +257,9 @@ public static DataType toPaimonType(ConnectorType type) { case "DECIMAL256": return new DecimalType(type.getPrecision(), type.getScale()); case "DATETIME": - return new TimestampType(); case "DATETIMEV2": - return new TimestampType(type.getPrecision()); + // Paimon interprets numeric timestamp strings in units derived from this precision. + return new TimestampType("DATETIMEV2".equals(name) ? type.getPrecision() : 0); case "TIMESTAMPTZ": // Preserve the instant contract when deriving an external schema. return new LocalZonedTimestampType(type.getPrecision()); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java index af0aad304742cb..055709ac090ceb 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTypeMappingToPaimonTest.java @@ -20,9 +20,13 @@ import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.paimon.casting.CastExecutor; +import org.apache.paimon.casting.CastExecutors; +import org.apache.paimon.data.BinaryString; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DateType; @@ -46,7 +50,7 @@ /** * P5-T11 — pins the Doris->Paimon reverse type mapping in - * {@link PaimonTypeMapping#toPaimonType} to byte-parity with the legacy fe-core + * {@link PaimonTypeMapping#toPaimonType}, the connector equivalent of the former fe-core * {@code DorisToPaimonTypeVisitor}. * *

The CREATE TABLE path produces these {@link ConnectorType} descriptors; this mapping is @@ -74,26 +78,56 @@ public void scalarPrimitivesMapExactly() { } @Test - public void charFamilyCollapsesToVarcharMaxDroppingLength() { - // WHY: legacy isCharFamily -> VarCharType(MAX_LENGTH) unconditionally; the declared length - // is intentionally dropped and CHAR is NOT mapped to paimon CharType. MUTATION: honoring - // the declared length (e.g. new VarCharType(10)) or mapping CHAR -> CharType makes these red. - DataType expected = new VarCharType(VarCharType.MAX_LENGTH); - Assertions.assertEquals(expected, PaimonTypeMapping.toPaimonType(ConnectorType.of("CHAR", 10, 0))); - Assertions.assertEquals(expected, PaimonTypeMapping.toPaimonType(ConnectorType.of("VARCHAR", 20, 0))); - Assertions.assertEquals(expected, PaimonTypeMapping.toPaimonType(ConnectorType.of("STRING"))); + public void characterTypesKeepDeclaredBounds() { + // WHY: the declared length is part of the Paimon schema; Paimon truncates to it when it casts + // a historical data file to the current column type. MUTATION: collapsing CHAR/VARCHAR to + // VarChar(MAX), or mapping CHAR to VarCharType, makes these red. + Assertions.assertEquals(new CharType(3), PaimonTypeMapping.toPaimonType(ConnectorType.of("CHAR", 3, 0))); + Assertions.assertEquals(new VarCharType(3), + PaimonTypeMapping.toPaimonType(ConnectorType.of("VARCHAR", 3, 0))); + Assertions.assertEquals(new VarCharType(VarCharType.MAX_LENGTH), + PaimonTypeMapping.toPaimonType(ConnectorType.of("STRING"))); } @Test public void datetimePreservesPrecisionAndTimestampKind() { - Assertions.assertEquals(new TimestampType(3), - PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIMEV2", 3, 0))); - Assertions.assertEquals(new TimestampType(), + for (int precision : new int[] {0, 3, 6}) { + Assertions.assertEquals(new TimestampType(precision), + PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIMEV2", precision, 0))); + } + // DATETIME (v1) has no fractional seconds. + Assertions.assertEquals(new TimestampType(0), PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIME"))); Assertions.assertEquals(new org.apache.paimon.types.LocalZonedTimestampType(6), PaimonTypeMapping.toPaimonType(ConnectorType.of("TIMESTAMPTZ", 6, 0))); } + @Test + @SuppressWarnings("unchecked") + public void mappedTypesDriveHistoricalCasts() { + // WHY: Paimon reads a data file written under an older schema by casting the file type to the + // current column type, so the mapped bounds decide the values a reader sees. + CastExecutor varcharCast = (CastExecutor) CastExecutors.resolve( + new VarCharType(10), PaimonTypeMapping.toPaimonType(ConnectorType.of("VARCHAR", 3, 0))); + Assertions.assertNotNull(varcharCast); + Assertions.assertEquals("abc", varcharCast.cast(BinaryString.fromString("abcdef")).toString()); + CastExecutor timestampCast = (CastExecutor) CastExecutors.resolve( + new VarCharType(VarCharType.MAX_LENGTH), + PaimonTypeMapping.toPaimonType(ConnectorType.of("DATETIMEV2", 3, 0))); + Assertions.assertNotNull(timestampCast); + Assertions.assertEquals("1970-01-01T00:00:00.042", + timestampCast.cast(BinaryString.fromString("42")).toString()); + } + + @Test + public void nestedTypesKeepBoundsAndPrecision() { + Assertions.assertEquals(new ArrayType(new VarCharType(3)), + PaimonTypeMapping.toPaimonType(ConnectorType.arrayOf(ConnectorType.of("VARCHAR", 3, 0)))); + Assertions.assertEquals(new MapType(new CharType(false, 2), new TimestampType(3)), + PaimonTypeMapping.toPaimonType(ConnectorType.mapOf( + ConnectorType.of("CHAR", 2, 0), ConnectorType.of("DATETIMEV2", 3, 0)))); + } + @Test public void decimalCarriesPrecisionAndScale() { // WHY: every decimal family member carries precision/scale through verbatim. MUTATION: