From 294d1f74f66eaee684cd052676ecbe68ca89acd3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 16 Oct 2026 13:11:35 +0800 Subject: [PATCH 1/4] [fix](external) Backport missing type mapping behavior to branch-4.2 --- .../iceberg/IcebergSysTableColumnValue.java | 36 +++- .../iceberg/IcebergSysTableJniScanner.java | 2 +- .../IcebergSysTableColumnValueTest.java | 80 +++++++++ .../jdbc/client/JdbcOracleClient.java | 6 +- .../jdbc/client/JdbcSQLServerClient.java | 129 +++++++++++++- .../ExternalFileTableValuedFunction.java | 9 +- .../jdbc/client/JdbcOracleClientTest.java | 54 ++++++ .../jdbc/client/JdbcSQLServerClientTest.java | 166 ++++++++++++++++++ .../ExternalFileTableValuedFunctionTest.java | 33 ++-- ...test_iceberg_files_timestamp_partition.out | 72 ++++++++ .../write/test_iceberg_write_stats2.out | 4 +- .../test_hdfs_orc_group0_orc_files.out | Bin 101666 -> 103658 bytes .../test_hdfs_orc_group5_orc_files.out | 12 +- .../test_hdfs_orc_group6_orc_files.out | Bin 10112 -> 10141 bytes .../test_hdfs_orc_group7_orc_files.out | Bin 654 -> 663 bytes .../tvf/test_hdfs_parquet_group0.out | Bin 26390 -> 28142 bytes .../tvf/test_hdfs_parquet_group2.out | 72 ++++---- .../tvf/test_hdfs_parquet_group3.out | Bin 11347 -> 11459 bytes .../tvf/test_hdfs_parquet_group4.out | Bin 105548 -> 105672 bytes .../tvf/test_hdfs_parquet_group5.out | Bin 613238 -> 613850 bytes .../tvf/test_hdfs_parquet_group6.out | 48 ++--- ...t_iceberg_files_timestamp_partition.groovy | 72 ++++++++ .../tvf/test_tvf_binary_mapping.groovy | 69 ++++++++ 23 files changed, 752 insertions(+), 112 deletions(-) create mode 100644 fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClientTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClientTest.java create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.groovy create mode 100644 regression-test/suites/external_table_p0/tvf/test_tvf_binary_mapping.groovy diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java index 1e404809b7737d..2fe0c5ba5742a0 100644 --- a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java +++ b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java @@ -20,6 +20,8 @@ import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.iceberg.StructLike; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; import java.math.BigDecimal; import java.math.BigInteger; @@ -38,14 +40,20 @@ public class IcebergSysTableColumnValue implements ColumnValue { private final Object fieldData; private final String timezone; + private final Type icebergType; public IcebergSysTableColumnValue(Object fieldData) { this(fieldData, DEFAULT_TIME_ZONE); } public IcebergSysTableColumnValue(Object fieldData, String timezone) { + this(fieldData, timezone, null); + } + + public IcebergSysTableColumnValue(Object fieldData, String timezone, Type icebergType) { this.fieldData = fieldData; this.timezone = timezone; + this.icebergType = icebergType; } @Override @@ -136,14 +144,22 @@ public LocalDate getDate() { @Override public LocalDateTime getDateTime() { - Instant instant = Instant.ofEpochMilli((((long) fieldData) / 1000)); - return LocalDateTime.ofInstant(instant, ZoneId.of(timezone)); + // NTZ is a wall-clock value; only timestamptz mapped to DATETIME uses the session zone. + ZoneId zone = icebergType instanceof Types.TimestampType + && !((Types.TimestampType) icebergType).shouldAdjustToUTC() + ? ZoneOffset.UTC : ZoneId.of(timezone); + return LocalDateTime.ofInstant(timestampInstant(), zone); } @Override public LocalDateTime getTimeStampTz() { - Instant instant = Instant.ofEpochMilli((((long) fieldData) / 1000)); - return LocalDateTime.ofInstant(instant, ZoneId.of("UTC")); + return LocalDateTime.ofInstant(timestampInstant(), ZoneOffset.UTC); + } + + private Instant timestampInstant() { + long micros = (long) fieldData; + return Instant.ofEpochSecond(Math.floorDiv(micros, 1_000_000L), + Math.floorMod(micros, 1_000_000L) * 1_000L); } @Override @@ -162,7 +178,8 @@ public byte[] getBytes() { public void unpackArray(List values) { List items = (List) fieldData; for (Object item : items) { - values.add(new IcebergSysTableColumnValue(item, timezone)); + values.add(new IcebergSysTableColumnValue(item, timezone, + icebergType == null ? null : icebergType.asListType().elementType())); } } @@ -170,8 +187,10 @@ public void unpackArray(List values) { public void unpackMap(List keys, List values) { Map data = (Map) fieldData; data.forEach((key, value) -> { - keys.add(new IcebergSysTableColumnValue(key, timezone)); - values.add(new IcebergSysTableColumnValue(value, timezone)); + keys.add(new IcebergSysTableColumnValue(key, timezone, + icebergType == null ? null : icebergType.asMapType().keyType())); + values.add(new IcebergSysTableColumnValue(value, timezone, + icebergType == null ? null : icebergType.asMapType().valueType())); }); } @@ -180,7 +199,8 @@ public void unpackStruct(List structFieldIndex, List value StructLike record = (StructLike) fieldData; for (Integer fieldIndex : structFieldIndex) { Object rawValue = record.get(fieldIndex, Object.class); - values.add(new IcebergSysTableColumnValue(rawValue, timezone)); + values.add(new IcebergSysTableColumnValue(rawValue, timezone, + icebergType == null ? null : icebergType.asStructType().fields().get(fieldIndex).type())); } } } diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java index b014a42706ff29..a35df84920868f 100644 --- a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java +++ b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java @@ -109,7 +109,7 @@ protected int getNext() throws IOException { for (int i = 0; i < fields.size(); i++) { SelectedField field = fields.get(i); Object value = row.get(field.sourceIndex, field.field.type().typeId().javaClass()); - ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone); + ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone, field.field.type()); appendData(i, columnValue); } rows++; diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java b/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java new file mode 100644 index 00000000000000..c18d51517bba36 --- /dev/null +++ b/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.iceberg; + +import org.apache.doris.common.jni.vec.ColumnValue; + +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class IcebergSysTableColumnValueTest { + @Test + public void preservesMicrosAndTimestampSemantics() { + long[] micros = {-1000001L, -1L, 1L, 999L, 1000L, 1234567L}; + String[] expected = {"1969-12-31T23:59:58.999999", "1969-12-31T23:59:59.999999", + "1970-01-01T00:00:00.000001", "1970-01-01T00:00:00.000999", + "1970-01-01T00:00:00.001", "1970-01-01T00:00:01.234567"}; + for (int i = 0; i < micros.length; i++) { + LocalDateTime utc = LocalDateTime.parse(expected[i]); + for (String zone : Arrays.asList("UTC", "Asia/Shanghai")) { + ColumnValue ntz = new IcebergSysTableColumnValue(micros[i], zone, Types.TimestampType.withoutZone()); + ColumnValue tz = new IcebergSysTableColumnValue(micros[i], zone, Types.TimestampType.withZone()); + Assertions.assertEquals(utc, ntz.getDateTime()); + Assertions.assertEquals(zone.equals("UTC") ? utc : utc.plusHours(8), tz.getDateTime()); + Assertions.assertEquals(utc, tz.getTimeStampTz()); + } + } + } + + @Test + public void propagatesTypesThroughNestedContainers() { + Types.StructType struct = Types.StructType.of( + Types.NestedField.optional(1, "ts", Types.TimestampType.withoutZone()), + Types.NestedField.optional(2, "tz", Types.TimestampType.withZone())); + PartitionData record = new PartitionData(struct); + record.set(0, -1L); + record.set(1, -1L); + List fields = new ArrayList<>(); + new IcebergSysTableColumnValue(record, "Asia/Shanghai", struct) + .unpackStruct(Arrays.asList(1, 0), fields); + LocalDateTime utc = LocalDateTime.parse("1969-12-31T23:59:59.999999"); + Assertions.assertEquals(utc.plusHours(8), fields.get(0).getDateTime()); + Assertions.assertEquals(utc, fields.get(1).getDateTime()); + + List elements = new ArrayList<>(); + new IcebergSysTableColumnValue(Collections.singletonList(-1L), "Asia/Shanghai", + Types.ListType.ofOptional(3, Types.TimestampType.withoutZone())).unpackArray(elements); + Assertions.assertEquals(utc, elements.get(0).getDateTime()); + + List keys = new ArrayList<>(); + List values = new ArrayList<>(); + new IcebergSysTableColumnValue(Collections.singletonMap(-1L, -1L), "Asia/Shanghai", + Types.MapType.ofOptional(4, 5, Types.TimestampType.withoutZone(), Types.TimestampType.withZone())) + .unpackMap(keys, values); + Assertions.assertEquals(utc, keys.get(0).getDateTime()); + Assertions.assertEquals(utc.plusHours(8), values.get(0).getDateTime()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 170db84b0d5da5..27a09005008610 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -208,6 +208,10 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { precision = scale; } return createDecimalOrStringType(precision, scale); + // Oracle binary floating-point types retain IEEE precision instead of becoming unsupported. + case "BINARY_FLOAT": + return Type.FLOAT; + case "BINARY_DOUBLE": case "FLOAT": return Type.DOUBLE; case "DATE": @@ -227,8 +231,6 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { return ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize()); case "NCLOB": case "BFILE": - case "BINARY_FLOAT": - case "BINARY_DOUBLE": default: return Type.UNSUPPORTED; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java index 35edd6c96babd3..e3300d8ce5b913 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java @@ -21,8 +21,19 @@ import org.apache.doris.catalog.Type; import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; +import java.sql.Types; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class JdbcSQLServerClient extends JdbcClient { + // TYPE_NAME of an IDENTITY column decorates the base type: "int identity", "decimal() identity", + // "numeric(18, 0) identity", "decimal(18,0) IDENTITY(1,1)". IDENTITY is only allowed on these base types. + private static final Pattern IDENTITY_TYPE_NAME = Pattern.compile( + "^(tinyint|smallint|int|bigint|decimal|numeric)\\s*(\\([^)]*\\))?\\s+identity(\\s*\\([^)]*\\))?$", + Pattern.CASE_INSENSITIVE); + protected JdbcSQLServerClient(JdbcClientConfig jdbcClientConfig) { super(jdbcClientConfig); } @@ -34,18 +45,48 @@ protected String getSchemaPatternForDatabaseNameList() { return "%"; } + /** + * The base type of an IDENTITY column's TYPE_NAME, or the name unchanged. + *

+ * The decoration is trusted only when {@code DATA_TYPE} is the code of the named base type: an alias type + * may legally be named like that ({@code CREATE TYPE dbo.[int identity] FROM varchar(10)}, or + * {@code dbo.[int alias]}), and it then has to be resolved by its code, not by the words of its name. + */ + static String identityBaseType(String typeName, int dataType) { + Matcher matcher = IDENTITY_TYPE_NAME.matcher(typeName); + if (!matcher.matches()) { + return typeName; + } + String baseType = matcher.group(1).toLowerCase(Locale.ROOT); + boolean codeMatches; + switch (baseType) { + case "tinyint": + codeMatches = dataType == Types.TINYINT; + break; + case "smallint": + codeMatches = dataType == Types.SMALLINT; + break; + case "int": + codeMatches = dataType == Types.INTEGER; + break; + case "bigint": + codeMatches = dataType == Types.BIGINT; + break; + default: + codeMatches = dataType == Types.DECIMAL || dataType == Types.NUMERIC; + break; + } + return codeMatches ? baseType : typeName; + } + @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { String originSqlserverType = fieldSchema.getDataTypeName().orElse("unknown"); - // For sqlserver IDENTITY type, such as 'INT IDENTITY' - // originSqlserverType is "int identity", so we only get "int". - // For types with parameters like 'decimal(18,0) IDENTITY(1,1)', we need to extract the base type - String sqlserverType = originSqlserverType.split(" ")[0]; - - // Handle types with parentheses like decimal(18,0), varchar(50), etc. - if (sqlserverType.contains("(")) { - sqlserverType = sqlserverType.substring(0, sqlserverType.indexOf("(")); - } + // An IDENTITY column is reported as "int identity" or "decimal(18,0) identity": only the base type + // is matched below. Any other name is matched as it is: system type names are single words, and a + // user-defined alias type may be named with spaces or parentheses ("int alias") and must not be + // mistaken for the system type its name starts with. + String sqlserverType = identityBaseType(originSqlserverType, fieldSchema.getDataType()); switch (sqlserverType) { case "bit": @@ -100,6 +141,76 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { case "binary": case "varbinary": return ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize()); + case "xml": + case "sql_variant": + case "geometry": + case "geography": + case "hierarchyid": + case "json": + case "vector": + // SQL Server system types that Doris does not support. They are listed explicitly + // so that they never reach the JDBC type code fallback below. + return Type.UNSUPPORTED; + default: + return jdbcTypeCodeToDoris(fieldSchema); + } + } + + /** + * Fallback for type names that are not SQL Server system types. + *

+ * User-defined alias types ({@code CREATE TYPE dbo.my_type FROM varchar(50)}) are reported by + * {@code DatabaseMetaData.getColumns()} with {@code TYPE_NAME} set to the alias name, so they can not be + * matched by name. {@code DATA_TYPE}, {@code COLUMN_SIZE} and {@code DECIMAL_DIGITS} still describe the + * base type, so the standard {@link Types} code is used to resolve the Doris type. The mapping mirrors + * the name based one above. + *

+ * Binary codes are deliberately not mapped: mssql-jdbc also reports CLR user-defined types + * (geometry, geography, hierarchyid, ...) as {@link Types#VARBINARY}, so they can not be told apart from + * an alias over a binary type by the type code alone. Vendor specific codes stay unsupported as well. + */ + private Type jdbcTypeCodeToDoris(JdbcFieldSchema fieldSchema) { + switch (fieldSchema.getDataType()) { + case Types.BIT: + case Types.BOOLEAN: + return Type.BOOLEAN; + // SQL Server tinyint is unsigned (0 to 255), so it needs SMALLINT + case Types.TINYINT: + case Types.SMALLINT: + return Type.SMALLINT; + case Types.INTEGER: + return Type.INT; + case Types.BIGINT: + return Type.BIGINT; + case Types.REAL: + return Type.FLOAT; + case Types.FLOAT: + case Types.DOUBLE: + return Type.DOUBLE; + case Types.DECIMAL: + case Types.NUMERIC: { + // money and smallmoney are reported as DECIMAL(19,4) and DECIMAL(10,4) + int precision = fieldSchema.getColumnSize().orElse(0); + int scale = fieldSchema.getDecimalDigits().orElse(0); + return createDecimalOrStringType(precision, scale); + } + case Types.DATE: + return ScalarType.createDateV2Type(); + case Types.TIMESTAMP: { + int scale = fieldSchema.getDecimalDigits().orElse(0); + if (scale > 6) { + scale = 6; + } + return ScalarType.createDatetimeV2Type(scale); + } + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + case Types.NCHAR: + case Types.NVARCHAR: + case Types.LONGNVARCHAR: + case Types.TIME: + return ScalarType.createStringType(); default: return Type.UNSUPPORTED; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 41466e0648a718..398cbb4e02cc25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -253,12 +253,9 @@ && getTFileType() != TFileType.FILE_S3) { throw new AnalysisException("Lance format is supported only by local() and s3() TVFs"); } - // The catalog property was removed, but this TVF-only option must remain explicit because - // changing an ad-hoc TVF result schema also breaks CTAS type inference. - String enableMappingVarbinaryStr = getOrDefaultAndRemove(copiedProps, - FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY, "false"); - fileFormatProperties.enableMappingVarbinary = Boolean.parseBoolean(enableMappingVarbinaryStr); - + // Binary file fields must preserve bytes even when a legacy opt-out is supplied. + copiedProps.remove(FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY); + fileFormatProperties.enableMappingVarbinary = true; // Consume the legacy option, but let file logical types determine timezone semantics. copiedProps.remove(FileFormatConstants.PROP_ENABLE_MAPPING_TIMESTAMP_TZ); fileFormatProperties.enableMappingTimestampTz = true; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClientTest.java new file mode 100644 index 00000000000000..6e14a38e95dc26 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClientTest.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.jdbc.client; + +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.mockito.Mockito; + +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; + +public class JdbcOracleClientTest { + private final JdbcOracleClient client = Mockito.mock(JdbcOracleClient.class, Answers.CALLS_REAL_METHODS); + + private static JdbcFieldSchema column(String typeName, int dataType) throws SQLException { + ResultSet resultSet = Mockito.mock(ResultSet.class); + Mockito.when(resultSet.getString("COLUMN_NAME")).thenReturn("col"); + Mockito.when(resultSet.getInt("DATA_TYPE")).thenReturn(dataType); + Mockito.when(resultSet.getString("TYPE_NAME")).thenReturn(typeName); + Mockito.when(resultSet.getInt("COLUMN_SIZE")).thenReturn(8); + Mockito.when(resultSet.getInt("DECIMAL_DIGITS")).thenReturn(0); + Mockito.when(resultSet.getInt("NULLABLE")).thenReturn(DatabaseMetaData.columnNullable); + return new JdbcFieldSchema(resultSet); + } + + @Test + public void testOracleBinaryFloatingPointTypes() throws SQLException { + Assertions.assertEquals(Type.FLOAT, + client.jdbcTypeToDoris(column("BINARY_FLOAT", Types.FLOAT))); + Assertions.assertEquals(Type.DOUBLE, + client.jdbcTypeToDoris(column("BINARY_DOUBLE", Types.DOUBLE))); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClientTest.java new file mode 100644 index 00000000000000..fa184a22218c6a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClientTest.java @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.jdbc.client; + +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.mockito.Mockito; + +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; + +public class JdbcSQLServerClientTest { + + // ODBC type codes that mssql-jdbc passes through DatabaseMetaData.getColumns() unchanged + private static final int SQL_VARIANT = -150; + private static final int SQL_SS_TIMESTAMPOFFSET = -155; + + private final JdbcSQLServerClient client = Mockito.mock(JdbcSQLServerClient.class, Answers.CALLS_REAL_METHODS); + + /** + * Builds the schema of one DatabaseMetaData.getColumns() row as reported by mssql-jdbc. + * For a user-defined alias type, TYPE_NAME is the alias name while DATA_TYPE, COLUMN_SIZE + * and DECIMAL_DIGITS describe the base type. + */ + private static JdbcFieldSchema column(String typeName, int dataType, int columnSize, int decimalDigits) + throws SQLException { + ResultSet rs = Mockito.mock(ResultSet.class); + Mockito.when(rs.getString("COLUMN_NAME")).thenReturn("col"); + Mockito.when(rs.getInt("DATA_TYPE")).thenReturn(dataType); + Mockito.when(rs.getString("TYPE_NAME")).thenReturn(typeName); + Mockito.when(rs.getInt("COLUMN_SIZE")).thenReturn(columnSize); + Mockito.when(rs.getInt("DECIMAL_DIGITS")).thenReturn(decimalDigits); + Mockito.when(rs.getInt("NULLABLE")).thenReturn(DatabaseMetaData.columnNullable); + return new JdbcFieldSchema(rs); + } + + @Test + public void testAliasTypeIsResolvedByJdbcTypeCode() throws SQLException { + // CREATE TYPE dbo.customtexttype FROM varchar(50), the case reported in #67793 + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("customtexttype", Types.VARCHAR, 50, 0))); + // sysname is a built-in alias over nvarchar(128) + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("sysname", Types.NVARCHAR, 128, 0))); + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("alias_nchar", Types.NCHAR, 10, 0))); + Assertions.assertEquals(Type.STRING, + client.jdbcTypeToDoris(column("alias_text", Types.LONGVARCHAR, Integer.MAX_VALUE, 0))); + Assertions.assertEquals(Type.STRING, + client.jdbcTypeToDoris(column("alias_ntext", Types.LONGNVARCHAR, Integer.MAX_VALUE / 2, 0))); + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("alias_time", Types.TIME, 16, 7))); + // uniqueidentifier is reported as CHAR(36) + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("alias_guid", Types.CHAR, 36, 0))); + + Assertions.assertEquals(Type.BOOLEAN, client.jdbcTypeToDoris(column("alias_bit", Types.BIT, 1, 0))); + // SQL Server tinyint is unsigned, so it keeps the SMALLINT mapping of the name based path + Assertions.assertEquals(Type.SMALLINT, client.jdbcTypeToDoris(column("alias_tinyint", Types.TINYINT, 3, 0))); + Assertions.assertEquals(Type.SMALLINT, client.jdbcTypeToDoris(column("alias_smallint", Types.SMALLINT, 5, 0))); + Assertions.assertEquals(Type.INT, client.jdbcTypeToDoris(column("alias_int", Types.INTEGER, 10, 0))); + Assertions.assertEquals(Type.BIGINT, client.jdbcTypeToDoris(column("alias_bigint", Types.BIGINT, 19, 0))); + Assertions.assertEquals(Type.FLOAT, client.jdbcTypeToDoris(column("alias_real", Types.REAL, 24, 0))); + Assertions.assertEquals(Type.DOUBLE, client.jdbcTypeToDoris(column("alias_float", Types.DOUBLE, 53, 0))); + + Assertions.assertEquals(ScalarType.createDecimalV3Type(10, 2), + client.jdbcTypeToDoris(column("alias_decimal", Types.DECIMAL, 10, 2))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(38, 10), + client.jdbcTypeToDoris(column("alias_numeric", Types.NUMERIC, 38, 10))); + // money is reported as DECIMAL(19,4), the same as the name based mapping produces + Assertions.assertEquals(ScalarType.createDecimalV3Type(19, 4), + client.jdbcTypeToDoris(column("alias_money", Types.DECIMAL, 19, 4))); + + Assertions.assertEquals(Type.DATEV2, client.jdbcTypeToDoris(column("alias_date", Types.DATE, 10, 0))); + Assertions.assertEquals(ScalarType.createDatetimeV2Type(3), + client.jdbcTypeToDoris(column("alias_datetime", Types.TIMESTAMP, 23, 3))); + // datetime2 defaults to 7 fractional digits, Doris supports at most 6 + Assertions.assertEquals(ScalarType.createDatetimeV2Type(6), + client.jdbcTypeToDoris(column("alias_datetime2", Types.TIMESTAMP, 27, 7))); + Assertions.assertEquals(ScalarType.createDatetimeV2Type(0), + client.jdbcTypeToDoris(column("alias_smalldatetime", Types.TIMESTAMP, 16, 0))); + } + + @Test + public void testUnknownTypesStayUnsupported() throws SQLException { + // CLR user-defined types are reported as VARBINARY, exactly like an alias over varbinary, + // so binary codes must not be resolved by the fallback + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("geometry", Types.VARBINARY, Integer.MAX_VALUE, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("my_clr_type", Types.VARBINARY, 8000, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, client.jdbcTypeToDoris(column("alias_binary", Types.BINARY, 20, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("alias_image", Types.LONGVARBINARY, Integer.MAX_VALUE, 0))); + // vendor specific type codes + Assertions.assertEquals(Type.UNSUPPORTED, client.jdbcTypeToDoris(column("sql_variant", SQL_VARIANT, 8000, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("alias_datetimeoffset", SQL_SS_TIMESTAMPOFFSET, 34, 7))); + // explicitly unsupported system types keep that behavior whatever type code the driver reports + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("xml", Types.LONGNVARCHAR, Integer.MAX_VALUE / 2, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("json", Types.LONGNVARCHAR, Integer.MAX_VALUE / 2, 0))); + Assertions.assertEquals(Type.UNSUPPORTED, + client.jdbcTypeToDoris(column("hierarchyid", Types.VARBINARY, 892, 0))); + } + + @Test + public void testAliasNamedLikeASystemTypeIsResolvedByJdbcTypeCode() throws SQLException { + // A delimited alias name may contain spaces and parentheses ([int alias], [decimal(18,0) identity]); + // it is reported as is, and the base type is still what DATA_TYPE says + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("int alias", Types.VARCHAR, 50, 0))); + Assertions.assertEquals(Type.STRING, + client.jdbcTypeToDoris(column("decimal(18,0) identity", Types.NVARCHAR, 20, 0))); + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("int identity", Types.VARCHAR, 10, 0))); + Assertions.assertEquals(Type.STRING, + client.jdbcTypeToDoris(column("bigint identity", Types.NVARCHAR, 20, 0))); + Assertions.assertEquals(ScalarType.createDatetimeV2Type(3), + client.jdbcTypeToDoris(column("varchar(50) alias", Types.TIMESTAMP, 23, 3))); + + // The IDENTITY decoration of a real system type, in the forms the driver versions report it in + Assertions.assertEquals(Type.INT, client.jdbcTypeToDoris(column("int identity", Types.INTEGER, 10, 0))); + Assertions.assertEquals(Type.BIGINT, client.jdbcTypeToDoris(column("bigint identity", Types.BIGINT, 19, 0))); + Assertions.assertEquals(Type.SMALLINT, + client.jdbcTypeToDoris(column("tinyint identity", Types.TINYINT, 3, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(18, 0), + client.jdbcTypeToDoris(column("decimal identity", Types.DECIMAL, 18, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(18, 0), + client.jdbcTypeToDoris(column("decimal() identity", Types.DECIMAL, 18, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(18, 0), + client.jdbcTypeToDoris(column("numeric(18, 0) identity", Types.NUMERIC, 18, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(18, 0), + client.jdbcTypeToDoris(column("decimal(18,0) IDENTITY(1,1)", Types.DECIMAL, 18, 0))); + } + + @Test + public void testSystemTypeNamesTakePrecedence() throws SQLException { + // the name based mapping is unchanged, the type code is only consulted for unknown names + Assertions.assertEquals(Type.SMALLINT, client.jdbcTypeToDoris(column("tinyint", Types.TINYINT, 3, 0))); + Assertions.assertEquals(Type.INT, client.jdbcTypeToDoris(column("int identity", Types.INTEGER, 10, 0))); + Assertions.assertEquals(ScalarType.createDecimalV3Type(19, 4), + client.jdbcTypeToDoris(column("money", Types.DECIMAL, 19, 4))); + Assertions.assertEquals(ScalarType.createVarbinaryType(20), + client.jdbcTypeToDoris(column("varbinary", Types.VARBINARY, 20, 0))); + Assertions.assertEquals(Type.STRING, client.jdbcTypeToDoris(column("timestamp", Types.BINARY, 8, 0))); + Assertions.assertEquals(ScalarType.createTimeStampTzType(6), + client.jdbcTypeToDoris(column("datetimeoffset", SQL_SS_TIMESTAMPOFFSET, 34, 7))); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java index cf03914f260299..7072738061667a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java @@ -99,24 +99,21 @@ public void testHiveParquetTimeZoneRejectsAmbiguousShortAlias() { } @Test - public void testTvfVarbinaryMappingRemainsExplicit() throws AnalysisException { - ExternalFileTableValuedFunction defaultTvf = Mockito.mock( - ExternalFileTableValuedFunction.class, Mockito.CALLS_REAL_METHODS); - Map defaultProperties = Maps.newHashMap(); - defaultProperties.put(FileFormatConstants.PROP_FORMAT, FileFormatConstants.FORMAT_PARQUET); - - defaultTvf.parseCommonProperties(defaultProperties); - - Assert.assertFalse(defaultTvf.fileFormatProperties.enableMappingVarbinary); - - ExternalFileTableValuedFunction enabledTvf = Mockito.mock( - ExternalFileTableValuedFunction.class, Mockito.CALLS_REAL_METHODS); - Map enabledProperties = Maps.newHashMap(defaultProperties); - enabledProperties.put(FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY, "true"); - - enabledTvf.parseCommonProperties(enabledProperties); - - Assert.assertTrue(enabledTvf.fileFormatProperties.enableMappingVarbinary); + public void testTvfVarbinaryMappingCannotBeDisabled() throws AnalysisException { + for (String format : new String[] {FileFormatConstants.FORMAT_PARQUET, FileFormatConstants.FORMAT_ORC}) { + for (String legacyValue : new String[] {null, "false", "true"}) { + ExternalFileTableValuedFunction tvf = Mockito.mock( + ExternalFileTableValuedFunction.class, Mockito.CALLS_REAL_METHODS); + Map properties = Maps.newHashMap(); + properties.put(FileFormatConstants.PROP_FORMAT, format); + if (legacyValue != null) { + properties.put(FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY, legacyValue); + } + Map remaining = tvf.parseCommonProperties(properties); + Assert.assertTrue(tvf.fileFormatProperties.enableMappingVarbinary); + Assert.assertFalse(remaining.containsKey(FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY)); + } + } } @Test diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out new file mode 100644 index 00000000000000..8c9e82c30c8acc --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out @@ -0,0 +1,72 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !false_UTC -- +{"ts":"1969-12-31 23:59:58.999999", "tz":"1969-12-31 23:59:58.999999+00:00"} +{"ts":"1969-12-31 23:59:59.999999", "tz":"1969-12-31 23:59:59.999999+00:00"} +{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 00:00:00.000001+00:00"} +{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 00:00:00.000999+00:00"} +{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 00:00:00.001000+00:00"} +{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 00:00:01.234567+00:00"} +{"ts":null, "tz":null} + +-- !false_UTC_ntz -- +\N +1969-12-31T23:59:58.999999 +1969-12-31T23:59:59.999999 +1970-01-01T00:00:00.000001 +1970-01-01T00:00:00.000999 +1970-01-01T00:00:00.001 +1970-01-01T00:00:01.234567 + +-- !false_Asia_Shanghai -- +{"ts":"1969-12-31 23:59:58.999999", "tz":"1970-01-01 07:59:58.999999+08:00"} +{"ts":"1969-12-31 23:59:59.999999", "tz":"1970-01-01 07:59:59.999999+08:00"} +{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 08:00:00.000001+08:00"} +{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 08:00:00.000999+08:00"} +{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 08:00:00.001000+08:00"} +{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 08:00:01.234567+08:00"} +{"ts":null, "tz":null} + +-- !false_Asia_Shanghai_ntz -- +\N +1969-12-31T23:59:58.999999 +1969-12-31T23:59:59.999999 +1970-01-01T00:00:00.000001 +1970-01-01T00:00:00.000999 +1970-01-01T00:00:00.001 +1970-01-01T00:00:01.234567 + +-- !true_UTC -- +{"ts":"1969-12-31 23:59:58.999999", "tz":"1969-12-31 23:59:58.999999+00:00"} +{"ts":"1969-12-31 23:59:59.999999", "tz":"1969-12-31 23:59:59.999999+00:00"} +{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 00:00:00.000001+00:00"} +{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 00:00:00.000999+00:00"} +{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 00:00:00.001000+00:00"} +{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 00:00:01.234567+00:00"} +{"ts":null, "tz":null} + +-- !true_UTC_ntz -- +\N +1969-12-31T23:59:58.999999 +1969-12-31T23:59:59.999999 +1970-01-01T00:00:00.000001 +1970-01-01T00:00:00.000999 +1970-01-01T00:00:00.001 +1970-01-01T00:00:01.234567 + +-- !true_Asia_Shanghai -- +{"ts":"1969-12-31 23:59:58.999999", "tz":"1970-01-01 07:59:58.999999+08:00"} +{"ts":"1969-12-31 23:59:59.999999", "tz":"1970-01-01 07:59:59.999999+08:00"} +{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 08:00:00.000001+08:00"} +{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 08:00:00.000999+08:00"} +{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 08:00:00.001000+08:00"} +{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 08:00:01.234567+08:00"} +{"ts":null, "tz":null} + +-- !true_Asia_Shanghai_ntz -- +\N +1969-12-31T23:59:58.999999 +1969-12-31T23:59:59.999999 +1970-01-01T00:00:00.000001 +1970-01-01T00:00:00.000999 +1970-01-01T00:00:00.001 +1970-01-01T00:00:01.234567 diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out index 0094b0f105f0c5..88a21fa45d5c5c 100644 --- a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out @@ -7,7 +7,7 @@ true 11 111 1.1 1.1 1111 1234.5678 1234.567890 123456789012345678.123456789012 a 0 PARQUET 2 {1:2, 2:2, 3:2, 4:2, 5:2, 6:2, 7:2, 8:2, 9:2, 10:2, 11:2, 12:2} {1:0, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0, 10:0, 11:0, 12:0} {1:0x00, 2:0x0B000000, 3:0x6F00000000000000, 4:0xCDCC8C3F, 5:0x9A9999999999F13F, 6:0x00000457, 7:0x00BC614E, 8:0x00000000499602D2, 9:0x000000018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000, 12:0x005CE70F33F10500} {1:0x01, 2:0x16000000, 3:0xDE00000000000000, 4:0xCDCC0C40, 5:0x9A99999999990140, 6:0x000008AE, 7:0x05397FB1, 8:0x000000020A75E124, 9:0x0000000C7748819DFFB62505316873CB, 10:0x626262, 11:0x434C0000, 12:0x40D91FA833FE0500} -- !sql_2 -- -{"bigint_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":33, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01 20:34:56.000000", "upper_bound":"2023-06-16 07:45:01.000000"}, "decimal_col1":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.567890, "upper_bound":8765.432100}, "decimal_col4":{"column_size":90, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":72, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} +{"bigint_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":33, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01 12:34:56.000000", "upper_bound":"2023-06-15 23:45:01.000000"}, "decimal_col1":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.567890, "upper_bound":8765.432100}, "decimal_col4":{"column_size":90, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":72, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} -- !sql_3 -- false 22 222 2.2 2.2 2222 8765.4321 8765.432100 987654321098765432.987654321099 bbb 2023-06-15 2023-06-15T23:45:01 @@ -17,7 +17,7 @@ true 11 111 1.1 1.1 1111 1234.5678 1234.567890 123456789012345678.123456789012 a 0 ORC 2 {1:2, 2:2, 3:2, 4:2, 5:2, 6:2, 7:2, 8:2, 9:2, 10:2, 11:2, 12:2} {} {1:0x00, 2:0x0B00000000000000, 3:0x6F00000000000000, 4:0xCDCC8C3F, 5:0x9A9999999999F13F, 6:0x0457, 7:0x00BC614E, 8:0x075BCD15, 9:0x018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000, 12:0x005CE70F33F10500} {1:0x01, 2:0x1600000000000000, 3:0xDE00000000000000, 4:0xCDCC0C40, 5:0x9A99999999990140, 6:0x08AE, 7:0x05397FB1, 8:0x05397FB1, 9:0x0C7748819DFFB62505316873CB, 10:0x626262, 11:0x434C0000, 12:0x40D91FA833FE0500} -- !sql_5 -- -{"bigint_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"2023-01-01 20:34:56.000000", "upper_bound":"2023-06-16 07:45:01.000000"}, "decimal_col1":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":123.456789, "upper_bound":87.654321}, "decimal_col4":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} +{"bigint_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"2023-01-01 12:34:56.000000", "upper_bound":"2023-06-15 23:45:01.000000"}, "decimal_col1":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":123.456789, "upper_bound":87.654321}, "decimal_col4":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":null, "value_count":2, "null_value_count":null, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} -- !sql_6 -- {1:0x00, 2:0x0B000000, 3:0x6F00000000000000, 4:0xCDCC8C3F, 5:0x9A9999999999F13F, 6:0x00000457, 7:0x00BC614E, 8:0x00000000499602D2, 9:0x000000018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000, 12:0x005CE70F33F10500} diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out index cb2f86d7ba47e4e1274e0b384cab58ba6795876e..c3b6fac5935862b70d865aa4a70ed097395e73a1 100644 GIT binary patch delta 8889 zcmbW7d#qK|IuCN!=7 z)26Adw$-2YG^V!3q)n(b)C~3)B7=Yk$ZJ4Q9)hn%qo4#qh>F&J_W513z`b{nn=r%u zo!{DzwZ7}SzH6QN*oR;I;H5AAuZOzB#3e!qDU{H{%pU2kc!9WN@1-yJ6Ls1z9&9{+ z_d5^y#jnC_yy)$3f7%Da?sfLMzfycj_Mxx6an@;PoZ>gxXC->er*3*?dpmv3??TOv zzH-+~Co{)ahCcJ3bAbncdgXa9nI0G(s9rk$=Hq*>+;RN$ahX2yJY4)==(W>)A@19H z%=Cr6AJ}K*9Mh5?+qd(i_Q-h`+D=aNp8aZXIUrlKgR{cnS8Y9Z`Wk0v1;fL3A2Y4- zj;qdDoY(iEU9oL4J-F?vZPUJOPZkf;ZCRWJv}(J>;y!wDr|CRzzk0r&xsL^lQ#t>oy<0X4|hd>*+v?P;&CVa-(PZx7es*!Q#cx zw(8S#+&qQrZP#vJge$hlx7h#u4EwJ==Rnr#v%A-_{>W#yKG$Qu__`gdLG9w}w*B&9 zyZ5^7>xb=(&+VFSHF*KRP1)cY5E{a#(0}}M3!kh4r6aE2dLZjRzWx9w`118DNNwO1 zPs7w3Z&-dlN&KlB7I9wZPOrRi+v<7#t{b->D9Ycwar=5mf79o;J(sk;{PX9noz^eE zX=(Mee)mngCo4kJApGx@9GOB3V&;IAbDJPC|~r zA*AwO-*WDHCwTCd#qykdf*R)!4RRCDoptL9fk+;=1sl^rh>Z$Zdhe|})?50x+jg!` z#D8(y^GKh^Zaeq+0C4yI=fv!{?q306Bi;XBY+ir+w&%0$M{i%`vRj)-54~gS0$9)7 z@@;qQTu|4u`T34J4oI8s3oFoCd*_dSVcXi!I_S;=RF{kI+_ARx{yWz}t+{LGYHa#X zcde6tPu{g{0j5?ssXz^oQ+0i+s_*{d0aM|IFD^D%HB(Rd(z;mxyDzO-g&zIVIg=w+ zO`*5ky^d74{O(1BSE2Fa?%Do)@b=z&)(Og^_blQKS3Fy`oO|!mT4=lK-gOo|`M#ZN z!{Bf4TNI;p;Qz>dZ(Y9}?)vg`RueaVc?AP&WB#=J7k61TSRcH9$Lbva!~0iK@!67b z>49wv3SWEA&poh0!K>5gYrb*-Xut0(E2Za|3itx4Lk-5(Xr!f+QAq@$tz$PKj0#5PBE4qMXIg*o`%kp9 z7H|8A-xP6;R_4A~ucUFbhJ(#U2ie=~)a=Q}cFhiY><&x@Y3Olfl*!s z2{;7BVX6pj?~{*-7GH(B%DutIv6mt$KSUpk0D}Bwbu3l!i1=Ri#PV$Z_!ltYk*w0z zKCm_#Fa_g_Gjfp1C_I@4g?O(p*FPD9K6JYguHrE=QfD z=?k!J?(dE7HM`f@z8~+LE&rr>KM3C&YfEoZ@-Zo+2UEz%WI?gEKKbnNr`)^!r@C!n zq#;WRVM)>}V`?B%TVu-Az!&IrF1Z|L+kd(|d(Y1fn{E2pt(awNhm6p=hULF+<^<_q?z>Y~1|9?glJV5HMMRQ-qA96{IzA zNGI$NjY+{pm8>>mj-~7n9fR@$g0p0JhW<48h_F&El`M0F9{w|Rh#S2)pYxtSzJKUm zKECnbZqxX$FX|p;1s7@sVIW=QROm%YVX4quD9A7F8C{8LSbc>P#`he|@lQCoYcO#_ z>Y-}e5{N=ov{k+a&AbC}hsX~{8DbunH*)d&H(>eB4b#OV9IDljQ!;iCB?rNGYlU1K zfc46`Vp8t=IhIrM@k;@{o*=@rB%x%6;VK&yazz1dFGR6%JoK<`*Z8tSr@$m;Ib1eX zsj3jnCNwz=)XX479|0T#DBR)Z7){ZKM*D1HfmFdr5WS$WcLh)=T}seJQUyx#^W|Sz z7-YeDDIeT9L;FcKt~b(`geS+T^wA@ ztxXgO@+Ok=qmJxOZJ>-4Gz*1DfXFC_2G5n=B^kAZ`wg2(DZSJq2D`@By#zmt*K`}l zTaW7YHFL;DsHj8LiCbIZG{mZP>^0@gRFg~yDys4Dmv+nJ6RCUc`0Gb^|AA#?JNPb9 zC`2jJHm=Nral%JZ#7dh(p*Wa1mXiamtiYEtMk=Umh6rCK2dhb)V;-z7y?{LA_=#gU z>D4dmZey1rLJ~kpwZd1bjr0wc($-j;Cjr+$?$XXNn&<|)^{!~hG%8}QLMu%WBaER; zp$Ab|5BTHFujrO%4?nnJJpH`x#PLI~=paje``EGF-%#HnLzDyf9TFFc1GT?k=!}hh zD`f^B^-0gqd_`v(JA3Oru_IkwEhKCp%@tRY5DzOfMr!ONH=6#!3&RCe?A?zaGydT# zNxuI%u6vvnGP@|Jqb11v4DdAyq%VjGLp>h=3}Gs2l@;{yU8{2je#$zRjH84eLWPif zjb<>8L`Vn}Tpq$OetZ+Ly!=F?*5x& z!$1I)Ni9Mtk{ki9rMf_sgj<`YX>x=0{EuF}z`9}F+(4M9&TNr{$Uuk6gEm8)_ltk)>gc2;+_de)V7dTogHeoL z*K&#o;09-DQII?&Vd2$^r*uI(yl+6<0wO*5`A5_OUj;0qB~Vh)&`JzJSVUn687}38 z3d$Dou6!ObSQ_8-hHlIFXQsQ9t*AOuyDGED)V-)GTRUCRXO*y~(zHy0-o}6FxzJCy zkp!Qx-RGMn^|dD1M>+&}-&811R+<1M6DpLBqgRF=W{&0Ly8wzSDv$8mVxo)2Hp!(i zM%qZy$@BoqkG~lj;d=tS)lUJ&zf={PI5HQ>2hqm?P9)(pTY3lrHWks$v78(T3 zaDG8`6v|Lb$@a;7j`_=4)C{uQGH2&?mv2H9Q7|Y{iR!(Y(Q#kOGwqTzH5AiJ$|roCD9_|Heg_Dd6yd=S-xlLdijxO7_0ee9`?7GN&MP zH=Yp*^S(3i;Mr%ah$s`0mWqn(DeROH-d;sYJtD?aN8kf(5x8?KCk-v(Cc@X`S{eFY zk<#r2SV67kf*3@#R9Or?rq;#67DbmUF8_P&_?Ujl!3<0Q~|8k7&N} z<~Maa#s_}4yQS@d_MOxt4FJ;vD}CrG_=eF$1>Fitww}5)$7oC2B~X!3!>D3UIf2Am zjU7-0wW6rJK=hFGvd)h`bK&Po=h7pq82R&6Ny&kdXd<8#0e4xl2~r*T4XuX1O^i%2dd8lW8mxtPjVCQKa2=JCYc>?htj z-HJ+xI7qrR8m%-!;dED0Hkx3p!<2y*2Kh9{Xu;<_#8i9QSSZ~q!N4NJxd=35%|Laa zsTzWF<1Kr@?vZ=CLudQHdHi_A`Q59=SMKTlvfXSDabUnk1M1NXZ3aV~q;wO50K%Cd zwVIOWEHM?;*5EUhL_^BV0HIZ5=wj$mo6L-8x}ah_Bj+bxxG-SsAZZ0tVQ7e`Xd@$m z1Y@QJ(NLm1G}0z%BL>UkrdPqBKe=e?@Te5%cuJ_H>7?PoZ?xg~6n<-^%nLNMN_9N( uV(Rx(B(h;V;|~@=TEF$V4E-V8M`@V|{AgEvZ_Q^`ds|;muhVL@sV(*ntKDL8TJ0TvuByYjK-OV#*c=`Gey+oZ8)vXK zi^7XGgjBW3?Il4O?c8l6%w72=W z9vjjIvwowqP2JYl3!&RX^?|I@+SBVGmDZz8LCk5fSv#x_yPxaqQGEp4YP9uPdeml{ z(P?dOx0YhGlyZd@6!wug1{cyNx4l{P)?q;K1?KxHWvBpr=)7#h9ukP7wwb^jh$B#7! zu}zMSj(+z#caGIhmavZ)y(Qiq(cWum)A5!Gi=I8+bl>YXo@kzQ-7_b2p=>j*V{vvE zoxK*=WbH|uYQlUkozza2$T&55nP*SIodCxNR%$u-`uWtBT*v>0p zAA76CD{XGSg?moaW9!>Zll9+x8yVmo;I)65?2qGrK_+@$fWmhgd^Y^qJCOHz1Qx!l z^WJpNyW0EKG}U{)w*1fcB;V}1^PX=gb-n-pc>8C(yjfr-_Sv>gJ-}6~~2u9(nU;i<>d13D7q}b<+!bPUAvR)+V`*L zVg+k8x|I#qR4H((mh9l4yb^f+5}-lH?OYbM0@Nm_DL#cxd-0p9tz-{C(L70V4vv17 z!HBWSxo}ZgXDSnOz!{qb8Smk^MnBWJ9{zkhJ;?#yI>8U~OoNc@m10O2PZ8wcz@2QC zpz!t9Mc|szh(QO{DC_Uuje%?6j=}ZXZI!F~%ONre3Ff2Get;``i+~?O+aegi&8|IP z>0NvO_k3o#hbM^oNn;E{lkx;}qhuf4^Hmm0!jRVVKuNNPoYxrzdMnxv+fKgBk4e0o zu``iikd>yJqawBFm#MmrebdoUX{M!Rh@~^-=BssWU38wA6Vj{fQpP$=3C!7N7Sl-CEaY^J%k z$Edhc%yp#H&lZ$bIbWoaLsZX$X0Wl?ycRM6g-^AE0;5LA0;kQe(d9Y5)AaoHKo-Kg z1KCMBN^t1q$H^v&mq7uhq0u#Vq}G+n(8<0V<={2DYe^j_4@6Lik3K_TB_9rEnJj_o z$_fgLG_*BnfU_=5mY||Vm&qZl9SA)d`r#bD$B`ZRK7t5b3L;%hDp$)$mX`E3Qp z+!gUzx?i<*-OL(_IKCEaol-qKq=vPXjI@ z-pS`0!hLl(bV~S}Q7o9h7{!Lk%Z+5b-9}hI0Oy+*Oj=EPR&M@=rUaU(EvmiqNT*IAlG_83QV2NUyNf0dd&Dx zEIS!}`o+YhwcS%@QQ?rJE-GRrLY_=6LpYbEDQ6RW;He(UJSA_}#0kkia)<^hWE=q5 zlqvzhWG~!}E`kC?hnCpLA!*>J;@L%Vr6f&{BA>*f0jy{}f)sf4fE@&0;u!vTBI3C< ziN*1RL^htXkh~DHUdeWkP93NdgFs6Dg7VS+C*{$aU@94N(7ZvSt>{@9eiVoX^6q2= zGs#k|S%db*Zcc&uiic{H%WcQ*AFcC&4BjKbzATkP|x>WSQ zaz+~buUX|c{(ZrtMQO9uR3snLo>bTvaER*p`{`^a|o z2fQeYGA)yxO`vF}!X72&M#xApbn4;_g407zlA(~!;^?px3RJY39smNkuX5@DD0S4TXPPyj^{eSrbOO_atmbhDURumsM(@Cy)ERfG!#$awi0$&qoP3HP1$^AKV4;t`!j!u@+?S!w4e-~be2qoLirTS1xe@8 zp~Y)~0Fp%dHE85u1OKuJdju7*aK5~ZJ)b~9bZHPsKw+guN2v<%LnHG5cFR!BWpQgO zv;iH;I|24Anm+a1M2zHWW!oxzS;SHzZQZ@CX6f{hZb`~-til*Tfg^xJPR|#wVdIGd zWf!Sj0F-(XOF<8yatv^aPF7ba{#LRH>`FBH;!9{&XV9BW>(}XlIE3~5awWT%uELPc zS=I$)LYw8>MUSR{{Kam=2IghunDPs=`N!QXf*-14Y98?byOopyDSx!2GC5W zh3_eZPVEL1D7USqCpyy#;^};|+#3jK<*}8ak?j5^5bU~4^yN`vK=E3Pwo44P718pO z8{vCJS4njDD<8hIqq=f~S}6&Tch2LGHSinNh;l;%JE|1D zUp8U=_y`Ks1wxAM3ZQyJ_AIVYY@p(hHSnkE;O^@!>{pry3Xy9+p?aNGma(!8y$6(x zoWn!)@ZD@38>hGh&|}~YO6Cyz*nXf^G~&rnOn3k0DXFEm0&x5{K>gX-6d(Y)SZfRr zKhlalK95EvyuwC8SpB4mp2NGL*9Je%z3`sb2LkKS~XXZh$7b zuMZvHVrH#(|L2&&|I5VOPgMEHfh$i6!Io+2hR~#47`XpDp_ODP;d6_Ea^e#>oJ3Zm z0mz@#f&``vlh3r_#tBN`%xzmEX6c&B8vv?Yd44#A_55lw9L?L=p(IS@rU?asBkRyV zL=PKam@Eg7Bzb2CGLKvqLRO_hMvE_4x`JYm3YSVn$;u&zYBdMGZnm=L;{pP}+=K?$ zPL;memI7Qg3uu6DxJ^#qgyW~zxHC-!4q#0OkW%yo2be+)K-tZqV{mM7O7pK7`L{U3~>eBB1ekv1Uap=&=I{`O)q7(OU^dXeg=QgetJ$Vf^2 zsGa>XS=m`RkP@#HmqqJiF@$~(}fiJ8F$%M3+04td;OVAAw=#6FyT^R~Y F^MA<_!Ycp( diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group5_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group5_orc_files.out index 3440963fcaff40..9052566df19922 100644 --- a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group5_orc_files.out +++ b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group5_orc_files.out @@ -4,9 +4,9 @@ 456 2 -- !test_2 -- -[{"elem1":1, "elem2":"123", "elem3":9.8, "nested":[{"elem1":1, "elem2":"123", "elem3":9.8}, {"elem1":2, "elem2":"456", "elem3":10.12}, {"elem1":3, "elem2":"789", "elem3":11.14}]}, {"elem1":2, "elem2":"456", "elem3":10.12, "nested":[{"elem1":4, "elem2":"101112", "elem3":123.8}, {"elem1":5, "elem2":"131415", "elem3":10.2}, {"elem1":6, "elem2":"161718", "elem3":11.414}]}, {"elem1":3, "elem2":"789", "elem3":11.14, "nested":[{"elem1":7, "elem2":"101", "elem3":13.08}, {"elem1":8, "elem2":"415", "elem3":1.12}, {"elem1":9, "elem2":"118", "elem3":0.414}]}] -[{"elem1":4, "elem2":"101112", "elem3":123.8, "nested":[{"elem1":1, "elem2":"123", "elem3":9.8}, {"elem1":2, "elem2":"456", "elem3":10.12}, {"elem1":3, "elem2":"789", "elem3":11.14}]}, {"elem1":5, "elem2":"131415", "elem3":10.2, "nested":[{"elem1":4, "elem2":"101112", "elem3":123.8}, {"elem1":5, "elem2":"131415", "elem3":10.2}, {"elem1":6, "elem2":"161718", "elem3":11.414}]}, {"elem1":6, "elem2":"161718", "elem3":11.414, "nested":[{"elem1":7, "elem2":"101", "elem3":13.08}, {"elem1":8, "elem2":"415", "elem3":1.12}, {"elem1":9, "elem2":"118", "elem3":0.414}]}] -[{"elem1":7, "elem2":"101", "elem3":13.08, "nested":[{"elem1":1, "elem2":"123", "elem3":9.8}, {"elem1":2, "elem2":"456", "elem3":10.12}, {"elem1":3, "elem2":"789", "elem3":11.14}]}, {"elem1":8, "elem2":"415", "elem3":1.12, "nested":[{"elem1":4, "elem2":"101112", "elem3":123.8}, {"elem1":5, "elem2":"131415", "elem3":10.2}, {"elem1":6, "elem2":"161718", "elem3":11.414}]}, {"elem1":9, "elem2":"118", "elem3":0.414, "nested":[{"elem1":7, "elem2":"101", "elem3":13.08}, {"elem1":8, "elem2":"415", "elem3":1.12}, {"elem1":9, "elem2":"118", "elem3":0.414}]}] +[{"elem1":1, "elem2":0x313233, "elem3":9.8, "nested":[{"elem1":1, "elem2":0x313233, "elem3":9.8}, {"elem1":2, "elem2":0x343536, "elem3":10.12}, {"elem1":3, "elem2":0x373839, "elem3":11.14}]}, {"elem1":2, "elem2":0x343536, "elem3":10.12, "nested":[{"elem1":4, "elem2":0x313031313132, "elem3":123.8}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414}]}, {"elem1":3, "elem2":0x373839, "elem3":11.14, "nested":[{"elem1":7, "elem2":0x313031, "elem3":13.08}, {"elem1":8, "elem2":0x343135, "elem3":1.12}, {"elem1":9, "elem2":0x313138, "elem3":0.414}]}] +[{"elem1":4, "elem2":0x313031313132, "elem3":123.8, "nested":[{"elem1":1, "elem2":0x313233, "elem3":9.8}, {"elem1":2, "elem2":0x343536, "elem3":10.12}, {"elem1":3, "elem2":0x373839, "elem3":11.14}]}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2, "nested":[{"elem1":4, "elem2":0x313031313132, "elem3":123.8}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414}]}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414, "nested":[{"elem1":7, "elem2":0x313031, "elem3":13.08}, {"elem1":8, "elem2":0x343135, "elem3":1.12}, {"elem1":9, "elem2":0x313138, "elem3":0.414}]}] +[{"elem1":7, "elem2":0x313031, "elem3":13.08, "nested":[{"elem1":1, "elem2":0x313233, "elem3":9.8}, {"elem1":2, "elem2":0x343536, "elem3":10.12}, {"elem1":3, "elem2":0x373839, "elem3":11.14}]}, {"elem1":8, "elem2":0x343135, "elem3":1.12, "nested":[{"elem1":4, "elem2":0x313031313132, "elem3":123.8}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414}]}, {"elem1":9, "elem2":0x313138, "elem3":0.414, "nested":[{"elem1":7, "elem2":0x313031, "elem3":13.08}, {"elem1":8, "elem2":0x343135, "elem3":1.12}, {"elem1":9, "elem2":0x313138, "elem3":0.414}]}] -- !test_3 -- 1 2024-06-30T20:00 @@ -16,9 +16,9 @@ 456 2 -- !test_5 -- -[{"elem1":1, "elem2":"123", "elem3":9.8}, {"elem1":2, "elem2":"456", "elem3":10.12}, {"elem1":3, "elem2":"789", "elem3":11.14}] -[{"elem1":4, "elem2":"101112", "elem3":123.8}, {"elem1":5, "elem2":"131415", "elem3":10.2}, {"elem1":6, "elem2":"161718", "elem3":11.414}] -[{"elem1":7, "elem2":"101", "elem3":13.08}, {"elem1":8, "elem2":"415", "elem3":1.12}, {"elem1":9, "elem2":"118", "elem3":0.414}] +[{"elem1":1, "elem2":0x313233, "elem3":9.8}, {"elem1":2, "elem2":0x343536, "elem3":10.12}, {"elem1":3, "elem2":0x373839, "elem3":11.14}] +[{"elem1":4, "elem2":0x313031313132, "elem3":123.8}, {"elem1":5, "elem2":0x313331343135, "elem3":10.2}, {"elem1":6, "elem2":0x313631373138, "elem3":11.414}] +[{"elem1":7, "elem2":0x313031, "elem3":13.08}, {"elem1":8, "elem2":0x343135, "elem3":1.12}, {"elem1":9, "elem2":0x313138, "elem3":0.414}] -- !test_6 -- \N \N diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out index f31f2a50d2fba93ab4e61f812eb7235da7845a65..d73f0dce7558e72cfd609c4ca25c59448b8e37e8 100644 GIT binary patch delta 81 zcmZqhpX;jf1B1=h HVy=P!nmZKd delta 52 zcmbR1-{8MtHXmzD-uwM~H!tLS$tG%OXlP(yU}9UKJyV9uyz!MF$lh_XaP5 zAb9d(a6R|~y!p6zQ1RqJy@+Q;5WlZGYsAeSCafz9y;D_PUw>8IT~l-CRma!&9dpk* zRaA|lJ`9RO%t3Py4u*r}kZ>q+NZs4cOM_8WD{XbEMJK&~2<~XtjSW>DX{{4j4Q+M7 zed@Zro}%)qVz7C^o$kIKFcrL%mVg-q_pQ5*sdT)6Nx|Lh$^3c(Hot8_aI2Qx@-`B% zT8H(5;O;HE?rjEKlDaJjZtLH!A9JzjpT9(|&VItolJ<`s2qYQ=-@bqrDB z3Ip|H$|>%}$;rhnwTb@EJqM$zRW`P|SK>ZpQf(tCMg+HFjkUeWWU`=F)Yqp^s@VKx z@-D8t_H4#^h}~#lSkaL+aS0jbzNL7A1~xo#X&}Pt=UI3~_{I2VGL6Q`@QdaH%UX(- zA1rHEpiQV{OG{f5wESRM%Xc&%wXB_oZ4qkSJi4`HU7-Uvfi-|Nf`2&KG0!?qx6+Bf z2@2<5@pf320a)=A<5F8l3NXUkfHhDLd7*QHWi3U^50 zTGr0Pwg|Nze{|iO4Ku5wswmkji5L@0D$2=VIzS~vD5Y!!>M{(M1`pwtFDVR4%I@VL zJAfb><$8zquY@=a$H?1{3~gKG6%{y$SKS_(;+#DmmTNs9Iza*@;hc)#R!PO#;bZIP zl|X|Lyo@qNjJrR~P6P{ptTFE0@HygXDb7KeeA%Da?#Pg(QKYASwaS9+=8V zogko+dsrxq{yhc$$Oj#ac0URi$PQHeIxx74W2YIGk;J4XGU7gtomxW$To`Eu2_Vgx z@lA9^Rrk&s3A#GIoe2%Q=~G}*_hfuG@kon%soeMReca-ps6Y)OG}QJ_d>Qb4N$6Gn z>{xc1(6KaFYUj<*&VSIc)E*T-uu!A4qIEwj{?HtssJIq!9MFK4%SpSaP;6c zDtPhm(3{{vJc|K6c=arT7r~Q=s32ZMQ512~+OAiHwwK4e_x{Xp-<#PtpI)VIKTAD$ z+BdiYr*I$c$7w9#49?;lK7rs%-$hI{lj*2uVW$5oD(R+*h_Eo6zLcY+WFSHbJWS7E zN=?d8&w?WfB~^M#s=x;+MojB@!a|(6F2Y&~5n-W_ok1$mg(oaL&R)h;XSx6i3r_Be z;OhnveBJ22%iT-o*iJ%s5);AQf#QPGeKYV*+8Q=nh$)z=W=SK=AOy8L5z>o#5~4_m zAT_wAU!)*EcrFe#RgK!DBbz2OWkys}ZnnaX9JRHO5G7=qs$hZ^OBOE+HG#%USZhK} zpweuIyEoPqQn6N2)K*Dp$`PZw%xD`LP#Frb;~nG9xP7E@*Y$_IMeUmR{` zeb3>p0KXr;kQw*+G5au_9+_TQk=kZrD$z(bDA8rL73!2}A&pE!{0fUDi zHK8U@X|}`Nv*R!R;{d^$+Olv7;O`p&N2j*sl=-`IYpR$yFGZ#_pILjRm^h6y+ZNur Y-lLbFe*q~{2r*^CWI_XC>>?E7-^R)KGXMYp diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out index 899acd4edd5f0c..69260dd3acbd57 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out @@ -39,14 +39,14 @@ apple_banana_mango9 [{"one":"2 - 0 - 1", "two":"2 - 0 - 2", "three":"2 - 0 - 3"}, {"one":"2 - 1 - 1", "two":"2 - 1 - 2", "three":"2 - 1 - 3"}] -- !test_4 -- -1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 +1001-01-07 17:07:46.123456+08:05:43 1001-01-07 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-08 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-09 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-10 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-11 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-12 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-13 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-14 17:07:46.123456+08:05:43 -- !test_5 -- ["good", "bye"] @@ -92,14 +92,14 @@ apple_banana_mango9 1981-01-10T00:00 20 -- !test_13 -- -1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-08T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-09T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-10T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-11T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-12T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-13T17:07:46.123456 -1001-01-07T17:07:46.123456 1001-01-14T17:07:46.123456 +1001-01-07 17:07:46.123456+08:05:43 1001-01-07 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-08 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-09 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-10 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-11 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-12 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-13 17:07:46.123456+08:05:43 +1001-01-07 17:07:46.123456+08:05:43 1001-01-14 17:07:46.123456+08:05:43 -- !test_14 -- [{"one":"First inner", "two":null, "three":null}, {"one":null, "two":"Second inner", "three":null}, {"one":null, "two":null, "three":"Third inner"}] @@ -171,14 +171,14 @@ apple_banana_mango9 1001-01-07 1001-01-14 -- !test_22 -- -1001-01-07T17:07:46.123 1001-01-07T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-08T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-09T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-10T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-11T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-12T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-13T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-14T17:07:46.123 +1001-01-07 17:07:46.123+08:05:43 1001-01-07 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-08 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-09 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-10 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-11 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-12 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-13 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-14 17:07:46.123+08:05:43 -- !test_23 -- 1001-01-07T09:02:03.123456 1001-01-07T09:02:03.123456 @@ -203,7 +203,7 @@ true 6 7 8 60 6.2 val_6 val_6 DIAMONDS \N \N \N \N \N \N \N \N \N ["arr_6", "arr true 8 9 10 80 8.199999999999999 val_8 val_8 SPADES true 8 9 10 80 8.199999999999999 val_8 val_8 SPADES ["arr_8", "arr_9", "arr_10"] [8] {8:"val_8", 9:"val_9", 10:"val_10"} {8:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 9:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 10:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}]} -- !test_25 -- -{"duration":"111222333444"} +{"duration":0x313131323232333333343434} -- !test_26 -- 1001-01-07 17:07:46.123+08:05:43 1001-01-07 17:07:46.123+08:05:43 @@ -260,9 +260,9 @@ true 8 9 10 80 8.199999999999999 val_8 val_8 SPADES true 8 9 10 80 8.19999999999 93.7 -- !test_32 -- -1970-01-01T08:00:00.010 -1970-01-01T08:00:00.010 -1970-01-01T08:00:00.010 +1970-01-01 08:00:00.010+08:00 +1970-01-01 08:00:00.010+08:00 +1970-01-01 08:00:00.010+08:00 -- !test_33 -- 1001-01-07T09:02:03.123456 1001-01-07T09:02:03.123456 @@ -285,14 +285,14 @@ true 8 9 10 80 8.199999999999999 val_8 val_8 SPADES true 8 9 10 80 8.19999999999 1001-01-07 1001-01-14 -- !test_35 -- -1001-01-07T17:07:46.123 1001-01-07T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-08T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-09T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-10T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-11T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-12T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-13T17:07:46.123 -1001-01-07T17:07:46.123 1001-01-14T17:07:46.123 +1001-01-07 17:07:46.123+08:05:43 1001-01-07 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-08 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-09 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-10 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-11 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-12 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-13 17:07:46.123+08:05:43 +1001-01-07 17:07:46.123+08:05:43 1001-01-14 17:07:46.123+08:05:43 -- !test_36 -- 1001-01-07T17:07:46.123456 1001-01-07T17:07:46.123456 diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group3.out index 93c2fd8c672e39e980185530e1369a95225b4229..b079c6f3de820dfe6e0ddf6092b93f25d3653c56 100644 GIT binary patch delta 568 zcmcZ{aX50q3{`^)0}}%iP7^LuP6$j(%qdRgG}SWz^MTT)2Bwo6cqG_>Ofv(siRUF% zfGl$Zb53(E3r+|uDJo6nv;b-Wae=ZH1{Rag^GJfUSQ=PvR%E=&2C@hUI1RWAIUx{Y zks(kQh!2!DG%%c;CkM93$iQf_E(h2EV*}&O9vspvbaCk$9)CuVtASwhL`_W)6XdFW z5GKe~uOUpJt0qfmffWFQZ}Mk8FcTD13jAOuFsLT`LYTm?np_28f&wN`7OVgiFxU7$ d@POcc{yUj3?>$q$^AT%OsuSv*YQX&v9V1)!y{?Q&dzDhWx)x7B}JvFoECZpU@iv- zrv;ZKCj_P?<`k!LS^}j&{1_nIY|eO)4bCPv1|h8z-1OiYvgIV72wnK$Qg zNV8DeLJeMjMiwOo2Bpche3~pujEqW?*MJyIOiGh)gBZ-rN|U+xH93`7Sd?NpqfKHb ze~?jOQDS9Pnk*#?lxJg8n(PT;u(K;ot^hGOIFu$Ym(^rb0{KX3vx>k69um|~z9xGS WX9#f0zv3dswJadlZr-ZJ#{>YcTv5>g diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group4.out index 231277cdb6318990f01786bcd2f54cc553828a62..56c373b451069128ddbe51d73a4d4085f5af6fa1 100644 GIT binary patch delta 458 zcmX@JgYCplwh41N3@Qvjz;NS&Pkd}(j?v_Rji#GV@(atsB_}R0-7KP)B8{wIz32RN zD1URnMxC^Jpk^ZjL$GN^2F?bCItog;sU?Xii6x0jRzOKZu%xk>v6-8hb1i3dwNhDP zPH8Gw4q2lSc8#WHhUP}*MnKZCHkPwmDLb{&P{|5mNIk0c#@J0UHgf@*U{G5NG@$}n zJF0FI?7Gd&jm=Gft}rzN(m*pxQj1FzKpy1c(%o(=!T8Ta3>u!`U_cPl9n%=Cw>S7Q cb}}Oe>UPIK#&05c^qj#XXS+Qlo$-eW0GUsH;s5{u delta 323 zcmX@HlkLn7wh42Y85kI+7X&hzZoIXZkBNb4^IU!pIk@0vCA}1Bpy=ee{HB`^dd^RW zv1f1mo?5TO#K^$F#KWkhqo9m{Xbx6oe>c z!c@wdSX7i)sZ<-wS*?_vT4|_cg|3MiQxkJ;Vu6xzZ7onw1*QfTObu+sB}JvlB}&^H sB^Y^4xftMJ`o=Uy>+Sb_89SLlzTOTD3T;s|`7>yI+wB+98Go1n06GUx?f?J) diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group5.out index 2b47e7d4575b97ea29ad1a9054c879248dedd652..29bac94b8d89805df3d830b85430930cccd85639 100644 GIT binary patch delta 1821 zcmcIlOKTKC5GETnsaf!`xZou(aATIZo9?cjo?gL&D7hu>BWy5)-~&902*Tne2M^+c zCj))*;N61|a05Ae^59*6gq~FJpjAB%voGCjz?rFn{;I0GtE;~0Z|@494hwJ9loTx? z0#cHdVuh?UtB{pp6|u6cD*LNbXS4Yh3e;+qwDh_HqJ=Maa^>Nc{k_~lrY>4W8f6f) z3S?M8BeWHQB8@hoF%Xrb6qk%A?q#XMK9di5+Ri^~=O;)qB$Fg3NKTTRBAFtY?zHo8 zA9m+ObVUgI%ECX(!qNclfD$q)2}$2@mq39itJF;mcZ0kCbeXVD%)xo{m*9J8Fb;8X zTXXPcSG2UHbGlA19XyDDkEjchu)WT^?Z#;10ro=3Zn_uKpo6d%3fawcVOMEY115;N zFoGiXv%Q!F8*oQ-ZO%QKhh^U!al>@Ajr)EJ;=UO$bw1C-O!hxCPRCq^g@tKqT8RqL zZLdKo9et_6MILDk(YOnBSmaTJ7Ax*t6=vM8HOQt8XVzimw4aLT;9uolHDNKvkPuB0 zx(`h##F(LRL}K0RbttBjf25gsB$-&w*5ZZZ=;|gEVjIW3sKKmjHzC`<@bjC=6r9(H zOqTk$c&!TM{u#$eW!8PICR<8#oUDO3a#sE{;`|S5W27?UZr78d@L@l!A3u|cMqCsw zrgbZgWE-Q3a?pUq8(cn5O;nzZh@OF{J?Sv?L__1bg^{NdTF)S?c-GLOoyiTzW#Tg% zRmKNRxOCi-{b<6{kt)-9vI&nfWAoRQwxH;vRR3k+cDDYoe$cd+MBzZx`y^`6^fQSX ZdNCD;URjcUyrNPPn*RSAd_Tw__zl7Mp#%T` delta 1245 zcmbtU&ubGw6t>%Lvu|>dn23k8W(N~3xXH}y?;s+GC?1+LNg+Z?X=8dyDaGnlgP^?& z4){0pAQX%sc=Beix!8jzLGa?qvu}TN(;lJ`VIO?&doypodHXS6J`{hwE4~$}>^U++ z6f#TZh)U*3ifE(*huzfqv}RQpuP`58?PtnKa`-v(Ub(rk*#%1wZu_RgERQjG*xLfP z?2ai8(V{XJ_GuOuJ|E>?j&d^?3dStP9EOT9kCDRAhNB#8!NnGHtJr)41mmv3IDSaU z-OG@@71uV$H4#z71Xec3v>C7Lo~5K(fP<&^dj0+m14cN6kV9vghzJW1*ej&XfVLuR zfko^H+5zoE*a3&Q5p)CEi?9bC@gwL5G%<{VzSn!Sw=SH5A)o1RNy9ORlYZTSa*_;x z>adXh&ruK84bXL!a$!=rzYInU2Ti!5Qprla?7^m`MBj_LQ8PuVOY_MT> z6|&N30+B~`NRJQs7HtgVPS$wAzFwfces5dQY9i=h1&24S*rl=b7Y$%jNzKKp?l;4` z{ZZnz_{QnhKZshCN3CFj;efuj{yxD{k)v+~`qf0ET4+!ko#~(>U9_Ty7WC13e0aPH z83p@JmRxJY)#--!qYZeu$$xhE Date: Fri, 16 Oct 2026 16:12:51 +0800 Subject: [PATCH 2/4] [fix](regression) Align external mapping cases with branch readers --- .../paimon/test_paimon_catalog_varbinary.out | Bin 734 -> 742 bytes .../tvf/test_hdfs_parquet_group0.out | 20 ++-- .../tvf/test_hdfs_parquet_group2.out | 92 +++++++++--------- .../tvf/test_hdfs_parquet_group5.out | 6 +- .../tvf/test_hdfs_parquet_group6.out | 34 +++---- .../test_paimon_catalog_varbinary.groovy | 2 +- ...test_paimon_write_variant_shredding.groovy | 13 +-- .../tvf/test_ctas_with_hdfs.groovy | 17 +++- 8 files changed, 96 insertions(+), 88 deletions(-) diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_varbinary.out b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_varbinary.out index e65f32a2ad61135d6e68d1b986bc346707db70bf..09b85d61c88549d7c12c988180320b00c7d6b6c4 100644 GIT binary patch delta 15 Xcmcb|`iyl$9n<6kOmdT_FnIw0G@k|K delta 33 dcmaFHdXIHO9h2z${d<9k0SY)F+{v? String payloadType = rawPayloadType(filePath) - assertTrue(payloadType.contains("metadata:text")) - assertTrue(payloadType.contains("value:text")) + // Raw TVFs expose the Variant metadata and residual value as opaque binary bytes. + assertTrue(payloadType.contains("metadata:varbinary")) + assertTrue(payloadType.contains("value:varbinary")) assertTrue(payloadType.contains("typed_value:struct")) assertTrue(payloadType.contains("age:struct")) assertTrue(payloadType.contains("profile:struct")) @@ -210,8 +211,8 @@ suite("test_paimon_write_variant_shredding", "p0,external,paimon,nonConcurrent") assertTrue(!unshreddedFiles.isEmpty()) unshreddedFiles.each { filePath -> String payloadType = rawPayloadType(filePath) - assertTrue(payloadType.contains("value:text")) - assertTrue(payloadType.contains("metadata:text")) + assertTrue(payloadType.contains("value:varbinary")) + assertTrue(payloadType.contains("metadata:varbinary")) assertFalse(payloadType.contains("typed_value")) } @@ -257,8 +258,8 @@ suite("test_paimon_write_variant_shredding", "p0,external,paimon,nonConcurrent") assertTrue(!firstInferredFiles.isEmpty()) firstInferredFiles.each { filePath -> String payloadType = rawPayloadType(filePath) - assertTrue(payloadType.contains("metadata:text")) - assertTrue(payloadType.contains("value:text")) + assertTrue(payloadType.contains("metadata:varbinary")) + assertTrue(payloadType.contains("value:varbinary")) assertTrue(payloadType.contains("typed_value:struct")) assertTrue(payloadType.contains("age:struct")) assertTrue(payloadType.contains("profile:struct")) diff --git a/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy b/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy index c02f7258d37b9d..4142259643d711 100644 --- a/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy @@ -29,6 +29,13 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { String tableName = "ctas_tvf_1"; def uri = "${defaultFS}" + "/user/doris/preinstalled_data/orc/orc_all_types/p1_col=desktops/p2_col=bigint_col/*" + // Internal tables cannot store VARBINARY. This fixture contains UTF-8 text bytes; + // decode them explicitly so CTAS still tests partitioning and distribution. + String fileColumns = """tinyint_col, smallint_col, int_col, bigint_col, boolean_col, + float_col, double_col, string_col, CAST(binary_col AS STRING) AS binary_col, + timestamp_col, decimal_col, char_col, varchar_col, date_col, + list_double_col, list_string_col""" + // test varchar type as partition column sql """drop table if exists ${tableName}; """ sql """ create table ${tableName} @@ -38,7 +45,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { ) PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns}, p1_col, p2_col from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -58,7 +65,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { ) PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns} from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -79,7 +86,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { ) PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns} from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -100,7 +107,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { ) PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns} from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", @@ -116,7 +123,7 @@ suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { DISTRIBUTED BY HASH(p1_col) BUCKETS 10 PROPERTIES("replication_num" = "1") as - select * from + select ${fileColumns}, p1_col, p2_col from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", From e18aff1aae086842946cc98237cb89dc45698b8f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 16 Oct 2026 23:07:14 +0800 Subject: [PATCH 3/4] [fix](external) preserve logical annotations and Oracle binary bindings --- be/src/format/orc/vorc_reader.cpp | 11 +- be/src/format/parquet/schema_desc.cpp | 12 +- be/src/format_v2/orc/orc_reader.cpp | 9 +- .../format_v2/parquet/native_schema_desc.cpp | 11 +- .../format/parquet/parquet_reader_test.cpp | 27 ++--- be/test/format_v2/orc/orc_reader_test.cpp | 14 +-- .../format_v2/parquet/parquet_reader_test.cpp | 5 +- .../format_v2/parquet/parquet_schema_test.cpp | 42 +++++++ .../apache/doris/jdbc/OracleJdbcExecutor.java | 2 + .../doris/jdbc/OracleSessionTimeZoneTest.java | 84 ++++++++++++++ .../uuid/test_uuid_orc_varbinary_mapping.out | 20 ++-- .../uuid/test_uuid_parquet_native.out | 104 +++++++++--------- .../tvf/test_hdfs_parquet_group6.out | 28 ++--- .../test_uuid_orc_varbinary_mapping.groovy | 16 ++- .../uuid/test_uuid_parquet_native.groovy | 4 +- .../jdbc/test_oracle_jdbc_catalog.groovy | 21 ++++ .../tvf/test_hdfs_parquet_group6.groovy | 22 +++- 17 files changed, 291 insertions(+), 141 deletions(-) diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 43621c30fca490..35aca9a99f817a 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -1736,13 +1736,14 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) { case orc::TypeKind::STRING: return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true); case orc::TypeKind::BINARY: - if (_scan_params.__isset.enable_mapping_varbinary && - _scan_params.enable_mapping_varbinary) { + // Logical UUID annotations take precedence over the default for raw binary fields. + if (orc_type->hasAttributeKey("doris.logical_type") && + orc_type->getAttributeValue("doris.logical_type") == "uuid") { + return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_UUID, true); + } else if (_scan_params.__isset.enable_mapping_varbinary && + _scan_params.enable_mapping_varbinary) { return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_VARBINARY, true); - } else if (orc_type->hasAttributeKey("doris.logical_type") && - orc_type->getAttributeValue("doris.logical_type") == "uuid") { - return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_UUID, true); } else { return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true); } diff --git a/be/src/format/parquet/schema_desc.cpp b/be/src/format/parquet/schema_desc.cpp index 09921f52496ccd..09dda646fa5007 100644 --- a/be/src/format/parquet/schema_desc.cpp +++ b/be/src/format/parquet/schema_desc.cpp @@ -268,7 +268,8 @@ std::pair FieldDescriptor::convert_to_doris_type( tparquet::LogicalType logicalType, bool nullable) { std::pair ans = {std::make_shared(), false}; bool& is_type_compatibility = ans.second; - if (logicalType.__isset.STRING) { + // ENUM is UTF-8 text, not an unannotated binary payload. + if (logicalType.__isset.STRING || logicalType.__isset.ENUM) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); } else if (logicalType.__isset.DECIMAL) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL128I, nullable, @@ -316,12 +317,8 @@ std::pair FieldDescriptor::convert_to_doris_type( } else if (logicalType.__isset.JSON) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); } else if (logicalType.__isset.UUID) { - if (_enable_mapping_varbinary) { - ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable, -1, - -1, 16); - } else { - ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); - } + // The raw-binary default must not override UUID's existing canonical-text contract. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); } else if (logicalType.__isset.FLOAT16) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); } else { @@ -336,6 +333,7 @@ std::pair FieldDescriptor::convert_to_doris_type( bool& is_type_compatibility = ans.second; switch (physical_schema.converted_type) { case tparquet::ConvertedType::type::UTF8: + case tparquet::ConvertedType::type::ENUM: ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); break; case tparquet::ConvertedType::type::DECIMAL: diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index b6d4e46c710abd..615beaf233d136 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -1131,11 +1131,12 @@ DataTypePtr OrcReader::_convert_to_doris_type(const ::orc::Type& type) const { data_type = std::make_shared(); break; case ::orc::TypeKind::BINARY: - if (_enable_mapping_varbinary) { - data_type = std::make_shared(); - } else if (type.hasAttributeKey("doris.logical_type") && - type.getAttributeValue("doris.logical_type") == "uuid") { + // A logical UUID remains a UUID when the raw-binary default is enabled. + if (type.hasAttributeKey("doris.logical_type") && + type.getAttributeValue("doris.logical_type") == "uuid") { data_type = std::make_shared(); + } else if (_enable_mapping_varbinary) { + data_type = std::make_shared(); } else { data_type = std::make_shared(); } diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index 6a21ce165cd4b4..93adc748feca20 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -891,14 +891,9 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( ans.first = DataTypeFactory::instance().create_data_type( TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); } else if (logicalType.__isset.UUID) { - // Keep binary UUID carriers consistent with table defaults and other physical formats. - // Ordinary Parquet scans still expose the native UUID unless VARBINARY was requested. - auto uuid_type = TYPE_UUID; - if (_enable_mapping_varbinary) { - uuid_type = TYPE_VARBINARY; - } else if (_preserve_binary_uuid) { - uuid_type = TYPE_STRING; - } + // UUID is a logical type, so the raw-binary default must not override it. + // Preserve table-requested raw carriers for Iceberg equality deletes and defaults. + auto uuid_type = _preserve_binary_uuid ? TYPE_STRING : TYPE_UUID; ans.first = DataTypeFactory::instance().create_data_type(uuid_type, nullable, -1, -1, 16); } else if (logicalType.__isset.FLOAT16) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); diff --git a/be/test/format/parquet/parquet_reader_test.cpp b/be/test/format/parquet/parquet_reader_test.cpp index 22d669e0f4a9be..6c6c112d7f8978 100644 --- a/be/test/format/parquet/parquet_reader_test.cpp +++ b/be/test/format/parquet/parquet_reader_test.cpp @@ -369,14 +369,14 @@ TEST_F(ParquetReaderTest, normal) { delete p_reader; } -TEST_F(ParquetReaderTest, uuid_text_and_varbinary_cached_mappings) { - // Reuse the same footer cache with both mappings, then repeat each cache lookup. +TEST_F(ParquetReaderTest, uuid_annotation_preserves_text_across_mapping_flags) { + // Changing the raw-binary option must not change a cached UUID field into VARBINARY. for (const bool mapping : {false, true, false, true}) { TDescriptorTable t_desc_table; TTableDescriptor t_table_desc; std::vector table_column_names = {"id", "col1"}; - std::vector table_column_types = { - TPrimitiveType::INT, mapping ? TPrimitiveType::VARBINARY : TPrimitiveType::STRING}; + std::vector table_column_types = {TPrimitiveType::INT, + TPrimitiveType::STRING}; create_table_desc(t_desc_table, t_table_desc, table_column_names, table_column_types); DescriptorTbl* desc_tbl; ObjectPool obj_pool; @@ -438,20 +438,11 @@ TEST_F(ParquetReaderTest, uuid_text_and_varbinary_cached_mappings) { } auto col = block->safe_get_by_position(1).column; auto nullable_column = assert_cast(col.get()); - if (mapping) { - auto varbinary_column = assert_cast( - nullable_column->get_nested_column_ptr().get()); - auto& data = varbinary_column->get_data(); - EXPECT_EQ(data[0].dump_hex(), "0x550E8400E29B41D4A716446655440000"); - EXPECT_EQ(data[1].dump_hex(), "0x123E4567E89B12D3A456426614174000"); - EXPECT_EQ(data[2].dump_hex(), "0x00000000000000000000000000000000"); - } else { - const auto& strings = - assert_cast(nullable_column->get_nested_column()); - EXPECT_EQ(strings.get_data_at(0).to_string(), "550e8400-e29b-41d4-a716-446655440000"); - EXPECT_EQ(strings.get_data_at(1).to_string(), "123e4567-e89b-12d3-a456-426614174000"); - EXPECT_EQ(strings.get_data_at(2).to_string(), "00000000-0000-0000-0000-000000000000"); - } + const auto& strings = + assert_cast(nullable_column->get_nested_column()); + EXPECT_EQ(strings.get_data_at(0).to_string(), "550e8400-e29b-41d4-a716-446655440000"); + EXPECT_EQ(strings.get_data_at(1).to_string(), "123e4567-e89b-12d3-a456-426614174000"); + EXPECT_EQ(strings.get_data_at(2).to_string(), "00000000-0000-0000-0000-000000000000"); } } diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 18d3c8220a625a..0953d5f2525c05 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -4772,21 +4772,21 @@ TEST_F(NewOrcReaderTest, UuidBinaryRequiresLogicalTypeAttribute) { EXPECT_EQ(remove_nullable(array_type.get_nested_type())->get_primitive_type(), TYPE_UUID); } -TEST_F(NewOrcReaderTest, BinaryMappingOverridesUuidAnnotation) { +TEST_F(NewOrcReaderTest, UuidAnnotationOverridesBinaryMapping) { auto reader = create_reader(nullptr, std::nullopt, true); auto binary = ::orc::createPrimitiveType(::orc::BINARY); EXPECT_EQ(remove_nullable(reader->_convert_to_doris_type(*binary))->get_primitive_type(), TYPE_VARBINARY); binary->setAttribute("doris.logical_type", "uuid"); EXPECT_EQ(remove_nullable(reader->_convert_to_doris_type(*binary))->get_primitive_type(), - TYPE_VARBINARY); + TYPE_UUID); auto string = ::orc::createPrimitiveType(::orc::STRING); string->setAttribute("doris.logical_type", "uuid"); EXPECT_EQ(remove_nullable(reader->_convert_to_doris_type(*string))->get_primitive_type(), TYPE_STRING); } -TEST_F(NewOrcReaderTest, BinaryMappingOverridesNestedUuidAnnotations) { +TEST_F(NewOrcReaderTest, NestedUuidAnnotationsOverrideBinaryMapping) { auto reader = create_reader(nullptr, std::nullopt, true); const auto uuid_type = [] { auto binary = ::orc::createPrimitiveType(::orc::BINARY); @@ -4796,19 +4796,19 @@ TEST_F(NewOrcReaderTest, BinaryMappingOverridesNestedUuidAnnotations) { auto list = ::orc::createListType(uuid_type()); const auto array = remove_nullable(reader->_convert_to_doris_type(*list)); const auto& array_type = assert_cast(*array); - EXPECT_EQ(remove_nullable(array_type.get_nested_type())->get_primitive_type(), TYPE_VARBINARY); + EXPECT_EQ(remove_nullable(array_type.get_nested_type())->get_primitive_type(), TYPE_UUID); auto map = ::orc::createMapType(uuid_type(), uuid_type()); const auto mapped = remove_nullable(reader->_convert_to_doris_type(*map)); const auto& map_type = assert_cast(*mapped); - EXPECT_EQ(remove_nullable(map_type.get_key_type())->get_primitive_type(), TYPE_VARBINARY); - EXPECT_EQ(remove_nullable(map_type.get_value_type())->get_primitive_type(), TYPE_VARBINARY); + EXPECT_EQ(remove_nullable(map_type.get_key_type())->get_primitive_type(), TYPE_UUID); + EXPECT_EQ(remove_nullable(map_type.get_value_type())->get_primitive_type(), TYPE_UUID); auto structure = ::orc::createStructType(); structure->addStructField("uuid", uuid_type()); const auto converted = remove_nullable(reader->_convert_to_doris_type(*structure)); const auto& struct_type = assert_cast(*converted); - EXPECT_EQ(remove_nullable(struct_type.get_elements()[0])->get_primitive_type(), TYPE_VARBINARY); + EXPECT_EQ(remove_nullable(struct_type.get_elements()[0])->get_primitive_type(), TYPE_UUID); } TEST_F(NewOrcReaderTest, AggregatePushdownReturnsCountFromFileMetadata) { diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index cf222bbe9b582e..a810510126ac78 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -2288,9 +2288,6 @@ TEST_F(NewParquetReaderTest, UuidPlainDictionaryNullableAndMappingMatrix) { ASSERT_TRUE(reader->get_schema(&schema).ok()); ASSERT_EQ(schema.size(), 1); auto expected_type = preserve_binary_uuid ? TYPE_STRING : TYPE_UUID; - if (mapping) { - expected_type = TYPE_VARBINARY; - } EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), expected_type); auto request = std::make_shared(); @@ -2313,7 +2310,7 @@ TEST_F(NewParquetReaderTest, UuidPlainDictionaryNullableAndMappingMatrix) { if (expected_null) { continue; } - if (mapping || preserve_binary_uuid) { + if (preserve_binary_uuid) { const auto value = column.get_nested_column().get_data_at(row); EXPECT_EQ(value.size, bytes.size()); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 97a10c3bd69aa0..58b49b17248d11 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -62,6 +62,48 @@ TEST(ParquetSchemaTest, UnannotatedInt96DoesNotAcquireTimezoneFromMappingFlag) { } } +TEST(ParquetSchemaTest, LogicalAnnotationsOverrideBinaryDefault) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(3); + tparquet::SchemaElement modern_enum; + modern_enum.__set_name("modern_enum"); + modern_enum.__set_type(tparquet::Type::BYTE_ARRAY); + modern_enum.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + modern_enum.__set_logicalType(tparquet::LogicalType()); + modern_enum.logicalType.__set_ENUM(tparquet::EnumType()); + auto legacy_enum = modern_enum; + legacy_enum.__set_name("legacy_enum"); + legacy_enum.__isset.logicalType = false; + legacy_enum.__set_converted_type(tparquet::ConvertedType::ENUM); + auto uuid = modern_enum; + uuid.__set_name("uuid"); + uuid.__set_type(tparquet::Type::FIXED_LEN_BYTE_ARRAY); + uuid.__set_type_length(16); + uuid.__set_logicalType(tparquet::LogicalType()); + uuid.logicalType.__set_UUID(tparquet::UUIDType()); + for (bool binary_mapping : {false, true}) { + const std::vector schema {root, modern_enum, legacy_enum, uuid}; + doris::FieldDescriptor legacy; + legacy.set_enable_mapping_varbinary(binary_mapping); + ASSERT_TRUE(legacy.parse_from_thrift(schema).ok()); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(legacy.get_column(i)->data_type->get_primitive_type(), TYPE_STRING); + } + NativeFieldDescriptor native; + native.set_enable_mapping_varbinary(binary_mapping); + ASSERT_TRUE(native.parse_from_thrift(schema).ok()); + EXPECT_EQ(native.get_column(0)->data_type->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(native.get_column(1)->data_type->get_primitive_type(), TYPE_STRING); + EXPECT_EQ(native.get_column(2)->data_type->get_primitive_type(), TYPE_UUID); + NativeFieldDescriptor iceberg; + iceberg.set_enable_mapping_varbinary(binary_mapping); + iceberg.set_preserve_binary_uuid(true); + ASSERT_TRUE(iceberg.parse_from_thrift(schema).ok()); + EXPECT_EQ(iceberg.get_column(2)->data_type->get_primitive_type(), TYPE_STRING); + } +} + namespace { std::vector unshredded_variant_schema( diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java index 4332c1472f2f13..1fdcc3cd12f3a9 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java @@ -78,6 +78,8 @@ protected void initializeStatement(Connection conn, JdbcDataSourceConfig config, @Override protected void setValidationQuery(HikariDataSource ds) { ds.setConnectionTestQuery("SELECT 1 FROM dual"); + // NUMBER binding rejects finite IEEE values outside NUMBER's exponent range. + ds.addDataSourceProperty("SetFloatAndDoubleUseBinary", "true"); } @Override diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java index ee525d7f1e4481..f1901c4f48b9c8 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java @@ -18,18 +18,102 @@ package org.apache.doris.jdbc; +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.VectorColumn; + +import com.zaxxer.hikari.HikariDataSource; import oracle.jdbc.OracleConnection; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.mockito.InOrder; import org.mockito.Mockito; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; class OracleSessionTimeZoneTest { + @Test + void binaryFloatingPointBindingsAreEnabledBeforeOpeningConnections() { + OracleJdbcExecutor handler = Mockito.mock(OracleJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + try (HikariDataSource ds = new HikariDataSource()) { + handler.setValidationQuery(ds); + Assertions.assertEquals("true", ds.getDataSourceProperties().getProperty( + OracleConnection.CONNECTION_PROPERTY_SET_FLOAT_AND_DOUBLE_USE_BINARY)); + } + } + + @Test + void binaryEncodingPreservesValuesOutsideNumberRange() throws Exception { + Assertions.assertThrows(SQLException.class, () -> new oracle.sql.NUMBER(1e200)); + for (double value : new double[] {1e200, -1e200, 1e-200, Double.MAX_VALUE, Double.MIN_VALUE}) { + byte[] encoded = new oracle.sql.BINARY_DOUBLE(value).getBytes(); + Assertions.assertEquals(value, new oracle.sql.BINARY_DOUBLE(encoded).doubleValue()); + } + for (float value : new float[] {1.25f, -1.25f, Float.MAX_VALUE, Float.MIN_VALUE}) { + byte[] encoded = new oracle.sql.BINARY_FLOAT(value).getBytes(); + Assertions.assertEquals(value, new oracle.sql.BINARY_FLOAT(encoded).floatValue()); + } + } + + @Test + @EnabledIfSystemProperty(named = "oracle.integration.url", matches = ".+") + void binaryFloatingPointInsertRoundTrip() throws Exception { + OracleJdbcExecutor handler = Mockito.mock(OracleJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + try (HikariDataSource ds = new HikariDataSource()) { + handler.setValidationQuery(ds); + ds.setJdbcUrl(System.getProperty("oracle.integration.url")); + ds.setUsername(System.getProperty("oracle.integration.user")); + ds.setPassword(System.getProperty("oracle.integration.password")); + ds.setMaximumPoolSize(1); + try (Connection connection = ds.getConnection(); Statement ddl = connection.createStatement()) { + String table = "doris_binary_" + java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 8); + ddl.execute("CREATE TABLE " + table + " (id INT, f BINARY_FLOAT, d BINARY_DOUBLE)"); + try (PreparedStatement insert = connection.prepareStatement("INSERT INTO " + table + " VALUES (?, ?, ?)")) { + handler.preparedStatement = insert; + VectorColumn floats = Mockito.mock(VectorColumn.class); + VectorColumn doubles = Mockito.mock(VectorColumn.class); + Mockito.when(floats.getColumnPrimitiveType()).thenReturn(ColumnType.Type.FLOAT); + Mockito.when(doubles.getColumnPrimitiveType()).thenReturn(ColumnType.Type.DOUBLE); + float[] floatValues = {1.25f, -1.25f, 0.0f}; + double[] doubleValues = {1e200, -1e200, 1e-200}; + java.lang.reflect.Method insertColumn = BaseJdbcExecutor.class.getDeclaredMethod( + "insertColumn", int.class, int.class, VectorColumn.class); + insertColumn.setAccessible(true); + Mockito.when(floats.isNullAt(3)).thenReturn(true); + Mockito.when(doubles.isNullAt(3)).thenReturn(true); + for (int row = 0; row < 4; ++row) { + if (row < 3) { + Mockito.when(floats.getFloat(row)).thenReturn(floatValues[row]); + Mockito.when(doubles.getDouble(row)).thenReturn(doubleValues[row]); + } + insert.setInt(1, row); + insertColumn.invoke(handler, row, 1, floats); + insertColumn.invoke(handler, row, 2, doubles); + insert.executeUpdate(); + } + try (ResultSet rows = ddl.executeQuery("SELECT f, d FROM " + table + " ORDER BY id")) { + for (int row = 0; row < 3; ++row) { + Assertions.assertTrue(rows.next()); + Assertions.assertEquals(floatValues[row], rows.getFloat(1)); + Assertions.assertEquals(doubleValues[row], rows.getDouble(2)); + } + Assertions.assertTrue(rows.next()); + Assertions.assertNull(rows.getObject(1)); + Assertions.assertNull(rows.getObject(2)); + Assertions.assertFalse(rows.next()); + } + } finally { + ddl.execute("DROP TABLE " + table); + } + } + } + } + @Test void initializesEveryBorrowedConnectionBeforePreparingQuery() throws Exception { OracleJdbcExecutor handler = Mockito.mock(OracleJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); diff --git a/regression-test/data/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.out b/regression-test/data/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.out index 3f50d70ed3fc54..b26e248febe64b 100644 --- a/regression-test/data/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.out +++ b/regression-test/data/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !schema_false_false -- a array Yes false \N NONE -b text Yes false \N NONE +b varbinary(65533) Yes false \N NONE id int Yes false \N NONE m map Yes false \N NONE s struct Yes false \N NONE @@ -20,12 +20,12 @@ u uuid Yes false \N NONE 4 00000000-0000-0000-0000-000000000000 \N \N -- !schema_false_true -- -a array Yes false \N NONE +a array Yes false \N NONE b varbinary(65533) Yes false \N NONE id int Yes false \N NONE -m map Yes false \N NONE -s struct Yes false \N NONE -u varbinary(65533) Yes false \N NONE +m map Yes false \N NONE +s struct Yes false \N NONE +u uuid Yes false \N NONE -- !values_false_true -- 1 00112233-4455-6677-8899-aabbccddeeff ["00112233-4455-6677-8899-aabbccddeeff", null] ["00000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff"] ["80000000-0000-0000-0000-000000000000", null] 80000000-0000-0000-0000-000000000000 0000FF 6 @@ -41,7 +41,7 @@ u varbinary(65533) Yes false \N NONE -- !schema_true_false -- a array Yes false \N NONE -b text Yes false \N NONE +b varbinary(65533) Yes false \N NONE id int Yes false \N NONE m map Yes false \N NONE s struct Yes false \N NONE @@ -60,12 +60,12 @@ u uuid Yes false \N NONE 4 00000000-0000-0000-0000-000000000000 \N \N -- !schema_true_true -- -a array Yes false \N NONE +a array Yes false \N NONE b varbinary(65533) Yes false \N NONE id int Yes false \N NONE -m map Yes false \N NONE -s struct Yes false \N NONE -u varbinary(65533) Yes false \N NONE +m map Yes false \N NONE +s struct Yes false \N NONE +u uuid Yes false \N NONE -- !values_true_true -- 1 00112233-4455-6677-8899-aabbccddeeff ["00112233-4455-6677-8899-aabbccddeeff", null] ["00000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff"] ["80000000-0000-0000-0000-000000000000", null] 80000000-0000-0000-0000-000000000000 0000FF 6 diff --git a/regression-test/data/datatype_p0/uuid/test_uuid_parquet_native.out b/regression-test/data/datatype_p0/uuid/test_uuid_parquet_native.out index 7fe0885ecc2ab1..9862669c7bbe5b 100644 --- a/regression-test/data/datatype_p0/uuid/test_uuid_parquet_native.out +++ b/regression-test/data/datatype_p0/uuid/test_uuid_parquet_native.out @@ -1,23 +1,23 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !binary_schema -- id int Yes false \N NONE -u varbinary(16) Yes false \N NONE -a array Yes false \N NONE -s struct Yes false \N NONE +u text Yes false \N NONE +a array Yes false \N NONE +s struct Yes false \N NONE -- !binary_values -- 0 \N \N -1 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -2 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -3 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -4 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -5 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +1 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +2 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +3 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +4 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +5 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff 6 \N \N -7 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -8 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -9 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -10 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -11 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +7 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +8 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +9 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +10 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +11 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff -- !external_schema -- id int Yes false \N NONE @@ -111,23 +111,23 @@ ffffffff-ffff-ffff-ffff-ffffffffffff 682 1396736 -- !binary_schema -- id int Yes false \N NONE -u varbinary(16) Yes false \N NONE -a array Yes false \N NONE -s struct Yes false \N NONE +u text Yes false \N NONE +a array Yes false \N NONE +s struct Yes false \N NONE -- !binary_values -- 0 \N \N -1 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -2 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -3 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -4 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -5 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +1 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +2 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +3 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +4 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +5 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff 6 \N \N -7 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -8 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -9 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -10 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -11 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +7 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +8 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +9 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +10 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +11 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff -- !external_schema -- id int Yes false \N NONE @@ -221,23 +221,23 @@ ffffffff-ffff-ffff-ffff-ffffffffffff 682 1396736 -- !binary_schema -- id int Yes false \N NONE -u varbinary(16) Yes false \N NONE -a array Yes false \N NONE -s struct Yes false \N NONE +u text Yes false \N NONE +a array Yes false \N NONE +s struct Yes false \N NONE -- !binary_values -- 0 \N \N -1 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -2 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -3 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -4 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -5 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +1 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +2 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +3 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +4 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +5 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff 6 \N \N -7 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -8 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -9 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -10 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -11 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +7 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +8 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +9 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +10 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +11 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff -- !external_schema -- id int Yes false \N NONE @@ -331,23 +331,23 @@ ffffffff-ffff-ffff-ffff-ffffffffffff 682 1396736 -- !binary_schema -- id int Yes false \N NONE -u varbinary(16) Yes false \N NONE -a array Yes false \N NONE -s struct Yes false \N NONE +u text Yes false \N NONE +a array Yes false \N NONE +s struct Yes false \N NONE -- !binary_values -- 0 \N \N -1 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -2 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -3 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -4 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -5 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +1 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +2 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +3 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +4 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +5 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff 6 \N \N -7 00000000000000000000000000000000 00000000-0000-0000-0000-000000000000 -8 00000000000000000000000000000001 00000000-0000-0000-0000-000000000001 -9 7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 7fffffff-ffff-ffff-ffff-ffffffffffff -10 80000000000000000000000000000000 80000000-0000-0000-0000-000000000000 -11 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF ffffffff-ffff-ffff-ffff-ffffffffffff +7 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +8 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 +9 7fffffff-ffff-ffff-ffff-ffffffffffff 7fffffff-ffff-ffff-ffff-ffffffffffff +10 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +11 ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff -- !external_schema -- id int Yes false \N NONE diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out index b160f8fa5590ff..a31770af13554e 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out @@ -657,10 +657,10 @@ hello {"x":"world", "y":"danny"} 5 -- !test_74 -- \N \N \N \N \N -string_smith 0x656E756D5F736D697468 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D -string_smith 0x656E756D5F736D697468 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D -string_smith 0x656E756D5F736D697468 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D -string_smith 0x656E756D5F736D697468 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D +string_smith enum_smith 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D +string_smith enum_smith 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D +string_smith enum_smith 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D +string_smith enum_smith 6.7 {"name": "smith"} 0x7B226E616D65223A2022736D697468227D -- !test_75 -- 20230208203804485 20230208203804485_0_0 AA1 07eeba07-b04d-42b5-9e31-35b1a22ea31d-0_0-89-83_20230208203804485.parquet AA1 20 1 1 2021-01-01 00:00:00.000000+08:00 @@ -808,15 +808,15 @@ true -- !test_98 -- \N \N \N -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 -- !test_98_desc -- decimal_flba decimal(5,2) Yes false \N NONE interval text Yes false \N NONE -uuid varbinary(16) Yes false \N NONE +uuid text Yes false \N NONE -- !test_100 -- 1317017856 1 18752152 809291 1089176 19951117 3-MEDIUM 0 40 4801000 16034243 9 4368910 72015 3 19951228 RAIL Customer#018752152 q4gN2btSpiKXdN,6 ALGERIA 1 ALGERIA AFRICA 10-753-996-8708 MACHINERY Supplier#001089176 ROidEL1L6yeFsJqnUjD EGYPT 5 EGYPT MIDDLE EAST 14-807-108-7869 blanched gainsboro MFGR#4 MFGR#43 MFGR#433 brown MEDIUM BRUSHED STEEL 42 MED BAG @@ -871,12 +871,12 @@ uuid varbinary(16) Yes false \N NONE -- !test_107 -- \N \N \N -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 +61626344-6546-4768-696a-6b4c6d6e4f70 682.56 1212 -- !test_107_desc -- decimal_flba decimal(5,2) Yes false \N NONE interval text Yes false \N NONE -uuid varbinary(16) Yes false \N NONE +uuid text Yes false \N NONE diff --git a/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy b/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy index e1ad61f64dffee..f983b5cc730060 100644 --- a/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy +++ b/regression-test/suites/datatype_p0/uuid/test_uuid_orc_varbinary_mapping.groovy @@ -57,16 +57,14 @@ suite("test_uuid_orc_varbinary_mapping") { "format"="orc", "backend_id"="${backend.BackendId}", "enable_mapping_varbinary"="${binaryMapping}")""" "order_qt_schema_${scannerV2}_${binaryMapping}" "DESC FUNCTION ${source}" - String uuidValue = binaryMapping ? "CAST(HEX(u) AS UUID)" : "u" - String nestedValue = binaryMapping ? "CAST(HEX(value) AS UUID)" : "value" - String structValue = binaryMapping ? "CAST(HEX(s.k) AS UUID)" : "s.k" - "qt_values_${scannerV2}_${binaryMapping}" """SELECT id, ${uuidValue}, - ARRAY_MAP(value -> ${nestedValue}, a), - ARRAY_MAP(value -> ${nestedValue}, MAP_KEYS(m)), - ARRAY_MAP(value -> ${nestedValue}, MAP_VALUES(m)), - ${structValue}, HEX(b), LENGTH(HEX(b)) + // UUID annotations preserve scalar and nested types independently of the binary option. + "qt_values_${scannerV2}_${binaryMapping}" """SELECT id, u, + ARRAY_MAP(value -> value, a), + ARRAY_MAP(value -> value, MAP_KEYS(m)), + ARRAY_MAP(value -> value, MAP_VALUES(m)), + s.k, HEX(b), LENGTH(HEX(b)) FROM ${source} ORDER BY id""" - "qt_filtered_${scannerV2}_${binaryMapping}" """SELECT id, ${uuidValue}, HEX(b), LENGTH(HEX(b)) + "qt_filtered_${scannerV2}_${binaryMapping}" """SELECT id, u, HEX(b), LENGTH(HEX(b)) FROM ${source} WHERE u IS NOT NULL AND id > 1 ORDER BY id""" } } diff --git a/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy b/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy index 32567370e980c4..f0675e1b02dba4 100644 --- a/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy +++ b/regression-test/suites/datatype_p0/uuid/test_uuid_parquet_native.groovy @@ -50,11 +50,11 @@ suite("test_uuid_parquet_native", "p0,external") { DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')""" sql """INSERT INTO uuid_parquet_native SELECT id,CAST(u AS UUID),CAST(a AS ARRAY), CAST(s AS STRUCT) FROM ${source}""" - // Alternate mappings of the same file to exercise parsed-footer cache isolation. + // A raw-binary option must not override the UUID annotation in the same cached footer. String binarySource = source.replace("'format'='parquet'", "'format'='parquet','enable_mapping_varbinary'='true'") qt_binary_schema "DESC FUNCTION ${binarySource}" - qt_binary_values """SELECT id,HEX(u),CAST(HEX(u) AS UUID) FROM ${binarySource} + qt_binary_values """SELECT id,u,CAST(u AS UUID) FROM ${binarySource} WHERE id < 12 ORDER BY id""" qt_external_schema "DESC FUNCTION ${source}" qt_external_values "SELECT id,u,a,s,LENGTH(u),CAST(u AS UUID) FROM ${source} WHERE id < 12 ORDER BY id" diff --git a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy index 75bd21bc186c50..a807f6d99d62f3 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy @@ -48,6 +48,27 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d "driver_url" = "${driver_url}", "driver_class" = "oracle.jdbc.driver.OracleDriver" );""" + // NUMBER binding cannot represent 1e200, although it is a finite BINARY_DOUBLE. + String binaryTable = "DORIS_BINARY_FLOAT_" + UUID.randomUUID().toString().replace("-", "").substring(0, 8).toUpperCase() + def executeOracle = { statement -> + sql """CALL EXECUTE_STMT("${catalog_name}", "${statement}")""" + } + executeOracle("CREATE TABLE ${binaryTable} (id INT, f BINARY_FLOAT, d BINARY_DOUBLE)") + try { + sql """INSERT INTO ${catalog_name}.${ex_db_name}.${binaryTable} + VALUES (1, CAST(1.25 AS FLOAT), CAST(1e200 AS DOUBLE)), + (2, CAST(-1.25 AS FLOAT), CAST(-1e200 AS DOUBLE)), + (3, CAST(0 AS FLOAT), CAST(1e-200 AS DOUBLE)), + (4, NULL, NULL)""" + def binaryRows = sql """SELECT id, f, d FROM ${catalog_name}.${ex_db_name}.${binaryTable} ORDER BY id""" + assertEquals(4, binaryRows.size()) + assertEquals([1.25d, -1.25d, 0.0d], binaryRows.take(3).collect { it[1].doubleValue() }) + assertEquals([1e200d, -1e200d, 1e-200d], binaryRows.take(3).collect { it[2].doubleValue() }) + assertNull(binaryRows[3][1]) + assertNull(binaryRows[3][2]) + } finally { + executeOracle("DROP TABLE ${binaryTable}") + } order_qt_show_db """ show databases from ${catalog_name}; """ sql """use ${internal_db_name}""" sql """ drop table if exists ${internal_db_name}.${inDorisTable} """ diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy index 08d57af6d4cbb3..8ab05b46b017e4 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group6.groovy @@ -574,6 +574,26 @@ suite("test_hdfs_parquet_group6","external,hive,tvf,external_docker") { "format" = "parquet") limit 10; """ + // ENUM is text even though its physical representation is BYTE_ARRAY. + for (boolean mapping : [false, true]) { + String enumSource = """HDFS("uri"="${uri}", "hadoop.username"="${hdfsUserName}", + "format"="parquet", "enable_mapping_varbinary"="${mapping}")""" + def enumSchema = sql "DESC FUNCTION ${enumSource}" + def enumColumn = enumSchema.find { it[0].toString().toLowerCase().contains("enum") } + assertNotNull(enumColumn) + assertEquals("text", enumColumn[1].toString().toLowerCase()) + String enumTable = "parquet_enum_ctas" + sql "DROP TABLE IF EXISTS ${enumTable}" + try { + sql """CREATE TABLE ${enumTable} PROPERTIES("replication_num"="1") AS + SELECT `${enumColumn[0]}` AS enum_value FROM ${enumSource}""" + assertEquals(sql("SELECT `${enumColumn[0]}` FROM ${enumSource} ORDER BY 1"), + sql("SELECT enum_value FROM ${enumTable} ORDER BY 1")) + } finally { + sql "DROP TABLE IF EXISTS ${enumTable}" + } + } + uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/07eeba07-b04d-42b5-9e31-35b1a22ea31d-0_0-89-83_20230208203804485.parquet" order_qt_test_75 """ select * from HDFS( "uri" = "${uri}", @@ -736,7 +756,7 @@ suite("test_hdfs_parquet_group6","external,hive,tvf,external_docker") { // The first field is FIXED_LEN_BYTE_ARRAY(16) with a UUID logical annotation. - // TVFs preserve its bytes as VARBINARY even when the legacy mapping option is absent. + // The UUID annotation preserves canonical text independently of the raw-binary default. // test_107 verifies that explicitly enabling the option produces the same result. uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group6/fixed_len_byte_array.parquet" order_qt_test_98 """ select * from HDFS( From 32fd2fedcdbfe7a51ab8df129f12b3f71638a2a6 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 18 Oct 2026 17:50:07 +0800 Subject: [PATCH 4/4] [fix](external) Preserve UUID bytes for binary table columns --- be/src/exprs/function/cast/function_cast.cpp | 23 ++- be/src/format/orc/vorc_reader.cpp | 6 + .../format/parquet/parquet_column_convert.cpp | 7 +- .../function/cast/cast_uuid_binary_test.cpp | 92 ++++++++++++ .../parquet/parquet_column_convert_test.cpp | 13 +- .../parquet/parquet_uuid_binary_test.cpp | 133 ++++++++++++++++++ .../hive/test_hive_uuid_binary.out | 81 +++++++++++ .../tvf/test_hdfs_parquet_group2.out | 20 +-- .../hive/test_hive_uuid_binary.groovy | 73 ++++++++++ .../tvf/test_hdfs_parquet_group2.groovy | 1 + 10 files changed, 430 insertions(+), 19 deletions(-) create mode 100644 be/test/exprs/function/cast/cast_uuid_binary_test.cpp create mode 100644 be/test/format/parquet/parquet_uuid_binary_test.cpp create mode 100644 regression-test/data/external_table_p0/hive/test_hive_uuid_binary.out create mode 100644 regression-test/suites/external_table_p0/hive/test_hive_uuid_binary.groovy diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp index 027bdc4db6cf17..3b647399805304 100644 --- a/be/src/exprs/function/cast/function_cast.cpp +++ b/be/src/exprs/function/cast/function_cast.cpp @@ -17,6 +17,7 @@ #include +#include "core/column/column_varbinary.h" #include "core/data_type/data_type_agg_state.h" #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" // IWYU pragma: keep @@ -26,6 +27,7 @@ #include "core/data_type/data_type_variant.h" #include "core/data_type/data_type_variant_v2.h" #include "core/data_type/primitive_type.h" +#include "core/value/uuid_value.h" #include "exprs/function/cast/cast_to_array.h" #include "exprs/function/cast/cast_to_jsonb.h" #include "exprs/function/cast/cast_to_map.h" @@ -65,7 +67,26 @@ WrapperType create_varbinary_wrapper(const DataTypePtr& from_type_untyped) { return cast_from_string_to_generic; } - return CastWrapper::create_unsupport_wrapper("Cast to Varbinary only support from String type"); + if (from_type_untyped->get_primitive_type() == TYPE_UUID) { + return [](FunctionContext*, Block& block, const ColumnNumbers& arguments, uint32_t result, + size_t rows, const NullMap::value_type*) { + const auto& source = + assert_cast(*block.get_by_position(arguments[0]).column); + auto binary = ColumnVarbinary::create(); + binary->reserve(rows); + for (size_t i = 0; i < rows; ++i) { + // File Scanner V2 decodes UUIDs before mapping Hive BINARY columns. Recreate the + // file's big-endian bytes, not the host integer layout or canonical UUID text. + const auto bytes = UUIDValue::to_big_endian(source.get_data()[i]); + binary->insert_data(reinterpret_cast(bytes.data()), bytes.size()); + } + block.get_by_position(result).column = std::move(binary); + return Status::OK(); + }; + } + + return CastWrapper::create_unsupport_wrapper( + "Cast to Varbinary only support from String or UUID type"); } WrapperType prepare_unpack_dictionaries(FunctionContext* context, const DataTypePtr& from_type, diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 35aca9a99f817a..12e89b3f983085 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -2415,6 +2415,12 @@ Status OrcReader::_orc_column_to_doris_column( MutableColumnPtr data_column; if (orc_column_type != nullptr) { auto src_type = convert_to_doris_type(orc_column_type); + if (src_type->get_primitive_type() == TYPE_UUID && + data_type->get_primitive_type() == TYPE_VARBINARY) { + // Hive BINARY keeps the file bytes even when the ORC carrier has a UUID annotation. + // Decode that carrier directly, including recursively visited complex-type children. + src_type = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, true); + } bool is_dict_filter_col = false; for (const std::pair& dict_col : _dict_filter_cols) { if (col_name == dict_col.first) { diff --git a/be/src/format/parquet/parquet_column_convert.cpp b/be/src/format/parquet/parquet_column_convert.cpp index 9637ff6a6bb740..fbafb58af799d5 100644 --- a/be/src/format/parquet/parquet_column_convert.cpp +++ b/be/src/format/parquet/parquet_column_convert.cpp @@ -261,9 +261,10 @@ std::unique_ptr PhysicalToLogicalConverter::get_conv } else if (is_parquet_native_type(src_logical_primitive)) { bool is_string_logical_type = is_string_type(src_logical_primitive); if (is_string_logical_type && src_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) { - if (parquet_schema.logicalType.__isset.UUID && !preserve_binary_uuid) { - // Ordinary Parquet scans expose canonical text, as in File Scanner V2. - // Iceberg retains its raw 16-byte STRING carrier for equality deletes/defaults. + if (parquet_schema.logicalType.__isset.UUID && !preserve_binary_uuid && + dst_logical_type->get_primitive_type() != TYPE_VARBINARY) { + // Inferred STRING fields expose canonical text. Explicit BINARY targets and + // Iceberg's equality-delete carrier must retain the original 16 bytes. if (parquet_schema.type_length != UUIDValue::BINARY_LENGTH) { physical_converter = std::make_unique(src_physical_type, src_logical_type); diff --git a/be/test/exprs/function/cast/cast_uuid_binary_test.cpp b/be/test/exprs/function/cast/cast_uuid_binary_test.cpp new file mode 100644 index 00000000000000..6af7065aacb1a1 --- /dev/null +++ b/be/test/exprs/function/cast/cast_uuid_binary_test.cpp @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_nullable.h" +#include "core/column/column_varbinary.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_uuid.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/value/uuid_value.h" +#include "exprs/function/simple_function_factory.h" +#include "exprs/function_context.h" +#include "gtest/gtest.h" +#include "runtime/runtime_state.h" + +namespace doris { + +TEST(UuidBinaryCastTest, PreservesFileBytesAndNestedNulls) { + for (bool strict : {false, true}) { + RuntimeState state; + for (bool nested : {false, true}) { + const std::vector bytes = {std::string(16, '\0'), std::string(16, '\xff'), + std::string("\x80", 1) + std::string(15, '\0'), + "abcDeFGhijkLmnOp"}; + auto values = ColumnUUID::create(); + auto nulls = ColumnUInt8::create(); + for (const auto& value : bytes) { + values->insert_value( + UUIDValue::from_big_endian(reinterpret_cast(value.data()))); + nulls->insert_value(0); + } + values->insert_default(); + nulls->insert_value(1); + ColumnPtr source = ColumnNullable::create(std::move(values), std::move(nulls)); + DataTypePtr source_type = make_nullable(std::make_shared()); + DataTypePtr target_type = make_nullable(std::make_shared()); + if (nested) { + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(3); + offsets->insert_value(3); + offsets->insert_value(5); + source = ColumnArray::create(source, std::move(offsets)); + source_type = std::make_shared(source_type); + target_type = std::make_shared(target_type); + } + auto context = FunctionContext::create_context(&state, {}, {}); + context->set_enable_strict_mode(strict); + Block block; + block.insert({source, source_type, "uuid"}); + auto function = SimpleFunctionFactory::instance().get_function( + "CAST", {{source, source_type, "uuid"}}, target_type); + ASSERT_NE(function, nullptr); + const int result_id = block.columns(); + block.insert({nullptr, target_type, "binary"}); + auto status = function->execute(context.get(), block, {0}, result_id, source->size()); + ASSERT_TRUE(status.ok()) << status; + ColumnPtr result = block.get_by_position(result_id).column; + if (nested) { + const auto& array = assert_cast(*result); + EXPECT_EQ(array.get_offsets(), + assert_cast(*source).get_offsets()); + result = array.get_data_ptr(); + } + const auto& nullable = assert_cast(*result); + EXPECT_EQ(nullable.size(), 5); + for (size_t i = 0; i < bytes.size(); ++i) { + EXPECT_FALSE(nullable.is_null_at(i)); + EXPECT_EQ(nullable.get_nested_column().get_data_at(i).to_string(), bytes[i]); + } + EXPECT_TRUE(nullable.is_null_at(4)); + } + } +} + +} // namespace doris diff --git a/be/test/format/parquet/parquet_column_convert_test.cpp b/be/test/format/parquet/parquet_column_convert_test.cpp index 563a1795d63301..49758c281721ae 100644 --- a/be/test/format/parquet/parquet_column_convert_test.cpp +++ b/be/test/format/parquet/parquet_column_convert_test.cpp @@ -63,11 +63,14 @@ TEST(ParquetColumnConvertTest, UuidTextAndBinaryCarriersPreserveNullsAcrossBatch "00000000-0000-0000-0000-000000000000", "7f7f7f7f-7f7f-7f7f-7f7f-7f7f7f7f7f7f", "80000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff", "61626344-6546-4768-696a-6b4c6d6e4f70", "00000000-0000-0000-0000-000000000000"}; - for (const bool preserve_binary : {false, true}) { + for (int carrier = 0; carrier < 3; ++carrier) { + const bool preserve_binary = carrier == 1; + const auto dst_type = DataTypeFactory::instance().create_data_type( + carrier == 2 ? TYPE_VARBINARY : TYPE_STRING, true); auto converter = PhysicalToLogicalConverter::get_converter( - &field, field.data_type, field.data_type, nullptr, false, preserve_binary); + &field, field.data_type, dst_type, nullptr, false, preserve_binary); ASSERT_TRUE(converter->support()); - ColumnPtr dst = field.data_type->create_column(); + ColumnPtr dst = dst_type->create_column(); for (size_t batch = 0; batch < 2; ++batch) { auto values = ColumnFixedLengthObject::create(16); auto nulls = ColumnUInt8::create(); @@ -76,14 +79,14 @@ TEST(ParquetColumnConvertTest, UuidTextAndBinaryCarriersPreserveNullsAcrossBatch nulls->insert_value(i == 5); } ColumnPtr src = ColumnNullable::create(std::move(values), std::move(nulls)); - ASSERT_TRUE(converter->convert(src, field.data_type, field.data_type, dst, false).ok()); + ASSERT_TRUE(converter->convert(src, field.data_type, dst_type, dst, false).ok()); EXPECT_EQ(dst->size(), (batch + 1) * 3); } const auto& nullable = assert_cast(*dst); for (size_t i = 0; i < raw_values.size(); ++i) { EXPECT_EQ(nullable.get_null_map_data()[i], i == 5); EXPECT_EQ(nullable.get_nested_column().get_data_at(i).to_string(), - preserve_binary ? raw_values[i] : text_values[i]); + carrier != 0 ? raw_values[i] : text_values[i]); } } } diff --git a/be/test/format/parquet/parquet_uuid_binary_test.cpp b/be/test/format/parquet/parquet_uuid_binary_test.cpp new file mode 100644 index 00000000000000..fa7c8e3f8eefcd --- /dev/null +++ b/be/test/format/parquet/parquet_uuid_binary_test.cpp @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +#include "core/column/column_array.h" +#include "core/column/column_struct.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" +#include "format/parquet/vparquet_reader.h" +#include "io/fs/file_meta_cache.h" +#include "io/fs/local_file_system.h" +#include "util/timezone_utils.h" + +namespace doris { + +TEST(ParquetUuidBinaryTest, LegacyReaderPreservesNestedBytesAcrossBatches) { + // Timestamp-semantics negotiation can force V2 even with its session flag disabled. + // Exercise the legacy reader directly to keep its nested BINARY mapping covered. + const char* source_root = std::getenv("ROOT"); + ASSERT_NE(source_root, nullptr); + const std::vector bytes = {std::string(16, '\0'), + std::string(16, '\0'), + std::string(15, '\0') + std::string(1, '\1'), + std::string(1, '\x7f') + std::string(15, '\xff'), + std::string(1, '\x80') + std::string(15, '\0'), + std::string(16, '\xff')}; + auto check_value = [&](const IColumn& column, size_t row, size_t id) { + EXPECT_EQ(column.is_null_at(row), id % 6 == 0); + if (id % 6 != 0) { + EXPECT_EQ(assert_cast(column) + .get_nested_column() + .get_data_at(row) + .to_string(), + bytes[id % 6]); + } + }; + const auto binary = make_nullable(std::make_shared()); + const DataTypes types = { + make_nullable(std::make_shared()), binary, + make_nullable(std::make_shared(binary)), + make_nullable(std::make_shared(DataTypes {binary}, Strings {"k"}))}; + const std::vector names = {"id", "u", "a", "s"}; + std::unordered_map indexes = {{"id", 0}, {"u", 1}, {"a", 2}, {"s", 3}}; + cctz::time_zone zone; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("UTC", zone)); + FileMetaCache cache(1024); + for (const std::string encoding : {"plain", "dictionary"}) { + io::FileReaderSPtr file; + ASSERT_TRUE( + io::global_local_filesystem() + ->open_file( + std::string(source_root) + + "/regression-test/data/datatype_p0/uuid/test_data/uuid_" + + encoding + ".parquet", + &file) + .ok()); + TFileScanRangeParams params; + params.__set_enable_mapping_varbinary(true); + params.__set_file_type(TFileType::FILE_LOCAL); + TFileRangeDesc range; + range.__set_start_offset(0); + range.__set_size(file->size()); + range.__set_file_size(file->size()); + range.__set_path(file->path().native()); + ParquetReader reader(nullptr, params, range, 37, &zone, nullptr, nullptr, &cache); + phmap::flat_hash_map>> predicates; + auto status = reader.init_reader(names, &indexes, {}, predicates, nullptr, nullptr, nullptr, + nullptr, nullptr); + ASSERT_TRUE(status.ok()) << status; + std::unordered_map> partitions; + std::unordered_map missing; + ASSERT_TRUE(reader.set_fill_columns(partitions, missing).ok()); + bool eof = false; + size_t total = 0; + while (!eof) { + Block block; + for (size_t i = 0; i < names.size(); ++i) { + block.insert({types[i]->create_column(), types[i], names[i]}); + } + size_t rows = 0; + status = reader.get_next_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + for (size_t row = 0; row < rows; ++row) { + const size_t id = total + row; + check_value(*block.get_by_position(1).column, row, id); + const auto& array_nullable = + assert_cast(*block.get_by_position(2).column); + EXPECT_EQ(array_nullable.is_null_at(row), id % 7 == 0); + const auto& array = + assert_cast(array_nullable.get_nested_column()); + const auto begin = array.get_offsets()[static_cast(row) - 1]; + const auto end = array.get_offsets()[row]; + if (id % 7 > 1) { + ASSERT_EQ(end - begin, 2); + check_value(array.get_data(), begin, id); + EXPECT_TRUE(array.get_data().is_null_at(begin + 1)); + } else { + EXPECT_EQ(end, begin); + } + const auto& structure = + assert_cast(*block.get_by_position(3).column); + EXPECT_EQ(structure.is_null_at(row), id % 5 == 0); + if (id % 5 != 0) { + const auto& fields = + assert_cast(structure.get_nested_column()); + check_value(fields.get_column(0), row, id); + } + } + total += rows; + } + EXPECT_EQ(total, 4097); + } +} + +} // namespace doris diff --git a/regression-test/data/external_table_p0/hive/test_hive_uuid_binary.out b/regression-test/data/external_table_p0/hive/test_hive_uuid_binary.out new file mode 100644 index 00000000000000..226cdf02386d19 --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_uuid_binary.out @@ -0,0 +1,81 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !values_parquet_false_false -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_parquet_false_false -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_parquet_false_true -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_parquet_false_true -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_parquet_true_false -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_parquet_true_false -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_parquet_true_true -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_parquet_true_true -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_orc_false_false -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_orc_false_false -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_orc_false_true -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_orc_false_true -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_orc_true_false -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_orc_true_false -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + +-- !values_orc_true_true -- +1 00112233445566778899AABBCCDDEEFF 16 ["00112233445566778899AABBCCDDEEFF", null] ["00000000000000000000000000000000"] ["80000000000000000000000000000000"] 80000000000000000000000000000000 +2 80000000000000000000000000000000 16 [] [] [] \N +3 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF 16 \N \N \N \N +4 00000000000000000000000000000000 16 [null] ["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"] [null] 00000000000000000000000000000000 +5 \N \N \N \N \N \N + +-- !filtered_orc_true_true -- +1 00112233445566778899AABBCCDDEEFF 00112233445566778899AABBCCDDEEFF 80000000000000000000000000000000 + diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out index f4c37abf789c1e..2a8a902028d631 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group2.out @@ -191,16 +191,16 @@ apple_banana_mango9 1001-01-07T09:02:03.123456 1001-01-14T09:02:03.123456 -- !test_24 -- -false 1 2 3 10 1.2 val_1 val_1 0x484541525453 false 1 2 3 10 1.2 val_1 val_1 0x484541525453 ["arr_1", "arr_2", "arr_3"] [1] {1:"val_1", 2:"val_2", 3:"val_3"} {1:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 2:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 3:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}]} -false 3 4 5 30 3.2 val_3 val_3 0x434C554253 \N \N \N \N \N \N \N \N \N ["arr_3", "arr_4", "arr_5"] [3] {3:"val_3", 4:"val_4", 5:"val_5"} {3:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 4:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 5:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}]} -false 5 6 7 50 5.2 val_5 val_5 0x484541525453 false 5 6 7 50 5.2 val_5 val_5 0x484541525453 ["arr_5", "arr_6", "arr_7"] [5] {5:"val_5", 6:"val_6", 7:"val_7"} {5:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 6:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 7:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}]} -false 7 8 9 70 7.2 val_7 val_7 0x434C554253 false 7 8 9 70 7.2 val_7 val_7 0x434C554253 ["arr_7", "arr_8", "arr_9"] [7] {7:"val_7", 8:"val_8", 9:"val_9"} {7:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 8:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 9:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}]} -false 9 10 11 90 9.199999999999999 val_9 val_9 0x484541525453 \N \N \N \N \N \N \N \N \N ["arr_9", "arr_10", "arr_11"] [9] {9:"val_9", 10:"val_10", 11:"val_11"} {9:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 10:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 11:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}]} -true 0 1 2 0 0.2 val_0 val_0 0x535041444553 \N \N \N \N \N \N \N \N \N ["arr_0", "arr_1", "arr_2"] [0] {0:"val_0", 1:"val_1", 2:"val_2"} {0:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 1:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 2:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}]} -true 2 3 4 20 2.2 val_2 val_2 0x4449414D4F4E4453 true 2 3 4 20 2.2 val_2 val_2 0x4449414D4F4E4453 ["arr_2", "arr_3", "arr_4"] [2] {2:"val_2", 3:"val_3", 4:"val_4"} {2:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 3:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 4:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}]} -true 4 5 6 40 4.2 val_4 val_4 0x535041444553 true 4 5 6 40 4.2 val_4 val_4 0x535041444553 ["arr_4", "arr_5", "arr_6"] [4] {4:"val_4", 5:"val_5", 6:"val_6"} {4:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 5:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 6:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}]} -true 6 7 8 60 6.2 val_6 val_6 0x4449414D4F4E4453 \N \N \N \N \N \N \N \N \N ["arr_6", "arr_7", "arr_8"] [6] {6:"val_6", 7:"val_7", 8:"val_8"} {6:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 7:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 8:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}]} -true 8 9 10 80 8.199999999999999 val_8 val_8 0x535041444553 true 8 9 10 80 8.199999999999999 val_8 val_8 0x535041444553 ["arr_8", "arr_9", "arr_10"] [8] {8:"val_8", 9:"val_9", 10:"val_10"} {8:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 9:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 10:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}]} +false 1 2 3 10 1.2 val_1 val_1 HEARTS false 1 2 3 10 1.2 val_1 val_1 HEARTS ["arr_1", "arr_2", "arr_3"] [1] {1:"val_1", 2:"val_2", 3:"val_3"} {1:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 2:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}], 3:[{"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}]} +false 3 4 5 30 3.2 val_3 val_3 CLUBS \N \N \N \N \N \N \N \N \N ["arr_3", "arr_4", "arr_5"] [3] {3:"val_3", 4:"val_4", 5:"val_5"} {3:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 4:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}], 5:[{"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}]} +false 5 6 7 50 5.2 val_5 val_5 HEARTS false 5 6 7 50 5.2 val_5 val_5 HEARTS ["arr_5", "arr_6", "arr_7"] [5] {5:"val_5", 6:"val_6", 7:"val_7"} {5:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 6:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}], 7:[{"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}]} +false 7 8 9 70 7.2 val_7 val_7 CLUBS false 7 8 9 70 7.2 val_7 val_7 CLUBS ["arr_7", "arr_8", "arr_9"] [7] {7:"val_7", 8:"val_8", 9:"val_9"} {7:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 8:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}], 9:[{"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}]} +false 9 10 11 90 9.2 val_9 val_9 HEARTS \N \N \N \N \N \N \N \N \N ["arr_9", "arr_10", "arr_11"] [9] {9:"val_9", 10:"val_10", 11:"val_11"} {9:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 10:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}], 11:[{"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}, {"nestedintscolumn":[11, 12, 13], "nestedstringcolumn":"val_11"}]} +true 0 1 2 0 0.2 val_0 val_0 SPADES \N \N \N \N \N \N \N \N \N ["arr_0", "arr_1", "arr_2"] [0] {0:"val_0", 1:"val_1", 2:"val_2"} {0:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 1:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}], 2:[{"nestedintscolumn":[0, 1, 2], "nestedstringcolumn":"val_0"}, {"nestedintscolumn":[1, 2, 3], "nestedstringcolumn":"val_1"}, {"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}]} +true 2 3 4 20 2.2 val_2 val_2 DIAMONDS true 2 3 4 20 2.2 val_2 val_2 DIAMONDS ["arr_2", "arr_3", "arr_4"] [2] {2:"val_2", 3:"val_3", 4:"val_4"} {2:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 3:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}], 4:[{"nestedintscolumn":[2, 3, 4], "nestedstringcolumn":"val_2"}, {"nestedintscolumn":[3, 4, 5], "nestedstringcolumn":"val_3"}, {"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}]} +true 4 5 6 40 4.2 val_4 val_4 SPADES true 4 5 6 40 4.2 val_4 val_4 SPADES ["arr_4", "arr_5", "arr_6"] [4] {4:"val_4", 5:"val_5", 6:"val_6"} {4:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 5:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}], 6:[{"nestedintscolumn":[4, 5, 6], "nestedstringcolumn":"val_4"}, {"nestedintscolumn":[5, 6, 7], "nestedstringcolumn":"val_5"}, {"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}]} +true 6 7 8 60 6.2 val_6 val_6 DIAMONDS \N \N \N \N \N \N \N \N \N ["arr_6", "arr_7", "arr_8"] [6] {6:"val_6", 7:"val_7", 8:"val_8"} {6:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 7:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}], 8:[{"nestedintscolumn":[6, 7, 8], "nestedstringcolumn":"val_6"}, {"nestedintscolumn":[7, 8, 9], "nestedstringcolumn":"val_7"}, {"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}]} +true 8 9 10 80 8.2 val_8 val_8 SPADES true 8 9 10 80 8.2 val_8 val_8 SPADES ["arr_8", "arr_9", "arr_10"] [8] {8:"val_8", 9:"val_9", 10:"val_10"} {8:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 9:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}], 10:[{"nestedintscolumn":[8, 9, 10], "nestedstringcolumn":"val_8"}, {"nestedintscolumn":[9, 10, 11], "nestedstringcolumn":"val_9"}, {"nestedintscolumn":[10, 11, 12], "nestedstringcolumn":"val_10"}]} -- !test_25 -- {"duration":"111222333444"} diff --git a/regression-test/suites/external_table_p0/hive/test_hive_uuid_binary.groovy b/regression-test/suites/external_table_p0/hive/test_hive_uuid_binary.groovy new file mode 100644 index 00000000000000..d0fe04a20d159c --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_uuid_binary.groovy @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_hive_uuid_binary", "p0,external,hive,external_docker,external_docker_hive") { + if (!context.config.otherConfigs.get("enableHiveTest")?.equalsIgnoreCase("true")) { + return + } + setHivePrefix("hive2") + String host = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get("hive2HmsPort") + String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort") + String root = "hdfs://${host}:${hdfsPort}/tmp/test_hive_uuid_binary_${UUID.randomUUID()}" + hive_docker "CREATE DATABASE IF NOT EXISTS test_hive_uuid_binary" + sql "DROP CATALOG IF EXISTS hive_uuid_binary" + sql """CREATE CATALOG hive_uuid_binary PROPERTIES ( + "type"="hms", "hive.metastore.uris"="thrift://${host}:${hmsPort}", + "hadoop.username"="doris", "enable.mapping.varbinary"="true")""" + sql "DROP TABLE IF EXISTS hive_uuid_binary_source" + sql """CREATE TABLE hive_uuid_binary_source ( + id INT, u UUID, a ARRAY, m MAP, s STRUCT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO hive_uuid_binary_source VALUES + (1, '00112233445566778899aabbccddeeff', + ['00112233445566778899aabbccddeeff', NULL], + {'00000000000000000000000000000000':'80000000000000000000000000000000'}, + {'80000000000000000000000000000000'}), + (2, '80000000000000000000000000000000', [], {}, {NULL}), + (3, 'ffffffffffffffffffffffffffffffff', NULL, NULL, NULL), + (4, '00000000000000000000000000000000', [NULL], + {'ffffffffffffffffffffffffffffffff':NULL}, {'00000000000000000000000000000000'}), + (5, NULL, NULL, NULL, NULL)""" + for (String format : ["parquet", "orc"]) { + sql """SELECT * FROM hive_uuid_binary_source ORDER BY id + INTO OUTFILE "${root}/${format}/part_" FORMAT AS ${format} + PROPERTIES("fs.defaultFS"="hdfs://${host}:${hdfsPort}", "hadoop.username"="doris")""" + hive_docker "DROP TABLE IF EXISTS test_hive_uuid_binary.uuid_${format}" + // The file carries UUID annotations, but Hive's declared BINARY contract is the raw bytes. + hive_docker """CREATE EXTERNAL TABLE test_hive_uuid_binary.uuid_${format} ( + id INT, u BINARY, a ARRAY, m MAP, s STRUCT) + STORED AS ${format} LOCATION '${root}/${format}'""" + sql "REFRESH CATALOG hive_uuid_binary" + for (boolean scannerV2 : [false, true]) { + sql "SET enable_file_scanner_v2 = ${scannerV2}" + for (boolean strict : [false, true]) { + sql "SET enable_strict_cast = ${strict}" + String table = "hive_uuid_binary.test_hive_uuid_binary.uuid_${format}" + "qt_values_${format}_${scannerV2}_${strict}" """SELECT id, HEX(u), LENGTH(u), + ARRAY_MAP(x -> HEX(x), a), ARRAY_MAP(x -> HEX(x), MAP_KEYS(m)), + ARRAY_MAP(x -> HEX(x), MAP_VALUES(m)), HEX(s.k) + FROM ${table} ORDER BY id""" + "qt_filtered_${format}_${scannerV2}_${strict}" """SELECT id, HEX(u), HEX(a[1]), HEX(s.k) + FROM ${table} WHERE HEX(u) = '00112233445566778899AABBCCDDEEFF' + ORDER BY id""" + } + } + } +} diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy index c91edb5e201ba7..e269dc64823529 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group2.groovy @@ -198,6 +198,7 @@ suite("test_hdfs_parquet_group2","external,hive,tvf,external_docker") { "format" = "parquet") limit 10; """ + // ENUM is textual despite its BYTE_ARRAY storage; its expected cells must stay strings. uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group2/parquet-thrift-compat.snappy.parquet" order_qt_test_24 """ select * from HDFS( "uri" = "${uri}",